diff --git a/Cargo.lock b/Cargo.lock index cf099ef73..ea465529e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2331,7 +2331,7 @@ dependencies = [ [[package]] name = "pi-ast" -version = "15.3.2" +version = "15.4.3" dependencies = [ "anyhow", "ast-grep-core", @@ -2399,7 +2399,7 @@ dependencies = [ [[package]] name = "pi-iso" -version = "15.3.2" +version = "15.4.3" dependencies = [ "async-trait", "libc", @@ -2411,7 +2411,7 @@ dependencies = [ [[package]] name = "pi-natives" -version = "15.3.2" +version = "15.4.3" dependencies = [ "anyhow", "arboard", @@ -2457,7 +2457,7 @@ dependencies = [ [[package]] name = "pi-shell" -version = "15.3.2" +version = "15.4.3" dependencies = [ "anyhow", "brush-builtins", diff --git a/Cargo.toml b/Cargo.toml index 989083708..7cb4cdd7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["crates/brush-core-vendored", "crates/brush-builtins-vendored"] resolver = "3" [workspace.package] -version = "15.3.2" +version = "15.4.3" edition = "2024" license = "MIT" authors = ["Can Boluk"] diff --git a/bun.lock b/bun.lock index 1c6252fba..8aa3d5462 100644 --- a/bun.lock +++ b/bun.lock @@ -15,7 +15,7 @@ }, "packages/agent": { "name": "@oh-my-pi/pi-agent-core", - "version": "15.3.2", + "version": "15.4.3", "dependencies": { "@oh-my-pi/pi-ai": "catalog:", "@oh-my-pi/pi-natives": "catalog:", @@ -30,7 +30,7 @@ }, "packages/ai": { "name": "@oh-my-pi/pi-ai", - "version": "15.3.2", + "version": "15.4.3", "bin": { "pi-ai": "./src/cli.ts", }, @@ -48,7 +48,7 @@ }, "packages/coding-agent": { "name": "@oh-my-pi/pi-coding-agent", - "version": "15.3.2", + "version": "15.4.3", "bin": { "omp": "src/cli.ts", }, @@ -83,7 +83,7 @@ }, "packages/natives": { "name": "@oh-my-pi/pi-natives", - "version": "15.3.2", + "version": "15.4.3", "devDependencies": { "@napi-rs/cli": "catalog:", "@types/bun": "catalog:", @@ -91,7 +91,7 @@ }, "packages/stats": { "name": "@oh-my-pi/omp-stats", - "version": "15.3.2", + "version": "15.4.3", "bin": { "omp-stats": "./src/index.ts", }, @@ -116,7 +116,7 @@ }, "packages/swarm-extension": { "name": "@oh-my-pi/swarm-extension", - "version": "15.3.2", + "version": "15.4.3", "bin": { "omp-swarm": "src/cli.ts", }, @@ -132,7 +132,7 @@ }, "packages/tui": { "name": "@oh-my-pi/pi-tui", - "version": "15.3.2", + "version": "15.4.3", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "@oh-my-pi/pi-utils": "catalog:", @@ -172,7 +172,7 @@ }, "packages/utils": { "name": "@oh-my-pi/pi-utils", - "version": "15.3.2", + "version": "15.4.3", "dependencies": { "@oh-my-pi/pi-natives": "catalog:", "beautiful-mermaid": "catalog:", @@ -212,13 +212,13 @@ "@bufbuild/protoc-gen-es": "^2.12.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.6.2", - "@oh-my-pi/omp-stats": "15.3.2", - "@oh-my-pi/pi-agent-core": "15.3.2", - "@oh-my-pi/pi-ai": "15.3.2", - "@oh-my-pi/pi-coding-agent": "15.3.2", - "@oh-my-pi/pi-natives": "15.3.2", - "@oh-my-pi/pi-tui": "15.3.2", - "@oh-my-pi/pi-utils": "15.3.2", + "@oh-my-pi/omp-stats": "15.4.3", + "@oh-my-pi/pi-agent-core": "15.4.3", + "@oh-my-pi/pi-ai": "15.4.3", + "@oh-my-pi/pi-coding-agent": "15.4.3", + "@oh-my-pi/pi-natives": "15.4.3", + "@oh-my-pi/pi-tui": "15.4.3", + "@oh-my-pi/pi-utils": "15.4.3", "@opentelemetry/api": "^1.9.0", "@opentelemetry/context-async-hooks": "^2.0.0", "@opentelemetry/sdk-trace-base": "^2.0.0", @@ -831,7 +831,7 @@ "data-uri-to-buffer": ["data-uri-to-buffer@6.0.2", "", {}, "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw=="], - "date-fns": ["date-fns@4.2.1", "", {}, "sha512-37RhSdxaG1suen6VDCza6rNrQfooyQh57HFVPwQGEq2QWliVLzPQZ8Oa017weOu+HZCnzI7N3Pf/wyoBKfEqrA=="], + "date-fns": ["date-fns@4.3.0", "", {}, "sha512-OYcL+3N/jyWbYdFGqoMAhytDgxP9pbYPUUiRCOgn4Fewaadk9l/Wam4Avciiyp2BgkpfQyBV9B+ehnVJych+eQ=="], "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], @@ -1203,7 +1203,7 @@ "text-hex": ["text-hex@1.0.0", "", {}, "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="], - "tinyexec": ["tinyexec@1.1.2", "", {}, "sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA=="], + "tinyexec": ["tinyexec@1.2.2", "", {}, "sha512-M/Q0B2cp4K7kynaT/vnED1j8TlLY+Pp7C6Wl2bl/7u/F0mUVwdyOpwomQb8JpYLitHUssAJRmLZdMCGsrx7i+g=="], "token-types": ["token-types@6.1.2", "", { "dependencies": { "@borewit/text-codec": "^0.2.1", "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" } }, "sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww=="], diff --git a/crates/brush-core-vendored/src/interp.rs b/crates/brush-core-vendored/src/interp.rs index 802c9e450..408182333 100644 --- a/crates/brush-core-vendored/src/interp.rs +++ b/crates/brush-core-vendored/src/interp.rs @@ -2152,22 +2152,69 @@ fn setup_process_substitution( Ok((candidate_fd_num, target_file)) } +// LOCAL DIVERGENCE (vs upstream reubeno/brush@main): +// Upstream writes the entire heredoc/here-string body into the pipe +// synchronously on the calling thread. That deadlocks any time the body +// exceeds the OS pipe buffer because the reader is not handed to the +// downstream command (and therefore not drained) until after this +// function returns. Concrete buffer sizes: +// +// * Linux: 64 KiB default, growable via `F_SETPIPE_SZ` up to +// `/proc/sys/fs/pipe-max-size` (1 MiB default). +// * macOS: 16-64 KiB, no `F_SETPIPE_SZ` equivalent. +// * Windows: ~4 KiB (`CreatePipe(nSize = 0)`), no portable knob to +// raise it. +// +// We keep the `F_SETPIPE_SZ` fast path for Linux (avoids a thread spawn +// for the common in-process case) but fall through to a detached writer +// thread on every other platform with OS threads, and on Linux when the +// kernel rejects the requested size (body > `pipe-max-size`). The thread +// owns the writer; it terminates naturally when the consumer drains the +// pipe or drops the reader (`BrokenPipe`), so no `JoinHandle` is retained. +// Targets without OS thread support keep upstream's synchronous write path +// so heredocs continue to work there instead of failing at thread spawn. fn setup_open_file_with_contents(contents: &str) -> Result { let (reader, mut writer) = std::io::pipe()?; - let bytes = contents.as_bytes(); + // Linux fast path: grow the pipe so the entire body fits inline. + // Falls through to the generic writer when (a) `bytes.len()` + // overflows `i32`, or (b) the kernel rejects the requested size + // (body > /proc/sys/fs/pipe-max-size, default 1 MiB). #[cfg(any(target_os = "linux", target_os = "android"))] { use std::os::fd::AsFd as _; - let len = i32::try_from(bytes.len()) - .map_err(|_err| error::Error::from(error::ErrorKind::TooMuchData))?; - nix::fcntl::fcntl(reader.as_fd(), nix::fcntl::FcntlArg::F_SETPIPE_SZ(len))?; + if let Ok(len) = i32::try_from(bytes.len()) + && nix::fcntl::fcntl(reader.as_fd(), nix::fcntl::FcntlArg::F_SETPIPE_SZ(len)).is_ok() + { + writer.write_all(bytes)?; + drop(writer); + return Ok(reader.into()); + } + } + #[cfg(target_family = "wasm")] + { + writer.write_all(bytes)?; + drop(writer); + return Ok(reader.into()); + } + #[cfg(not(target_family = "wasm"))] + { + // Generic path: detached writer thread. Writing inline deadlocks + // once `bytes.len()` exceeds the OS pipe buffer (Windows ~4 KiB, + // macOS 16-64 KiB), neither of which has a `F_SETPIPE_SZ` + // equivalent. + let payload = bytes.to_vec(); + std::thread::Builder::new() + .name("brush-heredoc-writer".into()) + .spawn(move || { + // `BrokenPipe` is expected when the consumer drops the + // reader before the body is fully written; there is + // nothing useful to do with that error here. + let _ = writer.write_all(&payload); + })?; } - - writer.write_all(bytes)?; - drop(writer); Ok(reader.into()) } diff --git a/crates/pi-natives/src/grep.rs b/crates/pi-natives/src/grep.rs index 27b144a62..058dd1b58 100644 --- a/crates/pi-natives/src/grep.rs +++ b/crates/pi-natives/src/grep.rs @@ -922,6 +922,7 @@ mod tests { use std::{ fs, path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -934,11 +935,14 @@ mod tests { impl TempDirGuard { fn new() -> Self { - let unique = SystemTime::now() + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("system time is after UNIX_EPOCH") .as_nanos(); - let path = std::env::temp_dir().join(format!("pi-grep-test-{unique}")); + let seq = COUNTER.fetch_add(1, Ordering::Relaxed); + let pid = std::process::id(); + let path = std::env::temp_dir().join(format!("pi-grep-test-{pid}-{nanos}-{seq}")); fs::create_dir_all(&path).expect("create temp test directory"); Self(path) } diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index 77665724e..36c141a8b 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -67,5 +67,5 @@ use napi_derive::napi; /// MUST stay in sync with `VERSION_SENTINEL_EXPORT` in /// `packages/natives/native/index.js` (which derives the name from /// `package.json#version`). -#[napi(js_name = "__piNativesV15_3_2")] +#[napi(js_name = "__piNativesV15_4_3")] pub const fn pi_natives_version_sentinel() {} diff --git a/crates/pi-shell/src/shell.rs b/crates/pi-shell/src/shell.rs index c8ce084de..919f0ec6d 100644 --- a/crates/pi-shell/src/shell.rs +++ b/crates/pi-shell/src/shell.rs @@ -1919,4 +1919,32 @@ mod tests { } assert_eq!(stdout, b"prod:8080"); } + + /// Regression for a Windows/macOS deadlock in + /// `brush_core::interp::setup_open_file_with_contents`. The body is + /// 256 KiB — well past the default pipe buffer on every platform + /// (Windows ~4 KiB, macOS 16-64 KiB, Linux 64 KiB), so any inline + /// `write_all` on the calling thread blocks forever. The `:` builtin + /// never reads its stdin, so the only way `echo done` runs is if the + /// heredoc writer is decoupled from the main thread (or, on Linux, + /// the pipe buffer was grown via `F_SETPIPE_SZ`). The + /// `tokio::time::timeout` is the safety net that turns a regression + /// into a 10 s failure instead of hanging CI for the full + /// hard-timeout window. + #[tokio::test(flavor = "multi_thread")] + async fn large_heredoc_does_not_deadlock() { + let body = "X".repeat(256 * 1024); + let command = format!(": <<'EOF'\n{body}\nEOF\necho done"); + let options = ShellExecuteOptions { command, ..Default::default() }; + + let result = time::timeout( + Duration::from_secs(10), + execute_shell(options, None, CancelToken::default()), + ) + .await + .expect("execute_shell hung past 10 s — heredoc writer deadlocked") + .expect("execute_shell errored"); + + assert_eq!(result.exit_code, Some(0), "command did not run to completion"); + } } diff --git a/docs/auth-broker-gateway.md b/docs/auth-broker-gateway.md index 07aabda59..e13219a91 100644 --- a/docs/auth-broker-gateway.md +++ b/docs/auth-broker-gateway.md @@ -47,8 +47,9 @@ The broker is the only writer of OAuth refresh tokens. Clients (including the ga ``` omp auth-broker serve [--bind=host:port] # boot the broker omp auth-broker token [--regenerate] [--json] # print or rotate the bearer token -omp auth-broker login [--via=user@host] [--dry-run] -omp auth-broker logout +omp auth-broker login [] [--via=user@host] [--dry-run] +omp auth-broker logout [] +omp auth-broker list [--json] omp auth-broker import [--provider=] [--include-disabled] [--dry-run] [--json] omp auth-broker migrate --from-local [--dry-run] [--json] omp auth-broker status [--json] @@ -56,8 +57,9 @@ omp auth-broker status [--json] - `serve` opens the local SQLite store at `getAgentDbPath()` and binds an HTTP listener (default `127.0.0.1:8765`). On startup a token is ensured at `/auth-broker.token` (mode `0600`, `0700` parent dir). The background refresher refreshes any OAuth credential whose `expires - Date.now() < refreshSkewMs` (default 5 min) every `refreshIntervalMs` (default 60 s). - `token` prints the cached bearer or generates a new one. `--regenerate` rotates it. -- `login ` runs the per-provider OAuth flow locally, or — with `--via=user@host` — `ssh -L :127.0.0.1: user@host omp auth-broker login ` so the OAuth callback hits the local browser but the credential is written on the broker host. Built-in callback ports: `anthropic:54545`, `openai-codex:1455`, `google-gemini-cli:8085`, `google-antigravity:51121`, `gitlab-duo:8080`. -- `logout ` deletes every credential row for ``. +- `login []` runs the per-provider OAuth flow locally — when no provider is supplied, it falls back to an interactive numbered picker. With `--via=user@host` it shells out `ssh -L :127.0.0.1: user@host omp auth-broker login ` so the OAuth callback hits the local browser but the credential is written on the broker host (`--via` requires ``). Built-in callback ports: `anthropic:54545`, `openai-codex:1455`, `google-gemini-cli:8085`, `google-antigravity:51121`, `gitlab-duo:8080`. The OAuth dance is driven in-process via `AuthStorage.login()` — there is no longer a `pi-ai` bin to spawn. +- `logout []` deletes every credential row for ``. With no argument it shows an interactive numbered picker of currently-stored providers. +- `list` enumerates every registered OAuth provider id/name (the union of built-ins + `registerOAuthProvider` custom providers). `--json` emits a machine-readable array. - `import ` imports CLIProxyAPI-style JSON credentials into the local SQLite store. Maps `type` field → omp provider (`claude → anthropic`, `codex → openai-codex`, `gemini → google-gemini-cli`, `antigravity → google-antigravity`, `gemini-cli → google-gemini-cli`). - `migrate --from-local` walks the local SQLite store + env-derived credentials and idempotently uploads them to the configured broker (`POST /v1/credential`). - `status` health-pings the configured remote broker. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 01ec224b7..c03561c07 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -131,7 +131,7 @@ When `CLAUDE_CODE_USE_FOUNDRY` is enabled, Anthropic requests switch to Foundry | `AWS_DEFAULT_REGION` | Fallback if `AWS_REGION` unset | | `AWS_PROFILE` | Enables named profile auth path | | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | Enables IAM key auth path | -| `AWS_BEARER_TOKEN_BEDROCK` | Enables bearer token auth path | +| `AWS_BEARER_TOKEN_BEDROCK` | Highest-precedence bearer token auth path; skips AWS profile/credential-chain lookup when set | | `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` / `AWS_CONTAINER_CREDENTIALS_FULL_URI` | Enables ECS task credential path | | `AWS_WEB_IDENTITY_TOKEN_FILE` + `AWS_ROLE_ARN` | Enables web identity auth path | | `AWS_BEDROCK_SKIP_AUTH` | If `1`, injects dummy credentials (proxy/non-auth scenarios) | @@ -298,7 +298,7 @@ Extra conditional behavior: | `LM_STUDIO_BASE_URL` | Default implicit LM Studio discovery base URL override (`http://127.0.0.1:1234/v1` if unset) | | `OLLAMA_BASE_URL` | Default implicit Ollama discovery base URL override (`http://127.0.0.1:11434` if unset) | | `LLAMA_CPP_BASE_URL` | Default implicit Llama.cpp discovery base URL override (`http://127.0.0.1:8080` if unset) | -| `PI_EDIT_VARIANT` | Forces edit tool variant when valid (`patch`, `replace`, `hashline`, `atom`, `vim`, `apply_patch`) | +| `PI_EDIT_VARIANT` | Forces edit tool variant when valid (`patch`, `replace`, `hashline`, `apply_patch`) | | `PI_FORCE_IMAGE_PROTOCOL` | Forces supported image protocol (`kitty`, `iterm2`/`iterm`, `sixel`, `none`) where used | | `PI_ALLOW_SIXEL_PASSTHROUGH` | Allows SIXEL passthrough when `PI_FORCE_IMAGE_PROTOCOL=sixel` | | `PI_NO_PTY` | If `1`, disables interactive PTY path for bash tool | diff --git a/docs/tools/ast-grep.md b/docs/tools/ast-grep.md index 97282d6ef..9bda347e8 100644 --- a/docs/tools/ast-grep.md +++ b/docs/tools/ast-grep.md @@ -10,7 +10,7 @@ - `crates/pi-natives/src/language/mod.rs` — language aliases and extension inference - `packages/coding-agent/src/tools/path-utils.ts` — path/glob parsing and multi-path resolution - `packages/coding-agent/src/tools/render-utils.ts` — parse-error dedupe and display caps - - `packages/coding-agent/src/tools/match-line-format.ts` — anchor-prefixed match rendering + - `packages/coding-agent/src/tools/match-line-format.ts` — hashline match rendering - `packages/coding-agent/src/utils/file-display-mode.ts` — hashline vs line-number output mode - `packages/natives/native/index.d.ts` — JS-visible native binding contract @@ -36,7 +36,7 @@ Pattern grammar and language support exposed to the model: - Single-shot tool result. - Model-facing `content` is one text block: - grouped by file for directory/multi-file searches, - - match lines rendered as `*LINE+HASH|text` in hashline mode or `*LINE|text` otherwise, + - match lines rendered under `¶PATH#HASH` as `*LINE:text` in hashline mode or `*LINE|text` otherwise, - continuation lines for multi-line matches rendered with a leading space, - optional `meta: NAME=value` lines when ast-grep captured metavariables. - If no matches are found, text is `No matches found` or `No matches found. Parse issues mean the query may be mis-scoped; narrow paths before concluding absence.` plus formatted parse issues. diff --git a/docs/tools/edit.md b/docs/tools/edit.md index 57b6eb020..99e77f5ec 100644 --- a/docs/tools/edit.md +++ b/docs/tools/edit.md @@ -13,7 +13,7 @@ - `packages/coding-agent/src/hashline/apply.ts` — validates anchors and applies edits - `packages/coding-agent/src/hashline/anchors.ts` — stale-anchor mismatch formatting - `packages/coding-agent/src/hashline/recovery.ts` — cache-based stale-anchor recovery - - `packages/coding-agent/src/hashline/hash.ts` — computes `LINEhh|` anchors shared with `read`/`search` + - `packages/coding-agent/src/hashline/hash.ts` — computes 4-hex file hashes and `LINE:TEXT` display lines shared with `read`/`search` - `packages/coding-agent/src/edit/file-read-cache.ts` — per-session read snapshot cache - `packages/coding-agent/src/tools/read.ts` — emits anchored lines and records read snapshots - `packages/coding-agent/src/tools/search.ts` — records sparse snapshots from matches/context @@ -26,23 +26,27 @@ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `input` | `string` | Yes | One or more edit sections. First non-blank line must be `¶PATH` unless the caller supplies the legacy fallback `path` outside the model schema and the body already looks like hashline ops (`packages/coding-agent/src/hashline/input.ts`). Optional `*** Begin Patch` / `*** End Patch` envelope is ignored if present. | +| `input` | `string` | Yes | One or more edit sections. Anchored sections must start with `¶PATH#HASH`; unbound `¶PATH` is allowed only for new-file / `BOF` / `EOF` boundary inserts. Optional `*** Begin Patch` / `*** End Patch` envelope is ignored if present. | Patch language inside `input`: -- Section header: `¶PATH` -- Insert after: `ANCHOR↓` -- Insert before: `ANCHOR↑` -- Replace/delete range: `A-B→` -- Single-line replace/delete sugar: `A→` means `A-A→` -- `A-B→` with no payload deletes the range. To keep a blank line, include one explicit empty payload line. -- Inline payload: content after `↓`/`↑`/`→` on the same line is the first payload line; subsequent lines append to it -- Special anchors: `BOF`, `EOF` -- Anchor token: `<2-char-hash>`, for example `41th` +- Section header: `¶PATH#HASH` for anchored edits, `¶PATH` for BOF/EOF-only inserts +- Insert after: `LINE↓[payload]` +- Insert before: `LINE↑[payload]` +- Replace range: `A-B:[payload]` +- Single-line replace sugar: `A:[payload]` means `A-A:[payload]` +- Delete range: `A-B!` +- Single-line delete sugar: `A!` means `A-A!` +- **Payload semantics:** the first payload line is whatever follows the sigil on the op line itself; additional payload lines follow on subsequent lines and append after the inline first line. An empty inline (just the sigil followed by a newline) means the first payload line is empty. So bare `A↑` / `A↓` insert one blank line, and bare `A:` / `A-B:` replace the line/range with one blank line. But `A↓\nfoo` inserts blank-then-`foo`, not just `foo` — for a single-line insert, put `foo` inline as `A↓foo`. +- `!` deletes and forbids payload. +- Read lines like `84:content` are already valid single-line replacements. +- Special anchors: `BOF`, `EOF` (both support inline payload, e.g. `BOF↓export const done = true;`). +- Anchor token: bare line number, for example `41` +- File binding: 4-hex hash in the section header, for example `¶src/a.ts#1a2b` -Anchors come from `read`/`search` output. `read` formats lines as `LINEhh|TEXT` via `formatHashLine` / `formatHashLines` in `packages/coding-agent/src/hashline/hash.ts`; copy only the token left of `|` into op lines. +Anchors come from `read`/`search` output. `read` emits a `¶PATH#HASH` header and lines as `LINE:TEXT`; copy the header into the edit section and copy only the line number into op lines. -Other edit modes exist (`replace`, `patch`, `vim`, `apply_patch`) and are selected outside the tool payload by `resolveEditMode()` in `packages/coding-agent/src/utils/edit-mode.ts`. Their schemas are different; this document covers the default hashline mode. +Other edit modes exist (`replace`, `patch`, `apply_patch`) and are selected outside the tool payload by `resolveEditMode()` in `packages/coding-agent/src/utils/edit-mode.ts`. Their schemas are different; this document covers the default hashline mode. ## Outputs - Single-shot tool result; hashline mode does not use a `resolve` preview/apply handshake. @@ -67,20 +71,20 @@ Warnings: - While the model is still typing arguments, the TUI can compute a diff preview with `packages/coding-agent/src/edit/streaming.ts`; that preview is not a deferred action and does not block execution. ## Flow -1. `EditTool.execute()` in `packages/coding-agent/src/edit/index.ts` resolves the active mode. Default is `hashline`; `customFormat` exposes `packages/coding-agent/src/hashline/grammar.lark` with `$HFMT$` / `$HOP_INSERT_BEFORE$` / `$HOP_INSERT_AFTER$` / `$HOP_REPLACE$` / `$HOP_CHARS$` / `$HFILE$` placeholders filled from `packages/coding-agent/src/hashline/hash.ts`. -2. `executeHashlineSingle()` in `packages/coding-agent/src/hashline/execute.ts` splits the raw `input` into `¶PATH` sections with `splitHashlineInputs()`. +1. `EditTool.execute()` in `packages/coding-agent/src/edit/index.ts` resolves the active mode. Default is `hashline`; `customFormat` exposes `packages/coding-agent/src/hashline/grammar.lark` with `$HFILE_HASH$` / `$HOP_INSERT_BEFORE$` / `$HOP_INSERT_AFTER$` / `$HOP_REPLACE$` / `$HOP_CHARS$` / `$HFILE$` placeholders filled from `packages/coding-agent/src/hashline/hash.ts`. +2. `executeHashlineSingle()` in `packages/coding-agent/src/hashline/execute.ts` splits the raw `input` into `¶PATH#HASH` / `¶PATH` sections with `splitHashlineInputs()`. 3. If multiple sections target the same path, `mergeSamePathSections()` concatenates them before execution so every op still refers to the original file snapshot. 4. Multi-section calls run a preflight pass (`preflightHashlineSection()`): parse ops, enforce plan-mode write rules, load the current file, reject anchor-scoped edits against missing files, reject auto-generated files, apply edits in memory, and fail if the result is a no-op. This prevents partial batches. 5. `parseHashlineWithWarnings()` in `packages/coding-agent/src/hashline/parser.ts` tokenizes the diff body: - ignores blank lines and optional `*** Begin Patch` - stops at `*** End Patch` - stops at `*** Abort` and emits `ABORT_WARNING` - - turns `↓` / `↑` payload runs into one `insert` edit per payload line - - turns `A-B→` with payload into inserts before `A`, then deletes for `A-B` - - turns `A-B→` with no payload into one `delete` edit per line in the range; a blank-in-place edit requires one explicit empty payload line -6. `applyHashlineEdits()` in `packages/coding-agent/src/hashline/apply.ts` validates every referenced anchor before mutating anything. Each anchor hash is recomputed from current file content with `computeLineHash()`. -7. If any anchor hash differs, `applyHashlineEdits()` throws `HashlineMismatchError`. `execute.ts` catches only that class and calls `tryRecoverHashlineWithCache()`. -8. Recovery replays the edits against the most recent cached read/search snapshot for that path (`packages/coding-agent/src/edit/file-read-cache.ts`), then 3-way merges the result onto current disk content using `Diff.applyPatch(..., { fuzzFactor: 3 })` in `packages/coding-agent/src/hashline/recovery.ts`. On success the edit proceeds with a warning; on failure the original mismatch error is re-thrown. + - turns `↓` / `↑` payload runs (inline plus subsequent lines) into one `insert` edit per payload line + - turns `A-B:` with payload into inserts before `A`, then deletes for `A-B` + - turns `A-B!` into one `delete` edit per line in the range; payload is forbidden +6. `executeHashlineSingle()` computes the current file hash before applying anchored edits. If it differs from the section `#HASH`, recovery tries the read/search snapshot cache before any write. +7. `applyHashlineEdits()` validates only line bounds, then applies the already hash-bound line-number edits. +8. Recovery replays the edits against the cached snapshot for the section hash (`packages/coding-agent/src/edit/file-read-cache.ts`), then 3-way merges the result onto current disk content using `Diff.applyPatch(..., { fuzzFactor: 0 })` in `packages/coding-agent/src/hashline/recovery.ts`. On success the edit proceeds with a warning; on failure a `HashlineMismatchError` is surfaced. 9. Before splicing lines, `absorbReplacementBoundaryDuplicates()` normalizes some malformed-but-recoverable ranges: - duplicate prefix/suffix lines adjacent to a replacement can be absorbed by widening the delete range - pure inserts can auto-drop duplicated leading/trailing payload lines when `edit.hashlineAutoDropPureInsertDuplicates` is enabled @@ -98,61 +102,71 @@ Warnings: - `replace` — exact/fuzzy old/new text replacement (`packages/coding-agent/src/edit/modes/replace.ts`). - `patch` — structured JSON diff-hunk mode (`packages/coding-agent/src/edit/modes/patch.ts`). - `apply_patch` — freeform Codex-style `*** Begin Patch` envelope, internally expanded into patch-mode entries (`packages/coding-agent/src/edit/modes/apply-patch.ts`). -- `vim` — persistent modal editing buffer (`packages/coding-agent/src/tools/vim.ts`). -Hashline op examples: +Hashline op examples (single-line payloads are inline; multi-line payloads continue on subsequent lines): ```text -¶src/a.ts -4fb↓ -const added = true; +¶src/a.ts#1a2b +4↓const added = true; ``` ```text -¶src/a.ts -4fb↑ -const addedBefore = true; +¶src/a.ts#1a2b +4↑const addedBefore = true; ``` ```text -¶src/a.ts -4fb-6qx→ +¶src/a.ts#1a2b +4-6:const replacement = true; ``` ```text -¶src/a.ts -4fb-5dm→ -const clean = (name || DEF).trim(); +¶src/a.ts#1a2b +4-5:const clean = (name || DEF).trim(); return clean.length === 0 ? DEF : clean.toUpperCase(); ``` +```text +¶src/a.ts#1a2b +4:const clean = (name || DEF).trim(); +``` + BOF/EOF examples: ```text ¶src/a.ts -BOF↓ -const HEADER = true; +BOF↓const HEADER = true; ``` ```text ¶src/a.ts -EOF↓ -export const done = true; +EOF↓export const done = true; ``` Delete / blank examples: ```text -¶src/a.ts -4fb→ +¶src/a.ts#1a2b +4! ``` ```text -¶src/a.ts -4fb→ +¶src/a.ts#1a2b +4: +``` -EOF↓ -export const done = true; +```text +¶src/a.ts#1a2b +4-6! +``` + +Multi-file example: + +```text +¶src/a.ts#1a2b +4:const enabled = true; +¶src/b.ts#3c4d +20! ``` ## Side Effects @@ -173,50 +187,51 @@ export const done = true; ## Limits & Caps - Default mode is `hashline` (`DEFAULT_EDIT_MODE`) in `packages/coding-agent/src/utils/edit-mode.ts`. -- Anchor hashes are always 2 lowercase letters from a stable 647-entry bigram table (`HL_BIGRAMS_COUNT`) in `packages/coding-agent/src/hashline/hash.ts`. +- File hashes are 4 lowercase hex chars from `computeFileHash()` in `packages/coding-agent/src/hashline/hash.ts`. - The visible mismatch report shows 2 lines of context on each side (`MISMATCH_CONTEXT`) in `packages/coding-agent/src/hashline/constants.ts`. -- Stale-anchor recovery uses `fuzzFactor: 3` (`HASHLINE_RECOVERY_FUZZ_FACTOR`) in `packages/coding-agent/src/hashline/recovery.ts`. +- Stale-anchor recovery uses `fuzzFactor: 0` (`HASHLINE_RECOVERY_FUZZ_FACTOR`) in `packages/coding-agent/src/hashline/recovery.ts`. - The per-session read cache keeps at most 30 paths (`MAX_PATHS_PER_SESSION`) in `packages/coding-agent/src/edit/file-read-cache.ts`. - Hashline streaming chunk defaults are 200 lines or 64 KiB per chunk (`packages/coding-agent/src/hashline/types.ts`, consumed by `packages/coding-agent/src/hashline/stream.ts`). -- `HL_OP_INSERT_BEFORE` is `↑`, `HL_OP_INSERT_AFTER` is `↓`, `HL_OP_REPLACE` is `→`, `HL_OP_CHARS` is `↑↓→`, `HL_FILE_PREFIX` is `¶`, and `HL_BODY_SEP` is `|` (`packages/coding-agent/src/hashline/hash.ts`). +- `HL_OP_INSERT_BEFORE` is `↑`, `HL_OP_INSERT_AFTER` is `↓`, `HL_OP_REPLACE` is `:`, `HL_OP_DELETE` is `!`, `HL_OP_CHARS` is `↑↓:!`, `HL_FILE_PREFIX` is `¶`, `HL_FILE_HASH_SEP` is `#`, and `HL_LINE_BODY_SEP` is `:` (`packages/coding-agent/src/hashline/hash.ts`). ## Errors - Missing section header: - - `input must begin with "¶PATH" on the first non-blank line; got: ... Example: "¶src/foo.ts" then edit ops.` + - `input must begin with "¶PATH#HASH" on the first non-blank line for anchored edits; got: ...` - Empty header: - `Input header "¶" is empty; provide a file path.` +- Missing hash for anchored edit: + - `Missing hashline file hash for anchored edit to ; use ¶#hash from your latest read.` +- Line-hash anchors in edit ops: + - `line N: edit ops use bare line numbers. Copy the ¶PATH#hash header, then use anchors like 42, 42-45, BOF, or EOF.` - Bad anchor token: - - `line N: expected a full anchor such as "119sr"; got "...".` + - `line N: expected a line number such as "119"; got "...".` - Bad range syntax: - - `line N: range must be ANCHOR or ANCHOR-ANCHOR (one dash, no spaces); got ...` + - `line N: range must be LINE or LINE-LINE (one dash, no spaces); got ...` - `line N: range A-B ends before it starts.` - - `line N: range A-B uses two different hashes for the same line.` -- Missing payload for `↓` / `↑`: - - `line N: ↑ and ↓ operations require at least one verbatim payload line.` +- Payload forbidden for `!`: + - `line N: ! deletes only. Payload is forbidden after !; use : to replace.` - Stray payload line: - - `line N: payload line has no preceding ↑, ↓, or → operation.` + - `line N: payload line has no preceding ↑, ↓, :, or ! operation.` - Unknown op: - - `line N: unrecognized op. Use ANCHOR↑ (insert before), ANCHOR↓ (insert after), or A-B→ (replace/delete).` -- Delete vs blank: - - `A-B→` with no payload deletes. To blank in place, include one explicit empty payload line before the next op/header/EOF. + - `line N: unrecognized op. Use LINE↑ (insert before), LINE↓ (insert after), LINE: / A-B: (replace), or LINE! / A-B! (delete).` - Missing file for anchor-scoped edits: - `File not found: ` - Out-of-range anchor: - `Line N does not exist (file has M lines)` -- Stale anchors throw `HashlineMismatchError`. The error message contains re-read guidance and reprints nearby current file lines as `LINEhh|TEXT`; mismatched lines are marked `*`. `displayMessage` renders the same information in a code-frame style. +- Stale file hash throws `HashlineMismatchError`. The error contains both hashes, re-read guidance, and nearby current file lines as `*LINE:TEXT` / ` LINE:TEXT`. - No-op edit: - `Edits to resulted in no changes being made.` -- Recovery failure is silent internally: if cache-based merge cannot prove a valid result, the original mismatch error is surfaced unchanged. +- Recovery failure is silent internally: if cache-based merge cannot prove a valid result, the mismatch error is surfaced unchanged. ## Notes -- `read` and `search` are the authoritative source of anchors. The edit parser does not want the trailing `|TEXT`; copy only the `LINEhh` token. +- `read` and `search` are the authoritative source of section hashes. Copy `¶PATH#HASH`; op lines use bare line numbers and do not want the trailing `:TEXT`. - Multi-op patches are parsed against the original file snapshot. Do not renumber later anchors after earlier ops; `applyHashlineEdits()` buckets and applies them bottom-up. -- `A-B→` is not a primitive replace in the parser. With payload, it expands to inserts before `A` plus deletes for `A-B`; with no payload, it only deletes `A-B`. To blank in place, include one explicit empty payload line. Stale-anchor checking still happens on the original range lines. -- Interior lines of a multi-line range use hash `**` (`RANGE_INTERIOR_HASH`) and are not individually verified; only the first and last anchor hashes are checked. -- `computeLineHash()` trims trailing whitespace before hashing. Anchors survive line-ending changes and trailing-space-only changes, but not substantive line edits. -- For punctuation-only lines, the hash mixes in the line number; identical `}` lines on different lines intentionally get different anchors. -- `splitHashlineInputs()` normalizes absolute `¶PATH` headers back to a cwd-relative path when the file is inside the current working tree. Headers with any run of leading `¶` chars (e.g. `¶foo.ts`, `¶¶foo.ts`, `¶¶¶foo.ts`) are accepted; the canonical form is `¶PATH`. -- Optional `*** Begin Patch` / `*** End Patch` markers are accepted in hashline mode, but the file sections are still `¶PATH`-based, not Codex `*** Update File:` hunks. +- Failed hand-edits often come from sequentially shifting later anchors inside the same patch. Treat every op as using the line numbers from the original section header. +- `A-B:` is not a primitive replace in the parser. With payload, it expands to inserts before `A` plus deletes for `A-B`. `A-B!` is the direct delete form. Bare `A:` / `A-B:` (no payload) replaces with a single blank line; bare `↑` / `↓` insert a blank line. +- Inline payload tip: trailing whitespace on the op line is trimmed. To preserve trailing spaces in the inserted/replacement content, put that content on the next line instead of inline. +- `computeFileHash()` normalizes CR characters and trailing whitespace before hashing. The section survives line-ending and trailing-space-only changes, but not substantive file edits. +- `splitHashlineInputs()` normalizes absolute `¶PATH#HASH` headers back to a cwd-relative path when the file is inside the current working tree. Headers with any run of leading `¶` chars (e.g. `¶foo.ts`, `¶¶foo.ts`, `¶¶¶foo.ts`) are accepted; the canonical form is `¶PATH#HASH` for anchored edits. +- Optional `*** Begin Patch` / `*** End Patch` markers are accepted in hashline mode, but the file sections are still `¶PATH#HASH`-based, not Codex `*** Update File:` hunks. - `*** Abort` terminates parsing early and returns `ABORT_WARNING`; ops parsed before the marker still apply. - File-read cache invalidation is conflict-based, not write-through invalidation. If `read` later records content for a line that disagrees with the cached snapshot, the entire snapshot for that path is replaced with the newly observed lines (`packages/coding-agent/src/edit/file-read-cache.ts`). - There is no resolve-style apply/discard phase for hashline edits. The only preview path is the transient TUI diff preview in `packages/coding-agent/src/edit/streaming.ts`. diff --git a/docs/tools/eval.md b/docs/tools/eval.md index 573241f3b..56753e4f5 100644 --- a/docs/tools/eval.md +++ b/docs/tools/eval.md @@ -90,21 +90,22 @@ Side-channel artifacts: - `js` is gated on `eval.js !== false`. - A disabled or unavailable requested backend throws `ToolError`; there is no auto-fallback or sniffing. 3. The tool allocates an `OutputSink`, a `TailBuffer`, per-cell result objects, and a `sessionAbortController`. `session.trackEvalExecution?.(...)` can wrap the whole run for external cancellation tracking. -4. Cells execute sequentially. For each cell, `execute()`: +4. It resolves the executor session id from `session.getEvalSessionId?.()`, falling back to `defaultEvalSessionId(session)`. Subagents inherit the parent's id so both sides share the same JS VM and Python kernel for each backend. +5. Cells execute sequentially within one eval tool call. For each cell, `execute()`: - clamps `(cell.timeout ?? 30) * 1000` ms through `clampTimeout("eval", ...)` - builds a combined abort signal from the tool signal, the timeout, and the session abort controller - marks the cell `running` and emits an update - calls the backend’s `execute()` with `cwd`, `sessionId`, `sessionFile`, `kernelOwnerId`, `deadlineMs`, `reset` (defaults to `false`), artifact info, and chunk callback -5. JS cells dispatch through `packages/coding-agent/src/eval/js/index.ts` into `executeJs()`; Python cells dispatch through `packages/coding-agent/src/eval/py/index.ts` into `executePython()`. -6. Backend text chunks stream into the shared `OutputSink`; rich outputs are accumulated separately as JSON, images, markdown markers, and status events. -7. After each cell: +6. JS cells dispatch through `packages/coding-agent/src/eval/js/index.ts` into `executeJs()`; Python cells dispatch through `packages/coding-agent/src/eval/py/index.ts` into `executePython()`. +7. Backend text chunks stream into the shared `OutputSink`; rich outputs are accumulated separately as JSON, images, markdown markers, and status events. +8. After each cell: - text output is trimmed and stored on that cell result - multi-cell runs prefix text with `[i/n]` and the optional title - cancellations return early with `isError: true` and a cell-specific abort message - non-zero exit codes return early with `isError: true` and a message naming the failed cell - later cells are skipped after the first error, but earlier cell state persists in the underlying runtime -8. On success, the tool joins all cell outputs, synthesizes `(no text output)` or `(no output)` when needed, and attaches truncation metadata from `summarizeFinal()`. -9. The renderer uses `details.cells`, `details.jsonOutputs`, and `details.statusEvents` to build notebook-style output. `mergeCallAndResult = true` and `inline = true`, so call and result render together in the transcript. +9. On success, the tool joins all cell outputs, synthesizes `(no text output)` or `(no output)` when needed, and attaches truncation metadata from `summarizeFinal()`. +10. The renderer uses `details.cells`, `details.jsonOutputs`, and `details.statusEvents` to build notebook-style output. `mergeCallAndResult = true` and `inline = true`, so call and result render together in the transcript. ## Modes / Variants @@ -121,8 +122,8 @@ If the requested backend is disabled or unavailable, the tool throws `ToolError` Implemented in `packages/coding-agent/src/eval/js/context-manager.ts` and `packages/coding-agent/src/eval/js/prelude.txt`. -- Persistent `vm.Context` instances keyed by `js:${sessionId}` in `vmContexts` -- `reset: true` calls `resetVmContext(sessionKey)` before the cell executes +- Persistent worker-backed VM sessions keyed by `js:${sessionId}` +- `reset: true` calls `resetVmContext(sessionKey)` before the cell executes; reset is destructive for all live runs on that JS session - Top-level `await` and bare `return` are supported by wrapping code in an async IIFE when `wrapCode()` sees `await` or `return` - Top-level static `import ... from ...` and dynamic `import(...)` calls are routed through `rewriteImports()`, which sends them via `__omp_import__` so the specifier resolves against the session cwd - Module cache is busted for **local** imports between cells so edits to source files are picked up without restarting the runtime. `__omp_import__` deletes `require.cache[absPath]` before re-importing whenever the original specifier is a filesystem path: relative (`./x`, `../x`, `.`, `..`), POSIX-absolute (`/...`), home-prefixed (`~/...`), or Windows drive-letter (`C:\...` / `C:/...`). Bare specifiers (`react`, `lodash/x`) and URL/scheme specifiers (`node:fs`, `file://...`, `https://...`) are left in cache so package identity stays stable across cells. The cache-bust only fires when the resolved target is an absolute path — unresolved bare-package fallbacks (`resolveImportSpecifier()` returning the original specifier) skip it. @@ -136,7 +137,7 @@ Implemented in `packages/coding-agent/src/eval/js/context-manager.ts` and `packa - `{ type: "image", data, mimeType }` becomes an image output - scalars become text - The VM exposes a restricted `process` subset plus `Buffer`, `fetch`, `Blob`, `File`, `Headers`, `Request`, `Response`, `fs`, `require`, and browser-style globals -- Per-session VM runs are serialized with `runQueued()` +- Concurrent runs on the same VM are not queued end-to-end. Synchronous JS still runs on the single event loop; awaited regions can interleave with sibling runs. ### Python runtime @@ -150,9 +151,9 @@ Implemented in `packages/coding-agent/src/eval/py/executor.ts`, `packages/coding - create/connect kernel - initialize cwd / env / `sys.path` - execute `PYTHON_PRELUDE` -- Python cells run inside IPython/Jupyter, so top-level `await` works; the prompt warns not to use `asyncio.run(...)` -- The Python prelude defines synchronous helpers with the same surface as JS (except `tool.` exists only in JS) -- `display(value)` wraps dict/list/tuple values in `IPython.display.JSON`; rich display MIME bundles are preserved +- Python cells run in the runner's persistent asyncio event loop, so top-level `await` works; the prompt warns not to use `asyncio.run(...)` +- The Python prelude defines helpers with the same surface as JS where practical, including `tool.(args)` through a per-run loopback bridge +- Synchronous statement blocks run in the default executor with ContextVar state copied in; the GIL still serializes bytecode execution, but awaited regions can interleave with sibling cells - Kernel `display_data` / `execute_result` messages map to: - `application/x-omp-status` → status event - `image/png` → image output @@ -184,14 +185,14 @@ A single tool call can mix Python and JS cells. Persistence is per language runt - Session state - `session.assertEvalExecutionAllowed?.()` can block execution. - `session.trackEvalExecution?.(...)` can register cancellable eval work. - - `session.getSessionFile?.()` and `session.getEvalKernelOwnerId?.()` influence kernel reuse and artifact lookup. - - JS VM contexts persist in `vmContexts` across eval calls until reset/disposal. - - Python retained kernels persist in `kernelSessions` until reset, eviction, idle cleanup, or owner cleanup. + - `session.getSessionFile?.()`, `session.getEvalSessionId?.()`, and `session.getEvalKernelOwnerId?.()` influence VM/kernel reuse and artifact lookup. + - JS VM contexts persist across eval calls until reset/disposal. + - Python retained kernels persist until reset, owner cleanup, or process exit. - User-visible prompts / interactive UI - none; stdin requests are rejected programmatically - Background work / cancellation - Python retained kernels have heartbeat and idle cleanup timers. - - Cancellation interrupts a running Python kernel and aborts JS promise waits. + - Cancellation hard-kills/resets the shared executor for that backend: JS terminates the worker, Python sends SIGINT and may escalate to subprocess shutdown. ## Limits & Caps @@ -224,14 +225,21 @@ A single tool call can mix Python and JS cells. Persistence is per language runt - Cancellation is returned, not thrown, once backend execution has started. The tool formats it as a cell failure and sets `details.isError = true`. - If output truncates, the tool still succeeds; truncation is surfaced through `details.meta` and artifact-backed full output when available. +## Shared executor trade-offs + +- Parent agents and subagents share eval state bidirectionally when a subagent inherits the parent's executor id. Mutations in either direction are visible to the other participant. +- Async regions of concurrent runs can interleave. Synchronous JS still blocks the VM event loop; synchronous Python still contends on the GIL. +- Cancelling one run is destructive to the shared backend executor. This is intentional: JS worker termination and Python SIGINT/subprocess shutdown are the only reliable way to interrupt arbitrary user code. +- `reset: true` is destructive for every live run on that backend session id. New starts on that backend are rejected while reset is in flight. + ## Notes - Backend selection is now strictly explicit per cell: `language` must be `"py"` or `"js"`. The previous `*** Cell` header parser, the `eval.lark` constrained grammar, and the sniffer-based fallback have all been removed. - `EvalTool.customFormat` no longer exists. Tool calls flow through the standard JSON schema; there is no Lark-constrained sampling path. -- `tool.()` exists only in JS. Python prelude helpers do not call back into the full tool registry. +- `tool.()` exists in both JS and Python. Python calls route through a per-run loopback bridge keyed by the current cell id. - JS helper paths reject protocol URIs (`://`) in `resolvePath()`; the JS prelude is filesystem-only unless the code calls `tool.read(...)` or another tool explicitly. - Python helper `output(...)` depends on `PI_SESSION_FILE`; it fails outside a session-backed run. - `display()` can produce text and structured outputs from the same value; the renderer prefers markdown over `text/plain` when both exist. - JS static imports are rewritten only at top level. Nested imports stay invalid and surface normal JS syntax/runtime errors. -- `EvalTool` is `concurrency = "exclusive"`, so eval calls do not overlap within a session. +- `EvalTool` is `concurrency = "exclusive"` within one agent session, but parent and subagent sessions can run eval concurrently when they share an inherited executor id. - The tool description shown to the model is templated by backend availability (`getEvalToolDescription()`); if Python is unavailable, the prompt omits Python-specific instructions. diff --git a/docs/tools/read.md b/docs/tools/read.md index f99e75edb..19559390d 100644 --- a/docs/tools/read.md +++ b/docs/tools/read.md @@ -101,11 +101,11 @@ URL selectors are parsed separately in `packages/coding-agent/src/tools/fetch.ts - Default open-ended limit is `min(session setting read.defaultLimit, DEFAULT_MAX_LINES)`. - Explicit ranges expand by `RANGE_LEADING_CONTEXT_LINES = 1` / `RANGE_TRAILING_CONTEXT_LINES = 3` on the constrained sides only. - Non-raw output uses `resolveFileDisplayMode()`: - - hashline anchors when edit mode is hashline, read is not raw, source is mutable, edit tool exists, and `readHashLines !== false` + - hashline numbered output when edit mode is hashline, read is not raw, source is mutable, edit tool exists, and `readHashLines !== false` - otherwise optional line numbers when `readLineNumbers === true` - raw mode suppresses both -- Prefix format in hashline mode is `lineNumber + 2-char line hash + "|"`, e.g. `41th|def alpha():`, from `formatHashLine()` in `packages/coding-agent/src/hashline/hash.ts`. -- Those anchors are what the `edit`/hashline path consumes later; immutable sources and `:raw` intentionally suppress them. +- Prefix format in hashline mode is a `¶PATH#HASH` header followed by `LINE:TEXT`, e.g. `¶src/foo.ts#1a2b` and `41:def alpha():`, from `computeFileHash()` / `formatNumberedLine()` in `packages/coding-agent/src/hashline/hash.ts`. +- The `edit`/hashline path consumes that header plus bare line numbers later; immutable sources and `:raw` intentionally suppress them. ### Directory listings - `#readDirectory()` calls `buildDirectoryTree()` with: diff --git a/docs/tools/search.md b/docs/tools/search.md index 6fbe4badd..855d5fbe3 100644 --- a/docs/tools/search.md +++ b/docs/tools/search.md @@ -21,7 +21,7 @@ | Field | Type | Required | Description | | --- | --- | --- | --- | | `pattern` | `string` | Yes | Regex pattern. `search.ts` trims it and rejects empty input. The native matcher enables multiline only when the pattern text contains a literal newline or the two-character sequence `\\n`. The model prompt explicitly documents literal-brace escaping such as ``interface\\{\\}``, although the native layer also auto-escapes braces that cannot be valid repetition quantifiers. | -| `paths` | `string[]` | Yes | One or more file paths, directory paths, glob-like paths, or internal URLs. Empty strings are rejected after trimming/quote stripping. Internal URLs must resolve to a backing file and cannot contain glob characters. | +| `paths` | `string \| string[]` | Yes | One file path, directory path, glob-like path, internal URL, or an array of those. Empty strings are rejected after trimming/quote stripping. Internal URLs must resolve to a backing file and cannot contain glob characters. | | `i` | `boolean` | No | Case-insensitive search. Defaults to `false`. Passed to native `ignoreCase`. | | `gitignore` | `boolean` | No | Respect `.gitignore` during directory scans. Defaults to `true`. Passed to native `gitignore`. | | `skip` | `number` | No | Global match offset. Defaults to `0`. `search.ts` floors finite numbers and rejects negative or non-finite values. | @@ -29,8 +29,8 @@ ## Outputs The tool returns a single text block in `content[0].text` plus structured `details`. -- Match lines are formatted by `formatMatchLine()` as `*|` for matches and ` |` for context. - - Hashline mode: `*5th|content`, ` 9x}|content`. +- Match lines are formatted by `formatMatchLine()` as `*LINE:content` for matches and ` LINE:content` for context under a `¶PATH#HASH` header in hashline mode. + - Hashline mode: `¶src/login.ts#3c4d`, `*5:content`, ` 9:content`. - Plain mode: `*5|content`, ` 9|content`. - Directory results are grouped by file, with `# ` headings and blank lines between groups. - `details` may include: @@ -49,12 +49,13 @@ The tool returns a single text block in `content[0].text` plus structured `detai - trims `pattern`, rejects empty patterns; - normalizes `skip` to a non-negative integer; - reads `search.contextBefore` and `search.contextAfter` from session settings (`1` and `3` by default); - - enables multiline only when `pattern` contains `\n` or an actual newline. + - enables multiline only when `pattern` contains `\n` or an actual newline; + - wraps a single string `paths` value into a one-element list before path resolution. 2. Each `paths` entry is normalized with `normalizePathLikeInput()`. 3. Internal URLs are resolved through `session.internalRouter`: - glob metacharacters (`*`, `?`, `[`, `{`) are rejected for internal URLs; - URLs without `resource.sourcePath` fail; - - immutable sources are tracked so output can suppress editable hashline anchors per file. + - immutable sources are tracked so output can suppress editable hashline numbered output per file. 4. For multi-path calls, `partitionExistingPaths()` skips only ENOENT entries. If every entry is missing, the tool errors. 5. Path resolution branches: - one entry: `parseSearchPath()` splits `basePath` and optional glob; @@ -116,7 +117,7 @@ The tool returns a single text block in `content[0].text` plus structured `detai ## Limits & Caps - Visible page limit: `100` matches (`DEFAULT_MATCH_LIMIT` in `packages/coding-agent/src/tools/search.ts`). - Native preselection limit: `500` matches (`internalLimit = Math.min(DEFAULT_MATCH_LIMIT * 5, 2000)` in `packages/coding-agent/src/tools/search.ts`). -- Line truncation: `1024` characters per emitted line (`DEFAULT_MAX_COLUMN` in `packages/coding-agent/src/session/streaming-output.ts`). Native grep marks truncated lines; JS reports `linesTruncated`. +- Line truncation: `512` characters per emitted line (`DEFAULT_MAX_COLUMN` in `packages/coding-agent/src/session/streaming-output.ts`). Native grep marks truncated lines; JS reports `linesTruncated`. - Final text truncation: `truncateHead()` default byte cap `50 * 1024` bytes (`DEFAULT_MAX_BYTES` in `packages/coding-agent/src/session/streaming-output.ts`). `search.ts` overrides `maxLines` to `Number.MAX_SAFE_INTEGER`, so normal search output is byte-capped, not line-capped. - Context defaults: `search.contextBefore = 1`, `search.contextAfter = 3` in `packages/coding-agent/src/config/settings-schema.ts`. - Pagination: `skip` is a global match offset. In single-base searches it is pushed into native `offset`; in exact-file/multi-target aggregation it is applied in JS with `matches.slice(skip)`. @@ -140,4 +141,4 @@ The tool returns a single text block in `content[0].text` plus structured `detai - `hidden:true` is hard-coded in `search.ts`; there is no model-facing flag to exclude dotfiles. - `gitignore:false` only affects native directory traversal. It does not disable the tool's own path normalization or explicit-file handling. - When `paths` resolves to multiple exact files, `search.ts` does not apply the native `500` match cap and reports `totalMatches` internally as the post-skip length for that branch. -- The anchor suffix in hashline mode comes from `computeLineHash()` in `packages/coding-agent/src/hashline/hash.ts`; `search` itself only formats it. +- The section hash in hashline mode comes from `computeFileHash()` in `packages/coding-agent/src/hashline/hash.ts`; `search` emits bare line numbers beneath it. diff --git a/docs/tools/write.md b/docs/tools/write.md index 18b5aa97b..5b4047ba0 100644 --- a/docs/tools/write.md +++ b/docs/tools/write.md @@ -49,7 +49,7 @@ Single-shot result. - Archive writes return empty `details`. ## Flow -1. `WriteTool.execute()` in `packages/coding-agent/src/tools/write.ts` strips `LINE+ID|` hashline prefixes from `content` when the session is in hashline display mode. +1. `WriteTool.execute()` in `packages/coding-agent/src/tools/write.ts` strips pasted `¶PATH#HASH` headers and `LINE:` hashline prefixes from `content` when the session is in hashline display mode. 2. It calls `#resolveArchiveWritePath()` first. That uses `parseArchivePathCandidates()` from `packages/coding-agent/src/tools/archive-reader.ts`, checks candidate archive files on disk, and falls back to the longest matching archive suffix even when the archive file does not exist yet. 3. Archive writes call `enforcePlanModeWrite(..., { op: exists ? "update" : "create" })`, then `#writeArchiveEntry()`. - The parent directory of the archive file is created with `fs.mkdir(..., { recursive: true })`. diff --git a/package.json b/package.json index de568d5e0..9c4498b43 100644 --- a/package.json +++ b/package.json @@ -20,13 +20,13 @@ "@bufbuild/protoc-gen-es": "^2.12.0", "@mozilla/readability": "^0.6.0", "@napi-rs/cli": "3.6.2", - "@oh-my-pi/omp-stats": "15.3.2", - "@oh-my-pi/pi-agent-core": "15.3.2", - "@oh-my-pi/pi-ai": "15.3.2", - "@oh-my-pi/pi-coding-agent": "15.3.2", - "@oh-my-pi/pi-natives": "15.3.2", - "@oh-my-pi/pi-tui": "15.3.2", - "@oh-my-pi/pi-utils": "15.3.2", + "@oh-my-pi/omp-stats": "15.4.3", + "@oh-my-pi/pi-agent-core": "15.4.3", + "@oh-my-pi/pi-ai": "15.4.3", + "@oh-my-pi/pi-coding-agent": "15.4.3", + "@oh-my-pi/pi-natives": "15.4.3", + "@oh-my-pi/pi-tui": "15.4.3", + "@oh-my-pi/pi-utils": "15.4.3", "@opentelemetry/api": "^1.9.0", "@opentelemetry/context-async-hooks": "^2.0.0", "@opentelemetry/sdk-trace-base": "^2.0.0", diff --git a/packages/agent/package.json b/packages/agent/package.json index 96976a8a4..2ca3e1155 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-agent-core", - "version": "15.3.2", + "version": "15.4.3", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/agent/src/harmony-leak.ts b/packages/agent/src/harmony-leak.ts index db743a8f4..545c8e16e 100644 --- a/packages/agent/src/harmony-leak.ts +++ b/packages/agent/src/harmony-leak.ts @@ -38,7 +38,7 @@ const SCRIPT_CLASS = const SCRIPT_RUN_RE = new RegExp(`[${SCRIPT_CLASS}]{2,}`, "u"); // Recovery registry. Each entry's parser must recognize the configured -// sentinel (per-tool, see eval/parse.ts and hashline/parser.ts) and surface +// sentinel (per-tool, see eval/parse.ts and hashline/executor.ts) and surface // a warning to the model so it knows to re-issue any remaining work. // `accepts` gates on input shape: tools whose contaminated input doesn't // match the parser's expected DSL fall through to abort-and-retry. diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index ea61a96fc..ca14d03bf 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,6 +1,60 @@ # Changelog ## [Unreleased] +### Removed + +- Removed the `pi-ai` CLI binary (`packages/ai/src/cli.ts`) and its `bin` entry. Use the in-process equivalent in the omp coding-agent CLI: `omp auth-broker login [provider]`, `omp auth-broker logout [provider]`, and `omp auth-broker list`. The library API (`AuthStorage.login()`, `getOAuthProviders()`, etc.) is unchanged. + +### Fixed + +- Fixed usage polling to disable credentials when OAuth refresh fails definitively (for example `invalid_grant`) and clear cached last-good usage data so stale reports no longer remain visible + +## [15.4.3] - 2026-05-26 + +### Fixed + +- Fixed Google Vertex model discovery to use the project-scoped OpenAI-compatible model list so Vertex Model Garden models such as GLM and Claude are available through ADC auth ([#1412](https://github.com/can1357/oh-my-pi/issues/1412)). + +## [15.4.2] - 2026-05-26 + +### Fixed + +- Fixed OpenCode Zen `big-pickle` follow-up requests replaying assistant tool-call turns without DeepSeek-required `reasoning_content`, which caused HTTP 400 errors in thinking mode. + +## [15.4.1] - 2026-05-26 +### Added + +- Added `isOpenAICompletionsProgressChunk` export to identify real progress chunks vs. keepalives in OpenAI completions streams +- Added per-provider stream watchdog overrides via `getStreamIdleTimeoutMs(fallbackMs)` and `getStreamFirstEventTimeoutMs(idleTimeoutMs, fallbackMs)` to allow providers like Google Gemini CLI to extend first-event timeouts without affecting global defaults +- Added `promptCacheKey` to `StreamOptions` and passed it through stream option mapping so callers can specify an explicit prompt-cache key separate from `sessionId` +- Added `promptCacheKey` support to the native server option whitelist so `promptCacheKey` is accepted by `pi-native-server` streams +- Restored the per-provider stream watchdog (`iterateWithIdleTimeout`) on top of the abortable iterator. The lazy stream forwarder in `register-builtins` now wraps every provider's event stream with the first-event + steady-state idle watchdog (`PI_STREAM_FIRST_EVENT_TIMEOUT_MS`, `PI_STREAM_IDLE_TIMEOUT_MS`; aliases honored), and Anthropic / OpenAI Completions / OpenAI Responses / Azure OpenAI Responses / Codex SSE re-emit their per-provider progress predicates so empty keepalive frames cannot keep a stalled stream alive. Reverts the partial regression from #1392 that left Codex WebSocket subagent runs hanging silently for hours when the broker dropped frames between deltas. The Codex WebSocket transport additionally now resets `lastProgressAt` only on progress events (not keepalives), giving the 300s WS-internal idle ceiling the same liveness semantics as the SSE path. + +### Changed + +- Enabled OpenAI Codex WebSocket streams to apply `streamIdleTimeoutMs` and `streamFirstEventTimeoutMs` from `StreamOptions` per request instead of fixed internal defaults +- Changed stream idle watchdog implementation from `iterateUntilAbort` to `iterateWithIdleTimeout`, which now enforces maximum idle gaps between streamed events and distinguishes between first-event and steady-state timeouts +- Changed Anthropic, OpenAI Responses, OpenAI Completions, Azure OpenAI Responses, and OpenAI Codex Responses providers to use the new idle-timeout iterator with per-provider progress predicates so empty keepalive frames cannot keep a stalled stream alive +- Changed Codex WebSocket transport to reset `lastProgressAt` only on progress events (not keepalives), giving the 300s WS-internal idle ceiling the same liveness semantics as the SSE path +- Changed Google Gemini CLI stream forwarding defaults to use a 5-minute first-event floor via per-provider lazy-stream limits to avoid premature first-event timeouts on slow startup +- Changed OpenAI Responses and OpenAI Codex request handling to keep `sessionId` for provider routing and conversation headers while `promptCacheKey` controls the `prompt_cache_key` payload independently +- Changed `StreamOptions.streamIdleTimeoutMs` documentation to clarify it is now wired into every built-in provider and the lazy stream forwarder, and that `streamFirstEventTimeoutMs` is honored at both the SDK-request layer and the iterator-watchdog layer +- Changed OpenAI Responses and OpenAI Codex request handling so `sessionId` continues to drive provider routing and state while `promptCacheKey` controls the `prompt_cache_key` payload +- Changed Google Gemini CLI stream forwarding defaults to use a 5-minute first-event floor to avoid premature first-event timeouts on slow startup +- Changed auth-gateway request mapping to preserve incoming `prompt_cache_key` as both `promptCacheKey` and `sessionId` when routing OpenAI-compatible sessions +- Un-deprecated `StreamOptions.streamIdleTimeoutMs`; the option is wired into every built-in provider and the lazy stream forwarder again. `streamFirstEventTimeoutMs` is now honored at both the SDK-request layer (via `createSdkStreamRequestOptions`) and the iterator-watchdog layer, in cooperation. + +### Removed + +- Removed `installH2Fetch` and the `fetch` patch that forced HTTP/2 on HTTPS requests; callers now use the default Bun `fetch` transport + +### Fixed + +- Fixed first-item timeout handling so `iterateWithIdleTimeout` no longer keeps first-event timers active after the source throws or the consumer stops before semantic progress +- Fixed silent multi-hour hangs on Codex WebSocket subagent runs when the broker dropped frames between deltas by restoring per-provider stream watchdogs with progress-event filtering +- Fixed z.ai/GLM-via-OpenRouter subagent stalls where no-op keepalive chunks reset the idle watchdog indefinitely by filtering non-progress items before resetting the deadline + +## [15.4.0] - 2026-05-26 ### Breaking Changes - Removed `findAnthropicAuth` from `anthropic-auth` and replaced store-driven auth discovery with `buildAnthropicAuthConfig`, requiring callers to provide an already-resolved API key before building Anthropic auth config @@ -9,6 +63,7 @@ - Added `PI_CODEX_WEBSOCKET_FIRST_EVENT_TIMEOUT_MS` and `PI_CODEX_WEBSOCKET_IDLE_TIMEOUT_MS` options to tune Codex WebSocket timeout behavior before fallback - Added `AuthStorage.getOAuthAccess` to return a refreshed OAuth access token with identity metadata (`accountId`, `email`, `projectId`, `enterpriseUrl`) for callers that need bearer-token headers together +- Added Codex WebSocket forwarding to the `onSseEvent` observer so the raw provider-stream debug viewer captures the inbound JSON frames and the outbound request frame from the WS transport using the same synthesized SSE-wire shape (`event:` + `data:` lines, prefixed with a `: ws ← ` (inbound) or `: ws → ` (outbound) comment). ### Changed @@ -26,12 +81,14 @@ ### Fixed - Dropped truncated, thinking-only assistant turns with only `thinking`/`redacted_thinking` blocks and no `text` or `tool` content during message transformation, preventing Anthropic requests from sending consecutive assistant messages after a `max_tokens`/`error`/`aborted` interruption +- Fixed Amazon Bedrock bearer-token authentication to honor `AWS_BEARER_TOKEN_BEDROCK` before resolving AWS profiles or running `credential_process`, matching Bedrock API-key precedence. ([#1399](https://github.com/can1357/oh-my-pi/issues/1399)) - Updated `isRetryableError` to treat Bun HTTP/2 transport errors (`HTTP2StreamReset`, `HTTP2RefusedStream`) as retryable so transient stream-reset failures can be retried - Fixed Codex WebSocket streaming to recover from stalled sessions by falling back to SSE when the first event or subsequent progress is delayed beyond the configured websocket timeout - Fixed expired OAuth handling so provider-level paths no longer attempt direct token refresh calls for expired credentials and instead rely on `AuthStorage` for rotation - Fixed provider streams aborting slow-but-valid first tokens or silent inter-event gaps with OMP-owned first-event/idle watchdog errors. Built-in lazy streams, OpenAI/Anthropic/Azure/Codex SSE, and Codex WebSocket streams now wait for provider output, provider/socket errors, caller aborts, or explicit request-layer timeouts instead of treating provider silence as failure ([#1392](https://github.com/can1357/oh-my-pi/issues/1392)). - Fixed Claude Opus 4.7 on Amazon Bedrock streaming no reasoning output (and appearing to hang on long reasoning runs) because Anthropic silently switched the adaptive-thinking display default to `"omitted"`. The Bedrock provider now sends `thinking.display = "summarized"` by default on Opus 4.7+ adaptive models and on budget-based Claude models, mirroring the existing direct-Anthropic behavior. `BedrockOptions.thinkingDisplay` (`"summarized" | "omitted"`) is exposed for callers that want to opt out, and `hideThinkingSummary` now wires through to the Bedrock case ([#1373](https://github.com/can1357/oh-my-pi/issues/1373)). - Fixed Cursor Composer resume/tool-continuation turns failing with `Cannot send empty user message to Cursor API`. Empty current user turns now use Cursor's `resumeAction` instead of constructing an invalid `userMessageAction` ([#1376](https://github.com/can1357/oh-my-pi/issues/1376)). +- Fixed `pi-ai login moonshot` failing with `invalid temperature: only 1 is allowed for this model` (HTTP 400) because the API-key validator probed `kimi-k2.5` with `temperature: 0`. Moonshot login now validates against `GET /v1/models`, matching the DeepSeek/Fireworks/NanoGPT/ZenMux pattern and authenticating the key without invoking model-specific parameter restrictions. ## [15.3.2] - 2026-05-25 ### Added diff --git a/packages/ai/README.md b/packages/ai/README.md index 3ada12a74..99be71531 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1057,13 +1057,14 @@ Official docs: [Application Default Credentials](https://cloud.google.com/docs/a ### CLI Login -The quickest way to authenticate: +Authenticate via the [`omp`](https://omp.sh) coding-agent CLI, which drives this library's OAuth/API-key flows in-process and persists into `agent.db`: ```bash -bunx @oh-my-pi/pi-ai login # interactive provider selection -bunx @oh-my-pi/pi-ai login anthropic # login to specific provider -bunx @oh-my-pi/pi-ai login vllm # store vLLM API key (or placeholder for local no-auth) -bunx @oh-my-pi/pi-ai list # list available providers +omp auth-broker login # interactive provider selection +omp auth-broker login anthropic # login to a specific provider +omp auth-broker login vllm # store vLLM API key (or placeholder for local no-auth) +omp auth-broker list # list supported providers +omp auth-broker logout # interactive — pick a stored credential to remove ``` Credentials are saved to `agent.db` in the agent directory. `/login qianfan` opens the Qianfan console and stores the pasted API key. diff --git a/packages/ai/package.json b/packages/ai/package.json index 97cdad9d7..a4de241be 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-ai", - "version": "15.3.2", + "version": "15.4.3", "description": "Unified LLM API with automatic model discovery and provider configuration", "homepage": "https://omp.sh", "author": "Can Boluk", @@ -28,9 +28,6 @@ ], "main": "./src/index.ts", "types": "./src/index.ts", - "bin": { - "pi-ai": "./src/cli.ts" - }, "scripts": { "check": "biome check . && bun run check:types", "check:types": "tsgo -p tsconfig.json --noEmit", diff --git a/packages/ai/src/auth-broker/refresher.ts b/packages/ai/src/auth-broker/refresher.ts index 6c226dd5a..551f0a405 100644 --- a/packages/ai/src/auth-broker/refresher.ts +++ b/packages/ai/src/auth-broker/refresher.ts @@ -10,7 +10,7 @@ * snapshot pull surfaces a clean delete on the client. */ import { logger } from "@oh-my-pi/pi-utils"; -import type { AuthStorage } from "../auth-storage"; +import { type AuthStorage, isDefinitiveOAuthFailure } from "../auth-storage"; import { DEFAULT_REFRESH_INTERVAL_MS, DEFAULT_REFRESH_SKEW_MS } from "./types"; export interface AuthBrokerRefresherOptions { @@ -23,16 +23,6 @@ export interface AuthBrokerRefresherOptions { now?: () => number; } -const INVALID_GRANT_REGEX = /invalid_grant|invalid_token|revoked|unauthorized|expired.*refresh|refresh.*expired/i; -const TRANSIENT_REGEX = /timeout|network|fetch failed|ECONNREFUSED/i; -const HTTP_401_403_REGEX = /\b(401|403)\b/; - -function isDefinitiveFailure(errorMsg: string): boolean { - if (INVALID_GRANT_REGEX.test(errorMsg)) return true; - if (HTTP_401_403_REGEX.test(errorMsg) && !TRANSIENT_REGEX.test(errorMsg)) return true; - return false; -} - export interface AuthBrokerRefresherSchedule { enabled: boolean; intervalMs: number; @@ -113,7 +103,7 @@ export class AuthBrokerRefresher { await this.#storage.refreshCredentialById(id); } catch (error) { const errorMsg = String(error); - if (isDefinitiveFailure(errorMsg)) { + if (isDefinitiveOAuthFailure(errorMsg)) { logger.warn("auth-broker refresh failed definitively; disabling credential", { id, error: errorMsg, diff --git a/packages/ai/src/auth-gateway/server.ts b/packages/ai/src/auth-gateway/server.ts index 47233f994..4292bfd21 100644 --- a/packages/ai/src/auth-gateway/server.ts +++ b/packages/ai/src/auth-gateway/server.ts @@ -145,7 +145,9 @@ function buildStreamOptions(parsed: ParsedFormatRequest, api: Api, signal: Abort // Client-supplied `prompt_cache_key` wins; otherwise derive a stable // key from the model + system + tools so prefix caching engages on // Codex-class backends across turns of the same logical conversation. - opts.sessionId = options.promptCacheKey ?? deriveSessionId(parsed.modelId, parsed.context); + const promptCacheKey = options.promptCacheKey ?? deriveSessionId(parsed.modelId, parsed.context); + opts.promptCacheKey = promptCacheKey; + opts.sessionId = promptCacheKey; if (options.thinkingBudgets) { opts.thinkingBudgets = { ...(opts.thinkingBudgets ?? {}), ...options.thinkingBudgets }; } diff --git a/packages/ai/src/auth-gateway/types.ts b/packages/ai/src/auth-gateway/types.ts index 34e5c4b3e..7d9b6fefe 100644 --- a/packages/ai/src/auth-gateway/types.ts +++ b/packages/ai/src/auth-gateway/types.ts @@ -67,7 +67,7 @@ export interface AuthGatewayParsedRequestOptions { serviceTier?: ServiceTier; /** Cache retention hint derived from inbound `cache_control` markers. */ cacheRetention?: CacheRetention; - /** OpenAI Responses `prompt_cache_key`; bridges to pi-ai `sessionId`. */ + /** OpenAI Responses `prompt_cache_key`; also seeds provider routing when no separate session id exists. */ promptCacheKey?: string; /** OpenAI Responses `previous_response_id` for response chaining. */ previousResponseId?: string; diff --git a/packages/ai/src/auth-storage.ts b/packages/ai/src/auth-storage.ts index 3ec460696..786950444 100644 --- a/packages/ai/src/auth-storage.ts +++ b/packages/ai/src/auth-storage.ts @@ -414,6 +414,29 @@ const OAUTH_REFRESH_SKEW_MS = 60_000; */ const MAX_PENDING_DISABLED_EVENTS = 32; +/** + * Classify an OAuth refresh error as a definitive credential failure (the + * refresh token is dead — re-login required) versus a transient blip + * (network/5xx — retry next sweep). + * + * Anchored at module scope so all three refresh sites — in-stream + * {@link AuthStorage.getApiKey}, the usage probe in + * {@link AuthStorage.fetchUsageReports}, and the auth-broker background + * refresher — disable rows on the same criteria. A drifting classifier + * between sites would let stale last-good usage reports surface indefinitely + * while streaming requests correctly tear the row down. + */ +const OAUTH_DEFINITIVE_FAILURE_REGEX = + /invalid_grant|invalid_token|revoked|unauthorized|expired.*refresh|refresh.*expired/i; +const OAUTH_TRANSIENT_FAILURE_REGEX = /timeout|network|fetch failed|ECONNREFUSED/i; +const OAUTH_HTTP_AUTH_REGEX = /\b(401|403)\b/; + +export function isDefinitiveOAuthFailure(errorMsg: string): boolean { + if (OAUTH_DEFINITIVE_FAILURE_REGEX.test(errorMsg)) return true; + if (OAUTH_HTTP_AUTH_REGEX.test(errorMsg) && !OAUTH_TRANSIENT_FAILURE_REGEX.test(errorMsg)) return true; + return false; +} + type UsageCacheEntry = { value: T; expiresAt: number; @@ -1832,9 +1855,50 @@ export class AuthStorage { credential: refreshedCredential, }; } catch (error) { + const errorMsg = String(error); + // Definitive failure (invalid_grant / 401 not from a network blip) means + // the refresh token itself is dead — probing with the original credential + // will 401, the catch below will return null, and #fetchUsageCached's + // last-good fallback will surface yesterday's report indefinitely + // (including its already-elapsed `resetsAt`). CAS-disable the row and + // clear the cache so the credential drops out of the report instead of + // freezing in place until the user notices and re-logs in. + if (isDefinitiveOAuthFailure(errorMsg)) { + const credentialId = this.#findStoredCredentialIdForUsageCredential( + request.provider, + request.credential, + ); + if (credentialId !== undefined) { + const entries = this.#getStoredCredentials(request.provider); + const index = entries.findIndex(entry => entry.id === credentialId); + if (index !== -1) { + const disabled = this.#tryDisableCredentialAtIfMatches( + request.provider, + index, + refreshableCredential, + `oauth refresh failed during usage probe: ${errorMsg}`, + ); + if (disabled) { + this.#usageLogger?.warn( + "Usage credential refresh failed definitively; credential disabled", + { provider: request.provider, credentialId, error: errorMsg }, + ); + // Neutralize last-good for this cache key: write a null + // entry with an immediately-elapsed expiry so a future + // getStale lookup (e.g. on re-login under the same + // account identity) can't replay the stale report. + this.#usageCache.set(this.#buildUsageReportCacheKey(request), { + value: null, + expiresAt: 0, + }); + return null; + } + } + } + } this.#usageLogger?.debug("Usage credential refresh failed, using original credential", { provider: request.provider, - error: String(error), + error: errorMsg, }); } } @@ -2877,9 +2941,7 @@ export class AuthStorage { const errorMsg = String(error); // Only remove credentials for definitive auth failures // Keep credentials for transient errors (network, 5xx) and block temporarily - const isDefinitiveFailure = - /invalid_grant|invalid_token|revoked|unauthorized|expired.*refresh|refresh.*expired/i.test(errorMsg) || - (/\b(401|403)\b/.test(errorMsg) && !/timeout|network|fetch failed|ECONNREFUSED/i.test(errorMsg)); + const isDefinitiveFailure = isDefinitiveOAuthFailure(errorMsg); logger.warn("OAuth token refresh failed", { provider, diff --git a/packages/ai/src/cli.ts b/packages/ai/src/cli.ts deleted file mode 100755 index 66c5c69db..000000000 --- a/packages/ai/src/cli.ts +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env bun -import * as readline from "node:readline"; -import { AuthStorage, SqliteAuthCredentialStore } from "./auth-storage"; -import { getOAuthProviders } from "./utils/oauth"; -import type { OAuthProvider } from "./utils/oauth/types"; - -const PROVIDERS = getOAuthProviders(); - -function prompt(rl: readline.Interface, question: string): Promise { - const { promise, resolve, reject } = Promise.withResolvers(); - const input = process.stdin as NodeJS.ReadStream; - const supportsRawMode = input.isTTY && typeof input.setRawMode === "function"; - const wasRaw = supportsRawMode ? input.isRaw : false; - let settled = false; - - const cleanup = () => { - rl.off("SIGINT", onSigint); - if (supportsRawMode) { - input.off("keypress", onKeypress); - input.setRawMode?.(wasRaw); - } - }; - - const finish = (result: () => void) => { - if (settled) return; - settled = true; - cleanup(); - result(); - }; - - const cancel = () => { - finish(() => reject(new Error("Login cancelled"))); - }; - - const onSigint = () => { - cancel(); - }; - - const onKeypress = (_str: string, key: readline.Key) => { - if (key.name === "escape" || (key.ctrl && key.name === "c")) { - cancel(); - rl.close(); - } - }; - - if (supportsRawMode) { - readline.emitKeypressEvents(input, rl); - input.setRawMode(true); - input.on("keypress", onKeypress); - } - - rl.once("SIGINT", onSigint); - rl.question(question, answer => { - finish(() => resolve(answer)); - }); - return promise; -} - -async function login(provider: OAuthProvider): Promise { - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - const promptFn = (msg: string) => prompt(rl, `${msg} `); - const store = await SqliteAuthCredentialStore.open(); - const storage = new AuthStorage(store); - await storage.reload(); - - try { - await storage.login(provider, { - onAuth(info) { - const { url, instructions } = info; - console.log(`\nOpen this URL in your browser:\n${url}`); - if (instructions) console.log(instructions); - console.log(); - }, - onProgress(message) { - console.log(message); - }, - onPrompt(p) { - return promptFn(`${p.message}${p.placeholder ? ` (${p.placeholder})` : ""}:`); - }, - }); - console.log(`\nCredentials saved to ~/.omp/agent/agent.db`); - } finally { - store.close(); - rl.close(); - } -} - -async function main(): Promise { - const args = process.argv.slice(2); - const command = args[0]; - - if (!command || command === "help" || command === "--help" || command === "-h") { - console.log(`Usage: bunx @oh-my-pi/pi-ai [provider] - -Commands: - login [provider] Login to a provider - logout [provider] Logout from a provider - status Show logged-in providers - list List available providers - -Providers: - anthropic Anthropic (Claude Pro/Max) - github-copilot GitHub Copilot - google-gemini-cli Google Gemini CLI - google-antigravity Antigravity (Gemini 3, Claude, GPT-OSS) - openai-codex OpenAI Codex (ChatGPT Plus/Pro) - kimi-code Kimi Code - kilo Kilo Gateway - kagi Kagi - tavily Tavily - zai Z.AI (GLM Coding Plan) - deepseek DeepSeek - nanogpt NanoGPT - minimax-code MiniMax Coding Plan (International) - minimax-code-cn MiniMax Coding Plan (China) - cursor Cursor (Claude, GPT, etc.) - zenmux ZenMux - ollama-cloud Ollama Cloud - -Examples: - bunx @oh-my-pi/pi-ai login # interactive provider selection - bunx @oh-my-pi/pi-ai login anthropic # login to specific provider - bunx @oh-my-pi/pi-ai logout anthropic # logout from specific provider - bunx @oh-my-pi/pi-ai status # show logged-in providers - bunx @oh-my-pi/pi-ai list # list providers -`); - return; - } - - if (command === "status") { - const storage = await SqliteAuthCredentialStore.open(); - try { - const providers = storage.listProviders(); - if (providers.length === 0) { - console.log("No credentials stored."); - console.log(`Use 'bunx @oh-my-pi/pi-ai login' to authenticate.`); - } else { - console.log("Logged-in providers:\n"); - for (const provider of providers) { - const oauth = storage.getOAuth(provider); - if (oauth) { - const expires = new Date(oauth.expires); - const expired = Date.now() >= oauth.expires; - const status = expired ? "(expired)" : `(expires ${expires.toLocaleString()})`; - console.log(` ${provider.padEnd(20)} ${status}`); - continue; - } - const apiKey = storage.getApiKey(provider); - if (apiKey) { - console.log(` ${provider.padEnd(20)} (api key)`); - } - } - } - } finally { - storage.close(); - } - return; - } - - if (command === "list") { - console.log("Available providers:\n"); - for (const p of PROVIDERS) { - console.log(` ${p.id.padEnd(20)} ${p.name}`); - } - return; - } - - if (command === "logout") { - let provider = args[1] as OAuthProvider | undefined; - const storage = await SqliteAuthCredentialStore.open(); - - try { - if (!provider) { - const providers = storage.listProviders(); - if (providers.length === 0) { - console.log("No credentials stored."); - return; - } - - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - console.log("Select a provider to logout:\n"); - for (let i = 0; i < providers.length; i++) { - console.log(` ${i + 1}. ${providers[i]}`); - } - console.log(); - - const choice = await prompt(rl, `Enter number (1-${providers.length}): `); - rl.close(); - - const index = parseInt(choice, 10) - 1; - if (index < 0 || index >= providers.length) { - console.error("Invalid selection"); - process.exit(1); - } - provider = providers[index] as OAuthProvider; - } - if (!provider) { - console.error("No provider selected"); - process.exit(1); - } - - const oauth = storage.getOAuth(provider); - const apiKey = storage.getApiKey(provider); - if (!oauth && !apiKey) { - console.error(`Not logged in to ${provider}`); - process.exit(1); - } - - storage.deleteProvider(provider); - console.log(`Logged out from ${provider}`); - } finally { - storage.close(); - } - return; - } - - if (command === "login") { - let provider = args[1] as OAuthProvider | undefined; - - if (!provider) { - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - console.log("Select a provider:\n"); - for (let i = 0; i < PROVIDERS.length; i++) { - console.log(` ${i + 1}. ${PROVIDERS[i].name}`); - } - console.log(); - - const choice = await prompt(rl, `Enter number (1-${PROVIDERS.length}): `); - rl.close(); - - const index = parseInt(choice, 10) - 1; - if (index < 0 || index >= PROVIDERS.length) { - console.error("Invalid selection"); - process.exit(1); - } - provider = PROVIDERS[index].id as OAuthProvider; - } - if (!provider) { - console.error("No provider selected"); - process.exit(1); - } - - if (!PROVIDERS.some(p => p.id === provider)) { - console.error(`Unknown provider: ${provider}`); - console.error(`Use 'bunx @oh-my-pi/pi-ai list' to see available providers`); - process.exit(1); - } - - console.log(`Logging in to ${provider}…`); - await login(provider); - return; - } - - console.error(`Unknown command: ${command}`); - console.error(`Use 'bunx @oh-my-pi/pi-ai --help' for usage`); - process.exit(1); -} - -main().catch(err => { - console.error("Error:", err.message); - process.exit(1); -}); diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 7d71d9e7e..213f30a2b 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -40,7 +40,6 @@ export * from "./usage/zai"; export * from "./utils/anthropic-auth"; export * from "./utils/discovery"; export * from "./utils/event-stream"; -export * from "./utils/h2-fetch"; export * from "./utils/oauth"; export type { OAuthCredentials, diff --git a/packages/ai/src/models.json b/packages/ai/src/models.json index 665f17a4f..c5d223f7c 100644 --- a/packages/ai/src/models.json +++ b/packages/ai/src/models.json @@ -16288,6 +16288,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "stealth/claude-opus-4.6": { + "id": "stealth/claude-opus-4.6", + "name": "Stealth: Claude Opus 4.6 (20% off)", + "api": "openai-completions", + "provider": "kilo", + "baseUrl": "https://api.kilo.ai/api/gateway", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "stealth/claude-opus-4.7": { "id": "stealth/claude-opus-4.7", "name": "Stealth: Claude Opus 4.7 (20% off)", @@ -16307,6 +16326,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "stealth/claude-sonnet-4.6": { + "id": "stealth/claude-sonnet-4.6", + "name": "Stealth: Claude Sonnet 4.6 (20% off)", + "api": "openai-completions", + "provider": "kilo", + "baseUrl": "https://api.kilo.ai/api/gateway", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "stepfun/step-3.5-flash": { "id": "stepfun/step-3.5-flash", "name": "Step 3.5 Flash", @@ -18562,6 +18600,25 @@ "maxLevel": "xhigh" } }, + "baidu/ernie-5.1": { + "id": "baidu/ernie-5.1", + "name": "baidu/ernie-5.1", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "baidu/ernie-x1.1-preview": { "id": "baidu/ernie-x1.1-preview", "name": "baidu/ernie-x1.1-preview", @@ -19802,6 +19859,63 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "claw-high": { + "id": "claw-high", + "name": "claw-high", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "claw-low": { + "id": "claw-low", + "name": "claw-low", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "claw-medium": { + "id": "claw-medium", + "name": "claw-medium", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "codex-auto-review": { "id": "codex-auto-review", "name": "codex-auto-review", @@ -19878,6 +19992,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "command-a-plus-05-2026": { + "id": "command-a-plus-05-2026", + "name": "command-a-plus-05-2026", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "command-a-reasoning-08-2025": { "id": "command-a-reasoning-08-2025", "name": "command-a-reasoning-08-2025", @@ -20948,6 +21081,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "ernie-5.1": { + "id": "ernie-5.1", + "name": "ernie-5.1", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "ernie-x1-32k": { "id": "ernie-x1-32k", "name": "ernie-x1-32k", @@ -21949,6 +22101,253 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "Gemma-4-31B-Claude-4.6-Opus-Reasoning-Distilled": { + "id": "Gemma-4-31B-Claude-4.6-Opus-Reasoning-Distilled", + "name": "Gemma-4-31B-Claude-4.6-Opus-Reasoning-Distilled", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-Cognitive-Unshackled": { + "id": "Gemma-4-31B-Cognitive-Unshackled", + "name": "Gemma-4-31B-Cognitive-Unshackled", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-DarkIdol": { + "id": "Gemma-4-31B-DarkIdol", + "name": "Gemma-4-31B-DarkIdol", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "gemma-4-31B-Fabled": { + "id": "gemma-4-31B-Fabled", + "name": "gemma-4-31B-Fabled", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "gemma-4-31B-Garnet": { + "id": "gemma-4-31B-Garnet", + "name": "gemma-4-31B-Garnet", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-GarnetV2": { + "id": "Gemma-4-31B-GarnetV2", + "name": "Gemma-4-31B-GarnetV2", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-Gemopus": { + "id": "Gemma-4-31B-Gemopus", + "name": "Gemma-4-31B-Gemopus", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-it": { + "id": "Gemma-4-31B-it", + "name": "Gemma-4-31B-it", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "gemma-4-31B-K1-v5": { + "id": "gemma-4-31B-K1-v5", + "name": "gemma-4-31B-K1-v5", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "gemma-4-31B-Larkspur-v0.5": { + "id": "gemma-4-31B-Larkspur-v0.5", + "name": "gemma-4-31B-Larkspur-v0.5", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "gemma-4-31B-MeroMero": { + "id": "gemma-4-31B-MeroMero", + "name": "gemma-4-31B-MeroMero", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-Musica-v1": { + "id": "Gemma-4-31B-Musica-v1", + "name": "Gemma-4-31B-Musica-v1", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Gemma-4-31B-Queen": { + "id": "Gemma-4-31B-Queen", + "name": "Gemma-4-31B-Queen", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "glm-4": { "id": "glm-4", "name": "glm-4", @@ -22845,6 +23244,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "google/gemini-3.1-flash-lite": { + "id": "google/gemini-3.1-flash-lite", + "name": "google/gemini-3.1-flash-lite", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "google/gemini-3.1-flash-lite-preview": { "id": "google/gemini-3.1-flash-lite-preview", "name": "Gemini 3.1 Flash Lite Preview", @@ -22951,6 +23369,63 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "google/gemini-3.5-flash": { + "id": "google/gemini-3.5-flash", + "name": "google/gemini-3.5-flash", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "google/gemini-3.5-flash-free": { + "id": "google/gemini-3.5-flash-free", + "name": "google/gemini-3.5-flash-free", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "google/gemini-3.5-flash-thinking": { + "id": "google/gemini-3.5-flash-thinking", + "name": "google/gemini-3.5-flash-thinking", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "google/gemini-flash-1.5": { "id": "google/gemini-flash-1.5", "name": "google/gemini-flash-1.5", @@ -23479,6 +23954,30 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "gpt-oss-120b": { + "id": "gpt-oss-120b", + "name": "GPT OSS 120B", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": true, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 131072, + "maxTokens": 32768, + "thinking": { + "mode": "effort", + "minLevel": "minimal", + "maxLevel": "xhigh" + } + }, "grok-3-beta": { "id": "grok-3-beta", "name": "grok-3-beta", @@ -23724,6 +24223,63 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "hermes-high": { + "id": "hermes-high", + "name": "hermes-high", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "hermes-low": { + "id": "hermes-low", + "name": "hermes-low", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "hermes-medium": { + "id": "hermes-medium", + "name": "hermes-medium", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "hf:deepseek-ai/DeepSeek-R1-0528": { "id": "hf:deepseek-ai/DeepSeek-R1-0528", "name": "hf:deepseek-ai/DeepSeek-R1-0528", @@ -24470,6 +25026,25 @@ "maxLevel": "xhigh" } }, + "inclusionai/ring-2.6-1t": { + "id": "inclusionai/ring-2.6-1t", + "name": "inclusionai/ring-2.6-1t", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "inclusionai/ring-flash-2.0": { "id": "inclusionai/ring-flash-2.0", "name": "inclusionai/ring-flash-2.0", @@ -26522,6 +27097,44 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "mirothinker-1-7-deepresearch": { + "id": "mirothinker-1-7-deepresearch", + "name": "mirothinker-1-7-deepresearch", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "mirothinker-1-7-deepresearch-mini": { + "id": "mirothinker-1-7-deepresearch-mini", + "name": "mirothinker-1-7-deepresearch-mini", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "Mistral-Nemo-12B-Instruct-2407": { "id": "Mistral-Nemo-12B-Instruct-2407", "name": "Mistral-Nemo-12B-Instruct-2407", @@ -27272,6 +27885,101 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "nanogpt/coding-router": { + "id": "nanogpt/coding-router", + "name": "nanogpt/coding-router", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "nanogpt/coding-router:high": { + "id": "nanogpt/coding-router:high", + "name": "nanogpt/coding-router:high", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "nanogpt/coding-router:low": { + "id": "nanogpt/coding-router:low", + "name": "nanogpt/coding-router:low", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "nanogpt/coding-router:max": { + "id": "nanogpt/coding-router:max", + "name": "nanogpt/coding-router:max", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "nanogpt/coding-router:medium": { + "id": "nanogpt/coding-router:medium", + "name": "nanogpt/coding-router:medium", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "NeverSleep/Llama-3-Lumimaid-70B-v0.1": { "id": "NeverSleep/Llama-3-Lumimaid-70B-v0.1", "name": "NeverSleep/Llama-3-Lumimaid-70B-v0.1", @@ -27597,6 +28305,25 @@ "maxLevel": "xhigh" } }, + "openai/chat-latest": { + "id": "openai/chat-latest", + "name": "openai/chat-latest", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "openai/chatgpt-4o-latest": { "id": "openai/chatgpt-4o-latest", "name": "openai/chatgpt-4o-latest", @@ -28859,6 +29586,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "perceptron/perceptron-mk1": { + "id": "perceptron/perceptron-mk1", + "name": "perceptron/perceptron-mk1", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "phi-4-mini-instruct": { "id": "phi-4-mini-instruct", "name": "phi-4-mini-instruct", @@ -29709,6 +30455,25 @@ "maxLevel": "high" } }, + "qwen/qwen3.7-max": { + "id": "qwen/qwen3.7-max", + "name": "qwen/qwen3.7-max", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "qwen/qwq-32b-preview": { "id": "qwen/qwq-32b-preview", "name": "qwen/qwq-32b-preview", @@ -30279,6 +31044,44 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted": { + "id": "Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted", + "name": "Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted-Lite": { + "id": "Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted-Lite", + "name": "Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted-Lite", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "Qwen3.5-27B-Queen-Derestricted": { "id": "Qwen3.5-27B-Queen-Derestricted", "name": "Qwen3.5-27B-Queen-Derestricted", @@ -30526,6 +31329,30 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "qwen3.7-max": { + "id": "qwen3.7-max", + "name": "Qwen3.7 Max", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": true, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 1000000, + "maxTokens": 65536, + "thinking": { + "mode": "effort", + "minLevel": "minimal", + "maxLevel": "high" + } + }, "qwq-32b": { "id": "qwq-32b", "name": "qwq-32b", @@ -30779,6 +31606,63 @@ "maxLevel": "xhigh" } }, + "sapiens-ai/agnes-2.0-flash": { + "id": "sapiens-ai/agnes-2.0-flash", + "name": "sapiens-ai/agnes-2.0-flash", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "sarvam-105b": { + "id": "sarvam-105b", + "name": "sarvam-105b", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, + "sarvam-30b": { + "id": "sarvam-30b", + "name": "sarvam-30b", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "sarvan-medium": { "id": "sarvan-medium", "name": "sarvan-medium", @@ -31202,6 +32086,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "stepfun-ai/step-3.6": { + "id": "stepfun-ai/step-3.6", + "name": "stepfun-ai/step-3.6", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "stepfun/step-3": { "id": "stepfun/step-3", "name": "Step-3", @@ -31384,6 +32287,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "TEE/gemma-4-26b-a4b-uncensored": { + "id": "TEE/gemma-4-26b-a4b-uncensored", + "name": "TEE/gemma-4-26b-a4b-uncensored", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "TEE/gemma4-31b": { "id": "TEE/gemma4-31b", "name": "TEE/gemma4-31b", @@ -31859,6 +32781,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "TEE/qwen3.6-35b-a3b-uncensored": { + "id": "TEE/qwen3.6-35b-a3b-uncensored", + "name": "TEE/qwen3.6-35b-a3b-uncensored", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "tencent/hunyuan-2.0-thinking": { "id": "tencent/hunyuan-2.0-thinking", "name": "tencent/hunyuan-2.0-thinking", @@ -33003,6 +33944,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "x-ai/grok-build-0.1": { + "id": "x-ai/grok-build-0.1", + "name": "x-ai/grok-build-0.1", + "api": "openai-completions", + "provider": "litellm", + "baseUrl": "http://localhost:4000/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "x-ai/grok-code-fast-1": { "id": "x-ai/grok-code-fast-1", "name": "Grok Code Fast 1", @@ -47074,6 +48034,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "TEE/gemma-4-31b-it": { + "id": "TEE/gemma-4-31b-it", + "name": "TEE/gemma-4-31b-it", + "api": "openai-completions", + "provider": "nanogpt", + "baseUrl": "https://nano-gpt.com/api/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "TEE/gemma4-31b": { "id": "TEE/gemma4-31b", "name": "TEE/gemma4-31b", @@ -47516,6 +48495,25 @@ "contextWindow": 222222, "maxTokens": 8888 }, + "TEE/qwen3.5-122b-a10b": { + "id": "TEE/qwen3.5-122b-a10b", + "name": "TEE/qwen3.5-122b-a10b", + "api": "openai-completions", + "provider": "nanogpt", + "baseUrl": "https://nano-gpt.com/api/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 222222, + "maxTokens": 8888 + }, "TEE/qwen3.5-27b": { "id": "TEE/qwen3.5-27b", "name": "TEE/qwen3.5-27b", @@ -53089,7 +54087,7 @@ "cacheWrite": 0 }, "contextWindow": 200000, - "maxTokens": 128000, + "maxTokens": 32000, "thinking": { "mode": "effort", "minLevel": "minimal", @@ -55606,13 +56604,13 @@ "text" ], "cost": { - "input": 0.32, - "output": 0.8899999999999999, + "input": 0.2288, + "output": 0.9144, "cacheRead": 0.15, "cacheWrite": 0 }, - "contextWindow": 163840, - "maxTokens": 16384 + "contextWindow": 131072, + "maxTokens": 16000 }, "deepseek/deepseek-chat-v3-0324": { "id": "deepseek/deepseek-chat-v3-0324", @@ -59517,8 +60515,8 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 131072, - "maxTokens": 8192, + "contextWindow": 262144, + "maxTokens": 32768, "thinking": { "mode": "effort", "minLevel": "minimal", @@ -60601,12 +61599,12 @@ ], "cost": { "input": 0.3, - "output": 3.1999999999999997, - "cacheRead": 0, + "output": 2, + "cacheRead": 0.15, "cacheWrite": 0 }, "contextWindow": 262144, - "maxTokens": 262144, + "maxTokens": 65536, "thinking": { "mode": "effort", "minLevel": "minimal", @@ -61506,9 +62504,9 @@ "image" ], "cost": { - "input": 0.39999999999999997, - "output": 2, - "cacheRead": 0.08, + "input": 0.14, + "output": 0.28, + "cacheRead": 0.0028, "cacheWrite": 0 }, "contextWindow": 1048576, @@ -61530,13 +62528,13 @@ "text" ], "cost": { - "input": 1, - "output": 3, - "cacheRead": 0.19999999999999998, + "input": 0.435, + "output": 0.87, + "cacheRead": 0.0036, "cacheWrite": 0 }, "contextWindow": 1048576, - "maxTokens": 16384, + "maxTokens": 131072, "thinking": { "mode": "effort", "minLevel": "minimal", @@ -62697,7 +63695,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 1000000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -62934,7 +63932,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 64000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -62956,7 +63954,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 32000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -63176,7 +64174,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 32000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -63198,7 +64196,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 128000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -63286,7 +64284,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 1000000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -63639,7 +64637,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 256000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -64348,7 +65346,7 @@ "cacheRead": 0, "cacheWrite": 0 }, - "contextWindow": 222222, + "contextWindow": 1000000, "maxTokens": 8888, "compat": { "supportsUsageInStreaming": false @@ -72397,6 +73395,46 @@ "maxLevel": "xhigh" } }, + "openai/gpt-image-1.5": { + "id": "openai/gpt-image-1.5", + "name": "OpenAI: GPT-Image-1.5", + "api": "openai-completions", + "provider": "zenmux", + "baseUrl": "https://zenmux.ai/api/v1", + "reasoning": false, + "input": [ + "text", + "image" + ], + "cost": { + "input": 5, + "output": 10, + "cacheRead": 1.25, + "cacheWrite": 0 + }, + "contextWindow": 10000, + "maxTokens": 8888 + }, + "openai/gpt-image-2": { + "id": "openai/gpt-image-2", + "name": "OpenAI: GPT-Image-2", + "api": "openai-completions", + "provider": "zenmux", + "baseUrl": "https://zenmux.ai/api/v1", + "reasoning": false, + "input": [ + "text", + "image" + ], + "cost": { + "input": 5, + "output": 0, + "cacheRead": 1.25, + "cacheWrite": 0 + }, + "contextWindow": 10000, + "maxTokens": 8888 + }, "openai/o4-mini": { "id": "openai/o4-mini", "name": "o4-mini", @@ -72422,6 +73460,44 @@ "maxLevel": "xhigh" } }, + "openai/text-embedding-3-large": { + "id": "openai/text-embedding-3-large", + "name": "OpenAI: Text Embedding 3 Large", + "api": "openai-completions", + "provider": "zenmux", + "baseUrl": "https://zenmux.ai/api/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0.13, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 8192, + "maxTokens": 8888 + }, + "openai/text-embedding-3-small": { + "id": "openai/text-embedding-3-small", + "name": "OpenAI: Text Embedding 3 Small", + "api": "openai-completions", + "provider": "zenmux", + "baseUrl": "https://zenmux.ai/api/v1", + "reasoning": false, + "input": [ + "text" + ], + "cost": { + "input": 0.02, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0 + }, + "contextWindow": 8192, + "maxTokens": 8888 + }, "qwen/qwen3-14b": { "id": "qwen/qwen3-14b", "name": "Qwen: Qwen3-14B", @@ -73301,6 +74377,26 @@ "contextWindow": 1000000, "maxTokens": 8888 }, + "x-ai/grok-build-0.1": { + "id": "x-ai/grok-build-0.1", + "name": "xAI: Grok Build 0.1", + "api": "openai-completions", + "provider": "zenmux", + "baseUrl": "https://zenmux.ai/api/v1", + "reasoning": false, + "input": [ + "text", + "image" + ], + "cost": { + "input": 1, + "output": 2, + "cacheRead": 0.2, + "cacheWrite": 0 + }, + "contextWindow": 256000, + "maxTokens": 8888 + }, "x-ai/grok-code-fast-1": { "id": "x-ai/grok-code-fast-1", "name": "Grok Code Fast 1", diff --git a/packages/ai/src/provider-models/descriptors.ts b/packages/ai/src/provider-models/descriptors.ts index fa96664c0..8dbdce383 100644 --- a/packages/ai/src/provider-models/descriptors.ts +++ b/packages/ai/src/provider-models/descriptors.ts @@ -6,7 +6,7 @@ import type { ModelManagerOptions } from "../model-manager"; import type { Api, KnownProvider } from "../types"; import type { OAuthProvider } from "../utils/oauth/types"; -import { googleModelManagerOptions } from "./google"; +import { googleModelManagerOptions, googleVertexModelManagerOptions } from "./google"; import { ollamaCloudModelManagerOptions } from "./ollama"; import { alibabaCodingPlanModelManagerOptions, @@ -283,6 +283,9 @@ export const PROVIDER_DESCRIPTORS: readonly ProviderDescriptor[] = [ catalogDescriptor("zai", "glm-5.1", config => zaiModelManagerOptions(config), catalog("zAI", ["ZAI_API_KEY"])), descriptor("github-copilot", "gpt-4o", config => githubCopilotModelManagerOptions(config)), descriptor("google", "gemini-2.5-pro", config => googleModelManagerOptions(config)), + descriptor("google-vertex", "gemini-3-pro-preview", config => googleVertexModelManagerOptions(config), { + allowUnauthenticated: true, + }), catalogDescriptor( "cursor", "claude-sonnet-4-6", diff --git a/packages/ai/src/provider-models/google.ts b/packages/ai/src/provider-models/google.ts index 9574b0d82..fe40b250c 100644 --- a/packages/ai/src/provider-models/google.ts +++ b/packages/ai/src/provider-models/google.ts @@ -1,6 +1,8 @@ import type { ModelManagerOptions } from "../model-manager"; +import type { FetchImpl } from "../types"; import { fetchAntigravityDiscoveryModels } from "../utils/discovery/antigravity"; import { fetchGeminiModels } from "../utils/discovery/gemini"; +import { fetchVertexOpenAIModels } from "../utils/discovery/vertex"; export interface GoogleModelManagerConfig { apiKey?: string; @@ -8,6 +10,10 @@ export interface GoogleModelManagerConfig { export interface GoogleVertexModelManagerConfig { apiKey?: string; + project?: string; + location?: string; + signal?: AbortSignal; + fetch?: FetchImpl; } export interface GoogleAntigravityModelManagerConfig { @@ -32,15 +38,43 @@ export function googleModelManagerOptions( }; } -export function googleVertexModelManagerOptions( - _config?: GoogleVertexModelManagerConfig, -): ModelManagerOptions<"google-vertex"> { - // Vertex AI uses Application Default Credentials (ADC) for authentication, - // which is handled at stream time rather than during model discovery. - // Dynamic model discovery is not yet implemented for this provider. - return { - providerId: "google-vertex", - }; +export function googleVertexModelManagerOptions(config?: GoogleVertexModelManagerConfig): ModelManagerOptions { + const project = resolveVertexProject(config); + const hasApiKey = (config?.apiKey ?? Bun.env.GOOGLE_CLOUD_API_KEY ?? "").trim().length > 0; + const location = resolveVertexLocation(config); + if (hasApiKey) { + return { providerId: "google-vertex" }; + } + if (project && location) { + return { + providerId: "google-vertex", + staticModels: [], + fetchDynamicModels: () => + fetchVertexOpenAIModels({ + project, + location, + signal: config?.signal, + fetch: config?.fetch, + }), + }; + } + // With neither ADC project+location nor API key auth configured, drop the + // bundled static catalog so stale fallbacks (e.g. `gemini-1.5-*`) cannot leak + // into `/models` alongside an authoritative cached Vertex project catalog on + // the next refresh. + return { providerId: "google-vertex", staticModels: [] }; +} +function resolveVertexProject(config?: GoogleVertexModelManagerConfig): string | undefined { + const project = config?.project ?? Bun.env.GOOGLE_CLOUD_PROJECT ?? Bun.env.GCP_PROJECT ?? Bun.env.GCLOUD_PROJECT; + const trimmed = project?.trim(); + return trimmed ? trimmed : undefined; +} + +function resolveVertexLocation(config?: GoogleVertexModelManagerConfig): string | undefined { + const location = + config?.location ?? Bun.env.GOOGLE_VERTEX_LOCATION ?? Bun.env.GOOGLE_CLOUD_LOCATION ?? Bun.env.VERTEX_LOCATION; + const trimmed = location?.trim(); + return trimmed ? trimmed : undefined; } export function googleAntigravityModelManagerOptions( diff --git a/packages/ai/src/providers/amazon-bedrock.ts b/packages/ai/src/providers/amazon-bedrock.ts index bf74b62c8..666a62294 100644 --- a/packages/ai/src/providers/amazon-bedrock.ts +++ b/packages/ai/src/providers/amazon-bedrock.ts @@ -42,6 +42,8 @@ export type BedrockThinkingDisplay = "summarized" | "omitted"; export interface BedrockOptions extends StreamOptions { region?: string; profile?: string; + /** Amazon Bedrock API key sent as `Authorization: Bearer`, ahead of SigV4 credential resolution. */ + bearerToken?: string; toolChoice?: "auto" | "any" | "none" | { type: "tool"; name: string }; /* See https://docs.aws.amazon.com/bedrock/latest/userguide/inference-reasoning.html for supported models. */ reasoning?: Effort; @@ -63,6 +65,12 @@ export interface BedrockOptions extends StreamOptions { */ thinkingDisplay?: BedrockThinkingDisplay; } +const AUTHENTICATED_API_KEY_SENTINEL = ""; + +function resolveBearerToken(options: BedrockOptions): string | undefined { + const apiKey = options.apiKey === AUTHENTICATED_API_KEY_SENTINEL ? undefined : options.apiKey; + return options.bearerToken || apiKey || $env.AWS_BEARER_TOKEN_BEDROCK; +} type Block = (TextContent | ThinkingContent | ToolCall) & { index?: number; partialJson?: string }; @@ -225,34 +233,40 @@ export const streamBedrock: StreamFunction<"bedrock-converse-stream"> = ( body: commandInput, }; - let credentials: { accessKeyId: string; secretAccessKey: string; sessionToken?: string }; - if ($flag("AWS_BEDROCK_SKIP_AUTH")) { - credentials = { accessKeyId: "dummy-access-key", secretAccessKey: "dummy-secret-key" }; - } else { - credentials = await resolveAwsCredentials({ - profile: options.profile, - region, - signal: options.signal, - }); - } - const bodyText = JSON.stringify(commandInput); const body = new TextEncoder().encode(bodyText); const baseHeaders: Record = { "content-type": "application/json", accept: "application/vnd.amazon.eventstream", }; - const signed = await signRequest({ - method: "POST", - host, - path: urlPath, - body, - region, - service: "bedrock", - credentials, - headers: baseHeaders, - }); - const requestHeaders: Record = { ...baseHeaders, ...signed }; + + const bearerToken = resolveBearerToken(options); + let requestHeaders: Record; + if (bearerToken) { + requestHeaders = { ...baseHeaders, Authorization: `Bearer ${bearerToken}` }; + } else { + let credentials: { accessKeyId: string; secretAccessKey: string; sessionToken?: string }; + if ($flag("AWS_BEDROCK_SKIP_AUTH")) { + credentials = { accessKeyId: "dummy-access-key", secretAccessKey: "dummy-secret-key" }; + } else { + credentials = await resolveAwsCredentials({ + profile: options.profile, + region, + signal: options.signal, + }); + } + const signed = await signRequest({ + method: "POST", + host, + path: urlPath, + body, + region, + service: "bedrock", + credentials, + headers: baseHeaders, + }); + requestHeaders = { ...baseHeaders, ...signed }; + } const response = await fetchWithRetry(url, { method: "POST", diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index 1ba4eb02c..ff8f0b569 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -2,7 +2,10 @@ import * as nodeCrypto from "node:crypto"; import * as fs from "node:fs"; import { scheduler } from "node:timers/promises"; import * as tls from "node:tls"; -import Anthropic, { type ClientOptions as AnthropicSdkClientOptions } from "@anthropic-ai/sdk"; +import Anthropic, { + APIConnectionTimeoutError as AnthropicConnectionTimeoutError, + type ClientOptions as AnthropicSdkClientOptions, +} from "@anthropic-ai/sdk"; import type { ContentBlockParam, MessageCreateParamsStreaming, @@ -53,17 +56,16 @@ import { resolveCacheRetention, } from "../utils"; import { createAbortSourceTracker } from "../utils/abort"; -import { iterateUntilAbort } from "../utils/abortable-iterator"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { isFoundryEnabled } from "../utils/foundry"; import { finalizeErrorMessage, type RawHttpRequestDump, rewriteCopilotError } from "../utils/http-inspector"; +import { getStreamFirstEventTimeoutMs, getStreamIdleTimeoutMs, iterateWithIdleTimeout } from "../utils/idle-iterator"; import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse"; import { parseGitHubCopilotApiKey } from "../utils/oauth/github-copilot"; import { notifyProviderResponse } from "../utils/provider-response"; import { isCopilotTransientModelError } from "../utils/retry"; import { COMBINATOR_KEYS, NO_STRICT, toolWireSchema } from "../utils/schema"; import { spillToDescription } from "../utils/schema/spill"; -import { createSdkStreamRequestOptions } from "../utils/sdk-stream-timeout"; import { notifyRawSseEvent, wrapFetchForSseDebug } from "../utils/sse-debug"; import { buildCopilotDynamicHeaders, @@ -1089,35 +1091,71 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( | TextContent | (ToolCall & { partialJson: string }) ) & { index: number }; + const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getStreamIdleTimeoutMs(); + const firstEventTimeoutMs = options?.streamFirstEventTimeoutMs ?? getStreamFirstEventTimeoutMs(idleTimeoutMs); + const requestTimeoutMs = + firstEventTimeoutMs !== undefined && firstEventTimeoutMs > 0 ? firstEventTimeoutMs : undefined; const blocks = output.content as Block[]; stream.push({ type: "start", partial: output }); // Retry loop for transient errors from the stream. // Provider-level transport/rate-limit failures: only before any streamed content starts. // Malformed envelopes/JSON: only before replay-unsafe text/tool events are visible on this stream. let providerRetryAttempt = 0; + const firstEventTimeoutAbortError = new Error("Anthropic stream timed out while waiting for the first event"); + const idleTimeoutAbortError = new Error("Anthropic stream stalled while waiting for the next event"); while (true) { activeAbortTracker = createAbortSourceTracker(options?.signal); const { requestSignal } = activeAbortTracker; - const requestOptions = createSdkStreamRequestOptions(requestSignal, options?.streamFirstEventTimeoutMs); + const requestOptions = + requestTimeoutMs === undefined + ? { signal: requestSignal } + : { signal: requestSignal, timeout: requestTimeoutMs }; const anthropicRequest = client.messages.create({ ...params, stream: true }, requestOptions); let streamedReplayUnsafeContent = false; try { - const { - events: anthropicStream, - response, - requestId, - } = await getAnthropicStreamResponse( - anthropicRequest, - requestSignal, - options?.client ? event => options?.onSseEvent?.(event, model) : undefined, - ); + let requestTimeout: NodeJS.Timeout | undefined; + if (requestTimeoutMs !== undefined) { + requestTimeout = setTimeout( + () => activeAbortTracker.abortLocally(firstEventTimeoutAbortError), + requestTimeoutMs, + ); + } + let anthropicStream: AsyncIterable; + let response: Response; + let requestId: string | null; + try { + ({ + events: anthropicStream, + response, + requestId, + } = await getAnthropicStreamResponse( + anthropicRequest, + requestSignal, + options?.client ? event => options?.onSseEvent?.(event, model) : undefined, + )); + } catch (error) { + if (error instanceof AnthropicConnectionTimeoutError && !activeAbortTracker.wasCallerAbort()) { + throw firstEventTimeoutAbortError; + } + throw error; + } finally { + if (requestTimeout !== undefined) clearTimeout(requestTimeout); + } await notifyProviderResponse(options, response, model, requestId); let sawEvent = false; let sawMessageStart = false; let sawTerminalEnvelope = false; - for await (const event of iterateUntilAbort(anthropicStream, options?.signal)) { + for await (const event of iterateWithIdleTimeout(anthropicStream, { + idleTimeoutMs, + firstItemTimeoutMs: firstEventTimeoutMs, + errorMessage: idleTimeoutAbortError.message, + firstItemErrorMessage: firstEventTimeoutAbortError.message, + onIdle: () => activeAbortTracker.abortLocally(idleTimeoutAbortError), + onFirstItemTimeout: () => activeAbortTracker.abortLocally(firstEventTimeoutAbortError), + abortSignal: options?.signal, + })) { sawEvent = true; if (event.type === "message_start") { @@ -1425,9 +1463,10 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = ( delete (block as { index?: number }).index; delete (block as { partialJson?: string }).partialJson; } + const firstEventTimeoutError = activeAbortTracker.getLocalAbortReason(); output.stopReason = activeAbortTracker.wasCallerAbort() ? "aborted" : "error"; output.errorStatus = extractHttpStatusFromError(error); - output.errorMessage = await finalizeErrorMessage(error, rawRequestDump); + output.errorMessage = firstEventTimeoutError?.message ?? (await finalizeErrorMessage(error, rawRequestDump)); output.errorMessage = rewriteCopilotError(output.errorMessage, error, model.provider); output.duration = Date.now() - startTime; if (firstTokenTime) output.ttft = firstTokenTime - startTime; diff --git a/packages/ai/src/providers/azure-openai-responses.ts b/packages/ai/src/providers/azure-openai-responses.ts index af1eb779a..cb1b4e659 100644 --- a/packages/ai/src/providers/azure-openai-responses.ts +++ b/packages/ai/src/providers/azure-openai-responses.ts @@ -1,9 +1,10 @@ import { $env, extractHttpStatusFromError } from "@oh-my-pi/pi-utils"; -import { AzureOpenAI } from "openai"; +import { AzureOpenAI, APIConnectionTimeoutError as OpenAIConnectionTimeoutError } from "openai"; import type { Tool as OpenAITool, ResponseCreateParamsStreaming, ResponseInput, + ResponseStreamEvent, } from "openai/resources/responses/responses"; import { getEnvApiKey } from "../stream"; import type { @@ -18,11 +19,14 @@ import type { } from "../types"; import { normalizeSystemPrompts } from "../utils"; import { createAbortSourceTracker } from "../utils/abort"; -import { iterateUntilAbort } from "../utils/abortable-iterator"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; +import { + getOpenAIStreamIdleTimeoutMs, + getStreamFirstEventTimeoutMs, + iterateWithIdleTimeout, +} from "../utils/idle-iterator"; import { sanitizeSchemaForOpenAIResponses, toolWireSchema } from "../utils/schema"; -import { createSdkStreamRequestOptions, resolveSdkTimeoutMs } from "../utils/sdk-stream-timeout"; import { wrapFetchForSseDebug } from "../utils/sse-debug"; import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice"; import { normalizeOpenAIResponsesPromptCacheKey, supportsDeveloperRole } from "./openai-responses"; @@ -33,12 +37,15 @@ import { convertResponsesAssistantMessage, convertResponsesInputContent, createInitialResponsesAssistantMessage, + isOpenAIResponsesProgressEvent, normalizeResponsesToolCallIdForTransform, processResponsesStream, } from "./openai-responses-shared"; import { transformMessages } from "./transform-messages"; const DEFAULT_AZURE_API_VERSION = "v1"; +const AZURE_OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE = + "Azure OpenAI responses stream timed out while waiting for the first event"; function parseDeploymentNameMap(value: string | undefined): Map { const map = new Map(); @@ -104,7 +111,8 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses" ); let rawRequestDump: RawHttpRequestDump | undefined; const abortTracker = createAbortSourceTracker(options?.signal); - const { requestSignal } = abortTracker; + const firstEventTimeoutAbortError = new Error(AZURE_OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE); + const { requestAbortController, requestSignal } = abortTracker; try { // Create Azure OpenAI client @@ -113,6 +121,10 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses" const { baseUrl } = resolveAzureConfig(model, options); const params = buildParams(model, context, options, deploymentName, baseUrl); options?.onPayload?.(params); + const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(); + const firstEventTimeoutMs = options?.streamFirstEventTimeoutMs ?? getStreamFirstEventTimeoutMs(idleTimeoutMs); + const requestTimeoutMs = + firstEventTimeoutMs !== undefined && firstEventTimeoutMs > 0 ? firstEventTimeoutMs : undefined; rawRequestDump = { provider: model.provider, api: output.api, @@ -121,15 +133,52 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses" url: `${baseUrl}/responses`, body: params, }; - const requestOptions = createSdkStreamRequestOptions(requestSignal, options?.streamFirstEventTimeoutMs); - const openaiStream = await client.responses.create(params, requestOptions); + let requestTimeout: NodeJS.Timeout | undefined; + if (requestTimeoutMs !== undefined) { + requestTimeout = setTimeout(() => abortTracker.abortLocally(firstEventTimeoutAbortError), requestTimeoutMs); + } + let openaiStream: AsyncIterable; + try { + const requestOptions = + requestTimeoutMs === undefined + ? { signal: requestSignal } + : { signal: requestSignal, timeout: requestTimeoutMs }; + openaiStream = await client.responses.create(params, requestOptions); + } catch (error) { + if (error instanceof OpenAIConnectionTimeoutError && !abortTracker.wasCallerAbort()) { + throw firstEventTimeoutAbortError; + } + throw error; + } finally { + if (requestTimeout !== undefined) clearTimeout(requestTimeout); + } stream.push({ type: "start", partial: output }); - await processResponsesStream(iterateUntilAbort(openaiStream, options?.signal), output, stream, model, { - onFirstToken: () => { - if (!firstTokenTime) firstTokenTime = Date.now(); + await processResponsesStream( + iterateWithIdleTimeout(openaiStream, { + idleTimeoutMs, + firstItemTimeoutMs: firstEventTimeoutMs, + firstItemErrorMessage: AZURE_OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE, + errorMessage: "Azure OpenAI responses stream stalled while waiting for the next event", + onIdle: () => requestAbortController.abort(), + onFirstItemTimeout: () => abortTracker.abortLocally(firstEventTimeoutAbortError), + abortSignal: options?.signal, + isProgressItem: isOpenAIResponsesProgressEvent, + }), + output, + stream, + model, + { + onFirstToken: () => { + if (!firstTokenTime) firstTokenTime = Date.now(); + }, }, - }); + ); + + const firstEventTimeoutError = abortTracker.getLocalAbortReason(); + if (firstEventTimeoutError) { + throw firstEventTimeoutError; + } if (abortTracker.wasCallerAbort()) { throw new Error("Request was aborted"); @@ -145,9 +194,10 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses" stream.end(); } catch (error) { for (const block of output.content) delete (block as { index?: number }).index; + const firstEventTimeoutError = abortTracker.getLocalAbortReason(); output.stopReason = abortTracker.wasCallerAbort() ? "aborted" : "error"; output.errorStatus = extractHttpStatusFromError(error); - output.errorMessage = await finalizeErrorMessage(error, rawRequestDump); + output.errorMessage = firstEventTimeoutError?.message ?? (await finalizeErrorMessage(error, rawRequestDump)); output.duration = Date.now() - startTime; if (firstTokenTime) output.ttft = firstTokenTime - startTime; stream.push({ type: "error", reason: output.stopReason, error: output }); @@ -218,7 +268,6 @@ function createClient(model: Model<"azure-openai-responses">, apiKey: string, op const baseFetch = options?.fetch ?? fetch; const onSseEvent = options?.onSseEvent; - const sdkTimeoutMs = resolveSdkTimeoutMs(options?.streamFirstEventTimeoutMs); return new AzureOpenAI({ apiKey, apiVersion, @@ -227,7 +276,6 @@ function createClient(model: Model<"azure-openai-responses">, apiKey: string, op defaultHeaders: headers, baseURL: baseUrl, fetch: onSseEvent ? wrapFetchForSseDebug(baseFetch, event => onSseEvent(event, model)) : baseFetch, - ...(sdkTimeoutMs !== undefined ? { timeout: sdkTimeoutMs } : {}), }); } @@ -244,7 +292,7 @@ function buildParams( model: deploymentName, input: messages, stream: true, - prompt_cache_key: normalizeOpenAIResponsesPromptCacheKey(options?.sessionId), + prompt_cache_key: normalizeOpenAIResponsesPromptCacheKey(options?.promptCacheKey ?? options?.sessionId), }; applyCommonResponsesSamplingParams(params, options, model.provider); diff --git a/packages/ai/src/providers/google-vertex.ts b/packages/ai/src/providers/google-vertex.ts index bcac9c78a..a8c07f2c4 100644 --- a/packages/ai/src/providers/google-vertex.ts +++ b/packages/ai/src/providers/google-vertex.ts @@ -67,10 +67,10 @@ function resolveApiKey(options?: GoogleVertexOptions): string | undefined { } function resolveProject(options?: GoogleVertexOptions): string { - const project = options?.project || $env.GOOGLE_CLOUD_PROJECT || $env.GCLOUD_PROJECT; + const project = options?.project || $env.GOOGLE_CLOUD_PROJECT || $env.GCP_PROJECT || $env.GCLOUD_PROJECT; if (!project) { throw new Error( - "Vertex AI requires a project ID. Set GOOGLE_CLOUD_PROJECT/GCLOUD_PROJECT or pass project in options.", + "Vertex AI requires a project ID. Set GOOGLE_CLOUD_PROJECT/GCP_PROJECT/GCLOUD_PROJECT or pass project in options.", ); } return project; @@ -80,9 +80,12 @@ function resolveEndpointHost(location: string): string { return location === "global" ? "aiplatform.googleapis.com" : `${location}-aiplatform.googleapis.com`; } function resolveLocation(options?: GoogleVertexOptions): string { - const location = options?.location || $env.GOOGLE_CLOUD_LOCATION; + const location = + options?.location || $env.GOOGLE_VERTEX_LOCATION || $env.GOOGLE_CLOUD_LOCATION || $env.VERTEX_LOCATION; if (!location) { - throw new Error("Vertex AI requires a location. Set GOOGLE_CLOUD_LOCATION or pass location in options."); + throw new Error( + "Vertex AI requires a location. Set GOOGLE_VERTEX_LOCATION/GOOGLE_CLOUD_LOCATION/VERTEX_LOCATION or pass location in options.", + ); } return location; } diff --git a/packages/ai/src/providers/mock.ts b/packages/ai/src/providers/mock.ts index 1fffb1356..cc18c0d96 100644 --- a/packages/ai/src/providers/mock.ts +++ b/packages/ai/src/providers/mock.ts @@ -216,10 +216,6 @@ export class MockModel implements Model { this.toolCallCounter = 0; } } - -/** @deprecated Use {@link MockModel}; the class IS the handle. */ -export type MockModelHandle = MockModel; - /** Check whether `model` was produced by `createMockModel`. */ export function isMockModel(model: Model): model is MockModel { return model instanceof MockModel; diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts index 328d2a497..62198b89a 100644 --- a/packages/ai/src/providers/openai-codex-responses.ts +++ b/packages/ai/src/providers/openai-codex-responses.ts @@ -29,6 +29,7 @@ import { type FetchImpl, type Model, type ProviderSessionState, + type RawSseEvent, resolveServiceTier, type ServiceTier, type StreamFunction, @@ -45,11 +46,16 @@ import { getOpenAIResponsesHistoryPayload, normalizeSystemPrompts, } from "../utils"; -import { iterateUntilAbort } from "../utils/abortable-iterator"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; +import { + getOpenAIStreamIdleTimeoutMs, + getStreamFirstEventTimeoutMs, + iterateWithIdleTimeout, +} from "../utils/idle-iterator"; import { parseStreamingJson } from "../utils/json-parse"; import { adaptSchemaForStrict, NO_STRICT, sanitizeSchemaForOpenAIResponses, toolWireSchema } from "../utils/schema"; +import { notifyRawSseEvent } from "../utils/sse-debug"; import { compactGrammarDefinition } from "./grammar"; import { CODEX_BASE_URL, getCodexAccountId, OPENAI_HEADER_VALUES, OPENAI_HEADERS } from "./openai-codex/constants"; import { @@ -66,6 +72,7 @@ import { convertResponsesInputContent, encodeResponsesToolCallId, encodeTextSignatureV1, + isOpenAIResponsesProgressEvent, mapOpenAIResponsesStopReason, populateResponsesUsageFromResponse, } from "./openai-responses-shared"; @@ -116,6 +123,14 @@ const X_REASONING_INCLUDED_HEADER = "x-reasoning-included"; const CODEX_WEBSOCKET_FATAL_PATTERNS = ["websocket error:", "websocket closed before open", "connection timeout"]; /** Max total time to spend retrying 429s with server-provided delays (5 minutes). */ const CODEX_RATE_LIMIT_BUDGET_MS = 5 * 60 * 1000; +const CODEX_ADDITIONAL_PROGRESS_EVENT_TYPES = new Set(["response.done", "response.incomplete"]); + +function isCodexStreamProgressEvent(event: unknown): boolean { + if (isOpenAIResponsesProgressEvent(event)) return true; + if (!event || typeof event !== "object") return false; + const type = (event as { type?: unknown }).type; + return typeof type === "string" && CODEX_ADDITIONAL_PROGRESS_EVENT_TYPES.has(type); +} type CodexTransport = "sse" | "websocket"; type CodexEventItem = ResponseReasoningItem | ResponseOutputMessage | ResponseFunctionToolCall | ResponseCustomToolCall; @@ -157,6 +172,7 @@ interface CodexRequestContext { baseUrl: string; url: string; requestHeaders: Record; + transportSessionId?: string; providerSessionState?: CodexProviderSessionState; websocketState?: CodexWebSocketSessionState; transformedBody: RequestBody; @@ -165,6 +181,10 @@ interface CodexRequestContext { interface CodexRequestSetup { requestSignal: AbortSignal; + wrapCodexSseStream: (source: AsyncGenerator>) => AsyncGenerator>; + requestAbortController: AbortController; + websocketIdleTimeoutMs: number | undefined; + websocketFirstEventTimeoutMs: number | undefined; } interface CodexStreamRuntime { @@ -356,6 +376,52 @@ function extractCodexWebSocketHandshakeHeaders(socket: Bun.WebSocket, openEvent? ); } +// Synthesizes a `RawSseEvent` for a Codex WebSocket frame so the same debug +// pipeline used for HTTP SSE (`onSseEvent` → `RawSseDebugBuffer.recordEvent`) +// also captures WebSocket traffic. The `raw` array mirrors SSE wire format +// (one line per field) so the existing TUI viewer renders it identically: +// : ws ← +// event: +// data: +// Outbound (client → server) uses `: ws → `. The viewer pretty-prints +// `data:` JSON lines, so we keep the wire JSON single-line here and let the +// renderer expand it. +function notifyCodexWebSocketInbound( + observer: ((event: RawSseEvent) => void) | undefined, + parsed: Record, + text: string, +): void { + const type = typeof parsed.type === "string" ? parsed.type : null; + const raw: string[] = [`: ws ← ${type ?? "(untyped)"}`]; + if (type) raw.push(`event: ${type}`); + raw.push(`data: ${text}`); + notifyRawSseEvent(observer, { event: type, data: text, raw }); +} + +function notifyCodexWebSocketOutbound( + observer: ((event: RawSseEvent) => void) | undefined, + request: Record, + payload: string, +): void { + const type = typeof request.type === "string" ? request.type : null; + const raw: string[] = [`: ws → ${type ?? "(untyped)"}`]; + if (type) raw.push(`event: ${type}`); + raw.push(`data: ${payload}`); + notifyRawSseEvent(observer, { event: type, data: payload, raw }); +} + +function notifyCodexWebSocketMalformed( + observer: ((event: RawSseEvent) => void) | undefined, + data: unknown, + error: unknown, +): void { + const text = typeof data === "string" ? data : ""; + const reason = error instanceof Error ? error.message : String(error); + const raw: string[] = [`: ws ← (parse-error: ${reason})`]; + if (text) raw.push(`data: ${text}`); + notifyRawSseEvent(observer, { event: "parse_error", data: text, raw }); +} + /** @internal Exported for tests. */ export function normalizeCodexToolChoice( choice: ToolChoice | undefined, @@ -472,7 +538,34 @@ function removeTransientBlockIndices(output: AssistantMessage): void { } function createRequestSetup(options: OpenAICodexResponsesOptions | undefined): CodexRequestSetup { - return { requestSignal: options?.signal ?? new AbortController().signal }; + const requestAbortController = new AbortController(); + const requestSignal = options?.signal + ? AbortSignal.any([options.signal, requestAbortController.signal]) + : requestAbortController.signal; + const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(); + const websocketIdleTimeoutMs = options?.streamIdleTimeoutMs ?? getCodexWebSocketIdleTimeoutMs(); + const firstEventTimeoutMs = options?.streamFirstEventTimeoutMs ?? getStreamFirstEventTimeoutMs(idleTimeoutMs); + const websocketFirstEventTimeoutMs = options?.streamFirstEventTimeoutMs ?? getCodexWebSocketFirstEventTimeoutMs(); + const wrapCodexSseStream = ( + source: AsyncGenerator>, + ): AsyncGenerator> => + iterateWithIdleTimeout(source, { + idleTimeoutMs, + firstItemTimeoutMs: firstEventTimeoutMs, + firstItemErrorMessage: "OpenAI Codex SSE stream timed out while waiting for the first event", + errorMessage: "OpenAI Codex SSE stream stalled while waiting for the next event", + onIdle: () => requestAbortController.abort(), + onFirstItemTimeout: () => requestAbortController.abort(), + abortSignal: options?.signal, + isProgressItem: isCodexStreamProgressEvent, + }); + return { + requestAbortController, + requestSignal, + wrapCodexSseStream, + websocketIdleTimeoutMs, + websocketFirstEventTimeoutMs, + }; } async function buildCodexRequestContext( @@ -489,8 +582,9 @@ async function buildCodexRequestContext( const accountId = getAccountId(apiKey); const baseUrl = model.baseUrl || CODEX_BASE_URL; const url = resolveCodexResponsesUrl(baseUrl); - const promptCacheKey = normalizeOpenAIResponsesPromptCacheKey(options?.sessionId); - const transformedBody = await buildTransformedCodexRequestBody(model, context, options); + const promptCacheKey = resolveCodexPromptCacheKey(options); + const transportSessionId = resolveCodexTransportSessionId(options); + const transformedBody = await buildTransformedCodexRequestBody(model, context, options, promptCacheKey); options?.onPayload?.(transformedBody); const requestHeaders = { ...(model.headers ?? {}), ...(options?.headers ?? {}) }; @@ -504,20 +598,20 @@ async function buildCodexRequestContext( }; const providerSessionState = getCodexProviderSessionState(options?.providerSessionState); - const sessionKey = getCodexWebSocketSessionKey(promptCacheKey, model, accountId, baseUrl); - const publicSessionKey = getCodexPublicSessionKey(promptCacheKey, model, baseUrl); + const sessionKey = getCodexWebSocketSessionKey(transportSessionId, model, accountId, baseUrl); + const publicSessionKey = getCodexPublicSessionKey(transportSessionId, model, baseUrl); if (sessionKey && publicSessionKey) { providerSessionState?.webSocketPublicToPrivate.set(publicSessionKey, sessionKey); } const websocketState = sessionKey && providerSessionState ? getCodexWebSocketSessionState(sessionKey, providerSessionState) : undefined; - return { apiKey, accountId, baseUrl, url, requestHeaders, + transportSessionId, providerSessionState, websocketState, transformedBody, @@ -529,12 +623,13 @@ async function buildTransformedCodexRequestBody( model: Model<"openai-codex-responses">, context: Context, options: OpenAICodexResponsesOptions | undefined, + promptCacheKey = resolveCodexPromptCacheKey(options), ): Promise { const params: RequestBody = { model: model.id, input: [...convertMessages(model, context)], stream: true, - prompt_cache_key: normalizeOpenAIResponsesPromptCacheKey(options?.sessionId), + prompt_cache_key: promptCacheKey, }; if (options?.maxTokens) { @@ -613,7 +708,13 @@ async function openInitialCodexEventStream( let websocketRetries = 0; while (true) { try { - return await openCodexWebSocketTransport(requestContext, requestSetup, websocketState, websocketRetries); + return await openCodexWebSocketTransport( + requestContext, + requestSetup, + websocketState, + websocketRetries, + options ? event => options.onSseEvent?.(event, model) : undefined, + ); } catch (error) { const websocketError = error instanceof Error ? error : new Error(String(error)); const isFatal = isCodexWebSocketFatalError(websocketError); @@ -644,6 +745,7 @@ async function openCodexWebSocketTransport( requestSetup: CodexRequestSetup, websocketState: CodexWebSocketSessionState, retry: number, + onSseEvent?: (event: RawSseEvent) => void, ): Promise<{ eventStream: AsyncGenerator>; requestBodyForState: RequestBody; @@ -654,7 +756,7 @@ async function openCodexWebSocketTransport( requestContext.requestHeaders, requestContext.accountId, requestContext.apiKey, - requestContext.transformedBody.prompt_cache_key, + requestContext.transportSessionId, "websocket", websocketState, ); @@ -675,7 +777,12 @@ async function openCodexWebSocketTransport( websocketHeaders, websocketRequest, websocketState, + { + idleTimeoutMs: requestSetup.websocketIdleTimeoutMs, + firstEventTimeoutMs: requestSetup.websocketFirstEventTimeoutMs, + }, requestSetup.requestSignal, + onSseEvent, ); return { eventStream, requestBodyForState, transport: "websocket" }; } @@ -692,20 +799,19 @@ async function openCodexSseTransport( requestBodyForState: RequestBody; transport: CodexTransport; }> { - const eventStream = iterateUntilAbort( + const eventStream = requestSetup.wrapCodexSseStream( await openCodexSseEventStream( requestContext.url, requestContext.requestHeaders, requestContext.accountId, requestContext.apiKey, - body.prompt_cache_key, + requestContext.transportSessionId, body, state, requestSetup.requestSignal, event => options?.onSseEvent?.(event, model), options?.fetch, ), - requestSetup.requestSignal, ); return { eventStream, requestBodyForState: structuredCloneJSON(body), transport: "sse" }; } @@ -721,6 +827,7 @@ async function reopenCodexWebSocketRuntimeStream( context.requestSetup, state, runtime.websocketStreamRetries, + context.options ? event => context.options?.onSseEvent?.(event, context.model) : undefined, ); runtime.eventStream = next.eventStream; runtime.requestBodyForState = next.requestBodyForState; @@ -1602,6 +1709,18 @@ export async function prewarmOpenAICodexResponses( state.prewarmed = true; } +function resolveCodexPromptCacheKey( + options: Pick | undefined, +): string | undefined { + return normalizeOpenAIResponsesPromptCacheKey(options?.promptCacheKey ?? options?.sessionId); +} + +function resolveCodexTransportSessionId( + options: Pick | undefined, +): string | undefined { + return normalizeOpenAIResponsesPromptCacheKey(options?.sessionId); +} + function getCodexWebSocketSessionKey( sessionId: string | undefined, model: Model<"openai-codex-responses">, @@ -1848,29 +1967,29 @@ function headersToRecord(headers: Headers): Record { return result; } +interface CodexWebSocketRequestTimeouts { + idleTimeoutMs?: number; + firstEventTimeoutMs?: number; +} + interface CodexWebSocketConnectionOptions { - idleTimeoutMs: number; - firstEventTimeoutMs: number; onHandshakeHeaders?: (headers: Headers) => void; } class CodexWebSocketConnection { #url: string; #headers: Record; - #idleTimeoutMs: number; - #firstEventTimeoutMs: number; #onHandshakeHeaders?: (headers: Headers) => void; #socket: Bun.WebSocket | null = null; #queue: Array | Error | null> = []; #waiters: Array<() => void> = []; #connectPromise?: Promise; #activeRequest = false; + #streamObserver?: (event: RawSseEvent) => void; constructor(url: string, headers: Record, options: CodexWebSocketConnectionOptions) { this.#url = url; this.#headers = headers; - this.#idleTimeoutMs = options.idleTimeoutMs; - this.#firstEventTimeoutMs = options.firstEventTimeoutMs; this.#onHandshakeHeaders = options.onHandshakeHeaders; } @@ -1983,8 +2102,10 @@ class CodexWebSocketConnection { parsed.message = inner.message; } } + notifyCodexWebSocketInbound(this.#streamObserver, parsed, text); this.#push(parsed); } catch (error) { + notifyCodexWebSocketMalformed(this.#streamObserver, event.data, error); this.#push(createCodexWebSocketTransportError(String(error))); } }; @@ -1999,7 +2120,9 @@ class CodexWebSocketConnection { async *streamRequest( request: Record, + timeouts: CodexWebSocketRequestTimeouts, signal?: AbortSignal, + onSseEvent?: (event: RawSseEvent) => void, ): AsyncGenerator> { if (!this.#socket || this.#socket.readyState !== WebSocket.OPEN) { throw createCodexWebSocketTransportError("websocket connection is unavailable"); @@ -2008,6 +2131,7 @@ class CodexWebSocketConnection { throw createCodexWebSocketTransportError("websocket request already in progress"); } this.#activeRequest = true; + this.#streamObserver = onSseEvent; const onAbort = () => { this.close("aborted"); this.#push(createCodexWebSocketTransportError("request was aborted")); @@ -2021,21 +2145,28 @@ class CodexWebSocketConnection { } try { - this.#socket.send(JSON.stringify(request)); + const requestPayload = JSON.stringify(request); + notifyCodexWebSocketOutbound(onSseEvent, request, requestPayload); + this.#socket.send(requestPayload); let sawFirstEvent = false; + const { idleTimeoutMs, firstEventTimeoutMs } = timeouts; let lastProgressAt = Date.now(); while (true) { - let timeoutMs: number; + let timeoutMs: number | undefined; let timeoutReason: string; if (sawFirstEvent) { - timeoutMs = this.#idleTimeoutMs - (Date.now() - lastProgressAt); timeoutReason = "idle timeout waiting for websocket"; - if (timeoutMs <= 0) { - throw createCodexWebSocketTransportError(timeoutReason); + if (idleTimeoutMs !== undefined && idleTimeoutMs > 0) { + timeoutMs = idleTimeoutMs - (Date.now() - lastProgressAt); + if (timeoutMs <= 0) { + throw createCodexWebSocketTransportError(timeoutReason); + } } } else { - timeoutMs = this.#firstEventTimeoutMs; timeoutReason = "timeout waiting for first websocket event"; + if (firstEventTimeoutMs !== undefined && firstEventTimeoutMs > 0) { + timeoutMs = firstEventTimeoutMs; + } } const next = await this.#nextMessage(timeoutMs, timeoutReason); if (next instanceof Error) { @@ -2045,7 +2176,9 @@ class CodexWebSocketConnection { throw createCodexWebSocketTransportError("websocket closed before response completion"); } sawFirstEvent = true; - lastProgressAt = Date.now(); + if (isCodexStreamProgressEvent(next)) { + lastProgressAt = Date.now(); + } yield next; const eventType = typeof next.type === "string" ? next.type : ""; if ( @@ -2060,6 +2193,7 @@ class CodexWebSocketConnection { } } finally { this.#activeRequest = false; + this.#streamObserver = undefined; if (signal) { signal.removeEventListener("abort", onAbort); } @@ -2079,13 +2213,16 @@ class CodexWebSocketConnection { if (waiter) waiter(); } - async #nextMessage(timeoutMs: number, timeoutReason: string): Promise | Error | null> { + async #nextMessage( + timeoutMs: number | undefined, + timeoutReason: string, + ): Promise | Error | null> { while (this.#queue.length === 0) { const { promise, resolve } = Promise.withResolvers(); this.#waiters.push(resolve); let timedOut = false; let timeout: NodeJS.Timeout | undefined; - if (timeoutMs > 0) { + if (timeoutMs !== undefined && timeoutMs > 0) { timeout = setTimeout(() => { timedOut = true; const waiterIndex = this.#waiters.indexOf(resolve); @@ -2124,8 +2261,6 @@ async function getOrCreateCodexWebSocketConnection( resetCodexWebSocketAppendState(state); logger.time("codexWs:newSocket"); state.connection = new CodexWebSocketConnection(url, headerRecord, { - idleTimeoutMs: getCodexWebSocketIdleTimeoutMs(), - firstEventTimeoutMs: getCodexWebSocketFirstEventTimeoutMs(), onHandshakeHeaders: handshakeHeaders => { updateCodexSessionMetadataFromHeaders(state, handshakeHeaders); }, @@ -2192,17 +2327,19 @@ async function openCodexWebSocketEventStream( headers: Headers, request: Record, state: CodexWebSocketSessionState, + timeouts: CodexWebSocketRequestTimeouts, signal?: AbortSignal, + onSseEvent?: (event: RawSseEvent) => void, ): Promise>> { const connection = await getOrCreateCodexWebSocketConnection(state, url, headers, signal); - return connection.streamRequest(request, signal); + return connection.streamRequest(request, timeouts, signal, onSseEvent); } function createCodexHeaders( initHeaders: Record | undefined, accountId: string, accessToken: string, - promptCacheKey?: string, + sessionId?: string, transport: CodexTransport = "sse", state?: CodexWebSocketSessionState, ): Headers { @@ -2219,10 +2356,10 @@ function createCodexHeaders( headers.set(OPENAI_HEADERS.BETA, betaHeader); headers.set(OPENAI_HEADERS.ORIGINATOR, OPENAI_HEADER_VALUES.ORIGINATOR_CODEX); headers.set("User-Agent", getCodexUserAgent()); - if (promptCacheKey) { - headers.set(OPENAI_HEADERS.CONVERSATION_ID, promptCacheKey); - headers.set(OPENAI_HEADERS.SESSION_ID, promptCacheKey); - headers.set("x-client-request-id", promptCacheKey); + if (sessionId) { + headers.set(OPENAI_HEADERS.CONVERSATION_ID, sessionId); + headers.set(OPENAI_HEADERS.SESSION_ID, sessionId); + headers.set("x-client-request-id", sessionId); } else { headers.delete(OPENAI_HEADERS.CONVERSATION_ID); headers.delete(OPENAI_HEADERS.SESSION_ID); diff --git a/packages/ai/src/providers/openai-completions-compat.ts b/packages/ai/src/providers/openai-completions-compat.ts index 90e5393ac..98795ec24 100644 --- a/packages/ai/src/providers/openai-completions-compat.ts +++ b/packages/ai/src/providers/openai-completions-compat.ts @@ -74,11 +74,16 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB // applies when thinking mode is actually engaged. const lowerId = model.id.toLowerCase(); const lowerName = (model.name ?? "").toLowerCase(); + // OpenCode Zen's `big-pickle` is a DeepSeek reasoning alias; the upstream + // 400s come from DeepSeek and require exact reasoning_content replay. + const isOpenCodeDeepseekAlias = + provider === "opencode-zen" && (lowerId === "big-pickle" || lowerName === "big pickle"); const isDeepseekFamily = provider === "deepseek" || baseUrl.includes("deepseek.com") || lowerId.includes("deepseek") || - lowerName.includes("deepseek"); + lowerName.includes("deepseek") || + isOpenCodeDeepseekAlias; const isDirectDeepseekApi = provider === "deepseek" || baseUrl.includes("api.deepseek.com"); const isDirectDeepseekReasoning = isDirectDeepseekApi && isDeepseekFamily && Boolean(model.reasoning); const isNonStandard = @@ -181,7 +186,13 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB return { supportsStore: !isNonStandard, - supportsDeveloperRole: !isNonStandard, + // `developer` is an OpenAI-Responses-era extension to the chat-completions schema. Almost + // every OpenAI-compatible host other than OpenAI itself (and Azure OpenAI, which mirrors + // the schema exactly) treats it as an unknown role: Moonshot returns a 400 "tokenization + // failed", Groq/Cerebras/etc. error or silently misroute. Default to `system` and require + // callers to opt in via `compat.supportsDeveloperRole: true` for hosts known to mirror + // OpenAI's reasoning-API surface. + supportsDeveloperRole: isOpenAIHost || isAzureHost, supportsMultipleSystemMessages: supportsMultipleSystemMessagesDefault, supportsReasoningEffort: !isGrok && !isZai, reasoningEffortMap, @@ -205,12 +216,14 @@ export function detectOpenAICompat(model: Model<"openai-completions">, resolvedB reasoningContentField: "reasoning_content", // Backends that 400 follow-up requests when prior assistant tool-call turns lack `reasoning_content`: // - Kimi: documented invariant on its native API. - // - Any reasoning-capable model reached through OpenRouter: DeepSeek V4 Pro and similar enforce - // this server-side whenever the request is in thinking mode. We can't translate Anthropic's - // redacted/encrypted reasoning into DeepSeek's plaintext form, so cross-provider continuations - // rely on a placeholder — see `convertMessages` for the placeholder injection. - // - OpenCode-Go and OpenCode-Zen handle reasoning content internally and reject - // `reasoning_content` in client-sent messages — exclude them even for Kimi models. + // - DeepSeek-family reasoning models, including aliased OpenCode Zen models + // like `big-pickle`, validate exact thinking-mode replay. + // - Any reasoning-capable model reached through OpenRouter can enforce this + // server-side whenever the request is in thinking mode. We can't translate + // Anthropic's redacted/encrypted reasoning into provider-native plaintext, + // so cross-provider continuations rely on a placeholder. + // OpenCode Kimi aliases handle reasoning content internally and reject + // client-sent `reasoning_content`, so exclude only that Kimi-on-OpenCode path. requiresReasoningContentForToolCalls: (isKimiModel && !isOpenCodeProvider) || (isDeepseekFamily && Boolean(model.reasoning)) || diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 9e30cc716..b8fa7a95f 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -1,5 +1,5 @@ import { $env, extractHttpStatusFromError } from "@oh-my-pi/pi-utils"; -import OpenAI from "openai"; +import OpenAI, { APIConnectionTimeoutError as OpenAIConnectionTimeoutError } from "openai"; import type { ChatCompletionAssistantMessageParam, ChatCompletionChunk, @@ -37,7 +37,6 @@ import { } from "../types"; import { normalizeSystemPrompts } from "../utils"; import { createAbortSourceTracker } from "../utils/abort"; -import { iterateUntilAbort } from "../utils/abortable-iterator"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { toFirepassWireModelId, toFireworksWireModelId } from "../utils/fireworks-model-id"; import { @@ -46,13 +45,17 @@ import { type RawHttpRequestDump, rewriteCopilotError, } from "../utils/http-inspector"; +import { + getOpenAIStreamIdleTimeoutMs, + getStreamFirstEventTimeoutMs, + iterateWithIdleTimeout, +} from "../utils/idle-iterator"; import { parseStreamingJson } from "../utils/json-parse"; import { parseGitHubCopilotApiKey } from "../utils/oauth/github-copilot"; import { getKimiCommonHeaders } from "../utils/oauth/kimi"; import { notifyProviderResponse } from "../utils/provider-response"; import { callWithCopilotModelRetry } from "../utils/retry"; import { adaptSchemaForStrict, NO_STRICT, toolWireSchema } from "../utils/schema"; -import { resolveSdkTimeoutMs } from "../utils/sdk-stream-timeout"; import { wrapFetchForSseDebug } from "../utils/sse-debug"; import { type HealedToolCall, modelMayLeakKimiToolCalls, ToolCallHealer } from "../utils/tool-call-healing"; import { isForcedToolChoice, mapToOpenAICompletionsToolChoice } from "../utils/tool-choice"; @@ -164,6 +167,52 @@ function hasToolHistory(messages: Message[]): boolean { } return false; } +/** + * Identify "real progress" stream chunks vs. keepalives, role-only preambles, + * and empty `{choices:[]}` no-ops emitted by some OpenAI-compatible endpoints. + * Without this filter, every keepalive resets `iterateWithIdleTimeout`'s + * deadline, so a provider that streams nothing but pings keeps the watchdog + * asleep indefinitely — observed against z.ai/GLM via OpenRouter where a + * subagent stalled for hours with no error surfaced. + * + * A chunk counts as progress when it carries terminal usage, a finish reason, + * or any model-produced delta (content / tool calls / reasoning / refusal). + * Role-only `delta: { role: "assistant" }` preambles do NOT count; we want the + * (longer) first-event timeout to keep governing until real output appears. + */ +export function isOpenAICompletionsProgressChunk(chunk: unknown): boolean { + if (!chunk || typeof chunk !== "object") return false; + const record = chunk as { + usage?: unknown; + choices?: ReadonlyArray<{ + finish_reason?: unknown; + usage?: unknown; + delta?: { + content?: unknown; + tool_calls?: unknown; + reasoning?: unknown; + reasoning_content?: unknown; + reasoning_text?: unknown; + refusal?: unknown; + }; + }>; + }; + if (record.usage) return true; + const choice = Array.isArray(record.choices) ? record.choices[0] : undefined; + if (!choice) return false; + if (choice.finish_reason) return true; + if (choice.usage) return true; + const delta = choice.delta; + if (!delta) return false; + const content = delta.content; + if (typeof content === "string" ? content.length > 0 : Array.isArray(content) && content.length > 0) return true; + if (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) return true; + if (typeof delta.reasoning === "string" && delta.reasoning.length > 0) return true; + if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) return true; + if (typeof delta.reasoning_text === "string" && delta.reasoning_text.length > 0) return true; + if (typeof delta.refusal === "string" && delta.refusal.length > 0) return true; + return false; +} export interface OpenAICompletionsOptions extends StreamOptions { toolChoice?: ToolChoice; @@ -321,6 +370,8 @@ function getTrailingPartialDeepseekToken(text: string): string { if (tail.length > 256) return ""; return tail; } +const OPENAI_COMPLETIONS_FIRST_EVENT_TIMEOUT_MESSAGE = + "OpenAI completions stream timed out while waiting for the first event"; export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( model: Model<"openai-completions">, @@ -337,10 +388,15 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( const output: AssistantMessage = createInitialResponsesAssistantMessage(model.api, model.provider, model.id); let rawRequestDump: RawHttpRequestDump | undefined; const abortTracker = createAbortSourceTracker(options?.signal); - const { requestSignal } = abortTracker; + const firstEventTimeoutAbortError = new Error(OPENAI_COMPLETIONS_FIRST_EVENT_TIMEOUT_MESSAGE); + const { requestAbortController, requestSignal } = abortTracker; try { const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; + const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(); + const firstEventTimeoutMs = options?.streamFirstEventTimeoutMs ?? getStreamFirstEventTimeoutMs(idleTimeoutMs); + const requestTimeoutMs = + firstEventTimeoutMs !== undefined && firstEventTimeoutMs > 0 ? firstEventTimeoutMs : undefined; const { client, copilotPremiumRequests, @@ -356,7 +412,6 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( options?.initiatorOverride, options?.onSseEvent, options?.fetch, - options?.streamFirstEventTimeoutMs, ); const premiumRequestsTotal = copilotPremiumRequests; getCapturedErrorResponse = captureErrorResponse; @@ -389,11 +444,31 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( headers: requestHeaders, body: params, }; - const { data, response, request_id } = await client.chat.completions - .create(params, { signal: requestSignal }) - .withResponse(); - await notifyProviderResponse(options, response, model, request_id); - return data; + const requestOptions = + requestTimeoutMs === undefined + ? { signal: requestSignal } + : { signal: requestSignal, timeout: requestTimeoutMs }; + let requestTimeout: NodeJS.Timeout | undefined; + if (requestTimeoutMs !== undefined) { + requestTimeout = setTimeout( + () => abortTracker.abortLocally(firstEventTimeoutAbortError), + requestTimeoutMs, + ); + } + try { + const { data, response, request_id } = await client.chat.completions + .create(params, requestOptions) + .withResponse(); + await notifyProviderResponse(options, response, model, request_id); + return data; + } catch (error) { + if (error instanceof OpenAIConnectionTimeoutError && !abortTracker.wasCallerAbort()) { + throw firstEventTimeoutAbortError; + } + throw error; + } finally { + if (requestTimeout !== undefined) clearTimeout(requestTimeout); + } }; let openaiStream: AsyncIterable; try { @@ -601,7 +676,16 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( for (const call of calls) emitHealedToolCall(call); }; - for await (const chunk of iterateUntilAbort(openaiStream, options?.signal)) { + for await (const chunk of iterateWithIdleTimeout(openaiStream, { + idleTimeoutMs, + firstItemTimeoutMs: firstEventTimeoutMs, + firstItemErrorMessage: OPENAI_COMPLETIONS_FIRST_EVENT_TIMEOUT_MESSAGE, + errorMessage: "OpenAI completions stream stalled while waiting for the next event", + onIdle: () => requestAbortController.abort(), + onFirstItemTimeout: () => abortTracker.abortLocally(firstEventTimeoutAbortError), + abortSignal: options?.signal, + isProgressItem: isOpenAICompletionsProgressChunk, + })) { if (!chunk || typeof chunk !== "object") continue; // OpenAI documents ChatCompletionChunk.id as the unique chat completion identifier, @@ -770,6 +854,10 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( finishCurrentBlock(currentBlock); + const firstEventTimeoutError = abortTracker.getLocalAbortReason(); + if (firstEventTimeoutError) { + throw firstEventTimeoutError; + } if (abortTracker.wasCallerAbort()) { throw new Error("Request was aborted"); } @@ -788,9 +876,12 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = ( stream.end(); } catch (error) { for (const block of output.content) delete (block as any).index; + const firstEventTimeoutError = abortTracker.getLocalAbortReason(); output.stopReason = abortTracker.wasCallerAbort() ? "aborted" : "error"; output.errorStatus = extractHttpStatusFromError(error) ?? getCapturedErrorResponse?.()?.status; - output.errorMessage = await finalizeErrorMessage(error, rawRequestDump, getCapturedErrorResponse?.()); + output.errorMessage = + firstEventTimeoutError?.message ?? + (await finalizeErrorMessage(error, rawRequestDump, getCapturedErrorResponse?.())); // Some providers via OpenRouter include extra details here. const rawMetadata = (error as { error?: { metadata?: { raw?: string } } })?.error?.metadata?.raw; if (rawMetadata) output.errorMessage += `\n${rawMetadata}`; @@ -813,7 +904,6 @@ async function createClient( initiatorOverride?: MessageAttribution, onSseEvent?: OpenAICompletionsOptions["onSseEvent"], fetchOverride?: FetchImpl, - streamFirstEventTimeoutOverride?: number, ): Promise<{ client: OpenAI; copilotPremiumRequests: number | undefined; @@ -912,7 +1002,6 @@ async function createClient( baseFetch.preconnect ? { preconnect: baseFetch.preconnect } : {}, ); const debugFetch = onSseEvent ? wrapFetchForSseDebug(wrappedFetch, event => onSseEvent(event, model)) : wrappedFetch; - const sdkTimeoutMs = resolveSdkTimeoutMs(streamFirstEventTimeoutOverride); return { client: new OpenAI({ apiKey, @@ -922,7 +1011,6 @@ async function createClient( defaultHeaders: headers, defaultQuery: azureDefaultQuery, fetch: debugFetch, - ...(sdkTimeoutMs !== undefined ? { timeout: sdkTimeoutMs } : {}), }), copilotPremiumRequests, baseUrl, diff --git a/packages/ai/src/providers/openai-responses-shared.ts b/packages/ai/src/providers/openai-responses-shared.ts index 8e1e16f3b..e8d6cc382 100644 --- a/packages/ai/src/providers/openai-responses-shared.ts +++ b/packages/ai/src/providers/openai-responses-shared.ts @@ -32,6 +32,31 @@ import { normalizeResponsesToolCallId } from "../utils"; import type { AssistantMessageEventStream } from "../utils/event-stream"; import { parseStreamingJson } from "../utils/json-parse"; import { joinTextWithImagePlaceholder, NON_VISION_IMAGE_PLACEHOLDER, partitionVisionContent } from "./vision-guard"; +export const OPENAI_RESPONSES_PROGRESS_EVENT_TYPES: ReadonlySet = new Set([ + "response.created", + "response.output_item.added", + "response.reasoning_summary_part.added", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_part.done", + "response.reasoning_text.delta", + "response.content_part.added", + "response.output_text.delta", + "response.refusal.delta", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.custom_tool_call_input.delta", + "response.custom_tool_call_input.done", + "response.output_item.done", + "response.completed", + "response.failed", + "error", +]); + +export function isOpenAIResponsesProgressEvent(event: unknown): boolean { + if (!event || typeof event !== "object") return false; + const type = (event as { type?: unknown }).type; + return typeof type === "string" && OPENAI_RESPONSES_PROGRESS_EVENT_TYPES.has(type); +} export function encodeTextSignatureV1(id: string, phase?: TextSignatureV1["phase"]): string { const payload: TextSignatureV1 = { v: 1, id }; diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index bb22d3cd8..e07cf83d8 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -1,5 +1,5 @@ import { $env, extractHttpStatusFromError, structuredCloneJSON } from "@oh-my-pi/pi-utils"; -import OpenAI from "openai"; +import OpenAI, { APIConnectionTimeoutError as OpenAIConnectionTimeoutError } from "openai"; import type { Tool as OpenAITool, ResponseCreateParamsStreaming, @@ -30,14 +30,17 @@ import { sanitizeOpenAIResponsesHistoryItemsForReplay, } from "../utils"; import { createAbortSourceTracker } from "../utils/abort"; -import { iterateUntilAbort } from "../utils/abortable-iterator"; import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump, rewriteCopilotError } from "../utils/http-inspector"; +import { + getOpenAIStreamIdleTimeoutMs, + getStreamFirstEventTimeoutMs, + iterateWithIdleTimeout, +} from "../utils/idle-iterator"; import { parseGitHubCopilotApiKey } from "../utils/oauth/github-copilot"; import { notifyProviderResponse } from "../utils/provider-response"; import { callWithCopilotModelRetry } from "../utils/retry"; import { adaptSchemaForStrict, NO_STRICT, sanitizeSchemaForOpenAIResponses, toolWireSchema } from "../utils/schema"; -import { createSdkStreamRequestOptions, resolveSdkTimeoutMs } from "../utils/sdk-stream-timeout"; import { wrapFetchForSseDebug } from "../utils/sse-debug"; import { mapToOpenAIResponsesToolChoice, type OpenAIResponsesToolChoice } from "../utils/tool-choice"; import { @@ -55,6 +58,7 @@ import { convertResponsesAssistantMessage, convertResponsesInputContent, createInitialResponsesAssistantMessage, + isOpenAIResponsesProgressEvent, normalizeResponsesToolCallIdForTransform, processResponsesStream, repairOrphanResponsesToolOutputs, @@ -96,6 +100,8 @@ export interface OpenAIResponsesOptions extends StreamOptions { } const OPENAI_RESPONSES_PROVIDER_SESSION_STATE_PREFIX = "openai-responses:"; +const OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE = + "OpenAI responses stream timed out while waiting for the first event"; interface OpenAIResponsesProviderSessionState extends ProviderSessionState { nativeHistoryReplayWarmed: boolean; @@ -165,11 +171,14 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( ); let rawRequestDump: RawHttpRequestDump | undefined; const abortTracker = createAbortSourceTracker(options?.signal); - const { requestSignal } = abortTracker; + const firstEventTimeoutAbortError = new Error(OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE); + const { requestAbortController, requestSignal } = abortTracker; try { - // Keep request headers and prompt-cache routing on the same session-derived value. - const cacheSessionId = getOpenAIResponsesCacheSessionId(options); + // Keep request routing on `sessionId` while allowing callers to pin a + // stable prompt-cache key independently. Side-channel calls use this to + // avoid perturbing provider conversation state without cold-starting the cache. + const routingSessionId = getOpenAIResponsesRoutingSessionId(options); const apiKey = options?.apiKey || getEnvApiKey(model.provider) || ""; const { client, copilotPremiumRequests, baseUrl } = createClient( model, @@ -177,14 +186,17 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( apiKey, options?.headers, options?.initiatorOverride, - cacheSessionId, + routingSessionId, options?.onSseEvent, options?.fetch, - options?.streamFirstEventTimeoutMs, ); const premiumRequestsTotal = copilotPremiumRequests; const providerSessionState = getOpenAIResponsesProviderSessionState(model, options?.providerSessionState); const { params } = buildParams(model, context, options, providerSessionState, baseUrl); + const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(); + const firstEventTimeoutMs = options?.streamFirstEventTimeoutMs ?? getStreamFirstEventTimeoutMs(idleTimeoutMs); + const requestTimeoutMs = + firstEventTimeoutMs !== undefined && firstEventTimeoutMs > 0 ? firstEventTimeoutMs : undefined; options?.onPayload?.(params); rawRequestDump = { provider: model.provider, @@ -194,14 +206,33 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( url: `${baseUrl ?? "https://api.openai.com/v1"}/responses`, body: params, }; - const requestOptions = createSdkStreamRequestOptions(requestSignal, options?.streamFirstEventTimeoutMs); const openaiStream = await callWithCopilotModelRetry( async () => { - const { data, response, request_id } = await client.responses - .create(params, requestOptions) - .withResponse(); - await notifyProviderResponse(options, response, model, request_id); - return data; + const requestOptions = + requestTimeoutMs === undefined + ? { signal: requestSignal } + : { signal: requestSignal, timeout: requestTimeoutMs }; + let requestTimeout: NodeJS.Timeout | undefined; + if (requestTimeoutMs !== undefined) { + requestTimeout = setTimeout( + () => abortTracker.abortLocally(firstEventTimeoutAbortError), + requestTimeoutMs, + ); + } + try { + const { data, response, request_id } = await client.responses + .create(params, requestOptions) + .withResponse(); + await notifyProviderResponse(options, response, model, request_id); + return data; + } catch (error) { + if (error instanceof OpenAIConnectionTimeoutError && !abortTracker.wasCallerAbort()) { + throw firstEventTimeoutAbortError; + } + throw error; + } finally { + if (requestTimeout !== undefined) clearTimeout(requestTimeout); + } }, { provider: model.provider, signal: requestSignal }, ); @@ -209,16 +240,35 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( stream.push({ type: "start", partial: output }); const nativeOutputItems: Array> = []; - await processResponsesStream(iterateUntilAbort(openaiStream, options?.signal), output, stream, model, { - onFirstToken: () => { - if (!firstTokenTime) firstTokenTime = Date.now(); + await processResponsesStream( + iterateWithIdleTimeout(openaiStream, { + idleTimeoutMs, + firstItemTimeoutMs: firstEventTimeoutMs, + firstItemErrorMessage: OPENAI_RESPONSES_FIRST_EVENT_TIMEOUT_MESSAGE, + errorMessage: "OpenAI responses stream stalled while waiting for the next event", + onFirstItemTimeout: () => abortTracker.abortLocally(firstEventTimeoutAbortError), + onIdle: () => requestAbortController.abort(), + abortSignal: options?.signal, + isProgressItem: isOpenAIResponsesProgressEvent, + }), + output, + stream, + model, + { + onFirstToken: () => { + if (!firstTokenTime) firstTokenTime = Date.now(); + }, + onOutputItemDone: item => { + nativeOutputItems.push(structuredCloneJSON(item) as unknown as Record); + }, }, - onOutputItemDone: item => { - nativeOutputItems.push(structuredCloneJSON(item) as unknown as Record); - }, - }); + ); if (premiumRequestsTotal !== undefined) output.usage.premiumRequests = premiumRequestsTotal; + const firstEventTimeoutError = abortTracker.getLocalAbortReason(); + if (firstEventTimeoutError) { + throw firstEventTimeoutError; + } if (abortTracker.wasCallerAbort()) { throw new Error("Request was aborted"); } @@ -236,9 +286,10 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = ( stream.end(); } catch (error) { for (const block of output.content) delete (block as { index?: number }).index; + const firstEventTimeoutError = abortTracker.getLocalAbortReason(); output.stopReason = abortTracker.wasCallerAbort() ? "aborted" : "error"; output.errorStatus = extractHttpStatusFromError(error); - output.errorMessage = await finalizeErrorMessage(error, rawRequestDump); + output.errorMessage = firstEventTimeoutError?.message ?? (await finalizeErrorMessage(error, rawRequestDump)); output.errorMessage = rewriteCopilotError(output.errorMessage, error, model.provider); output.duration = Date.now() - startTime; if (firstTokenTime) output.ttft = firstTokenTime - startTime; @@ -259,7 +310,6 @@ function createClient( sessionId?: string, onSseEvent?: OpenAIResponsesOptions["onSseEvent"], fetchOverride?: FetchImpl, - streamFirstEventTimeoutOverride?: number, ): { client: OpenAI; copilotPremiumRequests: number | undefined; @@ -298,7 +348,6 @@ function createClient( headers["x-client-request-id"] ??= sessionId; } const baseFetch = fetchOverride ?? fetch; - const sdkTimeoutMs = resolveSdkTimeoutMs(streamFirstEventTimeoutOverride); return { client: new OpenAI({ apiKey, @@ -307,19 +356,24 @@ function createClient( maxRetries: 5, defaultHeaders: headers, fetch: onSseEvent ? wrapFetchForSseDebug(baseFetch, event => onSseEvent(event, model)) : baseFetch, - ...(sdkTimeoutMs !== undefined ? { timeout: sdkTimeoutMs } : {}), }), copilotPremiumRequests, baseUrl, }; } -function getOpenAIResponsesCacheSessionId( +function getOpenAIResponsesPromptCacheKey( + options: Pick | undefined, +): string | undefined { + if (resolveCacheRetention(options?.cacheRetention) === "none") return undefined; + return normalizeOpenAIResponsesPromptCacheKey(options?.promptCacheKey ?? options?.sessionId); +} + +function getOpenAIResponsesRoutingSessionId( options: Pick | undefined, ): string | undefined { - return resolveCacheRetention(options?.cacheRetention) === "none" - ? undefined - : normalizeOpenAIResponsesPromptCacheKey(options?.sessionId); + if (resolveCacheRetention(options?.cacheRetention) === "none") return undefined; + return normalizeOpenAIResponsesPromptCacheKey(options?.sessionId); } function buildParams( @@ -359,7 +413,7 @@ function buildParams( } const cacheRetention = resolveCacheRetention(options?.cacheRetention); - const promptCacheKey = getOpenAIResponsesCacheSessionId(options); + const promptCacheKey = getOpenAIResponsesPromptCacheKey(options); const params: OpenAIResponsesSamplingParams = { model: model.id, input: messages, diff --git a/packages/ai/src/providers/pi-native-server.ts b/packages/ai/src/providers/pi-native-server.ts index 2be0e9f7e..6092aeca4 100644 --- a/packages/ai/src/providers/pi-native-server.ts +++ b/packages/ai/src/providers/pi-native-server.ts @@ -58,6 +58,7 @@ const ALLOWED_OPTION_KEYS: ReadonlySet = new Set([ "maxRetryDelayMs", "metadata", "sessionId", + "promptCacheKey", "streamFirstEventTimeoutMs", "streamIdleTimeoutMs", "reasoning", diff --git a/packages/ai/src/providers/register-builtins.ts b/packages/ai/src/providers/register-builtins.ts index 64a90d9a9..813751071 100644 --- a/packages/ai/src/providers/register-builtins.ts +++ b/packages/ai/src/providers/register-builtins.ts @@ -20,8 +20,8 @@ import type { OptionsForApi, } from "../types"; import { type AbortSourceTracker, createAbortSourceTracker } from "../utils/abort"; -import { iterateUntilAbort } from "../utils/abortable-iterator"; import { AssistantMessageEventStream as EventStreamImpl } from "../utils/event-stream"; +import { getStreamFirstEventTimeoutMs, getStreamIdleTimeoutMs, iterateWithIdleTimeout } from "../utils/idle-iterator"; import type { BedrockOptions } from "./amazon-bedrock"; import type { AnthropicOptions } from "./anthropic"; import type { AzureOpenAIResponsesOptions } from "./azure-openai-responses"; @@ -157,22 +157,82 @@ export function setBedrockProviderModule(module: BedrockProviderModule): void { // Stream forwarding / error helpers // --------------------------------------------------------------------------- +const LAZY_STREAM_IDLE_TIMEOUT_ERROR = "Provider stream stalled while waiting for the next event"; +const LAZY_STREAM_FIRST_EVENT_TIMEOUT_ERROR = "Provider stream timed out while waiting for the first event"; + function hasFinalResult( source: AsyncIterable, ): source is AsyncIterable & { result(): Promise } { return typeof (source as { result?: unknown }).result === "function"; } +/** + * Per-provider default overrides for the lazy stream watchdogs. These widen the + * floor used when neither caller option nor env var pins a value. The env vars + * (`PI_STREAM_FIRST_EVENT_TIMEOUT_MS`, `PI_STREAM_IDLE_TIMEOUT_MS`) still take + * precedence; `StreamOptions.streamFirstEventTimeoutMs` / `streamIdleTimeoutMs` + * still trump everything. + */ +interface LazyStreamLimits { + defaultFirstEventTimeoutMs?: number; + defaultIdleTimeoutMs?: number; + /** + * The provider implementation already wraps its upstream transport with + * stream timeouts. Keep the lazy loader from racing it with generic errors. + */ + providerHandlesStreamTimeouts?: boolean; +} +/** + * Cloud Code Assist (google-gemini-cli / google-antigravity) routinely takes + * longer than the global 100s default to emit its first SSE event when serving + * the heavier Gemini 3.x Pro tiers at high thinking levels. Bump the first-event + * floor to five minutes so callers stop seeing spurious "stream timed out while + * waiting for the first event" aborts on legitimate cold reasoning starts. + * The steady-state idle watchdog stays on the global default since the upstream + * emits thinking tokens frequently once it gets going. + */ +const GOOGLE_GEMINI_CLI_LAZY_STREAM_LIMITS: LazyStreamLimits = { + defaultFirstEventTimeoutMs: 300_000, +}; + +const PROVIDER_HANDLED_STREAM_TIMEOUTS: LazyStreamLimits = { + providerHandlesStreamTimeouts: true, +}; + function forwardStream( target: EventStreamImpl, source: AsyncIterable, model: Model, options: OptionsForApi, abortTracker: AbortSourceTracker, + limits?: LazyStreamLimits, ): void { (async () => { try { - for await (const event of iterateUntilAbort(source, options.signal)) { + const providerHandlesStreamTimeouts = limits?.providerHandlesStreamTimeouts === true; + const idleTimeoutMs = providerHandlesStreamTimeouts + ? undefined + : (options.streamIdleTimeoutMs ?? getStreamIdleTimeoutMs(limits?.defaultIdleTimeoutMs)); + const watchedSource = iterateWithIdleTimeout(source, { + idleTimeoutMs, + firstItemTimeoutMs: providerHandlesStreamTimeouts + ? 0 + : (options.streamFirstEventTimeoutMs ?? + getStreamFirstEventTimeoutMs(idleTimeoutMs, limits?.defaultFirstEventTimeoutMs)), + errorMessage: LAZY_STREAM_IDLE_TIMEOUT_ERROR, + firstItemErrorMessage: LAZY_STREAM_FIRST_EVENT_TIMEOUT_ERROR, + onIdle: () => abortTracker.abortLocally(new Error(LAZY_STREAM_IDLE_TIMEOUT_ERROR)), + onFirstItemTimeout: () => abortTracker.abortLocally(new Error(LAZY_STREAM_FIRST_EVENT_TIMEOUT_ERROR)), + abortSignal: options.signal, + // The synthetic `start` event is yielded immediately by every provider before + // the upstream model has emitted any tokens. Treating it as the first "real" + // item would flip the watchdog from `firstItemTimeoutMs` to the much shorter + // `idleTimeoutMs` while we're still legitimately waiting on the model's + // first response (slow first-token from reasoning models, cold proxies, etc.). + isProgressItem: event => (event as AssistantMessageEvent).type !== "start", + }); + + for await (const event of watchedSource) { target.push(event); } if (hasFinalResult(source)) { @@ -221,6 +281,7 @@ function createLazyLoadErrorMessage( function createLazyStream( loadModule: () => Promise>, + limits?: LazyStreamLimits, ): (model: Model, context: Context, options: OptionsForApi) => EventStreamImpl { return (model, context, options) => { const outer = new EventStreamImpl(); @@ -231,7 +292,7 @@ function createLazyStream( const abortTracker = createAbortSourceTracker(streamOptions.signal); const providerOptions = { ...streamOptions, signal: abortTracker.requestSignal } as OptionsForApi; const inner = module.stream(model, context, providerOptions); - forwardStream(outer, inner, model, streamOptions, abortTracker); + forwardStream(outer, inner, model, streamOptions, abortTracker, limits); }) .catch(error => { const message = createLazyLoadErrorMessage(model, error); @@ -346,14 +407,26 @@ function loadBedrockProviderModule(): Promise): boolean { + return ( + model.provider === "google-vertex" && + model.api === "openai-completions" && + model.baseUrl.includes("/endpoints/openapi") + ); +} + +function createVertexOpenAIFetch(options: StreamOptions | undefined): FetchImpl { + const baseFetch = options?.fetch ?? fetch; + const vertexFetch = async (input: string | URL | Request, init?: RequestInit): Promise => { + const token = await getVertexAccessToken({ signal: options?.signal, fetch: baseFetch }); + const headers = new Headers(init?.headers); + headers.set("Authorization", `Bearer ${token}`); + return baseFetch(input, { ...init, headers }); + }; + return Object.assign(vertexFetch, baseFetch.preconnect ? { preconnect: baseFetch.preconnect } : {}); +} type KeyResolver = string | (() => string | undefined); @@ -119,16 +139,16 @@ const serviceProviderMap: Record = { return $env.GOOGLE_CLOUD_API_KEY; } const hasCredentials = hasVertexAdcCredentials(); - const hasProject = !!($env.GOOGLE_CLOUD_PROJECT || $env.GCLOUD_PROJECT); - const hasLocation = !!$env.GOOGLE_CLOUD_LOCATION; + const hasProject = !!($env.GOOGLE_CLOUD_PROJECT || $env.GCP_PROJECT || $env.GCLOUD_PROJECT); + const hasLocation = !!($env.GOOGLE_VERTEX_LOCATION || $env.GOOGLE_CLOUD_LOCATION || $env.VERTEX_LOCATION); if (hasCredentials && hasProject && hasLocation) { return ""; } }, // Amazon Bedrock supports multiple credential sources: - // 1. AWS_PROFILE - named profile from ~/.aws/credentials + // 1. AWS_BEARER_TOKEN_BEDROCK - Bedrock API keys (bearer token) // 2. AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY - standard IAM keys - // 3. AWS_BEARER_TOKEN_BEDROCK - Bedrock API keys (bearer token) + // 3. AWS_PROFILE - named profile from ~/.aws/credentials // 4. AWS_CONTAINER_CREDENTIALS_* - ECS/Task IAM role credentials // 5. AWS_WEB_IDENTITY_TOKEN_FILE + AWS_ROLE_ARN - IRSA (EKS) web identity "amazon-bedrock": () => { @@ -222,7 +242,9 @@ export function stream( if (!apiKey) { throw new Error(`No API key for provider: ${model.provider}`); } - const providerOptions = { ...options, apiKey }; + const providerOptions = isGoogleVertexOpenAIModel(model) + ? { ...options, apiKey: "vertex-adc", fetch: createVertexOpenAIFetch(options as StreamOptions | undefined) } + : { ...options, apiKey }; const api: Api = model.api; switch (api) { @@ -564,6 +586,9 @@ function mapOptionsForApi( maxRetryDelayMs: options?.maxRetryDelayMs, metadata: options?.metadata, sessionId: options?.sessionId, + promptCacheKey: options?.promptCacheKey, + streamFirstEventTimeoutMs: options?.streamFirstEventTimeoutMs, + streamIdleTimeoutMs: options?.streamIdleTimeoutMs, providerSessionState: options?.providerSessionState, onPayload: options?.onPayload, onResponse: options?.onResponse, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 60db6d081..6a25de26a 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -316,11 +316,17 @@ export interface StreamOptions { */ metadata?: Record; /** - * Optional session identifier for providers that support session-based caching. - * Providers can use this to enable prompt caching, request routing, or other - * session-aware features. Ignored by providers that don't support it. + * Optional session identifier for providers that support session-based + * routing, request affinity, or transport reuse. Providers may also use this + * as the prompt-cache key when `promptCacheKey` is not set. */ sessionId?: string; + /** + * Optional prompt-cache identity. When set, OpenAI Responses-compatible + * providers use this for `prompt_cache_key` while keeping `sessionId` for + * provider routing / conversation headers. + */ + promptCacheKey?: string; /** * Provider-scoped mutable state store for this agent session. * Providers can use this to persist transport/session state between turns. @@ -336,30 +342,37 @@ export interface StreamOptions { */ onResponse?: (response: ProviderResponseMetadata, model?: Model) => void | Promise; /** - * Optional callback for raw Server-Sent Events as they arrive from HTTP streaming providers. + * Optional callback for raw Server-Sent Events as they arrive from HTTP streaming providers, + * plus synthesized SSE-shaped frames for the Codex WebSocket transport (one synthetic frame + * per JSON request/response message). WebSocket frames are tagged with a leading + * `: ws → ` (outbound) or `: ws ← ` (inbound) comment line in `RawSseEvent.raw`. * * Diagnostic only: provider implementations must ignore callback failures and must not * let observers alter stream contents. */ onSseEvent?: (event: RawSseEvent, model?: Model) => void; /** - * Optional SDK/request timeout hint in milliseconds applied to the underlying HTTP - * request when the provider's transport exposes a per-request timeout. + * Optional override for the first-event watchdog in milliseconds. Built-in + * providers apply this budget twice when they can: once to the underlying + * SDK/request while waiting for the HTTP stream object to exist, then again + * in the iterator while waiting for the first semantic stream event. Set to + * `0` to disable both layers for this request. After the first semantic + * event arrives, `streamIdleTimeoutMs` governs inter-event stalls. Falls + * back to `PI_STREAM_FIRST_EVENT_TIMEOUT_MS` and then to a 100s default. * - * Honored by: `openai-completions`, `openai-responses`, `azure-openai-responses`, - * `anthropic-messages`. - * Ignored by: `openai-codex-responses` (uses its own websocket/SSE transport), - * `google`, `google-gemini-cli`, `google-vertex`, `bedrock-converse`, `cursor-agent`. - * - * Provider stream silence is never treated as failure on its own — once the request - * has started, callers must abort to interrupt a silent stream. + * Iterator-level honored by: every built-in provider (via the lazy-stream + * forwarder in `register-builtins`). SDK-request honored by: + * `openai-completions`, `openai-responses`, `azure-openai-responses`, + * `anthropic-messages`. */ streamFirstEventTimeoutMs?: number; /** - * @deprecated Stream idle watchdogs were removed; provider streams now wait for - * provider output, provider/socket errors, caller aborts, or request-layer timeouts. - * This field is accepted for backwards compatibility but no longer wired anywhere. - * Will be removed in the next major release. + * Optional override for the maximum idle gap between streamed events in + * milliseconds. Once the first event arrives, this guards against silent + * mid-stream stalls (broker dies, half-open socket, model produces no real + * progress for too long). Set to `0` to disable. Falls back to + * `PI_STREAM_IDLE_TIMEOUT_MS` (alias: `PI_OPENAI_STREAM_IDLE_TIMEOUT_MS`) + * and then to a 120s default. */ streamIdleTimeoutMs?: number; /** diff --git a/packages/ai/src/utils/discovery/index.ts b/packages/ai/src/utils/discovery/index.ts index 7af3bebdf..d6ae7ae90 100644 --- a/packages/ai/src/utils/discovery/index.ts +++ b/packages/ai/src/utils/discovery/index.ts @@ -2,3 +2,4 @@ export * from "./antigravity"; export * from "./codex"; export * from "./gemini"; export * from "./openai-compatible"; +export * from "./vertex"; diff --git a/packages/ai/src/utils/discovery/vertex.ts b/packages/ai/src/utils/discovery/vertex.ts new file mode 100644 index 000000000..c0b0e4ba4 --- /dev/null +++ b/packages/ai/src/utils/discovery/vertex.ts @@ -0,0 +1,210 @@ +import { UNK_CONTEXT_WINDOW, UNK_MAX_TOKENS } from "@oh-my-pi/pi-ai"; +import * as z from "zod/v4"; +import { getVertexAccessToken } from "../../providers/google-auth"; +import type { FetchImpl, Model } from "../../types"; + +const API_VERSION = "v1"; +const DEFAULT_PAGE_SIZE = 100; +const DEFAULT_MAX_PAGES = 25; + +const vertexOpenAIModelSchema = z.object({ + id: z.string().optional().catch(undefined), + name: z.string().optional().catch(undefined), + displayName: z.string().optional().catch(undefined), +}); + +const vertexOpenAIModelsResponseSchema = z.object({ + data: z + .array(z.unknown()) + .optional() + .transform(items => { + if (!items) return []; + const parsedItems: VertexOpenAIModelItem[] = []; + for (const item of items) { + const parsed = vertexOpenAIModelSchema.safeParse(item); + if (parsed.success) parsedItems.push(parsed.data); + } + return parsedItems; + }), + nextPageToken: z.string().optional().catch(undefined), +}); + +type VertexOpenAIModelItem = z.infer; + +/** Configuration for Vertex AI OpenAI-compatible model discovery. */ +export interface VertexDiscoveryOptions { + /** Google Cloud project ID hosting the Vertex AI endpoint. */ + project: string; + /** Vertex AI location, for example `global` or `us-central1`. */ + location: string; + /** Optional requested page size for model listing. */ + pageSize?: number; + /** Maximum number of pages to request before stopping pagination. */ + maxPages?: number; + /** Optional abort signal for HTTP requests. */ + signal?: AbortSignal; + /** Optional fetch implementation override for tests. */ + fetch?: FetchImpl; +} + +/** + * Fetches models exposed by Vertex AI's OpenAI-compatible endpoint. + * + * Returns `null` on auth, transport, or protocol failures so callers can fall + * back to cache/static models without surfacing discovery noise at startup. + */ +export async function fetchVertexOpenAIModels( + options: VertexDiscoveryOptions, +): Promise[] | null> { + const project = options.project.trim(); + const location = options.location.trim(); + if (!project || !location) return null; + + const fetchImpl = options.fetch ?? fetch; + const baseUrl = buildVertexOpenAIBaseUrl(project, location); + const pageSize = normalizePositiveInt(options.pageSize, DEFAULT_PAGE_SIZE); + const maxPages = normalizePositiveInt(options.maxPages, DEFAULT_MAX_PAGES); + let accessToken: string; + try { + accessToken = await getVertexAccessToken({ signal: options.signal, fetch: fetchImpl }); + } catch { + return null; + } + + const modelsById = new Map>(); + const seenTokens = new Set(); + let nextPageToken: string | undefined; + for (let page = 0; page < maxPages; page += 1) { + const requestUrl = buildModelsUrl(baseUrl, pageSize, nextPageToken); + let response: Response; + try { + response = await fetchImpl(requestUrl, { + method: "GET", + headers: { Authorization: `Bearer ${accessToken}` }, + signal: options.signal, + }); + } catch { + return null; + } + + if (!response.ok) return null; + + let payload: unknown; + try { + payload = await response.json(); + } catch { + return null; + } + + const parsed = vertexOpenAIModelsResponseSchema.safeParse(payload); + if (!parsed.success) return null; + + for (const item of parsed.data.data) { + const model = normalizeModel(item, baseUrl); + if (model) modelsById.set(model.id, model); + } + + const token = normalizePageToken(parsed.data.nextPageToken); + if (!token || seenTokens.has(token)) break; + seenTokens.add(token); + nextPageToken = token; + } + + return Array.from(modelsById.values()).sort((left, right) => left.id.localeCompare(right.id)); +} + +/** Returns the stable Vertex AI OpenAI-compatible endpoint base URL. */ +export function buildVertexOpenAIBaseUrl(project: string, location: string): string { + const host = location === "global" ? "aiplatform.googleapis.com" : `${location}-aiplatform.googleapis.com`; + return `https://${host}/${API_VERSION}/projects/${project}/locations/${location}/endpoints/openapi`; +} + +function buildModelsUrl(baseUrl: string, pageSize: number, pageToken?: string): URL { + const url = new URL(`${baseUrl}/models`); + url.searchParams.set("pageSize", String(pageSize)); + if (pageToken) url.searchParams.set("pageToken", pageToken); + return url; +} + +function normalizePositiveInt(value: number | undefined, fallback: number): number { + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) return fallback; + const normalized = Math.floor(value); + return normalized > 0 ? normalized : fallback; +} + +function normalizePageToken(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const token = value.trim(); + return token.length > 0 ? token : undefined; +} + +function normalizeModel(item: VertexOpenAIModelItem, baseUrl: string): Model<"openai-completions"> | null { + const id = normalizeModelId(item.id ?? item.name); + if (!id) return null; + return { + id, + name: normalizeModelName(item.displayName, id), + api: "openai-completions", + provider: "google-vertex", + baseUrl, + reasoning: inferReasoning(id), + input: inferInput(id), + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: UNK_CONTEXT_WINDOW, + maxTokens: UNK_MAX_TOKENS, + }; +} + +function normalizeModelId(value: string | undefined): string | null { + if (!value) return null; + const trimmed = value.trim(); + if (!trimmed) return null; + const marker = "/models/"; + const markerIndex = trimmed.lastIndexOf(marker); + if (markerIndex >= 0) { + const modelId = trimmed.slice(markerIndex + marker.length); + const publisher = extractPublisher(trimmed.slice(0, markerIndex)); + return publisher ? `${publisher}/${modelId}` : modelId; + } + return trimmed; +} + +function extractPublisher(prefix: string): string | undefined { + const marker = "/publishers/"; + const markerIndex = prefix.lastIndexOf(marker); + if (markerIndex < 0) return undefined; + const publisher = prefix.slice(markerIndex + marker.length).trim(); + return publisher.length > 0 ? publisher : undefined; +} + +function normalizeModelName(displayName: string | undefined, id: string): string { + const trimmed = displayName?.trim(); + return trimmed ? trimmed : id; +} + +function inferReasoning(id: string): boolean { + const normalized = id.toLowerCase(); + return ( + normalized.includes("thinking") || + normalized.includes("reasoning") || + normalized.includes("glm-4.5") || + normalized.includes("glm-4.6") || + normalized.includes("glm-4.7") || + normalized.includes("glm-5") || + normalized.includes("gemini-2.5") || + normalized.includes("gemini-3") + ); +} + +function inferInput(id: string): ("text" | "image")[] { + const normalized = id.toLowerCase(); + if ( + normalized.includes("gemini") || + normalized.includes("vision") || + normalized.includes("image") || + normalized.includes("vl") + ) { + return ["text", "image"]; + } + return ["text"]; +} diff --git a/packages/ai/src/utils/h2-fetch.ts b/packages/ai/src/utils/h2-fetch.ts deleted file mode 100644 index c11c8c867..000000000 --- a/packages/ai/src/utils/h2-fetch.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Patch `globalThis.fetch` to advertise HTTP/2 in TLS ALPN, with transparent - * HTTP/1.1 fallback when the server doesn't negotiate `h2`. - * - * Bun's HTTP/2 client is gated on `BUN_FEATURE_FLAG_EXPERIMENTAL_HTTP2_CLIENT`, - * read by the native runtime before any JS executes; assigning to - * `process.env` from inside JS is a no-op. Per-request `protocol: "http2"` - * activates h2 over TLS ALPN and rejects with `error.code === "HTTP2Unsupported"` - * if the server picks anything else, so we catch and retry without the hint. - * - * Some HTTPS endpoints (e.g. corporate API gateways behind reverse proxies) - * advertise h2 via ALPN but then refuse or reset the connection at the HTTP/2 - * framing layer. Bun surfaces these as `ConnectionRefused`, `ConnectionReset`, - * or `ConnectionClosed` rather than `HTTP2Unsupported`, so we treat those - * codes as h2-fallback triggers as well. - * - * Bun negotiates h2 via ALPN over TLS only (no h2c), so plain `http://` URLs - * skip the attempt entirely — avoids the throw/retry round-trip for localhost. - * - * Idempotent. - */ - -const installed: unique symbol = Symbol.for("oh-my-pi.h2fetch.installed"); - -interface PatchedFetch { - [installed]?: true; -} - -export function installH2Fetch(): void { - const original = globalThis.fetch as typeof fetch & PatchedFetch; - if (original[installed]) return; - - /** Error codes that indicate h2 negotiation/transport failure (not an application error). */ - const h2FallbackCodes: ReadonlySet = new Set([ - "HTTP2Unsupported", // Server selected h1 in ALPN - "ConnectionRefused", // Server refused the h2 connection - "ConnectionReset", // Server reset during h2 handshake - "ConnectionClosed", // Server closed before h2 response - ]); - const wrapper = async function h2fetch(input: string | URL | Request, init?: RequestInit): Promise { - if (!isHttps(input)) return original(input, init); - try { - return await original(input, { ...init, protocol: "http2" }); - } catch (err) { - if (!h2FallbackCodes.has((err as { code?: string }).code ?? "")) throw err; - return original(input, init); - } - } as typeof fetch & PatchedFetch; - - // Preserve `fetch.preconnect` and any other statics SDK code might poke at. - Object.assign(wrapper, original); - wrapper[installed] = true; - globalThis.fetch = wrapper; -} - -function isHttps(input: string | URL | Request): boolean { - if (typeof input === "string") return input.startsWith("https:"); - if (input instanceof URL) return input.protocol === "https:"; - return input.url.startsWith("https:"); -} diff --git a/packages/ai/src/utils/idle-iterator.ts b/packages/ai/src/utils/idle-iterator.ts new file mode 100644 index 000000000..7385612c9 --- /dev/null +++ b/packages/ai/src/utils/idle-iterator.ts @@ -0,0 +1,239 @@ +import { $env } from "@oh-my-pi/pi-utils"; + +const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 120_000; +const DEFAULT_STREAM_FIRST_EVENT_TIMEOUT_MS = 100_000; + +function normalizeIdleTimeoutMs(value: string | undefined, fallback: number): number | undefined { + if (value === undefined) return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + if (parsed <= 0) return undefined; + return Math.trunc(parsed); +} + +/** + * Returns the idle timeout used for provider streaming transports. + * + * `PI_OPENAI_STREAM_IDLE_TIMEOUT_MS` is accepted as a backward-compatible alias. + * Set `PI_STREAM_IDLE_TIMEOUT_MS=0` to disable the watchdog. + * + * Providers that legitimately stream much slower than the global default can pass + * `fallbackMs` to widen the floor used when neither env var nor caller option is set. + * Caller options still take precedence; env overrides still trump the fallback. + */ +export function getStreamIdleTimeoutMs(fallbackMs: number = DEFAULT_STREAM_IDLE_TIMEOUT_MS): number | undefined { + return normalizeIdleTimeoutMs($env.PI_STREAM_IDLE_TIMEOUT_MS ?? $env.PI_OPENAI_STREAM_IDLE_TIMEOUT_MS, fallbackMs); +} + +/** + * Returns the idle timeout used for OpenAI-family streaming transports. + * + * Set `PI_OPENAI_STREAM_IDLE_TIMEOUT_MS=0` to disable the watchdog. + */ +export function getOpenAIStreamIdleTimeoutMs(): number | undefined { + return normalizeIdleTimeoutMs( + $env.PI_OPENAI_STREAM_IDLE_TIMEOUT_MS ?? $env.PI_STREAM_IDLE_TIMEOUT_MS, + DEFAULT_STREAM_IDLE_TIMEOUT_MS, + ); +} + +/** + * Returns the timeout used while waiting for the first stream event. + * The first token can legitimately take longer than later inter-event gaps, + * so the default never undershoots the steady-state idle timeout. + * + * Set `PI_STREAM_FIRST_EVENT_TIMEOUT_MS=0` to disable the watchdog. + * + * Providers whose first response can legitimately take longer (heavy reasoning, + * slow cold-start proxies) can pass `fallbackMs` to widen the floor used when + * neither env var nor caller option is set. Caller options still take precedence; + * env overrides still trump the fallback. + */ +export function getStreamFirstEventTimeoutMs( + idleTimeoutMs?: number, + fallbackMs: number = DEFAULT_STREAM_FIRST_EVENT_TIMEOUT_MS, +): number | undefined { + const fallback = idleTimeoutMs === undefined ? fallbackMs : Math.max(fallbackMs, idleTimeoutMs); + return normalizeIdleTimeoutMs($env.PI_STREAM_FIRST_EVENT_TIMEOUT_MS, fallback); +} + +export interface IdleTimeoutIteratorOptions { + idleTimeoutMs?: number; + firstItemTimeoutMs?: number; + errorMessage: string; + firstItemErrorMessage?: string; + onIdle?: () => void; + onFirstItemTimeout?: () => void; + /** + * Optional semantic-progress predicate. Non-progress items are still yielded, + * but they do not reset the idle deadline. This prevents provider + * keepalive/no-op events from keeping a stalled tool call alive forever. + */ + isProgressItem?: (item: unknown) => boolean; + /** + * Cancel iteration as soon as this signal aborts. Required for caller-driven + * cancellation (ESC) when the underlying transport does not surface signal + * aborts to the iterator (HTTP/2 proxies, native sockets, mocked fetch). + * Without this, the consumer sleeps on iterator.next() until the idle/first + * -event watchdog fires — observable as the issue #912 "Working… forever" + * symptom on the github-copilot provider. + */ + abortSignal?: AbortSignal; +} + +/** + * Yields items from an async iterable while enforcing a maximum idle gap between items. + * + * The first item may use a shorter timeout so stuck requests can be aborted and retried + * before any user-visible content has streamed. + */ +export async function* iterateWithIdleTimeout( + iterable: AsyncIterable, + options: IdleTimeoutIteratorOptions, +): AsyncGenerator { + const firstItemTimeoutMs = options.firstItemTimeoutMs ?? options.idleTimeoutMs; + const firstItemDeadlineMs = + firstItemTimeoutMs !== undefined && firstItemTimeoutMs > 0 ? Date.now() + firstItemTimeoutMs : undefined; + const abortSignal = options.abortSignal; + const iterator = iterable[Symbol.asyncIterator](); + + const closeIterator = (): void => { + const returnPromise = iterator.return?.(); + if (returnPromise) { + void returnPromise.catch(() => {}); + } + }; + + if (abortSignal?.aborted) { + closeIterator(); + throw abortReason(abortSignal); + } + + const withRacy = (promise: Promise) => + promise.then( + result => ({ kind: "next" as const, result }), + error => ({ kind: "error" as const, error }), + ); + + let awaitingFirstItem = true; + const markFirstItemReceived = () => { + awaitingFirstItem = false; + }; + const isProgressItem = (item: T): boolean => { + if (!options.isProgressItem) return true; + try { + return options.isProgressItem(item); + } catch { + return true; + } + }; + let lastProgressAt = Date.now(); + + const noTimeoutEnforced = + (firstItemTimeoutMs === undefined || firstItemTimeoutMs <= 0) && + (options.idleTimeoutMs === undefined || options.idleTimeoutMs <= 0); + + while (true) { + let activeTimeoutMs: number | undefined; + if (awaitingFirstItem) { + if (firstItemDeadlineMs !== undefined) { + activeTimeoutMs = firstItemDeadlineMs - Date.now(); + if (activeTimeoutMs <= 0) { + options.onFirstItemTimeout?.(); + closeIterator(); + throw new Error(options.firstItemErrorMessage ?? options.errorMessage); + } + } + } else if (options.idleTimeoutMs !== undefined && options.idleTimeoutMs > 0) { + activeTimeoutMs = options.idleTimeoutMs - (Date.now() - lastProgressAt); + if (activeTimeoutMs <= 0) { + options.onIdle?.(); + closeIterator(); + throw new Error(options.errorMessage); + } + } + + const nextResultPromise = withRacy(iterator.next()); + + const racers: Array< + Promise< + | { kind: "next"; result: IteratorResult } + | { kind: "error"; error: unknown } + | { kind: "timeout" } + | { kind: "abort" } + > + > = [nextResultPromise]; + + let timer: NodeJS.Timeout | undefined; + let resolveTimeout: ((value: { kind: "timeout" }) => void) | undefined; + const enforceTimeout = !noTimeoutEnforced && activeTimeoutMs !== undefined && activeTimeoutMs > 0; + if (enforceTimeout) { + const { promise, resolve } = Promise.withResolvers<{ kind: "timeout" }>(); + resolveTimeout = resolve; + timer = setTimeout(() => resolve({ kind: "timeout" }), activeTimeoutMs); + racers.push(promise); + } + + let abortListener: (() => void) | undefined; + let resolveAbort: ((value: { kind: "abort" }) => void) | undefined; + if (abortSignal) { + const { promise, resolve } = Promise.withResolvers<{ kind: "abort" }>(); + resolveAbort = resolve; + abortListener = () => resolve({ kind: "abort" }); + abortSignal.addEventListener("abort", abortListener, { once: true }); + racers.push(promise); + } + + try { + const outcome = await Promise.race(racers); + if (outcome.kind === "abort") { + closeIterator(); + throw abortReason(abortSignal!); + } + if (outcome.kind === "timeout") { + if (!awaitingFirstItem) { + options.onIdle?.(); + } else { + options.onFirstItemTimeout?.(); + } + closeIterator(); + throw new Error( + !awaitingFirstItem ? options.errorMessage : (options.firstItemErrorMessage ?? options.errorMessage), + ); + } + if (outcome.kind === "error") { + throw outcome.error; + } + if (outcome.result.done) { + markFirstItemReceived(); + return; + } + const item = outcome.result.value; + // Non-progress items (e.g. provider keepalives, synthetic `start` events that + // arrive before the model has produced any tokens) MUST NOT flip us out of + // `awaitingFirstItem`. Otherwise the next iteration switches from the (longer) + // first-item watchdog to the (shorter) idle watchdog while we're still waiting + // on the model's first real output. + if (isProgressItem(item)) { + markFirstItemReceived(); + lastProgressAt = Date.now(); + } + yield item; + } finally { + if (timer !== undefined) clearTimeout(timer); + // Resolve dangling promises so the racers don't leak (Promise.race is one-shot). + resolveTimeout?.({ kind: "timeout" }); + if (abortListener && abortSignal) { + abortSignal.removeEventListener("abort", abortListener); + } + resolveAbort?.({ kind: "abort" }); + } + } +} + +function abortReason(signal: AbortSignal): Error { + const reason = signal.reason; + if (reason instanceof Error) return reason; + if (typeof reason === "string") return new Error(reason); + return new Error("Request was aborted"); +} diff --git a/packages/ai/src/utils/oauth/moonshot.ts b/packages/ai/src/utils/oauth/moonshot.ts index a87ebc694..b825b9273 100644 --- a/packages/ai/src/utils/oauth/moonshot.ts +++ b/packages/ai/src/utils/oauth/moonshot.ts @@ -1,4 +1,12 @@ -/** Moonshot login flow (API key paste against https://api.moonshot.ai/v1). */ +/** + * Moonshot login flow (API key paste against https://api.moonshot.ai/v1). + * + * Validation hits `GET /v1/models` rather than a chat completion. Moonshot's + * thinking models (e.g. kimi-k2.5/k2.6) reject the `temperature: 0` probe used + * by the chat-completions validator with `invalid temperature: only 1 is + * allowed for this model`, so a hello-world chat call cannot authenticate the + * key reliably across the Moonshot catalog. + */ import { createApiKeyLogin } from "./api-key-login"; export const loginMoonshot = createApiKeyLogin({ @@ -8,9 +16,8 @@ export const loginMoonshot = createApiKeyLogin({ promptMessage: "Paste your Moonshot API key", placeholder: "sk-...", validation: { - kind: "chat-completions", + kind: "models-endpoint", provider: "moonshot", - baseUrl: "https://api.moonshot.ai/v1", - model: "kimi-k2.5", + modelsUrl: "https://api.moonshot.ai/v1/models", }, }); diff --git a/packages/ai/src/utils/schema/fields.ts b/packages/ai/src/utils/schema/fields.ts index 55457d0ef..41e9aacf1 100644 --- a/packages/ai/src/utils/schema/fields.ts +++ b/packages/ai/src/utils/schema/fields.ts @@ -34,6 +34,7 @@ export const UNSUPPORTED_SCHEMA_FIELDS: Record = { maximum: true, exclusiveMinimum: true, exclusiveMaximum: true, + multipleOf: true, pattern: true, format: true, }; diff --git a/packages/ai/test/anthropic-stream-timeout.test.ts b/packages/ai/test/anthropic-stream-timeout.test.ts index fda30d3d8..fbcb7a55d 100644 --- a/packages/ai/test/anthropic-stream-timeout.test.ts +++ b/packages/ai/test/anthropic-stream-timeout.test.ts @@ -4,6 +4,8 @@ import { streamAnthropic } from "../src/providers/anthropic"; import type { Context, Model } from "../src/types"; import { waitForDelayOrAbort } from "./helpers"; +const originalFetch = global.fetch; + const model: Model<"anthropic-messages"> = { id: "claude-sonnet-4-5", name: "Claude Sonnet 4.5", @@ -90,13 +92,11 @@ function createSuccessfulAnthropicEvents(text: string): MockAnthropicEvent[] { function createAnthropicMockStream({ signal, connectDelayMs = 0, - firstEventDelayMs = 0, events, hangAfterEvents = false, }: { signal: AbortSignal | undefined; connectDelayMs?: number; - firstEventDelayMs?: number; events?: MockAnthropicEvent[]; hangAfterEvents?: boolean; }): MockAnthropicRequest { @@ -111,9 +111,6 @@ function createAnthropicMockStream({ await waitForAbortAndThrowAbortError(signal); return; } - if (firstEventDelayMs > 0) { - await waitForDelayOrAbort(firstEventDelayMs, signal); - } for (const event of events) { yield event; } @@ -138,18 +135,20 @@ function createAnthropicMockStream({ } afterEach(() => { - // No shared globals to restore; keep hook so the suite stays explicit. + global.fetch = originalFetch; + vi.restoreAllMocks(); }); -describe("anthropic stream silence", () => { - it("waits for delayed first stream events without retrying", async () => { +describe("anthropic first-event timeout retries", () => { + it("retries when the provider never sends the first stream event", async () => { let attempt = 0; - const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal }) => { + const requestTimeouts: Array = []; + const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal; timeout?: number }) => { attempt += 1; + requestTimeouts.push(requestOptions?.timeout); return createAnthropicMockStream({ signal: requestOptions?.signal, - firstEventDelayMs: 10, - events: createSuccessfulAnthropicEvents("slow first event"), + events: attempt === 1 ? undefined : createSuccessfulAnthropicEvents("retry recovered"), }) as never; }) as unknown as Anthropic["messages"]["create"]; const client = { messages: { create } } as Anthropic; @@ -161,15 +160,18 @@ describe("anthropic stream silence", () => { providerRetryWait, }).result(); - expect(attempt).toBe(1); - expect(providerRetryWait).not.toHaveBeenCalled(); + expect(attempt).toBe(2); + expect(providerRetryWait).toHaveBeenCalledWith(2000, undefined); + expect(requestTimeouts).toEqual([1, 1]); expect(result.stopReason).toBe("stop"); - expect(result.content).toEqual([{ type: "text", text: "slow first event" }]); + expect(result.content).toEqual([{ type: "text", text: "retry recovered" }]); expect(result.responseId).toBe("msg_retry_success"); }); it("does not arm the Anthropic first-event watchdog before the stream connects", async () => { - const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal }) => { + let seenRequestTimeout: number | undefined; + const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal; timeout?: number }) => { + seenRequestTimeout = requestOptions?.timeout; return createAnthropicMockStream({ signal: requestOptions?.signal, connectDelayMs: 2, @@ -180,13 +182,41 @@ describe("anthropic stream silence", () => { const result = await streamAnthropic(model, context, { client, - streamFirstEventTimeoutMs: 1, + streamFirstEventTimeoutMs: 20, }).result(); expect(result.stopReason).toBe("stop"); + expect(seenRequestTimeout).toBe(20); expect(result.content).toEqual([{ type: "text", text: "delayed connect" }]); }); + it("times out before the Anthropic stream connects and forwards the budget to the SDK request", async () => { + let attempt = 0; + const requestTimeouts: Array = []; + const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal; timeout?: number }) => { + attempt += 1; + requestTimeouts.push(requestOptions?.timeout); + return createAnthropicMockStream({ + signal: requestOptions?.signal, + connectDelayMs: 20, + events: createSuccessfulAnthropicEvents("too late"), + }) as never; + }) as unknown as Anthropic["messages"]["create"]; + const client = { messages: { create } } as Anthropic; + const providerRetryWait = vi.fn(async () => {}); + + const result = await streamAnthropic(model, context, { + client, + streamFirstEventTimeoutMs: 1, + providerRetryWait, + }).result(); + + expect(attempt).toBe(4); + expect(providerRetryWait).toHaveBeenCalledTimes(3); + expect(requestTimeouts).toEqual([1, 1, 1, 1]); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Anthropic stream timed out while waiting for the first event"); + }); it("keeps caller aborts as aborted instead of retrying them as first-event timeouts", async () => { let attempt = 0; const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal }) => { @@ -209,7 +239,7 @@ describe("anthropic stream silence", () => { expect(result.errorMessage).not.toBe("Anthropic stream timed out while waiting for the first event"); expect((result.errorMessage ?? "").toLowerCase()).toContain("abort"); }); - it("waits through silent gaps between tool-call events until caller aborts", async () => { + it("fails hung Anthropic streams between tool-call events instead of waiting forever", async () => { let attempt = 0; const create = ((_body: unknown, requestOptions?: { signal?: AbortSignal }) => { attempt += 1; @@ -243,19 +273,16 @@ describe("anthropic stream silence", () => { }) as never; }) as unknown as Anthropic["messages"]["create"]; const client = { messages: { create } } as Anthropic; - const controller = new AbortController(); - setTimeout(() => controller.abort(), 10); const result = await streamAnthropic(model, context, { client, - signal: controller.signal, streamFirstEventTimeoutMs: 5000, streamIdleTimeoutMs: 1, }).result(); expect(attempt).toBe(1); - expect(result.stopReason).toBe("aborted"); - expect(result.errorMessage).not.toBe("Anthropic stream stalled while waiting for the next event"); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Anthropic stream stalled while waiting for the next event"); expect(result.content).toEqual([ { type: "toolCall", diff --git a/packages/ai/test/auth-storage-usage-cache.test.ts b/packages/ai/test/auth-storage-usage-cache.test.ts index b57d28484..15bff0d2d 100644 --- a/packages/ai/test/auth-storage-usage-cache.test.ts +++ b/packages/ai/test/auth-storage-usage-cache.test.ts @@ -272,3 +272,167 @@ describe("AuthStorage usage cache: jitter", () => { } }); }); + +describe("AuthStorage usage cache: terminal refresh failure", () => { + // Regression: a revoked refresh token used to fail the in-line OAuth refresh + // inside the usage probe, get silently swallowed, then trigger the upstream + // 401 → null → last-good fallback chain. The credential was therefore never + // removed from the candidate set and the /usage TUI kept rendering yesterday's + // report — including its now-elapsed `resetsAt`, which the renderer printed + // as e.g. `(-612090ms)`. The fix CAS-disables the row on a definitive refresh + // failure and clears the cache, so the credential drops out cleanly. + it("disables credential and suppresses last-good when OAuth refresh fails with invalid_grant", async () => { + // Row whose access token has just expired — within the 60s refresh skew so + // the usage probe is forced to refresh before issuing the upstream call. + const row = oauthRow(1, "a@example.com"); + (row.credential as { expires: number }).expires = Date.now() - 1000; + const rows = [row]; + + // `makeStore` returns `false` from `tryDisableAuthCredentialIfMatches`, + // which would short-circuit our disable. Use a local store that actually + // performs the soft-delete so we can observe the AuthStorage-side effects. + const cache = new Map(); + let disableCalls = 0; + const store: ObservableStore = { + cache, + close() {}, + listAuthCredentials: () => rows.filter(r => !r.disabledCause), + updateAuthCredential() {}, + deleteAuthCredential(id: number, cause: string) { + const target = rows.find(r => r.id === id); + if (target) target.disabledCause = cause; + }, + tryDisableAuthCredentialIfMatches(id: number, _data: string, cause: string) { + disableCalls += 1; + const target = rows.find(r => r.id === id); + if (!target) return false; + target.disabledCause = cause; + return true; + }, + replaceAuthCredentialsForProvider: () => rows, + upsertAuthCredentialForProvider: () => rows, + deleteAuthCredentialsForProvider() {}, + getCache(key: string, options?: { includeExpired?: boolean }) { + const entry = cache.get(key); + if (!entry) return null; + if (!options?.includeExpired && entry.expiresAtSec * 1000 <= Date.now()) return null; + return entry.value; + }, + setCache(key: string, value: string, expiresAtSec: number) { + cache.set(key, { value, expiresAtSec }); + }, + cleanExpiredCache() {}, + }; + + // Pre-populate the cache with a "last good" report whose inner expiresAt + // is in the past (so `get()` misses) but the entry is still reachable via + // `getStale()`. Mirrors what the prior poll would have written. + const lastGood = makeReport("a@example.com"); + const cacheKey = "usage_cache:report:anthropic:default:oauth|account:account-1|email:a@example.com"; + cache.set(cacheKey, { + value: JSON.stringify({ value: lastGood, expiresAt: 1 }), + expiresAtSec: Math.floor((Date.now() + 24 * 60 * 60_000) / 1000), + }); + + const storage = new AuthStorage(store, { + usageProviderResolver: provider => (provider === "anthropic" ? claudeUsage.claudeUsageProvider : undefined), + refreshOAuthCredential: async () => { + throw new Error("OAuth refresh failed: 400 invalid_grant: refresh token revoked"); + }, + }); + await storage.reload(); + + const fetchSpy = vi.spyOn(claudeUsage.claudeUsageProvider, "fetchUsage"); + + try { + const reports = anthropicReports(await storage.fetchUsageReports()); + + // No last-good fallback: the row was disabled before lastGood could leak. + expect(reports).toHaveLength(0); + // CAS disable was attempted exactly once on the failing row. + expect(disableCalls).toBe(1); + expect(rows[0].disabledCause).toContain("invalid_grant"); + // Upstream probe is short-circuited — no point asking the provider + // with a credential we've just torn down. + expect(fetchSpy).not.toHaveBeenCalled(); + // Cache entry was neutralized: a future `getStale` lookup (e.g. on + // re-login under the same account identity) returns null, not the + // stale report with its already-elapsed `resetsAt`. + const rawAfter = cache.get(cacheKey); + expect(rawAfter).toBeDefined(); + const parsedAfter = JSON.parse(rawAfter!.value); + expect(parsedAfter.value).toBeNull(); + // And a second poll surfaces nothing — the credential is gone from + // `listAuthCredentials`, so `#collectUsageRequests` doesn't even + // look it up. + const secondPoll = anthropicReports(await storage.fetchUsageReports()); + expect(secondPoll).toHaveLength(0); + } finally { + storage.close(); + vi.restoreAllMocks(); + } + }); + + it("preserves last-good fallback for transient (non-definitive) refresh failures", async () => { + // Mirror image: a 502 from the token endpoint is transient — we keep the + // row, fall back to the prior good report, and try again next poll. + const row = oauthRow(2, "b@example.com"); + (row.credential as { expires: number }).expires = Date.now() - 1000; + const rows = [row]; + + const cache = new Map(); + const store: ObservableStore = { + cache, + close() {}, + listAuthCredentials: () => rows.filter(r => !r.disabledCause), + updateAuthCredential() {}, + deleteAuthCredential() {}, + tryDisableAuthCredentialIfMatches() { + return true; + }, + replaceAuthCredentialsForProvider: () => rows, + upsertAuthCredentialForProvider: () => rows, + deleteAuthCredentialsForProvider() {}, + getCache(key: string, options?: { includeExpired?: boolean }) { + const entry = cache.get(key); + if (!entry) return null; + if (!options?.includeExpired && entry.expiresAtSec * 1000 <= Date.now()) return null; + return entry.value; + }, + setCache(key: string, value: string, expiresAtSec: number) { + cache.set(key, { value, expiresAtSec }); + }, + cleanExpiredCache() {}, + }; + + const lastGood = makeReport("b@example.com"); + const cacheKey = "usage_cache:report:anthropic:default:oauth|account:account-2|email:b@example.com"; + cache.set(cacheKey, { + value: JSON.stringify({ value: lastGood, expiresAt: 1 }), + expiresAtSec: Math.floor((Date.now() + 24 * 60 * 60_000) / 1000), + }); + + const storage = new AuthStorage(store, { + usageProviderResolver: provider => (provider === "anthropic" ? claudeUsage.claudeUsageProvider : undefined), + refreshOAuthCredential: async () => { + throw new Error("fetch failed: connect ECONNREFUSED 1.2.3.4:443"); + }, + }); + await storage.reload(); + + // The provider probe runs with the stale credential and fails — we don't + // need a real upstream response, just a deterministic null so the lastGood + // path is the one being tested. + vi.spyOn(claudeUsage.claudeUsageProvider, "fetchUsage").mockResolvedValue(null); + + try { + const reports = anthropicReports(await storage.fetchUsageReports()); + expect(reports).toHaveLength(1); + expect(reports[0]?.metadata?.email).toBe("b@example.com"); + expect(rows[0].disabledCause).toBeNull(); + } finally { + storage.close(); + vi.restoreAllMocks(); + } + }); +}); diff --git a/packages/ai/test/deepseek-reasoning-content.test.ts b/packages/ai/test/deepseek-reasoning-content.test.ts index 48b8407b5..d70d69c91 100644 --- a/packages/ai/test/deepseek-reasoning-content.test.ts +++ b/packages/ai/test/deepseek-reasoning-content.test.ts @@ -369,6 +369,27 @@ describe("DeepSeek reasoning_content tool-call replay", () => { expect(rc).toBe(""); }); + it("sets reasoning_content to empty string for OpenCode Zen big-pickle tool-call turns", () => { + const model = getBundledModel("opencode-zen", "big-pickle") as Model<"openai-completions">; + const compat = detectCompat(model); + expect(compat.requiresReasoningContentForToolCalls).toBe(true); + expect(compat.allowsSyntheticReasoningContentForToolCalls).toBe(false); + + const msg = assistantToolCall(model, [ + { + type: "toolCall", + id: "call_big_pickle", + name: "bash", + arguments: { command: "git status --short" }, + } as ToolCall, + ]); + const messages = convertMessages(model, { messages: [msg] }, compat); + const assistant = messages.find(m => m.role === "assistant"); + expect(assistant).toBeDefined(); + expect(Reflect.get(assistant as object, "reasoning_content")).toBe(""); + expect((assistant as { content: unknown }).content).toBe(""); + }); + it("sets content to empty string (not null) when reasoning_content is present", () => { const model = deepseekModel({ provider: "nvidia", diff --git a/packages/ai/test/google-vertex-discovery.test.ts b/packages/ai/test/google-vertex-discovery.test.ts new file mode 100644 index 000000000..f47870fcc --- /dev/null +++ b/packages/ai/test/google-vertex-discovery.test.ts @@ -0,0 +1,131 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { resolveProviderModels } from "../src/model-manager"; +import { googleVertexModelManagerOptions } from "../src/provider-models/google"; +import { __resetVertexTokenCache } from "../src/providers/google-auth"; + +const OAUTH_TOKEN_URL = "https://oauth2.googleapis.com/token"; +const METADATA_TOKEN_URL = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; + +describe("google-vertex model discovery", () => { + let tempDir = ""; + let dbPath = ""; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "pi-ai-vertex-models-")); + dbPath = path.join(tempDir, "models.db"); + }); + + afterEach(async () => { + __resetVertexTokenCache(); + if (tempDir) { + await fs.rm(tempDir, { recursive: true, force: true }); + tempDir = ""; + dbPath = ""; + } + }); + + it("uses the Vertex OpenAI-compatible model list as the authoritative project catalog", async () => { + const urls: string[] = []; + const options = googleVertexModelManagerOptions({ + project: "vertex-project", + location: "global", + fetch: async input => { + const url = input instanceof Request ? input.url : input.toString(); + urls.push(url); + if (url === METADATA_TOKEN_URL || url === OAUTH_TOKEN_URL) { + return new Response(JSON.stringify({ access_token: "vertex-token", expires_in: 3600 })); + } + if ( + url.startsWith( + "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi/models", + ) + ) { + return new Response( + JSON.stringify({ + data: [ + { id: "zai-org/glm-4.7-maas", displayName: "GLM-4.7" }, + { + name: "projects/vertex-project/locations/global/publishers/anthropic/models/claude-sonnet-4-5", + displayName: "Claude Sonnet 4.5", + }, + ], + }), + ); + } + return new Response("not found", { status: 404 }); + }, + }); + + const result = await resolveProviderModels({ ...options, cacheDbPath: dbPath }, "online"); + + expect(result.stale).toBe(false); + expect(result.models.map(model => model.id)).toEqual(["anthropic/claude-sonnet-4-5", "zai-org/glm-4.7-maas"]); + expect(result.models.every(model => model.provider === "google-vertex")).toBe(true); + expect(result.models.every(model => model.api === "openai-completions")).toBe(true); + expect( + result.models.every( + model => + model.baseUrl === + "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi", + ), + ).toBe(true); + expect(result.models.some(model => model.id === "gemini-1.5-pro")).toBe(false); + expect(urls).toContain( + "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi/models?pageSize=100", + ); + }); + + it("keeps the API-key Vertex Gemini catalog when project and location are also configured", async () => { + const options = googleVertexModelManagerOptions({ + apiKey: "vertex-api-key", + project: "vertex-project", + location: "global", + }); + + const result = await resolveProviderModels({ ...options, cacheDbPath: dbPath }, "offline"); + + expect(result.models.some(model => model.id === "gemini-2.5-pro")).toBe(true); + expect(result.models.every(model => model.provider === "google-vertex")).toBe(true); + expect(result.models.every(model => model.api === "google-vertex")).toBe(true); + }); + + it("omits the bundled Vertex Gemini static fallback when neither ADC project nor API key are configured", async () => { + const previousProject = Bun.env.GOOGLE_CLOUD_PROJECT; + const previousGcpProject = Bun.env.GCP_PROJECT; + const previousGcloudProject = Bun.env.GCLOUD_PROJECT; + const previousVertexLocation = Bun.env.GOOGLE_VERTEX_LOCATION; + const previousCloudLocation = Bun.env.GOOGLE_CLOUD_LOCATION; + const previousLocation = Bun.env.VERTEX_LOCATION; + const previousApiKey = Bun.env.GOOGLE_CLOUD_API_KEY; + delete Bun.env.GOOGLE_CLOUD_PROJECT; + delete Bun.env.GCP_PROJECT; + delete Bun.env.GCLOUD_PROJECT; + delete Bun.env.GOOGLE_VERTEX_LOCATION; + delete Bun.env.GOOGLE_CLOUD_LOCATION; + delete Bun.env.VERTEX_LOCATION; + delete Bun.env.GOOGLE_CLOUD_API_KEY; + try { + const options = googleVertexModelManagerOptions(); + const result = await resolveProviderModels({ ...options, cacheDbPath: dbPath }, "offline"); + expect(result.models).toEqual([]); + } finally { + if (previousProject === undefined) delete Bun.env.GOOGLE_CLOUD_PROJECT; + else Bun.env.GOOGLE_CLOUD_PROJECT = previousProject; + if (previousGcpProject === undefined) delete Bun.env.GCP_PROJECT; + else Bun.env.GCP_PROJECT = previousGcpProject; + if (previousGcloudProject === undefined) delete Bun.env.GCLOUD_PROJECT; + else Bun.env.GCLOUD_PROJECT = previousGcloudProject; + if (previousVertexLocation === undefined) delete Bun.env.GOOGLE_VERTEX_LOCATION; + else Bun.env.GOOGLE_VERTEX_LOCATION = previousVertexLocation; + if (previousCloudLocation === undefined) delete Bun.env.GOOGLE_CLOUD_LOCATION; + else Bun.env.GOOGLE_CLOUD_LOCATION = previousCloudLocation; + if (previousLocation === undefined) delete Bun.env.VERTEX_LOCATION; + else Bun.env.VERTEX_LOCATION = previousLocation; + if (previousApiKey === undefined) delete Bun.env.GOOGLE_CLOUD_API_KEY; + else Bun.env.GOOGLE_CLOUD_API_KEY = previousApiKey; + } + }); +}); diff --git a/packages/ai/test/issue-1399-repro.test.ts b/packages/ai/test/issue-1399-repro.test.ts new file mode 100644 index 000000000..f03e74a90 --- /dev/null +++ b/packages/ai/test/issue-1399-repro.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { hookFetch } from "@oh-my-pi/pi-utils"; +import { streamBedrock } from "../src/providers/amazon-bedrock"; +import { clearAwsCredentialCache } from "../src/providers/aws-credentials"; +import type { Context, Model } from "../src/types"; + +const model: Model<"bedrock-converse-stream"> = { + id: "zai.glm-5", + name: "GLM-5", + api: "bedrock-converse-stream", + provider: "amazon-bedrock", + baseUrl: "https://bedrock-runtime.us-west-2.amazonaws.com", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 131_072, + maxTokens: 16_384, +}; + +const context: Context = { + systemPrompt: [], + messages: [{ role: "user", content: "say hi", timestamp: Date.now() }], +}; + +const awsEnvKeys = [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_PROFILE", + "AWS_REGION", + "AWS_DEFAULT_REGION", + "AWS_CONFIG_FILE", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_EC2_METADATA_DISABLED", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_BEDROCK_SKIP_AUTH", +] as const; +function snapshotAwsEnv(): () => void { + const previous = new Map(); + for (const key of awsEnvKeys) previous.set(key, process.env[key]); + return () => { + for (const key of awsEnvKeys) { + const value = previous.get(key); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + clearAwsCredentialCache(); + }; +} + +describe("issue #1399: Bedrock bearer token precedence", () => { + it("uses AWS_BEARER_TOKEN_BEDROCK without invoking profile credential_process", async () => { + const restoreAwsEnv = snapshotAwsEnv(); + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "omp-bedrock-auth-")); + try { + const configPath = path.join(tempDir, "config"); + await Bun.write( + configPath, + [ + "[default]", + "region = us-west-2", + "credential_process = /bin/sh -c 'echo should-not-run >&2; exit 17'", + "", + ].join("\n"), + ); + + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + delete process.env.AWS_SESSION_TOKEN; + delete process.env.AWS_PROFILE; + delete process.env.AWS_DEFAULT_REGION; + delete process.env.AWS_BEDROCK_SKIP_AUTH; + process.env.AWS_REGION = "us-west-2"; + process.env.AWS_CONFIG_FILE = configPath; + process.env.AWS_SHARED_CREDENTIALS_FILE = path.join(tempDir, "credentials"); + process.env.AWS_EC2_METADATA_DISABLED = "true"; + process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-api-key"; + clearAwsCredentialCache(); + + let requestHeaders: Headers | undefined; + using _hook = hookFetch((_input, init) => { + requestHeaders = new Headers(init?.headers); + return new Response('{"message":"unauthorized"}', { status: 401 }); + }); + + const result = await streamBedrock(model, context, {}).result(); + + expect(requestHeaders?.get("authorization")).toBe("Bearer bedrock-api-key"); + expect(requestHeaders?.has("x-amz-date")).toBe(false); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("Bedrock HTTP 401"); + expect(result.errorMessage).not.toContain("credential_process"); + } finally { + restoreAwsEnv(); + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it("ignores agent sentinel apiKey when AWS_BEARER_TOKEN_BEDROCK is available", async () => { + const restoreAwsEnv = snapshotAwsEnv(); + try { + delete process.env.AWS_ACCESS_KEY_ID; + delete process.env.AWS_SECRET_ACCESS_KEY; + delete process.env.AWS_SESSION_TOKEN; + delete process.env.AWS_PROFILE; + delete process.env.AWS_CONFIG_FILE; + delete process.env.AWS_DEFAULT_REGION; + delete process.env.AWS_BEDROCK_SKIP_AUTH; + process.env.AWS_REGION = "us-west-2"; + process.env.AWS_SHARED_CREDENTIALS_FILE = path.join(os.tmpdir(), "missing-aws-credentials"); + process.env.AWS_EC2_METADATA_DISABLED = "true"; + process.env.AWS_BEARER_TOKEN_BEDROCK = "bedrock-api-key"; + clearAwsCredentialCache(); + + let requestHeaders: Headers | undefined; + using _hook = hookFetch((_input, init) => { + requestHeaders = new Headers(init?.headers); + return new Response('{"message":"unauthorized"}', { status: 401 }); + }); + + const result = await streamBedrock(model, context, { apiKey: "" }).result(); + + expect(requestHeaders?.get("authorization")).toBe("Bearer bedrock-api-key"); + expect(requestHeaders?.get("authorization")).not.toBe("Bearer "); + expect(requestHeaders?.has("x-amz-date")).toBe(false); + expect(result.stopReason).toBe("error"); + } finally { + restoreAwsEnv(); + } + }); + + it("ignores agent sentinel apiKey when signing with AWS credentials", async () => { + const restoreAwsEnv = snapshotAwsEnv(); + try { + process.env.AWS_ACCESS_KEY_ID = "AKIDEXAMPLE"; + process.env.AWS_SECRET_ACCESS_KEY = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"; + delete process.env.AWS_SESSION_TOKEN; + delete process.env.AWS_PROFILE; + delete process.env.AWS_CONFIG_FILE; + delete process.env.AWS_SHARED_CREDENTIALS_FILE; + delete process.env.AWS_DEFAULT_REGION; + delete process.env.AWS_BEARER_TOKEN_BEDROCK; + delete process.env.AWS_BEDROCK_SKIP_AUTH; + process.env.AWS_REGION = "us-west-2"; + process.env.AWS_EC2_METADATA_DISABLED = "true"; + clearAwsCredentialCache(); + + let requestHeaders: Headers | undefined; + using _hook = hookFetch((_input, init) => { + requestHeaders = new Headers(init?.headers); + return new Response('{"message":"unauthorized"}', { status: 401 }); + }); + + const result = await streamBedrock(model, context, { apiKey: "" }).result(); + + const authorization = requestHeaders?.get("authorization"); + expect(authorization).toStartWith("AWS4-HMAC-SHA256 "); + expect(authorization).toContain("Credential=AKIDEXAMPLE/"); + expect(authorization).not.toBe("Bearer "); + expect(requestHeaders?.has("x-amz-date")).toBe(true); + expect(result.stopReason).toBe("error"); + } finally { + restoreAwsEnv(); + } + }); +}); diff --git a/packages/ai/test/mock-provider.test.ts b/packages/ai/test/mock-provider.test.ts index f81258b90..bdd189b3a 100644 --- a/packages/ai/test/mock-provider.test.ts +++ b/packages/ai/test/mock-provider.test.ts @@ -5,7 +5,7 @@ import { isMockModel, MOCK_API, type MockHandler, - type MockModelHandle, + type MockModel, registerMockApi, streamMock, } from "../src/providers/mock"; @@ -298,7 +298,7 @@ describe("mock provider — async-iterable response sources", () => { test("generator can react to call state via the shared handle", async () => { // Each yield reads `mock.calls` to react to what just happened. - let handle!: MockModelHandle; + let handle!: MockModel; const gen = async function* () { yield { content: ["turn 0"] }; // At this point mock.calls.length === 1. diff --git a/packages/ai/test/openai-codex-stream.test.ts b/packages/ai/test/openai-codex-stream.test.ts index 4e5c42fd6..be0d34d95 100644 --- a/packages/ai/test/openai-codex-stream.test.ts +++ b/packages/ai/test/openai-codex-stream.test.ts @@ -286,7 +286,6 @@ describe("openai-codex streaming", () => { const result = await streamOpenAICodexResponses(model, context, { apiKey: token, signal: controller.signal, - streamIdleTimeoutMs: 5, }).result(); expect(result.stopReason).toBe("aborted"); @@ -359,6 +358,63 @@ describe("openai-codex streaming", () => { expect(result.stopReason).toBe("stop"); }); + it("forwards websocket frames through onSseEvent for the raw-SSE debug viewer", async () => { + const tempDir = TempDir.createSync("@pi-codex-stream-"); + setAgentDir(tempDir.path()); + const token = createCodexTestToken(); + + class ObservedWebSocket extends MockWebSocket { + constructor(url: string, options?: { headers?: WsHeaders }) { + super(url, options); + this.scheduleOpen(); + } + + send(): void { + this.emitCodexResponse({ messageId: "msg_obs", responseId: "resp_obs", text: "Observed" }); + } + } + global.WebSocket = ObservedWebSocket as unknown as typeof WebSocket; + + const observed: Array<{ event: string | null; data: string; raw: string[] }> = []; + const result = await streamOpenAICodexResponses( + createCodexTestModel("https://chatgpt.com/backend-api"), + createCodexTestContext(), + { + apiKey: token, + sessionId: "ws-observer-session", + providerSessionState: new Map(), + onSseEvent: event => { + observed.push({ event: event.event, data: event.data, raw: [...event.raw] }); + }, + }, + ).result(); + + expect(result.stopReason).toBe("stop"); + + // First record is the outbound request frame (the JSON we sent). + const [outbound, ...inbound] = observed; + expect(outbound).toBeDefined(); + expect(outbound.raw[0]).toMatch(/^: ws → /); + expect(outbound.data.length).toBeGreaterThan(0); + expect(() => JSON.parse(outbound.data)).not.toThrow(); + + // Inbound frames mirror the Codex response sequence emitted by `emitCodexResponse`. + expect(inbound.map(e => e.event)).toEqual([ + "response.output_item.added", + "response.content_part.added", + "response.output_text.delta", + "response.output_item.done", + "response.done", + ]); + for (const event of inbound) { + expect(event.raw[0]).toBe(`: ws ← ${event.event}`); + // Synthesized SSE wire shape: `event:` line then `data:` line. + expect(event.raw[1]).toBe(`event: ${event.event}`); + expect(event.raw[2]).toBe(`data: ${event.data}`); + expect(JSON.parse(event.data)).toMatchObject({ type: event.event }); + } + }); + it("omits request-body headers and replaces stale beta headers for websocket handshakes", async () => { const tempDir = TempDir.createSync("@pi-codex-stream-"); setAgentDir(tempDir.path()); @@ -907,6 +963,48 @@ describe("openai-codex streaming", () => { const streamResult = streamOpenAICodexResponses(model, context, { apiKey: token, sessionId }); await streamResult.result(); }); + it("keeps prompt_cache_key separate from Codex conversation headers", async () => { + const tempDir = TempDir.createSync("@pi-codex-stream-"); + setAgentDir(tempDir.path()); + + const token = createCodexTestToken(); + const model = createCodexTestModel("https://chatgpt.com/backend-api"); + const sessionId = "side-channel-session"; + const promptCacheKey = "main-session-cache"; + let capturedHeaders: Headers | undefined; + let capturedBody: Record | undefined; + + global.fetch = vi.fn(async (input: string | URL, init?: RequestInit) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://api.github.com/repos/openai/codex/releases/latest") { + return new Response(JSON.stringify({ tag_name: "rust-v0.0.0" }), { status: 200 }); + } + if (url.startsWith("https://raw.githubusercontent.com/openai/codex/")) { + return new Response("PROMPT", { status: 200, headers: { etag: '"etag"' } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses") { + capturedHeaders = init?.headers instanceof Headers ? init.headers : new Headers(init?.headers); + capturedBody = + typeof init?.body === "string" ? (JSON.parse(init.body) as Record) : undefined; + return new Response(createCompletedCodexSse("Hello"), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response("not found", { status: 404 }); + }) as unknown as typeof fetch; + + await streamOpenAICodexResponses(model, createCodexTestContext(), { + apiKey: token, + sessionId, + promptCacheKey, + }).result(); + + expect(capturedHeaders?.get("conversation_id")).toBe(sessionId); + expect(capturedHeaders?.get("session_id")).toBe(sessionId); + expect(capturedHeaders?.get("x-client-request-id")).toBe(sessionId); + expect(capturedBody?.prompt_cache_key).toBe(promptCacheKey); + }); it("rejects gpt-5.3-codex minimal reasoning effort instead of clamping", async () => { const tempDir = TempDir.createSync("@pi-codex-stream-"); @@ -1788,13 +1886,12 @@ describe("openai-codex streaming", () => { expect(fetchMock).not.toHaveBeenCalled(); }); - it("waits for caller abort when websocket status events do not make semantic progress", async () => { + it("surfaces a websocket idle-timeout error when status events never make semantic progress", async () => { const tempDir = TempDir.createSync("@pi-codex-stream-"); setAgentDir(tempDir.path()); const token = createCodexTestToken(); - const sse = createCompletedCodexSse("Hello fallback"); const fetchMock = vi.fn(async () => { - return new Response(sse, { headers: { "content-type": "text/event-stream" } }); + throw new Error("SSE fallback should not run once the websocket stream becomes replay-unsafe"); }); global.fetch = fetchMock as unknown as typeof fetch; @@ -1818,6 +1915,16 @@ describe("openai-codex streaming", () => { arguments: "", }, }); + this.sendJson({ + type: "response.output_item.done", + item: { + type: "function_call", + id: "fc_ws_stalled", + call_id: "call_ws_stalled", + name: "todo_write", + arguments: "{}", + }, + }); interval = setInterval(() => { this.sendJson({ type: "response.in_progress", @@ -1835,18 +1942,25 @@ describe("openai-codex streaming", () => { const model = createCodexTestModel("https://chatgpt.com/backend-api"); const providerSessionState = new Map(); - const controller = new AbortController(); - setTimeout(() => controller.abort(), 30); const result = await streamOpenAICodexResponses(model, createCodexTestContext(), { apiKey: token, sessionId: "ws-no-progress-session", providerSessionState, - signal: controller.signal, streamIdleTimeoutMs: 5, }).result(); expect(sendCount).toBe(1); - expect(result.stopReason).toBe("aborted"); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("idle timeout waiting for websocket"); + expect(result.content).toEqual([ + expect.objectContaining({ + type: "toolCall", + id: "call_ws_stalled|fc_ws_stalled", + name: "todo_write", + arguments: {}, + partialJson: "", + }), + ]); expect(fetchMock).not.toHaveBeenCalled(); }); diff --git a/packages/ai/test/openai-completions-compat.test.ts b/packages/ai/test/openai-completions-compat.test.ts index e96c3d6ae..f1e4231f7 100644 --- a/packages/ai/test/openai-completions-compat.test.ts +++ b/packages/ai/test/openai-completions-compat.test.ts @@ -183,6 +183,57 @@ describe("openai-completions compatibility", () => { ]); }); + it("defaults supportsDeveloperRole to off for non-OpenAI/Azure hosts", () => { + // Regression: Moonshot's Kimi chat template rejects the `developer` role + // with `400 Invalid request: tokenization failed` because `developer` is + // an OpenAI extension and most other hosts don't carry it through their + // tokenizer. The default for any non-OpenAI/Azure host MUST be `system`, + // so reasoning models on those hosts cannot accidentally emit `developer`. + const cases: Array<{ provider: string; baseUrl: string; expected: boolean }> = [ + { provider: "openai", baseUrl: "https://api.openai.com/v1", expected: true }, + { provider: "azure", baseUrl: "https://example.openai.azure.com/openai", expected: true }, + { provider: "moonshot", baseUrl: "https://api.moonshot.ai/v1", expected: false }, + { provider: "openrouter", baseUrl: "https://openrouter.ai/api/v1", expected: false }, + { provider: "groq", baseUrl: "https://api.groq.com/openai/v1", expected: false }, + { provider: "github-copilot", baseUrl: "https://api.githubcopilot.com", expected: false }, + ]; + for (const { provider, baseUrl, expected } of cases) { + const model: Model<"openai-completions"> = { + ...getBundledModel("openai", "gpt-4o-mini"), + api: "openai-completions", + provider: provider as Model["provider"], + baseUrl, + reasoning: true, + }; + expect(detectCompat(model).supportsDeveloperRole).toBe(expected); + } + }); + + it("emits system role for reasoning models on Moonshot (kimi tokenization rejects developer)", () => { + const model: Model<"openai-completions"> = { + ...getBundledModel("openai", "gpt-4o-mini"), + api: "openai-completions", + provider: "moonshot", + baseUrl: "https://api.moonshot.ai/v1", + id: "kimi-k2.5", + reasoning: true, + }; + + const messages = convertMessages( + model, + { + systemPrompt: ["you are a helpful assistant"], + messages: [{ role: "user", content: "hi", timestamp: Date.now() }], + }, + detectCompat(model), + ); + + expect(messages.slice(0, 2)).toEqual([ + { role: "system", content: "you are a helpful assistant" }, + { role: "user", content: "hi" }, + ]); + }); + it("coalesces ordered system prompts when the host disables multi-system support", () => { const model: Model<"openai-completions"> = { ...getBundledModel("openai", "gpt-4o-mini"), diff --git a/packages/ai/test/openai-completions-progress-chunk.test.ts b/packages/ai/test/openai-completions-progress-chunk.test.ts new file mode 100644 index 000000000..e8821a2dc --- /dev/null +++ b/packages/ai/test/openai-completions-progress-chunk.test.ts @@ -0,0 +1,262 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { getBundledModel } from "../src/models"; +import { isOpenAICompletionsProgressChunk, streamOpenAICompletions } from "../src/providers/openai-completions"; +import type { Context, Model } from "../src/types"; + +const originalFetch = global.fetch; + +const openAICompletionsModel = { + ...(getBundledModel("openai", "gpt-4o-mini") as Model<"openai-completions">), + api: "openai-completions", +} satisfies Model<"openai-completions">; + +function baseContext(): Context { + return { + messages: [{ role: "user", content: "Say hello", timestamp: Date.now() }], + }; +} + +function getRequestSignal(input: string | URL | Request, init: RequestInit | undefined): AbortSignal | undefined { + if (init?.signal) { + return init.signal; + } + if (input instanceof Request) { + return input.signal; + } + return undefined; +} + +function createKeepaliveOnlyCompletionsResponse(modelId: string, signal: AbortSignal | undefined): Response { + const encoder = new TextEncoder(); + let interval: NodeJS.Timeout | undefined; + let abortListener: (() => void) | undefined; + const encode = (event: unknown): Uint8Array => encoder.encode(`data: ${JSON.stringify(event)}\n\n`); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + encode({ + id: "chatcmpl-stalled", + object: "chat.completion.chunk", + created: 0, + model: modelId, + choices: [{ index: 0, delta: { content: "Hello" } }], + }), + ); + interval = setInterval(() => { + controller.enqueue( + encode({ + id: "chatcmpl-stalled", + object: "chat.completion.chunk", + created: 0, + model: modelId, + choices: [{ index: 0, delta: { role: "assistant" } }], + }), + ); + }, 2); + abortListener = () => { + if (interval) clearInterval(interval); + if (abortListener) signal?.removeEventListener("abort", abortListener); + const reason = signal?.reason; + controller.error(reason instanceof Error ? reason : new Error("request aborted")); + }; + if (signal?.aborted) { + queueMicrotask(() => abortListener?.()); + } else { + signal?.addEventListener("abort", abortListener, { once: true }); + } + }, + cancel() { + if (interval) clearInterval(interval); + if (abortListener) signal?.removeEventListener("abort", abortListener); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +afterEach(() => { + global.fetch = originalFetch; +}); +/** + * Contract: `isOpenAICompletionsProgressChunk` decides whether a streamed chunk + * resets the idle-watchdog deadline in `iterateWithIdleTimeout`. A false + * positive (counting a no-op chunk as progress) silently disables the + * watchdog and is the root cause of the z.ai/GLM-via-OpenRouter hang where + * a subagent stalled for hours with no error surfaced. A false negative is + * cheap (delays the watchdog by at most the first-event window). + */ +describe("isOpenAICompletionsProgressChunk", () => { + describe("non-progress chunks (MUST NOT reset the watchdog)", () => { + it("rejects null/non-object", () => { + expect(isOpenAICompletionsProgressChunk(null)).toBe(false); + expect(isOpenAICompletionsProgressChunk(undefined)).toBe(false); + expect(isOpenAICompletionsProgressChunk("hi")).toBe(false); + expect(isOpenAICompletionsProgressChunk(42)).toBe(false); + }); + + it("rejects empty {} keepalives", () => { + expect(isOpenAICompletionsProgressChunk({})).toBe(false); + }); + + it("rejects {choices: []} keepalives", () => { + expect(isOpenAICompletionsProgressChunk({ choices: [] })).toBe(false); + }); + + it("rejects role-only preambles", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { role: "assistant" } }], + }), + ).toBe(false); + }); + + it("rejects empty-string content deltas", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { content: "" } }], + }), + ).toBe(false); + }); + + it("rejects empty-array content deltas", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { content: [] } }], + }), + ).toBe(false); + }); + + it("rejects empty tool_calls arrays", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { tool_calls: [] } }], + }), + ).toBe(false); + }); + + it("rejects empty reasoning fields", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { reasoning: "" } }], + }), + ).toBe(false); + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { reasoning_content: "" } }], + }), + ).toBe(false); + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { reasoning_text: "" } }], + }), + ).toBe(false); + }); + }); + + describe("progress chunks (MUST reset the watchdog)", () => { + it("accepts a top-level usage chunk (terminal token report)", () => { + expect( + isOpenAICompletionsProgressChunk({ + usage: { prompt_tokens: 12, completion_tokens: 4 }, + }), + ).toBe(true); + }); + + it("accepts choice-level usage", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ usage: { prompt_tokens: 12 } }], + }), + ).toBe(true); + }); + + it("accepts finish_reason", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ finish_reason: "stop" }], + }), + ).toBe(true); + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ finish_reason: "tool_calls" }], + }), + ).toBe(true); + }); + + it("accepts text content deltas", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { content: "Hello" } }], + }), + ).toBe(true); + }); + + it("accepts array-shape content parts (Mistral-style)", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { content: [{ type: "text", text: "Hi" }] } }], + }), + ).toBe(true); + }); + + it("accepts tool call deltas", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [ + { + delta: { + tool_calls: [{ id: "call_1", function: { name: "search", arguments: "{}" } }], + }, + }, + ], + }), + ).toBe(true); + }); + + it("accepts reasoning deltas in all three field names", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { reasoning: "thinking..." } }], + }), + ).toBe(true); + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { reasoning_content: "thinking..." } }], + }), + ).toBe(true); + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { reasoning_text: "thinking..." } }], + }), + ).toBe(true); + }); + + it("accepts refusal deltas", () => { + expect( + isOpenAICompletionsProgressChunk({ + choices: [{ delta: { refusal: "I can't help with that." } }], + }), + ).toBe(true); + }); + }); +}); +describe("provider integration", () => { + it("times out a completions stream whose keepalives never make progress", async () => { + global.fetch = ((input: string | URL | Request, init?: RequestInit) => + Promise.resolve( + createKeepaliveOnlyCompletionsResponse(openAICompletionsModel.id, getRequestSignal(input, init)), + )) as typeof fetch; + + const result = await streamOpenAICompletions(openAICompletionsModel, baseContext(), { + apiKey: "test-key", + streamFirstEventTimeoutMs: 1_000, + streamIdleTimeoutMs: 20, + }).result(); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("OpenAI completions stream stalled while waiting for the next event"); + expect(result.content).toEqual([{ type: "text", text: "Hello" }]); + }); +}); diff --git a/packages/ai/test/openai-first-event-timeout.test.ts b/packages/ai/test/openai-first-event-timeout.test.ts index aa7c88eb7..e74afff67 100644 --- a/packages/ai/test/openai-first-event-timeout.test.ts +++ b/packages/ai/test/openai-first-event-timeout.test.ts @@ -3,6 +3,7 @@ import { getBundledModel } from "../src/models"; import { streamAzureOpenAIResponses } from "../src/providers/azure-openai-responses"; import { streamOpenAICompletions } from "../src/providers/openai-completions"; import { streamOpenAIResponses } from "../src/providers/openai-responses"; +import { streamSimple } from "../src/stream"; import type { Context, Model, TextContent } from "../src/types"; import { waitForDelayOrAbort } from "./helpers"; @@ -42,6 +43,15 @@ function getRequestSignal(input: string | URL | Request, init: RequestInit | und return undefined; } +function getRequestHeader(input: string | URL | Request, init: RequestInit | undefined, name: string): string | null { + if (init?.headers) { + return new Headers(init.headers).get(name); + } + if (input instanceof Request) { + return input.headers.get(name); + } + return null; +} function createHangingSseResponse(signal: AbortSignal | undefined): Response { let abortListener: (() => void) | undefined; const stream = new ReadableStream({ @@ -143,28 +153,19 @@ function createNoProgressOpenAIResponsesStream(signal: AbortSignal | undefined): }); } -function createDelayedFetch(delayMs: number, responseFactory: () => Response): typeof fetch { +function createDelayedFetch( + delayMs: number, + responseFactory: () => Response, + onRequest?: (input: string | URL | Request, init: RequestInit | undefined) => void, +): typeof fetch { async function mockFetch(input: string | URL | Request, init?: RequestInit): Promise { + onRequest?.(input, init); await waitForDelayOrAbort(delayMs, getRequestSignal(input, init)); return responseFactory(); } return Object.assign(mockFetch, { preconnect: originalFetch.preconnect }); } -function createAbortObservingDelayedFetch(delayMs: number, onAbort: () => void): typeof fetch { - async function mockFetch(input: string | URL | Request, init?: RequestInit): Promise { - const signal = getRequestSignal(input, init); - try { - await waitForDelayOrAbort(delayMs, signal); - } catch (error) { - if (signal?.aborted) onAbort(); - throw error; - } - return createOpenAIResponsesSuccessResponse(); - } - - return Object.assign(mockFetch, { preconnect: originalFetch.preconnect }); -} function createOpenAIResponsesSuccessResponse(): Response { return createSseResponse([ @@ -227,22 +228,33 @@ function createOpenAICompletionsSuccessResponse(modelId: string): Response { ]); } -async function expectProviderSilenceWaitsForCallerAbort( - run: ( - signal: AbortSignal, - streamFirstEventTimeoutMs: number, - ) => Promise<{ stopReason: string; errorMessage?: string }>, - unexpectedMessage: string, +async function expectFirstEventTimeout( + run: (streamFirstEventTimeoutMs: number) => Promise<{ stopReason: string; errorMessage?: string }>, + expectedMessage: string, ): Promise { global.fetch = createHangingFetch(); - const controller = new AbortController(); - setTimeout(() => controller.abort(), 30); - const result = await run(controller.signal, 5); + const result = await run(20); - expect(result.stopReason).toBe("aborted"); - expect(result.errorMessage).not.toBe(unexpectedMessage); - expect((result.errorMessage ?? "").toLowerCase()).toContain("abort"); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe(expectedMessage); +} + +async function expectRequestSetupTimeout( + run: (streamFirstEventTimeoutMs: number) => Promise<{ stopReason: string; errorMessage?: string }>, + expectedMessage: string, + responseFactory: () => Response, +): Promise { + const timeoutHeaders: string[] = []; + global.fetch = createDelayedFetch(30, responseFactory, (input, init) => { + timeoutHeaders.push(getRequestHeader(input, init, "X-Stainless-Timeout") ?? ""); + }); + + const result = await run(20); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe(expectedMessage); + expect(timeoutHeaders).toContain("0"); } async function expectCallerAbort( @@ -270,12 +282,12 @@ function getFirstTextContent(result: { content: unknown[] }): TextContent | unde } async function expectDelayedRequestSetupSucceeds( - run: () => Promise<{ stopReason: string; content: unknown[] }>, + run: (streamFirstEventTimeoutMs: number) => Promise<{ stopReason: string; content: unknown[] }>, responseFactory: () => Response, ): Promise { global.fetch = createDelayedFetch(30, responseFactory); - const result = await run(); + const result = await run(50); expect(result.stopReason).toBe("stop"); expect(getFirstTextContent(result)).toMatchObject({ type: "text", text: "Hello delayed" }); @@ -285,34 +297,41 @@ afterEach(() => { global.fetch = originalFetch; }); -describe("OpenAI-family provider stream silence", () => { - it("waits for caller abort when OpenAI responses emits no first event", async () => { - await expectProviderSilenceWaitsForCallerAbort( - (signal, streamFirstEventTimeoutMs) => +describe("OpenAI-family first-event timeouts", () => { + it("surfaces the OpenAI responses first-event timeout message instead of a generic abort", async () => { + await expectFirstEventTimeout( + streamFirstEventTimeoutMs => streamOpenAIResponses(openAIResponsesModel, baseContext(), { apiKey: "test-key", - signal, streamFirstEventTimeoutMs, }).result(), "OpenAI responses stream timed out while waiting for the first event", ); }); + it("times out OpenAI responses before the stream opens and forwards the budget to the SDK request", async () => { + await expectRequestSetupTimeout( + streamFirstEventTimeoutMs => + streamOpenAIResponses(openAIResponsesModel, baseContext(), { + apiKey: "test-key", + streamFirstEventTimeoutMs, + }).result(), + "OpenAI responses stream timed out while waiting for the first event", + createOpenAIResponsesSuccessResponse, + ); + }); - it("waits for caller abort when OpenAI responses streams only no-progress status events", async () => { + it("times out OpenAI responses streams that only emit no-progress status events", async () => { global.fetch = ((input: string | URL | Request, init?: RequestInit) => Promise.resolve(createNoProgressOpenAIResponsesStream(getRequestSignal(input, init)))) as typeof fetch; - const controller = new AbortController(); - setTimeout(() => controller.abort(), 30); const result = await streamOpenAIResponses(openAIResponsesModel, baseContext(), { apiKey: "test-key", - signal: controller.signal, streamFirstEventTimeoutMs: 1_000, - streamIdleTimeoutMs: 5, + streamIdleTimeoutMs: 20, }).result(); - expect(result.stopReason).toBe("aborted"); - expect(result.errorMessage).not.toBe("OpenAI responses stream stalled while waiting for the next event"); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("OpenAI responses stream stalled while waiting for the next event"); expect(result.content as unknown[]).toEqual([ { type: "toolCall", @@ -324,31 +343,98 @@ describe("OpenAI-family provider stream silence", () => { ]); }); - it("waits for caller abort when OpenAI completions emits no first event", async () => { - await expectProviderSilenceWaitsForCallerAbort( - (signal, streamFirstEventTimeoutMs) => + it("forwards streamSimple per-call timeout options to OpenAI-family providers", async () => { + global.fetch = createHangingFetch(); + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(new Error("fallback abort")), 200); + abortTimer.unref(); + + try { + const result = await streamSimple(openAIResponsesModel, baseContext(), { + apiKey: "test-key", + signal: controller.signal, + streamFirstEventTimeoutMs: 20, + streamIdleTimeoutMs: 20, + }).result(); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("OpenAI responses stream timed out while waiting for the first event"); + } finally { + clearTimeout(abortTimer); + } + }); + + it("surfaces the OpenAI completions first-event timeout message", async () => { + await expectFirstEventTimeout( + streamFirstEventTimeoutMs => streamOpenAICompletions(openAICompletionsModel, baseContext(), { apiKey: "test-key", - signal, streamFirstEventTimeoutMs, }).result(), "OpenAI completions stream timed out while waiting for the first event", ); }); + it("times out OpenAI completions before the stream opens and forwards the budget to the SDK request", async () => { + await expectRequestSetupTimeout( + streamFirstEventTimeoutMs => + streamOpenAICompletions(openAICompletionsModel, baseContext(), { + apiKey: "test-key", + streamFirstEventTimeoutMs, + }).result(), + "OpenAI completions stream timed out while waiting for the first event", + () => createOpenAICompletionsSuccessResponse(openAICompletionsModel.id), + ); + }); - it("waits for caller abort when Azure OpenAI responses emits no first event", async () => { - await expectProviderSilenceWaitsForCallerAbort( - (signal, streamFirstEventTimeoutMs) => + it("surfaces the Azure OpenAI responses first-event timeout message", async () => { + await expectFirstEventTimeout( + streamFirstEventTimeoutMs => streamAzureOpenAIResponses(azureOpenAIResponsesModel, baseContext(), { apiKey: "test-key", azureBaseUrl: azureOpenAIResponsesModel.baseUrl, azureApiVersion: "v1", - signal, streamFirstEventTimeoutMs, }).result(), "Azure OpenAI responses stream timed out while waiting for the first event", ); }); + it("times out Azure OpenAI responses before the stream opens and forwards the budget to the SDK request", async () => { + await expectRequestSetupTimeout( + streamFirstEventTimeoutMs => + streamAzureOpenAIResponses(azureOpenAIResponsesModel, baseContext(), { + apiKey: "test-key", + azureBaseUrl: azureOpenAIResponsesModel.baseUrl, + azureApiVersion: "v1", + streamFirstEventTimeoutMs, + }).result(), + "Azure OpenAI responses stream timed out while waiting for the first event", + createOpenAIResponsesSuccessResponse, + ); + }); + + it("times out Azure responses streams that only emit no-progress status events", async () => { + global.fetch = ((input: string | URL | Request, init?: RequestInit) => + Promise.resolve(createNoProgressOpenAIResponsesStream(getRequestSignal(input, init)))) as typeof fetch; + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(new Error("fallback abort")), 200); + abortTimer.unref(); + + try { + const result = await streamAzureOpenAIResponses(azureOpenAIResponsesModel, baseContext(), { + apiKey: "test-key", + azureBaseUrl: azureOpenAIResponsesModel.baseUrl, + azureApiVersion: "v1", + signal: controller.signal, + streamFirstEventTimeoutMs: 1_000, + streamIdleTimeoutMs: 20, + }).result(); + + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Azure OpenAI responses stream stalled while waiting for the next event"); + } finally { + clearTimeout(abortTimer); + } + }); it("keeps caller aborts as aborted for OpenAI responses", async () => { await expectCallerAbort( @@ -388,69 +474,38 @@ describe("OpenAI-family provider stream silence", () => { ); }); - it("does not apply a default first-event timeout before OpenAI responses stream setup finishes", async () => { + it("does not arm the first-event watchdog before OpenAI responses stream setup finishes", async () => { await expectDelayedRequestSetupSucceeds( - () => + streamFirstEventTimeoutMs => streamOpenAIResponses(openAIResponsesModel, baseContext(), { apiKey: "test-key", + streamFirstEventTimeoutMs, }).result(), createOpenAIResponsesSuccessResponse, ); }); - it("maps explicit OpenAI responses first-event timeout to the SDK request timeout", async () => { - let abortObserved = false; - global.fetch = createAbortObservingDelayedFetch(1_000, () => { - abortObserved = true; - }); - - const result = await streamOpenAIResponses(openAIResponsesModel, baseContext(), { - apiKey: "test-key", - streamFirstEventTimeoutMs: 10, - }).result(); - - expect(abortObserved).toBe(true); - expect(result.stopReason).toBe("error"); - expect(getFirstTextContent(result)).toBeUndefined(); - }); - - it("does not apply a default first-event timeout before OpenAI completions stream setup finishes", async () => { + it("does not arm the first-event watchdog before OpenAI completions stream setup finishes", async () => { await expectDelayedRequestSetupSucceeds( - () => + streamFirstEventTimeoutMs => streamOpenAICompletions(openAICompletionsModel, baseContext(), { apiKey: "test-key", + streamFirstEventTimeoutMs, }).result(), () => createOpenAICompletionsSuccessResponse(openAICompletionsModel.id), ); }); - it("does not apply a default first-event timeout before Azure OpenAI responses setup finishes", async () => { + it("does not arm the first-event watchdog before Azure OpenAI responses setup finishes", async () => { await expectDelayedRequestSetupSucceeds( - () => + streamFirstEventTimeoutMs => streamAzureOpenAIResponses(azureOpenAIResponsesModel, baseContext(), { apiKey: "test-key", azureBaseUrl: azureOpenAIResponsesModel.baseUrl, azureApiVersion: "v1", + streamFirstEventTimeoutMs, }).result(), createOpenAIResponsesSuccessResponse, ); }); - - it("maps explicit Azure OpenAI responses first-event timeout to the SDK request timeout", async () => { - let abortObserved = false; - global.fetch = createAbortObservingDelayedFetch(1_000, () => { - abortObserved = true; - }); - - const result = await streamAzureOpenAIResponses(azureOpenAIResponsesModel, baseContext(), { - apiKey: "test-key", - azureBaseUrl: azureOpenAIResponsesModel.baseUrl, - azureApiVersion: "v1", - streamFirstEventTimeoutMs: 10, - }).result(); - - expect(abortObserved).toBe(true); - expect(result.stopReason).toBe("error"); - expect(getFirstTextContent(result)).toBeUndefined(); - }); }); diff --git a/packages/ai/test/openai-responses-cache-affinity.test.ts b/packages/ai/test/openai-responses-cache-affinity.test.ts index 31a8cb7dc..93ae55db4 100644 --- a/packages/ai/test/openai-responses-cache-affinity.test.ts +++ b/packages/ai/test/openai-responses-cache-affinity.test.ts @@ -89,6 +89,16 @@ describe("openai-responses cache affinity", () => { expect(captured.clientRequestId).toBe("session-123"); expect(captured.body?.prompt_cache_key).toBe("session-123"); }); + it("keeps prompt cache key separate from OpenAI routing headers when both are provided", async () => { + const captured = await captureOpenAIResponseHeaders({ + sessionId: "side-channel-456", + promptCacheKey: "session-123", + }); + + expect(captured.sessionId).toBe("side-channel-456"); + expect(captured.clientRequestId).toBe("side-channel-456"); + expect(captured.body?.prompt_cache_key).toBe("session-123"); + }); it("lets explicit headers override the default OpenAI session routing headers", async () => { const captured = await captureOpenAIResponseHeaders({ diff --git a/packages/ai/test/register-builtins.test.ts b/packages/ai/test/register-builtins.test.ts index f4dfc51f4..8e6628786 100644 --- a/packages/ai/test/register-builtins.test.ts +++ b/packages/ai/test/register-builtins.test.ts @@ -93,35 +93,41 @@ describe("register-builtins lazy streams", () => { expect(result.errorMessage).toContain("bedrock exploded"); }); - it("forwards lazy provider streams without first-event or idle watchdogs", async () => { + it("turns idle lazy provider streams into retryable terminal errors", async () => { const partialMessage = createAssistantMessage("stop"); - const finalMessage = createAssistantMessage("stop"); - finalMessage.content = [{ type: "text", text: "slow ok" }]; + let providerSignal: AbortSignal | undefined; const source = { async *[Symbol.asyncIterator]() { - await Bun.sleep(30); yield { type: "start", partial: partialMessage } as const; - await Bun.sleep(30); - yield { type: "text_delta", contentIndex: 0, delta: "slow ok", partial: finalMessage } as const; + yield { type: "text_delta", contentIndex: 0, delta: "hello", partial: partialMessage } as const; + const { promise, reject } = Promise.withResolvers(); + if (providerSignal?.aborted) { + reject(new Error("Request was aborted")); + } + providerSignal?.addEventListener("abort", () => reject(new Error("Request was aborted")), { + once: true, + }); + await promise; }, - result: async () => finalMessage, } as unknown as AssistantMessageEventStream; setBedrockProviderModule({ - streamBedrock: () => source, + streamBedrock: (_model, _context, options) => { + providerSignal = options.signal; + return source; + }, }); - const stream = streamBedrock(createModel(), baseContext, { - streamFirstEventTimeoutMs: 10, - streamIdleTimeoutMs: 10, - }); + const stream = streamBedrock(createModel(), baseContext, { streamIdleTimeoutMs: 10 }); const result = await Promise.race([stream.result(), Bun.sleep(500).then(() => "timeout" as const)]); expect(result).not.toBe("timeout"); if (result === "timeout") { - throw new Error("Timed out waiting for forwarded slow stream result"); + throw new Error("Timed out waiting for forwarded stream stall result"); } - expect(result).toEqual(finalMessage); + expect(providerSignal?.aborted).toBe(true); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toBe("Provider stream stalled while waiting for the next event"); }); it("preserves caller aborts while forwarding lazy provider streams", async () => { diff --git a/packages/ai/test/stream-timeout-defaults.test.ts b/packages/ai/test/stream-timeout-defaults.test.ts new file mode 100644 index 000000000..35bbad1ab --- /dev/null +++ b/packages/ai/test/stream-timeout-defaults.test.ts @@ -0,0 +1,182 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { + getStreamFirstEventTimeoutMs, + getStreamIdleTimeoutMs, + iterateWithIdleTimeout, +} from "../src/utils/idle-iterator"; + +/** + * Per-provider fallback overrides on the stream-watchdog helpers. + * + * These are the gear that lets `google-gemini-cli` widen its first-event floor + * beyond the 100s global default without forcing every other provider to wait + * just as long. Tests pin the precedence contract callers depend on: + * caller option > env var > per-provider fallback > base default. + */ + +const ENV_KEYS = [ + "PI_STREAM_IDLE_TIMEOUT_MS", + "PI_OPENAI_STREAM_IDLE_TIMEOUT_MS", + "PI_STREAM_FIRST_EVENT_TIMEOUT_MS", +] as const; + +const originalEnv: Partial> = {}; + +beforeEach(() => { + for (const key of ENV_KEYS) { + originalEnv[key] = Bun.env[key]; + delete Bun.env[key]; + } +}); + +afterEach(() => { + for (const key of ENV_KEYS) { + const prior = originalEnv[key]; + if (prior === undefined) { + delete Bun.env[key]; + } else { + Bun.env[key] = prior; + } + } +}); + +describe("getStreamIdleTimeoutMs(fallbackMs)", () => { + it("returns the per-provider fallback when env vars are unset", () => { + expect(getStreamIdleTimeoutMs(300_000)).toBe(300_000); + }); + + it("lets PI_STREAM_IDLE_TIMEOUT_MS override the per-provider fallback", () => { + Bun.env.PI_STREAM_IDLE_TIMEOUT_MS = "42"; + expect(getStreamIdleTimeoutMs(300_000)).toBe(42); + }); + + it("treats PI_STREAM_IDLE_TIMEOUT_MS=0 as a watchdog disable", () => { + Bun.env.PI_STREAM_IDLE_TIMEOUT_MS = "0"; + expect(getStreamIdleTimeoutMs(300_000)).toBeUndefined(); + }); +}); + +describe("getStreamFirstEventTimeoutMs(idleTimeoutMs, fallbackMs)", () => { + it("returns the per-provider fallback when env unset and idle timeout is undefined", () => { + expect(getStreamFirstEventTimeoutMs(undefined, 300_000)).toBe(300_000); + }); + + it("floors the first-event timeout at the per-provider fallback even when idle is shorter", () => { + expect(getStreamFirstEventTimeoutMs(50_000, 300_000)).toBe(300_000); + }); + + it("never undershoots the steady-state idle timeout", () => { + expect(getStreamFirstEventTimeoutMs(500_000, 300_000)).toBe(500_000); + }); + + it("lets PI_STREAM_FIRST_EVENT_TIMEOUT_MS override the per-provider fallback", () => { + Bun.env.PI_STREAM_FIRST_EVENT_TIMEOUT_MS = "42"; + expect(getStreamFirstEventTimeoutMs(undefined, 300_000)).toBe(42); + }); + + it("treats PI_STREAM_FIRST_EVENT_TIMEOUT_MS=0 as a watchdog disable", () => { + Bun.env.PI_STREAM_FIRST_EVENT_TIMEOUT_MS = "0"; + expect(getStreamFirstEventTimeoutMs(undefined, 300_000)).toBeUndefined(); + }); + + it("falls back to the 100s global default when no fallback or env is provided", () => { + expect(getStreamFirstEventTimeoutMs()).toBe(100_000); + }); +}); + +async function expectRejectsWithMessage(run: () => Promise, message: string): Promise { + let caught: unknown; + try { + await run(); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toBe(message); +} + +describe("iterateWithIdleTimeout", () => { + it("does not reset the first-progress deadline for no-progress items", async () => { + const abortController = new AbortController(); + const abortTimer = setTimeout(() => abortController.abort(new Error("fallback abort")), 150); + abortTimer.unref(); + + async function* noProgressItems(): AsyncGenerator<{ type: "keepalive" }> { + while (true) { + await Bun.sleep(2); + yield { type: "keepalive" }; + } + } + + try { + const run = async (): Promise => { + for await (const _item of iterateWithIdleTimeout(noProgressItems(), { + firstItemTimeoutMs: 20, + idleTimeoutMs: 1_000, + errorMessage: "idle timeout", + firstItemErrorMessage: "first progress timeout", + abortSignal: abortController.signal, + isProgressItem: () => false, + })) { + // Consume until the watchdog fires. + } + }; + + await expectRejectsWithMessage(run, "first progress timeout"); + } finally { + clearTimeout(abortTimer); + } + }); + + it("cleans first-item timers when the source throws before progress", async () => { + let firstItemTimedOut = false; + + // biome-ignore lint/correctness/useYield: intentionally yields nothing — the test exercises the path where the source generator throws before its first yield. + async function* failingStream(): AsyncGenerator { + throw new Error("stream failed"); + } + + await expectRejectsWithMessage(async () => { + for await (const _item of iterateWithIdleTimeout(failingStream(), { + firstItemTimeoutMs: 10, + errorMessage: "idle timeout", + firstItemErrorMessage: "first progress timeout", + onFirstItemTimeout: () => { + firstItemTimedOut = true; + }, + })) { + // Unreachable. + } + }, "stream failed"); + + await Bun.sleep(20); + expect(firstItemTimedOut).toBe(false); + }); + + it("cleans first-item timers when the consumer returns before progress", async () => { + let firstItemTimedOut = false; + + async function* noProgressItems(): AsyncGenerator<{ type: "keepalive" }> { + while (true) { + await Bun.sleep(2); + yield { type: "keepalive" }; + } + } + + for await (const _item of iterateWithIdleTimeout(noProgressItems(), { + firstItemTimeoutMs: 10, + idleTimeoutMs: 1_000, + errorMessage: "idle timeout", + firstItemErrorMessage: "first progress timeout", + onFirstItemTimeout: () => { + firstItemTimedOut = true; + }, + isProgressItem: () => false, + })) { + break; + } + + await Bun.sleep(20); + expect(firstItemTimedOut).toBe(false); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 17669f482..df4fb68a5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,6 +1,106 @@ # Changelog ## [Unreleased] +### Added + +- Added interactive provider selection to `omp auth-broker logout` when no provider argument is supplied +- Added `--json` flag to `omp auth-broker list` for machine-readable output +- Added `omp auth-broker list` to enumerate supported OAuth providers (replaces `bunx @oh-my-pi/pi-ai list`). +- Added interactive provider selection to `omp auth-broker login` and `omp auth-broker logout` when no provider argument is supplied (replaces `bunx @oh-my-pi/pi-ai login` / `logout` interactive flows). + +### Changed + +- Changed `omp auth-broker login` to support interactive provider selection when invoked without a provider argument +- Changed `omp auth-broker logout` to support interactive provider selection from stored credentials when invoked without a provider argument +- Changed `omp auth-broker login` to drive the per-provider OAuth/API-key flow in-process via `AuthStorage.login()` instead of spawning the `pi-ai` CLI subprocess. The pi-ai bin is being removed; the same login surface now lives entirely inside `omp`. +- Changed default per-line truncation cap for search/grep output (`DEFAULT_MAX_COLUMN`) from `1024` to `512` characters. + +## [15.4.3] - 2026-05-26 + +### Fixed + +- Fixed Google Vertex cached project discovery replacing the bundled fallback catalog so `/models` does not keep showing outdated Gemini entries after authoritative Vertex discovery, while keeping the bundled fallback in place when the cached snapshot is stale or non-authoritative (e.g. after an ADC discovery failure) ([#1412](https://github.com/can1357/oh-my-pi/issues/1412)). + +## [15.4.2] - 2026-05-26 + +### Fixed + +- Fixed plan-mode subagents being unable to terminate because `yield` was registered but missing from the active tool set when `requireYieldTool` was combined with an explicit `toolNames` list ([#1408](https://github.com/can1357/oh-my-pi/issues/1408)) + +## [15.4.1] - 2026-05-26 + +### Breaking Changes +- The `vim` edit mode option is no longer available; configurations using `edit.mode: vim` will be automatically mapped to `hashline` mode +- Hashline payload semantics are now strictly inline-first: the first payload line is whatever follows the sigil on the op line itself, and subsequent lines append after it. A newline immediately after `↑`/`↓`/`:` is no longer a free separator — it produces a blank first payload line. Use `LINE↓content` for a one-line insert, `LINE↓firstline\nsecondline` for two lines; bare `LINE↓` / `LINE↑` / `LINE:` (no inline payload) still insert/replace with one blank line as before. + +### Added +- Added `irc.timeoutMs` setting to configure IRC message timeout duration with a default of 120 seconds +- Added timeout enforcement for IRC send operations to prevent indefinite hangs when recipients are unresponsive +- Added evaluator state inheritance for `task`-spawned subagents so JavaScript and Python variables are visible between a parent agent and its child sessions +- Added `hashline-per` edit mode to restore the legacy per-line hashline dialect alongside the default file-hash dialect +- Added file-hash computation and validation for hashline sections to detect stale edits +- Added file-read snapshot caching with multi-snapshot ring per path for recovery from agent's own writes +- Added delete operation (`!`) support to hashline grammar for explicit line deletion +- Added structural bracket/brace balance warnings when deleting lines with unclosed constructs +- Added file-hash computation and validation for hashline sections to detect stale edits +- Added file-read snapshot caching with multi-snapshot ring per path for recovery from agent's own writes +- Added delete operation (`!`) support to hashline grammar for explicit line deletion +- Added structural bracket/brace balance warnings when deleting lines with unclosed constructs +- Added resource metadata URL (RFC 9728) support to OAuth discovery for chaining authorization server resolution from protected-resource metadata ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) +- Added path-prefixed well-known URL fallback in OAuth discovery to support authorization servers behind gateways with sub-path routing ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) +- Added relative URL resolution for `Mcp-Auth-Server` header values against the server URL ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) + +### Changed + +- Changed Python shared eval sessions to be keyed by `sessionId` and `cwd` so code state no longer leaks across different directories when reusing a session +- Changed shared JavaScript and Python startup to deduplicate concurrent first-time session initialization so parallel first calls share one warm session +- Changed shared JavaScript and Python execution output handling so interleaved async runs keep their `display` output scoped to the originating run +- Changed Python tool bridge to use per-run identifiers alongside session IDs for correct routing of tool responses and output in concurrent evaluations +- Changed JavaScript and Python `eval` execution to allow overlapping asynchronous cells on the same session ID to run concurrently instead of being strictly queued +- Updated the edit mode option set to support `replace`, `patch`, `hashline`, and `apply_patch` variants +- Bare `A:` / `A-B:` (no payload, no inline body) now replaces the line/range with a single blank line, symmetric with bare `A↑` / `A↓` inserting a blank line; previously rejected as ambiguous +- Simplified hashline anchor format from `LINE+HASH` to bare `LINE` numbers in edit operations +- Updated hashline file headers to include 4-hex file hash: `¶PATH#HASH` format for anchored edits +- Changed hashline line separator from `|` to `:` in editable output (e.g., `42:content` instead of `42ab|content`) +- Removed per-line hash validation; file-level hash now validates entire section integrity +- Updated read/search output to emit file-hash headers (`¶PATH#HASH`) followed by numbered lines for hashline mode +- Modified hashline grammar to accept optional file hash in headers and removed hash requirements from line anchors +- Changed hashline diff preview format to use `LINE:content` instead of `LINE+HASH|content` +- Updated prompt documentation to reflect new `¶PATH#HASH` header and bare line-number syntax +- Bare `A:` / `A-B:` (no payload, no inline body) now replaces the line/range with a single blank line, symmetric with bare `A↑` / `A↓` inserting a blank line; previously rejected as ambiguous +- Simplified hashline anchor format from `LINE+HASH` to bare `LINE` numbers in edit operations +- Updated hashline file headers to include 4-hex file hash: `¶PATH#HASH` format for anchored edits +- Changed hashline line separator from `|` to `:` in editable output (e.g., `42:content` instead of `42ab|content`) +- Removed per-line hash validation; file-level hash now validates entire section integrity +- Updated read/search output to emit file-hash headers (`¶PATH#HASH`) followed by numbered lines for hashline mode +- Modified hashline grammar to accept optional file hash in headers and removed hash requirements from line anchors +- Changed hashline diff preview format to use `LINE:content` instead of `LINE+HASH|content` +- Updated prompt documentation to reflect new `¶PATH#HASH` header and bare line-number syntax +- Updated `discoverOAuthEndpoints` to accept `resourceMetadataUrl` parameter and prioritize the resource-metadata chain ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) +- Updated `parseMcpAuthServerUrl` and `extractMcpAuthServerUrl` to accept optional `serverUrl` for relative URL resolution ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) +- Updated `MCPOAuthFlow.#resolveRegistrationEndpoint` to try origin-root well-known first, then fall back to path-prefixed well-known ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) + +### Removed +- Removed the `installH2Fetch()` activation from CLI startup; HTTPS fetches now use Bun's default transport +- Removed the `vim` edit mode along with the `VimTool` module, prompt, and supporting buffer/engine/renderer stack +- Removed per-line hash anchors (2-letter bigram hashes) from hashline format +- Removed `RANGE_INTERIOR_HASH` constant; multi-line ranges no longer use `**` filler +- Removed `HashMismatch` type and hash mismatch error reporting; replaced with file-level validation +- Removed per-line hash anchors (2-letter bigram hashes) from hashline format +- Removed `RANGE_INTERIOR_HASH` constant; multi-line ranges no longer use `**` filler +- Removed `HashMismatch` type and hash mismatch error reporting; replaced with file-level validation + +### Fixed + +- Fixed missing `await` on `#tryWellKnownForRegistration` call in `#resolveRegistrationEndpoint` that caused path-prefixed well-known fallback to never actually execute, returning the unresolved Promise object instead of the registration endpoint ([1407](https://github.com/can1357/oh-my-pi/pull/1407) by [@faizhasim](https://github.com/faizhasim)) +- Fixed JavaScript module reloading to refresh local re-exports when transitive dependency files are edited +- Fixed Python tool calls in warm kernels to initialize once bridge environment variables appear after startup and to return a clear `tool bridge is unavailable` error when missing +- Fixed IRC `send` handling to preserve recipient incoming messages when auto-reply timeouts instead of dropping them +- Fixed Python session disposal to cancel all concurrent active executions in a shared kernel +- Fixed JavaScript `eval` imports to preserve module-level singletons across re-imports of unchanged local files and reload them only after edits +- Fixed concurrent Python evaluator tool calls to use per-run identifiers so tool responses and output are routed to the correct execution +- Fixed the `search` tool argument validation to accept a single string `paths` value as a one-path search. +## [15.4.0] - 2026-05-26 ### Breaking Changes @@ -8,8 +108,10 @@ ### Added +- Added resolved subagent model badge to the task widget status line showing `/` (with optional `:` suffix when thinking is set explicitly), opt-in via `task.showResolvedModelBadge` Appearance setting (default off) - Added `codex` and `gemini` to the web search provider settings so users can configure OpenAI and Gemini web search directly from provider selection - Added OpenAI (`codex`) and Gemini web search options with updated setup descriptions for `omp /login openai-codex` and Gemini OAuth login +- Added pretty-printing for wide JSON `data:` payloads in the raw provider-stream debug viewer so streamed event bodies expand across multiple `data:` lines instead of getting clipped by the per-line truncator, and updated the viewer header to read `raw provider stream (SSE + WS)` now that Codex WebSocket frames also flow through the buffer ### Changed @@ -33,11 +135,15 @@ - Fixed three correctness issues in the `find` tool. (1) `onMatch` guard checked the outer task `signal?.aborted` instead of `combinedSignal.aborted` (which also includes the per-call timeout signal), so late matches accumulated past the timeout. (2) Timeout-drained partial results were emitted in insertion order while the normal path sorts by mtime descending; callers relying on "most recently modified first" got an inconsistent ordering when the call timed out. The timeout drain now tracks per-entry mtime in a parallel array and applies the same comparator. (3) `validateFindPathInputs` now skips backslash-escaped commas (`\,`) when checking for top-level commas, matching `search.ts:containsTopLevelComma`. - Fixed `throwPreferredDapStartError` using a single `await Promise.resolve()` (one microtask) to let a concurrent launch/attach rejection settle before deciding which error to surface. That worked for synchronous-rejection test fakes but not real adapter I/O: the launch failure arrives via socket and lands several ticks after the `configurationDone` failure. `DapStartRequestFailure` now carries a `settled?: Promise` that resolves when the underlying request settles either way, and `throwPreferredDapStartError` races against it with a 50 ms ceiling. The preferred error message (the underlying launch/attach failure rather than the cascade) now surfaces under real I/O. - Fixed `adapter: "debugpy"` over-promising in the `debug` tool. `resolveAdapter("debugpy", cwd)` only checks for `python` in `PATH`; it does not verify that the `debugpy` module is importable. Both failure modes — `python` missing and `debugpy` module missing — used to collapse onto a generic `"No suitable debug adapter found"` error. The launch/attach action now throws a targeted `ToolError` naming `python` when `resolveAdapter` returns null for an explicit `adapter: "debugpy"`, and the `DapSessionManager` spawn-catch detects `"No module named debugpy"` in adapter stderr and surfaces a `pip install debugpy` hint. The Python debug tool documentation lists the install hint so the prompt and runtime diagnostics agree. +- Fixed the LSP symbol-resolver `BARE_IDENTIFIER_RE` (`/^[A-Za-z_][\w]*$/`) rejecting `$`-prefixed identifiers (`$store`, `$count`, RxJS observables, Svelte stores, Angular signals). Without the word-boundary check, searching for `$store` on a line containing `bar$store` returned the offset inside the compound identifier rather than the standalone occurrence, feeding a wrong column to the LSP server. Pattern now `/^[$A-Za-z_][\w$]*$/`; the companion `IDENTIFIER_CHAR_RE` already contained `$`. +- Fixed `applyWorkspaceEdit` writing all text edits before walking `documentChanges` for resource operations. LSP §3.16.2 requires clients to apply `documentChanges` in declared order, so any server emitting `{kind: "create", uri: X}` followed by a `TextDocumentEdit` for `X` (e.g. "Extract to new file" code actions, some rename responses) broke: the edit ran against a non-existent file, then the create happened. `applyWorkspaceEdit` now walks `documentChanges` once in declared order; per-URI text edits are coalesced into a pending Map and flushed immediately before any subsequent resource op for the same URI. Legacy `changes`-map-only payloads are unchanged. Folder-level `rename`/`delete` ops now flush every pending URI under the affected subtree (not just the exact target) so child-file edits queued before a parent-folder move land at the original location instead of dangling against a non-existent path on the final flush. Rename ops additionally flush pending edits queued against `renameOp.newUri` (and its descendants) **before** `fs.rename` runs, so edits intended for the pre-rename target file are applied before the rename clobbers or replaces it (relevant under `options.overwrite`/`options.ignoreIfExists`). When a `WorkspaceEdit` payload supplies both `changes` and `documentChanges`, the `documentChanges` arm is now used exclusively per LSP §3.16.2 ("if documentChanges are supplied … servers should use them in preference to changes"); previously the two were merged. ### Fixed - Fixed built-in `explore` agent failing every invocation with `schema_violation: files.0.ref: must not be present` on releases prior to 15.3.2 by renaming the `files[].ref` property to `files[].path` in the agent's output schema; `ref` is a JTD-reserved keyword (RFC 8927) and collides with JSON Type Definition's schema-reference form, so the converter previously dropped it from the generated JSON Schema. Defense-in-depth alongside the 15.3.2 converter fix ([#1379](https://github.com/can1357/oh-my-pi/issues/1379)). - Increased the `yield` tool's schema-validation retry budget from 1 to 3 so subagents whose first structured-output attempt mismatches the declared output schema get up to three retries before the parent's post-mortem `schema_violation` check hard-fails the task. The tool now also surfaces remaining retry attempts and an explicit "call yield again with the corrected shape" directive in each rejection message, giving the model the context it needs to converge — particularly helpful for models like GLM that tend to invent per-element field names instead of following the declared schema. +- Fixed CLI PDF file arguments being decoded as raw bytes for local vision models; `.pdf` and other supported document files now go through the same Markit conversion path as the `read` tool before entering the prompt ([#1401](https://github.com/can1357/oh-my-pi/issues/1401)). +- Fixed the `bash` tool hanging until the 305 s hard timeout when a command writes a file via heredoc on Windows (bodies > ~4 KiB) or macOS (bodies > 16-64 KiB). Root cause was in the embedded brush shell; see `@oh-my-pi/pi-natives` changelog for the underlying fix. ## [15.3.2] - 2026-05-25 ### Added diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index a2f0a5454..c4083bee0 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-coding-agent", - "version": "15.3.2", + "version": "15.4.3", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/coding-agent/src/cli.ts b/packages/coding-agent/src/cli.ts index e9461782c..701352956 100755 --- a/packages/coding-agent/src/cli.ts +++ b/packages/coding-agent/src/cli.ts @@ -1,14 +1,6 @@ #!/usr/bin/env bun -import { installH2Fetch } from "@oh-my-pi/pi-ai"; import { APP_NAME, MIN_BUN_VERSION, procmgr, VERSION } from "@oh-my-pi/pi-utils"; -// Activate HTTP/2 for all `fetch()` calls (provider streams, OAuth, model -// discovery, web tools). Bun's HTTP/2 client is gated on a startup flag we -// can't toggle from JS, so we patch globalThis.fetch to pass -// `protocol: "http2"` per request, with transparent HTTP/1.1 fallback on -// `HTTP2Unsupported`. See @oh-my-pi/pi-ai/utils/h2-fetch for details. -installH2Fetch(); - // Strip macOS malloc-stack-logging env vars before any subprocess is spawned. // Otherwise every child bun process (subagents, plugin installs, ptree spawns, // etc.) prints a `MallocStackLogging: can't turn off …` warning to stderr. diff --git a/packages/coding-agent/src/cli/auth-broker-cli.ts b/packages/coding-agent/src/cli/auth-broker-cli.ts index 651d139b9..9356b5df2 100644 --- a/packages/coding-agent/src/cli/auth-broker-cli.ts +++ b/packages/coding-agent/src/cli/auth-broker-cli.ts @@ -17,6 +17,7 @@ import * as crypto from "node:crypto"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; +import * as readline from "node:readline"; import { AuthBrokerClient, type AuthCredential, @@ -28,6 +29,7 @@ import { listProvidersWithEnvKey, type OAuthCredential, type OAuthProvider, + type OAuthProviderInfo, SqliteAuthCredentialStore, startAuthBroker, } from "@oh-my-pi/pi-ai"; @@ -36,7 +38,7 @@ import { $ } from "bun"; import chalk from "chalk"; import { resolveAuthBrokerConfig } from "../session/auth-broker-config"; -export type AuthBrokerAction = "serve" | "token" | "login" | "logout" | "status" | "import" | "migrate"; +export type AuthBrokerAction = "serve" | "token" | "login" | "logout" | "status" | "import" | "migrate" | "list"; export interface AuthBrokerCommandArgs { action: AuthBrokerAction; @@ -60,7 +62,16 @@ export interface AuthBrokerCommandArgs { }; } -const ACTIONS: readonly AuthBrokerAction[] = ["serve", "token", "login", "logout", "import", "migrate", "status"]; +const ACTIONS: readonly AuthBrokerAction[] = [ + "serve", + "token", + "login", + "logout", + "import", + "migrate", + "status", + "list", +]; /** Callback ports baked from the per-provider OAuth flow modules. */ const CALLBACK_PORTS: Record = { @@ -168,13 +179,23 @@ async function runToken(flags: AuthBrokerCommandArgs["flags"]): Promise { } async function runLogin(flags: AuthBrokerCommandArgs["flags"]): Promise { - const providerArg = flags.provider; + const providers = getOAuthProviders(); + let providerArg = flags.provider; if (!providerArg) { - throw new Error("Usage: omp auth-broker login [--via=user@host]"); + if (flags.via) { + throw new Error( + "Usage: omp auth-broker login --via=user@host (provider required for remote login)", + ); + } + providerArg = await pickProviderInteractively(providers); } - const oauthProviders = new Set(getOAuthProviders().map(p => p.id)); - if (!oauthProviders.has(providerArg)) { - throw new Error(`Unknown OAuth provider '${providerArg}'. Known: ${[...oauthProviders].sort().join(", ")}`); + if (!providers.some(p => p.id === providerArg)) { + throw new Error( + `Unknown OAuth provider '${providerArg}'. Known: ${providers + .map(p => p.id) + .sort() + .join(", ")}`, + ); } if (flags.via) { await runRemoteLogin(providerArg, flags.via, flags.dryRun ?? false); @@ -184,18 +205,107 @@ async function runLogin(flags: AuthBrokerCommandArgs["flags"]): Promise { } async function runLocalLogin(provider: OAuthProvider): Promise { - // Spawn the pi-ai CLI in-process — it handles the per-provider OAuth dance - // and persists into the same SQLite store the broker uses. - const piAiCli = Bun.fileURLToPath(import.meta.resolve("@oh-my-pi/pi-ai/cli")); - const proc = Bun.spawn({ - cmd: [process.execPath, piAiCli, "login", provider], - stdin: "inherit", - stdout: "inherit", - stderr: "inherit", + // Drive the per-provider OAuth dance in-process. Persists into the same + // SQLite store the broker uses. + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const ask = (msg: string) => promptLine(rl, `${msg} `); + const store = await SqliteAuthCredentialStore.open(getAgentDbPath()); + const storage = new AuthStorage(store); + await storage.reload(); + try { + await storage.login(provider, { + onAuth({ url, instructions }) { + process.stdout.write(`\nOpen this URL in your browser:\n${url}\n`); + if (instructions) process.stdout.write(`${instructions}\n`); + process.stdout.write("\n"); + }, + onProgress(message) { + process.stdout.write(`${message}\n`); + }, + onPrompt(p) { + return ask(`${p.message}${p.placeholder ? ` (${p.placeholder})` : ""}:`); + }, + }); + process.stdout.write(`\nCredentials saved to ${getAgentDbPath()}\n`); + } finally { + store.close(); + rl.close(); + } +} + +/** + * Interactive `readline` prompt that cleanly tears down on Ctrl-C / Escape so + * cancelling a half-finished login flow doesn't leave the terminal in raw mode. + */ +function promptLine(rl: readline.Interface, question: string): Promise { + const { promise, resolve, reject } = Promise.withResolvers(); + const input = process.stdin as NodeJS.ReadStream; + const supportsRawMode = input.isTTY && typeof input.setRawMode === "function"; + const wasRaw = supportsRawMode ? input.isRaw : false; + let settled = false; + + const cleanup = () => { + rl.off("SIGINT", onSigint); + if (supportsRawMode) { + input.off("keypress", onKeypress); + input.setRawMode?.(wasRaw); + } + }; + + const finish = (result: () => void) => { + if (settled) return; + settled = true; + cleanup(); + result(); + }; + + const cancel = () => { + finish(() => reject(new Error("Login cancelled"))); + }; + + const onSigint = () => { + cancel(); + }; + + const onKeypress = (_str: string, key: readline.Key) => { + if (key.name === "escape" || (key.ctrl && key.name === "c")) { + cancel(); + rl.close(); + } + }; + + if (supportsRawMode) { + readline.emitKeypressEvents(input, rl); + input.setRawMode(true); + input.on("keypress", onKeypress); + } + + rl.once("SIGINT", onSigint); + rl.question(question, answer => { + finish(() => resolve(answer)); }); - const exitCode = await proc.exited; - if (exitCode !== 0) { - throw new Error(`pi-ai login exited with code ${exitCode}`); + return promise; +} + +async function pickProviderInteractively(providers: readonly OAuthProviderInfo[]): Promise { + if (providers.length === 0) { + throw new Error("No OAuth providers registered"); + } + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + try { + process.stdout.write("Select a provider:\n\n"); + for (let i = 0; i < providers.length; i++) { + process.stdout.write(` ${i + 1}. ${providers[i].name}\n`); + } + process.stdout.write("\n"); + const choice = await promptLine(rl, `Enter number (1-${providers.length}): `); + const index = Number.parseInt(choice, 10) - 1; + if (Number.isNaN(index) || index < 0 || index >= providers.length) { + throw new Error(`Invalid selection: ${choice}`); + } + return providers[index].id; + } finally { + rl.close(); } } @@ -235,12 +345,17 @@ async function runRemoteLogin(provider: string, via: string, dryRun: boolean): P } async function runLogout(flags: AuthBrokerCommandArgs["flags"]): Promise { - const providerArg = flags.provider; - if (!providerArg) { - throw new Error("Usage: omp auth-broker logout "); - } + let providerArg = flags.provider; const store = await SqliteAuthCredentialStore.open(getAgentDbPath()); try { + if (!providerArg) { + const stored = store.listProviders(); + if (stored.length === 0) { + process.stdout.write("No credentials stored.\n"); + return; + } + providerArg = await pickStoredProviderInteractively(stored); + } store.deleteAuthCredentialsForProvider(providerArg, "logged out by user"); process.stdout.write(`Logged out of ${providerArg}\n`); } finally { @@ -248,6 +363,37 @@ async function runLogout(flags: AuthBrokerCommandArgs["flags"]): Promise { } } +async function pickStoredProviderInteractively(providers: string[]): Promise { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + try { + process.stdout.write("Select a provider to logout:\n\n"); + for (let i = 0; i < providers.length; i++) { + process.stdout.write(` ${i + 1}. ${providers[i]}\n`); + } + process.stdout.write("\n"); + const choice = await promptLine(rl, `Enter number (1-${providers.length}): `); + const index = Number.parseInt(choice, 10) - 1; + if (Number.isNaN(index) || index < 0 || index >= providers.length) { + throw new Error(`Invalid selection: ${choice}`); + } + return providers[index]; + } finally { + rl.close(); + } +} + +async function runList(flags: AuthBrokerCommandArgs["flags"]): Promise { + const providers = getOAuthProviders(); + if (flags.json) { + process.stdout.write(`${JSON.stringify(providers.map(p => ({ id: p.id, name: p.name })))}\n`); + return; + } + process.stdout.write("Available providers:\n\n"); + for (const p of providers) { + process.stdout.write(` ${p.id.padEnd(20)} ${p.name}\n`); + } +} + // ─── CLIProxyAPI import ───────────────────────────────────────────────── /** @@ -732,6 +878,9 @@ export async function runAuthBrokerCommand(cmd: AuthBrokerCommandArgs): Promise< case "status": await runStatus(cmd.flags); return; + case "list": + await runList(cmd.flags); + return; default: { // Exhaustive check. const _exhaustive: never = cmd.action; diff --git a/packages/coding-agent/src/cli/file-processor.ts b/packages/coding-agent/src/cli/file-processor.ts index 34d90f6af..355337f8f 100644 --- a/packages/coding-agent/src/cli/file-processor.ts +++ b/packages/coding-agent/src/cli/file-processor.ts @@ -1,5 +1,5 @@ /** - * Process @file CLI arguments into text content and image attachments + * Process @file CLI arguments into text, document content, and image attachments */ import * as fs from "node:fs"; import * as path from "node:path"; @@ -9,11 +9,13 @@ import chalk from "chalk"; import { resolveReadPath } from "../tools/path-utils"; import { formatBytes } from "../tools/render-utils"; import { formatDimensionNote, resizeImage } from "../utils/image-resize"; +import { convertFileWithMarkit } from "../utils/markit"; // Keep CLI startup responsive and avoid OOM when users pass huge files. // If a file exceeds these limits, we include it as a path-only block. const MAX_CLI_TEXT_BYTES = 5 * 1024 * 1024; // 5MB const MAX_CLI_IMAGE_BYTES = 25 * 1024 * 1024; // 25MB +const CONVERTIBLE_EXTENSIONS = new Set([".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".rtf", ".epub"]); export interface ProcessedFiles { text: string; @@ -25,7 +27,7 @@ export interface ProcessFileOptions { autoResizeImages?: boolean; } -/** Process @file arguments into text content and image attachments */ +/** Process @file arguments into text, document content, and image attachments */ export async function processFileArguments(fileArgs: string[], options?: ProcessFileOptions): Promise { const autoResizeImages = options?.autoResizeImages ?? true; let text = ""; @@ -43,6 +45,7 @@ export async function processFileArguments(fileArgs: string[], options?: Process const imageMetadata = await readImageMetadata(absolutePath); const mimeType = imageMetadata?.mimeType; + const ext = path.extname(absolutePath).toLowerCase(); const maxBytes = mimeType ? MAX_CLI_IMAGE_BYTES : MAX_CLI_TEXT_BYTES; if (stat.size > maxBytes) { console.error( @@ -106,6 +109,13 @@ export async function processFileArguments(fileArgs: string[], options?: Process } else { text += `\n`; } + } else if (CONVERTIBLE_EXTENSIONS.has(ext)) { + const result = await convertFileWithMarkit(absolutePath); + if (result.ok) { + text += `\n${result.content}\n\n`; + } else { + text += `[Cannot read ${ext} file: ${result.error || "conversion failed"}]\n`; + } } else { // Handle text file try { diff --git a/packages/coding-agent/src/commands/auth-broker.ts b/packages/coding-agent/src/commands/auth-broker.ts index c6beb1f14..85bb5906e 100644 --- a/packages/coding-agent/src/commands/auth-broker.ts +++ b/packages/coding-agent/src/commands/auth-broker.ts @@ -56,8 +56,11 @@ export default class AuthBroker extends Command { "# Boot on a non-default port\n omp auth-broker serve --bind=127.0.0.1:9000", "# Print the bearer token\n omp auth-broker token", "# Rotate the bearer token\n omp auth-broker token --regenerate", + "# List supported OAuth providers\n omp auth-broker list", "# Local login (run on the broker host)\n omp auth-broker login anthropic", + "# Interactive provider selection\n omp auth-broker login", "# Remote login over SSH tunnel\n omp auth-broker login anthropic --via=user@broker", + "# Log out of a provider (interactive without provider arg)\n omp auth-broker logout anthropic", "# Import a CLIProxyAPI auth dump\n omp auth-broker import ~/.cliproxy/auth", "# Import a single CLIProxyAPI JSON, overriding the provider mapping\n omp auth-broker import ~/.cliproxy/auth/claude-foo.json --provider anthropic", "# Preview a migration from local store + env vars to the configured broker\n omp auth-broker migrate --from-local --include-env --dry-run", diff --git a/packages/coding-agent/src/commands/commit.ts b/packages/coding-agent/src/commands/commit.ts index 428efd36f..d5abd4c38 100644 --- a/packages/coding-agent/src/commands/commit.ts +++ b/packages/coding-agent/src/commands/commit.ts @@ -32,14 +32,14 @@ export default class Commit extends Command { }; await initTheme(); - // The agentic commit flow opens HTTP/2 keep-alive sockets to the model - // provider (via `installH2Fetch`) and spins up an AgentSession with - // background async-job + extension machinery. `session.dispose()` releases - // what it can, but Bun's fetch keeps idle connections warm and a few - // timers (Settings autosave, OAuth refresh) stay armed long enough to - // pin the event loop after the commit is already written. Mirror the - // `runPrintMode` exit pattern from `main.ts` so the CLI returns to the - // shell instead of stranding the user on Ctrl+C (issue #1041). + // The agentic commit flow opens keep-alive sockets to the model provider + // and spins up an AgentSession with background async-job + extension + // machinery. `session.dispose()` releases what it can, but Bun's fetch + // keeps idle connections warm and a few timers (Settings autosave, OAuth + // refresh) stay armed long enough to pin the event loop after the commit + // is already written. Mirror the `runPrintMode` exit pattern from + // `main.ts` so the CLI returns to the shell instead of stranding the user + // on Ctrl+C (issue #1041). await runCommitCommand(cmd); await postmortem.quit(0); } diff --git a/packages/coding-agent/src/config/model-registry.ts b/packages/coding-agent/src/config/model-registry.ts index ad7bae9cf..1655f6273 100644 --- a/packages/coding-agent/src/config/model-registry.ts +++ b/packages/coding-agent/src/config/model-registry.ts @@ -291,6 +291,28 @@ export function mergeDiscoveredModel( return model; } +function isAuthoritativeProjectCatalogModel(model: Model): boolean { + return ( + model.provider === "google-vertex" && + model.api === "openai-completions" && + model.baseUrl.includes("/endpoints/openapi") + ); +} + +function providersWithAuthoritativeProjectCatalog(models: readonly Model[]): Set { + const providers = new Set(); + for (const model of models) { + if (isAuthoritativeProjectCatalogModel(model)) { + providers.add(model.provider); + } + } + return providers; +} + +function dropProviderModels(models: readonly Model[], providers: ReadonlySet): Model[] { + return models.filter(model => !providers.has(model.provider)); +} + interface DiscoveryProviderConfig { provider: string; api: Api; @@ -877,9 +899,24 @@ export class ModelRegistry { this.#equivalenceConfig = equivalence; this.#addImplicitDiscoverableProviders(configuredProviders); - const builtInModels = this.#applyHardcodedModelPolicies(this.#loadBuiltInModels(overrides)); - const cachedStandardModels = this.#applyHardcodedModelPolicies(this.#loadCachedStandardProviderModels()); + let builtInModels = this.#applyHardcodedModelPolicies(this.#loadBuiltInModels(overrides)); + const cachedStandardResult = this.#loadCachedStandardProviderModels(); + const cachedStandardModels = this.#applyHardcodedModelPolicies(cachedStandardResult.models); const cachedDiscoveries = this.#applyHardcodedModelPolicies(this.#loadCachedDiscoverableModels()); + // Only drop bundled fallback models when the cached project-catalog row is + // itself fresh AND authoritative. A stale or non-authoritative snapshot + // (e.g. after ADC discovery failure rewrote the row with authoritative=0) + // must not strip bundled Vertex Gemini entries — that would leave only the + // stale project-scoped rows in API-key-only environments. + const cachedAuthoritativeProviders = new Set(); + for (const provider of providersWithAuthoritativeProjectCatalog(cachedStandardModels)) { + if (cachedStandardResult.authoritativeFreshProviders.has(provider)) { + cachedAuthoritativeProviders.add(provider); + } + } + if (cachedAuthoritativeProviders.size > 0) { + builtInModels = dropProviderModels(builtInModels, cachedAuthoritativeProviders); + } const resolvedDefaults = this.#mergeResolvedModels( this.#mergeResolvedModels(builtInModels, cachedStandardModels), cachedDiscoveries, @@ -982,9 +1019,10 @@ export class ModelRegistry { return merged; } - #loadCachedStandardProviderModels(): Model[] { + #loadCachedStandardProviderModels(): { models: Model[]; authoritativeFreshProviders: Set } { const configuredDiscoveryProviders = new Set(this.#discoverableProviders.map(provider => provider.provider)); const cachedModels: Model[] = []; + const authoritativeFreshProviders = new Set(); for (const descriptor of PROVIDER_DESCRIPTORS) { if (configuredDiscoveryProviders.has(descriptor.providerId)) { continue; @@ -993,6 +1031,9 @@ export class ModelRegistry { if (!cache) { continue; } + if (cache.fresh && cache.authoritative) { + authoritativeFreshProviders.add(descriptor.providerId); + } const models = cache.models.map(model => model.provider === descriptor.providerId ? model : { ...model, provider: descriptor.providerId }, ); @@ -1005,7 +1046,7 @@ export class ModelRegistry { : withTransport; cachedModels.push(...this.#applyProviderModelOverrides(descriptor.providerId, withCompat)); } - return cachedModels; + return { models: cachedModels, authoritativeFreshProviders }; } #loadCachedDiscoverableModels(): Model[] { @@ -1229,7 +1270,10 @@ export class ModelRegistry { ), ), ); - const resolved = this.#mergeResolvedModels(this.#models, discoveredModels); + const authoritativeProviders = providersWithAuthoritativeProjectCatalog(discoveredModels); + const baseModels = + authoritativeProviders.size > 0 ? dropProviderModels(this.#models, authoritativeProviders) : this.#models; + const resolved = this.#mergeResolvedModels(baseModels, discoveredModels); const withConfigModels = this.#mergeCustomModels(resolved, this.#customModelOverlays); // Merge runtime extension models so they survive online discovery completion const combined = this.#mergeCustomModels(withConfigModels, this.#runtimeModelOverlays); diff --git a/packages/coding-agent/src/config/prompt-templates.ts b/packages/coding-agent/src/config/prompt-templates.ts index 5f833e787..7daa327de 100644 --- a/packages/coding-agent/src/config/prompt-templates.ts +++ b/packages/coding-agent/src/config/prompt-templates.ts @@ -8,7 +8,7 @@ import { parseFrontmatter, prompt, } from "@oh-my-pi/pi-utils"; -import { computeLineHash, HL_BODY_SEP } from "../hashline/hash"; +import { HL_LINE_BODY_SEP } from "../hashline/hash"; import { jtdToTypeScript } from "../tools/jtd-to-typescript"; import { parseCommandArgs, substituteArgs } from "../utils/command-args"; @@ -34,7 +34,7 @@ function formatHashlineRef(lineNum: unknown, content: unknown): { num: number; t const num = typeof lineNum === "number" ? lineNum : Number.parseInt(String(lineNum), 10); const raw = typeof content === "string" ? content : String(content ?? ""); const text = raw.replace(/\\t/g, "\t").replace(/\\n/g, "\n").replace(/\\r/g, "\r"); - const ref = `${num}${computeLineHash(num, text)}`; + const ref = `${num}`; return { num, text, ref }; } @@ -124,11 +124,11 @@ function resolveHashlineRef(state: HashlineHelperState, args: unknown[]): string } /** - * {{href lineNum "content"}} — compute a real hashline ref for prompt examples. + * {{href lineNum "content"}} — compute a hashline line ref for prompt examples. * {{href lineNum}} — quote the ref remembered by the earlier {{hline lineNum "..."}} * {{href}} — quote the ref from the previous {{hline}} call. * {{href "[" "]"}} — wrap the previous {{hline}} ref with pre/post chars. - * Returns `"lineNumBIGRAM"` (e.g., `"42nd"`), or `"[42nd]"` when pre/post are supplied. + * Returns `"lineNum"` (e.g., `"42"`), or `"[42]"` when pre/post are supplied. */ prompt.registerHelper("href", function (this: unknown, ...args: unknown[]): string { const { positional, options } = splitHelperArgs(args); @@ -143,7 +143,7 @@ prompt.registerHelper("hrefr", function (this: unknown, ...args: unknown[]): str /** * {{hline lineNum "content"}} — format a full read-style line with prefix. - * Returns `"lineNumBIGRAM|content"` (pipe between anchor and content). + * Returns `"lineNum:content"` (colon between line number and content). */ prompt.registerHelper("hline", function (this: unknown, ...args: unknown[]): string { const { positional, options } = splitHelperArgs(args); @@ -151,7 +151,7 @@ prompt.registerHelper("hline", function (this: unknown, ...args: unknown[]): str const { num, ref, text } = formatHashlineRef(lineNum, content); const state = getHashlineHelperState(this, options); rememberHashlineRef(state, num, ref); - return `${ref}${HL_BODY_SEP}${text}`; + return `${ref}${HL_LINE_BODY_SEP}${text}`; }); const INLINE_ARG_SHELL_PATTERN = /\$(?:ARGUMENTS|@(?:\[\d+(?::\d*)?\])?|\d+)/; diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 7472b4f8c..ece417830 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -1528,7 +1528,7 @@ export const SETTINGS_SCHEMA = { ui: { tab: "editing", label: "Edit Mode", - description: "Select the edit tool variant (replace, patch, hashline, vim, or apply_patch)", + description: "Select the edit tool variant (replace, patch, hashline, or apply_patch)", }, }, @@ -1603,7 +1603,8 @@ export const SETTINGS_SCHEMA = { ui: { tab: "editing", label: "Hash Lines", - description: "Include line hashes in read output for hashline edit mode (LINE+ID|content)", + description: + "Include file-hash headers and line numbers in read output for hashline edit mode (¶PATH#hash plus LINE:content)", }, }, @@ -1905,6 +1906,24 @@ export const SETTINGS_SCHEMA = { }, }, + "irc.timeoutMs": { + type: "number", + default: 120_000, + ui: { + tab: "tools", + label: "IRC Timeout", + description: + "Drop IRC messages whose recipient does not respond within this many milliseconds (0 disables the timeout)", + options: [ + { value: "0", label: "Disabled" }, + { value: "30000", label: "30 seconds" }, + { value: "60000", label: "1 minute" }, + { value: "120000", label: "2 minutes" }, + { value: "300000", label: "5 minutes" }, + ], + }, + }, + // Optional tools "renderMermaid.enabled": { @@ -2458,6 +2477,16 @@ export const SETTINGS_SCHEMA = { }, }, + "task.showResolvedModelBadge": { + type: "boolean", + default: false, + ui: { + tab: "appearance", + label: "Show Resolved Model Badge", + description: "Display the actual model ID used by each subagent in the task widget status line", + }, + }, + // Skills "skills.enabled": { type: "boolean", default: true }, diff --git a/packages/coding-agent/src/config/settings.ts b/packages/coding-agent/src/config/settings.ts index 85142fcd5..6a14bb026 100644 --- a/packages/coding-agent/src/config/settings.ts +++ b/packages/coding-agent/src/config/settings.ts @@ -408,7 +408,7 @@ export class Settings { /** * Get the edit variant for a specific model. - * Returns "patch", "replace", "hashline", "vim", "apply_patch", or null (use global default). + * Returns "patch", "replace", "hashline", "apply_patch", or null (use global default). */ getEditVariantForModel(model: string | undefined): EditMode | null { if (!model) return null; @@ -643,22 +643,22 @@ export class Settings { } } - // edit.mode: removed "atom" variant is now "hashline" + // edit.mode: removed "atom" and "vim" variants map back to "hashline" const editObj = raw.edit as Record | undefined; if (editObj) { - if (editObj.mode === "atom") { + if (editObj.mode === "atom" || editObj.mode === "vim") { editObj.mode = "hashline"; } const modelVariants = editObj.modelVariants as Record | undefined; if (modelVariants && typeof modelVariants === "object" && !Array.isArray(modelVariants)) { for (const [pattern, variant] of Object.entries(modelVariants)) { - if (variant === "atom") { + if (variant === "atom" || variant === "vim") { modelVariants[pattern] = "hashline"; } } } } - if (raw["edit.mode"] === "atom") { + if (raw["edit.mode"] === "atom" || raw["edit.mode"] === "vim") { raw["edit.mode"] = "hashline"; } diff --git a/packages/coding-agent/src/debug/raw-sse.ts b/packages/coding-agent/src/debug/raw-sse.ts index c3d812264..a369fb246 100644 --- a/packages/coding-agent/src/debug/raw-sse.ts +++ b/packages/coding-agent/src/debug/raw-sse.ts @@ -2,15 +2,56 @@ import { type Component, matchesKey, padding, replaceTabs, truncateToWidth, visi import { sanitizeText } from "@oh-my-pi/pi-utils"; import { theme } from "../modes/theme/theme"; import { copyToClipboard } from "../utils/clipboard"; -import { formatRawSseIsoTime, type RawSseDebugBuffer, rawSseRecordLines } from "./raw-sse-buffer"; +import { + formatRawSseIsoTime, + type RawSseDebugBuffer, + type RawSseDebugRecord, + rawSseRecordLines, +} from "./raw-sse-buffer"; const MIN_VIEWER_WIDTH = 20; const VIEWER_FRAME_LINES = 5; +// `data:` lines below this width render fine on a single row; anything wider gets pretty-printed +// across multiple `data:` lines so streamed JSON blobs stop getting clipped by `truncateToWidth`. +const PRETTY_PRINT_DATA_THRESHOLD = 100; function sanitizeFrameLine(line: string, width: number): string { return truncateToWidth(replaceTabs(sanitizeText(line)), width); } +// Walks the SSE wire lines and replaces single-line `data: ` payloads with +// multi-line `data: ` entries when the JSON is wide enough to clip. +// Multi-line `data:` is still valid SSE (the spec joins lines with `\n`), so the +// transformed view round-trips back to the same event when copied. +/** @internal Exported for tests. */ +export function expandPrettyDataLines(raw: readonly string[]): string[] { + const out: string[] = []; + for (const line of raw) { + if (!line.startsWith("data: ") || line.length <= PRETTY_PRINT_DATA_THRESHOLD) { + out.push(line); + continue; + } + const body = line.slice("data: ".length); + const trimmed = body.trim(); + if (trimmed.length === 0 || (trimmed[0] !== "{" && trimmed[0] !== "[")) { + out.push(line); + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + out.push(line); + continue; + } + const pretty = JSON.stringify(parsed, null, 2); + for (const prettyLine of pretty.split("\n")) { + out.push(`data: ${prettyLine}`); + } + } + return out; +} + export interface RawSseViewerOptions { buffer: RawSseDebugBuffer; terminalRows: number; @@ -30,6 +71,12 @@ export class RawSseViewerComponent implements Component { #followTail = true; #lastRenderWidth = MIN_VIEWER_WIDTH; #statusMessage: string | undefined; + // Pretty-printed wire lines keyed by `record.sequence`. Pretty-printing is + // the JSON.parse + JSON.stringify per `data:` line, so we cache the result — + // the render path runs on every keypress and from `#maxScrollOffset()`. + // Sequences are monotonic; we prune entries below the oldest live record + // after each render so the cache tracks the buffer's eviction window. + readonly #prettyLinesCache = new Map(); constructor(options: RawSseViewerOptions) { this.#buffer = options.buffer; @@ -131,8 +178,9 @@ export class RawSseViewerComponent implements Component { ); lines.push(""); } + const firstSequence = snapshot.records[0]?.sequence; for (const record of snapshot.records) { - for (const line of rawSseRecordLines(record)) { + for (const line of this.#prettyLinesFor(record)) { lines.push(sanitizeFrameLine(line, innerWidth)); } if (record.kind === "event" && record.truncated) { @@ -140,14 +188,31 @@ export class RawSseViewerComponent implements Component { } lines.push(""); } + if (firstSequence !== undefined) this.#pruneCache(firstSequence); return lines; } + #prettyLinesFor(record: RawSseDebugRecord): string[] { + const cached = this.#prettyLinesCache.get(record.sequence); + if (cached) return cached; + const expanded = expandPrettyDataLines(rawSseRecordLines(record)); + this.#prettyLinesCache.set(record.sequence, expanded); + return expanded; + } + + #pruneCache(firstSequence: number): void { + // Bounded by the buffer eviction rate; with `MAX_RAW_SSE_EVENTS = 1000` + // this rarely runs and only walks freshly-evicted entries. + for (const key of this.#prettyLinesCache.keys()) { + if (key < firstSequence) this.#prettyLinesCache.delete(key); + } + } + #summaryText(): string { const snapshot = this.#buffer.snapshot(); const last = snapshot.lastUpdatedAt ? ` last=${formatRawSseIsoTime(snapshot.lastUpdatedAt)}` : ""; const follow = this.#followTail ? "follow:on" : "follow:off"; - return ` # raw SSE | events=${snapshot.totalEvents} records=${snapshot.records.length}${last} | ${follow} | Esc back Ctrl+C copy End follow`; + return ` # raw provider stream (SSE + WS) | events=${snapshot.totalEvents} records=${snapshot.records.length}${last} | ${follow} | Esc back Ctrl+C copy End follow`; } #statusText(): string { diff --git a/packages/coding-agent/src/edit/file-read-cache.ts b/packages/coding-agent/src/edit/file-read-cache.ts index 33065e52a..54a2f1897 100644 --- a/packages/coding-agent/src/edit/file-read-cache.ts +++ b/packages/coding-agent/src/edit/file-read-cache.ts @@ -11,68 +11,101 @@ * Scoped per `ToolSession`: the cache lives on the session object itself, so * different sessions never share snapshots and entries get reclaimed when * the session goes out of scope. Each session keeps a small LRU window of - * paths; the cache always reflects what *this* session most recently saw, - * so it stays correct by construction even when this session writes the - * file itself — the next read after the write refreshes the entry. + * paths; each path keeps a short ring of recent snapshots so follow-up edits + * can recover from the agent's own prior writes as well as stale reads. */ import { LRUCache } from "lru-cache/raw"; import type { ToolSession } from "../tools"; const MAX_PATHS_PER_SESSION = 30; +const MAX_SNAPSHOTS_PER_PATH = 4; export interface FileReadSnapshot { /** 1-indexed line number → exact line content as observed by `read`/`search`. */ lines: Map; + /** Full normalized text when the read path observed the whole file. */ + fullText?: string; + /** 4-hex hash of `fullText`, or a sparse snapshot hash supplied by search. */ + fileHash?: string; recordedAt: number; } +interface FileReadSnapshotMetadata { + fullText?: string; + fileHash?: string; +} + export class FileReadCache { - #snapshots = new LRUCache({ max: MAX_PATHS_PER_SESSION }); + #snapshots = new LRUCache({ max: MAX_PATHS_PER_SESSION }); /** Look up the most recent snapshot for `absPath`, or `null` if absent. */ get(absPath: string): FileReadSnapshot | null { - return this.#snapshots.get(absPath) ?? null; + return this.#snapshots.get(absPath)?.[0] ?? null; + } + + /** Look up the most recent snapshot for `absPath` whose file hash matches. */ + getByHash(absPath: string, fileHash: string): FileReadSnapshot | null { + const history = this.#snapshots.get(absPath); + return history?.find(snapshot => snapshot.fileHash === fileHash) ?? null; } /** Record a contiguous run of lines (e.g. from a `read` tool). `startLine` is 1-indexed. */ - recordContiguous(absPath: string, startLine: number, lines: readonly string[]): void { - if (lines.length === 0) return; + recordContiguous( + absPath: string, + startLine: number, + lines: readonly string[], + metadata: FileReadSnapshotMetadata = {}, + ): void { + if (lines.length === 0 && metadata.fullText === undefined) return; const entries: Array = lines.map((line, idx) => [startLine + idx, line] as const); - this.#record(absPath, entries); + this.#record(absPath, entries, metadata); } /** Record sparse `(lineNumber, content)` pairs (e.g. `search` matches plus context). */ - recordSparse(absPath: string, entries: Iterable): void { + recordSparse( + absPath: string, + entries: Iterable, + metadata: FileReadSnapshotMetadata = {}, + ): void { const arr = Array.from(entries); - if (arr.length === 0) return; - this.#record(absPath, arr); + if (arr.length === 0 && metadata.fullText === undefined) return; + this.#record(absPath, arr, metadata); } - /** Drop the snapshot for a single path. */ + /** Drop the snapshot history for a single path. */ invalidate(absPath: string): void { this.#snapshots.delete(absPath); } - /** Drop every snapshot. */ + /** Drop every snapshot history. */ clear(): void { this.#snapshots.clear(); } - #record(absPath: string, entries: ReadonlyArray): void { - const existing = this.#snapshots.get(absPath); - if (existing && hasConflict(existing.lines, entries)) { - // File content has changed since we last recorded. Drop the stale - // snapshot and start fresh with whatever we just observed. - this.#snapshots.set(absPath, { lines: new Map(entries), recordedAt: Date.now() }); - return; - } - if (existing) { - for (const [lineNum, content] of entries) existing.lines.set(lineNum, content); - existing.recordedAt = Date.now(); + #record( + absPath: string, + entries: ReadonlyArray, + metadata: FileReadSnapshotMetadata, + ): void { + const history = this.#snapshots.get(absPath) ?? []; + const head = history[0]; + const now = Date.now(); + if (head && !hasConflict(head.lines, entries) && !hasHashConflict(head, metadata)) { + for (const [lineNum, content] of entries) head.lines.set(lineNum, content); + if (metadata.fullText !== undefined) head.fullText = metadata.fullText; + if (metadata.fileHash !== undefined) head.fileHash = metadata.fileHash; + head.recordedAt = now; // `get` above already touched LRU recency for this key. return; } - this.#snapshots.set(absPath, { lines: new Map(entries), recordedAt: Date.now() }); + + const nextSnapshot: FileReadSnapshot = { + lines: new Map(entries), + ...metadata, + recordedAt: now, + }; + const dedupedHistory = history.filter(snapshot => !isSameSnapshotIdentity(snapshot, nextSnapshot)); + this.#snapshots.set(absPath, [nextSnapshot, ...dedupedHistory].slice(0, MAX_SNAPSHOTS_PER_PATH)); } } @@ -84,6 +117,16 @@ function hasConflict(existing: Map, incoming: ReadonlyArray; -type EditParams = ReplaceParams | PatchParams | HashlineParams | VimParams | ApplyPatchParams; -type EditToolResultDetails = EditToolDetails | VimToolDetails; +type EditParams = ReplaceParams | PatchParams | HashlineParams | ApplyPatchParams; type EditModeDefinition = { description: (session: ToolSession) => string; @@ -65,8 +59,8 @@ type EditModeDefinition = { params: EditParams, signal: AbortSignal | undefined, batchRequest: LspBatchRequest | undefined, - onUpdate?: (partialResult: AgentToolResult) => void, - ) => Promise>; + onUpdate?: (partialResult: AgentToolResult) => void, + ) => Promise>; }; function resolveConfiguredEditMode(rawEditMode: string): EditMode | undefined { @@ -284,7 +278,6 @@ export class EditTool implements AgentTool { readonly #fuzzyThreshold: number; readonly #writethrough: WritethroughCallback; readonly #editMode?: EditMode; - readonly #vimTool: VimTool; readonly #pendingDeferredFetches = new Map(); constructor(private readonly session: ToolSession) { @@ -298,7 +291,6 @@ export class EditTool implements AgentTool { this.#allowFuzzy = resolveAllowFuzzy(session, editFuzzy); this.#fuzzyThreshold = resolveFuzzyThreshold(session, editFuzzyThreshold); this.#writethrough = createEditWritethrough(session); - this.#vimTool = new VimTool(session); } get mode(): EditMode { @@ -341,9 +333,9 @@ export class EditTool implements AgentTool { _toolCallId: string, params: EditParams, signal?: AbortSignal, - onUpdate?: AgentToolUpdateCallback, + onUpdate?: AgentToolUpdateCallback, context?: AgentToolContext, - ): Promise> { + ): Promise> { const modeDefinition = this.#getModeDefinition(); return modeDefinition.execute(this, params, signal, getLspBatchRequest(context?.toolCall), onUpdate); } @@ -460,29 +452,6 @@ export class EditTool implements AgentTool { return executeSinglePathEntries(path, runs, batchRequest, onUpdate); }, }, - vim: { - description: () => this.#vimTool.description, - parameters: vimSchema, - execute: async ( - tool: EditTool, - params: EditParams, - signal: AbortSignal | undefined, - _batchRequest: LspBatchRequest | undefined, - onUpdate?: (partialResult: AgentToolResult) => void, - ) => { - const handleUpdate = onUpdate - ? (partialResult: AgentToolResult) => { - onUpdate(partialResult as AgentToolResult); - } - : undefined; - return (await tool.#vimTool.execute( - "edit", - params as VimParams, - signal, - handleUpdate, - )) as AgentToolResult; - }, - }, }[this.mode]; } diff --git a/packages/coding-agent/src/edit/renderer.ts b/packages/coding-agent/src/edit/renderer.ts index 5c0020575..f28c4cdeb 100644 --- a/packages/coding-agent/src/edit/renderer.ts +++ b/packages/coding-agent/src/edit/renderer.ts @@ -25,10 +25,8 @@ import { shortenPath, truncateDiffByHunk, } from "../tools/render-utils"; -import { type VimRenderArgs, vimToolRenderer } from "../tools/vim"; import { fileHyperlink, Hasher, type RenderCache, renderStatusLine, truncateToWidth } from "../tui"; import type { EditMode } from "../utils/edit-mode"; -import type { VimToolDetails } from "../vim/types"; import type { DiffError, DiffResult } from "./diff"; import { type ApplyPatchEntry, expandApplyPatchToEntries, expandApplyPatchToPreviewEntries } from "./modes/apply-patch"; import type { Operation } from "./modes/patch"; @@ -127,31 +125,6 @@ interface ApplyPatchRenderSummary { error?: string; } -function isVimRenderArgs(args: EditRenderArgs | VimRenderArgs): args is VimRenderArgs { - return ( - typeof args === "object" && - args !== null && - typeof (args as { file?: unknown }).file === "string" && - !("path" in args) && - !("file_path" in args) && - !("edits" in args) - ); -} - -function isVimToolDetails(details: unknown): details is VimToolDetails { - if (!details || typeof details !== "object" || Array.isArray(details)) { - return false; - } - const cursor = (details as { cursor?: unknown }).cursor; - const viewportLines = (details as { viewportLines?: unknown }).viewportLines; - return ( - typeof (details as { file?: unknown }).file === "string" && - typeof cursor === "object" && - cursor !== null && - Array.isArray(viewportLines) - ); -} - /** Extended context for edit tool rendering */ export interface EditRenderContext { /** Edit mode resolved by the caller; lets the renderer dispatch without shape-sniffing */ @@ -332,13 +305,15 @@ const MISSING_APPLY_PATCH_END_ERROR = "The last line of the patch must be '*** E function normalizeHashlineInputPreviewPath(rawPath: string): string { const trimmed = rawPath.trim(); - if (trimmed.length < 2) return trimmed; - const first = trimmed[0]; - const last = trimmed[trimmed.length - 1]; + const hashStart = /#[0-9a-f]{4}$/u.exec(trimmed)?.index; + const withoutHash = hashStart === undefined ? trimmed : trimmed.slice(0, hashStart); + if (withoutHash.length < 2) return withoutHash; + const first = withoutHash[0]; + const last = withoutHash[withoutHash.length - 1]; if ((first === '"' || first === "'") && first === last) { - return trimmed.slice(1, -1); + return withoutHash.slice(1, -1); } - return trimmed; + return withoutHash; } function parseHashlineInputPreviewHeader(line: string): string | null { @@ -460,16 +435,11 @@ export const editToolRenderer = { mergeCallAndResult: true, renderCall( - args: EditRenderArgs | VimRenderArgs, + args: EditRenderArgs, options: RenderResultOptions & { renderContext?: EditRenderContext }, uiTheme: Theme, ): Component { const renderContext = options.renderContext; - // Dispatch on the explicit editMode when available; fall back to the - // shape probe for legacy call sites that don't thread renderContext. - if (renderContext?.editMode === "vim" || isVimRenderArgs(args)) { - return vimToolRenderer.renderCall(args as VimRenderArgs, options, uiTheme); - } const editArgs = args as EditRenderArgs; const hashlineInputSummary = getHashlineInputRenderSummary(editArgs, renderContext?.editMode); @@ -514,14 +484,6 @@ export const editToolRenderer = { uiTheme: Theme, args?: EditRenderArgs, ): Component { - if (options.renderContext?.editMode === "vim" || isVimToolDetails(result.details)) { - return vimToolRenderer.renderResult( - result as { content: Array<{ type: string; text?: string }>; details?: VimToolDetails; isError?: boolean }, - options, - uiTheme, - ); - } - const perFileResults = result.details?.perFileResults; const totalFiles = args?.edits ? countEditFiles(args.edits) : 0; if (perFileResults && (perFileResults.length > 1 || totalFiles > 1)) { diff --git a/packages/coding-agent/src/edit/streaming.ts b/packages/coding-agent/src/edit/streaming.ts index 76463f658..a0ad805c0 100644 --- a/packages/coding-agent/src/edit/streaming.ts +++ b/packages/coding-agent/src/edit/streaming.ts @@ -22,8 +22,7 @@ import { containsRecognizableHashlineOperations, END_PATCH_MARKER, type HashlineInputSection, - HL_FILE_PREFIX, - isHashlineOpLineText, + HashlineTokenizer, splitHashlineInputs, } from "../hashline"; import type { Theme } from "../modes/theme/theme"; @@ -78,39 +77,25 @@ export interface EditStreamingStrategy { const STREAMING_FALLBACK_LINES = 12; const STREAMING_FALLBACK_WIDTH = 80; -function isHashlineHeaderLine(line: string): boolean { - return line.trimEnd().startsWith(HL_FILE_PREFIX); -} - -function parseHashlineHeaderPath(line: string): string { - const trimmed = line.trimEnd(); - let prefixEnd = 0; - while (prefixEnd < trimmed.length && trimmed[prefixEnd] === HL_FILE_PREFIX) prefixEnd++; - return trimmed.slice(prefixEnd).trim(); -} - -function isHashlineOpLine(line: string): boolean { - return isHashlineOpLineText(line); -} - -function isHashlineEnvelopeMarkerLine(line: string): boolean { - const trimmed = line.trimEnd(); - return trimmed === BEGIN_PATCH_MARKER || trimmed === END_PATCH_MARKER || trimmed === ABORT_MARKER; -} +// Streaming-preview classification reuses one tokenizer instance for the +// stateless predicates and `tokenize`/`tokenizeAll` helpers; instances are +// cheap, but keeping a single module-level reference matches the rest of +// the hashline package. +const HASHLINE_TOKENIZER = new HashlineTokenizer(); function trimHashlineStreamingSyntax(lines: string[]): string[] { let index = lines.findIndex(line => line.trim().length > 0); if (index === -1) return []; - if (lines[index].trimEnd() === BEGIN_PATCH_MARKER) { + if (HASHLINE_TOKENIZER.tokenize(lines[index]).kind === "envelope-begin") { index++; while (index < lines.length && lines[index].trim().length === 0) index++; } - if (index < lines.length && isHashlineHeaderLine(lines[index])) { + if (index < lines.length && HASHLINE_TOKENIZER.tokenize(lines[index]).kind === "header") { index++; } - return lines.slice(index).filter(line => !isHashlineEnvelopeMarkerLine(line)); + return lines.slice(index).filter(line => !HASHLINE_TOKENIZER.isEnvelopeMarker(line)); } function renderHashlineInputFallback(input: string, uiTheme: Theme): string { @@ -380,32 +365,61 @@ function buildHashlineNaturalOrderPreviews( input: string, defaultPath: string | undefined, ): PerFileDiffPreview[] | null { - const lines = input.split("\n"); const groups = new Map(); let currentPath = defaultPath ?? ""; - const ensure = (path: string): string[] => { - let bucket = groups.get(path); + const ensure = (sectionPath: string): string[] => { + let bucket = groups.get(sectionPath); if (!bucket) { bucket = []; - groups.set(path, bucket); + groups.set(sectionPath, bucket); } return bucket; }; - for (const raw of lines) { - if (isHashlineEnvelopeMarkerLine(raw)) continue; - if (isHashlineHeaderLine(raw)) { - currentPath = parseHashlineHeaderPath(raw); - if (currentPath) ensure(currentPath); - continue; + + // Per-call instance: the streaming preview re-runs each tick with the + // cumulative input, and we need the line counter to start at 1. A + // dedicated tokenizer keeps the shared HASHLINE_TOKENIZER above free + // for stateless predicate use elsewhere in this module. + const streamer = new HashlineTokenizer(); + for (const token of streamer.tokenizeAll(input)) { + switch (token.kind) { + case "envelope-begin": + case "envelope-end": + case "abort": + case "op-delete": + continue; + case "header": + currentPath = token.path; + if (currentPath) ensure(currentPath); + continue; + case "op-insert": + case "op-replace": + // Inline body on the op line itself (`N↓payload`, `A-B:payload`) is + // payload content that just happens to share a line with the op + // header — render it the same as a standalone payload token so + // the very first character the model types after the sigil shows + // up in the streaming preview. Without this, the preview is + // empty until a newline arrives, and the renderer falls back to + // raw input ("A-B: bla bla bla") instead of "+ bla bla bla". + if (!currentPath || token.inlineBody === undefined) continue; + ensure(currentPath).push(`+${token.inlineBody}`); + continue; + case "blank": + if (!currentPath) continue; + ensure(currentPath).push("+"); + continue; + case "payload": + if (!currentPath) continue; + ensure(currentPath).push(`+${token.text}`); + continue; } - if (isHashlineOpLine(raw) || !currentPath) continue; - ensure(currentPath).push(`+${raw}`); } + if (groups.size === 0) return null; const previews: PerFileDiffPreview[] = []; - for (const [path, body] of groups) { + for (const [sectionPath, body] of groups) { if (body.length === 0) continue; - previews.push({ path, diff: body.join("\n") }); + previews.push({ path: sectionPath, diff: body.join("\n") }); } return previews.length > 0 ? previews : null; } @@ -523,27 +537,11 @@ const applyPatchStrategy: EditStreamingStrategy = { return ""; }, }; - -// Vim streaming preview is handled by the existing vimToolRenderer inside -// edit/renderer.ts. The strategy here is a no-op so the registry is total. -const vimStrategy: EditStreamingStrategy = { - extractCompleteEdits(args) { - return args; - }, - async computeDiffPreview() { - return null; - }, - renderStreamingFallback() { - return ""; - }, -}; - export const EDIT_MODE_STRATEGIES: Record> = { replace: replaceStrategy as EditStreamingStrategy, patch: patchStrategy as EditStreamingStrategy, hashline: hashlineStrategy as EditStreamingStrategy, apply_patch: applyPatchStrategy as EditStreamingStrategy, - vim: vimStrategy, }; export { resolveEditMode }; diff --git a/packages/coding-agent/src/eval/__tests__/shared-executors.test.ts b/packages/coding-agent/src/eval/__tests__/shared-executors.test.ts new file mode 100644 index 000000000..009fbe97f --- /dev/null +++ b/packages/coding-agent/src/eval/__tests__/shared-executors.test.ts @@ -0,0 +1,520 @@ +import { afterAll, afterEach, describe, expect, it, vi } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import type { AssistantMessage } from "@oh-my-pi/pi-ai"; +import { TempDir } from "@oh-my-pi/pi-utils"; +import type { ModelRegistry } from "../../config/model-registry"; +import { Settings } from "../../config/settings"; +import type { LoadExtensionsResult } from "../../extensibility/extensions/types"; +import type { CreateAgentSessionOptions, CreateAgentSessionResult } from "../../sdk"; +import * as sdkModule from "../../sdk"; +import type { AgentSession, AgentSessionEvent, PromptOptions } from "../../session/agent-session"; +import { TaskTool } from "../../task"; +import * as discoveryModule from "../../task/discovery"; +import type { AgentDefinition, TaskParams } from "../../task/types"; +import type { ToolSession } from "../../tools"; +import { EventBus } from "../../utils/event-bus"; +import { disposeAllVmContexts } from "../js/context-manager"; +import { executeJs } from "../js/executor"; +import { disposeAllKernelSessions, executePython } from "../py/executor"; + +function createToolSession(cwd: string, sessionFile: string | null, evalSessionId?: string): ToolSession { + const modelRegistry = { + authStorage: undefined, + refresh: async () => {}, + getAvailable: () => [], + getApiKey: async () => null, + } as unknown as ModelRegistry; + return { + cwd, + hasUI: false, + settings: Settings.isolated({ + "async.enabled": false, + "task.isolation.mode": "none", + }), + getSessionFile: () => sessionFile, + getSessionSpawns: () => "*", + getEvalSessionId: evalSessionId ? () => evalSessionId : undefined, + modelRegistry, + } as unknown as ToolSession; +} + +function createBridgeToolSession(resultText: string, calls: unknown[]): ToolSession { + const readTool = { + name: "read", + label: "read", + description: "read", + parameters: { type: "object" }, + async execute(_id: string, args: unknown) { + calls.push(args); + return { content: [{ type: "text" as const, text: resultText }] }; + }, + }; + const tools = new Map([["read", readTool]]); + return { getToolByName: (name: string) => tools.get(name) } as unknown as ToolSession; +} + +function assistantStopMessage(text: string): AssistantMessage { + return { + role: "assistant", + content: text ? [{ type: "text", text }] : [], + api: "openai-responses", + provider: "openai", + model: "mock", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; +} + +function createYieldingSubagentSession(onPrompt: () => Promise): AgentSession { + const listeners: Array<(event: AgentSessionEvent) => void> = []; + const state = { messages: [] as AssistantMessage[] }; + const emit = (event: AgentSessionEvent) => { + for (const listener of listeners) listener(event); + }; + return { + state, + agent: { state: { systemPrompt: ["test"] } }, + model: undefined, + extensionRunner: undefined, + sessionManager: { + appendSessionInit: () => {}, + }, + getActiveToolNames: () => ["eval", "yield"], + setActiveToolsByName: async () => {}, + subscribe: (listener: (event: AgentSessionEvent) => void) => { + listeners.push(listener); + return () => { + const index = listeners.indexOf(listener); + if (index >= 0) listeners.splice(index, 1); + }; + }, + prompt: async (_text: string, _options?: PromptOptions) => { + await onPrompt(); + state.messages.push(assistantStopMessage("done")); + emit({ + type: "tool_execution_end", + toolCallId: "yield-call", + toolName: "yield", + result: { + content: [{ type: "text", text: "Result submitted." }], + details: { status: "success", data: { ok: true } }, + }, + isError: false, + }); + }, + waitForIdle: async () => {}, + getLastAssistantMessage: () => state.messages[state.messages.length - 1], + abort: async () => {}, + dispose: async () => {}, + } as unknown as AgentSession; +} + +const taskAgent: AgentDefinition = { + name: "task", + description: "Task agent", + systemPrompt: "Read eval state and yield.", + source: "bundled", + tools: ["eval", "yield"], +}; + +const taskParams: TaskParams = { + agent: "task", + tasks: [{ id: "ReadEval", description: "Read eval state", assignment: "Read parent eval state." }], +}; + +describe("shared eval executors", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + afterAll(async () => { + await disposeAllVmContexts(); + await disposeAllKernelSessions(); + }); + + it("shares JavaScript state across executeJs calls with one session id", async () => { + using tempDir = TempDir.createSync("@omp-eval-js-shared-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `js-shared:${crypto.randomUUID()}`; + const session = createToolSession(tempDir.path(), sessionFile); + + await executeJs("globalThis.x = 41;", { sessionId, session, sessionFile }); + const result = await executeJs("return globalThis.x + 1;", { sessionId, session, sessionFile }); + + expect(result.exitCode).toBe(0); + expect(result.output.trim()).toBe("42"); + }); + + it("shares Python state across executePython calls with one session id", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-shared-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-shared:${crypto.randomUUID()}`; + + await executePython("x = 41", { cwd: tempDir.path(), sessionId, sessionFile }); + const result = await executePython("print(x + 1)", { cwd: tempDir.path(), sessionId, sessionFile }); + + expect(result.exitCode).toBe(0); + expect(result.output.trim()).toBe("42"); + }); + + it("deduplicates concurrent first JavaScript session acquisition", async () => { + using tempDir = TempDir.createSync("@omp-eval-js-cold-start-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `js-cold-start:${crypto.randomUUID()}`; + const session = createToolSession(tempDir.path(), sessionFile); + + const [first, second] = await Promise.all([ + executeJs( + "globalThis.sharedMarker ??= crypto.randomUUID(); await Bun.sleep(50); return globalThis.sharedMarker;", + { + sessionId, + session, + sessionFile, + }, + ), + executeJs("globalThis.sharedMarker ??= crypto.randomUUID(); return globalThis.sharedMarker;", { + sessionId, + session, + sessionFile, + }), + ]); + const third = await executeJs("return globalThis.sharedMarker;", { sessionId, session, sessionFile }); + + expect(first.exitCode).toBe(0); + expect(second.exitCode).toBe(0); + expect(third.exitCode).toBe(0); + expect(first.output.trim()).toBe(second.output.trim()); + expect(third.output.trim()).toBe(first.output.trim()); + }); + + it("deduplicates concurrent first Python session acquisition", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-cold-start-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-cold-start:${crypto.randomUUID()}`; + + const [first, second] = await Promise.all([ + executePython( + `import asyncio, uuid +shared_marker = globals().get("shared_marker") or str(uuid.uuid4()) +globals()["shared_marker"] = shared_marker +await asyncio.sleep(0.05) +print(shared_marker)`, + { cwd: tempDir.path(), sessionId, sessionFile }, + ), + executePython( + `import uuid +shared_marker = globals().get("shared_marker") or str(uuid.uuid4()) +globals()["shared_marker"] = shared_marker +print(shared_marker)`, + { cwd: tempDir.path(), sessionId, sessionFile }, + ), + ]); + const third = await executePython("print(shared_marker)", { cwd: tempDir.path(), sessionId, sessionFile }); + + expect(first.exitCode).toBe(0); + expect(second.exitCode).toBe(0); + expect(third.exitCode).toBe(0); + expect(first.output.trim()).toBe(second.output.trim()); + expect(third.output.trim()).toBe(first.output.trim()); + }); + + it("splits retained Python kernels by cwd for one shared session id", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-cwd-"); + const dirA = path.join(tempDir.path(), "a"); + const dirB = path.join(tempDir.path(), "b"); + await fs.mkdir(dirA); + await fs.mkdir(dirB); + const realDirA = await fs.realpath(dirA); + const realDirB = await fs.realpath(dirB); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-cwd:${crypto.randomUUID()}`; + + const first = await executePython( + `import os +token = "from-a" +print(os.getcwd())`, + { + cwd: dirA, + sessionId, + sessionFile, + }, + ); + const second = await executePython( + `import os +print(os.getcwd()) +print("token" in globals())`, + { + cwd: dirB, + sessionId, + sessionFile, + }, + ); + const third = await executePython("print(token)", { cwd: dirA, sessionId, sessionFile }); + + expect(first.exitCode).toBe(0); + expect(first.output.trim()).toBe(realDirA); + expect(second.exitCode).toBe(0); + expect(second.output.trim().split("\n")).toEqual([realDirB, "False"]); + expect(third.exitCode).toBe(0); + expect(third.output.trim()).toBe("from-a"); + }); + + it("interrupts timed out synchronous Python cells before they mutate shared state", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-sync-timeout-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-sync-timeout:${crypto.randomUUID()}`; + + const timedOut = await executePython("import time\ntime.sleep(0.2)\nleaked_after_timeout = True", { + cwd: tempDir.path(), + sessionId, + sessionFile, + timeoutMs: 20, + }); + await Bun.sleep(250); + const probe = await executePython('print("leaked_after_timeout" in globals())', { + cwd: tempDir.path(), + sessionId, + sessionFile, + }); + + expect(timedOut.cancelled).toBe(true); + expect(probe.exitCode).toBe(0); + expect(probe.output.trim()).toBe("False"); + }); + + it("settles Python cells that raise SystemExit", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-system-exit-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-system-exit:${crypto.randomUUID()}`; + + const result = await executePython('raise SystemExit("bye")', { + cwd: tempDir.path(), + sessionId, + sessionFile, + timeoutMs: 500, + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain("SystemExit"); + expect(result.output).toContain("bye"); + }); + + it("lets a subagent inherit parent JavaScript and Python eval state", async () => { + using tempDir = TempDir.createSync("@omp-eval-subagent-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const evalSessionId = `session:${sessionFile}:cwd:${tempDir.path()}`; + const parentSession = createToolSession(tempDir.path(), sessionFile, evalSessionId); + let seenJs = ""; + let seenPy = ""; + let capturedOptions: CreateAgentSessionOptions | undefined; + + await executeJs('globalThis.parentSecret = "hello-js";', { + sessionId: `js:${evalSessionId}`, + session: parentSession, + sessionFile, + }); + await executePython('parent_secret = "hello-py"', { + cwd: tempDir.path(), + sessionId: `python:${evalSessionId}`, + sessionFile, + }); + + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents: [taskAgent], projectAgentsDir: null }); + vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async (options = {}) => { + capturedOptions = options; + const inherited = options.parentEvalSessionId; + if (!inherited) throw new Error("Missing parent eval session id"); + return { + session: createYieldingSubagentSession(async () => { + const jsResult = await executeJs("return globalThis.parentSecret;", { + sessionId: `js:${inherited}`, + session: parentSession, + sessionFile, + }); + const pyResult = await executePython("print(parent_secret)", { + cwd: tempDir.path(), + sessionId: `python:${inherited}`, + sessionFile, + }); + seenJs = jsResult.output.trim(); + seenPy = pyResult.output.trim(); + }), + extensionsResult: {} as unknown as LoadExtensionsResult, + setToolUIContext: () => {}, + eventBus: new EventBus(), + } satisfies CreateAgentSessionResult; + }); + + const tool = await TaskTool.create(parentSession); + await tool.execute("tool-call", taskParams); + + expect(capturedOptions?.parentEvalSessionId).toBe(evalSessionId); + expect(seenJs).toBe("hello-js"); + expect(seenPy).toBe("hello-py"); + }); + + it("routes interleaved JavaScript display output to the matching run", async () => { + using tempDir = TempDir.createSync("@omp-eval-js-interleave-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `js-interleave:${crypto.randomUUID()}`; + const session = createToolSession(tempDir.path(), sessionFile); + + const first = executeJs('await Bun.sleep(80); display({ label: "A" });', { + sessionId, + session, + sessionFile, + }); + await Bun.sleep(10); + const second = executeJs('display({ label: "B" });', { + sessionId, + session, + sessionFile, + }); + + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(firstResult.exitCode).toBe(0); + expect(secondResult.exitCode).toBe(0); + expect(firstResult.displayOutputs).toEqual([{ type: "json", data: { label: "A" } }]); + expect(secondResult.displayOutputs).toEqual([{ type: "json", data: { label: "B" } }]); + }); + + it("routes interleaved Python display output to the matching run", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-interleave-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-interleave:${crypto.randomUUID()}`; + + const first = executePython( + `import asyncio +await asyncio.sleep(0.08) +display({"label": "A"})`, + { + cwd: tempDir.path(), + sessionId, + sessionFile, + }, + ); + await Bun.sleep(10); + const second = executePython('display({"label": "B"})', { + cwd: tempDir.path(), + sessionId, + sessionFile, + }); + + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(firstResult.exitCode).toBe(0); + expect(secondResult.exitCode).toBe(0); + expect(firstResult.displayOutputs).toEqual([{ type: "json", data: { label: "A" } }]); + expect(secondResult.displayOutputs).toEqual([{ type: "json", data: { label: "B" } }]); + }); + it("preserves module-level singleton state across re-imports of an unchanged file", async () => { + using tempDir = TempDir.createSync("@omp-eval-js-mtime-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `js-mtime:${crypto.randomUUID()}`; + const session = createToolSession(tempDir.path(), sessionFile); + const modulePath = path.join(tempDir.path(), "singleton.ts"); + const moduleSpec = JSON.stringify(modulePath); + await Bun.write( + modulePath, + "let value = 0;\nexport function set(v) { value = v; }\nexport function get() { return value; }\n", + ); + + const initResult = await executeJs(`const mod = await import(${moduleSpec}); mod.set(42); return mod.get();`, { + sessionId, + session, + sessionFile, + }); + expect(initResult.exitCode).toBe(0); + expect(initResult.output.trim()).toBe("42"); + + // Unchanged file: re-import must reuse the existing module namespace so the + // counter is still 42. This is the regression — the previous unconditional + // `delete require.cache[target]` reset singletons on every dynamic import. + const reuseResult = await executeJs(`const mod = await import(${moduleSpec}); return mod.get();`, { + sessionId, + session, + sessionFile, + }); + expect(reuseResult.exitCode).toBe(0); + expect(reuseResult.output.trim()).toBe("42"); + + // Bump mtime by 5s to simulate an edit; the next import must evict the cache + // and re-evaluate the file, dropping the counter back to its initializer. + const future = new Date(Date.now() + 5_000); + await fs.utimes(modulePath, future, future); + + const reloadResult = await executeJs(`const mod = await import(${moduleSpec}); return mod.get();`, { + sessionId, + session, + sessionFile, + }); + expect(reloadResult.exitCode).toBe(0); + expect(reloadResult.output.trim()).toBe("0"); + }); + + it("reloads a local re-export when a transitive dependency changes", async () => { + using tempDir = TempDir.createSync("@omp-eval-js-transitive-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `js-transitive:${crypto.randomUUID()}`; + const session = createToolSession(tempDir.path(), sessionFile); + const leafPath = path.join(tempDir.path(), "leaf.ts"); + const entryPath = path.join(tempDir.path(), "entry.ts"); + const entrySpec = JSON.stringify(entryPath); + await Bun.write(leafPath, "export const value = 1;\n"); + await Bun.write(entryPath, 'export { value } from "./leaf.ts";\n'); + + const initial = await executeJs(`const mod = await import(${entrySpec}); return mod.value;`, { + sessionId, + session, + sessionFile, + }); + expect(initial.exitCode).toBe(0); + expect(initial.output.trim()).toBe("1"); + + await Bun.write(leafPath, "export const value = 2;\n"); + const future = new Date(Date.now() + 5_000); + await fs.utimes(leafPath, future, future); + + const reloaded = await executeJs(`const mod = await import(${entrySpec}); return mod.value;`, { + sessionId, + session, + sessionFile, + }); + expect(reloaded.exitCode).toBe(0); + expect(reloaded.output.trim()).toBe("2"); + }); + + it("refreshes the Python tool proxy when bridge env appears after kernel warm-up", async () => { + using tempDir = TempDir.createSync("@omp-eval-py-tool-proxy-"); + const sessionFile = path.join(tempDir.path(), "session.jsonl"); + const sessionId = `py-tool-proxy:${crypto.randomUUID()}`; + const bridgeCalls: unknown[] = []; + const bridgeSession = createBridgeToolSession("bridge-ok", bridgeCalls); + + const withoutBridge = await executePython( + 'try:\n print(tool.read({"path": "foo.txt"}))\nexcept Exception as exc:\n print(type(exc).__name__)\n print(str(exc))', + { cwd: tempDir.path(), sessionId, sessionFile }, + ); + const withBridge = await executePython('print(tool.read({"path": "foo.txt"}))', { + cwd: tempDir.path(), + sessionId, + sessionFile, + toolSession: bridgeSession, + }); + + expect(withoutBridge.exitCode).toBe(0); + expect(withoutBridge.output).toContain("RuntimeError"); + expect(withoutBridge.output).toContain("tool bridge is unavailable"); + expect(withBridge.exitCode).toBe(0); + expect(withBridge.output.trim()).toBe("bridge-ok"); + expect(bridgeCalls).toEqual([{ path: "foo.txt", _i: "py prelude" }]); + }); +}); diff --git a/packages/coding-agent/src/eval/js/context-manager.ts b/packages/coding-agent/src/eval/js/context-manager.ts index 1b77736dd..cea0c97e5 100644 --- a/packages/coding-agent/src/eval/js/context-manager.ts +++ b/packages/coding-agent/src/eval/js/context-manager.ts @@ -48,10 +48,11 @@ interface JsSession { worker: WorkerHandle; state: "alive" | "dead"; pending: Map; - queue: Promise; } const sessions = new Map(); +const startingSessions = new Map>(); +const resettingSessions = new Set(); const READY_TIMEOUT_MS_DEFAULT = 5_000; export async function executeInVmContext(options: { @@ -66,45 +67,46 @@ export async function executeInVmContext(options: { runState: VmRunState; }): Promise<{ value: unknown }> { if (options.reset) { - await resetVmContext(options.sessionKey); + if (resettingSessions.has(options.sessionKey)) { + throw new ToolError("JS context reset already in progress"); + } + resettingSessions.add(options.sessionKey); + try { + await resetVmContext(options.sessionKey); + } finally { + resettingSessions.delete(options.sessionKey); + } + } else if (resettingSessions.has(options.sessionKey)) { + throw new ToolError("JS context reset in progress"); } const session = await acquireSession( options.sessionKey, { cwd: options.cwd, sessionId: options.sessionId }, options.timeoutMs, ); - return await runQueued(session, () => runOnce(session, options)); + return await runOnce(session, options); } export async function resetVmContext(sessionKey: string): Promise { - const session = sessions.get(sessionKey); + const session = sessions.get(sessionKey) ?? (await startingSessions.get(sessionKey)?.catch(() => undefined)); if (!session) return; sessions.delete(sessionKey); await killSession(session, new ToolError("JS context reset")); } export async function disposeAllVmContexts(): Promise { + const pending = [...startingSessions.values()]; + startingSessions.clear(); + const started = await Promise.allSettled(pending); const all = [...sessions.values()]; + for (const result of started) { + if (result.status !== "fulfilled") continue; + if (!all.includes(result.value)) all.push(result.value); + } sessions.clear(); await Promise.all(all.map(session => killSession(session, new ToolError("JS context disposed")))); } -async function runQueued(session: JsSession, work: () => Promise): Promise { - const previous = session.queue; - const { promise, resolve } = Promise.withResolvers(); - session.queue = promise; - try { - await previous; - } catch { - // Previous run's failure must not poison this one. - } - try { - return await work(); - } finally { - resolve(); - } -} - async function runOnce( session: JsSession, options: { @@ -162,43 +164,52 @@ async function runOnce( async function acquireSession(sessionKey: string, snapshot: SessionSnapshot, timeoutMs?: number): Promise { const existing = sessions.get(sessionKey); if (existing && existing.state === "alive") return existing; + const starting = startingSessions.get(sessionKey); + if (starting) return await starting; - const worker = await spawnJsWorker(); - const session: JsSession = { - sessionKey, - worker, - state: "alive", - pending: new Map(), - queue: Promise.resolve(), - }; - const { promise: readyPromise, resolve: resolveReady, reject: rejectReady } = Promise.withResolvers(); - let resolved = false; - const unsubscribe = worker.onMessage(msg => { - if (!resolved && msg.type === "ready") { - resolved = true; - resolveReady(); - return; + const startup = (async (): Promise => { + const worker = await spawnJsWorker(); + const session: JsSession = { + sessionKey, + worker, + state: "alive", + pending: new Map(), + }; + const { promise: readyPromise, resolve: resolveReady, reject: rejectReady } = Promise.withResolvers(); + let resolved = false; + const unsubscribe = worker.onMessage(msg => { + if (!resolved && msg.type === "ready") { + resolved = true; + resolveReady(); + return; + } + if (!resolved && msg.type === "init-failed") { + resolved = true; + rejectReady(errorFromPayload(msg.error)); + return; + } + handleSessionMessage(session, msg); + }); + try { + // Cold-start can exceed 5s on slow hosts. Let the caller's per-cell timeout dominate so + // users can grant more headroom when they raise `timeout` on a cell. + const readyTimeoutMs = Math.max(READY_TIMEOUT_MS_DEFAULT, timeoutMs ?? 0); + await raceWithTimeout(readyPromise, readyTimeoutMs, "Timed out initializing JS eval worker"); + worker.send({ type: "init", snapshot }); + sessions.set(sessionKey, session); + return session; + } catch (error) { + unsubscribe(); + await worker.terminate().catch(() => undefined); + throw error; } - if (!resolved && msg.type === "init-failed") { - resolved = true; - rejectReady(errorFromPayload(msg.error)); - return; - } - handleSessionMessage(session, msg); - }); + })(); + startingSessions.set(sessionKey, startup); try { - // Cold-start can exceed 5s on slow hosts. Let the caller's per-cell timeout dominate so - // users can grant more headroom when they raise `timeout` on a cell. - const readyTimeoutMs = Math.max(READY_TIMEOUT_MS_DEFAULT, timeoutMs ?? 0); - await raceWithTimeout(readyPromise, readyTimeoutMs, "Timed out initializing JS eval worker"); - } catch (error) { - unsubscribe(); - await worker.terminate().catch(() => undefined); - throw error; + return await startup; + } finally { + if (startingSessions.get(sessionKey) === startup) startingSessions.delete(sessionKey); } - worker.send({ type: "init", snapshot }); - sessions.set(sessionKey, session); - return session; } function handleSessionMessage(session: JsSession, msg: WorkerOutbound): void { diff --git a/packages/coding-agent/src/eval/js/shared/local-module-loader.ts b/packages/coding-agent/src/eval/js/shared/local-module-loader.ts new file mode 100644 index 000000000..a0ea8fa47 --- /dev/null +++ b/packages/coding-agent/src/eval/js/shared/local-module-loader.ts @@ -0,0 +1,265 @@ +import * as fs from "node:fs"; +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import * as vm from "node:vm"; +import { collectModuleSourceSpecifiers, stripTypeScriptSyntax } from "./rewrite-imports"; + +interface LocalModuleEntry { + version: number; + identifier: string; + module: vm.SourceTextModule; +} + +export type LocalImportResolution = { mode: "local"; value: unknown } | { mode: "external"; target: string }; + +const LOCAL_MODULE_EXTENSIONS = new Set([".js", ".jsx", ".mjs", ".ts", ".tsx", ".mts"]); + +export class LocalModuleLoader { + #context: vm.Context; + #sessionTag: string; + #moduleMtimes = new Map(); + #moduleDeps = new Map>(); + #moduleParents = new Map>(); + #moduleVersions = new Map(); + #moduleEntries = new Map(); + #moduleBuilds = new Map>(); + #externalModules = new Map>(); + #requireCache = new Map(); + + constructor(sessionId: string) { + this.#context = vm.createContext(globalThis); + this.#sessionTag = Bun.hash(sessionId).toString(16); + } + + async resolveForRun(cwd: string, source: string): Promise { + this.#refreshTrackedLocalModules(); + return await this.#resolveFromBase(cwd, source); + } + + async resolveForModule(moduleUrl: string, source: string, cwd: string): Promise { + this.#refreshTrackedLocalModules(); + const modulePath = this.filenameForUrl(moduleUrl); + const baseDir = modulePath ? path.dirname(modulePath) : cwd; + return await this.#resolveFromBase(baseDir, source); + } + + requireForFile(moduleUrlOrPath: string | undefined, cwd: string): NodeJS.Require { + const basePath = this.filenameForUrl(moduleUrlOrPath) ?? path.join(cwd, "[eval]"); + let cached = this.#requireCache.get(basePath); + if (!cached) { + cached = buildRequire(basePath); + this.#requireCache.set(basePath, cached); + } + return cached; + } + + filenameForUrl(moduleUrlOrPath: string | undefined): string | null { + if (!moduleUrlOrPath) return null; + if (moduleUrlOrPath.startsWith("file://")) return fileURLToPath(moduleUrlOrPath); + return path.isAbsolute(moduleUrlOrPath) ? moduleUrlOrPath : null; + } + + dirnameForUrl(moduleUrlOrPath: string | undefined, cwd: string): string { + const filename = this.filenameForUrl(moduleUrlOrPath); + return filename ? path.dirname(filename) : cwd; + } + + async #resolveFromBase(baseDir: string, source: string): Promise { + const resolved = resolveImportSpecifier(baseDir, source); + if (isLocalPathSpecifier(source) && isManagedLocalModulePath(resolved)) { + const entry = await this.#ensureLocalModule(resolved); + return { mode: "local", value: entry.module.namespace }; + } + return { mode: "external", target: normalizeImportTarget(resolved) }; + } + + async #ensureLocalModule(modulePath: string): Promise { + const existing = this.#moduleEntries.get(modulePath); + if (existing) return existing; + const building = this.#moduleBuilds.get(modulePath); + if (building) return await building; + const buildPromise = this.#buildLocalModule(modulePath).finally(() => { + if (this.#moduleBuilds.get(modulePath) === buildPromise) this.#moduleBuilds.delete(modulePath); + }); + this.#moduleBuilds.set(modulePath, buildPromise); + return await buildPromise; + } + + async #buildLocalModule(modulePath: string): Promise { + const rawSource = fs.readFileSync(modulePath, "utf8"); + const stripped = stripTypeScriptSyntax(rawSource); + const moduleDir = path.dirname(modulePath); + const localDeps = new Set(); + for (const specifier of collectModuleSourceSpecifiers(stripped)) { + const resolved = resolveImportSpecifier(moduleDir, specifier); + if (isLocalPathSpecifier(specifier) && isManagedLocalModulePath(resolved)) { + localDeps.add(resolved); + } + } + this.#setModuleDependencies(modulePath, localDeps); + this.#moduleMtimes.set(modulePath, fs.statSync(modulePath).mtimeMs); + const version = this.#moduleVersions.get(modulePath) ?? 1; + this.#moduleVersions.set(modulePath, version); + const fileUrl = pathToFileURL(modulePath).href; + const identifier = `${fileUrl}?omp-session=${this.#sessionTag}&v=${version}`; + const wrappedSource = buildModuleSource(stripped, modulePath); + const module = new vm.SourceTextModule(wrappedSource, { + context: this.#context, + identifier, + initializeImportMeta: meta => { + (meta as { url?: string; path?: string; dir?: string }).url = fileUrl; + (meta as { url?: string; path?: string; dir?: string }).path = modulePath; + (meta as { url?: string; path?: string; dir?: string }).dir = moduleDir; + }, + importModuleDynamically: async specifier => { + return await this.#resolveLinkedModule(modulePath, String(specifier)); + }, + }); + const entry: LocalModuleEntry = { version, identifier, module }; + this.#moduleEntries.set(modulePath, entry); + try { + await module.link(async specifier => await this.#resolveLinkedModule(modulePath, specifier)); + await module.evaluate(); + return entry; + } catch (error) { + this.#moduleEntries.delete(modulePath); + throw error; + } + } + + async #resolveLinkedModule(referrerPath: string, specifier: string): Promise { + const baseDir = path.dirname(referrerPath); + const resolved = resolveImportSpecifier(baseDir, specifier); + if (isLocalPathSpecifier(specifier) && isManagedLocalModulePath(resolved)) { + return (await this.#ensureLocalModule(resolved)).module; + } + return await this.#ensureExternalModule(normalizeImportTarget(resolved)); + } + + async #ensureExternalModule(target: string): Promise { + const existing = this.#externalModules.get(target); + if (existing) return await existing; + const loadPromise = (async () => { + const namespace = await import(target); + const exportNames = Object.keys(namespace); + const module = new vm.SyntheticModule( + exportNames, + function () { + for (const name of exportNames) { + this.setExport(name, namespace[name as keyof typeof namespace]); + } + }, + { context: this.#context, identifier: target }, + ); + await module.link(() => { + throw new Error("Synthetic external modules have no dependencies"); + }); + await module.evaluate(); + return module; + })(); + this.#externalModules.set(target, loadPromise); + try { + return await loadPromise; + } catch (error) { + if (this.#externalModules.get(target) === loadPromise) this.#externalModules.delete(target); + throw error; + } + } + + #refreshTrackedLocalModules(): void { + const changed: string[] = []; + for (const [modulePath, previousMtime] of this.#moduleMtimes.entries()) { + let nextMtime: number | undefined; + try { + nextMtime = fs.statSync(modulePath).mtimeMs; + } catch { + nextMtime = undefined; + } + if (nextMtime === previousMtime) continue; + if (nextMtime === undefined) this.#moduleMtimes.delete(modulePath); + else this.#moduleMtimes.set(modulePath, nextMtime); + changed.push(modulePath); + } + for (const modulePath of changed) { + this.#invalidateModuleAndParents(modulePath, new Set()); + } + } + + #invalidateModuleAndParents(modulePath: string, seen: Set): void { + if (seen.has(modulePath)) return; + seen.add(modulePath); + this.#moduleEntries.delete(modulePath); + this.#moduleBuilds.delete(modulePath); + this.#moduleVersions.set(modulePath, (this.#moduleVersions.get(modulePath) ?? 1) + 1); + const parents = [...(this.#moduleParents.get(modulePath) ?? [])]; + for (const parent of parents) this.#invalidateModuleAndParents(parent, seen); + } + + #setModuleDependencies(modulePath: string, deps: Set): void { + const previousDeps = this.#moduleDeps.get(modulePath); + if (previousDeps) { + for (const dep of previousDeps) { + const parents = this.#moduleParents.get(dep); + if (!parents) continue; + parents.delete(modulePath); + if (parents.size === 0) this.#moduleParents.delete(dep); + } + } + this.#moduleDeps.set(modulePath, new Set(deps)); + for (const dep of deps) { + const parents = this.#moduleParents.get(dep) ?? new Set(); + parents.add(modulePath); + this.#moduleParents.set(dep, parents); + } + } +} + +function buildRequire(fromPath: string): NodeJS.Require { + const basePath = path.extname(fromPath) ? fromPath : path.join(fromPath, "[eval]"); + return createRequire(pathToFileURL(basePath).href); +} + +function buildModuleSource(source: string, modulePath: string): string { + const moduleDir = path.dirname(modulePath); + return [ + `const require = globalThis.__omp_get_require__(${JSON.stringify(pathToFileURL(modulePath).href)});`, + `const __filename = ${JSON.stringify(modulePath)};`, + `const __dirname = ${JSON.stringify(moduleDir)};`, + source, + ].join("\n"); +} + +function resolveImportSpecifier(cwd: string, source: string): string { + if (/^[a-z][a-z0-9+.-]*:/i.test(source)) return source; + try { + return Bun.resolveSync(source, cwd); + } catch { + return source; + } +} + +function isLocalPathSpecifier(source: string): boolean { + return ( + source.startsWith("./") || + source.startsWith("../") || + source === "." || + source === ".." || + source.startsWith("/") || + source.startsWith("~/") || + /^[a-zA-Z]:[\\/]/.test(source) + ); +} + +function isManagedLocalModulePath(target: string): boolean { + return ( + path.isAbsolute(target) && + LOCAL_MODULE_EXTENSIONS.has(path.extname(target)) && + !target.includes(`${path.sep}node_modules${path.sep}`) + ); +} + +function normalizeImportTarget(target: string): string { + if (path.isAbsolute(target)) return pathToFileURL(target).href; + return target; +} diff --git a/packages/coding-agent/src/eval/js/shared/prelude.txt b/packages/coding-agent/src/eval/js/shared/prelude.txt index 0fd503eab..0a7ce544d 100644 --- a/packages/coding-agent/src/eval/js/shared/prelude.txt +++ b/packages/coding-agent/src/eval/js/shared/prelude.txt @@ -51,6 +51,10 @@ if (!globalThis.__omp_js_prelude_loaded__) { warn: (...args) => globalThis.__omp_log__("warn", ...formatArgs(args)), error: (...args) => globalThis.__omp_log__("error", ...formatArgs(args)), debug: (...args) => globalThis.__omp_log__("debug", ...formatArgs(args)), + table: (data, columns) => + columns === undefined + ? globalThis.__omp_table__(data) + : globalThis.__omp_table__(data, columns), }; globalThis.console = consoleBridge; diff --git a/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts b/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts index 2afa85947..73a8703a4 100644 --- a/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts +++ b/packages/coding-agent/src/eval/js/shared/rewrite-imports.ts @@ -57,6 +57,10 @@ type BabelExpressionStatement = { }; type BabelProgramNode = BabelImportDeclaration | BabelLexicalDecl | BabelExpressionStatement | { type: string }; +type BabelModuleSourceDeclaration = { + type: "ImportDeclaration" | "ExportNamedDeclaration" | "ExportAllDeclaration"; + source?: { value: string; start: number; end: number } | null; +}; type BabelNode = { type: string; start: number; end: number; [key: string]: unknown }; @@ -189,6 +193,84 @@ export function rewriteImports(code: string): string { } return result; } +export function collectModuleSourceSpecifiers(code: string): string[] { + const ast = parseProgram(code); + if (!ast) return []; + const sources: string[] = []; + for (const node of ast.program.body) { + if ( + (node.type === "ImportDeclaration" || + node.type === "ExportNamedDeclaration" || + node.type === "ExportAllDeclaration") && + typeof (node as BabelModuleSourceDeclaration).source?.value === "string" + ) { + sources.push((node as BabelModuleSourceDeclaration).source!.value); + } + } + return sources; +} + +export function rewriteModuleSourceSpecifiers(code: string, replacer: (source: string) => string): string { + const ast = parseProgram(code); + if (!ast) return code; + + type Edit = { start: number; end: number; text: string }; + const edits: Edit[] = []; + + for (const node of ast.program.body) { + if ( + node.type !== "ImportDeclaration" && + node.type !== "ExportNamedDeclaration" && + node.type !== "ExportAllDeclaration" + ) { + continue; + } + const source = (node as BabelModuleSourceDeclaration).source; + if (!source || typeof source.value !== "string") continue; + const next = replacer(source.value); + if (next === source.value) continue; + edits.push({ start: source.start, end: source.end, text: JSON.stringify(next) }); + } + + if (edits.length === 0) return code; + edits.sort((a, b) => b.start - a.start); + let result = code; + for (const edit of edits) { + result = result.slice(0, edit.start) + edit.text + result.slice(edit.end); + } + return result; +} + +export function rewriteDynamicImports(code: string, callee = "__omp_import__"): string { + if (!code.includes("import")) return code; + const ast = parseProgram(code); + if (!ast) return code; + + type Edit = { start: number; end: number; text: string }; + const edits: Edit[] = []; + walkNodes(ast, node => { + if (node.type !== "CallExpression") return; + const call = node as unknown as { callee?: { type?: string; start?: number; end?: number } }; + const callCallee = call.callee; + if ( + !callCallee || + callCallee.type !== "Import" || + typeof callCallee.start !== "number" || + typeof callCallee.end !== "number" + ) { + return; + } + edits.push({ start: callCallee.start, end: callCallee.end, text: callee }); + }); + + if (edits.length === 0) return code; + edits.sort((a, b) => b.start - a.start); + let result = code; + for (const edit of edits) { + result = result.slice(0, edit.start) + edit.text + result.slice(edit.end); + } + return result; +} function collectBindingNames(pattern: unknown, names: string[]): void { if (!pattern || typeof pattern !== "object") return; @@ -390,6 +472,9 @@ function stripTypeScript(code: string): string { return code; } } +export function stripTypeScriptSyntax(code: string): string { + return stripTypeScript(code); +} // Heuristic: any of the obvious TS-only tokens. Plain JS using `as` only inside strings // won't match because we require a leading word boundary plus a colon/keyword neighbor. diff --git a/packages/coding-agent/src/eval/js/shared/runtime.ts b/packages/coding-agent/src/eval/js/shared/runtime.ts index f02761756..51ccc21f2 100644 --- a/packages/coding-agent/src/eval/js/shared/runtime.ts +++ b/packages/coding-agent/src/eval/js/shared/runtime.ts @@ -1,23 +1,23 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { Console } from "node:console"; import * as fs from "node:fs"; import { createRequire } from "node:module"; import * as path from "node:path"; -import { pathToFileURL } from "node:url"; +import { Writable } from "node:stream"; import * as util from "node:util"; import { logger } from "@oh-my-pi/pi-utils"; -import { ToolError } from "../../../tools/tool-errors"; import { createHelpers, type HelperBundle } from "./helpers"; import { awaitMaybePromise, indirectEval } from "./indirect-eval"; +import { LocalModuleLoader } from "./local-module-loader"; import { JAVASCRIPT_PRELUDE_SOURCE } from "./prelude"; import { wrapCode } from "./rewrite-imports"; import type { JsDisplayOutput, JsStatusEvent } from "./types"; /** - * Per-run callbacks. Returned by `getHooks()` on each helper/tool/display invocation so - * the embedding worker can route emissions to the currently active run. Returning `null` - * makes status/display/tool calls reject with an error — useful for guarding against - * helpers being invoked outside a run window. + * Per-run callbacks. Runtime globals resolve these from AsyncLocalStorage so + * overlapping async cells can route output/tool calls back to their own run. */ export interface RuntimeHooks { onText(chunk: string): void; @@ -25,11 +25,17 @@ export interface RuntimeHooks { callTool(name: string, args: unknown): Promise; } +export interface RunContext { + runId: string; + hooks: RuntimeHooks; + cwd: string; + finalExpressionSet: boolean; + finalExpressionValue: unknown; +} + export interface RuntimeOptions { initialCwd: string; sessionId: string; - /** Resolve hooks for the run currently in flight, or `null` if nothing is active. */ - getHooks(): RuntimeHooks | null; /** * Extra globals installed alongside `__omp_helpers__` / prelude. Use for stable, lifetime- * of-the-worker bindings (e.g. browser's `page`, `browser`). Per-run scope should be set @@ -118,19 +124,18 @@ export class JsRuntime { #cwd: string; readonly sessionId: string; #env: Map; - #getHooks: () => RuntimeHooks | null; - #finalExpressionSet = false; - #finalExpressionValue: unknown; + #als = new AsyncLocalStorage(); + #moduleLoader: LocalModuleLoader; constructor(opts: RuntimeOptions) { this.#cwd = opts.initialCwd; this.sessionId = opts.sessionId; this.#env = new Map(); - this.#getHooks = opts.getHooks; + this.#moduleLoader = new LocalModuleLoader(this.sessionId); this.helpers = createHelpers({ - cwd: () => this.#cwd, + cwd: () => this.#activeCwd(), env: this.#env, - emitStatus: event => this.#getHooks()?.onDisplay({ type: "status", event }), + emitStatus: event => this.#activeHooks("emitStatus")?.onDisplay({ type: "status", event }), }); this.#install(opts.extraGlobals); } @@ -154,29 +159,42 @@ export class JsRuntime { Object.assign(globalThis, scope); } - async run(code: string, filename?: string): Promise { - this.#finalExpressionSet = false; - this.#finalExpressionValue = undefined; - const wrapped = wrapCode(code); - const value = indirectEval(wrapped.source, filename); - if (wrapped.finalExpressionReturned) { - const awaited = await awaitMaybePromise(value); - if (this.#finalExpressionSet) { - const finalValue = this.#finalExpressionValue; - this.#finalExpressionSet = false; - this.#finalExpressionValue = undefined; - const resolved = await awaitMaybePromise(finalValue); - return resolved; + async run( + code: string, + filename: string | undefined, + hooks: RuntimeHooks, + options: { runId?: string; cwd?: string } = {}, + ): Promise { + const context: RunContext = { + runId: options.runId ?? crypto.randomUUID(), + hooks, + cwd: options.cwd ?? this.#cwd, + finalExpressionSet: false, + finalExpressionValue: undefined, + }; + return await this.#als.run(context, async () => { + const wrapped = wrapCode(code); + const value = indirectEval(wrapped.source, filename); + if (wrapped.finalExpressionReturned) { + const awaited = await awaitMaybePromise(value); + if (context.finalExpressionSet) { + const finalValue = context.finalExpressionValue; + context.finalExpressionSet = false; + context.finalExpressionValue = undefined; + return await awaitMaybePromise(finalValue); + } + return awaited; } - return awaited; - } - return await awaitMaybePromise(value); + return await awaitMaybePromise(value); + }); } - displayValue(value: unknown): void { + displayValue(value: unknown, hooks: RuntimeHooks | undefined = this.#als.getStore()?.hooks): void { if (value === undefined) return; - const hooks = this.#getHooks(); - if (!hooks) return; + if (!hooks) { + logger.warn("js runtime display called outside an active run"); + return; + } if (value && typeof value === "object") { const record = value as Record; if (record.type === "image" && typeof record.mimeType === "string") { @@ -207,45 +225,108 @@ export class JsRuntime { hooks.onText(`${String(value)}\n`); } + #activeCwd(): string { + return this.#als.getStore()?.cwd ?? this.#cwd; + } + + #activeHooks(action: string): RuntimeHooks | undefined { + const hooks = this.#als.getStore()?.hooks; + if (!hooks) { + logger.warn("js runtime helper called outside an active run", { action }); + } + return hooks; + } + + #activeRequire(moduleUrlOrPath?: string): NodeJS.Require { + return this.#moduleLoader.requireForFile(moduleUrlOrPath, this.#activeCwd()); + } + + #moduleFilename(moduleUrlOrPath?: string): string { + return this.#moduleLoader.filenameForUrl(moduleUrlOrPath) ?? path.join(this.#activeCwd(), "[eval]"); + } + + #moduleDirname(moduleUrlOrPath?: string): string { + return this.#moduleLoader.dirnameForUrl(moduleUrlOrPath, this.#activeCwd()); + } + + #buildDynamicRequire(): NodeJS.Require { + const dynamicRequire = ((id: string) => this.#activeRequire()(id)) as NodeJS.Require; + const resolve = ((id: string, options?: { paths?: string[] }) => + this.#activeRequire().resolve(id, options)) as NodeJS.Require["resolve"] & { + paths(request: string): string[] | null; + }; + resolve.paths = request => this.#activeRequire().resolve.paths(request); + Object.defineProperties(dynamicRequire, { + resolve: { value: resolve, configurable: true }, + cache: { get: () => this.#activeRequire().cache, configurable: true }, + extensions: { get: () => this.#activeRequire().extensions, configurable: true }, + main: { get: () => this.#activeRequire().main, configurable: true }, + }); + return dynamicRequire; + } + #install(extraGlobals: Record | undefined): void { const injected: Record = { __omp_session__: { cwd: this.#cwd, sessionId: this.sessionId }, __omp_helpers__: this.helpers, __omp_call_tool__: async (name: string, args: unknown) => { - const hooks = this.#getHooks(); - if (!hooks) throw new ToolError("Tool calls are only valid inside an active run"); + const hooks = this.#activeHooks("tool"); + if (!hooks) return undefined; return await hooks.callTool(name, args); }, __omp_import__: async (source: string, options?: ImportCallOptions) => { - const target = resolveImportSpecifier(this.#cwd, source); - // Always invalidate cached module records for user-owned source files so edits - // between cells are picked up. Bun ignores query-string busting on `file:` URLs - // but honors `delete require.cache[absPath]`; bare specifiers and URL schemes are - // left alone to keep package identity stable across cells. - if (isLocalPathSpecifier(source) && path.isAbsolute(target)) { - delete require.cache[target]; - } + const resolved = await this.#moduleLoader.resolveForRun(this.#activeCwd(), source); + if (resolved.mode === "local") return resolved.value; + const target = resolved.target; return options !== undefined ? await import(target, options) : await import(target); }, + __omp_import_from__: async (moduleUrl: string, source: string, options?: ImportCallOptions) => { + const resolved = await this.#moduleLoader.resolveForModule(moduleUrl, source, this.#activeCwd()); + if (resolved.mode === "local") return resolved.value; + const target = resolved.target; + return options !== undefined ? await import(target, options) : await import(target); + }, + __omp_get_require__: (moduleUrl?: string) => this.#activeRequire(moduleUrl), + __omp_get_filename__: (moduleUrl?: string) => this.#moduleFilename(moduleUrl), + __omp_get_dirname__: (moduleUrl?: string) => this.#moduleDirname(moduleUrl), __omp_emit_status__: (op: string, data: Record = {}) => { const event: JsStatusEvent = { op, ...data }; - this.#getHooks()?.onDisplay({ type: "status", event }); + this.#activeHooks("emitStatus")?.onDisplay({ type: "status", event }); }, __omp_log__: (level: string, ...args: unknown[]) => { const prefix = level === "error" ? "[error] " : level === "warn" ? "[warn] " : ""; const text = `${prefix}${formatConsoleArgs(args)}`; - this.#getHooks()?.onText(text.endsWith("\n") ? text : `${text}\n`); + this.#activeHooks("log")?.onText(text.endsWith("\n") ? text : `${text}\n`); + }, + __omp_table__: (...args: unknown[]) => { + const hooks = this.#activeHooks("table"); + if (!hooks) return; + let buffer = ""; + const stream = new Writable({ + write(chunk, _enc, cb) { + buffer += typeof chunk === "string" ? chunk : (chunk as Buffer).toString("utf8"); + cb(); + }, + }); + const tableConsole = new Console({ stdout: stream, colorMode: false }); + (tableConsole.table as (...a: unknown[]) => void)(...args); + hooks.onText(buffer.endsWith("\n") ? buffer : `${buffer}\n`); }, __omp_display__: (value: unknown) => this.displayValue(value), __omp_set_final_expr__: (value: unknown) => { - this.#finalExpressionSet = true; - this.#finalExpressionValue = value; + const context = this.#als.getStore(); + if (!context) { + logger.warn("js runtime final expression set outside an active run"); + return; + } + context.finalExpressionSet = true; + context.finalExpressionValue = value; }, webcrypto: crypto, // `process` is intentionally not overridden — user code gets the host worker's real // `process` object. Subsetting it caused segfaults in workers that share state with // puppeteer/worker_threads internals. - require: buildRequire(this.#cwd), + require: this.#buildDynamicRequire(), createRequire, fs, }; @@ -261,41 +342,3 @@ function formatConsoleArgs(args: unknown[]): string { .map(arg => (typeof arg === "string" ? arg : util.inspect(arg, { depth: 6, colors: false, breakLength: 120 }))) .join(" "); } - -function buildRequire(cwd: string): NodeJS.Require { - return createRequire(pathToFileURL(path.join(cwd, "[eval]")).href); -} - -/** - * Resolve an import specifier emitted by `rewriteImports` against the active session - * cwd. Relative paths (`./`, `../`, `/`) and bare specifiers (`pkg`, `@scope/pkg`) both go - * through `Bun.resolveSync` rooted at the cwd so user-pasted ESM behaves as if it lived in - * the project — not next to the worker module. URL-like specifiers (`file://`, `data:`, - * `node:`, `http:`) are passed through unchanged. - */ -function resolveImportSpecifier(cwd: string, source: string): string { - if (/^[a-z][a-z0-9+.-]*:/i.test(source)) return source; - try { - return Bun.resolveSync(source, cwd); - } catch { - return source; - } -} - -/** - * Returns true when the original specifier is a relative or absolute filesystem path - * (i.e. user-owned source the agent is iterating on). Bare specifiers and URL schemes - * are excluded — `node:` built-ins cannot be reloaded, and busting bare packages would - * defeat module identity for every cell while bringing no editing benefit. - */ -function isLocalPathSpecifier(source: string): boolean { - return ( - source.startsWith("./") || - source.startsWith("../") || - source === "." || - source === ".." || - source.startsWith("/") || - source.startsWith("~/") || - /^[a-zA-Z]:[\\/]/.test(source) - ); -} diff --git a/packages/coding-agent/src/eval/js/worker-core.ts b/packages/coding-agent/src/eval/js/worker-core.ts index 590571213..552e9af9a 100644 --- a/packages/coding-agent/src/eval/js/worker-core.ts +++ b/packages/coding-agent/src/eval/js/worker-core.ts @@ -3,6 +3,7 @@ import { JsRuntime, type RuntimeHooks } from "./shared/runtime"; import type { RunErrorPayload, SessionSnapshot, ToolReply, Transport, WorkerInbound } from "./worker-protocol"; interface PendingTool { + runId: string; resolve(value: unknown): void; reject(error: Error): void; } @@ -36,8 +37,7 @@ function errorFromPayload(payload: RunErrorPayload): Error { export class WorkerCore { #transport: Transport; #runtime: JsRuntime | null = null; - #queue: Promise = Promise.resolve(); - #active: ActiveRun | null = null; + #runs = new Map(); #unsubscribe: () => void; constructor(transport: Transport) { @@ -52,7 +52,7 @@ export class WorkerCore { this.#ensureRuntime(msg.snapshot); return; case "run": - this.#enqueueRun(msg.runId, msg.code, msg.filename, msg.snapshot); + void this.#runOne(msg.runId, msg.code, msg.filename, msg.snapshot); return; case "tool-reply": this.#deliverToolReply(msg.id, msg.reply); @@ -71,73 +71,58 @@ export class WorkerCore { this.#runtime = new JsRuntime({ initialCwd: snapshot.cwd, sessionId: snapshot.sessionId, - getHooks: () => this.#hooksForCurrentRun(), }); return this.#runtime; } - #hooksForCurrentRun(): RuntimeHooks | null { - const active = this.#active; - if (!active) return null; - const runId = active.runId; - return { - onText: chunk => this.#transport.send({ type: "text", runId, chunk }), - onDisplay: output => this.#transport.send({ type: "display", runId, output }), - callTool: (name, args) => this.#callTool(active, name, args), - }; - } - - #enqueueRun(runId: string, code: string, filename: string, snapshot: SessionSnapshot): void { - const previous = this.#queue; - const next = (async () => { - await previous.catch(() => undefined); - await this.#runOne(runId, code, filename, snapshot); - })(); - this.#queue = next.catch(() => undefined); - } - async #runOne(runId: string, code: string, filename: string, snapshot: SessionSnapshot): Promise { const runtime = this.#ensureRuntime(snapshot); runtime.setCwd(snapshot.cwd); - this.#active = { runId, pendingTools: new Map() }; + const active: ActiveRun = { runId, pendingTools: new Map() }; + this.#runs.set(runId, active); + const hooks: RuntimeHooks = { + onText: chunk => this.#transport.send({ type: "text", runId, chunk }), + onDisplay: output => this.#transport.send({ type: "display", runId, output }), + callTool: (name, args) => this.#callTool(active, name, args), + }; try { - const value = await runtime.run(code, filename); - runtime.displayValue(value); + const value = await runtime.run(code, filename, hooks, { runId, cwd: snapshot.cwd }); + runtime.displayValue(value, hooks); this.#transport.send({ type: "result", runId, ok: true }); } catch (error) { this.#transport.send({ type: "result", runId, ok: false, error: errorPayload(error) }); } finally { - this.#active = null; + this.#runs.delete(runId); } } async #callTool(active: ActiveRun, name: string, args: unknown): Promise { const id = `tc-${active.runId}-${crypto.randomUUID()}`; const { promise, resolve, reject } = Promise.withResolvers(); - active.pendingTools.set(id, { resolve, reject }); + active.pendingTools.set(id, { runId: active.runId, resolve, reject }); this.#transport.send({ type: "tool-call", id, runId: active.runId, name, args }); return await promise; } #deliverToolReply(id: string, reply: ToolReply): void { - const active = this.#active; - if (!active) return; - const pending = active.pendingTools.get(id); - if (!pending) return; - active.pendingTools.delete(id); - if (reply.ok) pending.resolve(reply.value); - else pending.reject(errorFromPayload(reply.error)); + for (const active of this.#runs.values()) { + const pending = active.pendingTools.get(id); + if (!pending) continue; + active.pendingTools.delete(id); + if (reply.ok) pending.resolve(reply.value); + else pending.reject(errorFromPayload(reply.error)); + return; + } } #close(): void { - const active = this.#active; - if (active) { + for (const active of this.#runs.values()) { for (const pending of active.pendingTools.values()) { pending.reject(new ToolError("JS worker closed")); } active.pendingTools.clear(); } - this.#active = null; + this.#runs.clear(); this.#runtime = null; this.#transport.send({ type: "closed" }); this.#unsubscribe(); diff --git a/packages/coding-agent/src/eval/py/executor.ts b/packages/coding-agent/src/eval/py/executor.ts index 42ec13b7f..d0a1d8e0e 100644 --- a/packages/coding-agent/src/eval/py/executor.ts +++ b/packages/coding-agent/src/eval/py/executor.ts @@ -1,14 +1,17 @@ +import * as path from "node:path"; + import { getProjectDir, logger } from "@oh-my-pi/pi-utils"; import { Settings } from "../../config/settings"; import { OutputSink } from "../../session/streaming-output"; import type { ToolSession } from "../../tools"; import { resolveOutputMaxColumns, resolveOutputSinkHeadBytes } from "../../tools/output-meta"; import type { JsStatusEvent } from "../js/shared/types"; -import type { KernelDisplayOutput } from "./display"; import { checkPythonKernelAvailability, + type KernelDisplayOutput, type KernelExecuteOptions, type KernelExecuteResult, + type KernelRuntimeEnv, PythonKernel, } from "./kernel"; import { ensurePyToolBridge, registerPyToolBridge } from "./tool-bridge"; @@ -92,20 +95,32 @@ export interface PythonResult { // --------------------------------------------------------------------------- // Session bookkeeping // -// One PythonKernel subprocess per session id. Sessions are reused until they -// die or are explicitly disposed. Multiple agent owners can register against -// the same session id; the kernel stays alive until the last owner detaches. +// One PythonKernel subprocess per (session id, cwd) tuple. The runner mutates +// process-global cwd/sys.path during execution, so cross-directory work MUST +// never share a live kernel. Multiple agent owners can still register against +// the same tuple; the kernel stays alive until the last owner detaches. // --------------------------------------------------------------------------- interface PythonSession { + sessionKey: string; sessionId: string; + cwd: string; kernel: PythonKernel; ownerIds: Set; hasFallbackOwner: boolean; - queue: Promise; } const sessions = new Map(); +const startingSessions = new Map>(); +const resettingSessions = new Set(); + +function normalizeSessionCwd(cwd: string): string { + return path.resolve(cwd); +} + +function buildSessionKey(sessionId: string, cwd: string): string { + return `${sessionId}\0${normalizeSessionCwd(cwd)}`; +} // --------------------------------------------------------------------------- // Cancellation plumbing @@ -240,19 +255,40 @@ function createCancelledPythonResult(timedOut: boolean, timeoutMs?: number): Pyt // Kernel start helpers // --------------------------------------------------------------------------- +const MANAGED_KERNEL_ENV_KEYS = [ + "PI_SESSION_FILE", + "PI_ARTIFACTS_DIR", + "PI_TOOL_BRIDGE_URL", + "PI_TOOL_BRIDGE_TOKEN", + "PI_TOOL_BRIDGE_SESSION", +] as const; + +function buildKernelEnvPatch(options: { + sessionFile?: string; + artifactsDir?: string; + bridgeSessionId?: string; + bridge?: { url: string; token: string }; +}): KernelRuntimeEnv { + return { + PI_SESSION_FILE: options.sessionFile ?? null, + PI_ARTIFACTS_DIR: options.artifactsDir ?? null, + PI_TOOL_BRIDGE_URL: options.bridge?.url ?? null, + PI_TOOL_BRIDGE_TOKEN: options.bridge?.token ?? null, + PI_TOOL_BRIDGE_SESSION: options.bridge && options.bridgeSessionId ? options.bridgeSessionId : null, + }; +} + function buildKernelEnv(options: { sessionFile?: string; artifactsDir?: string; bridgeSessionId?: string; bridge?: { url: string; token: string }; }): Record | undefined { + const patch = buildKernelEnvPatch(options); const env: Record = {}; - if (options.sessionFile) env.PI_SESSION_FILE = options.sessionFile; - if (options.artifactsDir) env.PI_ARTIFACTS_DIR = options.artifactsDir; - if (options.bridge && options.bridgeSessionId) { - env.PI_TOOL_BRIDGE_URL = options.bridge.url; - env.PI_TOOL_BRIDGE_TOKEN = options.bridge.token; - env.PI_TOOL_BRIDGE_SESSION = options.bridgeSessionId; + for (const key of MANAGED_KERNEL_ENV_KEYS) { + const value = patch[key]; + if (value !== null) env[key] = value; } return Object.keys(env).length > 0 ? env : undefined; } @@ -282,23 +318,44 @@ function attachOwner(session: PythonSession, sessionId: string, ownerId: string } } -async function acquireSession(sessionId: string, cwd: string, options: PythonExecutorOptions): Promise { - const existing = sessions.get(sessionId); +async function acquireSession( + sessionKey: string, + sessionId: string, + cwd: string, + options: PythonExecutorOptions, +): Promise { + const existing = sessions.get(sessionKey); if (existing) { attachOwner(existing, sessionId, options.kernelOwnerId); return existing; } - const kernel = await startKernel(cwd, options); - const session: PythonSession = { - sessionId, - kernel, - ownerIds: new Set(), - hasFallbackOwner: false, - queue: Promise.resolve(), - }; - attachOwner(session, sessionId, options.kernelOwnerId); - sessions.set(sessionId, session); - return session; + const starting = startingSessions.get(sessionKey); + if (starting) { + const session = await starting; + attachOwner(session, sessionId, options.kernelOwnerId); + return session; + } + const startup = (async () => { + const kernel = await startKernel(cwd, options); + const session: PythonSession = { + sessionKey, + sessionId, + cwd, + kernel, + ownerIds: new Set(), + hasFallbackOwner: false, + }; + sessions.set(sessionKey, session); + return session; + })(); + startingSessions.set(sessionKey, startup); + try { + const session = await startup; + attachOwner(session, sessionId, options.kernelOwnerId); + return session; + } finally { + if (startingSessions.get(sessionKey) === startup) startingSessions.delete(sessionKey); + } } async function replaceSessionKernel( @@ -311,52 +368,40 @@ async function replaceSessionKernel( await old .shutdown(remaining !== undefined ? { timeoutMs: Math.max(0, remaining) } : undefined) .catch(() => undefined); - if (sessions.get(session.sessionId) !== session) { + if (sessions.get(session.sessionKey) !== session) { throw new PythonExecutionCancelledError(false); } requireRemainingTimeoutMs(options.deadlineMs); const next = await startKernel(cwd, options); - if (sessions.get(session.sessionId) !== session) { + if (sessions.get(session.sessionKey) !== session) { await next.shutdown().catch(() => undefined); throw new PythonExecutionCancelledError(false); } session.kernel = next; } -async function resetSession(sessionId: string): Promise { - const existing = sessions.get(sessionId); +async function resetSession(sessionKey: string): Promise { + const existing = sessions.get(sessionKey) ?? (await startingSessions.get(sessionKey)?.catch(() => undefined)); if (!existing) return; - sessions.delete(sessionId); + sessions.delete(sessionKey); await existing.kernel.shutdown().catch(() => undefined); } -async function runQueued( - session: PythonSession, - options: Pick, - work: () => Promise, -): Promise { - const previous = session.queue; - const { promise: ourSlot, resolve: releaseSlot } = Promise.withResolvers(); - // Keep the queue chained even if WE bail out: future runs must still wait - // for `previous` to finish before they touch the kernel. - session.queue = previous.catch(() => undefined).then(() => ourSlot); - try { - await waitForPromiseWithCancellation( - previous.catch(() => undefined), - options, - ); - return await work(); - } finally { - releaseSlot(); - } -} - // --------------------------------------------------------------------------- // Public dispose entry points // --------------------------------------------------------------------------- export async function disposeAllKernelSessions(): Promise { + const pending = [...startingSessions.values()]; + startingSessions.clear(); + const started = await Promise.allSettled(pending); const all = [...sessions.entries()]; + for (const result of started) { + if (result.status !== "fulfilled") continue; + if (!all.some(([, session]) => session === result.value)) { + all.push([result.value.sessionKey, result.value]); + } + } for (const [id, session] of all) { if (sessions.get(id) === session) sessions.delete(id); } @@ -366,7 +411,12 @@ export async function disposeAllKernelSessions(): Promise { const result = results[i]; if (result.status === "fulfilled" && result.value?.confirmed !== false) continue; const reason = result.status === "rejected" ? result.reason : "not confirmed"; - logger.warn("Python kernel shutdown not confirmed", { sessionId: id, reason }); + logger.warn("Python kernel shutdown not confirmed", { + sessionId: session.sessionId, + sessionKey: id, + cwd: session.cwd, + reason, + }); if (!sessions.has(id)) sessions.set(id, session); } } @@ -382,7 +432,7 @@ export async function disposeKernelSessionsByOwner(ownerId: string): Promise session.kernel.shutdown())); for (let i = 0; i < toShutdown.length; i += 1) { @@ -393,8 +443,13 @@ export async function disposeKernelSessionsByOwner(ownerId: string): Promise { displayOutputs.push({ type: "status", event }); }); + const runId = `py-${crypto.randomUUID()}`; const unregisterBridge = options?.toolSession && options?.bridgeSessionId - ? registerPyToolBridge(options.bridgeSessionId, { + ? registerPyToolBridge(options.bridgeSessionId, runId, { toolSession: options.toolSession, signal: options.signal, emitStatus, @@ -436,6 +492,9 @@ async function executeWithKernel( try { executionTimeoutMs = requireRemainingTimeoutMs(deadlineMs); const result = await kernel.execute(code, { + cwd: options?.cwd, + env: buildKernelEnvPatch(options ?? {}), + id: runId, signal: options?.signal, timeoutMs: executionTimeoutMs, onChunk: text => sink.push(text), @@ -516,7 +575,7 @@ async function executePerCall(code: string, cwd: string, options: PythonExecutor } const kernel = await startKernel(cwd, options); try { - return await executeWithKernel(kernel, code, options); + return await executeWithKernel(kernel, code, { ...options, cwd: undefined }); } finally { await kernel.shutdown().catch(() => undefined); } @@ -524,42 +583,53 @@ async function executePerCall(code: string, cwd: string, options: PythonExecutor async function executeOnSession(code: string, cwd: string, options: PythonExecutorOptions): Promise { const sessionId = options.sessionId ?? `session:${cwd}`; + const sessionKey = buildSessionKey(sessionId, cwd); if (options.bridge && !options.bridgeSessionId) { options.bridgeSessionId = sessionId; } if (options.reset) { - await resetSession(sessionId); - } - const session = await acquireSession(sessionId, cwd, options); - return await runQueued(session, options, async () => { - if (options.signal?.aborted) { - throw new PythonExecutionCancelledError(isTimedOutCancellation(options.signal.reason, options.signal)); + if (resettingSessions.has(sessionKey)) { + throw new Error("Python kernel reset already in progress"); } - if (sessions.get(session.sessionId) !== session) { + resettingSessions.add(sessionKey); + try { + await resetSession(sessionKey); + } finally { + resettingSessions.delete(sessionKey); + } + } else if (resettingSessions.has(sessionKey)) { + throw new Error("Python kernel reset in progress"); + } + const session = await acquireSession(sessionKey, sessionId, cwd, options); + if (options.signal?.aborted) { + throw new PythonExecutionCancelledError(isTimedOutCancellation(options.signal.reason, options.signal)); + } + if (sessions.get(session.sessionKey) !== session) { + throw new PythonExecutionCancelledError(false); + } + if (!session.kernel.isAlive()) { + await replaceSessionKernel(session, cwd, options); + if (sessions.get(session.sessionKey) !== session) { throw new PythonExecutionCancelledError(false); } - if (!session.kernel.isAlive()) { - await replaceSessionKernel(session, cwd, options); - if (sessions.get(session.sessionId) !== session) { - throw new PythonExecutionCancelledError(false); - } + } + const runOptions = { ...options, cwd: undefined }; + try { + return await executeWithKernel(session.kernel, code, runOptions); + } catch (err) { + if (isCancellationError(err) || options.signal?.aborted) throw err; + if (session.kernel.isAlive()) throw err; + if (sessions.get(session.sessionKey) !== session) { + throw new PythonExecutionCancelledError(false); } - try { - return await executeWithKernel(session.kernel, code, options); - } catch (err) { - if (isCancellationError(err) || options.signal?.aborted) throw err; - if (session.kernel.isAlive()) throw err; - if (sessions.get(session.sessionId) !== session) { - throw new PythonExecutionCancelledError(false); - } - // Kernel died during execute. Replace it and retry once on a fresh one. - await replaceSessionKernel(session, cwd, options); - if (sessions.get(session.sessionId) !== session) { - throw new PythonExecutionCancelledError(false); - } - return await executeWithKernel(session.kernel, code, options); + // Shared kernels are keyed by cwd, so a dead kernel can be recreated in place + // without risking cross-directory state bleed. + await replaceSessionKernel(session, cwd, options); + if (sessions.get(session.sessionKey) !== session) { + throw new PythonExecutionCancelledError(false); } - }); + return await executeWithKernel(session.kernel, code, runOptions); + } } export async function executePythonWithKernel( @@ -571,10 +641,11 @@ export async function executePythonWithKernel( } export async function executePython(code: string, options?: PythonExecutorOptions): Promise { - const cwd = options?.cwd ?? getProjectDir(); + const cwd = normalizeSessionCwd(options?.cwd ?? getProjectDir()); const deadlineMs = getExecutionDeadlineMs(options); const executionOptions: PythonExecutorOptions = { ...(options ?? {}), + cwd, deadlineMs, }; diff --git a/packages/coding-agent/src/eval/py/kernel.ts b/packages/coding-agent/src/eval/py/kernel.ts index 07d2eb223..60b93c73c 100644 --- a/packages/coding-agent/src/eval/py/kernel.ts +++ b/packages/coding-agent/src/eval/py/kernel.ts @@ -51,7 +51,14 @@ const STARTUP_TIMEOUT_MS = 10_000; // kernel's state, so we only kill as a last-resort recovery path. const INTERRUPT_ESCALATION_MS = 5_000; +export type KernelRuntimeEnv = Record; + export interface KernelExecuteOptions { + id?: string; + /** Runtime working directory applied immediately before this request executes. */ + cwd?: string; + /** Managed runtime environment variables applied immediately before this request executes. */ + env?: KernelRuntimeEnv; signal?: AbortSignal; onChunk?: (text: string) => Promise | void; onDisplay?: (output: KernelDisplayOutput) => Promise | void; @@ -260,7 +267,7 @@ export class PythonKernel { throw new Error("Python kernel is not running"); } - const msgId = Snowflake.next(); + const msgId = options?.id ?? Snowflake.next(); const { promise, resolve } = Promise.withResolvers(); const pending: PendingExecution = { resolve, @@ -345,6 +352,8 @@ export class PythonKernel { const payload = JSON.stringify({ id: msgId, code, + cwd: options?.cwd, + env: options?.env, silent: options?.silent ?? false, storeHistory: options?.storeHistory ?? !(options?.silent ?? false), }); diff --git a/packages/coding-agent/src/eval/py/prelude.py b/packages/coding-agent/src/eval/py/prelude.py index abf08f9da..7858c3286 100644 --- a/packages/coding-agent/src/eval/py/prelude.py +++ b/packages/coding-agent/src/eval/py/prelude.py @@ -377,13 +377,20 @@ if "__omp_prelude_loaded__" not in globals(): return current + def _tool_proxy_from_env() -> tuple[str, str, str]: + base = os.environ.get("PI_TOOL_BRIDGE_URL") + token = os.environ.get("PI_TOOL_BRIDGE_TOKEN") + session = os.environ.get("PI_TOOL_BRIDGE_SESSION") + if not base or not token or not session: + raise RuntimeError("tool bridge is unavailable in this kernel") + return (base.rstrip("/"), token, session) + class _ToolCallable: """Invokes one host-side tool via the loopback HTTP bridge.""" - __slots__ = ("_proxy", "_name") + __slots__ = ("_name",) - def __init__(self, proxy: "_ToolProxy", name: str): - self._proxy = proxy + def __init__(self, name: str): self._name = name def __repr__(self) -> str: @@ -402,16 +409,19 @@ if "__omp_prelude_loaded__" not in globals(): merged.update(kwargs) if "_i" not in merged: merged["_i"] = "py prelude" + base, token, session = _tool_proxy_from_env() + _run_id_getter = globals().get("__omp_current_run_id__") + _run_id = _run_id_getter() if callable(_run_id_getter) else globals().get("__omp_run_id__") payload = json.dumps( - {"session": self._proxy._session, "name": self._name, "args": merged} + {"session": session, "run": _run_id, "name": self._name, "args": merged} ).encode("utf-8") req = urllib.request.Request( - f"{self._proxy._base}/v1/tool", + f"{base}/v1/tool", data=payload, method="POST", headers={ "Content-Type": "application/json", - "Authorization": f"Bearer {self._proxy._token}", + "Authorization": f"Bearer {token}", }, ) try: @@ -433,30 +443,18 @@ if "__omp_prelude_loaded__" not in globals(): class _ToolProxy: """`tool.(args)` proxy mirroring the JS runtime bridge.""" - __slots__ = ("_base", "_token", "_session") - - def __init__(self, base: str, token: str, session: str): - self._base = base.rstrip("/") - self._token = token - self._session = session + __slots__ = () def __getattr__(self, name: str) -> _ToolCallable: if name.startswith("_"): raise AttributeError(name) - return _ToolCallable(self, name) + return _ToolCallable(name) def __getitem__(self, name: str) -> _ToolCallable: - return _ToolCallable(self, name) + return _ToolCallable(name) def __repr__(self) -> str: - return f"" + session = os.environ.get("PI_TOOL_BRIDGE_SESSION") + return f"" if session else "" - if all( - _k in os.environ - for _k in ("PI_TOOL_BRIDGE_URL", "PI_TOOL_BRIDGE_TOKEN", "PI_TOOL_BRIDGE_SESSION") - ): - tool = _ToolProxy( - os.environ["PI_TOOL_BRIDGE_URL"], - os.environ["PI_TOOL_BRIDGE_TOKEN"], - os.environ["PI_TOOL_BRIDGE_SESSION"], - ) + tool = _ToolProxy() diff --git a/packages/coding-agent/src/eval/py/runner.py b/packages/coding-agent/src/eval/py/runner.py index 280590c6f..25bbc11d9 100644 --- a/packages/coding-agent/src/eval/py/runner.py +++ b/packages/coding-agent/src/eval/py/runner.py @@ -5,6 +5,7 @@ wrapper writes typed frames back. Host -> wrapper: {"id": str, "code": str, "silent": bool?, "storeHistory": bool?} + {"id": str, "code": str, "silent": bool?, "storeHistory": bool?, "cwd": str?, "env": dict?} {"type": "exit"} # graceful shutdown Wrapper -> host: @@ -27,6 +28,7 @@ from __future__ import annotations import asyncio import ast +import contextvars import base64 import builtins import inspect @@ -43,7 +45,7 @@ import threading import time import traceback from pathlib import Path -from typing import Any, Callable +from typing import Any # --------------------------------------------------------------------------- # Frame writer @@ -93,7 +95,7 @@ class _StreamProxy(io.TextIOBase): data = str(data) if not data: return 0 - rid = _STATE.current_id + rid = _CURRENT_RID.get() if rid is None: _RAW_STDERR.write(data) _RAW_STDERR.flush() @@ -112,7 +114,6 @@ class _StreamProxy(io.TextIOBase): class _RunnerState: def __init__(self) -> None: - self.current_id: str | None = None self.execution_count: int = 0 self.cancel_requested: bool = False # User globals — kept across requests when running in session mode. @@ -123,8 +124,11 @@ class _RunnerState: } self.last_install_marker: int = 0 self.loop: asyncio.AbstractEventLoop | None = None + self.active_executions: int = 0 +_CURRENT_RID: contextvars.ContextVar[str | None] = contextvars.ContextVar("omp_current_rid", default=None) + _STATE = _RunnerState() @@ -284,7 +288,7 @@ def cell_magic(name: str) -> Callable[[Callable[[str, str], Any]], Callable[[str def _emit_status(op: str, **data: Any) -> None: bundle = {"application/x-omp-status": {"op": op, **data}} - rid = _STATE.current_id + rid = _CURRENT_RID.get() if rid is None: return _emit({"type": "display", "id": rid, "bundle": bundle}) @@ -424,7 +428,7 @@ def _magic_reset(_args: str) -> None: def _magic_load(args: str) -> None: path = Path(os.path.expanduser(args.strip())) source = path.read_text(encoding="utf-8") - _emit({"type": "display", "id": _STATE.current_id, "bundle": {"text/plain": source}}) + _emit({"type": "display", "id": _CURRENT_RID.get(), "bundle": {"text/plain": source}}) _exec_source(source, _STATE.user_ns) @@ -622,7 +626,7 @@ def _mime_bundle(value: Any) -> dict: def _emit_display(bundle: dict, *, kind: str = "display") -> None: - rid = _STATE.current_id + rid = _CURRENT_RID.get() if rid is None: return _emit({"type": kind, "id": rid, "bundle": bundle}) @@ -681,6 +685,7 @@ def _install_builtins(ns: dict) -> None: ns["__omp_magic"] = __omp_magic ns["__omp_magic_cell"] = __omp_magic_cell ns["__omp_shell"] = __omp_shell + ns["__omp_current_run_id__"] = lambda: _CURRENT_RID.get() _install_builtins(_STATE.user_ns) @@ -694,25 +699,20 @@ _install_builtins(_STATE.user_ns) _TLA_FLAG = getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x2000) -def _get_event_loop() -> asyncio.AbstractEventLoop: - loop = _STATE.loop - if loop is None or loop.is_closed(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - _STATE.loop = loop - return loop +def _await_sync(coro) -> Any: + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if running_loop is not None and running_loop.is_running(): + raise RuntimeError("top-level await is not supported from synchronous magic execution") + return asyncio.run(coro) -def _run_compiled(code, ns: dict, *, want_value: bool) -> Any: - """Execute a code object, awaiting it if compiled as a coroutine. - - ``want_value`` is True for the trailing expression — we return ``eval``'s - result (or the awaited coroutine's value). For statement blocks the - return is always ``None``. - """ +def _run_compiled_sync(code, ns: dict, *, want_value: bool) -> Any: + """Synchronous execution path used by nested magic helpers.""" if code.co_flags & inspect.CO_COROUTINE: - coro = eval(code, ns) - result = _get_event_loop().run_until_complete(coro) + result = _await_sync(eval(code, ns)) return result if want_value else None if want_value: return eval(code, ns) @@ -720,15 +720,27 @@ def _run_compiled(code, ns: dict, *, want_value: bool) -> Any: return None -def _exec_source(source: str, ns: dict) -> None: - """Compile + execute ``source``; if the last node is an expression, route - its value through ``__omp_display`` so dataframes/figures render rich. - Top-level ``await`` / ``async for`` / ``async with`` is permitted; the - cell is driven through the runner's persistent event loop.""" - module = ast.parse(source, mode="exec") +async def _run_compiled_async(code, ns: dict, *, want_value: bool) -> Any: + """Execute a code object in the persistent event loop. + + Coroutine code is awaited in this task so top-level ``await`` interleaves + with sibling requests. Plain statement/expression code runs on the main + runner thread so SIGINT can interrupt it reliably. + """ + if code.co_flags & inspect.CO_COROUTINE: + result = await eval(code, ns) + return result if want_value else None + if want_value: + return eval(code, ns) + exec(code, ns) + return None + + +def _compile_source(source: str) -> tuple[Any, Any | None, bool]: + module = ast.parse(source, mode="exec") if not module.body: - return + return None, None, False last = module.body[-1] if isinstance(last, ast.Expr): @@ -737,14 +749,36 @@ def _exec_source(source: str, ns: dict) -> None: ast.copy_location(expr_module, last) body_code = compile(body_module, "", "exec", flags=_TLA_FLAG) expr_code = compile(expr_module, "", "eval", flags=_TLA_FLAG) - _run_compiled(body_code, ns, want_value=False) - value = _run_compiled(expr_code, ns, want_value=True) + return body_code, expr_code, True + + return compile(module, "", "exec", flags=_TLA_FLAG), None, False + + +def _exec_source(source: str, ns: dict) -> None: + """Synchronous source execution for legacy magic helpers.""" + body_code, expr_code, has_expr = _compile_source(source) + if body_code is None: + return + _run_compiled_sync(body_code, ns, want_value=False) + if has_expr and expr_code is not None: + value = _run_compiled_sync(expr_code, ns, want_value=True) if value is not None: __omp_display(value, kind="result") - return - code = compile(module, "", "exec", flags=_TLA_FLAG) - _run_compiled(code, ns, want_value=False) + +async def _exec_source_async(source: str, ns: dict) -> None: + """Compile + execute ``source``; if the last node is an expression, route + its value through ``__omp_display`` so dataframes/figures render rich. + Top-level ``await`` / ``async for`` / ``async with`` is permitted; awaited + regions yield to other requests in the runner's persistent event loop.""" + body_code, expr_code, has_expr = _compile_source(source) + if body_code is None: + return + await _run_compiled_async(body_code, ns, want_value=False) + if has_expr and expr_code is not None: + value = await _run_compiled_async(expr_code, ns, want_value=True) + if value is not None: + __omp_display(value, kind="result") # --------------------------------------------------------------------------- @@ -767,6 +801,46 @@ def _install_exec_sigint() -> None: pass +def _begin_exec_sigint() -> None: + _STATE.active_executions += 1 + _install_exec_sigint() + + +def _end_exec_sigint() -> None: + if _STATE.active_executions > 0: + _STATE.active_executions -= 1 + if _STATE.active_executions == 0: + _install_idle_sigint() + + +_MANAGED_ENV_KEYS = ( + "PI_SESSION_FILE", + "PI_ARTIFACTS_DIR", + "PI_TOOL_BRIDGE_URL", + "PI_TOOL_BRIDGE_TOKEN", + "PI_TOOL_BRIDGE_SESSION", +) + + +def _apply_request_runtime(req: dict) -> None: + cwd = req.get("cwd") + if isinstance(cwd, str) and cwd: + os.chdir(cwd) + try: + sys.path.remove(cwd) + except ValueError: + pass + sys.path.insert(0, cwd) + + env = req.get("env") + if isinstance(env, dict): + for key in _MANAGED_ENV_KEYS: + value = env.get(key) + if isinstance(value, str): + os.environ[key] = value + elif value is None: + os.environ.pop(key, None) + def _start_parent_watchdog() -> None: """Self-terminate when the host process dies. @@ -802,61 +876,72 @@ def _start_parent_watchdog() -> None: # --------------------------------------------------------------------------- -def _handle_request(req: dict) -> None: - if req.get("type") == "exit": - sys.exit(0) - +async def _handle_request_async(req: dict) -> None: rid = str(req.get("id")) - code = req.get("code", "") - _STATE.current_id = rid + token = _CURRENT_RID.set(rid) + _STATE.user_ns["__omp_run_id__"] = rid _STATE.cancel_requested = False _STATE.execution_count += 1 + execution_count = _STATE.execution_count _emit({"type": "started", "id": rid}) status: str = "ok" cancelled = False try: - transformed = transform_cell(code) - except SyntaxError as exc: - _emit_error(rid, exc) + try: + _apply_request_runtime(req) + transformed = transform_cell(req.get("code", "")) + except SyntaxError as exc: + _emit_error(rid, exc) + _emit({ + "type": "done", + "id": rid, + "status": "error", + "executionCount": execution_count, + "cancelled": False, + }) + return + except BaseException as exc: # noqa: BLE001 - runtime setup errors must settle the request + _emit_error(rid, exc) + _emit({ + "type": "done", + "id": rid, + "status": "error", + "executionCount": execution_count, + "cancelled": False, + }) + return + + _begin_exec_sigint() + try: + await _exec_source_async(transformed, _STATE.user_ns) + except KeyboardInterrupt: + cancelled = True + status = "error" + _emit_error(rid, KeyboardInterrupt("Execution interrupted")) + except SystemExit as exc: + status = "error" + _emit_error(rid, exc) + except BaseException as exc: # noqa: BLE001 - we want to surface every user error + status = "error" + _emit_error(rid, exc) + finally: + _end_exec_sigint() + try: + _flush_matplotlib_figures() + except Exception: + pass + _emit({ "type": "done", "id": rid, - "status": "error", - "executionCount": _STATE.execution_count, - "cancelled": False, + "status": status, + "executionCount": execution_count, + "cancelled": cancelled, }) - _STATE.current_id = None - return - - _install_exec_sigint() - try: - _exec_source(transformed, _STATE.user_ns) - except KeyboardInterrupt: - cancelled = True - status = "error" - _emit_error(rid, KeyboardInterrupt("Execution interrupted")) - except SystemExit: - raise - except BaseException as exc: # noqa: BLE001 - we want to surface every user error - status = "error" - _emit_error(rid, exc) finally: - _install_idle_sigint() - try: - _flush_matplotlib_figures() - except Exception: - pass - - _emit({ - "type": "done", - "id": rid, - "status": status, - "executionCount": _STATE.execution_count, - "cancelled": cancelled, - }) - _STATE.current_id = None + _CURRENT_RID.reset(token) def _emit_error(rid: str, exc: BaseException) -> None: @@ -875,16 +960,7 @@ def _emit_error(rid: str, exc: BaseException) -> None: # --------------------------------------------------------------------------- -def main() -> None: - sys.stdout = _StreamProxy("stdout") - sys.stderr = _StreamProxy("stderr") - _install_idle_sigint() - _start_parent_watchdog() - - stdin = sys.__stdin__ - if stdin is None: - return - +def _read_stdin(loop: asyncio.AbstractEventLoop, queue: asyncio.Queue, stdin) -> None: for raw_line in stdin: line = raw_line.strip() if not line: @@ -900,10 +976,52 @@ def main() -> None: "traceback": [], }) continue + loop.call_soon_threadsafe(queue.put_nowait, req) + loop.call_soon_threadsafe(queue.put_nowait, {"type": "exit"}) + + +async def _main_async() -> None: + sys.stdout = _StreamProxy("stdout") + sys.stderr = _StreamProxy("stderr") + _install_idle_sigint() + _start_parent_watchdog() + + stdin = sys.__stdin__ + if stdin is None: + return + + loop = asyncio.get_running_loop() + _STATE.loop = loop + queue: asyncio.Queue = asyncio.Queue() + reader = threading.Thread(target=_read_stdin, args=(loop, queue, stdin), name="omp-stdin-reader", daemon=True) + reader.start() + + tasks: set[asyncio.Task] = set() + def _task_done(task: asyncio.Task) -> None: + tasks.discard(task) try: - _handle_request(req) - except SystemExit: + exc = task.exception() + except asyncio.CancelledError: return + if exc is not None: + _emit_error("", exc) + try: + while True: + req = await queue.get() + if req.get("type") == "exit": + break + task = asyncio.create_task(_handle_request_async(req)) + tasks.add(task) + task.add_done_callback(_task_done) + finally: + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + +def main() -> None: + asyncio.run(_main_async()) if __name__ == "__main__": diff --git a/packages/coding-agent/src/eval/py/tool-bridge.ts b/packages/coding-agent/src/eval/py/tool-bridge.ts index 0cbfe8f3f..7c8f27d71 100644 --- a/packages/coding-agent/src/eval/py/tool-bridge.ts +++ b/packages/coding-agent/src/eval/py/tool-bridge.ts @@ -44,21 +44,23 @@ async function startServer(): Promise { return new Response("Forbidden", { status: 403 }); } - let body: { session?: unknown; name?: unknown; args?: unknown }; + let body: { session?: unknown; run?: unknown; name?: unknown; args?: unknown }; try { - body = (await req.json()) as { session?: unknown; name?: unknown; args?: unknown }; + body = (await req.json()) as { session?: unknown; run?: unknown; name?: unknown; args?: unknown }; } catch { return Response.json({ ok: false, error: "Invalid JSON body" }, { status: 400 }); } const sessionId = typeof body.session === "string" ? body.session : ""; + const runId = typeof body.run === "string" ? body.run : ""; const name = typeof body.name === "string" ? body.name : ""; - if (!sessionId || !name) { - return Response.json({ ok: false, error: "Missing session/name" }, { status: 400 }); + if (!sessionId || !runId || !name) { + return Response.json({ ok: false, error: "Missing session/run/name" }, { status: 400 }); } - const entry = registrations.get(sessionId); + const registrationKey = bridgeRegistrationKey(sessionId, runId); + const entry = registrations.get(registrationKey) ?? registrations.get(sessionId); if (!entry) { return Response.json( - { ok: false, error: `No active Python tool bridge session: ${sessionId}` }, + { ok: false, error: `No active Python tool bridge session: ${registrationKey}` }, { status: 200 }, ); } @@ -111,11 +113,16 @@ export async function ensurePyToolBridge(): Promise { * Register a tool session for the duration of one execution. The returned * function MUST be called to remove the entry once execution finishes. */ -export function registerPyToolBridge(sessionId: string, entry: PyToolBridgeEntry): () => void { - registrations.set(sessionId, entry); +function bridgeRegistrationKey(sessionId: string, runId: string): string { + return `${sessionId}:${runId}`; +} + +export function registerPyToolBridge(sessionId: string, runId: string, entry: PyToolBridgeEntry): () => void { + const key = bridgeRegistrationKey(sessionId, runId); + registrations.set(key, entry); return () => { - if (registrations.get(sessionId) === entry) { - registrations.delete(sessionId); + if (registrations.get(key) === entry) { + registrations.delete(key); } }; } diff --git a/packages/coding-agent/src/eval/session-id.ts b/packages/coding-agent/src/eval/session-id.ts new file mode 100644 index 000000000..d339b6a9c --- /dev/null +++ b/packages/coding-agent/src/eval/session-id.ts @@ -0,0 +1,8 @@ +import type { ToolSession } from "../tools"; + +export type EvalSessionSource = Pick; + +export function defaultEvalSessionId(session: EvalSessionSource): string { + const sessionFile = session.getSessionFile?.() ?? undefined; + return sessionFile ? `session:${sessionFile}:cwd:${session.cwd}` : `cwd:${session.cwd}`; +} diff --git a/packages/coding-agent/src/extensibility/extensions/runner.ts b/packages/coding-agent/src/extensibility/extensions/runner.ts index 69f885a79..43c3f05c7 100644 --- a/packages/coding-agent/src/extensibility/extensions/runner.ts +++ b/packages/coding-agent/src/extensibility/extensions/runner.ts @@ -462,7 +462,6 @@ export class ExtensionRunner { hasPendingMessages: () => this.#hasPendingMessagesFn(), shutdown: () => this.#shutdownHandler(), getSystemPrompt: () => this.#getSystemPromptFn(), - hasQueuedMessages: () => this.#hasPendingMessagesFn(), // deprecated alias }; } diff --git a/packages/coding-agent/src/extensibility/extensions/types.ts b/packages/coding-agent/src/extensibility/extensions/types.ts index 05be8ccc3..332427184 100644 --- a/packages/coding-agent/src/extensibility/extensions/types.ts +++ b/packages/coding-agent/src/extensibility/extensions/types.ts @@ -283,8 +283,6 @@ export interface ExtensionContext { shutdown(): void; /** Get the current effective system prompt. */ getSystemPrompt(): string[]; - /** @deprecated Use hasPendingMessages() instead */ - hasQueuedMessages(): boolean; } /** diff --git a/packages/coding-agent/src/hashline/anchors.ts b/packages/coding-agent/src/hashline/anchors.ts index a0977e8e1..d7f482cff 100644 --- a/packages/coding-agent/src/hashline/anchors.ts +++ b/packages/coding-agent/src/hashline/anchors.ts @@ -1,113 +1,104 @@ -import { formatCodeFrameLine } from "../tools/render-utils"; import { MISMATCH_CONTEXT } from "./constants"; -import { computeLineHash, describeAnchorExamples, HL_ANCHOR_RE_RAW, HL_BODY_SEP } from "./hash"; -import type { HashMismatch } from "./types"; +import { formatNumberedLine, HL_FILE_HASH_SEP, HL_FILE_PREFIX } from "./hash"; -const HL_HASH_HINT_RE = /^[a-z]{2}$/i; -const HL_ANCHOR_EXAMPLES = describeAnchorExamples("160"); -const PARSE_TAG_RE = new RegExp(`^${HL_ANCHOR_RE_RAW}`); +const LINE_REF_RE = /^\s*[>+\-*]*\s*(\d+)(?::.*)?\s*$/; export function formatFullAnchorRequirement(raw?: string): string { - const suffix = typeof raw === "string" ? raw.trim() : ""; - const hashOnlyHint = HL_HASH_HINT_RE.test(suffix) - ? ` It looks like you supplied only the hash suffix (${JSON.stringify(suffix)}). ` + - `Copy the full anchor exactly as shown (for example, "160${suffix}").` - : ""; const received = raw === undefined ? "" : ` Received ${JSON.stringify(raw)}.`; return ( - `the full anchor exactly as shown by read/search output ` + - `(line number + hash, for example ${HL_ANCHOR_EXAMPLES})${received}${hashOnlyHint}` + `a bare line number from read/search output plus the section header file hash ` + + `(for example ${HL_FILE_PREFIX}src/foo.ts${HL_FILE_HASH_SEP}1a2b and line "160")${received}` ); } -export function parseTag(ref: string): { line: number; hash: string } { - const match = ref.match(PARSE_TAG_RE); +export function parseTag(ref: string): { line: number } { + const match = ref.match(LINE_REF_RE); if (!match) { throw new Error(`Invalid line reference. Expected ${formatFullAnchorRequirement(ref)}.`); } const line = Number.parseInt(match[1], 10); if (line < 1) throw new Error(`Line number must be >= 1, got ${line} in "${ref}".`); - return { line, hash: match[2] }; + return { line }; } -function getMismatchDisplayLines(mismatches: HashMismatch[], fileLines: string[]): number[] { +export interface HashlineMismatchDetails { + path?: string; + expectedFileHash: string; + actualFileHash: string; + fileLines: string[]; + anchorLines?: readonly number[]; +} + +function getMismatchDisplayLines(anchorLines: readonly number[], fileLines: string[]): number[] { const displayLines = new Set(); - for (const mismatch of mismatches) { - const lo = Math.max(1, mismatch.line - MISMATCH_CONTEXT); - const hi = Math.min(fileLines.length, mismatch.line + MISMATCH_CONTEXT); + for (const line of anchorLines) { + if (line < 1 || line > fileLines.length) continue; + const lo = Math.max(1, line - MISMATCH_CONTEXT); + const hi = Math.min(fileLines.length, line + MISMATCH_CONTEXT); for (let lineNum = lo; lineNum <= hi; lineNum++) displayLines.add(lineNum); } return [...displayLines].sort((a, b) => a - b); } export class HashlineMismatchError extends Error { - readonly remaps: ReadonlyMap; + readonly path: string | undefined; + readonly expectedFileHash: string; + readonly actualFileHash: string; + readonly fileLines: string[]; + readonly anchorLines: readonly number[]; - constructor( - public readonly mismatches: HashMismatch[], - public readonly fileLines: string[], - ) { - super(HashlineMismatchError.formatMessage(mismatches, fileLines)); + constructor(details: HashlineMismatchDetails) { + super(HashlineMismatchError.formatMessage(details)); this.name = "HashlineMismatchError"; - - const remaps = new Map(); - for (const mismatch of mismatches) { - const actual = computeLineHash(mismatch.line, fileLines[mismatch.line - 1] ?? ""); - remaps.set(`${mismatch.line}${mismatch.expected}`, `${mismatch.line}${actual}`); - } - this.remaps = remaps; + this.path = details.path; + this.expectedFileHash = details.expectedFileHash; + this.actualFileHash = details.actualFileHash; + this.fileLines = details.fileLines; + this.anchorLines = details.anchorLines ?? []; } get displayMessage(): string { - return HashlineMismatchError.formatDisplayMessage(this.mismatches, this.fileLines); + return HashlineMismatchError.formatDisplayMessage({ + path: this.path, + expectedFileHash: this.expectedFileHash, + actualFileHash: this.actualFileHash, + fileLines: this.fileLines, + anchorLines: this.anchorLines, + }); } - private static rejectionHeader(mismatches: HashMismatch[]): string[] { - const noun = mismatches.length > 1 ? "anchors do" : "anchor does"; + static rejectionHeader(details: HashlineMismatchDetails): string[] { + const pathText = details.path ? ` for ${details.path}` : ""; return [ - `Edit rejected: ${mismatches.length} ${noun} not match the current file (marked *).`, - "The edit was NOT applied, please use the updated file content shown below, and issue another edit tool-call.", + `Edit rejected${pathText}: file changed between read and edit.`, + `Section is bound to ${HL_FILE_HASH_SEP}${details.expectedFileHash}, but the current file hashes to ${HL_FILE_HASH_SEP}${details.actualFileHash}; re-read and try again.`, ]; } - static formatDisplayMessage(mismatches: HashMismatch[], fileLines: string[]): string { - const mismatchSet = new Set(mismatches.map(m => m.line)); - const displayLines = getMismatchDisplayLines(mismatches, fileLines); - const width = displayLines.reduce((cur, n) => Math.max(cur, String(n).length), 0); - - const out = [...HashlineMismatchError.rejectionHeader(mismatches), ""]; - let previous = -1; - for (const lineNum of displayLines) { - if (previous !== -1 && lineNum > previous + 1) out.push("..."); - previous = lineNum; - const marker = mismatchSet.has(lineNum) ? "*" : " "; - out.push(formatCodeFrameLine(marker, lineNum, fileLines[lineNum - 1] ?? "", width)); - } - return out.join("\n"); + static formatDisplayMessage(details: HashlineMismatchDetails): string { + return HashlineMismatchError.formatMessage(details); } - static formatMessage(mismatches: HashMismatch[], fileLines: string[]): string { - const mismatchSet = new Set(mismatches.map(m => m.line)); - const lines = HashlineMismatchError.rejectionHeader(mismatches); + static formatMessage(details: HashlineMismatchDetails): string { + const anchorSet = new Set(details.anchorLines ?? []); + const lines = HashlineMismatchError.rejectionHeader(details); + const displayLines = getMismatchDisplayLines(details.anchorLines ?? [], details.fileLines); + if (displayLines.length === 0) return lines.join("\n"); + lines.push(""); let previous = -1; - for (const lineNum of getMismatchDisplayLines(mismatches, fileLines)) { + for (const lineNum of displayLines) { if (previous !== -1 && lineNum > previous + 1) lines.push("..."); previous = lineNum; - const text = fileLines[lineNum - 1] ?? ""; - const hash = computeLineHash(lineNum, text); - const marker = mismatchSet.has(lineNum) ? "*" : " "; - lines.push(`${marker}${lineNum}${hash}${HL_BODY_SEP}${text}`); + const text = details.fileLines[lineNum - 1] ?? ""; + const marker = anchorSet.has(lineNum) ? "*" : " "; + lines.push(`${marker}${formatNumberedLine(lineNum, text)}`); } return lines.join("\n"); } } -export function validateLineRef(ref: { line: number; hash: string }, fileLines: string[]): void { +export function validateLineRef(ref: { line: number }, fileLines: string[]): void { if (ref.line < 1 || ref.line > fileLines.length) { throw new Error(`Line ${ref.line} does not exist (file has ${fileLines.length} lines)`); } - const actualHash = computeLineHash(ref.line, fileLines[ref.line - 1] ?? ""); - if (actualHash !== ref.hash) { - throw new HashlineMismatchError([{ line: ref.line, expected: ref.hash, actual: actualHash }], fileLines); - } } diff --git a/packages/coding-agent/src/hashline/apply.ts b/packages/coding-agent/src/hashline/apply.ts index ea32131c3..7ea0282e8 100644 --- a/packages/coding-agent/src/hashline/apply.ts +++ b/packages/coding-agent/src/hashline/apply.ts @@ -1,8 +1,5 @@ -import { HashlineMismatchError } from "./anchors"; -import { RANGE_INTERIOR_HASH } from "./constants"; -import { computeLineHash } from "./hash"; -import { cloneCursor } from "./parser"; -import type { Anchor, HashlineApplyOptions, HashlineCursor, HashlineEdit, HashMismatch } from "./types"; +import { cloneCursor } from "./tokenizer"; +import type { Anchor, HashlineApplyOptions, HashlineCursor, HashlineEdit } from "./types"; export interface HashlineApplyResult { lines: string; @@ -43,26 +40,17 @@ function getHashlineEditAnchors(edit: HashlineEdit): Anchor[] { } /** - * Verify every anchor's hash. Any mismatch is reported as a `HashMismatch`; - * there is no auto-rebase. Callers are expected to surface mismatches as - * `HashlineMismatchError` so the model re-reads and re-anchors. + * Verify every anchored edit points at an existing line. File-version binding is + * checked once per section via the header hash before this function runs. */ -function validateHashlineAnchors(edits: HashlineEdit[], fileLines: string[]): HashMismatch[] { - const mismatches: HashMismatch[] = []; +function validateHashlineLineBounds(edits: HashlineEdit[], fileLines: string[]): void { for (const edit of edits) { for (const anchor of getHashlineEditAnchors(edit)) { if (anchor.line < 1 || anchor.line > fileLines.length) { throw new Error(`Line ${anchor.line} does not exist (file has ${fileLines.length} lines)`); } - if (anchor.hash === RANGE_INTERIOR_HASH) continue; - - const actualHash = computeLineHash(anchor.line, fileLines[anchor.line - 1] ?? ""); - if (actualHash === anchor.hash) continue; - - mismatches.push({ line: anchor.line, expected: anchor.hash, actual: actualHash }); } } - return mismatches; } function insertAtStart(fileLines: string[], lineOrigins: HashlineLineOrigin[], lines: string[]): void { @@ -287,15 +275,10 @@ function contiguousRange(start: number, count: number): number[] { return Array.from({ length: count }, (_, offset) => start + offset); } -function deleteEditForAutoAbsorbedLine( - line: number, - sourceLineNum: number, - index: number, - fileLines: string[], -): HashlineEdit { +function deleteEditForAutoAbsorbedLine(line: number, sourceLineNum: number, index: number): HashlineEdit { return { kind: "delete", - anchor: { line, hash: computeLineHash(line, fileLines[line - 1] ?? "") }, + anchor: { line }, lineNum: sourceLineNum, index, }; @@ -314,7 +297,7 @@ function cursorMatches(a: HashlineCursor, b: HashlineCursor): boolean { if (a.kind === "bof" || a.kind === "eof") return true; const aAnchor = (a as { anchor: Anchor }).anchor; const bAnchor = (b as { anchor: Anchor }).anchor; - return aAnchor.line === bAnchor.line && aAnchor.hash === bAnchor.hash; + return aAnchor.line === bAnchor.line; } /** @@ -606,13 +589,13 @@ function absorbReplacementBoundaryDuplicates( } for (const line of contiguousRange(startLine - safePrefixCount, safePrefixCount)) { - absorbed.push(deleteEditForAutoAbsorbedLine(line, group.sourceLineNum, nextSyntheticIndex++, fileLines)); + absorbed.push(deleteEditForAutoAbsorbedLine(line, group.sourceLineNum, nextSyntheticIndex++)); } for (let groupIndex = group.startIndex; groupIndex <= group.endIndex; groupIndex++) { absorbed.push(edits[groupIndex]); } for (const line of contiguousRange(endLine + 1, safeSuffixCount)) { - absorbed.push(deleteEditForAutoAbsorbedLine(line, group.sourceLineNum, nextSyntheticIndex++, fileLines)); + absorbed.push(deleteEditForAutoAbsorbedLine(line, group.sourceLineNum, nextSyntheticIndex++)); } index = group.endIndex; @@ -653,8 +636,7 @@ export function applyHashlineEdits( if (firstChangedLine === undefined || line < firstChangedLine) firstChangedLine = line; }; - const mismatches = validateHashlineAnchors(edits, fileLines); - if (mismatches.length > 0) throw new HashlineMismatchError(mismatches, fileLines); + validateHashlineLineBounds(edits, fileLines); const normalizedEdits = absorbReplacementBoundaryDuplicates(edits, fileLines, warnings, options); @@ -669,10 +651,9 @@ export function applyHashlineEdits( continue; } const nextLineNum = anchorLine + 1; - const nextContent = fileLines[nextLineNum - 1] ?? ""; edit.cursor = { kind: "before_anchor", - anchor: { line: nextLineNum, hash: computeLineHash(nextLineNum, nextContent) }, + anchor: { line: nextLineNum }, }; } @@ -711,6 +692,23 @@ export function applyHashlineEdits( } if (beforeLines.length === 0 && !deleteLine) continue; + const replaceMode = beforeLines.length > 0; + if (deleteLine && !replaceMode) { + const balance = computeDelimiterBalance([currentLine]); + const trimmedCurrentLine = currentLine.trim(); + const touchesStructuralBoundary = + trimmedCurrentLine.startsWith(")") || + trimmedCurrentLine.startsWith("]") || + trimmedCurrentLine.startsWith("}") || + trimmedCurrentLine.endsWith("(") || + trimmedCurrentLine.endsWith("[") || + trimmedCurrentLine.endsWith("{"); + if (balance.paren !== 0 || balance.bracket !== 0 || balance.brace !== 0 || touchesStructuralBoundary) { + warnings.push( + `Deleted line ${line} contains a structural bracket/brace boundary (${JSON.stringify(trimmedCurrentLine)}); verify the file is still balanced or use 'A:' to keep the boundary intact.`, + ); + } + } const replacement = deleteLine ? beforeLines : [...beforeLines, currentLine]; const origins = replacement.map((): HashlineLineOrigin => (deleteLine ? "replacement" : "insert")); if (!deleteLine) { diff --git a/packages/coding-agent/src/hashline/constants.ts b/packages/coding-agent/src/hashline/constants.ts index 0172a5290..99d28c094 100644 --- a/packages/coding-agent/src/hashline/constants.ts +++ b/packages/coding-agent/src/hashline/constants.ts @@ -1,9 +1,6 @@ /** Lines of context shown either side of a hash mismatch. */ export const MISMATCH_CONTEXT = 2; -/** Filler hash used for the interior of a multi-line range; not validated. */ -export const RANGE_INTERIOR_HASH = "**"; - /** Optional patch envelope start marker; silently consumed when present. */ export const BEGIN_PATCH_MARKER = "*** Begin Patch"; diff --git a/packages/coding-agent/src/hashline/diff-preview.ts b/packages/coding-agent/src/hashline/diff-preview.ts index 8b41950dc..624ebd39f 100644 --- a/packages/coding-agent/src/hashline/diff-preview.ts +++ b/packages/coding-agent/src/hashline/diff-preview.ts @@ -1,4 +1,3 @@ -import { computeLineHash, HL_BODY_SEP } from "./hash"; import type { CompactHashlineDiffOptions, CompactHashlineDiffPreview } from "./types"; export function buildCompactHashlineDiffPreview( @@ -11,7 +10,7 @@ export function buildCompactHashlineDiffPreview( // `generateDiffString` numbers `+` lines with the post-edit line number, // `-` lines with the pre-edit line number, and context lines with the - // pre-edit line number. To emit fresh anchors usable for follow-up edits, + // pre-edit line number. To emit fresh line numbers usable for follow-up edits, // we convert context-line numbers to post-edit positions by tracking the // running offset (added so far - removed so far) as we walk the diff. const formatted = lines.map(line => { @@ -28,13 +27,13 @@ export function buildCompactHashlineDiffPreview( switch (kind) { case "+": addedLines++; - return `+${lineNumber}${computeLineHash(lineNumber, content)}${HL_BODY_SEP}${content}`; + return `+${lineNumber}:${content}`; case "-": removedLines++; - return `-${lineNumber}--${HL_BODY_SEP}${content}`; + return `-${lineNumber}:${content}`; default: { const newLineNumber = lineNumber + addedLines - removedLines; - return ` ${newLineNumber}${computeLineHash(newLineNumber, content)}${HL_BODY_SEP}${content}`; + return ` ${newLineNumber}:${content}`; } } }); diff --git a/packages/coding-agent/src/hashline/diff.ts b/packages/coding-agent/src/hashline/diff.ts index 36987e8b5..a6e97691d 100644 --- a/packages/coding-agent/src/hashline/diff.ts +++ b/packages/coding-agent/src/hashline/diff.ts @@ -3,9 +3,10 @@ import { normalizeToLF, stripBom } from "../edit/normalize"; import { readEditFileText } from "../edit/read-file"; import { resolveToCwd } from "../tools/path-utils"; import { applyHashlineEdits } from "./apply"; -import { type HashlineInputSection, splitHashlineInputs } from "./input"; -import { parseHashline } from "./parser"; -import type { HashlineApplyOptions } from "./types"; +import { parseHashline } from "./executor"; +import { computeFileHash } from "./hash"; +import { splitHashlineInputs } from "./input"; +import type { HashlineApplyOptions, HashlineEdit, HashlineInputSection } from "./types"; async function readHashlineFileText( _file: { text(): Promise }, @@ -20,6 +21,28 @@ async function readHashlineFileText( } } +function hasAnchorScopedEdit(edits: readonly HashlineEdit[]): boolean { + return edits.some(edit => { + if (edit.kind === "delete") return true; + return edit.cursor.kind === "before_anchor" || edit.cursor.kind === "after_anchor"; + }); +} + +function validateSectionHash( + section: HashlineInputSection, + text: string, + edits: readonly HashlineEdit[], +): string | null { + if (section.fileHash === undefined) { + return hasAnchorScopedEdit(edits) + ? `Missing hashline file hash for anchored edit to ${section.path}; use \`¶${section.path}#hash\` from your latest read.` + : null; + } + const currentHash = computeFileHash(text); + if (currentHash === section.fileHash) return null; + return `Hashline file hash mismatch for ${section.path}: section is bound to #${section.fileHash}, but current file hashes to #${currentHash}; re-read and try again.`; +} + export async function computeHashlineSectionDiff( section: HashlineInputSection, cwd: string, @@ -30,7 +53,10 @@ export async function computeHashlineSectionDiff( const rawContent = await readHashlineFileText(Bun.file(absolutePath), absolutePath, section.path); const { text: content } = stripBom(rawContent); const normalized = normalizeToLF(content); - const result = applyHashlineEdits(normalized, parseHashline(section.diff), options); + const { edits } = parseHashline(section.diff); + const hashError = validateSectionHash(section, normalized, edits); + if (hashError) return { error: hashError }; + const result = applyHashlineEdits(normalized, edits, options); if (normalized === result.lines) return { error: `No changes would be made to ${section.path}.` }; return generateDiffString(normalized, result.lines); } catch (err) { diff --git a/packages/coding-agent/src/hashline/execute.ts b/packages/coding-agent/src/hashline/execute.ts index 1cf666712..92b20239f 100644 --- a/packages/coding-agent/src/hashline/execute.ts +++ b/packages/coding-agent/src/hashline/execute.ts @@ -13,13 +13,15 @@ import { enforcePlanModeWrite, resolvePlanPath } from "../tools/plan-mode-guard" import { HashlineMismatchError } from "./anchors"; import { applyHashlineEdits, type HashlineApplyResult } from "./apply"; import { buildCompactHashlineDiffPreview } from "./diff-preview"; -import { type HashlineInputSection, splitHashlineInputs } from "./input"; -import { parseHashlineWithWarnings } from "./parser"; +import { parseHashline } from "./executor"; +import { computeFileHash } from "./hash"; +import { splitHashlineInputs } from "./input"; import { tryRecoverHashlineWithCache } from "./recovery"; import type { ExecuteHashlineSingleOptions, HashlineApplyOptions, HashlineEdit, + HashlineInputSection, hashlineEditParamsSchema, } from "./types"; @@ -46,6 +48,27 @@ function hasAnchorScopedEdit(edits: HashlineEdit[]): boolean { }); } +function collectAnchorLines(edits: HashlineEdit[]): number[] { + const lines = new Set(); + for (const edit of edits) { + if (edit.kind === "delete") { + lines.add(edit.anchor.line); + continue; + } + if (edit.cursor.kind === "before_anchor" || edit.cursor.kind === "after_anchor") { + lines.add(edit.cursor.anchor.line); + } + } + return [...lines].sort((a, b) => a - b); +} + +function assertSectionHashAllowed(sectionPath: string, fileHash: string | undefined, edits: HashlineEdit[]): void { + if (fileHash !== undefined || !hasAnchorScopedEdit(edits)) return; + throw new Error( + `Missing hashline file hash for anchored edit to ${sectionPath}; use \`¶${sectionPath}#hash\` from your latest read.`, + ); +} + function formatNoChangeDiagnostic(pathText: string): string { return `Edits to ${pathText} resulted in no changes being made.`; } @@ -65,36 +88,48 @@ function getEditDetails(result: AgentToolResult): EditToolDetai } /** - * Apply hashline edits with anchor-stale recovery: on `HashlineMismatchError`, - * consult the read-snapshot cache for the file and 3-way-merge the edits onto - * the current text. If recovery succeeds, return the merged result with a - * synthetic warning. Otherwise re-throw the original mismatch error. + * Apply hashline edits with file-hash stale recovery. The section hash gates + * line-number edits against the version shown to the model; if the live file + * drifted, snapshot recovery attempts a strict 3-way merge. */ function applyHashlineEditsWithRecovery( session: ToolSession, absolutePath: string, + pathText: string, text: string, + fileHash: string | undefined, edits: HashlineEdit[], options: HashlineApplyOptions, ): HashlineApplyResult { - try { - return applyHashlineEdits(text, edits, options); - } catch (err) { - if (!(err instanceof HashlineMismatchError)) throw err; - const recovered = tryRecoverHashlineWithCache({ - cache: getFileReadCache(session), - absolutePath, - currentText: text, - edits, - options, - }); - if (!recovered) throw err; + if (fileHash === undefined) return applyHashlineEdits(text, edits, options); + + const currentHash = computeFileHash(text); + if (currentHash === fileHash) return applyHashlineEdits(text, edits, options); + + const cache = getFileReadCache(session); + const recovered = tryRecoverHashlineWithCache({ + cache, + absolutePath, + currentText: text, + fileHash, + edits, + options, + }); + if (recovered) { return { lines: recovered.lines, firstChangedLine: recovered.firstChangedLine, warnings: recovered.warnings, }; } + + throw new HashlineMismatchError({ + path: pathText, + expectedFileHash: fileHash, + actualFileHash: currentHash, + fileLines: text.split("\n"), + anchorLines: collectAnchorLines(edits), + }); } /** @@ -103,10 +138,11 @@ function applyHashlineEditsWithRecovery( * any changes in a multi-section batch. */ async function preflightHashlineSection(options: ExecuteHashlineSingleOptions & HashlineInputSection): Promise { - const { session, path: sectionPath, diff } = options; + const { session, path: sectionPath, fileHash, diff } = options; const absolutePath = resolvePlanPath(session, sectionPath); - const { edits } = parseHashlineWithWarnings(diff); + const { edits } = parseHashline(diff); + assertSectionHashAllowed(sectionPath, fileHash, edits); enforcePlanModeWrite(session, sectionPath, { op: "update" }); const source = await readHashlineFile(absolutePath, sectionPath); @@ -118,7 +154,9 @@ async function preflightHashlineSection(options: ExecuteHashlineSingleOptions & const result = applyHashlineEditsWithRecovery( session, absolutePath, + sectionPath, normalized, + source.exists ? fileHash : undefined, edits, getHashlineApplyOptions(session), ); @@ -131,6 +169,7 @@ async function executeHashlineSection( const { session, path: sourcePath, + fileHash, diff, signal, batchRequest, @@ -139,7 +178,8 @@ async function executeHashlineSection( } = options; const absolutePath = resolvePlanPath(session, sourcePath); - const { edits, warnings: parseWarnings } = parseHashlineWithWarnings(diff); + const { edits, warnings: parseWarnings } = parseHashline(diff); + assertSectionHashAllowed(sourcePath, fileHash, edits); enforcePlanModeWrite(session, sourcePath, { op: "update" }); const source = await readHashlineFile(absolutePath, sourcePath); @@ -152,7 +192,9 @@ async function executeHashlineSection( const result = applyHashlineEditsWithRecovery( session, absolutePath, + sourcePath, originalNormalized, + source.exists ? fileHash : undefined, edits, getHashlineApplyOptions(session), ); @@ -182,7 +224,10 @@ async function executeHashlineSection( // of the file: the model just received it back as the diff/preview. Cache // it so a follow-up edit anchored against this state can still recover // if the file is touched out-of-band before the next edit lands. - getFileReadCache(session).recordContiguous(absolutePath, 1, result.lines.split("\n")); + getFileReadCache(session).recordContiguous(absolutePath, 1, result.lines.split("\n"), { + fullText: result.lines, + fileHash: computeFileHash(result.lines), + }); const diffResult = generateDiffString(originalNormalized, result.lines); const meta = outputMeta() @@ -257,11 +302,31 @@ export async function executeHashlineSingle( * Path order is preserved by first occurrence. */ function mergeSamePathSections(sections: HashlineInputSection[]): HashlineInputSection[] { - const byPath = new Map(); + const byPath = new Map(); for (const section of sections) { const existing = byPath.get(section.path); - if (existing) existing.push(section.diff); - else byPath.set(section.path, [section.diff]); + if (existing) { + if ( + existing.fileHash !== undefined && + section.fileHash !== undefined && + existing.fileHash !== section.fileHash + ) { + throw new Error( + `Conflicting hashline file hashes for ${section.path}: #${existing.fileHash} and #${section.fileHash}. Re-read the file and retry with one current header.`, + ); + } + if (existing.fileHash === undefined && section.fileHash !== undefined) existing.fileHash = section.fileHash; + existing.diffs.push(section.diff); + continue; + } + byPath.set(section.path, { + ...(section.fileHash !== undefined ? { fileHash: section.fileHash } : {}), + diffs: [section.diff], + }); } - return Array.from(byPath, ([path, diffs]) => ({ path, diff: diffs.join("\n") })); + return Array.from(byPath, ([path, entry]) => ({ + path, + ...(entry.fileHash !== undefined ? { fileHash: entry.fileHash } : {}), + diff: entry.diffs.join("\n"), + })); } diff --git a/packages/coding-agent/src/hashline/executor.ts b/packages/coding-agent/src/hashline/executor.ts new file mode 100644 index 000000000..4c6849762 --- /dev/null +++ b/packages/coding-agent/src/hashline/executor.ts @@ -0,0 +1,239 @@ +import { ABORT_WARNING } from "./constants"; +import { HL_OP_CHARS, HL_OP_DELETE, HL_OP_INSERT_AFTER, HL_OP_INSERT_BEFORE, HL_OP_REPLACE } from "./hash"; +import { + cloneCursor, + type HashlineToken, + HashlineTokenizer, + isDeleteOpWithPayload, + type ParsedRange, +} from "./tokenizer"; +import type { Anchor, HashlineCursor, HashlineEdit } from "./types"; + +function validateRangeOrder(range: ParsedRange, lineNum: number): void { + if (range.end.line < range.start.line) { + throw new Error(`line ${lineNum}: range ${range.start.line}-${range.end.line} ends before it starts.`); + } +} + +function expandRange(range: ParsedRange): Anchor[] { + const anchors: Anchor[] = []; + for (let line = range.start.line; line <= range.end.line; line++) { + anchors.push({ line }); + } + return anchors; +} + +type PendingOp = + | { kind: "insert"; cursor: HashlineCursor; lineNum: number } + | { kind: "replace"; range: ParsedRange; lineNum: number }; + +interface Pending { + op: PendingOp; + payload: string[]; + pendingBlanks: number; +} + +/** + * Token-driven state machine that turns a stream of {@link HashlineToken}s + * into the flat list of {@link HashlineEdit}s applied downstream by the + * apply/diff layers. + * + * The executor owns: + * - the running edit index (kept monotonic across pending flushes), + * - the pending-payload buffer (lines accumulated for the most recently + * opened insert/replace op), + * - all parse-time diagnostics (range order, "delete with payload", + * orphan payload, unrecognized op), + * - the {@link terminated} flag set by `envelope-end`/`abort`. + * + * Tokens are dispatched in the order they arrive; the matching tokenizer + * supplies the line numbers carried inside each token so diagnostics line + * up with the source. + */ +export class HashlineExecutor { + #edits: HashlineEdit[] = []; + #warnings: string[] = []; + #editIndex = 0; + #pending: Pending | undefined; + #terminated = false; + + /** True once an `envelope-end` or `abort` token has been observed. */ + get terminated(): boolean { + return this.#terminated; + } + + /** + * Consume one token. After `terminated` flips true subsequent feeds + * are silently ignored so callers can keep draining their tokenizer + * without explicit early-exit guards. + */ + feed(token: HashlineToken): void { + if (this.#terminated) return; + + switch (token.kind) { + case "envelope-begin": + return; + case "envelope-end": + this.#terminated = true; + return; + case "abort": + this.#warnings.push(ABORT_WARNING); + this.#terminated = true; + return; + case "header": + this.#flushPending(false); + return; + case "blank": + if (this.#pending) this.#pending.pendingBlanks++; + return; + case "payload": + this.#handlePayload(token.text, token.lineNum); + return; + case "op-delete": + this.#flushPending(false); + if (token.trailingPayload) { + throw new Error( + `line ${token.lineNum}: ${HL_OP_DELETE} deletes only. Payload is forbidden after ${HL_OP_DELETE}; use ${HL_OP_REPLACE} to replace.`, + ); + } + validateRangeOrder(token.range, token.lineNum); + for (const anchor of expandRange(token.range)) { + this.#edits.push({ kind: "delete", anchor, lineNum: token.lineNum, index: this.#editIndex++ }); + } + return; + case "op-insert": + this.#flushPending(false); + this.#pending = { + op: { kind: "insert", cursor: token.cursor, lineNum: token.lineNum }, + payload: [token.inlineBody ?? ""], + pendingBlanks: 0, + }; + return; + case "op-replace": + this.#flushPending(false); + validateRangeOrder(token.range, token.lineNum); + this.#pending = { + op: { kind: "replace", range: token.range, lineNum: token.lineNum }, + payload: [token.inlineBody ?? ""], + pendingBlanks: 0, + }; + return; + } + } + + /** + * Flush any open pending op (including its trailing blank lines, which + * are payload-significant) and return the accumulated edits and + * warnings. The executor is single-use; reset() is required for reuse. + */ + end(): { edits: HashlineEdit[]; warnings: string[] } { + this.#flushPending(true); + return { edits: this.#edits, warnings: this.#warnings }; + } + + /** Reset to a fresh state so the same instance can drive another parse. */ + reset(): void { + this.#edits = []; + this.#warnings = []; + this.#editIndex = 0; + this.#pending = undefined; + this.#terminated = false; + } + + #handlePayload(text: string, lineNum: number): void { + if (this.#pending) { + this.#flushPendingBlanks(); + this.#pending.payload.push(text); + return; + } + + // Whitespace-only payload outside any pending op is a visual + // separator (matches the legacy outer-loop isBlankLine skip); + // only fully-empty lines arrive as `blank` tokens. + if (text.trim().length === 0) return; + // Orphan payload outside any pending op: pick the most specific + // diagnostic so the model sees the actionable hint. + if (isDeleteOpWithPayload(text)) { + throw new Error( + `line ${lineNum}: ${HL_OP_DELETE} deletes only. Payload is forbidden after ${HL_OP_DELETE}; use ${HL_OP_REPLACE} to replace.`, + ); + } + + const firstChar = text[0]; + const startsWithOp = firstChar !== undefined && HL_OP_CHARS.includes(firstChar); + if (startsWithOp || firstChar === "-" || firstChar === "@" || firstChar === "«" || firstChar === "»") { + throw new Error( + `line ${lineNum}: unrecognized op. Use LINE${HL_OP_INSERT_BEFORE} (insert before), LINE${HL_OP_INSERT_AFTER} (insert after), LINE${HL_OP_REPLACE} / A-B${HL_OP_REPLACE} (replace), or LINE${HL_OP_DELETE} / A-B${HL_OP_DELETE} (delete). ` + + `Got ${JSON.stringify(text)}.`, + ); + } + + throw new Error( + `line ${lineNum}: payload line has no preceding ${HL_OP_INSERT_BEFORE}, ${HL_OP_INSERT_AFTER}, ${HL_OP_REPLACE}, or ${HL_OP_DELETE} operation. ` + + `Got ${JSON.stringify(text)}.`, + ); + } + + #flushPendingBlanks(): void { + if (!this.#pending) return; + for (let count = 0; count < this.#pending.pendingBlanks; count++) this.#pending.payload.push(""); + this.#pending.pendingBlanks = 0; + } + + #flushPending(includeTrailingBlanks: boolean): void { + const pending = this.#pending; + if (!pending) return; + if (includeTrailingBlanks) this.#flushPendingBlanks(); + + const { op, payload } = pending; + const linesToInsert = payload; + + if (op.kind === "insert") { + for (const text of linesToInsert) { + this.#edits.push({ + kind: "insert", + cursor: cloneCursor(op.cursor), + text, + lineNum: op.lineNum, + index: this.#editIndex++, + }); + } + } else { + for (const text of linesToInsert) { + this.#edits.push({ + kind: "insert", + cursor: { kind: "before_anchor", anchor: { ...op.range.start } }, + text, + lineNum: op.lineNum, + index: this.#editIndex++, + }); + } + for (const anchor of expandRange(op.range)) { + this.#edits.push({ kind: "delete", anchor, lineNum: op.lineNum, index: this.#editIndex++ }); + } + } + + this.#pending = undefined; + } +} + +/** + * Drive a full hashline diff through the tokenizer + executor pipeline and + * return the resulting edits plus any parse-time warnings. This is the + * convenience entry point most callers want; reach for {@link + * HashlineTokenizer}/{@link HashlineExecutor} directly only when you need + * streaming feeds, cross-section state, or custom token handling. + */ +export function parseHashline(diff: string): { edits: HashlineEdit[]; warnings: string[] } { + const tokenizer = new HashlineTokenizer(); + const executor = new HashlineExecutor(); + const drain = (tokens: HashlineToken[]): void => { + for (const token of tokens) { + if (executor.terminated) return; + executor.feed(token); + } + }; + drain(tokenizer.feed(diff)); + drain(tokenizer.end()); + return executor.end(); +} diff --git a/packages/coding-agent/src/hashline/grammar.lark b/packages/coding-agent/src/hashline/grammar.lark index 1d7c74ecd..417230a99 100644 --- a/packages/coding-agent/src/hashline/grammar.lark +++ b/packages/coding-agent/src/hashline/grammar.lark @@ -3,22 +3,21 @@ begin_patch: "*** Begin Patch" LF end_patch: "*** End Patch" LF? hunk: update_hunk -update_hunk: "$HFILE$" filename LF line_op* +update_hunk: "$HFILE$" filename ("#" file_hash)? LF line_op* -filename: /(.+)/ +filename: /([^\s#]+)/ +file_hash: /[0-9a-f]{4}/ -line_op: insert_before | insert_after | replace | blank -insert_before: anchor "$HOP_INSERT_BEFORE$" inline_body LF payload* - | anchor "$HOP_INSERT_BEFORE$" LF payload+ -insert_after: anchor "$HOP_INSERT_AFTER$" inline_body LF payload* - | anchor "$HOP_INSERT_AFTER$" LF payload+ +line_op: insert_before | insert_after | replace | delete +insert_before: anchor "$HOP_INSERT_BEFORE$" inline_body? LF payload* +insert_after: anchor "$HOP_INSERT_AFTER$" inline_body? LF payload* replace: range "$HOP_REPLACE$" inline_body? LF payload* +delete: range "$HOP_DELETE$" LF inline_body: /[^\n]+/ -payload: /[^$HFILE$\n][^\n]*/ LF | LF -blank: LF +payload: /(.*)/ LF anchor: LID | "EOF" | "BOF" range: LID ("-" LID)? -LID: /[1-9]\d*$HFMT$/ +LID: /[1-9]\d*/ %import common.LF diff --git a/packages/coding-agent/src/hashline/hash.ts b/packages/coding-agent/src/hashline/hash.ts index d86f83787..c04531a3e 100644 --- a/packages/coding-agent/src/hashline/hash.ts +++ b/packages/coding-agent/src/hashline/hash.ts @@ -3,70 +3,54 @@ * and prompt helpers. */ -import bigrams from "./bigrams.json" with { type: "json" }; +const regexEscape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); /** - * 647 single-token BPE bigrams for hashline anchors. Every entry tokenizes as - * exactly one token in modern BPE vocabularies (cl100k / o200k / Claude family), - * so a hashline anchor built from one bigram is exactly 1 token. - * - * This is the complete set of 2-letter lowercase combinations that are single - * tokens — the 29 missing combinations are rare-letter pairs (q/x/z heavy) - * that no major BPE vocabulary merges into a single token. - * - * Order is stable forever — changing it would invalidate every saved - * `LINE+ID` reference in transcripts and prompts. - */ -export const HL_BIGRAMS: readonly string[] = bigrams; - -export const HL_BIGRAMS_COUNT = HL_BIGRAMS.length; - -/** - * Decoration prefix that may precede a `LINE+HASH` anchor in tool output: + * Decoration prefix that may precede a line number in tool output: * `>` (context line in grep), `+` (added line in diff), `-` (removed line), * `*` (match line). Any combination, in any order, surrounded by optional - * whitespace. Output formatters emit at most one decoration per anchor; the - * regex stays liberal because anchor-ref parsers accept whatever the model - * echoes back. + * whitespace. Output formatters emit at most one decoration per line; the + * parser stays liberal because it accepts whatever the model echoes back. */ export const HL_ANCHOR_DECORATION_RE_RAW = `\\s*[>+\\-*]*\\s*`; -/** - * Capture-group regex source for a decorated `LINE+HASH` anchor. Group 1 - * captures the line number (digits only); group 2 captures the hash. The - * source is intentionally unanchored — anchoring with `^` (or composing into a - * larger pattern) is the caller's responsibility. - */ -export const HL_ANCHOR_RE_RAW = `${HL_ANCHOR_DECORATION_RE_RAW}(\\d+)([a-z]{2})`; +/** Capture-group regex source for a decorated bare line-number anchor. */ +export const HL_ANCHOR_RE_RAW = `${HL_ANCHOR_DECORATION_RE_RAW}(\\d+)`; + +/** Bare positive line-number Lid (no decorations, no captures, no anchors). */ +export const HL_LINE_RE_RAW = `[1-9]\\d*`; + +/** Capture-group form of {@link HL_LINE_RE_RAW}. */ +export const HL_LINE_CAPTURE_RE_RAW = `([1-9]\\d*)`; + +/** Four-hex-character file hash carried by a hashline section header. */ +export const HL_FILE_HASH_RE_RAW = `[0-9a-f]{4}`; + +/** Capture-group form of {@link HL_FILE_HASH_RE_RAW}. */ +export const HL_FILE_HASH_CAPTURE_RE_RAW = `(${HL_FILE_HASH_RE_RAW})`; + +/** Separator between a hashline file path and its file hash. */ +export const HL_FILE_HASH_SEP = "#"; + +/** Separator between a line number and displayed line content in hashline mode. */ +export const HL_LINE_BODY_SEP = ":"; + +/** Regex-escaped form of {@link HL_LINE_BODY_SEP}, safe for embedding inside a regex. */ +export const HL_LINE_BODY_SEP_RE_RAW = regexEscape(HL_LINE_BODY_SEP); /** - * Bare `LINE+HASH` Lid (no decorations, no captures, no anchors). Use for - * embedding inside larger patterns where the line+hash unit appears as a - * literal (e.g. range bounds, alternation arms, op-line heuristics). + * Representative file hashes for use in user-facing error messages and prompt + * examples. */ -export const HL_HASH_RE_RAW = `[1-9]\\d*[a-z]{2}`; - -/** - * Capture-group form of {@link HL_HASH_RE_RAW}: group 1 captures the - * line number, group 2 captures the hash. - */ -export const HL_HASH_CAPTURE_RE_RAW = `([1-9]\\d*)([a-z]{2})`; - -/** Width of a hash in display characters. */ -export const HL_HASH_WIDTH = 2; - -/** - * Representative hash suffixes for use in user-facing error messages and - * prompt examples. - */ -export const HL_HASH_EXAMPLES = ["sr", "ab", "th"] as const; +export const HL_FILE_HASH_EXAMPLES = ["1a2b", "3c4d", "9f3e"] as const; /** * Format a comma-separated list of example anchors with an optional line-number - * prefix, quoted for inclusion in error messages: `"160sr", "160ab", "160th"`. + * prefix, quoted for inclusion in error messages: `"160", "42", "7"`. */ export function describeAnchorExamples(linePrefix = ""): string { - return HL_HASH_EXAMPLES.map(e => `"${linePrefix}${e}"`).join(", "); + const examples = linePrefix ? [linePrefix, `${linePrefix.slice(0, -1) || "4"}2`, "7"] : ["160", "42", "7"]; + return examples.map(e => `"${e}"`).join(", "); } /** @@ -76,23 +60,21 @@ export function describeAnchorExamples(linePrefix = ""): string { */ export function resolveHashlineGrammarPlaceholders(grammar: string): string { return grammar - .replaceAll("$HFMT$", "[a-z]{2}") + .replaceAll("$HFMT$", "") + .replaceAll("$HFILE_HASH$", HL_FILE_HASH_RE_RAW) + .replaceAll("$HFILE_HASH_SEP$", HL_FILE_HASH_SEP) .replaceAll("$HOP_INSERT_BEFORE$", HL_OP_INSERT_BEFORE) .replaceAll("$HOP_INSERT_AFTER$", HL_OP_INSERT_AFTER) .replaceAll("$HOP_REPLACE$", HL_OP_REPLACE) + .replaceAll("$HOP_DELETE$", HL_OP_DELETE) .replaceAll("$HOP_CHARS$", HL_OP_CHARS) .replaceAll("$HFILE$", HL_FILE_PREFIX); } -/** @deprecated Use {@link resolveHashlineGrammarPlaceholders}. */ -export const resolveLarkLidPlaceholders = resolveHashlineGrammarPlaceholders; - -const regexEscape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - /** - * Hashline edit input markers. File section headers start with {@link HL_FILE_PREFIX}; * op lines have an `ANCHOR[INLINE_PAYLOAD]` shape, where SIGIL is one of - * {@link HL_OP_INSERT_BEFORE}, {@link HL_OP_INSERT_AFTER}, or {@link HL_OP_REPLACE}. + * {@link HL_OP_INSERT_BEFORE}, {@link HL_OP_INSERT_AFTER}, {@link HL_OP_REPLACE}, + * or {@link HL_OP_DELETE}. * Multi-line payloads follow on subsequent lines as verbatim file content with no * per-line marker. * @@ -101,74 +83,46 @@ const regexEscape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, */ export const HL_OP_INSERT_BEFORE = "↑"; export const HL_OP_INSERT_AFTER = "↓"; -export const HL_OP_REPLACE = "→"; +export const HL_OP_REPLACE = ":"; +export const HL_OP_DELETE = "!"; /** All hashline edit op sigils, concatenated for fast membership tests. */ -export const HL_OP_CHARS = `${HL_OP_INSERT_BEFORE}${HL_OP_INSERT_AFTER}${HL_OP_REPLACE}`; +export const HL_OP_CHARS = `${HL_OP_INSERT_BEFORE}${HL_OP_INSERT_AFTER}${HL_OP_REPLACE}${HL_OP_DELETE}`; /** Hashline edit file section header marker. */ export const HL_FILE_PREFIX = "¶"; -/** Stable separator for read/search/hashline display output. Intentionally not configurable. */ -export const HL_BODY_SEP = "|"; - -/** Regex-escaped form of {@link HL_BODY_SEP}, safe for embedding inside a regex. */ -export const HL_BODY_SEP_RE_RAW = regexEscape(HL_BODY_SEP); - -/** - * Compute a 2-character hash of a single line via xxHash32 mod 647 over - * {@link HL_BIGRAMS}. The hash depends only on the line's content (after - * stripping CR and trailing whitespace); the `idx` parameter is accepted - * for call-site symmetry with line numbers but is intentionally unused so - * that anchors remain stable across line shifts caused by sibling edits. - * - * The line input should not include a trailing newline. - */ -export function computeLineHash(idx: number, line: string): string { - void idx; - line = line.replace(/\r/g, "").trimEnd(); - // Seed is fixed so the hash depends only on line content. Earlier we mixed - // in `idx` for blank/punctuation-only lines, but that meant any line shift - // (e.g. from a sibling edit in the same batch) invalidated anchors whose - // content had not changed. Identical blank lines are intentionally allowed - // to collide — the edit op's line number disambiguates them. - return HL_BIGRAMS[Bun.hash.xxHash32(line, 0) % HL_BIGRAMS_COUNT]; +function normalizeFileHashText(text: string): string { + return text + .replace(/\r/g, "") + .split("\n") + .map(line => line.trimEnd()) + .join("\n"); } /** - * Formats an anchor reference given a line number and its text. - * Returns `LINE+ID` (e.g., `42sr`) — no separator between - * number and hash. + * Compute the 4-hex-character hash carried by a hashline section header. + * The hash normalizes CR characters and trailing whitespace before hashing so + * platform line endings and display-trimmed lines do not invalidate anchors. */ -export function formatLineHash(line: number, lines: string): string { - return `${line}${computeLineHash(line, lines)}`; +export function computeFileHash(text: string): string { + const normalized = normalizeFileHashText(text); + const low16 = Bun.hash.xxHash32(normalized, 0) & 0xffff; + return low16.toString(16).padStart(4, "0"); } -/** - * Formats a single line with a hashline anchor. - * Returns `LINE+ID|TEXT` (e.g., `42sr|function hi() {`, `3ab|}`). - */ -export function formatHashLine(lineNumber: number, line: string): string { - return `${lineNumber}${computeLineHash(lineNumber, line)}${HL_BODY_SEP}${line}`; +/** Format a hashline section header for a file path and file hash. */ +export function formatHashlineHeader(filePath: string, fileHash: string): string { + return `${HL_FILE_PREFIX}${filePath}${HL_FILE_HASH_SEP}${fileHash}`; } -/** - * Format file text with hashline prefixes for display. - * - * Each line becomes `LINE+ID|TEXT` where LINENUM is 1-indexed. - * No padding on line numbers; pipe separator between anchor and content. - * - * @param text - Raw file text string - * @param startLine - First line number (1-indexed, defaults to 1) - * @returns Formatted string with one hashline-prefixed line per input line - * - * @example - * ``` - * formatHashLines("function hi() {\n return;\n}") - * // "1bm|function hi() {\n2er| return;\n3ab|}" - * ``` - */ -export function formatHashLines(text: string, startLine = 1): string { +/** Formats a single numbered line as `LINE:TEXT`. */ +export function formatNumberedLine(lineNumber: number, line: string): string { + return `${lineNumber}${HL_LINE_BODY_SEP}${line}`; +} + +/** Format file text with hashline-mode line-number prefixes for display. */ +export function formatNumberedLines(text: string, startLine = 1): string { const lines = text.split("\n"); - return lines.map((line, i) => formatHashLine(startLine + i, line)).join("\n"); + return lines.map((line, i) => formatNumberedLine(startLine + i, line)).join("\n"); } diff --git a/packages/coding-agent/src/hashline/index.ts b/packages/coding-agent/src/hashline/index.ts index da6ecb341..1e3c264a0 100644 --- a/packages/coding-agent/src/hashline/index.ts +++ b/packages/coding-agent/src/hashline/index.ts @@ -4,10 +4,11 @@ export * from "./constants"; export * from "./diff"; export * from "./diff-preview"; export * from "./execute"; +export * from "./executor"; export * from "./hash"; export * from "./input"; -export * from "./parser"; export * from "./prefixes"; export * from "./recovery"; export * from "./stream"; +export * from "./tokenizer"; export * from "./types"; diff --git a/packages/coding-agent/src/hashline/input.ts b/packages/coding-agent/src/hashline/input.ts index 81faf6c52..7f71a6beb 100644 --- a/packages/coding-agent/src/hashline/input.ts +++ b/packages/coding-agent/src/hashline/input.ts @@ -1,13 +1,10 @@ import * as path from "node:path"; -import { ABORT_MARKER, BEGIN_PATCH_MARKER, END_PATCH_MARKER } from "./constants"; -import { HL_FILE_PREFIX } from "./hash"; -import { isHashlineOpLineText } from "./parser"; -import type { SplitHashlineOptions } from "./types"; +import { HL_FILE_HASH_SEP, HL_FILE_PREFIX } from "./hash"; +import { HashlineTokenizer } from "./tokenizer"; +import type { HashlineInputSection, SplitHashlineOptions } from "./types"; -export interface HashlineInputSection { - path: string; - diff: string; -} +// Pure classification — single shared tokenizer is safe. +const TOKENIZER = new HashlineTokenizer(); function unquoteHashlinePath(pathText: string): string { if (pathText.length < 2) return pathText; @@ -25,27 +22,30 @@ function normalizeHashlinePath(rawPath: string, cwd?: string): string { return isWithinCwd ? relative || "." : unquoted; } +/** + * Parse a `¶PATH[#hash]` header line. Returns `null` for lines that do not + * begin with the `¶` prefix; throws the existing "Input header must be …" + * error when a `¶`-prefixed line fails the strict shape (so malformed paths + * surface immediately instead of being silently re-classified as payload). + */ function parseHashlineHeaderLine(line: string, cwd?: string): HashlineInputSection | null { const trimmed = line.trimEnd(); if (!trimmed.startsWith(HL_FILE_PREFIX)) return null; - // Strip a run of leading header markers so canonical `¶PATH` and - // runaway-prefix forms like `¶¶PATH` / `¶¶¶PATH` route to the same file. - let prefixEnd = 0; - while (prefixEnd < trimmed.length && trimmed[prefixEnd] === HL_FILE_PREFIX) prefixEnd++; - const rest = trimmed.slice(prefixEnd); - if (rest.trim().length === 0) { - throw new Error(`Input header "${HL_FILE_PREFIX}" is empty; provide a file path.`); + + const token = TOKENIZER.tokenize(trimmed); + if (token.kind !== "header") { + throw new Error( + `Input header must be ${HL_FILE_PREFIX}PATH or ${HL_FILE_PREFIX}PATH${HL_FILE_HASH_SEP}HASH with a 4-hex file hash; got ${JSON.stringify(trimmed)}.`, + ); } - const parsedPath = normalizeHashlinePath(rest, cwd); + + const parsedPath = normalizeHashlinePath(token.path, cwd); if (parsedPath.length === 0) { throw new Error(`Input header "${HL_FILE_PREFIX}" is empty; provide a file path.`); } - return { path: parsedPath, diff: "" }; -} - -function isPatchEnvelopeMarker(line: string): boolean { - const trimmed = line.trimEnd(); - return trimmed === BEGIN_PATCH_MARKER || trimmed === END_PATCH_MARKER; + return token.fileHash !== undefined + ? { path: parsedPath, fileHash: token.fileHash, diff: "" } + : { path: parsedPath, diff: "" }; } function stripLeadingBlankLines(input: string): string { @@ -53,7 +53,7 @@ function stripLeadingBlankLines(input: string): string { const lines = stripped.split("\n"); while (lines.length > 0) { const head = lines[0].replace(/\r$/, ""); - if (head.trim().length === 0 || head.trimEnd() === BEGIN_PATCH_MARKER) { + if (head.trim().length === 0 || TOKENIZER.tokenize(head).kind === "envelope-begin") { lines.shift(); continue; } @@ -64,7 +64,7 @@ function stripLeadingBlankLines(input: string): string { export function containsRecognizableHashlineOperations(input: string): boolean { for (const line of input.split(/\r?\n/)) { - if (isHashlineOpLineText(line)) return true; + if (TOKENIZER.isOp(line)) return true; } return false; } @@ -82,7 +82,7 @@ function normalizeFallbackInput(input: string, options: SplitHashlineOptions): s return `${HL_FILE_PREFIX}${fallbackPath}\n${input}`; } -export function splitHashlineInput(input: string, options: SplitHashlineOptions = {}): { path: string; diff: string } { +export function splitHashlineInput(input: string, options: SplitHashlineOptions = {}): HashlineInputSection { const [section] = splitHashlineInputs(input, options); return section; } @@ -95,33 +95,42 @@ export function splitHashlineInputs(input: string, options: SplitHashlineOptions if (parseHashlineHeaderLine(firstLine, options.cwd) === null) { const preview = JSON.stringify(firstLine.slice(0, 120)); throw new Error( - `input must begin with "${HL_FILE_PREFIX}PATH" on the first non-blank line; got: ${preview}. ` + - `Example: "${HL_FILE_PREFIX}src/foo.ts" then edit ops.`, + `input must begin with "${HL_FILE_PREFIX}PATH${HL_FILE_HASH_SEP}HASH" on the first non-blank line for anchored edits; got: ${preview}. ` + + `Example: "${HL_FILE_PREFIX}src/foo.ts${HL_FILE_HASH_SEP}1a2b" then edit ops.`, ); } const sections: HashlineInputSection[] = []; - let currentPath = ""; + let current: HashlineInputSection | undefined; let currentLines: string[] = []; const flush = () => { - if (currentPath.length === 0) return; + if (!current) return; const hasOps = currentLines.some(line => line.trim().length > 0); - if (hasOps) sections.push({ path: currentPath, diff: currentLines.join("\n") }); + if (hasOps) sections.push({ ...current, diff: currentLines.join("\n") }); currentLines = []; }; for (const line of lines) { - if (line.trimEnd() === END_PATCH_MARKER || line.trimEnd() === ABORT_MARKER) break; - if (isPatchEnvelopeMarker(line)) continue; - const header = parseHashlineHeaderLine(line, options.cwd); - if (header !== null) { - flush(); - currentPath = header.path; - currentLines = []; - } else { - currentLines.push(line); + const trimmed = line.trimEnd(); + const token = TOKENIZER.tokenize(line); + if (token.kind === "envelope-end" || token.kind === "abort") break; + if (token.kind === "envelope-begin") continue; + + // Route every `¶`-prefixed line through parseHashlineHeaderLine so + // malformed headers still raise the strict "Input header must be …" + // diagnostic (the tokenizer alone would silently classify them as + // payload). + if (trimmed.startsWith(HL_FILE_PREFIX)) { + const header = parseHashlineHeaderLine(line, options.cwd); + if (header !== null) { + flush(); + current = header; + currentLines = []; + continue; + } } + currentLines.push(line); } flush(); return sections; diff --git a/packages/coding-agent/src/hashline/parser.ts b/packages/coding-agent/src/hashline/parser.ts deleted file mode 100644 index eae14a6d7..000000000 --- a/packages/coding-agent/src/hashline/parser.ts +++ /dev/null @@ -1,251 +0,0 @@ -import { ABORT_MARKER, ABORT_WARNING, BEGIN_PATCH_MARKER, END_PATCH_MARKER, RANGE_INTERIOR_HASH } from "./constants"; -import { - describeAnchorExamples, - HL_FILE_PREFIX, - HL_HASH_CAPTURE_RE_RAW, - HL_HASH_RE_RAW, - HL_OP_CHARS, - HL_OP_INSERT_AFTER, - HL_OP_INSERT_BEFORE, - HL_OP_REPLACE, -} from "./hash"; -import type { Anchor, HashlineCursor, HashlineEdit } from "./types"; - -const regexEscape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -const OP_CHARS_ESCAPED = regexEscape(HL_OP_CHARS); - -// Leniently accept anchors copied from read/search output: -// - optional leading line-marker decoration (`*`, `>`, `+`, `-`) -// - the required `LINE+HASH` -// - an optional trailing `|TEXT` body so users can paste a full -// `LINE+HASH|TEXT` line verbatim. -const LID_CAPTURE_RE = new RegExp(`^\\s*[>+\\-*]*\\s*${HL_HASH_CAPTURE_RE_RAW}(?:\\|.*)?\\s*$`); - -// Pre-op anchor part for insert ops: leading decoration, then a LID or -// BOF/EOF, then optional `|TEXT` paste decoration. The decoration MUST NOT -// contain any op sigil so the op-line regex below knows where the anchor part -// ends. Trailing `\s*` allows space between the anchor and the op sigil. -const INSERT_ANCHOR_PART_RE_RAW = `\\s*[>+\\-*]*\\s*(?:${HL_HASH_RE_RAW}|BOF|EOF)(?:\\|[^${OP_CHARS_ESCAPED}\\n]*)?\\s*`; - -// Pre-op range part for the replace op: optional decoration + LID, then an -// optional `-LID` end, then optional trailing `|TEXT` paste decoration. The -// `-` is the range separator; `|TEXT` between bounds is unsupported (TEXT may -// contain `-`), trailing decoration after the full range is still tolerated. -const RANGE_PART_RE_RAW = `\\s*[>+\\-*]*\\s*${HL_HASH_RE_RAW}(?:-${HL_HASH_RE_RAW})?(?:\\|[^${OP_CHARS_ESCAPED}\\n]*)?\\s*`; - -// Op lines place the operator AFTER the anchor/range. Group 1 captures the -// anchor (or range) part; group 2 captures the optional inline payload that -// follows the op sigil on the same line, with trailing whitespace eaten. -const INSERT_BEFORE_OP_RE = new RegExp(`^(${INSERT_ANCHOR_PART_RE_RAW})${regexEscape(HL_OP_INSERT_BEFORE)}(.*?)\\s*$`); -const INSERT_AFTER_OP_RE = new RegExp(`^(${INSERT_ANCHOR_PART_RE_RAW})${regexEscape(HL_OP_INSERT_AFTER)}(.*?)\\s*$`); -const REPLACE_OP_RE = new RegExp(`^(${RANGE_PART_RE_RAW})${regexEscape(HL_OP_REPLACE)}(.*?)\\s*$`); - -// Range parser: a bare `LINE+HASH` or `LINE+HASH-LINE+HASH` with optional -// leading decoration and optional trailing `|TEXT` paste decoration. Captures -// 1/2 = start line/hash, 3/4 = optional end line/hash. -const RANGE_PARSE_RE = new RegExp( - `^\\s*[>+\\-*]*\\s*${HL_HASH_CAPTURE_RE_RAW}(?:-${HL_HASH_CAPTURE_RE_RAW})?(?:\\|.*)?\\s*$`, -); - -function parseLid(raw: string, lineNum: number): Anchor { - const match = LID_CAPTURE_RE.exec(raw); - if (!match) { - throw new Error( - `line ${lineNum}: expected a full anchor such as ${describeAnchorExamples("119")}; ` + - `got ${JSON.stringify(raw)}.`, - ); - } - return { line: Number.parseInt(match[1], 10), hash: match[2] }; -} - -interface ParsedRange { - start: Anchor; - end: Anchor; -} - -function parseRange(raw: string, lineNum: number): ParsedRange { - const match = RANGE_PARSE_RE.exec(raw); - if (!match) { - throw new Error( - `line ${lineNum}: range must be ANCHOR or ANCHOR-ANCHOR (one dash, no spaces); ` + - `got ${JSON.stringify(raw)}.`, - ); - } - const start: Anchor = { line: Number.parseInt(match[1], 10), hash: match[2] }; - const end: Anchor = match[3] !== undefined ? { line: Number.parseInt(match[3], 10), hash: match[4] } : { ...start }; - if (end.line < start.line) { - throw new Error( - `line ${lineNum}: range ${start.line}${start.hash}-${end.line}${end.hash} ends before it starts.`, - ); - } - if (end.line === start.line && end.hash !== start.hash) { - throw new Error( - `line ${lineNum}: range ${start.line}${start.hash}-${end.line}${end.hash} uses two different hashes for the same line.`, - ); - } - return { start, end }; -} - -function expandRange(range: ParsedRange): Anchor[] { - const anchors: Anchor[] = []; - for (let line = range.start.line; line <= range.end.line; line++) { - const hash = - line === range.start.line ? range.start.hash : line === range.end.line ? range.end.hash : RANGE_INTERIOR_HASH; - anchors.push({ line, hash }); - } - return anchors; -} - -// `BOF`/`EOF` with optional leading decoration and optional `|TEXT` trailing -// paste decoration. The token is recognized verbatim; any `|TEXT` is discarded. -const BOF_RE = /^\s*[>+\-*]*\s*BOF(?:\|[^\n]*)?\s*$/; -const EOF_RE = /^\s*[>+\-*]*\s*EOF(?:\|[^\n]*)?\s*$/; - -function parseInsertTarget(raw: string, lineNum: number, kind: "before" | "after"): HashlineCursor { - if (BOF_RE.test(raw)) return { kind: "bof" }; - if (EOF_RE.test(raw)) return { kind: "eof" }; - const cursorKind = kind === "before" ? "before_anchor" : "after_anchor"; - return { kind: cursorKind, anchor: parseLid(raw, lineNum) }; -} - -function isEnvelopeOrAbortMarkerLine(line: string): boolean { - const trimmed = line.trimEnd(); - return trimmed === BEGIN_PATCH_MARKER || trimmed === END_PATCH_MARKER || trimmed === ABORT_MARKER; -} - -export function isHashlineOpLineText(line: string): boolean { - return INSERT_BEFORE_OP_RE.test(line) || INSERT_AFTER_OP_RE.test(line) || REPLACE_OP_RE.test(line); -} - -function isPayloadTerminatorLine(line: string): boolean { - if (line.startsWith(HL_FILE_PREFIX)) return true; - if (isHashlineOpLineText(line)) return true; - return isEnvelopeOrAbortMarkerLine(line); -} - -export function cloneCursor(cursor: HashlineCursor): HashlineCursor { - if (cursor.kind === "before_anchor") return { kind: "before_anchor", anchor: { ...cursor.anchor } }; - if (cursor.kind === "after_anchor") return { kind: "after_anchor", anchor: { ...cursor.anchor } }; - return cursor; -} - -function collectPayload( - lines: string[], - startIndex: number, - opLineNum: number, - requirePayload: boolean, -): { payload: string[]; nextIndex: number } { - const payload: string[] = []; - let index = startIndex; - while (index < lines.length) { - const line = lines[index]; - if (isPayloadTerminatorLine(line)) break; - payload.push(line); - index++; - } - if (payload.length === 0 && requirePayload) { - throw new Error( - `line ${opLineNum}: ${HL_OP_INSERT_BEFORE} and ${HL_OP_INSERT_AFTER} operations require at least one verbatim payload line.`, - ); - } - return { payload, nextIndex: index }; -} - -export function parseHashline(diff: string): HashlineEdit[] { - return parseHashlineWithWarnings(diff).edits; -} - -export function parseHashlineWithWarnings(diff: string): { edits: HashlineEdit[]; warnings: string[] } { - const edits: HashlineEdit[] = []; - const warnings: string[] = []; - const lines = diff.split(/\r?\n/); - if (diff.endsWith("\n") && lines.at(-1) === "") lines.pop(); - let editIndex = 0; - - const pushInsert = (cursor: HashlineCursor, text: string, lineNum: number) => { - edits.push({ kind: "insert", cursor: cloneCursor(cursor), text, lineNum, index: editIndex++ }); - }; - - for (let i = 0; i < lines.length; ) { - const lineNum = i + 1; - const line = lines[i]; - - if (line.trim().length === 0) { - i++; - continue; - } - if (line === END_PATCH_MARKER) { - break; - } - if (line === ABORT_MARKER) { - warnings.push(ABORT_WARNING); - break; - } - if (line === BEGIN_PATCH_MARKER) { - i++; - continue; - } - - const insertBeforeMatch = INSERT_BEFORE_OP_RE.exec(line); - if (insertBeforeMatch) { - const cursor = parseInsertTarget(insertBeforeMatch[1], lineNum, "before"); - const inlineBody = insertBeforeMatch[2].length > 0 ? insertBeforeMatch[2] : undefined; - const { payload, nextIndex } = collectPayload(lines, i + 1, lineNum, inlineBody === undefined); - if (inlineBody !== undefined) pushInsert(cursor, inlineBody, lineNum); - for (const text of payload) pushInsert(cursor, text, lineNum); - i = nextIndex; - continue; - } - - const insertAfterMatch = INSERT_AFTER_OP_RE.exec(line); - if (insertAfterMatch) { - const cursor = parseInsertTarget(insertAfterMatch[1], lineNum, "after"); - const inlineBody = insertAfterMatch[2].length > 0 ? insertAfterMatch[2] : undefined; - const { payload, nextIndex } = collectPayload(lines, i + 1, lineNum, inlineBody === undefined); - if (inlineBody !== undefined) pushInsert(cursor, inlineBody, lineNum); - for (const text of payload) pushInsert(cursor, text, lineNum); - i = nextIndex; - continue; - } - - const replaceMatch = REPLACE_OP_RE.exec(line); - if (replaceMatch) { - const range = parseRange(replaceMatch[1], lineNum); - const inlineBody = replaceMatch[2].length > 0 ? replaceMatch[2] : undefined; - const { payload, nextIndex } = collectPayload(lines, i + 1, lineNum, false); - const allPayload = inlineBody !== undefined ? [inlineBody, ...payload] : payload; - if (allPayload.length > 0) { - for (const text of allPayload) { - edits.push({ - kind: "insert", - cursor: { kind: "before_anchor", anchor: { ...range.start } }, - text, - lineNum, - index: editIndex++, - }); - } - } - for (const anchor of expandRange(range)) { - edits.push({ kind: "delete", anchor, lineNum, index: editIndex++ }); - } - i = nextIndex; - continue; - } - - const firstChar = line[0]; - const startsWithOp = firstChar !== undefined && HL_OP_CHARS.includes(firstChar); - if (startsWithOp || /^[-@«»\u2254\u00A7]/u.test(line)) { - throw new Error( - `line ${lineNum}: unrecognized op. Use ANCHOR${HL_OP_INSERT_BEFORE} (insert before), ANCHOR${HL_OP_INSERT_AFTER} (insert after), or A-B${HL_OP_REPLACE} (replace/delete). ` + - `Got ${JSON.stringify(line)}.`, - ); - } - - throw new Error( - `line ${lineNum}: payload line has no preceding ${HL_OP_INSERT_BEFORE}, ${HL_OP_INSERT_AFTER}, or ${HL_OP_REPLACE} operation. ` + - `Got ${JSON.stringify(line)}.`, - ); - } - - return { edits, warnings }; -} diff --git a/packages/coding-agent/src/hashline/prefixes.ts b/packages/coding-agent/src/hashline/prefixes.ts index e24e67b82..057870154 100644 --- a/packages/coding-agent/src/hashline/prefixes.ts +++ b/packages/coding-agent/src/hashline/prefixes.ts @@ -1,8 +1,6 @@ -import { HL_BODY_SEP_RE_RAW } from "./hash"; - -const HL_OUTPUT_PREFIX_SEPARATOR_RE = `[:${HL_BODY_SEP_RE_RAW}]`; -const HL_PREFIX_RE = new RegExp(`^\\s*(?:>>>|>>)?\\s*(?:[+*]\\s*)?\\d+[a-z]{2}${HL_OUTPUT_PREFIX_SEPARATOR_RE}`); -const HL_PREFIX_PLUS_RE = new RegExp(`^\\s*(?:>>>|>>)?\\s*\\+\\s*\\d+[a-z]{2}${HL_OUTPUT_PREFIX_SEPARATOR_RE}`); +const HL_PREFIX_RE = /^\s*(?:>>>|>>)?\s*(?:[+*-]\s*)?\d+:/; +const HL_PREFIX_PLUS_RE = /^\s*(?:>>>|>>)?\s*\+\s*\d+:/; +const HL_HEADER_RE = /^\s*¶\S+#[0-9a-f]{4}\s*$/; const DIFF_PLUS_RE = /^[+](?![+])/; const READ_TRUNCATION_NOTICE_RE = /^\[(?:Showing lines \d+-\d+ of \d+|\d+ more lines? in (?:file|\S+))\b.*\bUse :L?\d+/; @@ -20,12 +18,14 @@ function stripLeadingHashlinePrefixes(line: string): string { // 5. Read-output prefix stripping // // When a model echoes back content from a `read` or `search` response, every -// line is prefixed with either a hashline tag (`123ab|`) or, for diff-style -// echoes, a leading `+`. These helpers detect that and recover the raw text. +// line is prefixed with either a hashline-mode line number (`123:`) or, for +// diff-style echoes, a leading `+`. These helpers detect that and recover the +// raw text. // ─────────────────────────────────────────────────────────────────────────── type LinePrefixStats = { nonEmpty: number; + headerCount: number; hashPrefixCount: number; diffPlusHashPrefixCount: number; diffPlusCount: number; @@ -35,6 +35,7 @@ type LinePrefixStats = { function collectLinePrefixStats(lines: string[]): LinePrefixStats { const stats: LinePrefixStats = { nonEmpty: 0, + headerCount: 0, hashPrefixCount: 0, diffPlusHashPrefixCount: 0, diffPlusCount: 0, @@ -47,6 +48,11 @@ function collectLinePrefixStats(lines: string[]): LinePrefixStats { stats.truncationNoticeCount++; continue; } + if (HL_HEADER_RE.test(line)) { + stats.nonEmpty++; + stats.headerCount++; + continue; + } stats.nonEmpty++; if (HL_PREFIX_RE.test(line)) stats.hashPrefixCount++; if (HL_PREFIX_PLUS_RE.test(line)) stats.diffPlusHashPrefixCount++; @@ -59,7 +65,8 @@ export function stripNewLinePrefixes(lines: string[]): string[] { const stats = collectLinePrefixStats(lines); if (stats.nonEmpty === 0) return lines; - const stripHash = stats.hashPrefixCount > 0 && stats.hashPrefixCount === stats.nonEmpty; + const contentLineCount = stats.nonEmpty - stats.headerCount; + const stripHash = contentLineCount > 0 && stats.hashPrefixCount === contentLineCount; const stripPlus = !stripHash && stats.diffPlusHashPrefixCount === 0 && @@ -69,7 +76,7 @@ export function stripNewLinePrefixes(lines: string[]): string[] { if (!stripHash && !stripPlus && stats.diffPlusHashPrefixCount === 0) return lines; return lines - .filter(line => !READ_TRUNCATION_NOTICE_RE.test(line)) + .filter(line => !READ_TRUNCATION_NOTICE_RE.test(line) && !(stripHash && HL_HEADER_RE.test(line))) .map(line => { if (stripHash) return stripLeadingHashlinePrefixes(line); if (stripPlus) return line.replace(DIFF_PLUS_RE, ""); @@ -83,8 +90,11 @@ export function stripNewLinePrefixes(lines: string[]): string[] { export function stripHashlinePrefixes(lines: string[]): string[] { const stats = collectLinePrefixStats(lines); if (stats.nonEmpty === 0) return lines; - if (stats.hashPrefixCount !== stats.nonEmpty) return lines; - return lines.filter(line => !READ_TRUNCATION_NOTICE_RE.test(line)).map(line => stripLeadingHashlinePrefixes(line)); + const contentLineCount = stats.nonEmpty - stats.headerCount; + if (contentLineCount === 0 || stats.hashPrefixCount !== contentLineCount) return lines; + return lines + .filter(line => !READ_TRUNCATION_NOTICE_RE.test(line) && !HL_HEADER_RE.test(line)) + .map(line => stripLeadingHashlinePrefixes(line)); } /** diff --git a/packages/coding-agent/src/hashline/recovery.ts b/packages/coding-agent/src/hashline/recovery.ts index c5e147820..b1c818a9d 100644 --- a/packages/coding-agent/src/hashline/recovery.ts +++ b/packages/coding-agent/src/hashline/recovery.ts @@ -1,15 +1,15 @@ import * as Diff from "diff"; import { generateDiffString } from "../edit/diff"; -import type { FileReadCache } from "../edit/file-read-cache"; -import { HashlineMismatchError } from "./anchors"; +import type { FileReadCache, FileReadSnapshot } from "../edit/file-read-cache"; import { applyHashlineEdits, type HashlineApplyResult } from "./apply"; -import { computeLineHash } from "./hash"; -import type { Anchor, HashlineApplyOptions, HashlineEdit } from "./types"; +import { computeFileHash } from "./hash"; +import type { HashlineApplyOptions, HashlineEdit } from "./types"; export interface HashlineRecoveryArgs { cache: FileReadCache; absolutePath: string; currentText: string; + fileHash: string; edits: HashlineEdit[]; options: HashlineApplyOptions; } @@ -20,76 +20,28 @@ export interface HashlineRecoveryResult { warnings: string[]; } -// Anchors are line-precise; never let Diff.applyPatch slide a hunk onto a -// duplicate closer 100+ lines away. If the snapshot-based replay does not -// align by exact line number, refuse and let the model re-read. +// Section hashes are line-precise; never let Diff.applyPatch slide a hunk onto a +// duplicate closer 100+ lines away. If snapshot replay does not align exactly, +// refuse and let the model re-read. const HASHLINE_RECOVERY_FUZZ_FACTOR = 0; -const HASHLINE_RECOVERY_WARNING = - "Recovered from stale anchors using a previous read snapshot (file changed externally between read and edit)."; - -/** Collect every line anchor an edit batch depends on. */ -function collectEditAnchors(edits: HashlineEdit[]): Anchor[] { - const anchors: Anchor[] = []; - for (const edit of edits) { - if (edit.kind === "delete") { - anchors.push(edit.anchor); - continue; - } - const cursor = edit.cursor; - if (cursor.kind === "before_anchor" || cursor.kind === "after_anchor") { - anchors.push(cursor.anchor); - } - } - return anchors; -} - -/** - * Attempt to recover from a `HashlineMismatchError` by replaying the edits - * against a cached pre-edit snapshot of the file and 3-way-merging the result - * onto the current on-disk content. Returns `null` when no recovery is - * possible — callers should propagate the original mismatch error in that - * case. - * - * Recovery is gated on a strict precondition: every line the model anchored - * MUST be present in the cached snapshot AND its content MUST hash to the - * model-supplied hash. This prevents 3-way merges from silently sliding onto - * the wrong site when only tangential parts of the file went stale. - */ -export function tryRecoverHashlineWithCache(args: HashlineRecoveryArgs): HashlineRecoveryResult | null { - const { cache, absolutePath, currentText, edits, options } = args; - const snapshot = cache.get(absolutePath); - if (!snapshot || snapshot.lines.size === 0) return null; - - // Precondition: the model's anchors must be vouched-for by the cache. If - // even one anchored line is missing from the snapshot, or its cached - // content hashes to a different value than the model supplied, refuse — - // any merge from here is a guess. - const anchors = collectEditAnchors(edits); - for (const anchor of anchors) { - const cachedLine = snapshot.lines.get(anchor.line); - if (cachedLine === undefined) return null; - if (computeLineHash(anchor.line, cachedLine) !== anchor.hash) return null; - } - - const overlaid = currentText.split("\n"); - let maxCachedLine = 0; - for (const lineNum of snapshot.lines.keys()) { - if (lineNum > maxCachedLine) maxCachedLine = lineNum; - } - while (overlaid.length < maxCachedLine) overlaid.push(""); - for (const [lineNum, content] of snapshot.lines) { - overlaid[lineNum - 1] = content; - } - const previousText = overlaid.join("\n"); - if (previousText === currentText) return null; +const HASHLINE_RECOVERY_EXTERNAL_WARNING = + "Recovered from a stale file hash using a previous read snapshot (file changed externally between read and edit)."; +const HASHLINE_RECOVERY_SESSION_CHAIN_WARNING = + "Recovered from a stale file hash using an earlier in-session snapshot (the file hash advanced after a prior edit in this session)."; +function applyEditsToSnapshot( + previousText: string, + currentText: string, + edits: HashlineEdit[], + options: HashlineApplyOptions, + recoveryWarning: string, +): HashlineRecoveryResult | null { let applied: HashlineApplyResult; try { applied = applyHashlineEdits(previousText, edits, options); - } catch (err) { - if (err instanceof HashlineMismatchError) return null; - throw err; + } catch { + return null; } if (applied.lines === previousText) return null; @@ -98,11 +50,9 @@ export function tryRecoverHashlineWithCache(args: HashlineRecoveryArgs): Hashlin if (typeof merged !== "string" || merged === currentText) return null; const mergedDiff = generateDiffString(currentText, merged); - // Only surface the recovery warning when the merge actually changed - // something visible. A no-op merge (e.g. trailing-newline only) is noise. const hasNetChange = mergedDiff.firstChangedLine !== undefined; const recoveryWarnings = hasNetChange - ? [HASHLINE_RECOVERY_WARNING, ...(applied.warnings ?? [])] + ? [recoveryWarning, ...(applied.warnings ?? [])] : [...(applied.warnings ?? [])]; return { @@ -111,3 +61,45 @@ export function tryRecoverHashlineWithCache(args: HashlineRecoveryArgs): Hashlin warnings: recoveryWarnings, }; } + +function buildSparseOverlayText(currentText: string, snapshotLines: ReadonlyMap): string { + const overlaid = currentText.split("\n"); + let maxCachedLine = 0; + for (const lineNum of snapshotLines.keys()) { + if (lineNum > maxCachedLine) maxCachedLine = lineNum; + } + while (overlaid.length < maxCachedLine) overlaid.push(""); + for (const [lineNum, content] of snapshotLines) { + overlaid[lineNum - 1] = content; + } + return overlaid.join("\n"); +} + +function isHeadSnapshot(head: FileReadSnapshot | null, snapshot: FileReadSnapshot): boolean { + return head === snapshot; +} + +function resolveRecoveryWarning(head: FileReadSnapshot | null, snapshot: FileReadSnapshot): string { + return isHeadSnapshot(head, snapshot) ? HASHLINE_RECOVERY_EXTERNAL_WARNING : HASHLINE_RECOVERY_SESSION_CHAIN_WARNING; +} + +/** + * Attempt to recover from a section file-hash mismatch by replaying the edits + * against a cached pre-edit snapshot of the file and 3-way-merging the result + * onto the current on-disk content. Returns `null` when no recovery is possible. + */ +export function tryRecoverHashlineWithCache(args: HashlineRecoveryArgs): HashlineRecoveryResult | null { + const { cache, absolutePath, currentText, fileHash, edits, options } = args; + const head = cache.get(absolutePath); + const snapshot = cache.getByHash(absolutePath, fileHash); + if (!snapshot || snapshot.lines.size === 0) return null; + + const recoveryWarning = resolveRecoveryWarning(head, snapshot); + if (snapshot.fullText !== undefined) { + return applyEditsToSnapshot(snapshot.fullText, currentText, edits, options, recoveryWarning); + } + + const overlayText = buildSparseOverlayText(currentText, snapshot.lines); + if (computeFileHash(overlayText) !== fileHash) return null; + return applyEditsToSnapshot(overlayText, currentText, edits, options, recoveryWarning); +} diff --git a/packages/coding-agent/src/hashline/stream.ts b/packages/coding-agent/src/hashline/stream.ts index 6e241430b..05b4bd2b4 100644 --- a/packages/coding-agent/src/hashline/stream.ts +++ b/packages/coding-agent/src/hashline/stream.ts @@ -1,4 +1,4 @@ -import { formatHashLine } from "./hash"; +import { formatNumberedLine } from "./hash"; import type { HashlineStreamOptions } from "./types"; interface ResolvedHashlineStreamOptions { @@ -34,7 +34,7 @@ function createHashlineChunkEmitter(options: ResolvedHashlineStreamOptions): Has }; const pushLine = (line: string): string[] => { - const formatted = formatHashLine(lineNumber, line); + const formatted = formatNumberedLine(lineNumber, line); lineNumber++; const chunks: string[] = []; diff --git a/packages/coding-agent/src/hashline/tokenizer.ts b/packages/coding-agent/src/hashline/tokenizer.ts new file mode 100644 index 000000000..93d8d217e --- /dev/null +++ b/packages/coding-agent/src/hashline/tokenizer.ts @@ -0,0 +1,467 @@ +import { ABORT_MARKER, BEGIN_PATCH_MARKER, END_PATCH_MARKER } from "./constants"; +import { + describeAnchorExamples, + HL_FILE_HASH_SEP, + HL_FILE_PREFIX, + HL_OP_DELETE, + HL_OP_INSERT_AFTER, + HL_OP_INSERT_BEFORE, + HL_OP_REPLACE, +} from "./hash"; +import type { Anchor, HashlineCursor } from "./types"; + +const CHAR_LINE_FEED = 10; +const CHAR_CARRIAGE_RETURN = 13; +const CHAR_ZERO = 48; +const CHAR_NINE = 57; +const CHAR_HASH = 35; +const CHAR_TAB = 9; +const CHAR_SPACE = 32; +const CHAR_LOWER_A = 97; +const CHAR_LOWER_F = 102; +const CHAR_PILCROW = HL_FILE_PREFIX.charCodeAt(0); +const FILE_HASH_LENGTH = 4; + +function isDigitCode(code: number): boolean { + return code >= CHAR_ZERO && code <= CHAR_NINE; +} + +function isNonZeroDigitCode(code: number): boolean { + return code > CHAR_ZERO && code <= CHAR_NINE; +} + +function isDecorationCode(code: number): boolean { + return code === 42 || code === 43 || code === 45 || code === 62; +} + +function isHexDigitCode(code: number): boolean { + return isDigitCode(code) || (code >= CHAR_LOWER_A && code <= CHAR_LOWER_F); +} + +function skipWhitespace(line: string, index: number, end = line.length): number { + return end - line.slice(index, end).trimStart().length; +} + +function trimEndIndex(line: string): number { + return line.trimEnd().length; +} + +function isEmptyLine(line: string): boolean { + return line.length === 0; +} + +function markerLineEquals(line: string, marker: string): boolean { + return line.trimEnd() === marker; +} + +/** + * Split a hashline diff into individual lines without losing the trailing + * empty line that callers may rely on for explicit blank payloads. CRLF pairs + * are normalized to a single line break. + * + * This mirrors the line-splitting performed by {@link HashlineTokenizer}'s + * streaming drain loop and is kept for non-streaming callers that prefer + * a single-shot split. + */ +export function splitHashlineLines(text: string): string[] { + if (text.length === 0) return [""]; + + const lines: string[] = []; + let start = 0; + for (let index = 0; index < text.length; index++) { + if (text.charCodeAt(index) !== CHAR_LINE_FEED) continue; + let end = index; + if (end > start && text.charCodeAt(end - 1) === CHAR_CARRIAGE_RETURN) end--; + lines.push(text.slice(start, end)); + start = index + 1; + } + + if (start < text.length) { + let end = text.length; + if (end > start && text.charCodeAt(end - 1) === CHAR_CARRIAGE_RETURN) end--; + lines.push(text.slice(start, end)); + } + return lines; +} + +export function cloneCursor(cursor: HashlineCursor): HashlineCursor { + if (cursor.kind === "before_anchor") return { kind: "before_anchor", anchor: { ...cursor.anchor } }; + if (cursor.kind === "after_anchor") return { kind: "after_anchor", anchor: { ...cursor.anchor } }; + return cursor; +} + +// Leniently accept anchors copied from read/search output: +// - optional leading line-marker decoration (`*`, `>`, `+`, `-`) +// - the required bare line number +function skipDecoratedAnchorPrefix(line: string, end = trimEndIndex(line)): number { + let index = skipWhitespace(line, 0, end); + while (index < end && isDecorationCode(line.charCodeAt(index))) index++; + return skipWhitespace(line, index, end); +} + +interface NumberScan { + line: number; + nextIndex: number; +} + +function scanLineNumber(line: string, index: number, end: number): NumberScan | null { + if (index >= end || !isNonZeroDigitCode(line.charCodeAt(index))) return null; + + let lineNumber = 0; + let nextIndex = index; + while (nextIndex < end) { + const code = line.charCodeAt(nextIndex); + if (!isDigitCode(code)) break; + lineNumber = lineNumber * 10 + (code - CHAR_ZERO); + nextIndex++; + } + return { line: lineNumber, nextIndex }; +} + +/** Parse a bare line-number anchor (used by insert ops). Throws on malformed input. */ +export function parseLid(raw: string, lineNum: number): Anchor { + const end = trimEndIndex(raw); + const numberStart = skipDecoratedAnchorPrefix(raw, end); + const number = scanLineNumber(raw, numberStart, end); + if (number === null || skipWhitespace(raw, number.nextIndex, end) !== end) { + throw new Error( + `line ${lineNum}: expected a line number such as ${describeAnchorExamples("119")}; ` + + `got ${JSON.stringify(raw)}. Use ${HL_FILE_PREFIX}PATH${HL_FILE_HASH_SEP}hash from your latest read for file-version binding.`, + ); + } + return { line: number.line }; +} + +export interface ParsedRange { + start: Anchor; + end: Anchor; +} + +interface RangeScan { + range: ParsedRange; + nextIndex: number; +} + +function scanRange(line: string, end = trimEndIndex(line)): RangeScan | null { + const numberStart = skipDecoratedAnchorPrefix(line, end); + const start = scanLineNumber(line, numberStart, end); + if (start === null) return null; + + let nextIndex = start.nextIndex; + let rangeEnd = start.line; + if (nextIndex < end && line.charCodeAt(nextIndex) === 45) { + const endNumber = scanLineNumber(line, nextIndex + 1, end); + if (endNumber === null) return null; + rangeEnd = endNumber.line; + nextIndex = endNumber.nextIndex; + } + + return { + range: { start: { line: start.line }, end: { line: rangeEnd } }, + nextIndex: skipWhitespace(line, nextIndex, end), + }; +} + +function startsWithWord(line: string, index: number, end: number, word: string): boolean { + if (index + word.length > end) return false; + for (let offset = 0; offset < word.length; offset++) { + if (line.charCodeAt(index + offset) !== word.charCodeAt(offset)) return false; + } + return true; +} + +function parseInsertTarget(raw: string, lineNum: number, kind: "before" | "after"): HashlineCursor { + const end = trimEndIndex(raw); + const targetStart = skipDecoratedAnchorPrefix(raw, end); + + if (startsWithWord(raw, targetStart, end, "BOF") && skipWhitespace(raw, targetStart + 3, end) === end) { + return { kind: "bof" }; + } + if (startsWithWord(raw, targetStart, end, "EOF") && skipWhitespace(raw, targetStart + 3, end) === end) { + return { kind: "eof" }; + } + + const cursorKind = kind === "before" ? "before_anchor" : "after_anchor"; + return { kind: cursorKind, anchor: parseLid(raw, lineNum) }; +} + +function scanInlineBody(line: string, index: number): string | undefined { + const end = trimEndIndex(line); + return index < end ? line.slice(index, end) : undefined; +} + +interface ParsedInsertOp { + kind: "insert"; + cursor: HashlineCursor; + inlineBody: string | undefined; +} + +interface ParsedReplaceOp { + kind: "replace"; + range: ParsedRange; + inlineBody: string | undefined; +} + +interface ParsedDeleteOp { + kind: "delete"; + range: ParsedRange; + trailingPayload: boolean; +} + +type ParsedOp = ParsedInsertOp | ParsedReplaceOp | ParsedDeleteOp; + +function tryParseInsertOp(line: string, sigil: string, kind: "before" | "after"): ParsedInsertOp | null { + const end = trimEndIndex(line); + const targetStart = skipDecoratedAnchorPrefix(line, end); + + let targetEnd: number; + if (startsWithWord(line, targetStart, end, "BOF") || startsWithWord(line, targetStart, end, "EOF")) { + targetEnd = targetStart + 3; + } else { + const anchor = scanLineNumber(line, targetStart, end); + if (anchor === null) return null; + targetEnd = anchor.nextIndex; + } + + const opIndex = skipWhitespace(line, targetEnd, end); + if (opIndex >= end || line[opIndex] !== sigil) return null; + + // parseInsertTarget can only throw on inputs that already passed the + // BOF/EOF/line-number scan above, but guard the throw anyway — the + // tokenizer contract forbids it and a future refactor of the prefix + // scan must not silently start raising here. + try { + return { + kind: "insert", + cursor: parseInsertTarget(line.slice(0, opIndex), 0, kind), + inlineBody: scanInlineBody(line, opIndex + sigil.length), + }; + } catch { + return null; + } +} + +function tryParseReplaceOp(line: string): ParsedReplaceOp | null { + const end = trimEndIndex(line); + const range = scanRange(line, end); + if (range === null || range.nextIndex >= end || line[range.nextIndex] !== HL_OP_REPLACE) return null; + return { + kind: "replace", + range: range.range, + inlineBody: scanInlineBody(line, range.nextIndex + HL_OP_REPLACE.length), + }; +} + +function tryParseDeleteOp(line: string): ParsedDeleteOp | null { + const end = trimEndIndex(line); + const range = scanRange(line, end); + if (range === null || range.nextIndex >= end || line[range.nextIndex] !== HL_OP_DELETE) return null; + const afterSigil = range.nextIndex + HL_OP_DELETE.length; + return { kind: "delete", range: range.range, trailingPayload: afterSigil !== end }; +} + +function tryParseOp(line: string): ParsedOp | null { + return ( + tryParseInsertOp(line, HL_OP_INSERT_BEFORE, "before") ?? + tryParseInsertOp(line, HL_OP_INSERT_AFTER, "after") ?? + tryParseReplaceOp(line) ?? + tryParseDeleteOp(line) + ); +} + +/** + * Strict header scan: `¶+` prefix, optional whitespace, path body that excludes + * whitespace, `#`, and `¶`, optional `#[0-9a-f]{4}` hash suffix, optional + * trailing whitespace. Returns `null` when any byte deviates from the shape. + */ +function tryParseHeader(line: string): { path: string; fileHash?: string } | null { + const end = trimEndIndex(line); + if (end === 0 || line.charCodeAt(0) !== CHAR_PILCROW) return null; + + let index = 0; + while (index < end && line.charCodeAt(index) === CHAR_PILCROW) index++; + index = skipWhitespace(line, index, end); + if (index >= end) return null; + + const pathStart = index; + while (index < end) { + const code = line.charCodeAt(index); + if (code === CHAR_HASH || code === CHAR_PILCROW || code === CHAR_SPACE || code === CHAR_TAB) break; + index++; + } + if (index === pathStart) return null; + const path = line.slice(pathStart, index); + + let fileHash: string | undefined; + if (index < end && line.charCodeAt(index) === CHAR_HASH) { + const hashStart = index + 1; + const hashEnd = hashStart + FILE_HASH_LENGTH; + if (hashEnd > end) return null; + for (let probe = hashStart; probe < hashEnd; probe++) { + if (!isHexDigitCode(line.charCodeAt(probe))) return null; + } + fileHash = line.slice(hashStart, hashEnd); + index = hashEnd; + } + + // Anything other than trailing whitespace disqualifies the header. + if (skipWhitespace(line, index, end) !== end) return null; + + return fileHash !== undefined ? { path, fileHash } : { path }; +} + +/** + * Returns true when the line scans as `LINE!payload` (delete sigil followed by + * additional content). The executor uses this for the dedicated "deletes only" + * diagnostic, separate from the standard "unrecognized op" path. + */ +export function isDeleteOpWithPayload(line: string): boolean { + const range = scanRange(line, line.length); + return ( + range !== null && + range.nextIndex < line.length && + line[range.nextIndex] === HL_OP_DELETE && + range.nextIndex + HL_OP_DELETE.length < line.length + ); +} + +interface TokenBase { + /** 1-indexed line number in the original input stream. */ + lineNum: number; +} + +export type HashlineToken = + | (TokenBase & { kind: "blank" }) + | (TokenBase & { kind: "envelope-begin" }) + | (TokenBase & { kind: "envelope-end" }) + | (TokenBase & { kind: "abort" }) + | (TokenBase & { kind: "header"; path: string; fileHash?: string }) + | (TokenBase & { kind: "op-insert"; cursor: HashlineCursor; inlineBody: string | undefined }) + | (TokenBase & { kind: "op-replace"; range: ParsedRange; inlineBody: string | undefined }) + | (TokenBase & { kind: "op-delete"; range: ParsedRange; trailingPayload: boolean }) + | (TokenBase & { kind: "payload"; text: string }); + +function classifyLine(line: string, lineNum: number): HashlineToken { + if (isEmptyLine(line)) return { kind: "blank", lineNum }; + if (markerLineEquals(line, BEGIN_PATCH_MARKER)) return { kind: "envelope-begin", lineNum }; + if (markerLineEquals(line, END_PATCH_MARKER)) return { kind: "envelope-end", lineNum }; + if (markerLineEquals(line, ABORT_MARKER)) return { kind: "abort", lineNum }; + + if (line.charCodeAt(0) === CHAR_PILCROW) { + const header = tryParseHeader(line); + if (header !== null) { + return header.fileHash !== undefined + ? { kind: "header", lineNum, path: header.path, fileHash: header.fileHash } + : { kind: "header", lineNum, path: header.path }; + } + } + + const op = tryParseOp(line); + if (op !== null) { + if (op.kind === "insert") { + return { kind: "op-insert", lineNum, cursor: op.cursor, inlineBody: op.inlineBody }; + } + if (op.kind === "replace") { + return { kind: "op-replace", lineNum, range: op.range, inlineBody: op.inlineBody }; + } + return { kind: "op-delete", lineNum, range: op.range, trailingPayload: op.trailingPayload }; + } + + return { kind: "payload", lineNum, text: line }; +} + +/** + * Stateful, line-oriented classifier for hashline diff text. Use the streaming + * {@link feed}/{@link end} pair to ingest text in chunks (each completed line + * emits exactly one token; a trailing partial line stays buffered until the + * next chunk or {@link end}). Use the stateless {@link tokenize}/predicate + * methods for callers that already hold whole lines and only need + * classification without buffering. + */ +export class HashlineTokenizer { + #buffer = ""; + #nextLineNum = 1; + #closed = false; + + /** + * Ingest a chunk of input text. Each newline-terminated line in the + * combined buffer produces one token. A trailing partial line (no `\n` + * yet, possibly ending in a lone `\r`) stays buffered until the next + * `feed`/`end` call so CRLF pairs that straddle chunk boundaries are + * still normalized correctly. + */ + feed(chunk: string): HashlineToken[] { + if (this.#closed) throw new Error("HashlineTokenizer is closed; call reset() before reusing."); + if (chunk.length === 0) return []; + this.#buffer = this.#buffer ? this.#buffer + chunk : chunk; + return this.#drainCompleteLines(); + } + + /** + * Flush any buffered residual line (the last line of input when it lacks + * a trailing newline) and mark the tokenizer closed. Calling `end` a + * second time returns `[]`; reuse requires `reset`. + */ + end(): HashlineToken[] { + if (this.#closed) return []; + this.#closed = true; + const buf = this.#buffer; + this.#buffer = ""; + if (buf.length === 0) return []; + let stop = buf.length; + if (buf.charCodeAt(stop - 1) === CHAR_CARRIAGE_RETURN) stop--; + const token = classifyLine(buf.slice(0, stop), this.#nextLineNum++); + return [token]; + } + + /** Discard any buffered text and reset the line counter to 1. */ + reset(): void { + this.#buffer = ""; + this.#nextLineNum = 1; + this.#closed = false; + } + + /** Convenience: feed an entire text and immediately flush. */ + tokenizeAll(text: string): HashlineToken[] { + this.reset(); + const first = this.feed(text); + const last = this.end(); + return last.length === 0 ? first : first.concat(last); + } + + /** Stateless one-shot classification. Does not touch the streaming buffer. */ + tokenize(line: string, lineNum = 0): HashlineToken { + return classifyLine(line, lineNum); + } + + isOp(line: string): boolean { + return tryParseOp(line) !== null; + } + + isHeader(line: string): boolean { + return tryParseHeader(line) !== null; + } + + isEnvelopeMarker(line: string): boolean { + return ( + markerLineEquals(line, BEGIN_PATCH_MARKER) || + markerLineEquals(line, END_PATCH_MARKER) || + markerLineEquals(line, ABORT_MARKER) + ); + } + + #drainCompleteLines(): HashlineToken[] { + const tokens: HashlineToken[] = []; + const buf = this.#buffer; + let start = 0; + for (let index = 0; index < buf.length; index++) { + if (buf.charCodeAt(index) !== CHAR_LINE_FEED) continue; + let stop = index; + if (stop > start && buf.charCodeAt(stop - 1) === CHAR_CARRIAGE_RETURN) stop--; + tokens.push(classifyLine(buf.slice(start, stop), this.#nextLineNum++)); + start = index + 1; + } + this.#buffer = start < buf.length ? buf.slice(start) : ""; + return tokens; + } +} diff --git a/packages/coding-agent/src/hashline/types.ts b/packages/coding-agent/src/hashline/types.ts index 0747a5db2..1bd87f70a 100644 --- a/packages/coding-agent/src/hashline/types.ts +++ b/packages/coding-agent/src/hashline/types.ts @@ -3,16 +3,8 @@ import type { LspBatchRequest } from "../edit/renderer"; import type { WritethroughCallback, WritethroughDeferredHandle } from "../lsp"; import type { ToolSession } from "../tools"; -export interface HashMismatch { - line: number; - expected: string; - actual: string; -} - export type Anchor = { line: number; - hash: string; - contentHint?: string; }; export type HashlineCursor = @@ -25,6 +17,12 @@ export type HashlineEdit = | { kind: "insert"; cursor: HashlineCursor; text: string; lineNum: number; index: number } | { kind: "delete"; anchor: Anchor; lineNum: number; index: number; oldAssertion?: string }; +export interface HashlineInputSection { + path: string; + fileHash?: string; + diff: string; +} + /** `path` is accepted by the edit tool runtime; other extra keys are preserved. */ export const hashlineEditParamsSchema = z.object({ input: z.string(), path: z.string().optional() }).passthrough(); export type HashlineParams = z.infer; diff --git a/packages/coding-agent/src/lsp/edits.ts b/packages/coding-agent/src/lsp/edits.ts index 1159a0ca5..78c96b262 100644 --- a/packages/coding-agent/src/lsp/edits.ts +++ b/packages/coding-agent/src/lsp/edits.ts @@ -127,38 +127,91 @@ export async function applyTextEdits(filePath: string, edits: TextEdit[]): Promi export async function applyWorkspaceEdit(edit: WorkspaceEdit, cwd: string): Promise { const applied: string[] = []; - // Coalesce all text edits per URI before applying so a single file's edits - // are applied in one pass against a single snapshot — multiple TextDocumentEdits - // for the same URI would otherwise read stale positions on subsequent writes. - const textEditsByUri = flattenWorkspaceTextEdits(edit); - for (const [uri, textEdits] of textEditsByUri) { - const filePath = uriToFile(uri); - await applyTextEdits(filePath, textEdits); - applied.push(`Applied ${textEdits.length} edit(s) to ${formatPathRelativeToCwd(filePath, cwd)}`); - } - - // Resource operations (create/rename/delete) preserve their original order. if (edit.documentChanges) { - for (const change of edit.documentChanges) { - if (!("kind" in change) || !change.kind) continue; - if (change.kind === "create") { - const createOp = change as CreateFile; - const filePath = uriToFile(createOp.uri); - await Bun.write(filePath, ""); - applied.push(`Created ${formatPathRelativeToCwd(filePath, cwd)}`); - } else if (change.kind === "rename") { - const renameOp = change as RenameFile; - const oldPath = uriToFile(renameOp.oldUri); - const newPath = uriToFile(renameOp.newUri); - await fs.mkdir(path.dirname(newPath), { recursive: true }); - await fs.rename(oldPath, newPath); - applied.push(`Renamed ${formatPathRelativeToCwd(oldPath, cwd)} → ${formatPathRelativeToCwd(newPath, cwd)}`); - } else if (change.kind === "delete") { - const deleteOp = change as DeleteFile; - const filePath = uriToFile(deleteOp.uri); - await fs.rm(filePath, { recursive: true }); - applied.push(`Deleted ${formatPathRelativeToCwd(filePath, cwd)}`); + // Walk documentChanges in original order. Accumulate text edits per-URI and + // flush them before any resource op that touches the same URI (or, for folder + // rename/delete, any descendant URI) so that renames, creates, and deletes + // always see the correct prior file state. + const pending = new Map(); + + const flushUri = async (uri: string) => { + const edits = pending.get(uri); + if (!edits) return; + pending.delete(uri); + const filePath = uriToFile(uri); + await applyTextEdits(filePath, edits); + applied.push(`Applied ${edits.length} edit(s) to ${formatPathRelativeToCwd(filePath, cwd)}`); + }; + + // Flush the exact URI plus every pending descendant (for folder-level + // resource ops where the queued edits target child files of the target). + const flushSubtree = async (uri: string) => { + const prefix = uri.endsWith("/") ? uri : `${uri}/`; + const matches: string[] = []; + for (const candidate of pending.keys()) { + if (candidate === uri || candidate.startsWith(prefix)) matches.push(candidate); } + for (const target of matches) { + await flushUri(target); + } + }; + + for (const change of edit.documentChanges) { + if ("textDocument" in change && change.textDocument && "edits" in change && change.edits) { + const tdc = change as TextDocumentEdit; + const uri = tdc.textDocument.uri; + const textEdits = tdc.edits.filter((e): e is TextEdit => "range" in e && "newText" in e); + if (textEdits.length > 0) { + const prev = pending.get(uri); + if (prev) prev.push(...textEdits); + else pending.set(uri, [...textEdits]); + } + } else if ("kind" in change && change.kind) { + if (change.kind === "create") { + const createOp = change as CreateFile; + await flushUri(createOp.uri); + const filePath = uriToFile(createOp.uri); + await Bun.write(filePath, ""); + applied.push(`Created ${formatPathRelativeToCwd(filePath, cwd)}`); + } else if (change.kind === "rename") { + const renameOp = change as RenameFile; + // Per LSP §3.16.2 documentChanges are applied in declared order. + // Flush both the source subtree (so prior edits land before the move) + // AND the destination subtree (so prior edits land on whatever exists + // at newUri before the rename overwrites/replaces it — relevant under + // `options.overwrite` and `options.ignoreIfExists`). + await flushSubtree(renameOp.oldUri); + await flushSubtree(renameOp.newUri); + const oldPath = uriToFile(renameOp.oldUri); + const newPath = uriToFile(renameOp.newUri); + await fs.mkdir(path.dirname(newPath), { recursive: true }); + await fs.rename(oldPath, newPath); + applied.push( + `Renamed ${formatPathRelativeToCwd(oldPath, cwd)} → ${formatPathRelativeToCwd(newPath, cwd)}`, + ); + } else if (change.kind === "delete") { + const deleteOp = change as DeleteFile; + await flushSubtree(deleteOp.uri); + const filePath = uriToFile(deleteOp.uri); + await fs.rm(filePath, { recursive: true }); + applied.push(`Deleted ${formatPathRelativeToCwd(filePath, cwd)}`); + } + } + } + + // Flush text edits not followed by a resource op. + for (const [uri] of pending) { + await flushUri(uri); + } + } else if (edit.changes) { + // Legacy changes-map path: apply all text edits in one pass. + const changes = edit.changes; + for (const uri in changes) { + const textEdits = changes[uri]; + if (textEdits.length === 0) continue; + const filePath = uriToFile(uri); + await applyTextEdits(filePath, textEdits); + applied.push(`Applied ${textEdits.length} edit(s) to ${formatPathRelativeToCwd(filePath, cwd)}`); } } diff --git a/packages/coding-agent/src/lsp/utils.ts b/packages/coding-agent/src/lsp/utils.ts index 2ee9bea82..b74ed989b 100644 --- a/packages/coding-agent/src/lsp/utils.ts +++ b/packages/coding-agent/src/lsp/utils.ts @@ -581,7 +581,7 @@ function firstNonWhitespaceColumn(lineText: string): number { return match ? (match.index ?? 0) : 0; } -const BARE_IDENTIFIER_RE = /^[A-Za-z_][\w]*$/; +const BARE_IDENTIFIER_RE = /^[$A-Za-z_][\w$]*$/; const IDENTIFIER_CHAR_RE = /[A-Za-z0-9_$]/; function findSymbolMatchIndexes(lineText: string, symbol: string, caseInsensitive = false): number[] { diff --git a/packages/coding-agent/src/mcp/discoverable-tool-metadata.ts b/packages/coding-agent/src/mcp/discoverable-tool-metadata.ts deleted file mode 100644 index 9be4ce3a0..000000000 --- a/packages/coding-agent/src/mcp/discoverable-tool-metadata.ts +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Back-compat re-export layer. - * All types and functions have moved to src/tool-discovery/tool-index.ts. - * This file exists solely so existing imports continue to compile without changes. - */ -export type { - DiscoverableMCPSearchDocument, - DiscoverableMCPSearchIndex, - DiscoverableMCPSearchResult, - DiscoverableMCPTool, - DiscoverableMCPToolServerSummary, - DiscoverableMCPToolSummary, -} from "../tool-discovery/tool-index"; - -export { - buildDiscoverableMCPSearchIndex, - collectDiscoverableMCPTools, - formatDiscoverableMCPToolServerSummary, - getDiscoverableMCPTool, - isMCPToolName, - searchDiscoverableMCPTools, - selectDiscoverableMCPToolNamesByServer, - summarizeDiscoverableMCPTools, -} from "../tool-discovery/tool-index"; diff --git a/packages/coding-agent/src/mcp/oauth-discovery.ts b/packages/coding-agent/src/mcp/oauth-discovery.ts index 65e330c06..1a1db88e3 100644 --- a/packages/coding-agent/src/mcp/oauth-discovery.ts +++ b/packages/coding-agent/src/mcp/oauth-discovery.ts @@ -17,22 +17,23 @@ export interface AuthDetectionResult { authType?: "oauth" | "apikey" | "unknown"; oauth?: OAuthEndpoints; authServerUrl?: string; + resourceMetadataUrl?: string; message?: string; } -function parseMcpAuthServerUrl(errorMessage: string): string | undefined { +function parseMcpAuthServerUrl(errorMessage: string, serverUrl?: string): string | undefined { const match = errorMessage.match(/Mcp-Auth-Server:\s*([^;\]\s]+)/i); if (!match?.[1]) return undefined; try { - return new URL(match[1]).toString(); + return new URL(match[1], serverUrl).toString(); } catch { return undefined; } } -export function extractMcpAuthServerUrl(error: Error): string | undefined { - return parseMcpAuthServerUrl(error.message); +export function extractMcpAuthServerUrl(error: Error, serverUrl?: string): string | undefined { + return parseMcpAuthServerUrl(error.message, serverUrl); } /** @@ -189,12 +190,15 @@ export function extractOAuthEndpoints(error: Error): OAuthEndpoints | null { * Analyze an error to determine authentication requirements. * Returns structured info about what auth is needed. */ -export function analyzeAuthError(error: Error): AuthDetectionResult { +export function analyzeAuthError(error: Error, serverUrl?: string): AuthDetectionResult { if (!detectAuthError(error)) { return { requiresAuth: false }; } - const authServerUrl = extractMcpAuthServerUrl(error); + const authServerUrl = extractMcpAuthServerUrl(error, serverUrl); + // Extract resource_metadata URL from challenge entries in error message + const resourceMetaMatch = error.message.match(/resource_metadata\s*=\s*"([^"]+)"/i); + const resourceMetadataUrl = resourceMetaMatch?.[1]; // Try to extract OAuth endpoints const oauth = extractOAuthEndpoints(error); @@ -205,6 +209,7 @@ export function analyzeAuthError(error: Error): AuthDetectionResult { authType: "oauth", oauth, authServerUrl, + resourceMetadataUrl, message: "Server requires OAuth authentication. Launching authorization flow...", }; } @@ -221,6 +226,7 @@ export function analyzeAuthError(error: Error): AuthDetectionResult { requiresAuth: true, authType: "apikey", authServerUrl, + resourceMetadataUrl, message: "Server requires API key authentication.", }; } @@ -230,6 +236,7 @@ export function analyzeAuthError(error: Error): AuthDetectionResult { requiresAuth: true, authType: "unknown", authServerUrl, + resourceMetadataUrl, message: "Server requires authentication but type could not be determined.", }; } @@ -241,6 +248,7 @@ export function analyzeAuthError(error: Error): AuthDetectionResult { export async function discoverOAuthEndpoints( serverUrl: string, authServerUrl?: string, + resourceMetadataUrl?: string, ): Promise { const wellKnownPaths = [ "/.well-known/oauth-authorization-server", @@ -250,9 +258,44 @@ export async function discoverOAuthEndpoints( "/.mcp/auth", "/authorize", // Some MCP servers expose OAuth config here ]; - const urlsToQuery = [authServerUrl, serverUrl].filter((value): value is string => Boolean(value)); + const urlsToQuery: string[] = []; const visitedAuthServers = new Set(); + // Step 1: If a resource_metadata URL was provided, fetch it to discover auth servers. + // This follows the RFC 9728 chain: resource_metadata → authorization_servers. + if (resourceMetadataUrl && !visitedAuthServers.has(resourceMetadataUrl)) { + visitedAuthServers.add(resourceMetadataUrl); + try { + const metaResp = await fetch(resourceMetadataUrl, { + method: "GET", + headers: { Accept: "application/json" }, + redirect: "follow", + }); + if (metaResp.ok) { + const meta = (await metaResp.json()) as Record; + const authServers = Array.isArray(meta.authorization_servers) + ? meta.authorization_servers.filter((entry): entry is string => typeof entry === "string") + : []; + for (const s of authServers) { + if (!visitedAuthServers.has(s)) { + urlsToQuery.push(s); + visitedAuthServers.add(s); + } + } + } + } catch { + // Ignore errors, continue to try explicit URLs + } + } + + // Step 2: Add explicit authServerUrl and serverUrl (deduped against visited) + for (const url of [authServerUrl, serverUrl].filter((v): v is string => Boolean(v))) { + if (!visitedAuthServers.has(url)) { + urlsToQuery.push(url); + visitedAuthServers.add(url); + } + } + const findEndpoints = (metadata: Record): OAuthEndpoints | null => { if (metadata.authorization_endpoint && metadata.token_endpoint) { const scopesSupported = Array.isArray(metadata.scopes_supported) @@ -311,39 +354,84 @@ export async function discoverOAuthEndpoints( }; for (const baseUrl of urlsToQuery) { - visitedAuthServers.add(baseUrl); for (const path of wellKnownPaths) { - try { - const url = new URL(path, baseUrl); - const response = await fetch(url.toString(), { - method: "GET", - headers: { Accept: "application/json" }, - }); + // Try each well-known path at both the absolute origin and relative + const urlsToTry = buildWellKnownUrls(path, baseUrl); + for (const url of urlsToTry) { + try { + const response = await fetch(url.toString(), { + method: "GET", + headers: { Accept: "application/json" }, + redirect: "follow", + }); - if (response.ok) { - const metadata = (await response.json()) as Record; - const endpoints = findEndpoints(metadata); - if (endpoints) return endpoints; + if (response.ok) { + const metadata = (await response.json()) as Record; + const endpoints = findEndpoints(metadata); + if (endpoints) return endpoints; - if (path === "/.well-known/oauth-protected-resource") { - const authServers = Array.isArray(metadata.authorization_servers) - ? metadata.authorization_servers.filter((entry): entry is string => typeof entry === "string") - : []; + if (path === "/.well-known/oauth-protected-resource") { + const authServers = Array.isArray(metadata.authorization_servers) + ? metadata.authorization_servers.filter((entry): entry is string => typeof entry === "string") + : []; - for (const discoveredAuthServer of authServers) { - if (visitedAuthServers.has(discoveredAuthServer)) { - continue; + for (const discoveredAuthServer of authServers) { + if (visitedAuthServers.has(discoveredAuthServer)) { + continue; + } + const discovered = await discoverOAuthEndpoints(serverUrl, discoveredAuthServer); + if (discovered) return discovered; } - const discovered = await discoverOAuthEndpoints(serverUrl, discoveredAuthServer); - if (discovered) return discovered; } } + } catch { + // Ignore errors, try next path } - } catch { - // Ignore errors, try next path } } } return null; } + +function buildWellKnownUrls(wellKnownPath: string, baseUrl: string): URL[] { + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + return []; + } + + const absUrl = new URL(wellKnownPath, parsed); + if (!wellKnownPath.startsWith("/")) return [absUrl]; + + const normalizedPath = parsed.pathname.replace(/\/$/, ""); + const lastSlash = normalizedPath.lastIndexOf("/"); + // Bare origin (no path beyond "/") — only the origin-root candidate applies. + if (lastSlash < 0) return [absUrl]; + + // Path-prefixed well-known (common for gateways with sub-path routing). + // Multi-segment paths drop the trailing segment (typically the MCP endpoint); + // single-segment paths (lastSlash === 0) are themselves the gateway prefix. + const prefixPath = lastSlash === 0 ? normalizedPath : normalizedPath.slice(0, lastSlash); + const relUrl = new URL(wellKnownPath.slice(1), `${parsed.origin}${prefixPath}/`); + + const candidates: URL[] = [absUrl]; + const seen = new Set([absUrl.href]); + const push = (u: URL): void => { + if (!seen.has(u.href)) { + candidates.push(u); + seen.add(u.href); + } + }; + push(relUrl); + + // RFC 8414 §3.1 path-ful issuer form: /.well-known//. + // Only meaningful for well-known metadata documents. + if (wellKnownPath.startsWith("/.well-known/")) { + const pathfulUrl = new URL(`${wellKnownPath}${normalizedPath}`, parsed.origin); + push(pathfulUrl); + } + + return candidates; +} diff --git a/packages/coding-agent/src/mcp/oauth-flow.ts b/packages/coding-agent/src/mcp/oauth-flow.ts index 71ae572d9..4de143cbc 100644 --- a/packages/coding-agent/src/mcp/oauth-flow.ts +++ b/packages/coding-agent/src/mcp/oauth-flow.ts @@ -324,23 +324,51 @@ export class MCPOAuthFlow extends OAuthCallbackFlow { } async #resolveRegistrationEndpoint(): Promise { + const authorizationUrl = new URL(this.config.authorizationUrl); + + // origin-root well-known; most servers serve metadata here. + const rootUrl = new URL("/.well-known/oauth-authorization-server", authorizationUrl.origin).toString(); + const endpoint = await this.#tryWellKnownForRegistration(rootUrl); + if (endpoint) return endpoint; + + // path-prefixed well-known for gateways (e.g. https://gateway.example.com/my-service/). + const normalizedPath = authorizationUrl.pathname.replace(/\/$/, ""); + const lastSlash = normalizedPath.lastIndexOf("/"); + // Bare-origin authorization URL — nothing further to try. + if (lastSlash < 0) return null; + + // Single-segment paths are the gateway prefix itself; multi-segment paths + // drop the trailing segment (typically a service endpoint). + const prefixPath = lastSlash === 0 ? normalizedPath : normalizedPath.slice(0, lastSlash); + const prefixedUrl = new URL( + ".well-known/oauth-authorization-server", + `${authorizationUrl.origin}${prefixPath}/`, + ).toString(); + const prefixedEndpoint = await this.#tryWellKnownForRegistration(prefixedUrl); + if (prefixedEndpoint) return prefixedEndpoint; + + // RFC 8414 §3.1 path-ful issuer form: /.well-known/oauth-authorization-server/. + const pathfulUrl = new URL( + `/.well-known/oauth-authorization-server${normalizedPath}`, + authorizationUrl.origin, + ).toString(); + return await this.#tryWellKnownForRegistration(pathfulUrl); + } + + async #tryWellKnownForRegistration(wellKnownUrl: string): Promise { try { - const authorizationEndpoint = new URL(this.config.authorizationUrl); - const metadataUrl = new URL("/.well-known/oauth-authorization-server", authorizationEndpoint.origin); - const response = await fetch(metadataUrl.toString(), { + const response = await fetch(wellKnownUrl, { method: "GET", headers: { Accept: "application/json" }, }); - if (!response.ok) return null; const metadata = (await response.json()) as { registration_endpoint?: string }; if (metadata.registration_endpoint && metadata.registration_endpoint.trim() !== "") { return metadata.registration_endpoint; } } catch { - // Ignore metadata discovery failures. + // Ignore fetch/parse failures. } - return null; } diff --git a/packages/coding-agent/src/modes/components/extensions/types.ts b/packages/coding-agent/src/modes/components/extensions/types.ts index cdfe76edb..1b8f5ecd9 100644 --- a/packages/coding-agent/src/modes/components/extensions/types.ts +++ b/packages/coding-agent/src/modes/components/extensions/types.ts @@ -143,11 +143,6 @@ export interface DashboardState { selected: Extension | null; } -/** - * @deprecated Use FocusRegion instead - */ -export type FocusPane = "sidebar" | "main" | "inspector"; - /** * Callbacks from dashboard to parent. */ diff --git a/packages/coding-agent/src/modes/components/mcp-add-wizard.ts b/packages/coding-agent/src/modes/components/mcp-add-wizard.ts index 45420ad72..b7d4db071 100644 --- a/packages/coding-agent/src/modes/components/mcp-add-wizard.ts +++ b/packages/coding-agent/src/modes/components/mcp-add-wizard.ts @@ -965,14 +965,18 @@ export class MCPAddWizard extends Container { }, 1000); } catch (error) { // Connection failed - check if it's an auth error - const authResult = analyzeAuthError(error as Error); + const authResult = analyzeAuthError(error as Error, this.#state.url); if (authResult.requiresAuth) { // Prefer OAuth first: use error metadata, then well-known discovery fallback. let oauth = authResult.authType === "oauth" ? (authResult.oauth ?? null) : null; if (!oauth && this.#state.transport !== "stdio" && this.#state.url) { try { - oauth = await discoverOAuthEndpoints(this.#state.url, authResult.authServerUrl); + oauth = await discoverOAuthEndpoints( + this.#state.url, + authResult.authServerUrl, + authResult.resourceMetadataUrl, + ); } catch { // Ignore discovery failures and fallback to manual auth. } diff --git a/packages/coding-agent/src/modes/components/session-observer-overlay.ts b/packages/coding-agent/src/modes/components/session-observer-overlay.ts index 39e8804f8..27544a3e1 100644 --- a/packages/coding-agent/src/modes/components/session-observer-overlay.ts +++ b/packages/coding-agent/src/modes/components/session-observer-overlay.ts @@ -22,6 +22,7 @@ import { isSilentAbort } from "../../session/messages"; import type { SessionMessageEntry } from "../../session/session-manager"; import { parseSessionEntries } from "../../session/session-manager"; import { PREVIEW_LIMITS, replaceTabs, TRUNCATE_LENGTHS, truncateToWidth } from "../../tools/render-utils"; +import { toPathList } from "../../tools/search"; import type { ObservableSession, SessionObserverRegistry } from "../session-observer-registry"; import { getMarkdownTheme, theme } from "../theme/theme"; import { DynamicBorder } from "./dynamic-border"; @@ -533,13 +534,21 @@ export class SessionObserverOverlayComponent extends Container { case "write": case "edit": return args.path ? `path: ${args.path}` : ""; - case "search": + case "search": { + const searchPathsInput = + typeof args.paths === "string" || Array.isArray(args.paths) + ? args.paths + : typeof args.path === "string" + ? args.path + : undefined; + const searchPaths = toPathList(searchPathsInput); return [ args.pattern ? `pattern: ${args.pattern}` : "", - Array.isArray(args.paths) ? `paths: ${args.paths.join(", ")}` : "", + searchPaths.length > 0 ? `paths: ${searchPaths.join(", ")}` : "", ] .filter(Boolean) .join(", "); + } case "find": return Array.isArray(args.paths) ? `paths: ${args.paths.join(", ")}` : ""; case "bash": { diff --git a/packages/coding-agent/src/modes/components/settings-selector.ts b/packages/coding-agent/src/modes/components/settings-selector.ts index 6b1d4d00a..793c346ae 100644 --- a/packages/coding-agent/src/modes/components/settings-selector.ts +++ b/packages/coding-agent/src/modes/components/settings-selector.ts @@ -13,7 +13,7 @@ import { TabBar, Text, } from "@oh-my-pi/pi-tui"; -import { type SettingPath, settings } from "../../config/settings"; +import { getDefault, type SettingPath, settings } from "../../config/settings"; import type { SettingTab, StatusLinePreset, @@ -294,6 +294,7 @@ export class SettingsSelectorComponent extends Container { } const currentValue = this.#getCurrentValue(def); + const changed = this.#isChanged(def, currentValue); switch (def.type) { case "boolean": @@ -303,6 +304,7 @@ export class SettingsSelectorComponent extends Container { description: def.description, currentValue: currentValue ? "true" : "false", values: ["true", "false"], + changed, }; case "enum": @@ -312,6 +314,7 @@ export class SettingsSelectorComponent extends Container { description: def.description, currentValue: currentValue as string, values: [...def.values], + changed, }; case "submenu": @@ -321,6 +324,7 @@ export class SettingsSelectorComponent extends Container { description: def.description, currentValue: this.#getSubmenuCurrentValue(def.path, currentValue), submenu: (cv, done) => this.#createSubmenu(def, cv, done), + changed, }; case "text": @@ -330,6 +334,7 @@ export class SettingsSelectorComponent extends Container { description: def.description, currentValue: (currentValue as string) ?? "", submenu: (cv, done) => this.#createTextInput(def, cv, done), + changed, }; } } @@ -341,6 +346,10 @@ export class SettingsSelectorComponent extends Container { return settings.get(def.path); } + #isChanged(def: SettingDef, currentValue: unknown): boolean { + return !Object.is(currentValue, getDefault(def.path)); + } + #getSubmenuCurrentValue(path: SettingPath, value: unknown): string { const rawValue = String(value ?? ""); if (path === "compaction.thresholdPercent" && (rawValue === "-1" || rawValue === "")) { diff --git a/packages/coding-agent/src/modes/components/tree-selector.ts b/packages/coding-agent/src/modes/components/tree-selector.ts index ae2088132..76cda73d1 100644 --- a/packages/coding-agent/src/modes/components/tree-selector.ts +++ b/packages/coding-agent/src/modes/components/tree-selector.ts @@ -15,6 +15,7 @@ import { theme } from "../../modes/theme/theme"; import { matchesAppInterrupt } from "../../modes/utils/keybinding-matchers"; import type { SessionTreeNode } from "../../session/session-manager"; import { shortenPath } from "../../tools/render-utils"; +import { toPathList } from "../../tools/search"; import { DynamicBorder } from "./dynamic-border"; /** Gutter info: position (displayIndent where connector was) and whether to show │ */ @@ -690,8 +691,15 @@ class TreeList implements Component { } case "search": { const pattern = String(args.pattern || ""); - const paths = Array.isArray(args.paths) ? args.paths.join(", ") : String(args.path || "."); - return `[search: /${pattern}/ in ${shortenPath(paths)}]`; + const searchPathsInput = + typeof args.paths === "string" || Array.isArray(args.paths) + ? args.paths + : typeof args.path === "string" + ? args.path + : undefined; + const paths = toPathList(searchPathsInput); + const scope = paths.length > 0 ? paths.join(", ") : "."; + return `[search: /${pattern}/ in ${shortenPath(scope)}]`; } case "find": { const paths = Array.isArray(args.paths) ? args.paths.join(", ") : String(args.pattern || "."); diff --git a/packages/coding-agent/src/modes/controllers/command-controller.ts b/packages/coding-agent/src/modes/controllers/command-controller.ts index 0af272c0c..1f9b080e9 100644 --- a/packages/coding-agent/src/modes/controllers/command-controller.ts +++ b/packages/coding-agent/src/modes/controllers/command-controller.ts @@ -1368,10 +1368,12 @@ function formatUnlimitedReportLabel(report: UsageReport, index: number): string } function formatResetShort(limit: UsageLimit, nowMs: number): string | undefined { - if (limit.window?.resetsAt !== undefined) { - return formatDuration(limit.window.resetsAt - nowMs); - } - return undefined; + const resetsAt = limit.window?.resetsAt; + if (resetsAt === undefined) return undefined; + // Codex returns the prior window's reset_at until a new request opens a fresh window — + // rendering a negative delta is meaningless, so drop the suffix in that case. + if (resetsAt <= nowMs) return undefined; + return formatDuration(resetsAt - nowMs); } function formatAccountHeaderRow( diff --git a/packages/coding-agent/src/modes/controllers/extension-ui-controller.ts b/packages/coding-agent/src/modes/controllers/extension-ui-controller.ts index c83d71787..e795817b4 100644 --- a/packages/coding-agent/src/modes/controllers/extension-ui-controller.ts +++ b/packages/coding-agent/src/modes/controllers/extension-ui-controller.ts @@ -537,7 +537,6 @@ export class ExtensionUiController { model: this.ctx.session.model, isIdle: () => !this.ctx.session.isStreaming, hasPendingMessages: () => this.ctx.session.queuedMessageCount > 0, - hasQueuedMessages: () => this.ctx.session.queuedMessageCount > 0, abort: () => { this.ctx.session.abort(); }, diff --git a/packages/coding-agent/src/modes/controllers/mcp-command-controller.ts b/packages/coding-agent/src/modes/controllers/mcp-command-controller.ts index 717682096..8ad770813 100644 --- a/packages/coding-agent/src/modes/controllers/mcp-command-controller.ts +++ b/packages/coding-agent/src/modes/controllers/mcp-command-controller.ts @@ -401,12 +401,16 @@ export class MCPCommandController { ); return; } - const authResult = analyzeAuthError(error as Error); + const authResult = analyzeAuthError(error as Error, finalConfig.url); if (authResult.requiresAuth) { let oauth = authResult.authType === "oauth" ? (authResult.oauth ?? null) : null; if (!oauth && finalConfig.url) { try { - oauth = await discoverOAuthEndpoints(finalConfig.url, authResult.authServerUrl); + oauth = await discoverOAuthEndpoints( + finalConfig.url, + authResult.authServerUrl, + authResult.resourceMetadataUrl, + ); } catch { // Ignore discovery error and handle below. } @@ -742,11 +746,11 @@ export class MCPCommandController { } // Analyze the connection error to extract OAuth endpoints - const authResult = analyzeAuthError(connectionError!); + const authResult = analyzeAuthError(connectionError!, "url" in config ? config.url : undefined); let oauth = authResult.authType === "oauth" ? (authResult.oauth ?? null) : null; if (!oauth && (config.type === "http" || config.type === "sse") && config.url) { - oauth = await discoverOAuthEndpoints(config.url, authResult.authServerUrl); + oauth = await discoverOAuthEndpoints(config.url, authResult.authServerUrl, authResult.resourceMetadataUrl); } if (!oauth) { diff --git a/packages/coding-agent/src/modes/theme/theme.ts b/packages/coding-agent/src/modes/theme/theme.ts index d59f04d22..1a4a2c39f 100644 --- a/packages/coding-agent/src/modes/theme/theme.ts +++ b/packages/coding-agent/src/modes/theme/theme.ts @@ -2404,8 +2404,10 @@ export function getEditorTheme(): EditorTheme { export function getSettingsListTheme(): import("@oh-my-pi/pi-tui").SettingsListTheme { return { - label: (text: string, selected: boolean) => (selected ? theme.fg("accent", text) : text), - value: (text: string, selected: boolean) => (selected ? theme.fg("accent", text) : theme.fg("muted", text)), + label: (text: string, selected: boolean, changed: boolean) => + changed ? theme.fg("statusLineGitDirty", text) : selected ? theme.fg("accent", text) : text, + value: (text: string, selected: boolean, changed: boolean) => + selected ? theme.fg("accent", text) : changed ? theme.fg("statusLineGitDirty", text) : theme.fg("muted", text), description: (text: string) => theme.fg("dim", text), cursor: theme.fg("accent", `${theme.nav.cursor} `), hint: (text: string) => theme.fg("dim", text), diff --git a/packages/coding-agent/src/prompts/tools/ast-edit.md b/packages/coding-agent/src/prompts/tools/ast-edit.md index 29090214a..1be7238f9 100644 --- a/packages/coding-agent/src/prompts/tools/ast-edit.md +++ b/packages/coding-agent/src/prompts/tools/ast-edit.md @@ -14,7 +14,7 @@ Performs structural AST-aware rewrites via native ast-grep. -- Replacement summary, per-file replacement counts, and change diffs as `-LINE+ID|before` / `+LINE+ID|after` lines +- Replacement summary, per-file replacement counts, and change diffs as `¶src/foo.ts#1a2b`, `-12:before`, `+12:after` lines in hashline mode - Parse issues when files cannot be processed diff --git a/packages/coding-agent/src/prompts/tools/ast-grep.md b/packages/coding-agent/src/prompts/tools/ast-grep.md index 9cb49c440..c35809682 100644 --- a/packages/coding-agent/src/prompts/tools/ast-grep.md +++ b/packages/coding-agent/src/prompts/tools/ast-grep.md @@ -18,7 +18,7 @@ Performs structural code search using AST matching via native ast-grep. - Grouped matches with file path, byte range, line/column ranges, metavariable captures -- Match lines are anchor-prefixed: `*LINE+ID|content` for the matched line and ` LINE+ID|content` (leading space) for surrounding context +- Match lines are numbered under a file-hash header in hashline mode: `¶src/foo.ts#1a2b`, `*42:content` for the matched line, ` 43:content` for context - Summary counts (`totalMatches`, `filesWithMatches`, `filesSearched`) and parse issues when present diff --git a/packages/coding-agent/src/prompts/tools/eval.md b/packages/coding-agent/src/prompts/tools/eval.md index 78dec551d..b95b09677 100644 --- a/packages/coding-agent/src/prompts/tools/eval.md +++ b/packages/coding-agent/src/prompts/tools/eval.md @@ -1,7 +1,7 @@ Run code in a persistent kernel using a list of cells. -Each call submits one or more cells. Cells run in array order. State persists within each language across cells **and across tool calls**. +Each call submits one or more cells. Cells run in array order. State persists within each language across cells, tool calls, and subagents spawned with `task`; variables a parent or subagent declares are visible to the other on the same shared executor. Cell fields: diff --git a/packages/coding-agent/src/prompts/tools/find.md b/packages/coding-agent/src/prompts/tools/find.md index 4bf1fcd1f..3d0742495 100644 --- a/packages/coding-agent/src/prompts/tools/find.md +++ b/packages/coding-agent/src/prompts/tools/find.md @@ -1,4 +1,4 @@ -Finds files using fast pattern matching that works with any codebase size. +Finds files and directories using fast pattern matching that works with any codebase size. - `paths` is required and accepts an array of globs, files, or directories @@ -10,7 +10,7 @@ Finds files using fast pattern matching that works with any codebase size. -Matching file paths sorted by modification time (most recent first). Truncated at 1000 entries or 50KB (configurable via `limit`). +Matching file and directory paths sorted by modification time (most recent first). Directories are suffixed with `/`. Truncated at 1000 entries or 50KB (configurable via `limit`). @@ -20,6 +20,8 @@ Matching file paths sorted by modification time (most recent first). Truncated a `{"paths": ["src/**/*.ts", "test/**/*.ts"]}` # Find gitignored files like .env `{"paths": [".env*"], "gitignore": false}` +# Find directories matching a name (returns both files and dirs; directories are suffixed with `/`) +`{"paths": ["**/tests"]}` # Long-running search on a slow volume `{"paths": ["/Volumes/Storage/**/*.py"], "timeout": 30}` diff --git a/packages/coding-agent/src/prompts/tools/hashline.md b/packages/coding-agent/src/prompts/tools/hashline.md index f56d0a4c6..b0887c968 100644 --- a/packages/coding-agent/src/prompts/tools/hashline.md +++ b/packages/coding-agent/src/prompts/tools/hashline.md @@ -1,132 +1,109 @@ Your patch language is a compact, line-anchored edit format. -A patch contains one or more file sections. The first non-blank line of every edit section MUST be `¶PATH`. -Operations reference lines in the file by their line number and hash, called "Anchors", e.g. `5th`, `123ab`. -You MUST copy them verbatim from the latest output for the file you're editing. +A patch contains one or more file sections. Each anchored section starts with `¶PATH#HASH`, copied verbatim from the latest `read`/`search` output. `HASH` is a 4-hex file hash; `¶PATH` without `#HASH` is allowed only for new-file / `BOF` / `EOF` boundary inserts. -Purely textual format. The tool has NO awareness of language, indentation, brackets, fences, or table widths. You MUST emit valid syntax in replacements/insertions. +Operations reference lines by bare line number (`5`, `123`). Payload text is verbatim — NEVER escape unicode. The tool has NO awareness of language, indentation, brackets, fences, or table widths. Emit valid syntax in replacements/insertions. -¶PATH header: subsequent ops apply to PATH -Each op line is ONE of: -ANCHOR↑ insert ABOVE the anchored line (or BOF); payload may follow inline after `↑` and/or on subsequent lines -ANCHOR↓ insert BELOW the anchored line (or EOF); payload may follow inline after `↓` and/or on subsequent lines -A-B→ replace the inclusive range A..B with payload; delete the range if no payload follows -A→ shorthand for A-A→ +¶PATH#HASH header: subsequent anchored ops apply to PATH at file hash HASH +¶PATH unbound header: only BOF/EOF boundary inserts +LINE↑PAYLOAD insert ABOVE the anchored line (or BOF) +LINE↓PAYLOAD insert BELOW the anchored line (or EOF) +A-B:PAYLOAD replace the inclusive range A..B with PAYLOAD +A:PAYLOAD shorthand for A-A:PAYLOAD +A-B! delete the inclusive range A..B (payload forbidden) +A! shorthand for A-A! - -- The arrow points to where the content lands relative to the anchor: `↑` above, `↓` below, `→` overwrite. -- Payload text is verbatim — NEVER escape unicode. -- An op line is `ANCHOR[INLINE_PAYLOAD]`. Anything after the sigil on the same line is the first payload line; subsequent payload lines follow on the next lines. -- A payload run ends at the next op line, the next `¶PATH`, an envelope marker, or EOF. -- `A-B→` with no payload deletes the range. To keep a blank line, include one explicit empty payload line on the next line. -- **Payload is only what's NEW relative to your range:** - - `→` replaces inside; NEVER include lines outside. - - `↑`/`↓` adds at the anchor; NEVER repeat line A or neighbors. - - Payload matching nearby content duplicates — drop it or widen. -- **Pick a self-contained unit first.** Touching a multiline construct? Widen to the whole thing. -- Then smallest op: add → `↑`/`↓`; delete/replace → `→`. - + +- The first payload line is whatever follows the sigil on the op line. Additional payload lines follow on the next lines and append after the first. +- An empty inline IS an empty first line. So bare `A↓` / `A↑` insert one blank line; bare `A:` / `A-B:` replace with one blank line. `A↓\nfoo` inserts blank-then-`foo`, NOT just `foo`. +- Payload ends at the next op, next `¶PATH`, envelope marker, or EOF. Blank lines immediately before a next op or `¶PATH` are dropped; blank lines between content lines are preserved. + - -When braces bound your edit, you SHOULD prefer these shapes: -- **Whole block**: range spans `{` through matching `}`. -- **Signature only**: one-line `→` on the opener; body untouched. -- **Insert inside**: anchor on `{` or last interior line; NEVER repeat the braces. -- **End on `}`**: only when that `}` is part of the change. Otherwise extend or stop earlier. - + +- The sigil tells where content lands: `↑` above, `↓` below, `:` replaces, `!` deletes. +- **Payload is only what's NEW relative to your range.** `:` replaces inside; `↑`/`↓` add at anchor. NEVER repeat the anchor line or neighbors — that duplicates them. +- **Pick a self-contained unit.** Touching a multiline construct (return, array, brace block, JSX element)? Widen the range to span it. Don't bisect. +- Smallest op wins: add with `↑`/`↓`; replace with `:`; delete with `!`. +- Anchors reference the file as last read. ONE patch, ONE coordinate space — later ops still use original line numbers. + - **NEVER replay past your range.** Stop before B+1; extend B if it must go. -- **NEVER duplicate chunks inside one payload.** Caught re-emitting? Rewrite. -- **Anchor only inside the visible region.** B+1 truncated? Re-`read` first. -- **You SHOULD prefer the narrowest self-contained edit.** Narrow range beats wide range. -- **Anchors reference the file as last read.** NEVER shift for prior ops. -- **One `↓`/`↑` op per block, NOT per line.** N lines = ONE op, N payloads. Collapse adjacent ops. -- **NEVER fabricate anchor hashes.** Missing? Re-`read`. +- **NEVER duplicate chunks inside one payload.** +- **Read lines look like replace ops.** `84:content` already means "make line 84 equal to content" — don't echo a context line before it. +- **NEVER fabricate file hashes.** Missing? Re-`read`. +- **`A!` deletes silently.** Deleting a line that closes/opens a block (`}`, `} else {`, `})`, `*/`) breaks structure with no parse error. -{{hline 1 "const TITLE = \"Mr\";"}} -{{hline 2 "export function greet(name) {"}} -{{hline 3 "\treturn ["}} -{{hline 4 "\t\tTITLE,"}} -{{hline 5 "\t\tname?.trim() || \"guest\","}} -{{hline 6 "\t].join(\" \");"}} +¶mod.ts#1a2b +{{hline 1 'const TITLE = "Mr";'}} +{{hline 2 'export function greet(name) {'}} +{{hline 3 ' return ['}} +{{hline 4 ' TITLE,'}} +{{hline 5 ' name?.trim() || "guest",'}} +{{hline 6 ' ].join(" ");'}} {{hline 7 "}"}} -# Replace one line (the payload must re-emit the original indentation) -¶mod.ts -{{hrefr 1}}→ -const TITLE = "Mrs"; +# Replace one line (inline payload preserves original indentation) +¶mod.ts#1a2b +{{hrefr 1}}:const TITLE = "Mrs"; -# Replace a full multiline statement (widen to a self-contained boundary) -¶mod.ts -{{hrefr 3}}-{{hrefr 6}}→ - return [ +# Replace a multiline statement — first line inline, rest below +¶mod.ts#1a2b +{{hrefr 3}}-{{hrefr 6}}: return [ "Mrs", name?.trim() || "guest", ].join(" "); -# Insert ABOVE/BELOW a line -¶mod.ts -{{hrefr 4}}↓ - "Dr", -{{hrefr 5}}↑ - "Dr", +# Insert ABOVE / BELOW a line +¶mod.ts#1a2b +{{hrefr 4}}↓ "Dr", +{{hrefr 5}}↑ "Dr", -# Append to file -¶mod.ts -EOF↓ -export const done = true; +# Delete one line / blank a line / insert a blank line +¶mod.ts#1a2b +{{hrefr 5}}! +{{hrefr 6}}: +{{hrefr 7}}↑ -# Delete a line +# Create a file / append to one (hash optional for boundary-only inserts) +¶new.ts +BOF↓export const done = true; ¶mod.ts -{{hrefr 5}}→ +EOF↓export const done = true; -# Blank a line (replace with LF: the empty payload is the blank line before `EOF↓`) -¶mod.ts -{{hrefr 5}}→ - -EOF↓ -export const done = true; +# Multi-file patch +¶src/a.ts#1a2b +12:const enabled = true; +¶src/b.ts#3c4d +20! # WRONG — replaces 2 lines just to add one. -¶mod.ts -{{hrefr 1}}-{{hrefr 2}}→ -const TITLE = "Mr"; +¶mod.ts#1a2b +{{hrefr 1}}-{{hrefr 2}}:const TITLE = "Mr"; const DEBUG = false; export function greet(name) { -# RIGHT — same effect, one-line insert -¶mod.ts -{{hrefr 1}}↓ -const DEBUG = false; -# WRONG — replace from the middle of a larger statement (error-prone) -¶mod.ts -{{hrefr 4}}-{{hrefr 5}}→ - "Dr", +# RIGHT — one-line insert +¶mod.ts#1a2b +{{hrefr 1}}↓const DEBUG = false; + +# WRONG — bisects a multiline statement +¶mod.ts#1a2b +{{hrefr 4}}-{{hrefr 5}}: "Dr", name?.trim() || "guest", + # RIGHT — widen to the full statement -¶mod.ts -{{hrefr 3}}-{{hrefr 6}}→ - return [ +¶mod.ts#1a2b +{{hrefr 3}}-{{hrefr 6}}: return [ "Dr", name?.trim() || "guest", ].join(" "); - - -- Copy anchors verbatim (line number + 2-char hash); NEVER include the `|TEXT` body. -- NEVER write unified diff syntax. Headers are `¶PATH`; ops put `↑`/`↓`/`→` AFTER the anchor. -- `A-B→` deletes the range when no payload follows. To keep a blank line, include one explicit empty payload line. -- `A-B→` with payload writes exactly that payload. Edge line matches just outside? Widen, or it duplicates. -- Multiple ops are cheap. SHOULD prefer two narrow ops over one wide `→`. - - Before `A-B→`, mentally delete A..B. Splits an unclosed bracket/brace/string from above, or orphans a closer inside? You're bisecting a construct. -- NEVER use this tool to reformat code (indentation, whitespace, line wrapping, style). Run the project's formatter instead. - diff --git a/packages/coding-agent/src/prompts/tools/read.md b/packages/coding-agent/src/prompts/tools/read.md index 50d9e1cbe..b8b05fc3d 100644 --- a/packages/coding-agent/src/prompts/tools/read.md +++ b/packages/coding-agent/src/prompts/tools/read.md @@ -28,17 +28,17 @@ Append `:` to `path`. The bare path falls back to the default mode. - Reading a directory path returns a depth-limited dirent listing. {{#if IS_HL_MODE}} -- Reading a file with an explicit selector returns lines prefixed with `line+hash` anchors: `41th|def alpha():`. The 2-char hash is a content fingerprint that `edit` / `apply_patch` consume — copy it verbatim, NEVER fabricate. The pipe character after the hash is a separator, not part of the file content. +- Reading a file with an explicit selector emits a file-hash header and numbered lines: `¶src/foo.ts#1a2b` then `41:def alpha():`. Copy the `¶PATH#HASH` header for anchored edits; ops use bare line numbers. NEVER fabricate the hash. {{else}} {{#if IS_LINE_NUMBER_MODE}} - Reading a file with an explicit selector returns lines prefixed with line numbers: `41|def alpha():`. {{/if}} {{/if}} -- Parseable code without a selector returns a **structural summary**: declarations kept, large bodies collapsed to `..` (merged brace pair) or `…` (standalone). Summarized output ends with a footer of the form: +- Parseable code without a selector returns a **structural summary**: declarations kept, large bodies collapsed to `..` (merged brace pair) or `…` (standalone). Summarized output ends with a footer demonstrating the multi-range selector you can use to recover the elided bodies, e.g.: - `[NN lines across MM elided regions; read :raw or a line range like :1-9999 for verbatim content]` + `[NN lines elided; re-read needed ranges, e.g. :5-16,40-80]` - If the elided body is what you actually need, re-issue the **exact selector the footer names**. NEVER guess what's inside `..` / `…` — those markers carry no content. + Re-issue **only the relevant range(s)** using the multi-range selector (e.g. `:5-16,120-200`). NEVER guess what's inside `..` / `…` — those markers carry no content. NEVER re-read the whole file or use `:raw` when targeted ranges suffice. # Documents & Notebooks diff --git a/packages/coding-agent/src/prompts/tools/search.md b/packages/coding-agent/src/prompts/tools/search.md index 429a6ed60..3753b88b8 100644 --- a/packages/coding-agent/src/prompts/tools/search.md +++ b/packages/coding-agent/src/prompts/tools/search.md @@ -2,14 +2,14 @@ Searches files using powerful regex matching. - Supports Rust regex syntax (RE2-style — no lookaround or backreferences). Use line anchors or post-filters instead of (?!…)/(? {{#if IS_HL_MODE}} -- Text output is anchor-prefixed: `*5th|content` (match) or ` 9x}|content` (context, leading space). The 2-char suffix is a content fingerprint. The `|` before content is a separator, not part of the file content. +- Text output emits a file-hash header per matched file plus numbered lines: `¶src/login.ts#3c4d`, `*42:if (user.id) {` (match), ` 43:return user;` (context). Copy the header for anchored edits; ops use bare line numbers. {{else}} {{#if IS_LINE_NUMBER_MODE}} - Text output is line-number-prefixed diff --git a/packages/coding-agent/src/prompts/tools/vim.md b/packages/coding-agent/src/prompts/tools/vim.md deleted file mode 100644 index 40edff663..000000000 --- a/packages/coding-agent/src/prompts/tools/vim.md +++ /dev/null @@ -1,98 +0,0 @@ -Vim-style `edit` mode. The tool name stays `edit`; every call requires `file`, and the buffer loads automatically on first use. -- `{"file": "path"}` - view file -- `{"file": "path", "steps": [{"kbd": ["…"], "insert": "…"}]}` - edit file - -**Multi-location edits: always edit highest line number first (bottom-up).** Each insert shifts lines below it. - -## steps vs kbd vs insert - -`steps` = ordered editing steps. Each step runs `kbd`, then optionally types `insert`. -`kbd` = Vim commands only (`dd`, `G`, `o`, `cc`, `gg`, etc.). -`insert` = raw text content to type into the buffer. -`o`/`O` already create a new line — do not start `insert` with `\n`. A trailing `\n` in `insert` adds an extra blank line. - -Never put text content in `kbd`. Only Vim keystrokes go there. -- BAD: `{"steps": [{"kbd": ["1Gohello world"]}]}` -- BAD: `{"steps": [{"kbd": ["1Go", "hello world"]}]}` -- BAD: `{"steps": [{"kbd": ["1Ao"], "insert": "text"}]}` -- GOOD: `{"steps": [{"kbd": ["1Go"], "insert": "hello world"}]}` - -If a step uses `insert`, the last `kbd` entry in that step must leave INSERT mode active (`o`, `O`, `i`, `a`, `A`, `cc`, `C`, `s`, `S`). - -Each non-final `kbd` entry inside a step must end in NORMAL mode (add ``). - -Between steps, the tool auto-exits INSERT mode. - -Whitespace in `kbd` is literal. Do not use spaces as separators between keys; `ggdGi` is one sequence, not `ggdG i`. - -Common mistake: `Ni` means "insert N copies", NOT "insert at line N". To insert at line N, use `NGo` (below) or `NGO` (above). -## Editing patterns - -`NGo` = new line BELOW line N. `NGO` = new line ABOVE line N. - -Insert new line after line 3: -```json -{"file": "f.py", "steps": [{"kbd": ["3Go"], "insert": " new line here"}]} -``` - -Insert new line before line 3: -```json -{"file": "f.py", "steps": [{"kbd": ["3GO"], "insert": " new line here"}]} -``` - -Replace line N: -```json -{"file": "f.py", "steps": [{"kbd": ["5Gcc"], "insert": " replacement content"}]} -``` - -Replace entire file. `ggdGi` = go to top, delete all, enter INSERT. Use that exact sequence when rewriting the whole file: -```json -{"file": "f.py", "steps": [{"kbd": ["ggdGi"], "insert": "entire new file content"}]} -``` - -Multi-location edit — edit **highest line number first** (bottom-up) so inserts don't shift later targets: -```json -{"file": "f.py", "steps": [ - {"kbd": ["8Go"], "insert": " print(result)"}, - {"kbd": ["3Go"], "insert": "def helper(x):\n return x + 1"} -]} -``` -Each `o`/`O` insert adds lines, shifting everything below. Bottom-up order keeps all line numbers valid. Use `\n` within `insert` for multi-line content. - -Navigation or search step without insert: -```json -{"file": "f.py", "steps": [{"kbd": ["/pattern"]}]} -``` - -Find and replace: -```json -{"file": "f.py", "steps": [{"kbd": [":%s/old/new/g"]}]} -``` - -Delete line range: -```json -{"file": "f.py", "steps": [{"kbd": [":3,5d"]}]} -``` -Ex commands always start with `:` and end with ``. `3,5d` without `:` is NOT an ex command — it is interpreted as normal-mode keystrokes and will fail. - -## Undo mistakes -- `{"file": "f.py", "steps": [{"kbd": ["u"]}]}` - undo last change -- `{"file": "f.py", "steps": [{"kbd": ["3u"]}]}` - undo last 3 changes - -`:e!` reloads from disk. Warning: because non-paused calls auto-save, `:e!` reloads your last saved state, not the original file. Use `u` to undo instead. If stuck, use `ggdGi` with the full desired file content. - -## Session persistence - -The edit buffer in vim mode persists across tool calls. Cursor position, undo history, and file state are maintained until you close the buffer. Auto-save happens once after all steps in a non-paused call complete. - -## Supported - -Keys: `` `` `` `` `` `` `` `` `` -Motions: `h j k l w b e 0 $ ^ + - _ gg G { } f F t T % H M L ; ,` with counts -Operators: `d c y p` with motions and text objects (`iw aw ip ap i" a" i( a( i{ a{`) -Insert: `i a o O I A cc C s S R` - these all enter INSERT mode; do not add another `i` after them -Visual: `v V` with `d y c > < ~ r u U p P o J` -Other: `.` repeat, `u`/`` undo/redo, `/pattern` search, `n N * #`, `gv` `gJ` `gU` `gu` `ZZ` `ZQ` -Ex: `:w` `:q` `:wq` `:e` `:e!` `:N` `:s///` `:%s///` `:N,Md` `:%d` `:N,Mt N` `:sort` `:j` `:j!` `:g/pattern/d` `:v/pattern/d` -Addresses: absolute line numbers, `.`, `$`, and `+N`/`-N` relative offsets, including ranges like `:.,$d` and `:.+2,$g/pattern/d` -More ex: `:up` `:N,My` `:put` `:put!` `:N,Mco $` `:N,Mm $` diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index 3ea03426c..2dad4dfc0 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -51,6 +51,7 @@ import "./discovery"; import { resolveConfigValue } from "./config/resolve-config-value"; import { initializeWithSettings } from "./discovery"; import { disposeAllKernelSessions, disposeKernelSessionsByOwner } from "./eval/py/executor"; +import { defaultEvalSessionId } from "./eval/session-id"; import { TtsrManager } from "./export/ttsr"; import { type CustomCommandsLoadResult, @@ -84,11 +85,7 @@ import type { HindsightSessionState } from "./hindsight/state"; import { LocalProtocolHandler, type LocalProtocolOptions } from "./internal-urls"; import { LSP_STARTUP_EVENT_CHANNEL, type LspStartupEvent } from "./lsp/startup-events"; import { discoverAndLoadMCPTools, MCPManager, type MCPToolsLoadResult } from "./mcp"; -import { - collectDiscoverableMCPTools, - formatDiscoverableMCPToolServerSummary, - selectDiscoverableMCPToolNamesByServer, -} from "./mcp/discoverable-tool-metadata"; + import { resolveMemoryBackend } from "./memory-backend"; import asyncResultTemplate from "./prompts/tools/async-result.md" with { type: "text" }; import { AgentRegistry, MAIN_AGENT_ID } from "./registry/agent-registry"; @@ -117,6 +114,9 @@ import { parseThinkingLevel, resolveThinkingLevelForModel, toReasoningEffort } f import { collectDiscoverableTools, type DiscoverableTool, + filterBySource, + formatDiscoverableToolServerSummary, + selectDiscoverableToolNamesByServer, summarizeDiscoverableTools, } from "./tool-discovery/tool-index"; import { @@ -318,6 +318,8 @@ export interface CreateAgentSessionOptions { agentRegistry?: AgentRegistry; /** Parent task ID prefix for nested artifact naming (e.g., "6-Extensions") */ parentTaskPrefix?: string; + /** Inherited eval executor session id for subagents sharing parent eval state. */ + parentEvalSessionId?: string; /** Session manager. Default: session stored under the configured agentDir sessions root */ sessionManager?: SessionManager; @@ -1177,6 +1179,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} taskDepth: options.taskDepth ?? 0, getSessionFile: () => sessionManager.getSessionFile() ?? null, getEvalKernelOwnerId: () => evalKernelOwnerId, + getEvalSessionId: () => + session?.getEvalSessionId() ?? options.parentEvalSessionId ?? defaultEvalSessionId(toolSession), assertEvalExecutionAllowed: () => session?.assertEvalExecutionAllowed(), trackEvalExecution: (execution, abortController) => session ? session.trackEvalExecution(execution, abortController) : execution, @@ -1196,8 +1200,6 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} getTodoPhases: () => session.getTodoPhases(), setTodoPhases: phases => session.setTodoPhases(phases), isMCPDiscoveryEnabled: () => session.isMCPDiscoveryEnabled(), - getDiscoverableMCPTools: () => session.getDiscoverableMCPTools(), - getDiscoverableMCPSearchIndex: () => session.getDiscoverableMCPSearchIndex(), getSelectedMCPToolNames: () => session.getSelectedMCPToolNames(), activateDiscoveredMCPTools: toolNames => session.activateDiscoveredMCPTools(toolNames), // Generic tool discovery (unified — covers built-in + MCP + extension) @@ -1581,7 +1583,9 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} tools: Map, ): Promise => { toolContextStore.setToolNames(toolNames); - const discoverableMCPTools = mcpDiscoveryEnabled ? collectDiscoverableMCPTools(tools.values()) : []; + const discoverableMCPTools: DiscoverableTool[] = mcpDiscoveryEnabled + ? filterBySource(collectDiscoverableTools(tools.values()), "mcp") + : []; const activeToolNames = new Set(toolNames); const discoverableBuiltinTools: DiscoverableTool[] = effectiveDiscoveryMode === "all" @@ -1592,18 +1596,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} { source: "builtin" }, ) : []; - const discoverableToolsForDesc: DiscoverableTool[] = [ - ...discoverableBuiltinTools, - ...discoverableMCPTools.map(t => ({ - name: t.name, - label: t.label, - summary: t.description, - source: "mcp" as const, - serverName: t.serverName, - mcpToolName: t.mcpToolName, - schemaKeys: t.schemaKeys, - })), - ]; + const discoverableToolsForDesc: DiscoverableTool[] = [...discoverableBuiltinTools, ...discoverableMCPTools]; const discoverableToolSummary = summarizeDiscoverableTools(discoverableToolsForDesc); const hasDiscoverableTools = mcpDiscoveryEnabled && toolNames.includes("search_tool_bm25") && discoverableToolsForDesc.length > 0; @@ -1647,7 +1640,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} repeatToolDescriptions, intentField, mcpDiscoveryMode: hasDiscoverableTools, - mcpDiscoveryServerSummaries: discoverableToolSummary.servers.map(formatDiscoverableMCPToolServerSummary), + mcpDiscoveryServerSummaries: discoverableToolSummary.servers.map(formatDiscoverableToolServerSummary), eagerTasks, secretsEnabled, workspaceTree: workspaceTreePromise, @@ -1665,9 +1658,22 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} }; const toolNamesFromRegistry = Array.from(toolRegistry.keys()); - const requestedToolNames = - (options.toolNames ? [...new Set(options.toolNames.map(name => name.toLowerCase()))] : undefined) ?? - toolNamesFromRegistry; + const explicitlyRequestedToolNames = options.toolNames + ? [...new Set(options.toolNames.map(name => name.toLowerCase()))] + : undefined; + // When `requireYieldTool` is set, the subagent's prompts and idle-reminders demand a + // `yield` call to terminate. The tool registry already includes `yield` (see + // `createTools`), but an explicit `toolNames` list would otherwise drop it from the + // active set — leaving the model unable to satisfy the contract. Mirror the same + // invariant `parseAgentFields` enforces on frontmatter `tools`. + if ( + options.requireYieldTool === true && + explicitlyRequestedToolNames && + !explicitlyRequestedToolNames.includes("yield") + ) { + explicitlyRequestedToolNames.push("yield"); + } + const requestedToolNames = explicitlyRequestedToolNames ?? toolNamesFromRegistry; const normalizedRequested = requestedToolNames.filter(name => toolRegistry.has(name)); const requestedToolNameSet = new Set(normalizedRequested); // Effective discovery mode: tools.discoveryMode takes precedence; mcp.discoveryMode is back-compat alias. @@ -1693,8 +1699,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} (settings.get("mcp.discoveryDefaultServers") ?? []).map(serverName => serverName.trim()).filter(Boolean), ); const discoveryDefaultServerToolNames = mcpDiscoveryEnabled - ? selectDiscoverableMCPToolNamesByServer( - collectDiscoverableMCPTools(toolRegistry.values()), + ? selectDiscoverableToolNamesByServer( + filterBySource(collectDiscoverableTools(toolRegistry.values()), "mcp"), discoveryDefaultServers, ) : []; @@ -1994,6 +2000,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} agentId: resolvedAgentId, agentRegistry, providerSessionId: options.providerSessionId, + parentEvalSessionId: options.parentEvalSessionId, }); hasSession = true; if (asyncJobManager) { diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index 3a6e953ad..45c0faf28 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -108,6 +108,7 @@ import { executePython as executePythonCommand, type PythonResult, } from "../eval/py/executor"; +import { defaultEvalSessionId } from "../eval/session-id"; import { type BashResult, executeBash as executeBashCommand } from "../exec/bash-executor"; import { exportSessionToHtml } from "../export/html"; import type { TtsrManager, TtsrMatchContext } from "../export/ttsr"; @@ -141,14 +142,6 @@ import { GoalRuntime } from "../goals/runtime"; import type { Goal, GoalModeState } from "../goals/state"; import type { HindsightSessionState } from "../hindsight/state"; import { type LocalProtocolOptions, resolveLocalUrlToPath } from "../internal-urls"; -import { - buildDiscoverableMCPSearchIndex, - collectDiscoverableMCPTools, - type DiscoverableMCPSearchIndex, - type DiscoverableMCPTool, - isMCPToolName, - selectDiscoverableMCPToolNamesByServer, -} from "../mcp/discoverable-tool-metadata"; import { resolveMemoryBackend } from "../memory-backend"; import { getCurrentThemeName, theme } from "../modes/theme/theme"; import type { PlanModeState } from "../plan-mode/state"; @@ -171,6 +164,9 @@ import { collectDiscoverableTools, type DiscoverableTool, type DiscoverableToolSearchIndex, + filterBySource, + isMCPToolName, + selectDiscoverableToolNamesByServer, } from "../tool-discovery/tool-index"; import { assertEditableFile } from "../tools/auto-generated-guard"; import type { CheckpointState } from "../tools/checkpoint"; @@ -312,6 +308,8 @@ export interface AgentSessionConfig { ttsrManager?: TtsrManager; /** Secret obfuscator for deobfuscating streaming edit content */ obfuscator?: SecretObfuscator; + /** Inherited eval executor session id from a parent agent. */ + parentEvalSessionId?: string; /** Logical owner for retained Python kernels created by this session. */ evalKernelOwnerId?: string; /** @@ -808,6 +806,7 @@ export class AgentSession { // Python execution state #evalAbortControllers = new Set(); #evalKernelOwnerId: string; + #parentEvalSessionId: string | undefined; /** * AsyncJobManager owned by this session (top-level only). Subagents leave * this undefined and **MUST NOT** dispose the global instance on teardown. @@ -865,8 +864,7 @@ export class AgentSession { */ #lastAppliedToolSignature: string | undefined; #mcpDiscoveryEnabled = false; - #discoverableMCPTools = new Map(); - #discoverableMCPSearchIndex: DiscoverableMCPSearchIndex | null = null; + #discoverableMCPTools = new Map(); #selectedMCPToolNames = new Set(); // Generic tool discovery (covers built-in + MCP + extension when tools.discoveryMode === "all") #discoverableToolSearchIndex: DiscoverableToolSearchIndex | null = null; @@ -997,6 +995,7 @@ export class AgentSession { this.settings = config.settings; // Power assertions are taken per turn (see #beginInFlight); nothing acquired here. this.#evalKernelOwnerId = config.evalKernelOwnerId ?? `agent-session:${Snowflake.next()}`; + this.#parentEvalSessionId = config.parentEvalSessionId; this.#ownedAsyncJobManager = config.ownedAsyncJobManager; this.#scopedModels = config.scopedModels ?? []; this.#thinkingLevel = config.thinkingLevel; @@ -2876,11 +2875,12 @@ export class AgentSession { return this.#retryAttempt; } - #collectDiscoverableMCPToolsFromRegistry(): Map { - return new Map(collectDiscoverableMCPTools(this.#toolRegistry.values()).map(tool => [tool.name, tool] as const)); + #collectDiscoverableMCPToolsFromRegistry(): Map { + const mcpTools = filterBySource(collectDiscoverableTools(this.#toolRegistry.values()), "mcp"); + return new Map(mcpTools.map(tool => [tool.name, tool] as const)); } - #setDiscoverableMCPTools(discoverableMCPTools: Map): void { + #setDiscoverableMCPTools(discoverableMCPTools: Map): void { this.#discoverableMCPTools = discoverableMCPTools; this.#invalidateDiscoveryCaches(); } @@ -2889,7 +2889,6 @@ export class AgentSession { * affect which tools should be discoverable: registry mutations (refreshMCPTools, * refreshRpcHostTools) or active-tool mutations (#applyActiveToolsByName). */ #invalidateDiscoveryCaches(): void { - this.#discoverableMCPSearchIndex = null; this.#discoverableToolSearchIndex = null; } @@ -2900,7 +2899,7 @@ export class AgentSession { #getConfiguredDefaultSelectedMCPToolNames(): string[] { return this.#filterSelectableMCPToolNames([ ...this.#defaultSelectedMCPToolNames, - ...selectDiscoverableMCPToolNamesByServer( + ...selectDiscoverableToolNamesByServer( this.#discoverableMCPTools.values(), this.#defaultSelectedMCPServerNames, ), @@ -2993,28 +2992,6 @@ export class AgentSession { return this.#mcpDiscoveryEnabled; } - /** @deprecated Use {@link getDiscoverableTools} with `{ source: "mcp" }` instead. - * Preserves the legacy `description`-bearing MCP shape for back-compat callers. */ - getDiscoverableMCPTools(): DiscoverableMCPTool[] { - return Array.from(this.#discoverableMCPTools.values()).map(t => ({ - name: t.name, - label: t.label, - description: t.description, - serverName: t.serverName, - mcpToolName: t.mcpToolName, - schemaKeys: t.schemaKeys, - })); - } - - /** @deprecated Use {@link getDiscoverableToolSearchIndex} instead. - * Returns the legacy MCP search index whose documents expose `tool.description`. */ - getDiscoverableMCPSearchIndex(): DiscoverableMCPSearchIndex { - if (!this.#discoverableMCPSearchIndex) { - this.#discoverableMCPSearchIndex = buildDiscoverableMCPSearchIndex(this.#discoverableMCPTools.values()); - } - return this.#discoverableMCPSearchIndex; - } - getSelectedMCPToolNames(): string[] { if (!this.#mcpDiscoveryEnabled) { return this.getActiveToolNames().filter(name => isMCPToolName(name) && this.#toolRegistry.has(name)); @@ -3062,17 +3039,7 @@ export class AgentSession { // For "mcp-only" mode we only return MCP tools. const mode = this.#resolveEffectiveDiscoveryMode(); const activeNames = new Set(this.getActiveToolNames()); - const mcpTools: DiscoverableTool[] = Array.from(this.#discoverableMCPTools.values()) - .filter(t => !activeNames.has(t.name)) - .map(t => ({ - name: t.name, - label: t.label, - summary: t.description, - source: "mcp" as const, - serverName: t.serverName, - mcpToolName: t.mcpToolName, - schemaKeys: t.schemaKeys, - })); + const mcpTools = Array.from(this.#discoverableMCPTools.values()).filter(t => !activeNames.has(t.name)); const builtinTools: DiscoverableTool[] = mode === "all" ? this.#collectDiscoverableBuiltinTools() : []; const allTools = [...builtinTools, ...mcpTools]; return filter?.source ? allTools.filter(t => t.source === filter.source) : allTools; @@ -3693,6 +3660,13 @@ export class AgentSession { get sessionId(): string { return this.#providerSessionId ?? this.sessionManager.getSessionId(); } + getEvalSessionId(): string | null { + if (this.#parentEvalSessionId !== undefined) return this.#parentEvalSessionId; + return defaultEvalSessionId({ + cwd: this.sessionManager.getCwd(), + getSessionFile: () => this.sessionManager.getSessionFile() ?? null, + }); + } /** Current session display name, if set */ get sessionName(): string | undefined { @@ -4225,7 +4199,6 @@ export class AgentSession { void this.dispose(); process.exit(0); }, - hasQueuedMessages: () => this.queuedMessageCount > 0, getContextUsage: () => this.getContextUsage(), waitForIdle: () => this.waitForIdle(), newSession: async options => { @@ -7430,9 +7403,13 @@ export class AgentSession { } } - // Use the same session ID as eval's Python backend for kernel sharing - const sessionFile = this.sessionManager.getSessionFile(); - const sessionId = sessionFile ? `session:${sessionFile}:cwd:${cwd}` : `cwd:${cwd}`; + // Use the same session ID as eval's Python backend for kernel sharing. + const sessionId = + this.getEvalSessionId() ?? + defaultEvalSessionId({ + cwd, + getSessionFile: () => this.sessionManager.getSessionFile() ?? null, + }); const result = await executePythonCommand(code, { cwd, sessionId, @@ -7571,11 +7548,11 @@ export class AgentSession { * Generate an ephemeral reply to a background message (e.g. an IRC ping from * another agent) using this session's current model + system prompt + history. * - * The reply is computed via a side-channel `streamSimple` call (analogous to - * `/btw`) so it never blocks on the recipient's in-flight tool calls. After - * the reply is generated, both the incoming question and the auto-reply are - * queued for injection into the recipient's persisted history so the model - * sees the exchange on its next turn. Injection happens immediately when the + * The incoming message is queued for injection into the recipient's persisted + * history immediately so timeouts/abort still preserve delivery. The reply is + * computed via a side-channel `streamSimple` call (analogous to `/btw`) so it + * never blocks on the recipient's in-flight tool calls. When a reply is + * generated, it is queued separately. Injection happens immediately when the * session is idle, otherwise it is deferred until streaming ends. */ async respondAsBackground(args: { @@ -7604,8 +7581,8 @@ export class AgentSession { timestamp: incomingTimestamp, }); + this.#queueBackgroundExchangeInjection([incomingRecord]); if (!awaitReply) { - this.#queueBackgroundExchangeInjection([incomingRecord]); return { replyText: null }; } @@ -7635,7 +7612,7 @@ export class AgentSession { kind: "reply", timestamp: replyRecord.timestamp, }); - this.#queueBackgroundExchangeInjection([incomingRecord, replyRecord]); + this.#queueBackgroundExchangeInjection([replyRecord]); return { replyText }; } @@ -7717,10 +7694,17 @@ export class AgentSession { // removes the surface entirely. tools: [], }; + const cacheSessionId = this.sessionId; const options = this.prepareSimpleStreamOptions( { apiKey, - sessionId: this.sessionId, + // Side-channel turns must not share OpenAI/Codex append-only + // conversation state with the main agent turn: IRC and /btw can run + // while the main turn is mid-tool-call. Keep the prompt-cache key + // stable, but give provider routing a unique request lineage. + sessionId: `${cacheSessionId}:side:${Snowflake.next()}`, + promptCacheKey: cacheSessionId, + preferWebsockets: false, reasoning: toReasoningEffort(this.thinkingLevel), hideThinkingSummary: this.agent.hideThinkingSummary, serviceTier: this.serviceTier, diff --git a/packages/coding-agent/src/session/agent-storage.ts b/packages/coding-agent/src/session/agent-storage.ts index 17c226420..5fe1aa458 100644 --- a/packages/coding-agent/src/session/agent-storage.ts +++ b/packages/coding-agent/src/session/agent-storage.ts @@ -240,11 +240,10 @@ FROM model_usage_legacy } /** - * Retrieves all settings from storage (legacy, for migration only). - * Settings are now stored in config.yml. This method is only used - * during migration from agent.db to config.yml. + * Reads legacy settings persisted in the agent.db `settings` table. + * The canonical settings store is `config.yml`; this accessor only + * exists so the config loader can migrate values from older installs. * @returns Settings object, or null if no settings are stored - * @deprecated Use config.yml instead. This is only for migration. */ getSettings(): Settings | null { const rows = (this.#listSettingsStmt.all() as SettingsRow[]) ?? []; @@ -263,16 +262,6 @@ FROM model_usage_legacy return settings as Settings; } - /** - * @deprecated Settings are now stored in config.yml, not agent.db. - * This method is kept for backward compatibility but does nothing. - */ - saveSettings(settings: Settings): void { - logger.warn("AgentStorage.saveSettings is deprecated - settings are now stored in config.yml", { - keys: Object.keys(settings), - }); - } - /** * Records model usage, updating the last-used timestamp. * @param modelKey - Model key in "provider/modelId" format diff --git a/packages/coding-agent/src/session/streaming-output.ts b/packages/coding-agent/src/session/streaming-output.ts index 88da94201..26f97e2ae 100644 --- a/packages/coding-agent/src/session/streaming-output.ts +++ b/packages/coding-agent/src/session/streaming-output.ts @@ -9,7 +9,7 @@ import { sanitizeWithOptionalSixelPassthrough } from "../utils/sixel"; export const DEFAULT_MAX_LINES = 3000; export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB -export const DEFAULT_MAX_COLUMN = 1024; // Max chars per grep match line +export const DEFAULT_MAX_COLUMN = 512; // Max chars per grep match line const NL = "\n"; const ELLIPSIS = "…"; diff --git a/packages/coding-agent/src/slash-commands/acp-builtins.ts b/packages/coding-agent/src/slash-commands/acp-builtins.ts index 1a0e32a9e..1e28f9ddb 100644 --- a/packages/coding-agent/src/slash-commands/acp-builtins.ts +++ b/packages/coding-agent/src/slash-commands/acp-builtins.ts @@ -1,9 +1,9 @@ import type { AvailableCommand } from "@agentclientprotocol/sdk"; import { BUILTIN_SLASH_COMMANDS_INTERNAL, lookupBuiltinSlashCommand } from "./builtin-registry"; import { parseSlashCommand } from "./helpers/parse"; -import type { AcpBuiltinCommandRuntime, AcpBuiltinSlashCommandResult } from "./types"; +import type { AcpBuiltinSlashCommandResult, SlashCommandRuntime } from "./types"; -export type { AcpBuiltinCommandRuntime, AcpBuiltinSlashCommandResult } from "./types"; +export type { AcpBuiltinSlashCommandResult } from "./types"; /** * Commands advertised to ACP clients. Entries without a text-mode `handle` @@ -34,7 +34,7 @@ export const ACP_BUILTIN_SLASH_COMMANDS: AvailableCommand[] = BUILTIN_SLASH_COMM */ export async function executeAcpBuiltinSlashCommand( text: string, - runtime: AcpBuiltinCommandRuntime, + runtime: SlashCommandRuntime, ): Promise { const parsed = parseSlashCommand(text); if (!parsed) return false; diff --git a/packages/coding-agent/src/slash-commands/helpers/usage-report.ts b/packages/coding-agent/src/slash-commands/helpers/usage-report.ts index 9d737ce9f..bbef96730 100644 --- a/packages/coding-agent/src/slash-commands/helpers/usage-report.ts +++ b/packages/coding-agent/src/slash-commands/helpers/usage-report.ts @@ -56,7 +56,9 @@ function renderUsageReports(reports: UsageReport[], nowMs: number): string { lines.push(`- ${limit.label}${tier}${window ? ` — ${window}` : ""}`); lines.push(` ${formatUsageReportAccount(report, limit, index)}: ${formatUsageAmount(limit)}`); lines.push(` ${renderAsciiBar(limit.amount.usedFraction)}`); - if (limit.window?.resetsAt) lines.push(` resets in ${formatDuration(limit.window.resetsAt - nowMs)}`); + if (limit.window?.resetsAt && limit.window.resetsAt > nowMs) { + lines.push(` resets in ${formatDuration(limit.window.resetsAt - nowMs)}`); + } if (limit.notes && limit.notes.length > 0) lines.push(` ${limit.notes.join(" • ")}`); } } diff --git a/packages/coding-agent/src/slash-commands/types.ts b/packages/coding-agent/src/slash-commands/types.ts index 559d0e140..7bba2cdfd 100644 --- a/packages/coding-agent/src/slash-commands/types.ts +++ b/packages/coding-agent/src/slash-commands/types.ts @@ -116,11 +116,5 @@ export interface SlashCommandSpec extends BuiltinSlashCommand { ) => Promise | SlashCommandResult; } -/** - * @deprecated Use `SlashCommandRuntime` directly. Retained as an alias so - * downstream code that imported the ACP-specific name keeps compiling. - */ -export type AcpBuiltinCommandRuntime = SlashCommandRuntime; - /** Result returned by `executeAcpBuiltinSlashCommand`. */ export type AcpBuiltinSlashCommandResult = false | { consumed: true } | { prompt: string }; diff --git a/packages/coding-agent/src/task/executor.ts b/packages/coding-agent/src/task/executor.ts index e8ccf5af1..dcb1fed4d 100644 --- a/packages/coding-agent/src/task/executor.ts +++ b/packages/coding-agent/src/task/executor.ts @@ -185,6 +185,8 @@ export interface ExecutorOptions { */ parentArtifactManager?: ArtifactManager; parentHindsightSessionState?: HindsightSessionState; + /** Parent agent's eval executor session id. Subagents reuse it so eval state is shared. */ + parentEvalSessionId?: string; /** * Parent agent's OpenTelemetry configuration. When defined, the subagent's * loop is started with the same tracer/hooks but its own agent identity @@ -1146,6 +1148,11 @@ export async function runSubprocess(options: ExecutorOptions): Promise 0) { progress.contextWindow = model.contextWindow; } + if (model) { + progress.resolvedModel = explicitThinkingLevel + ? `${model.provider}/${model.id}:${resolvedThinkingLevel}` + : `${model.provider}/${model.id}`; + } const effectiveThinkingLevel = explicitThinkingLevel ? resolvedThinkingLevel : (thinkingLevel ?? resolvedThinkingLevel); @@ -1236,6 +1243,7 @@ export async function runSubprocess(options: ExecutorOptions): Promise 0 ? mcpProxyTools : undefined, localProtocolOptions: options.localProtocolOptions, telemetry: subagentTelemetry, + parentEvalSessionId: options.parentEvalSessionId, }), ); @@ -1603,6 +1611,7 @@ export async function runSubprocess(options: ExecutorOptions): Promise path.basename(file.path).toLowerCase() !== "agents.md", ); const promptTemplates = this.session.promptTemplates; + const parentEvalSessionId = this.session.getEvalSessionId?.() ?? undefined; // Initialize progress for all tasks for (let i = 0; i < tasksWithUniqueIds.length; i++) { @@ -911,6 +912,7 @@ export class TaskTool implements AgentTool 0) { line += `${theme.sep.dot}${theme.fg("statusLineCost", `$${opts.cost.toFixed(2)}`)}`; } + if (opts.resolvedModel && opts.showResolvedModelBadge) { + line += `${theme.sep.dot}${theme.fg("dim", truncateToWidth(replaceTabs(opts.resolvedModel), 30))}`; + } return line; } @@ -564,14 +570,15 @@ function renderAgentProgress( statusLine += ` ${formatBadge(statusLabel, iconColor, theme)}`; } + const showBadge = settings.get("task.showResolvedModelBadge"); if (progress.status === "running") { if (!description) { const taskPreview = truncateToWidth(progress.assignment ?? progress.task, 40); statusLine += ` ${theme.fg("muted", taskPreview)}`; } - statusLine = appendAgentStats(statusLine, progress, theme); + statusLine = appendAgentStats(statusLine, { ...progress, showResolvedModelBadge: showBadge }, theme); } else if (progress.status === "completed") { - statusLine = appendAgentStats(statusLine, progress, theme); + statusLine = appendAgentStats(statusLine, { ...progress, showResolvedModelBadge: showBadge }, theme); } lines.push(statusLine); @@ -838,6 +845,7 @@ function renderAgentResult(result: SingleResult, isLast: boolean, expanded: bool iconColor, theme, )}`; + const showBadge = settings.get("task.showResolvedModelBadge"); statusLine = appendAgentStats( statusLine, { @@ -845,6 +853,8 @@ function renderAgentResult(result: SingleResult, isLast: boolean, expanded: bool contextTokens: result.contextTokens, contextWindow: result.contextWindow, cost: result.usage?.cost.total ?? 0, + resolvedModel: result.resolvedModel, + showResolvedModelBadge: showBadge, }, theme, ); diff --git a/packages/coding-agent/src/task/types.ts b/packages/coding-agent/src/task/types.ts index 00295a8c6..ca669f3c1 100644 --- a/packages/coding-agent/src/task/types.ts +++ b/packages/coding-agent/src/task/types.ts @@ -210,6 +210,8 @@ export interface AgentProgress { cost: number; durationMs: number; modelOverride?: string | string[]; + /** Resolved model display string in the form `/`, optionally suffixed with `:` when the level was set explicitly. Undefined when the model could not be resolved. */ + resolvedModel?: string; /** Data extracted by registered subprocess tool handlers (keyed by tool name) */ extractedToolData?: Record; /** @@ -268,6 +270,8 @@ export interface SingleResult { /** Model's context window in tokens, when known. */ contextWindow?: number; modelOverride?: string | string[]; + /** Resolved model display string in the form `/`, optionally suffixed with `:` when the level was set explicitly. Omitted from tool-result JSON when undefined to keep wire payloads small. */ + resolvedModel?: string; error?: string; aborted?: boolean; abortReason?: string; diff --git a/packages/coding-agent/src/tool-discovery/tool-index.ts b/packages/coding-agent/src/tool-discovery/tool-index.ts index 71662ddb5..03ac1f6a4 100644 --- a/packages/coding-agent/src/tool-discovery/tool-index.ts +++ b/packages/coding-agent/src/tool-discovery/tool-index.ts @@ -44,47 +44,6 @@ export interface DiscoverableToolSearchResult { score: number; } -// ─── Legacy MCP-typed aliases (back-compat) ────────────────────────────────── - -/** @deprecated Use DiscoverableTool with source === "mcp" */ -export type DiscoverableMCPTool = Pick< - DiscoverableTool, - "name" | "label" | "schemaKeys" | "serverName" | "mcpToolName" -> & { description: string }; - -/** @deprecated Use DiscoverableToolServerSummary */ -export type DiscoverableMCPToolServerSummary = DiscoverableToolServerSummary; - -/** @deprecated Use DiscoverableToolSummary */ -export type DiscoverableMCPToolSummary = DiscoverableToolSummary; - -/** Tool object stored on legacy MCP index documents. Carries both legacy `description` and the - * generic `summary`/`source` so the legacy index is structurally assignable to - * DiscoverableToolSearchIndex (search functions read termFrequencies, not the tool fields). */ -export type DiscoverableMCPSearchTool = DiscoverableTool & { description: string }; - -/** @deprecated Use DiscoverableToolSearchDocument */ -export interface DiscoverableMCPSearchDocument { - tool: DiscoverableMCPSearchTool; - termFrequencies: Map; - length: number; -} - -/** @deprecated Use DiscoverableToolSearchIndex. - * Documents on this index expose `tool.description` (legacy MCP shape) while still being - * searchable via `searchDiscoverableTools`. */ -export interface DiscoverableMCPSearchIndex { - documents: DiscoverableMCPSearchDocument[]; - averageLength: number; - documentFrequencies: Map; -} - -/** @deprecated Use DiscoverableToolSearchResult */ -export interface DiscoverableMCPSearchResult { - tool: DiscoverableMCPSearchTool; - score: number; -} - // ─── BM25 Constants ─────────────────────────────────────────────────────────── const BM25_K1 = 1.2; @@ -295,96 +254,3 @@ export function searchDiscoverableTools( .sort((left, right) => right.score - left.score || left.tool.name.localeCompare(right.tool.name)) .slice(0, limit); } - -// ─── Legacy MCP-specific shims (back-compat wrappers) ──────────────────────── - -/** @deprecated Use getDiscoverableTool */ -export function getDiscoverableMCPTool(tool: AgentTool): DiscoverableMCPTool | null { - if (!isMCPToolName(tool.name)) return null; - const toolRecord = tool as AgentTool & { - label?: string; - description?: string; - mcpServerName?: string; - mcpToolName?: string; - parameters?: unknown; - }; - return { - name: tool.name, - label: typeof toolRecord.label === "string" ? toolRecord.label : tool.name, - description: typeof toolRecord.description === "string" ? toolRecord.description : "", - serverName: typeof toolRecord.mcpServerName === "string" ? toolRecord.mcpServerName : undefined, - mcpToolName: typeof toolRecord.mcpToolName === "string" ? toolRecord.mcpToolName : undefined, - schemaKeys: getSchemaPropertyKeys(toolRecord.parameters), - }; -} - -/** @deprecated Use collectDiscoverableTools with source filter */ -export function collectDiscoverableMCPTools(tools: Iterable): DiscoverableMCPTool[] { - const discoverable: DiscoverableMCPTool[] = []; - for (const tool of tools) { - const metadata = getDiscoverableMCPTool(tool); - if (metadata) { - discoverable.push(metadata); - } - } - return discoverable; -} - -/** @deprecated Use selectDiscoverableToolNamesByServer */ -export function selectDiscoverableMCPToolNamesByServer( - tools: Iterable, - serverNames: ReadonlySet, -): string[] { - if (serverNames.size === 0) return []; - return Array.from(tools) - .filter(tool => tool.serverName !== undefined && serverNames.has(tool.serverName)) - .map(tool => tool.name); -} - -/** @deprecated Use summarizeDiscoverableTools */ -export function summarizeDiscoverableMCPTools(tools: DiscoverableMCPTool[]): DiscoverableMCPToolSummary { - const serverToolCounts = new Map(); - for (const tool of tools) { - if (!tool.serverName) continue; - serverToolCounts.set(tool.serverName, (serverToolCounts.get(tool.serverName) ?? 0) + 1); - } - const servers = Array.from(serverToolCounts.entries()) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([name, toolCount]) => ({ name, toolCount })); - return { - servers, - toolCount: tools.length, - }; -} - -/** @deprecated Use buildDiscoverableToolSearchIndex. - * Builds an index whose documents preserve the legacy `description` field on each tool while - * also carrying the generic `summary` (set from `description`) so the index remains usable - * with `searchDiscoverableTools`. */ -export function buildDiscoverableMCPSearchIndex(tools: Iterable): DiscoverableMCPSearchIndex { - const adapted: DiscoverableMCPSearchTool[] = Array.from(tools).map(t => ({ - name: t.name, - label: t.label, - description: t.description, - summary: t.description, - source: "mcp" as DiscoverableToolSource, - serverName: t.serverName, - mcpToolName: t.mcpToolName, - schemaKeys: t.schemaKeys, - })); - const generic = buildDiscoverableToolSearchIndex(adapted); - // Documents reference `adapted` tools (with `description`), so the cast is sound. - return generic as unknown as DiscoverableMCPSearchIndex; -} - -/** @deprecated Use searchDiscoverableTools */ -export function searchDiscoverableMCPTools( - index: DiscoverableMCPSearchIndex | DiscoverableToolSearchIndex, - query: string, - limit: number, -): DiscoverableMCPSearchResult[] { - return searchDiscoverableTools(index as DiscoverableToolSearchIndex, query, limit) as DiscoverableMCPSearchResult[]; -} - -/** @deprecated Use formatDiscoverableToolServerSummary */ -export const formatDiscoverableMCPToolServerSummary = formatDiscoverableToolServerSummary; diff --git a/packages/coding-agent/src/tools/ast-edit.ts b/packages/coding-agent/src/tools/ast-edit.ts index 560c678d7..1f4d6a4e6 100644 --- a/packages/coding-agent/src/tools/ast-edit.ts +++ b/packages/coding-agent/src/tools/ast-edit.ts @@ -6,7 +6,7 @@ import { Text } from "@oh-my-pi/pi-tui"; import { $envpos, prompt, untilAborted } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; -import { computeLineHash, HL_BODY_SEP } from "../hashline/hash"; +import { computeFileHash, formatHashlineHeader } from "../hashline/hash"; import type { Theme } from "../modes/theme/theme"; import astEditDescription from "../prompts/tools/ast-edit.md" with { type: "text" }; import { Ellipsis, fileHyperlink, renderStatusLine, renderTreeList, truncateToWidth } from "../tui"; @@ -257,12 +257,26 @@ export class AstEditTool implements AgentTool(); + if (useHashLines) { + for (const relativePath of fileList) { + const absolutePath = path.resolve(this.session.cwd, relativePath); + try { + const fullText = await Bun.file(absolutePath).text(); + const fileHash = computeFileHash(fullText); + hashContexts.set(relativePath, { fileHash }); + } catch { + // Best-effort: if a file disappears between ast-edit and rendering, emit plain line output. + } + } + } const outputLines: string[] = []; const displayLines: string[] = []; const renderChangesForFile = (relativePath: string): { model: string[]; display: string[] } => { const modelOut: string[] = []; const displayOut: string[] = []; const fileChanges = changesByFile.get(relativePath) ?? []; + const hashContext = hashContexts.get(relativePath); const lineNumberWidth = fileChanges.reduce( (width, change) => Math.max(width, String(change.startLine).length), 0, @@ -272,13 +286,9 @@ export class AstEditTool implements AgentTool { const rendered = renderChangesForFile(relativePath); const count = fileReplacementCounts.get(relativePath) ?? 0; + const hashContext = hashContexts.get(relativePath); + const hashSuffix = hashContext ? `#${hashContext.fileHash}` : ""; return { - headerSuffix: ` (${formatCount("replacement", count)})`, + headerSuffix: `${hashSuffix} (${formatCount("replacement", count)})`, modelLines: rendered.model, displayLines: rendered.display, + skip: rendered.model.length === 0, }; }); outputLines.push(...grouped.model); @@ -302,6 +315,15 @@ export class AstEditTool implements AgentTool 0) { + outputLines.push(""); + displayLines.push(""); + } + const hashContext = hashContexts.get(relativePath); + if (hashContext) { + outputLines.push(formatHashlineHeader(relativePath, hashContext.fileHash)); + } outputLines.push(...rendered.model); displayLines.push(...rendered.display); } @@ -499,11 +521,12 @@ export const astEditToolRenderer = { let contextDir = searchBase ?? ""; return group.map(line => { if (line.startsWith("## ")) { - // Strip ` (3 replacements)` suffix attached by formatGroupedFiles. + // Strip ` (3 replacements)` and `#hash` suffixes from formatGroupedFiles. const fileName = line .slice(3) .trimEnd() - .replace(/\s+\([^)]*\)\s*$/, ""); + .replace(/\s+\([^)]*\)\s*$/, "") + .replace(/#[0-9a-f]+$/, ""); const absPath = contextDir && fileName ? path.join(contextDir, fileName) : undefined; const styled = uiTheme.fg("dim", line); return absPath ? fileHyperlink(absPath, styled) : styled; @@ -514,14 +537,14 @@ export const astEditToolRenderer = { .trimEnd() .replace(/\s+\([^)]*\)\s*$/, ""); const isDirectory = raw.endsWith("/"); - const name = raw.replace(/\/$/, ""); + const name = isDirectory ? raw.replace(/\/$/, "") : raw.replace(/#[0-9a-f]+$/, ""); if (isDirectory) { if (searchBase) { contextDir = name === "." ? searchBase : path.join(searchBase, name); } return uiTheme.fg("accent", line); } - // Root-level file with optional suffix, e.g. `# foo.ts (3 replacements)`. + // Root-level file with optional `#hash` and ` (3 replacements)` suffixes. const absPath = searchBase && name ? path.join(searchBase, name) : undefined; const styled = uiTheme.fg("accent", line); return absPath ? fileHyperlink(absPath, styled) : styled; diff --git a/packages/coding-agent/src/tools/ast-grep.ts b/packages/coding-agent/src/tools/ast-grep.ts index 20203eca9..119a2defc 100644 --- a/packages/coding-agent/src/tools/ast-grep.ts +++ b/packages/coding-agent/src/tools/ast-grep.ts @@ -5,7 +5,9 @@ import type { Component } from "@oh-my-pi/pi-tui"; import { Text } from "@oh-my-pi/pi-tui"; import { prompt, untilAborted } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; +import { getFileReadCache } from "../edit/file-read-cache"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; +import { computeFileHash, formatHashlineHeader } from "../hashline/hash"; import type { Theme } from "../modes/theme/theme"; import astGrepDescription from "../prompts/tools/ast-grep.md" with { type: "text" }; import { Ellipsis, fileHyperlink, renderStatusLine, renderTreeList, truncateToWidth } from "../tui"; @@ -216,25 +218,43 @@ export class AstGrepTool implements AgentTool(); + if (useHashLines) { + for (const relativePath of fileList) { + const absolutePath = path.resolve(this.session.cwd, relativePath); + try { + const fullText = await Bun.file(absolutePath).text(); + const fileHash = computeFileHash(fullText); + hashContexts.set(relativePath, { absolutePath, fileHash }); + } catch { + // Best-effort: if a file disappears between ast-grep and rendering, emit plain line output. + } + } + } const outputLines: string[] = []; const displayLines: string[] = []; const renderMatchesForFile = (relativePath: string): { model: string[]; display: string[] } => { const modelOut: string[] = []; const displayOut: string[] = []; const fileMatches = matchesByFile.get(relativePath) ?? []; + const hashContext = hashContexts.get(relativePath); const lineNumberWidth = fileMatches.reduce((width, match) => { const lineCount = match.text.split("\n").length; const endLine = match.startLine + lineCount - 1; return Math.max(width, String(match.startLine).length, String(endLine).length); }, 0); + const cacheEntries: Array = []; for (const match of fileMatches) { const matchLines = match.text.split("\n"); for (let index = 0; index < matchLines.length; index++) { const lineNumber = match.startLine + index; const isMatch = index === 0; const line = matchLines[index] ?? ""; - modelOut.push(formatMatchLine(lineNumber, line, isMatch, { useHashLines })); + modelOut.push( + formatMatchLine(lineNumber, line, isMatch, { useHashLines: hashContext !== undefined }), + ); displayOut.push(formatCodeFrameLine(isMatch ? "*" : " ", lineNumber, line, lineNumberWidth)); + cacheEntries.push([lineNumber, line] as const); } if (match.metaVariables && Object.keys(match.metaVariables).length > 0) { const serializedMeta = Object.entries(match.metaVariables) @@ -246,19 +266,39 @@ export class AstGrepTool implements AgentTool 0) { + getFileReadCache(this.session).recordSparse(hashContext.absolutePath, cacheEntries, { + fileHash: hashContext.fileHash, + }); + } return { model: modelOut, display: displayOut }; }; if (isDirectory) { const grouped = formatGroupedFiles(fileList, relativePath => { const rendered = renderMatchesForFile(relativePath); - return { modelLines: rendered.model, displayLines: rendered.display }; + const hashContext = hashContexts.get(relativePath); + return { + modelLines: rendered.model, + displayLines: rendered.display, + headerSuffix: hashContext ? `#${hashContext.fileHash}` : "", + skip: rendered.model.length === 0, + }; }); outputLines.push(...grouped.model); displayLines.push(...grouped.display); } else { for (const relativePath of fileList) { const rendered = renderMatchesForFile(relativePath); + if (rendered.model.length === 0) continue; + if (outputLines.length > 0) { + outputLines.push(""); + displayLines.push(""); + } + const hashContext = hashContexts.get(relativePath); + if (hashContext) { + outputLines.push(formatHashlineHeader(relativePath, hashContext.fileHash)); + } outputLines.push(...rendered.model); displayLines.push(...rendered.display); } @@ -385,7 +425,8 @@ export const astGrepToolRenderer = { const fileName = line .slice(3) .trimEnd() - .replace(/\s+\([^)]*\)\s*$/, ""); + .replace(/\s+\([^)]*\)\s*$/, "") + .replace(/#[0-9a-f]+$/, ""); const absPath = contextDir && fileName ? path.join(contextDir, fileName) : undefined; const styled = uiTheme.fg("dim", line); return absPath ? fileHyperlink(absPath, styled) : styled; @@ -396,7 +437,7 @@ export const astGrepToolRenderer = { .trimEnd() .replace(/\s+\([^)]*\)\s*$/, ""); const isDirectory = raw.endsWith("/"); - const name = raw.replace(/\/$/, ""); + const name = isDirectory ? raw.replace(/\/$/, "") : raw.replace(/#[0-9a-f]+$/, ""); if (isDirectory) { if (searchBase) { contextDir = name === "." ? searchBase : path.join(searchBase, name); diff --git a/packages/coding-agent/src/tools/browser/tab-worker.ts b/packages/coding-agent/src/tools/browser/tab-worker.ts index 8f23d2181..9cc0811e4 100644 --- a/packages/coding-agent/src/tools/browser/tab-worker.ts +++ b/packages/coding-agent/src/tools/browser/tab-worker.ts @@ -575,8 +575,10 @@ export class WorkerCore { if (signal.aborted) onCancel(); else signal.addEventListener("abort", onCancel, { once: true }); try { + const hooks = this.#hooksForActiveRun(); + if (!hooks) throw new ToolError("Browser runtime started without an active run"); const returnValue = await Promise.race([ - runtime.run(msg.code, `browser-run-${msg.id}.js`), + runtime.run(msg.code, `browser-run-${msg.id}.js`, hooks, { runId: msg.id, cwd: msg.session.cwd }), cancelRejection, ]); await this.#postReadyInfo(); @@ -601,7 +603,6 @@ export class WorkerCore { this.#runtime = new JsRuntime({ initialCwd: session.cwd, sessionId: `browser-tab-${this.#targetId ?? "unknown"}`, - getHooks: () => this.#hooksForActiveRun(), }); return this.#runtime; } diff --git a/packages/coding-agent/src/tools/eval.ts b/packages/coding-agent/src/tools/eval.ts index a931c45fb..9908711f5 100644 --- a/packages/coding-agent/src/tools/eval.ts +++ b/packages/coding-agent/src/tools/eval.ts @@ -6,6 +6,7 @@ import { prompt } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; import { jsBackend, pythonBackend } from "../eval"; import type { ExecutorBackend } from "../eval/backend"; +import { defaultEvalSessionId } from "../eval/session-id"; import type { EvalCellResult, EvalDisplayOutput, EvalLanguage, EvalStatusEvent, EvalToolDetails } from "../eval/types"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import { truncateToVisualLines } from "../modes/components/visual-truncate"; @@ -347,7 +348,7 @@ export class EvalTool implements AgentTool { pushUpdate(); }, }); - const sessionId = sessionFile ? `session:${sessionFile}:cwd:${session.cwd}` : `cwd:${session.cwd}`; + const sessionId = session.getEvalSessionId?.() ?? defaultEvalSessionId(session); for (let i = 0; i < cells.length; i++) { const cell = cells[i]; diff --git a/packages/coding-agent/src/tools/find.ts b/packages/coding-agent/src/tools/find.ts index 8f0f2ac87..60e0eb3be 100644 --- a/packages/coding-agent/src/tools/find.ts +++ b/packages/coding-agent/src/tools/find.ts @@ -341,7 +341,6 @@ export class FindTool implements AgentTool { { pattern: globPattern, path: searchPath, - fileType: natives.FileType.File, hidden: includeHidden, maxResults: effectiveLimit, sortByMtime: true, diff --git a/packages/coding-agent/src/tools/gh.ts b/packages/coding-agent/src/tools/gh.ts index 54947833c..313ee3dd7 100644 --- a/packages/coding-agent/src/tools/gh.ts +++ b/packages/coding-agent/src/tools/gh.ts @@ -1748,9 +1748,13 @@ async function fetchRunsForCommit( cwd: string, repo: string, headSha: string, - branch: string | undefined, signal?: AbortSignal, ): Promise { + // Filter only by `head_sha`. The SHA uniquely identifies the commit, so + // adding the GitHub `branch=` filter would wrongly exclude workflow runs + // whose `head_branch` is not the local checkout — e.g. tag-push triggered + // release workflows (`head_branch=v1.2.3`) or PR-triggered runs + // (`head_branch=`). See coding-agent issue tracker for details. const response = await git.github.json( cwd, [ @@ -1762,7 +1766,6 @@ async function fetchRunsForCommit( `head_sha=${headSha}`, "-F", `per_page=${RUN_JOBS_PAGE_SIZE}`, - ...(branch ? ["-F", `branch=${branch}`] : []), ], signal, { repoProvided: true }, @@ -3406,7 +3409,7 @@ async function executeRunWatch( throwIfAborted(signal); pollCount += 1; - let runs = await fetchRunsForCommit(session.cwd, repo, headSha, branch, signal); + let runs = await fetchRunsForCommit(session.cwd, repo, headSha, signal); const details = buildCommitRunWatchDetails(repo, headSha, branch, runs, { state: "watching", pollCount, @@ -3434,7 +3437,7 @@ async function executeRunWatch( }), }); await scheduler.wait(graceSeconds * 1000, { signal }); - runs = await fetchRunsForCommit(session.cwd, repo, headSha, branch, signal); + runs = await fetchRunsForCommit(session.cwd, repo, headSha, signal); } const failedJobLogs = await fetchFailedJobLogs( diff --git a/packages/coding-agent/src/tools/index.ts b/packages/coding-agent/src/tools/index.ts index 61c4a8b3e..a5cc255ce 100644 --- a/packages/coding-agent/src/tools/index.ts +++ b/packages/coding-agent/src/tools/index.ts @@ -91,7 +91,6 @@ export * from "./search"; export * from "./search-tool-bm25"; export * from "./ssh"; export * from "./todo-write"; -export * from "./vim"; export * from "./write"; export * from "./yield"; @@ -104,7 +103,6 @@ export type ContextFileEntry = { depth?: number; }; -export type { DiscoverableMCPTool } from "../mcp/discoverable-tool-metadata"; export type { DiscoverableTool, DiscoverableToolSearchIndex, @@ -140,6 +138,8 @@ export interface ToolSession { requireYieldTool?: boolean; /** Task recursion depth (0 = top-level, 1 = first child, etc.) */ taskDepth?: number; + /** Get shared eval executor session ID. Subagents inherit this to share JS/Python state. */ + getEvalSessionId?: () => string | null; /** Get session file */ getSessionFile: () => string | null; /** Get eval kernel owner ID for session-scoped retained-kernel cleanup. */ @@ -194,12 +194,6 @@ export interface ToolSession { setTodoPhases?: (phases: TodoPhase[]) => void; /** Whether MCP tool discovery is active for this session. */ isMCPDiscoveryEnabled?: () => boolean; - /** Get hidden-but-discoverable MCP tools for search_tool_bm25 prompts and fallbacks. - * @deprecated Use getDiscoverableTools with source filter instead. */ - getDiscoverableMCPTools?: () => import("../mcp/discoverable-tool-metadata").DiscoverableMCPTool[]; - /** Get the cached discoverable MCP search index for search_tool_bm25 execution. - * @deprecated Use getDiscoverableToolSearchIndex instead. */ - getDiscoverableMCPSearchIndex?: () => import("../tool-discovery/tool-index").DiscoverableMCPSearchIndex; /** Get MCP tools activated by prior search_tool_bm25 calls. */ getSelectedMCPToolNames?: () => string[]; /** Merge MCP tool selections into the active session tool set. */ diff --git a/packages/coding-agent/src/tools/irc.ts b/packages/coding-agent/src/tools/irc.ts index b1f169ba6..42382425a 100644 --- a/packages/coding-agent/src/tools/irc.ts +++ b/packages/coding-agent/src/tools/irc.ts @@ -25,6 +25,7 @@ import ircDescription from "../prompts/tools/irc.md" with { type: "text" }; import type { AgentRef, AgentRegistry } from "../registry/agent-registry"; import type { ToolSession } from "."; +const DEFAULT_IRC_TIMEOUT_MS = 120_000; const ircSchema = z.object({ op: z.enum(["send", "list"]).describe("irc operation"), to: z.string().optional().describe('recipient agent id or "all"'), @@ -159,6 +160,7 @@ export class IrcTool implements AgentTool { const awaitReply = params.awaitReply ?? !isBroadcast; + const timeoutMs = normalizeIrcTimeoutMs(this.session.settings.get("irc.timeoutMs")); const delivered: string[] = []; const replies: IrcReply[] = []; const failed: Array<{ id: string; error: string }> = []; @@ -174,12 +176,18 @@ export class IrcTool implements AgentTool { return; } try { - const result = await targetSession.respondAsBackground({ - from: senderId, - message, - awaitReply, + const result = await runIrcDispatchWithTimeout( + timeoutMs, signal, - }); + timeoutSignal => + targetSession.respondAsBackground({ + from: senderId, + message, + awaitReply, + signal: timeoutSignal, + }), + target.id, + ); delivered.push(target.id); if (awaitReply && result.replyText) { replies.push({ from: target.id, text: result.replyText }); @@ -237,3 +245,49 @@ function errorResult(text: string, details: IrcDetails): AgentToolResult( + timeoutMs: number, + parentSignal: AbortSignal | undefined, + run: (signal?: AbortSignal) => Promise, + targetId: string, +): Promise { + if (timeoutMs <= 0) { + return await run(parentSignal); + } + + const controller = new AbortController(); + const timeoutError = new Error(`IRC timed out waiting for ${targetId} after ${timeoutMs} ms`); + let timeout: NodeJS.Timeout | undefined; + let parentAbortListener: (() => void) | undefined; + + const timeoutDeferred = Promise.withResolvers(); + if (parentSignal) { + if (parentSignal.aborted) { + throw parentSignal.reason instanceof Error ? parentSignal.reason : new Error("IRC aborted"); + } + parentAbortListener = () => { + controller.abort(parentSignal.reason); + timeoutDeferred.reject(parentSignal.reason instanceof Error ? parentSignal.reason : new Error("IRC aborted")); + }; + parentSignal.addEventListener("abort", parentAbortListener, { once: true }); + } + + timeout = setTimeout(() => { + controller.abort(timeoutError); + timeoutDeferred.reject(timeoutError); + }, timeoutMs); + timeout.unref?.(); + + try { + return await Promise.race([run(controller.signal), timeoutDeferred.promise]); + } finally { + if (timeout) clearTimeout(timeout); + if (parentSignal && parentAbortListener) parentSignal.removeEventListener("abort", parentAbortListener); + } +} diff --git a/packages/coding-agent/src/tools/match-line-format.ts b/packages/coding-agent/src/tools/match-line-format.ts index 301323ae9..e56c6d4ac 100644 --- a/packages/coding-agent/src/tools/match-line-format.ts +++ b/packages/coding-agent/src/tools/match-line-format.ts @@ -1,12 +1,10 @@ -import { computeLineHash } from "../hashline/hash"; - /** * Format a single line of match output for grep/ast-grep style results. * - * The anchor/content separator is always `|`. Matched lines are prefixed - * with `*`; context lines are prefixed with a single space so anchors - * align in column. In hashline mode the anchor is `LINE+ID` (no `#`); in - * plain mode it is just the line number. Line numbers are never padded. + * Matched lines are prefixed with `*`; context lines are prefixed with a single + * space so line numbers align in column. In hashline mode the line uses the + * editable `LINE:content` shape under a file-hash header; in plain mode it keeps + * the legacy `LINE|content` display-only shape. Line numbers are never padded. */ export function formatMatchLine( lineNumber: number, @@ -16,7 +14,7 @@ export function formatMatchLine( ): string { const marker = isMatch ? "*" : " "; if (options.useHashLines) { - return `${marker}${lineNumber}${computeLineHash(lineNumber, line)}|${line}`; + return `${marker}${lineNumber}:${line}`; } return `${marker}${lineNumber}|${line}`; } diff --git a/packages/coding-agent/src/tools/read.ts b/packages/coding-agent/src/tools/read.ts index 395160449..c4052b14f 100644 --- a/packages/coding-agent/src/tools/read.ts +++ b/packages/coding-agent/src/tools/read.ts @@ -9,9 +9,10 @@ import { Text } from "@oh-my-pi/pi-tui"; import { getRemoteDir, logger, prompt, readImageMetadata, untilAborted } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; import { getFileReadCache } from "../edit/file-read-cache"; +import { normalizeToLF } from "../edit/normalize"; import { isNotebookPath, readEditableNotebookText } from "../edit/notebook"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; -import { formatHashLine, formatHashLines, formatLineHash, HL_BODY_SEP } from "../hashline/hash"; +import { computeFileHash, formatHashlineHeader, formatNumberedLine, formatNumberedLines } from "../hashline/hash"; import { InternalUrlRouter } from "../internal-urls"; import { parseInternalUrl } from "../internal-urls/parse"; import type { InternalUrl } from "../internal-urls/types"; @@ -113,13 +114,50 @@ function prependLineNumbers(text: string, startNum: number): string { return textLines.map((line, i) => `${startNum + i}|${line}`).join("\n"); } +interface HashlineHeaderContext { + header: string; + fileHash: string; + fullText: string; +} + +function buildHashlineHeaderContext(displayPath: string, fullText: string): HashlineHeaderContext { + const normalized = normalizeToLF(fullText); + const fileHash = computeFileHash(normalized); + return { + header: formatHashlineHeader(displayPath, fileHash), + fileHash, + fullText: normalized, + }; +} + +async function readHashlineHeaderContext(absolutePath: string, cwd: string): Promise { + const fullText = await Bun.file(absolutePath).text(); + return buildHashlineHeaderContext(formatPathRelativeToCwd(absolutePath, cwd), fullText); +} + +function prependHashlineHeader(text: string, context: HashlineHeaderContext | undefined): string { + return context ? `${context.header}\n${text}` : text; +} + +function recordHashlineSnapshot( + session: ToolSession, + absolutePath: string | undefined, + context: HashlineHeaderContext | undefined, +): void { + if (!context || !absolutePath || !path.isAbsolute(absolutePath)) return; + getFileReadCache(session).recordContiguous(absolutePath, 1, context.fullText.split("\n"), { + fullText: context.fullText, + fileHash: context.fileHash, + }); +} + function formatTextWithMode( text: string, startNum: number, shouldAddHashLines: boolean, shouldAddLineNumbers: boolean, ): string { - if (shouldAddHashLines) return formatHashLines(text, startNum); + if (shouldAddHashLines) return formatNumberedLines(text, startNum); if (shouldAddLineNumbers) return prependLineNumbers(text, startNum); return text; } @@ -150,7 +188,7 @@ function formatSingleLine( shouldAddHashLines: boolean, shouldAddLineNumbers: boolean, ): string { - if (shouldAddHashLines) return formatHashLine(line, text); + if (shouldAddHashLines) return formatNumberedLine(line, text); if (shouldAddLineNumbers) return `${line}|${text}`; return text; } @@ -165,9 +203,7 @@ function formatMergedBraceLine( ): { model: string; display: string } { const merged = `${headText.trimEnd()} .. ${tailText.trim()}`; if (shouldAddHashLines) { - const start = formatLineHash(startLine, headText); - const end = formatLineHash(endLine, tailText); - return { model: `${start}-${end}${HL_BODY_SEP}${merged}`, display: merged }; + return { model: `${startLine}-${endLine}:${merged}`, display: merged }; } if (shouldAddLineNumbers) { return { model: `${startLine}-${endLine}|${merged}`, display: merged }; @@ -180,17 +216,38 @@ function countTextLines(text: string): number { return text.split("\n").length; } +/** Inclusive line range describing one elided span in a structural summary. */ +interface ElidedRange { + start: number; + end: number; +} + +/** Sample ranges shown in the footer to demonstrate the multi-range syntax. */ +const FOOTER_RANGE_SAMPLES = 2; + /** * Footer appended to summarized reads telling the model how to recover the * elided body. Without this hint, agents either ignore the `...`/`{ .. }` - * markers or burn a turn guessing the right selector (see issue #1046). + * markers or burn a turn guessing the right selector (see issue #1046). The + * footer demonstrates the multi-range selector syntax with concrete sample + * ranges drawn from the actual elision so the model re-reads only what it + * needs instead of falling back to `:raw` or whole-file reads. */ -function formatSummaryElisionFooter(readPath: string, elidedSpans: number, elidedLines: number): string { - if (elidedSpans <= 0) return ""; - const spanWord = elidedSpans === 1 ? "region" : "regions"; +function formatSummaryElisionFooter( + readPath: string, + elidedRanges: ReadonlyArray, + elidedLines: number, +): string { + if (elidedRanges.length === 0) return ""; const lineWord = elidedLines === 1 ? "line" : "lines"; - const linePart = elidedLines > 0 ? `${elidedLines} ${lineWord} across ` : ""; - return `[${linePart}${elidedSpans} elided ${spanWord}; read ${readPath}:raw or a line range like ${readPath}:1-9999 for verbatim content]`; + const sampleCount = Math.min(elidedRanges.length, FOOTER_RANGE_SAMPLES); + const selector = elidedRanges + .slice(0, sampleCount) + .map(r => `${r.start}-${r.end}`) + .join(","); + const example = `${readPath}:${selector}`; + const tail = elidedRanges.length > sampleCount ? `, e.g. ${example}` : ` with ${example}`; + return `[${elidedLines} ${lineWord} elided; re-read needed ranges${tail}]`; } const READ_CHUNK_SIZE = 8 * 1024; @@ -844,9 +901,18 @@ export class ReadTool implements AgentTool { const shouldAddHashLines = displayMode.hashLines; const shouldAddLineNumbers = shouldAddHashLines ? false : displayMode.lineNumbers; + const hashContext = + shouldAddHashLines && options.sourcePath + ? buildHashlineHeaderContext(formatPathRelativeToCwd(options.sourcePath, this.session.cwd), text) + : undefined; + recordHashlineSnapshot(this.session, options.sourcePath, hashContext); + let emittedHashlineHeader = false; const formatText = (content: string, startNum: number): string => { details.displayContent = { text: content, startLine: startNum }; - return formatTextWithMode(content, startNum, shouldAddHashLines, shouldAddLineNumbers); + const formatted = formatTextWithMode(content, startNum, shouldAddHashLines, shouldAddLineNumbers); + if (!hashContext || emittedHashlineHeader) return formatted; + emittedHashlineHeader = true; + return prependHashlineHeader(formatted, hashContext); }; let outputText: string; @@ -862,7 +928,7 @@ export class ReadTool implements AgentTool { if (shouldAddHashLines) { outputText = `[Line ${startLineDisplay} is ${formatBytes( firstLineBytes, - )}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`; + )}, exceeds ${formatBytes(DEFAULT_MAX_BYTES)} limit. Hashline output requires full lines; cannot emit an editable numbered preview for a truncated line.]`; } else { outputText = formatText(snippet.text, startLineDisplay); } @@ -928,6 +994,12 @@ export class ReadTool implements AgentTool { const totalLines = allLines.length; const shouldAddHashLines = displayMode.hashLines; const shouldAddLineNumbers = shouldAddHashLines ? false : displayMode.lineNumbers; + const hashContext = + shouldAddHashLines && options.sourcePath + ? buildHashlineHeaderContext(formatPathRelativeToCwd(options.sourcePath, this.session.cwd), text) + : undefined; + recordHashlineSnapshot(this.session, options.sourcePath, hashContext); + let emittedHashlineHeader = false; const resultBuilder = toolResult(details); if (options.sourcePath) resultBuilder.sourcePath(options.sourcePath); @@ -943,7 +1015,9 @@ export class ReadTool implements AgentTool { } const effectiveEnd = Math.min(range.endLine ?? totalLines, totalLines); const sliced = allLines.slice(range.startLine - 1, effectiveEnd).join("\n"); - parts.push(formatTextWithMode(sliced, range.startLine, shouldAddHashLines, shouldAddLineNumbers)); + const formatted = formatTextWithMode(sliced, range.startLine, shouldAddHashLines, shouldAddLineNumbers); + parts.push(hashContext && !emittedHashlineHeader ? prependHashlineHeader(formatted, hashContext) : formatted); + if (hashContext) emittedHashlineHeader = true; } const outputText = parts.length > 0 ? parts.join("\n\n…\n\n") : ""; @@ -1002,6 +1076,11 @@ export class ReadTool implements AgentTool { const shouldAddHashLines = !rawSelector && displayMode.hashLines; const shouldAddLineNumbers = rawSelector ? false : shouldAddHashLines ? false : displayMode.lineNumbers; + const hashContext = shouldAddHashLines + ? await readHashlineHeaderContext(absolutePath, this.session.cwd) + : undefined; + recordHashlineSnapshot(this.session, absolutePath, hashContext); + let emittedHashlineHeader = false; const maxColumns = resolveOutputMaxColumns(this.session.settings); const blocks: string[] = []; @@ -1042,11 +1121,18 @@ export class ReadTool implements AgentTool { } if (collectedLines.length > 0) { - getFileReadCache(this.session).recordContiguous(absolutePath, range.startLine, collectedLines); + getFileReadCache(this.session).recordContiguous( + absolutePath, + range.startLine, + collectedLines, + hashContext ? { fullText: hashContext.fullText, fileHash: hashContext.fileHash } : {}, + ); } const blockText = collectedLines.join("\n"); - blocks.push(formatTextWithMode(blockText, range.startLine, shouldAddHashLines, shouldAddLineNumbers)); + const formatted = formatTextWithMode(blockText, range.startLine, shouldAddHashLines, shouldAddLineNumbers); + blocks.push(hashContext && !emittedHashlineHeader ? prependHashlineHeader(formatted, hashContext) : formatted); + if (hashContext) emittedHashlineHeader = true; } let outputText = blocks.join("\n\n…\n\n"); @@ -1335,7 +1421,7 @@ export class ReadTool implements AgentTool { #renderSummary(summary: SummaryResult): { text: string; displayText: string; - elidedSpans: number; + elidedRanges: ElidedRange[]; elidedLines: number; } { const displayMode = resolveFileDisplayMode(this.session); @@ -1396,13 +1482,13 @@ export class ReadTool implements AgentTool { const modelParts: string[] = []; const displayParts: string[] = []; - let elidedSpans = 0; + const elidedRanges: ElidedRange[] = []; let elidedLines = 0; for (const unit of units) { if (unit.kind === "elided") { modelParts.push("..."); displayParts.push("..."); - elidedSpans++; + elidedRanges.push({ start: unit.startLine, end: unit.endLine }); elidedLines += unit.endLine - unit.startLine + 1; continue; } @@ -1417,7 +1503,9 @@ export class ReadTool implements AgentTool { ); modelParts.push(formatted.model); displayParts.push(formatted.display); - elidedSpans++; + // Suggest the full brace range so re-reading shows both braces + // plus the elided body in one shot. + elidedRanges.push({ start: unit.startLine, end: unit.endLine }); // Merged brace pair encloses (start+1)..(end-1) as elided. elidedLines += Math.max(0, unit.endLine - unit.startLine - 1); continue; @@ -1426,7 +1514,7 @@ export class ReadTool implements AgentTool { displayParts.push(unit.text); } - return { text: modelParts.join("\n"), displayText: displayParts.join("\n"), elidedSpans, elidedLines }; + return { text: modelParts.join("\n"), displayText: displayParts.join("\n"), elidedRanges, elidedLines }; } async execute( @@ -1674,15 +1762,20 @@ export class ReadTool implements AgentTool { const renderedSummary = this.#renderSummary(summary); const footer = formatSummaryElisionFooter( localReadPath, - renderedSummary.elidedSpans, + renderedSummary.elidedRanges, renderedSummary.elidedLines, ); - const modelText = footer ? `${renderedSummary.text}\n\n${footer}` : renderedSummary.text; + const summaryHashContext = displayMode.hashLines + ? await readHashlineHeaderContext(absolutePath, this.session.cwd) + : undefined; + recordHashlineSnapshot(this.session, absolutePath, summaryHashContext); + const bodyText = footer ? `${renderedSummary.text}\n\n${footer}` : renderedSummary.text; + const modelText = prependHashlineHeader(bodyText, summaryHashContext); details = { displayContent: { text: renderedSummary.displayText, startLine: 1 }, summary: { lines: countTextLines(renderedSummary.text), - elidedSpans: renderedSummary.elidedSpans, + elidedSpans: renderedSummary.elidedRanges.length, elidedLines: renderedSummary.elidedLines, }, }; @@ -1820,16 +1913,29 @@ export class ReadTool implements AgentTool { firstLineExceedsLimit, }; - if (collectedLines.length > 0 && !firstLineExceedsLimit) { - getFileReadCache(this.session).recordContiguous(absolutePath, startLineDisplay, collectedLines); - } - const shouldAddHashLines = !rawSelector && displayMode.hashLines; const shouldAddLineNumbers = rawSelector ? false : shouldAddHashLines ? false : displayMode.lineNumbers; + const hashContext = shouldAddHashLines + ? await readHashlineHeaderContext(absolutePath, this.session.cwd) + : undefined; + + if (collectedLines.length > 0 && !firstLineExceedsLimit) { + getFileReadCache(this.session).recordContiguous( + absolutePath, + startLineDisplay, + collectedLines, + hashContext ? { fullText: hashContext.fullText, fileHash: hashContext.fileHash } : {}, + ); + } + let capturedDisplayContent: { text: string; startLine: number } | undefined; + let emittedHashlineHeader = false; const formatText = (text: string, startNum: number): string => { capturedDisplayContent = { text, startLine: startNum }; - return formatTextWithMode(text, startNum, shouldAddHashLines, shouldAddLineNumbers); + const formatted = formatTextWithMode(text, startNum, shouldAddHashLines, shouldAddLineNumbers); + if (!hashContext || emittedHashlineHeader) return formatted; + emittedHashlineHeader = true; + return prependHashlineHeader(formatted, hashContext); }; let outputText: string; @@ -1841,7 +1947,7 @@ export class ReadTool implements AgentTool { if (shouldAddHashLines) { outputText = `[Line ${startLineDisplay} is ${formatBytes( firstLineBytes, - )}, exceeds ${formatBytes(maxBytesForRead)} limit. Hashline output requires full lines; cannot compute hashes for a truncated preview.]`; + )}, exceeds ${formatBytes(maxBytesForRead)} limit. Hashline output requires full lines; cannot emit an editable numbered preview for a truncated line.]`; } else { outputText = formatText(snippet.text, startLineDisplay); } @@ -1964,7 +2070,12 @@ export class ReadTool implements AgentTool { const shouldAddLineNumbers = shouldAddHashLines ? false : displayMode.lineNumbers; const rawText = region.lines.join("\n"); - const formattedText = formatTextWithMode(rawText, region.startLine, shouldAddHashLines, shouldAddLineNumbers); + const hashContext = shouldAddHashLines + ? await readHashlineHeaderContext(entry.absolutePath, this.session.cwd) + : undefined; + recordHashlineSnapshot(this.session, entry.absolutePath, hashContext); + const formattedBody = formatTextWithMode(rawText, region.startLine, shouldAddHashLines, shouldAddLineNumbers); + const formattedText = prependHashlineHeader(formattedBody, hashContext); const details: ReadToolDetails = { resolvedPath: entry.absolutePath, diff --git a/packages/coding-agent/src/tools/search-tool-bm25.ts b/packages/coding-agent/src/tools/search-tool-bm25.ts index 25feebc46..a6793e84a 100644 --- a/packages/coding-agent/src/tools/search-tool-bm25.ts +++ b/packages/coding-agent/src/tools/search-tool-bm25.ts @@ -19,12 +19,6 @@ import type { ToolSession } from "."; import { formatCount, replaceTabs, TRUNCATE_LENGTHS } from "./render-utils"; import { ToolError } from "./tool-errors"; -// Re-export legacy MCP types for back-compat (tests and external callers may reference them) -export type { - DiscoverableMCPSearchIndex, - DiscoverableMCPTool, -} from "../mcp/discoverable-tool-metadata"; - const DEFAULT_LIMIT = 8; const TOOL_DISCOVERY_TITLE = "Tool Discovery"; const COLLAPSED_MATCH_LIMIT = 5; @@ -81,21 +75,7 @@ function buildSearchToolBm25Content(details: SearchToolBm25Details): string { /** Get discoverable tools for description rendering. Falls back to empty array on error. */ function getDiscoverableToolsForDescription(session: ToolSession): DiscoverableTool[] { try { - // Prefer generic method; fall back to legacy MCP-only - if (session.getDiscoverableTools) { - return session.getDiscoverableTools(); - } - // Legacy MCP path — adapt DiscoverableMCPTool (with `description`) → DiscoverableTool. - const legacy = session.getDiscoverableMCPTools?.() ?? []; - return legacy.map(t => ({ - name: t.name, - label: t.label, - summary: t.description, - source: "mcp" as const, - serverName: t.serverName, - mcpToolName: t.mcpToolName, - schemaKeys: t.schemaKeys, - })); + return session.getDiscoverableTools?.() ?? []; } catch { return []; } @@ -103,15 +83,8 @@ function getDiscoverableToolsForDescription(session: ToolSession): DiscoverableT function getDiscoverableToolSearchIndexForExecution(session: ToolSession): DiscoverableToolSearchIndex { try { - // Prefer generic cached index - if (session.getDiscoverableToolSearchIndex) { - const cached = session.getDiscoverableToolSearchIndex(); - if (cached) return cached; - } - // Legacy MCP: use cached MCP index. Its documents expose `tool.description` as well as - // `tool.summary`, so it is structurally compatible with DiscoverableToolSearchIndex. - const mcpCached = session.getDiscoverableMCPSearchIndex?.(); - if (mcpCached) return mcpCached as unknown as DiscoverableToolSearchIndex; + const cached = session.getDiscoverableToolSearchIndex?.(); + if (cached) return cached; } catch {} return buildDiscoverableToolSearchIndex(getDiscoverableToolsForDescription(session)); } diff --git a/packages/coding-agent/src/tools/search.ts b/packages/coding-agent/src/tools/search.ts index c93d398ea..d43af19ca 100644 --- a/packages/coding-agent/src/tools/search.ts +++ b/packages/coding-agent/src/tools/search.ts @@ -9,6 +9,7 @@ import { prompt, untilAborted } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; import { getFileReadCache } from "../edit/file-read-cache"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; +import { computeFileHash, formatHashlineHeader } from "../hashline/hash"; import type { Theme } from "../modes/theme/theme"; import searchDescription from "../prompts/tools/search.md" with { type: "text" }; import { DEFAULT_MAX_COLUMN, type TruncationResult, truncateHead } from "../session/streaming-output"; @@ -38,13 +39,13 @@ import { import { ToolError } from "./tool-errors"; import { toolResult } from "./tool-result"; +const searchPathEntrySchema = z.string().describe("file, directory, glob, or internal URL to search"); const searchSchema = z .object({ pattern: z.string().describe("regex pattern"), paths: z - .array(z.string().describe("file, directory, glob, or internal URL to search")) - .min(1) - .describe("files, directories, globs, or internal URLs to search"), + .union([searchPathEntrySchema, z.array(searchPathEntrySchema).min(1)]) + .describe("file, directory, glob, internal URL, or array of those to search"), i: z.boolean().optional().describe("case-insensitive search"), gitignore: z.boolean().optional().describe("respect gitignore"), skip: z @@ -55,6 +56,9 @@ const searchSchema = z .strict(); export type SearchToolInput = z.infer; +export function toPathList(input: string | string[] | undefined): string[] { + return typeof input === "string" ? [input] : (input ?? []); +} /** Maximum number of distinct files surfaced in a single response. The * agent paginates further pages via `skip`. */ @@ -236,7 +240,7 @@ export class SearchTool implements AgentTool, _toolContext?: AgentToolContext, ): Promise> { - const { pattern, paths, i, gitignore, skip } = params; + const { pattern, paths: rawPaths, i, gitignore, skip } = params; return untilAborted(signal, async () => { const normalizedPattern = pattern.trim(); @@ -248,6 +252,7 @@ export class SearchTool implements AgentTool(); + if (baseDisplayMode.hashLines) { + for (const relativePath of fileList) { + if (archiveDisplaySet.has(relativePath)) continue; + const absoluteFilePath = path.resolve(this.session.cwd, relativePath); + if (immutableSourcePaths.has(absoluteFilePath)) continue; + try { + const fullText = await Bun.file(absoluteFilePath).text(); + const fileHash = computeFileHash(fullText); + hashContexts.set(relativePath, { absolutePath: absoluteFilePath, fileHash }); + } catch { + // Best-effort: if the file disappeared between grep and render, fall back to plain line output. + } + } + } const renderMatchesForFile = (relativePath: string): { model: string[]; display: string[] } => { const modelOut: string[] = []; const displayOut: string[] = []; const fileMatches = matchesByFile.get(relativePath) ?? []; - const absoluteFilePath = path.resolve(this.session.cwd, relativePath); - const useHashLines = immutableSourcePaths.has(absoluteFilePath) - ? immutableDisplayMode.hashLines - : baseDisplayMode.hashLines; + const hashContext = hashContexts.get(relativePath); + const useHashLines = hashContext !== undefined; const lineNumberWidth = fileMatches.reduce((width, match) => { let nextWidth = Math.max(width, String(match.lineNumber).length); for (const ctx of match.contextBefore ?? []) { @@ -533,17 +550,21 @@ export class SearchTool implements AgentTool 0 && !archiveDisplaySet.has(relativePath)) { - getFileReadCache(this.session).recordSparse(path.resolve(searchPath, relativePath), cacheEntries); + if (cacheEntries.length > 0 && hashContext) { + getFileReadCache(this.session).recordSparse(hashContext.absolutePath, cacheEntries, { + fileHash: hashContext.fileHash, + }); } return { model: modelOut, display: displayOut }; }; if (isDirectory) { const grouped = formatGroupedFiles(fileList, relativePath => { const rendered = renderMatchesForFile(relativePath); + const hashContext = hashContexts.get(relativePath); return { modelLines: rendered.model, displayLines: rendered.display, + headerSuffix: hashContext ? `#${hashContext.fileHash}` : "", skip: rendered.model.length === 0, }; }); @@ -552,6 +573,15 @@ export class SearchTool implements AgentTool 0) { + outputLines.push(""); + displayLines.push(""); + } + const hashContext = hashContexts.get(relativePath); + if (hashContext) { + outputLines.push(formatHashlineHeader(relativePath, hashContext.fileHash)); + } outputLines.push(...rendered.model); displayLines.push(...rendered.display); } @@ -607,7 +637,7 @@ export class SearchTool implements AgentTool 0) meta.push(`skip:${args.skip}`); @@ -745,11 +776,12 @@ export const searchToolRenderer = { let contextDir = searchBase ?? ""; return group.map(line => { if (line.startsWith("## ")) { - // Strip optional ` (suffix)` like ` (3 replacements)` before resolving. + // Strip optional ` (suffix)` and `#hash` before resolving. const fileName = line .slice(3) .trimEnd() - .replace(/\s+\([^)]*\)\s*$/, ""); + .replace(/\s+\([^)]*\)\s*$/, "") + .replace(/#[0-9a-f]+$/, ""); const absPath = contextDir && fileName ? path.join(contextDir, fileName) : undefined; const styled = uiTheme.fg("dim", line); return absPath ? fileHyperlink(absPath, styled) : styled; @@ -760,7 +792,7 @@ export const searchToolRenderer = { .trimEnd() .replace(/\s+\([^)]*\)\s*$/, ""); const isDirectory = raw.endsWith("/"); - const name = raw.replace(/\/$/, ""); + const name = isDirectory ? raw.replace(/\/$/, "") : raw.replace(/#[0-9a-f]+$/, ""); if (isDirectory) { if (searchBase) { contextDir = name === "." ? searchBase : path.join(searchBase, name); diff --git a/packages/coding-agent/src/tools/vim.ts b/packages/coding-agent/src/tools/vim.ts deleted file mode 100644 index 862342b2f..000000000 --- a/packages/coding-agent/src/tools/vim.ts +++ /dev/null @@ -1,949 +0,0 @@ -import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core"; -import type { Component } from "@oh-my-pi/pi-tui"; -import { extractSegments, sliceWithWidth, Text } from "@oh-my-pi/pi-tui"; -import { isEnoent, logger, prompt, untilAborted } from "@oh-my-pi/pi-utils"; -import * as Diff from "diff"; -import * as z from "zod/v4"; -import type { RenderResultOptions } from "../extensibility/custom-tools/types"; -import { createLspWritethrough, type FileDiagnosticsResult, type WritethroughCallback, writethroughNoop } from "../lsp"; -import { getLanguageFromPath, highlightCode, type Theme } from "../modes/theme/theme"; -import vimDescription from "../prompts/tools/vim.md" with { type: "text" }; -import { CachedOutputBlock } from "../tui/output-block"; -import { renderStatusLine } from "../tui/status-line"; -import { VimBuffer } from "../vim/buffer"; -import { VimEngine, type VimSaveResult } from "../vim/engine"; -import { parseKeySequences } from "../vim/parser"; -import { - buildDetails, - computeViewport, - renderVimDetails, - VIM_DEFAULT_VIEWPORT_LINES, - VIM_OPEN_VIEWPORT_LINES, -} from "../vim/render"; -import type { VimFingerprint, VimKeyToken, VimLoadedFile, VimToolDetails, VimViewportLine } from "../vim/types"; -import { VimInputError } from "../vim/types"; -import type { ToolSession } from "."; -import { parseArchivePathCandidates } from "./archive-reader"; -import { assertEditableFile } from "./auto-generated-guard"; -import { isReadableUrlPath } from "./fetch"; -import { normalizePathLikeInput, resolveToCwd } from "./path-utils"; -import { enforcePlanModeWrite } from "./plan-mode-guard"; -import { formatDiagnostics, replaceTabs } from "./render-utils"; -import { isSqliteFile, parseSqlitePathCandidates } from "./sqlite-reader"; -import { ToolError } from "./tool-errors"; -import { toolResult } from "./tool-result"; - -const INTERNAL_URL_PREFIX = /^(agent|artifact|skill|rule|local|mcp):\/\//; -const utf8Decoder = new TextDecoder("utf-8", { fatal: true }); - -const vimStepSchema = z.object({ - kbd: z.array(z.string()).describe("vim key sequences"), - insert: z.string().optional().describe("raw text to insert"), -}); - -const vimSchema = z.object({ - file: z.string().describe("file path"), - steps: z.array(vimStepSchema).optional().describe("editing steps"), - pause: z.boolean().optional().describe("skip auto-save"), -}); - -type VimParams = z.infer; -type VimStep = z.infer; - -interface VimRenderStep { - kbd?: string[]; - insert?: string; -} - -export interface VimRenderArgs { - file?: string; - steps?: VimRenderStep[]; - pause?: boolean; - __partialJson?: string; -} - -function fingerprintEqual(left: VimFingerprint | null, right: VimFingerprint | null): boolean { - if (left === null || right === null) { - return left === right; - } - return ( - left.exists === right.exists && - left.size === right.size && - left.mtimeMs === right.mtimeMs && - left.hash === right.hash - ); -} - -function renderText(text: string): Component { - return new Text(replaceTabs(text), 0, 0); -} - -function serializeBufferText(buffer: Pick): string { - return `${buffer.getText()}${buffer.trailingNewline ? "\n" : ""}`; -} - -function buildModelDiff(beforeText: string, afterText: string): string | undefined { - if (beforeText === afterText) { - return undefined; - } - const patch = Diff.structuredPatch("", "", beforeText, afterText, "", "", { context: 3 }); - const diff = patch.hunks - .flatMap(hunk => [`@@ -${hunk.oldStart},${hunk.oldLines} +${hunk.newStart},${hunk.newLines} @@`, ...hunk.lines]) - .join("\n"); - return diff.length > 0 ? diff : undefined; -} - -function renderViewportCursor(line: VimViewportLine, styledText: string, uiTheme: Theme): string { - if (!line.isCursor || line.cursorCol === undefined) { - return styledText; - } - - const totalWidth = Bun.stringWidth(line.text); - const cursorCol = Math.max(0, Math.min(line.cursorCol, totalWidth)); - const cursorSlice = sliceWithWidth(line.text, cursorCol, 1, false); - const replaceWidth = cursorSlice.width; - const afterStart = Math.min(totalWidth, cursorCol + replaceWidth); - const segments = extractSegments(styledText, cursorCol, afterStart, Math.max(0, totalWidth - afterStart), true); - const cursorText = cursorSlice.text.length > 0 ? cursorSlice.text : " "; - const invertedCursor = uiTheme.inverse(cursorText); - const cursorHighlight = invertedCursor === cursorText ? `\x1b[7m${cursorText}\x1b[27m` : invertedCursor; - return `${segments.before}${cursorHighlight}${segments.after}`; -} - -function renderViewportLine(line: VimViewportLine, styledText: string, padWidth: number, uiTheme: Theme): string { - const marker = line.isCursor ? ">" : line.isSelected ? "*" : ""; - const gutterText = `${marker}${line.line}`.padStart(padWidth + 1, " "); - const gutterStyled = line.isCursor - ? uiTheme.fg("accent", gutterText) - : line.isSelected - ? uiTheme.fg("warning", gutterText) - : uiTheme.fg("dim", gutterText); - const separator = uiTheme.fg("dim", "│"); - return `${gutterStyled}${separator}${renderViewportCursor(line, styledText, uiTheme)}`; -} - -function splitTokensBySequence(kbd: string[]): Array<{ sequence: string; tokens: VimKeyToken[] }> { - const groups = new Map(); - for (const token of parseKeySequences(kbd)) { - const group = groups.get(token.sequenceIndex); - if (group) { - group.push(token); - continue; - } - groups.set(token.sequenceIndex, [token]); - } - return kbd.map((sequence, sequenceIndex) => ({ sequence, tokens: groups.get(sequenceIndex) ?? [] })); -} - -async function executeKeySequences( - engine: VimEngine, - groups: Array<{ sequence: string; tokens: VimKeyToken[] }>, - commandText: string, - onStep?: () => Promise, -): Promise { - for (let index = 0; index < groups.length; index += 1) { - const group = groups[index]!; - if (group.tokens.length === 0) { - continue; - } - await engine.executeTokens(group.tokens, commandText, onStep); - if (index < groups.length - 1 && engine.inputMode === "insert") { - // Roll back partial changes to prevent buffer corruption across calls. - engine.rollbackPendingInsert(); - const nextSeq = groups[index + 1]?.sequence ?? ""; - const looksLikeText = nextSeq.length > 0 && /\s/.test(nextSeq) && !/^[:/%]/.test(nextSeq); - let hint = - "Use the insert field for inserted text, or include to return to NORMAL mode before the next kbd entry."; - if (looksLikeText) { - hint += ` The next entry (\`${nextSeq.length > 40 ? `${nextSeq.slice(0, 37)}...` : nextSeq}\`) looks like text content — put it in the \`insert\` field instead. For another edit location, add a new \`steps\` entry instead of another kbd entry.`; - } - throw new VimInputError( - `Sequence ${index + 1} (\`${group.sequence}\`) entered INSERT mode — changes rolled back. ${hint}`, - group.tokens[group.tokens.length - 1], - ); - } - } -} - -// Module-level cache of the last real vim result so renderCall can reuse that -// viewport while the next tool call is still streaming. -let lastVimDetails: VimToolDetails | undefined; -function buildToolDetailsFromEngine( - engine: VimEngine, - viewportLines: number, - preferredStart?: number, - closed = false, - errorLocation?: VimToolDetails["errorLocation"], - statusMessage?: string, -): VimToolDetails { - const cursorLine = engine.buffer.cursor.line + 1; - const cursorCol = engine.buffer.cursor.col + 1; - const viewport = computeViewport(cursorLine, engine.buffer.lineCount(), viewportLines, preferredStart); - const details = buildDetails({ - file: engine.buffer.displayPath, - mode: engine.getPublicMode(), - cursor: { line: cursorLine, col: cursorCol }, - totalLines: engine.buffer.lineCount(), - modified: engine.buffer.modified, - lines: engine.buffer.lines, - viewport, - selection: engine.getSelection(), - lastCommand: engine.lastCommand, - statusMessage: statusMessage ?? engine.statusMessage, - pendingInput: engine.getPendingInput(), - errorLocation, - closed, - }); - details.diagnostics = engine.diagnostics; - return details; -} - -function getLastStepInsert(steps: readonly VimStep[] | undefined): string | undefined { - if (!steps || steps.length === 0) { - return undefined; - } - return steps[steps.length - 1]?.insert; -} - -function getNormalizedSteps(steps: VimRenderArgs["steps"]): VimStep[] | undefined { - if (!Array.isArray(steps)) { - return undefined; - } - return steps.map(step => ({ - kbd: Array.isArray(step?.kbd) ? [...step.kbd] : [], - ...(step?.insert !== undefined ? { insert: step.insert } : {}), - })); -} - -function getStepsForDisplay(args: VimRenderArgs): VimStep[] | undefined { - const steps = getNormalizedSteps(args.steps); - if (!steps || steps.length === 0) { - return steps; - } - - const partialInsert = extractPartialInsert(args.__partialJson); - if (partialInsert === undefined) { - return steps; - } - - const lastStep = steps[steps.length - 1]!; - if (lastStep.insert === undefined || partialInsert.length >= lastStep.insert.length) { - lastStep.insert = partialInsert; - } - return steps; -} - -function splitInsertIntoChunks(text: string): string[] { - const maxChunkChars = 32; - if (text.length <= maxChunkChars) { - return text.length === 0 ? [] : [text]; - } - - const chunks: string[] = []; - let start = 0; - while (start < text.length) { - let end = Math.min(start + maxChunkChars, text.length); - if (end < text.length) { - const lastNewline = text.lastIndexOf("\n", end - 1); - if (lastNewline >= start) { - end = lastNewline + 1; - } else { - const lastSpace = Math.max(text.lastIndexOf(" ", end - 1), text.lastIndexOf("\t", end - 1)); - if (lastSpace >= start + Math.floor(maxChunkChars / 2)) { - end = lastSpace + 1; - } - } - } - if (end <= start) { - end = Math.min(start + maxChunkChars, text.length); - } - chunks.push(text.slice(start, end)); - start = end; - } - return chunks; -} - -async function applyInsertWithStreaming( - engine: VimEngine, - text: string, - exitInsertMode: boolean, - onStep?: () => Promise, -): Promise { - const chunks = splitInsertIntoChunks(text); - if (chunks.length === 0) { - await engine.applyLiteralInsert("", exitInsertMode); - return; - } - - for (let index = 0; index < chunks.length; index += 1) { - await engine.applyLiteralInsert(chunks[index]!, exitInsertMode && index === chunks.length - 1); - await onStep?.(); - } -} - -interface ExecuteVimStepsOptions { - pauseLastStep?: boolean; - onKbdStep?: () => Promise; - onInsertStep?: () => Promise; -} - -// Auto-reorder line-positioned steps to descending order (bottom-up) when all steps -// are simple `NG` patterns and appear in ascending order (top-down). Bottom-up -// ordering is safe for any mix of insert/replace commands because edits at higher -// line numbers never shift lower line numbers. -function autoReorderSteps(steps: readonly VimStep[]): VimStep[] { - if (steps.length < 2) return [...steps]; - - // Match single kbd entry of `G` where cmd enters insert mode - const linePattern = /^(\d+)G(o|O|cc|C|S|s|i|I|a|A)$/; - const parsed: Array<{ line: number; step: VimStep }> = []; - for (const step of steps) { - if (step.kbd.length !== 1) return [...steps]; - const match = step.kbd[0]!.match(linePattern); - if (!match) return [...steps]; - parsed.push({ line: Number(match[1]), step }); - } - - // Only reorder if steps are in strictly ascending order (top-down, likely a mistake). - // If already descending, mixed, or equal, the model likely planned the order deliberately. - for (let i = 1; i < parsed.length; i++) { - if (parsed[i]!.line <= parsed[i - 1]!.line) { - return [...steps]; - } - } - - // Sort by descending line number (bottom-up) - parsed.sort((a, b) => b.line - a.line); - logger.debug("vim: auto-reordered steps to bottom-up", { - original: steps.map(s => s.kbd[0]), - reordered: parsed.map(p => p.step.kbd[0]), - }); - return parsed.map(p => p.step); -} - -async function executeVimSteps( - engine: VimEngine, - steps: readonly VimStep[], - options: ExecuteVimStepsOptions = {}, -): Promise { - // Auto-reorder ascending line-positioned steps to descending (bottom-up) - // to prevent line-shift corruption from top-down edits. - const orderedSteps = autoReorderSteps(steps); - for (let index = 0; index < orderedSteps.length; index += 1) { - if (engine.closed) { - break; - } - - const step = orderedSteps[index]!; - const isLast = index === orderedSteps.length - 1; - const hasKbd = step.kbd.some(sequence => sequence.length > 0); - const preservePausedState = !hasKbd && step.insert === undefined && isLast && options.pauseLastStep === true; - if (engine.inputMode === "insert" && (hasKbd || step.insert === undefined) && !preservePausedState) { - engine.rollbackPendingInsert(); - } - - if (step.kbd.length > 0) { - const commandText = step.kbd.join(" "); - const tokenGroups = splitTokensBySequence(step.kbd); - await executeKeySequences(engine, tokenGroups, commandText, options.onKbdStep); - } - - if (!engine.closed && step.insert !== undefined && (step.insert.length > 0 || engine.inputMode === "insert")) { - // Strip trailing newline from insert text — `o`/`O` already create a line boundary, - // so a trailing \n would produce an unwanted blank line. - const normalizedInsert = step.insert.endsWith("\n") ? step.insert.slice(0, -1) : step.insert; - const exitInsertMode = !(isLast && options.pauseLastStep === true); - await applyInsertWithStreaming(engine, normalizedInsert, exitInsertMode, options.onInsertStep); - } - - if (!isLast && engine.inputMode === "insert") { - engine.rollbackPendingInsert(); - } - } -} - -async function statFingerprint(absolutePath: string): Promise { - try { - const file = Bun.file(absolutePath); - const stat = await file.stat(); - if (!stat.isFile()) { - throw new ToolError(`Not a regular file: ${absolutePath}`); - } - const bytes = await file.bytes(); - return { - exists: true, - size: stat.size, - mtimeMs: stat.mtimeMs, - hash: String(Bun.hash(bytes)), - }; - } catch (error) { - if (isEnoent(error)) { - return null; - } - throw error; - } -} - -async function readTextFile( - absolutePath: string, -): Promise<{ lines: string[]; trailingNewline: boolean; fingerprint: VimFingerprint | null }> { - try { - const file = Bun.file(absolutePath); - const stat = await file.stat(); - if (!stat.isFile()) { - throw new ToolError(`Not a regular file: ${absolutePath}`); - } - const bytes = await file.bytes(); - for (const byte of bytes) { - if (byte === 0) { - throw new ToolError("Edit tool in vim mode only supports UTF-8 text files in v1"); - } - } - const text = utf8Decoder.decode(bytes); - const trailingNewline = text.endsWith("\n"); - const body = trailingNewline ? text.slice(0, -1) : text; - return { - lines: body.length === 0 ? [""] : body.split("\n"), - trailingNewline, - fingerprint: { - exists: true, - size: stat.size, - mtimeMs: stat.mtimeMs, - hash: String(Bun.hash(bytes)), - }, - }; - } catch (error) { - if (isEnoent(error)) { - return { - lines: [""], - trailingNewline: false, - fingerprint: null, - }; - } - if (error instanceof TypeError) { - throw new ToolError("Edit tool in vim mode only supports UTF-8 text files in v1"); - } - throw error; - } -} - -function normalizeTargetPath(inputPath: string, cwd: string): { absolutePath: string; displayPath: string } { - const normalized = normalizePathLikeInput(inputPath); - if (INTERNAL_URL_PREFIX.test(normalized)) { - throw new ToolError("Edit tool in vim mode only supports regular filesystem paths in v1"); - } - if (isReadableUrlPath(normalized)) { - throw new ToolError("Edit tool in vim mode only supports local filesystem paths in v1"); - } - if (parseArchivePathCandidates(normalized).some(candidate => candidate.archivePath === normalized)) { - throw new ToolError("Edit tool in vim mode does not support archive targets in v1"); - } - if (parseSqlitePathCandidates(normalized).some(candidate => candidate.sqlitePath === normalized)) { - throw new ToolError("Edit tool in vim mode does not support SQLite targets in v1"); - } - return { - absolutePath: resolveToCwd(normalized, cwd), - displayPath: normalized, - }; -} - -export class VimTool implements AgentTool { - readonly name = "vim"; - readonly label = "Vim"; - readonly description: string; - readonly parameters = vimSchema; - readonly concurrency = "exclusive"; - - #engines = new Map(); - #writethrough: WritethroughCallback; - - constructor(private readonly session: ToolSession) { - const enableLsp = session.enableLsp ?? true; - const enableFormat = enableLsp && session.settings.get("lsp.formatOnWrite"); - const enableDiagnostics = enableLsp && session.settings.get("lsp.diagnosticsOnWrite"); - this.#writethrough = enableLsp - ? createLspWritethrough(session.cwd, { enableFormat, enableDiagnostics }) - : writethroughNoop; - this.description = prompt.render(vimDescription); - } - - async #loadBuffer(targetPath: string): Promise { - const { absolutePath, displayPath } = normalizeTargetPath(targetPath, this.session.cwd); - if (await isSqliteFile(absolutePath)) { - throw new ToolError("Edit tool in vim mode does not support SQLite targets in v1"); - } - const loaded = await readTextFile(absolutePath); - return { - absolutePath, - displayPath, - lines: loaded.lines, - trailingNewline: loaded.trailingNewline, - fingerprint: loaded.fingerprint, - }; - } - - async #beforeMutate(buffer: VimBuffer): Promise { - enforcePlanModeWrite(this.session, buffer.displayPath, { op: buffer.baseFingerprint ? "update" : "create" }); - if (!buffer.editabilityChecked && buffer.baseFingerprint) { - await assertEditableFile(buffer.filePath, buffer.displayPath); - buffer.editabilityChecked = true; - } - } - - async #saveBuffer(buffer: VimBuffer, options?: { force?: boolean }): Promise { - enforcePlanModeWrite(this.session, buffer.displayPath, { op: buffer.baseFingerprint ? "update" : "create" }); - if (buffer.baseFingerprint) { - await assertEditableFile(buffer.filePath, buffer.displayPath); - } - if (!options?.force) { - const diskFingerprint = await statFingerprint(buffer.filePath); - if (!fingerprintEqual(buffer.baseFingerprint, diskFingerprint)) { - throw new ToolError("File changed on disk since open; reload with :e! before saving."); - } - } - const content = `${buffer.getText()}${buffer.trailingNewline ? "\n" : ""}`; - const diagnostics = (await this.#writethrough(buffer.filePath, content)) as FileDiagnosticsResult | undefined; - const loaded = await this.#loadBuffer(buffer.displayPath); - return { loaded, diagnostics }; - } - - #renderFromEngine( - engine: VimEngine, - viewportLines: number, - preferredStart?: number, - closed = false, - errorLocation?: VimToolDetails["errorLocation"], - statusMessage?: string, - modelDiff?: string, - ): AgentToolResult { - const details = buildToolDetailsFromEngine( - engine, - viewportLines, - preferredStart, - closed, - errorLocation, - statusMessage, - ); - const resultText = modelDiff ? `${renderVimDetails(details)}\n\nDiff:\n${modelDiff}` : renderVimDetails(details); - const builder = toolResult(details).text(resultText); - if (engine.diagnostics) { - builder.diagnostics(engine.diagnostics.summary, engine.diagnostics.messages ?? []); - } - lastVimDetails = details; - return builder.done(); - } - - #throwWithSnapshot(engine: VimEngine, error: unknown): never { - const location = error instanceof VimInputError ? error.location : undefined; - const statusMessage = error instanceof Error ? error.message : String(error); - const result = this.#renderFromEngine( - engine, - VIM_DEFAULT_VIEWPORT_LINES, - engine.viewportStart, - engine.closed, - location, - statusMessage, - ); - const text = result.content.find(block => block.type === "text")?.text ?? statusMessage; - throw new ToolError(text); - } - - async execute( - _toolCallId: string, - params: VimParams, - signal?: AbortSignal, - onUpdate?: AgentToolUpdateCallback, - _context?: AgentToolContext, - ): Promise> { - return untilAborted(signal, async () => { - // Resolve file path and get-or-create engine for this buffer - const { absolutePath } = normalizeTargetPath(params.file, this.session.cwd); - let engine = this.#engines.get(absolutePath); - let isNewBuffer = false; - if (!engine) { - const loaded = await this.#loadBuffer(params.file); - engine = new VimEngine(new VimBuffer(loaded), { - beforeMutate: buffer => this.#beforeMutate(buffer), - loadBuffer: path => this.#loadBuffer(path), - saveBuffer: (buffer, options) => this.#saveBuffer(buffer, options), - }); - engine.viewportStart = 1; - this.#engines.set(absolutePath, engine); - isNewBuffer = true; - } else if (!engine.buffer.modified) { - // Sync fingerprint from disk to handle LSP writethrough reformats - const fp = await statFingerprint(absolutePath); - if (fp) engine.buffer.baseFingerprint = fp; - } - - const steps = params.steps; - if (!steps || steps.length === 0) { - // No steps — just show the file viewport - if (isNewBuffer) { - engine.statusMessage = `Opened ${engine.buffer.displayPath}`; - } - return this.#renderFromEngine(engine, VIM_OPEN_VIEWPORT_LINES, engine.viewportStart); - } - - const beforeText = serializeBufferText(engine.buffer); - - if (this.session.getPlanModeState?.()?.enabled) { - if (steps.some(step => step.insert !== undefined)) { - throw new ToolError("Plan mode: edit is read-only in vim mode; insert payloads are not allowed."); - } - const preview = engine.clone({ - beforeMutate: async () => { - throw new VimInputError( - "Plan mode: edit is read-only in vim mode; only navigation, search, open, and close are allowed.", - ); - }, - saveBuffer: async () => { - throw new VimInputError("Plan mode: :w is not allowed."); - }, - }); - await executeVimSteps(preview, steps, { pauseLastStep: params.pause === true }); - } - - try { - const FRAME_INTERVAL_MS = 16; // ~60fps - let lastUpdateTime = 0; - - const emitUpdate = onUpdate - ? async (force = false) => { - const now = Date.now(); - if (!force && now - lastUpdateTime < FRAME_INTERVAL_MS) { - return; // throttle: skip if too soon - } - onUpdate(this.#renderFromEngine(engine, VIM_DEFAULT_VIEWPORT_LINES, engine.viewportStart)); - lastUpdateTime = Date.now(); - await Bun.sleep(FRAME_INTERVAL_MS); // real delay for terminal to render - } - : undefined; - - await executeVimSteps(engine, steps, { - pauseLastStep: params.pause === true, - onKbdStep: emitUpdate ? () => emitUpdate(true) : undefined, - onInsertStep: emitUpdate ? () => emitUpdate(true) : undefined, - }); - - if (params.pause === true && !engine.closed && engine.getPendingInput()) { - engine.statusMessage = engine.statusMessage ?? `Paused in ${engine.getPublicMode()} mode`; - } - } catch (error) { - this.#throwWithSnapshot(engine, error); - } - - if (beforeText !== serializeBufferText(engine.buffer)) { - engine.centerViewportOnCursor(); - } - - // Auto-save when buffer was modified - if (!engine.closed && engine.buffer.modified && params.pause !== true) { - try { - const result = await this.#saveBuffer(engine.buffer); - engine.buffer.markSaved(result.loaded); - engine.diagnostics = result.diagnostics; - if (beforeText !== serializeBufferText(engine.buffer)) { - engine.centerViewportOnCursor(); - } - } catch (error) { - this.#throwWithSnapshot(engine, error); - } - } - - const afterText = serializeBufferText(engine.buffer); - const modelDiff = buildModelDiff(beforeText, afterText); - - const result = this.#renderFromEngine( - engine, - VIM_DEFAULT_VIEWPORT_LINES, - engine.viewportStart, - engine.closed, - undefined, - undefined, - modelDiff, - ); - if (engine.closed) { - this.#engines.delete(absolutePath); - } - return result; - }); - } -} - -// Unescape JSON string escape sequences from a partial (potentially incomplete) JSON string value. -function unescapePartialJsonString(value: string): string { - let output = ""; - for (let index = 0; index < value.length; index += 1) { - const char = value[index]; - if (char !== "\\") { - output += char; - continue; - } - const next = value[index + 1]; - if (!next) { - output += "\\"; - break; - } - index += 1; - switch (next) { - case '"': - case "\\": - case "/": - output += next; - break; - case "b": - output += "\b"; - break; - case "f": - output += "\f"; - break; - case "n": - output += "\n"; - break; - case "r": - output += "\r"; - break; - case "t": - output += "\t"; - break; - case "u": { - const codePoint = value.slice(index + 1, index + 5); - if (codePoint.length === 4) { - const parsed = parseInt(codePoint, 16); - if (!Number.isNaN(parsed)) { - output += String.fromCharCode(parsed); - index += 4; - continue; - } - } - output += "\\u"; - break; - } - default: - output += `\\${next}`; - } - } - return output; -} - -// Extract partial insert text from raw JSON buffer during streaming. -// partial-json often doesn't surface string values until the closing quote is seen. -function extractPartialInsert(partialJson: string | undefined): string | undefined { - if (!partialJson) { - return undefined; - } - const matches = Array.from(partialJson.matchAll(/"insert"\s*:\s*"((?:\\.|[^"\\])*)(?:"|$)/gu)); - const match = matches[matches.length - 1]; - if (!match) { - return undefined; - } - return unescapePartialJsonString(match[1]!); -} - -function describeStepsForDisplay(args: VimRenderArgs): string { - const steps = getStepsForDisplay(args); - if (!steps || steps.length === 0) { - return ""; - } - - const kbdSummary = steps.map(step => step.kbd.join(" ")).filter(summary => summary.length > 0); - let description = steps.length === 1 ? (kbdSummary[0] ?? "1 step") : `${steps.length} steps`; - if (steps.length > 1 && kbdSummary.length > 0) { - description += ` · ${kbdSummary.join(" → ")}`; - } - - const insertText = getLastStepInsert(steps); - if (insertText !== undefined && insertText.length > 0) { - description += `${description.length > 0 ? " · " : ""}insert: ${insertText}`; - } - if (args.pause) { - description += `${description.length > 0 ? " · " : ""}pause`; - } - return description; -} - -export function resetVimRendererStateForTest(): void { - lastVimDetails = undefined; -} - -export const vimToolRenderer = { - renderCall(args: VimRenderArgs, options: RenderResultOptions, uiTheme: Theme): Component { - if (args.file && (!args.steps || args.steps.length === 0)) { - return renderText(`${uiTheme.bold("Edit")} open ${args.file}`); - } - - // Build a description of the streaming args for the header - const argsDescription = describeStepsForDisplay(args); - - // Reuse the last real vim result for the same file while the next call is still streaming. - const details = lastVimDetails?.file === args.file ? lastVimDetails : undefined; - if (details?.viewportLines && details.viewportLines.length > 0) { - const lang = getLanguageFromPath(details.file); - const langIcon = uiTheme.getLangIcon(lang); - const modified = details.modified ? " [+]" : ""; - const position = `L${details.cursor.line}:${details.cursor.col}`; - const padWidth = String(details.viewport.end).length; - const viewportLines = details.viewportLines; - const highlightedLines = highlightCode(viewportLines.map(line => line.text).join("\n"), lang); - const renderedLines = viewportLines.map((line, index) => - renderViewportLine(line, highlightedLines[index] ?? line.text, padWidth, uiTheme), - ); - if (details.statusMessage) { - renderedLines.push(uiTheme.fg("dim", details.statusMessage)); - } - - const outputBlock = new CachedOutputBlock(); - let cached: { key: string; result: string[] } | undefined; - - return { - render: (width: number): string[] => { - const cacheKey = `${width}|${options.spinnerFrame ?? -1}|${argsDescription}`; - if (cached?.key === cacheKey) { - return cached.result; - } - - const header = renderStatusLine( - { - icon: "pending", - spinnerFrame: options.spinnerFrame, - title: "Edit", - description: argsDescription || details.file + modified, - meta: [`${langIcon} ${details.totalLines} lines`, position], - }, - uiTheme, - ); - - const lines = outputBlock.render( - { - header, - state: "pending", - sections: [{ lines: renderedLines }], - width, - }, - uiTheme, - ); - cached = { key: cacheKey, result: lines }; - return lines; - }, - invalidate: () => { - cached = undefined; - outputBlock.invalidate(); - }, - }; - } - - // Fallback: no previous viewport available (first vim call) - if (argsDescription) { - return renderText(`${uiTheme.bold("Edit")} ${argsDescription}`); - } - return renderText(`${uiTheme.bold("Edit")}`); - }, - renderResult( - result: { content: Array<{ type: string; text?: string }>; details?: VimToolDetails; isError?: boolean }, - options: RenderResultOptions, - uiTheme: Theme, - ): Component { - const details = result.details; - const isError = result.isError === true; - - // No structured details (e.g. closed): fall back to plain text - if (!details?.viewportLines || details.viewportLines.length === 0) { - if (details) { - return renderText(renderVimDetails(details)); - } - const text = result.content.find(block => block.type === "text")?.text ?? ""; - return renderText(text); - } - - const lang = getLanguageFromPath(details.file); - const langIcon = uiTheme.getLangIcon(lang); - const modified = details.modified ? " [+]" : ""; - const position = `L${details.cursor.line}:${details.cursor.col}`; - const padWidth = String(details.viewport.end).length; - const viewportLines = details.viewportLines; - const highlightedLines = highlightCode(viewportLines.map(line => line.text).join("\n"), lang); - const renderedLines = viewportLines.map((line, index) => - renderViewportLine(line, highlightedLines[index] ?? line.text, padWidth, uiTheme), - ); - if (details.statusMessage) { - renderedLines.push(uiTheme.fg("dim", details.statusMessage)); - } - - const sections: Array<{ label?: string; lines: string[] }> = [{ lines: renderedLines }]; - if (details.diagnostics?.messages && details.diagnostics.messages.length > 0) { - const diagText = formatDiagnostics( - { - errored: isError, - summary: details.diagnostics.summary, - messages: details.diagnostics.messages, - }, - options.expanded, - uiTheme, - (filePath: string) => uiTheme.getLangIcon(getLanguageFromPath(filePath)), - ); - if (diagText) { - sections.push({ lines: [diagText] }); - } - } - - const outputBlock = new CachedOutputBlock(); - let cached: { key: string; result: string[] } | undefined; - - return { - render: (width: number): string[] => { - const cacheKey = `${width}|${options.isPartial ? 1 : 0}|${isError ? 1 : 0}|${options.spinnerFrame ?? -1}`; - if (cached?.key === cacheKey) { - return cached.result; - } - - const icon = options.isPartial ? "pending" : isError ? "error" : "success"; - - // Mode badge - const modeBadge = - details.mode === "NORMAL" - ? undefined - : { - label: details.mode, - color: - details.mode === "INSERT" - ? ("success" as const) - : details.mode === "VISUAL" || details.mode === "VISUAL-LINE" - ? ("warning" as const) - : ("accent" as const), - }; - - const header = renderStatusLine( - { - icon, - spinnerFrame: options.spinnerFrame, - title: "Edit", - description: details.file + modified, - badge: modeBadge, - meta: [`${langIcon} ${details.totalLines} lines`, position], - }, - uiTheme, - ); - - const lines = outputBlock.render( - { - header, - state: options.isPartial ? "pending" : isError ? "error" : "success", - sections, - width, - }, - uiTheme, - ); - cached = { key: cacheKey, result: lines }; - return lines; - }, - invalidate: () => { - cached = undefined; - outputBlock.invalidate(); - }, - }; - }, - mergeCallAndResult: true, -}; - -export { vimSchema }; diff --git a/packages/coding-agent/src/tools/write.ts b/packages/coding-agent/src/tools/write.ts index 4e44358a4..f318dd6ed 100644 --- a/packages/coding-agent/src/tools/write.ts +++ b/packages/coding-agent/src/tools/write.ts @@ -74,8 +74,8 @@ export interface WriteToolDetails { /** * Strip hashline display prefixes from write content. * - * Only active when hashline edit mode is enabled — the model sees `LINE+ID|` - * prefixes in read output and sometimes copies them into write content. + * Only active when hashline edit mode is enabled — the model sees `¶PATH#HASH` + * headers plus `LINE:` prefixes in read output and sometimes copies them into write content. */ function stripWriteContent(session: ToolSession, content: string): { text: string; stripped: boolean } { if (!resolveFileDisplayMode(session).hashLines) { @@ -658,7 +658,7 @@ export class WriteTool implements AgentTool> { return untilAborted(signal, async () => { - // Strip hashline display prefixes (LINE+ID|) if the model copied them from read output + // Strip hashline display prefixes (¶PATH#HASH + LINE:) if the model copied them from read output const { text: cleanContent, stripped } = stripWriteContent(this.session, content); const internalRouter = InternalUrlRouter.instance(); if (internalRouter.canHandle(path)) { diff --git a/packages/coding-agent/src/utils/edit-mode.ts b/packages/coding-agent/src/utils/edit-mode.ts index 5eeb5dd98..8c2cdfc85 100644 --- a/packages/coding-agent/src/utils/edit-mode.ts +++ b/packages/coding-agent/src/utils/edit-mode.ts @@ -1,6 +1,6 @@ import { $env } from "@oh-my-pi/pi-utils"; -export type EditMode = "replace" | "patch" | "hashline" | "vim" | "apply_patch"; +export type EditMode = "replace" | "patch" | "hashline" | "apply_patch"; export const DEFAULT_EDIT_MODE: EditMode = "hashline"; @@ -9,7 +9,6 @@ const EDIT_MODE_IDS = { hashline: "hashline", patch: "patch", replace: "replace", - vim: "vim", } as const satisfies Record; export const EDIT_MODES = Object.keys(EDIT_MODE_IDS) as EditMode[]; diff --git a/packages/coding-agent/src/utils/file-mentions.ts b/packages/coding-agent/src/utils/file-mentions.ts index 797d8dd04..b6335db94 100644 --- a/packages/coding-agent/src/utils/file-mentions.ts +++ b/packages/coding-agent/src/utils/file-mentions.ts @@ -12,7 +12,7 @@ import type { ImageContent } from "@oh-my-pi/pi-ai"; import { glob } from "@oh-my-pi/pi-natives"; import { fuzzyMatch } from "@oh-my-pi/pi-tui"; import { formatAge, formatBytes, readImageMetadata } from "@oh-my-pi/pi-utils"; -import { formatHashLines } from "../hashline/hash"; +import { computeFileHash, formatHashlineHeader, formatNumberedLines } from "../hashline/hash"; import type { FileMentionMessage } from "../session/messages"; import { DEFAULT_MAX_BYTES, @@ -356,7 +356,7 @@ export async function generateFileMentionMessages( const content = await Bun.file(absolutePath).text(); let { output, lineCount } = buildTextOutput(content); if (options?.useHashLines) { - output = formatHashLines(output); + output = `${formatHashlineHeader(resolvedPath, computeFileHash(content))}\n${formatNumberedLines(output)}`; } files.push({ path: resolvedPath, content: output, lineCount }); } catch { diff --git a/packages/coding-agent/src/vim/buffer.ts b/packages/coding-agent/src/vim/buffer.ts deleted file mode 100644 index a3f5f233e..000000000 --- a/packages/coding-agent/src/vim/buffer.ts +++ /dev/null @@ -1,309 +0,0 @@ -import { clonePosition, type Position, type VimBufferSnapshot, type VimFingerprint, type VimLoadedFile } from "./types"; - -function splitText(text: string): string[] { - if (text.length === 0) { - return [""]; - } - return text.split("\n"); -} - -export function snapshotEqual(left: VimBufferSnapshot, right: VimBufferSnapshot): boolean { - if ( - left.displayPath !== right.displayPath || - left.filePath !== right.filePath || - left.modified !== right.modified || - left.trailingNewline !== right.trailingNewline || - left.cursor.line !== right.cursor.line || - left.cursor.col !== right.cursor.col || - left.editabilityChecked !== right.editabilityChecked - ) { - return false; - } - - if (left.baseFingerprint === null || right.baseFingerprint === null) { - if (left.baseFingerprint !== right.baseFingerprint) { - return false; - } - } else if ( - left.baseFingerprint.exists !== right.baseFingerprint.exists || - left.baseFingerprint.size !== right.baseFingerprint.size || - left.baseFingerprint.mtimeMs !== right.baseFingerprint.mtimeMs || - left.baseFingerprint.hash !== right.baseFingerprint.hash - ) { - return false; - } - - if (left.lines.length !== right.lines.length) { - return false; - } - - for (let index = 0; index < left.lines.length; index += 1) { - if (left.lines[index] !== right.lines[index]) { - return false; - } - } - - return true; -} - -export class VimBuffer { - displayPath: string; - filePath: string; - lines: string[]; - cursor: Position; - modified: boolean; - trailingNewline: boolean; - baseFingerprint: VimFingerprint | null; - editabilityChecked: boolean; - - constructor(input: VimLoadedFile) { - this.displayPath = input.displayPath; - this.filePath = input.absolutePath; - this.lines = input.lines.length > 0 ? [...input.lines] : [""]; - this.cursor = { line: 0, col: 0 }; - this.modified = false; - this.trailingNewline = input.trailingNewline; - this.baseFingerprint = input.fingerprint ? { ...input.fingerprint } : null; - this.editabilityChecked = false; - } - - clone(): VimBuffer { - const clone = new VimBuffer({ - absolutePath: this.filePath, - displayPath: this.displayPath, - lines: [...this.lines], - trailingNewline: this.trailingNewline, - fingerprint: this.baseFingerprint ? { ...this.baseFingerprint } : null, - }); - clone.cursor = clonePosition(this.cursor); - clone.modified = this.modified; - clone.editabilityChecked = this.editabilityChecked; - return clone; - } - - createSnapshot(): VimBufferSnapshot { - return { - displayPath: this.displayPath, - filePath: this.filePath, - lines: [...this.lines], - cursor: clonePosition(this.cursor), - modified: this.modified, - trailingNewline: this.trailingNewline, - baseFingerprint: this.baseFingerprint ? { ...this.baseFingerprint } : null, - editabilityChecked: this.editabilityChecked, - }; - } - - restore(snapshot: VimBufferSnapshot): void { - this.displayPath = snapshot.displayPath; - this.filePath = snapshot.filePath; - this.lines = snapshot.lines.length > 0 ? [...snapshot.lines] : [""]; - this.cursor = clonePosition(snapshot.cursor); - this.modified = snapshot.modified; - this.trailingNewline = snapshot.trailingNewline; - this.baseFingerprint = snapshot.baseFingerprint ? { ...snapshot.baseFingerprint } : null; - this.editabilityChecked = snapshot.editabilityChecked; - this.clampCursor(); - } - - replaceLoadedFile(input: VimLoadedFile): void { - this.displayPath = input.displayPath; - this.filePath = input.absolutePath; - this.lines = input.lines.length > 0 ? [...input.lines] : [""]; - this.cursor = { line: 0, col: 0 }; - this.modified = false; - this.trailingNewline = input.trailingNewline; - this.baseFingerprint = input.fingerprint ? { ...input.fingerprint } : null; - this.editabilityChecked = false; - } - - markSaved(input: VimLoadedFile): void { - this.lines = input.lines.length > 0 ? [...input.lines] : [""]; - this.modified = false; - this.trailingNewline = input.trailingNewline; - this.baseFingerprint = input.fingerprint ? { ...input.fingerprint } : null; - this.clampCursor(); - } - - lineCount(): number { - return this.lines.length; - } - - lastLineIndex(): number { - return Math.max(0, this.lines.length - 1); - } - - getLine(line: number): string { - return this.lines[this.clampLine(line)] ?? ""; - } - - clampLine(line: number): number { - return Math.min(Math.max(line, 0), this.lastLineIndex()); - } - - clampCol(line: number, col: number): number { - return Math.min(Math.max(col, 0), this.getLine(line).length); - } - - setCursor(position: Position): void { - this.cursor = { - line: this.clampLine(position.line), - col: this.clampCol(position.line, position.col), - }; - } - - clampCursor(): void { - this.setCursor(this.cursor); - } - - firstNonBlank(line: number): number { - const content = this.getLine(line); - const index = content.search(/\S/); - return index === -1 ? 0 : index; - } - - getText(): string { - return this.lines.join("\n"); - } - - setText(text: string, trailingNewline = this.trailingNewline): void { - const normalizedText = trailingNewline && text.endsWith("\n") ? text.slice(0, -1) : text; - this.lines = splitText(normalizedText); - this.trailingNewline = trailingNewline; - this.clampCursor(); - } - - currentOffset(): number { - return this.positionToOffset(this.cursor); - } - - positionToOffset(position: Position): number { - const line = this.clampLine(position.line); - const col = this.clampCol(line, position.col); - let offset = 0; - for (let index = 0; index < line; index += 1) { - offset += this.lines[index]!.length + 1; - } - return offset + col; - } - - offsetToPosition(offset: number): Position { - const text = this.getText(); - const clamped = Math.min(Math.max(offset, 0), text.length); - let remaining = clamped; - for (let line = 0; line < this.lines.length; line += 1) { - const current = this.lines[line]!; - if (remaining <= current.length) { - return { line, col: remaining }; - } - remaining -= current.length; - if (line < this.lines.length - 1) { - if (remaining === 0) { - return { line: line + 1, col: 0 }; - } - remaining -= 1; - } - } - return { line: this.lastLineIndex(), col: this.getLine(this.lastLineIndex()).length }; - } - - setCursorFromOffset(offset: number): void { - this.cursor = this.offsetToPosition(offset); - } - - replaceOffsets(start: number, end: number, replacement: string, cursorOffset = start + replacement.length): void { - const text = this.getText(); - const normalizedStart = Math.min(Math.max(start, 0), text.length); - const normalizedEnd = Math.min(Math.max(end, normalizedStart), text.length); - const nextText = `${text.slice(0, normalizedStart)}${replacement}${text.slice(normalizedEnd)}`; - // getText() omits the trailing-newline marker, so any \n in the - // replacement is content (a line separator), not a file-trailing newline. - // Bypass setText() which would incorrectly strip it. - this.lines = splitText(nextText); - this.clampCursor(); - this.setCursorFromOffset(cursorOffset); - } - - deleteOffsets(start: number, end: number): string { - const text = this.getText(); - const normalizedStart = Math.min(Math.max(start, 0), text.length); - const normalizedEnd = Math.min(Math.max(end, normalizedStart), text.length); - const removed = text.slice(normalizedStart, normalizedEnd); - this.replaceOffsets(normalizedStart, normalizedEnd, "", normalizedStart); - return removed; - } - - deleteLines(startLine: number, endLine: number): string[] { - const start = this.clampLine(Math.min(startLine, endLine)); - const end = this.clampLine(Math.max(startLine, endLine)); - const removed = this.lines.slice(start, end + 1); - this.lines.splice(start, end - start + 1); - if (this.lines.length === 0) { - this.lines = [""]; - } - this.setCursor({ line: Math.min(start, this.lastLineIndex()), col: 0 }); - if (this.lines.length > 1 || removed.length > 1) { - this.trailingNewline = true; - } - return removed; - } - - insertLines(index: number, newLines: string[]): void { - const at = Math.min(Math.max(index, 0), this.lines.length); - const normalized = newLines.length > 0 ? newLines : [""]; - this.lines.splice(at, 0, ...normalized); - this.setCursor({ line: at, col: 0 }); - this.trailingNewline = true; - } - - replaceLine(line: number, content: string): void { - const target = this.clampLine(line); - this.lines[target] = content; - this.setCursor(this.cursor); - } - - joinLines(startLine: number, count: number): void { - const start = this.clampLine(startLine); - const end = this.clampLine(start + Math.max(count, 1)); - if (start >= end) { - return; - } - const joined = this.lines - .slice(start, end + 1) - .map(line => line.trim()) - .join(" "); - this.lines.splice(start, end - start + 1, joined); - this.setCursor({ line: start, col: Math.max(0, joined.length - 1) }); - } - - indentLines(startLine: number, endLine: number, indentUnit: string, direction: 1 | -1): void { - const start = this.clampLine(Math.min(startLine, endLine)); - const end = this.clampLine(Math.max(startLine, endLine)); - for (let line = start; line <= end; line += 1) { - const content = this.lines[line] ?? ""; - if (direction > 0) { - this.lines[line] = `${indentUnit}${content}`; - continue; - } - if (content.startsWith(indentUnit)) { - this.lines[line] = content.slice(indentUnit.length); - continue; - } - const spaces = content.match(/^ +/)?.[0].length ?? 0; - this.lines[line] = content.slice(Math.min(spaces, indentUnit.length)); - } - this.setCursor(this.cursor); - } - - getCharacterAtOffset(offset: number): string { - const text = this.getText(); - if (offset < 0 || offset >= text.length) { - return ""; - } - return text[offset] ?? ""; - } - - getCharacter(position: Position): string { - return this.getCharacterAtOffset(this.positionToOffset(position)); - } -} diff --git a/packages/coding-agent/src/vim/commands.ts b/packages/coding-agent/src/vim/commands.ts deleted file mode 100644 index 892562c7d..000000000 --- a/packages/coding-agent/src/vim/commands.ts +++ /dev/null @@ -1,382 +0,0 @@ -import type { VimExCommand, VimLineRange } from "./types"; -import { VimInputError } from "./types"; - -export interface VimExParseContext { - currentLine: number; - lastLine: number; -} - -interface ParsedLineAddress { - line: number; - nextIndex: number; -} - -function clampLine(line: number, context: VimExParseContext): number { - return Math.min(Math.max(line, 1), Math.max(1, context.lastLine)); -} - -function readDigits(raw: string, start: number): { digits: string; nextIndex: number } { - let index = start; - let digits = ""; - while (index < raw.length) { - const char = raw[index] ?? ""; - if (!/^\d$/.test(char)) { - break; - } - digits += char; - index += 1; - } - return { digits, nextIndex: index }; -} - -function parseLineAddress( - raw: string, - start: number, - context: VimExParseContext, - relativeBase = context.currentLine, -): ParsedLineAddress | undefined { - let index = start; - let line: number | undefined; - const first = raw[index] ?? ""; - - if (/^\d$/.test(first)) { - const { digits, nextIndex } = readDigits(raw, index); - line = Number.parseInt(digits, 10); - index = nextIndex; - } else if (first === ".") { - line = context.currentLine; - index += 1; - } else if (first === "$") { - line = context.lastLine; - index += 1; - } else if (first === "+" || first === "-") { - line = relativeBase; - } else { - return undefined; - } - - while (index < raw.length) { - const sign = raw[index]; - if (sign !== "+" && sign !== "-") { - break; - } - index += 1; - const { digits, nextIndex } = readDigits(raw, index); - index = nextIndex; - const offset = digits.length > 0 ? Number.parseInt(digits, 10) : 1; - line += sign === "+" ? offset : -offset; - } - - return { line: clampLine(line, context), nextIndex: index }; -} - -function parseLineRange(raw: string, context?: VimExParseContext): { range?: VimLineRange | "all"; rest: string } { - if (raw.startsWith("%")) { - return { range: "all", rest: raw.slice(1).trimStart() }; - } - - if (!context) { - const match = raw.match(/^(\d+)(?:\s*,\s*(\d+))?/); - if (!match) { - return { rest: raw }; - } - - const start = Number.parseInt(match[1] ?? "", 10); - const end = Number.parseInt(match[2] ?? match[1] ?? "", 10); - return { - range: { start, end }, - rest: raw.slice(match[0].length).trimStart(), - }; - } - - const first = parseLineAddress(raw, 0, context); - if (!first) { - return { rest: raw }; - } - - let index = first.nextIndex; - while (raw[index] === " ") { - index += 1; - } - - const separator = raw[index]; - if (separator !== "," && separator !== ";") { - return { - range: { start: first.line, end: first.line }, - rest: raw.slice(index).trimStart(), - }; - } - - index += 1; - while (raw[index] === " ") { - index += 1; - } - - const second = parseLineAddress(raw, index, context, separator === ";" ? first.line : context.currentLine); - if (!second) { - throw new VimInputError(`Missing line address after ${separator}`); - } - - return { - range: { start: first.line, end: second.line }, - rest: raw.slice(second.nextIndex).trimStart(), - }; -} - -function parseDelimitedSegments(raw: string): { pattern: string; replacement: string; flags: string } { - if (raw.length === 0) { - throw new VimInputError("Missing substitute delimiter"); - } - - const delimiter = raw[0] ?? "/"; - const segments: string[] = []; - let current = ""; - let escaped = false; - - for (let index = 1; index < raw.length; index += 1) { - const char = raw[index] ?? ""; - if (escaped) { - current += char; - escaped = false; - continue; - } - if (char === "\\") { - escaped = true; - current += char; - continue; - } - if (char === delimiter && segments.length < 2) { - segments.push(current); - current = ""; - continue; - } - current += char; - } - - if (segments.length !== 2) { - throw new VimInputError("Substitute command must look like :s/pattern/replacement/flags"); - } - - return { - pattern: segments[0] ?? "", - replacement: segments[1] ?? "", - flags: current.trim(), - }; -} - -function parseDestination(raw: string, context?: VimExParseContext): number { - const trimmed = raw.trim(); - if (trimmed.length === 0) { - throw new VimInputError("Missing destination"); - } - - if (/^\d+$/.test(trimmed)) { - return Number.parseInt(trimmed, 10); - } - - if (context) { - const address = parseLineAddress(trimmed, 0, context); - if (address && trimmed.slice(address.nextIndex).trim().length === 0) { - return address.line; - } - } - - const destination = Number.parseInt(trimmed, 10); - if (Number.isNaN(destination)) { - throw new VimInputError("Invalid destination"); - } - return destination; -} - -function matchGlobalCommand(rest: string): { pattern: string; command: string; invert: boolean } | undefined { - const globalMatch = rest.match(/^(g|v|g!|global|global!|vglobal)\s*([/|#])(.+?)\2(.*)$/); - if (!globalMatch) { - return undefined; - } - return { - invert: globalMatch[1] === "v" || globalMatch[1] === "vglobal" || globalMatch[1]?.endsWith("!") === true, - pattern: globalMatch[3] ?? "", - command: (globalMatch[4] ?? "d").trim() || "d", - }; -} - -function matchDestinationCommand(rest: string, prefixes: readonly string[]): string | undefined { - for (const prefix of prefixes) { - if (!rest.startsWith(prefix)) { - continue; - } - const suffix = rest.slice(prefix.length); - if (suffix.length === 0) { - return ""; - } - if (/^\s/.test(suffix) || /^[\d.$+-]/.test(suffix)) { - return suffix.trim(); - } - } - return undefined; -} - -export function parseExCommand(input: string, context?: VimExParseContext): VimExCommand { - const trimmed = input.trim(); - const normalized = trimmed.startsWith(":") ? trimmed.slice(1).trimStart() : trimmed; - if (normalized.length === 0) { - throw new VimInputError("Empty ex command"); - } - - if (/^\d+$/.test(normalized)) { - return { - kind: "goto-line", - line: Number.parseInt(normalized, 10), - }; - } - - if (normalized === "w" || normalized === "write") { - return { kind: "write", force: false }; - } - if (normalized === "w!" || normalized === "write!") { - return { kind: "write", force: true }; - } - if (normalized === "update" || normalized === "up") { - return { kind: "update", force: false }; - } - if (normalized === "update!" || normalized === "up!") { - return { kind: "update", force: true }; - } - if (normalized === "wq" || normalized === "x" || normalized === "xit" || normalized === "exit") { - return { kind: "write-quit", force: false }; - } - if (normalized === "wq!" || normalized === "x!" || normalized === "xit!" || normalized === "exit!") { - return { kind: "write-quit", force: true }; - } - if (normalized === "q" || normalized === "quit") { - return { kind: "quit", force: false }; - } - if (normalized === "q!" || normalized === "quit!") { - return { kind: "quit", force: true }; - } - if (normalized === "e" || normalized === "edit") { - return { kind: "edit", force: false }; - } - if (normalized === "e!" || normalized === "edit!") { - return { kind: "edit", force: true }; - } - if (normalized.startsWith("e ") || normalized.startsWith("edit ")) { - const path = normalized.startsWith("edit ") ? normalized.slice(5).trim() : normalized.slice(2).trim(); - return { kind: "edit", force: false, path }; - } - if (normalized.startsWith("e! ") || normalized.startsWith("edit! ")) { - const path = normalized.startsWith("edit! ") ? normalized.slice(6).trim() : normalized.slice(3).trim(); - return { kind: "edit", force: true, path }; - } - - const global = matchGlobalCommand(normalized); - if (global) { - return { kind: "global", ...global }; - } - - const { range, rest } = parseLineRange(normalized, context); - if (range && rest.length === 0) { - if (range === "all") { - throw new VimInputError(":% requires a following command"); - } - return { - kind: "goto-line", - line: range.start, - }; - } - - const rangedGlobal = matchGlobalCommand(rest); - if (rangedGlobal) { - return { kind: "global", range, ...rangedGlobal }; - } - - if (rest === "sort" || rest.startsWith("sort ") || rest.startsWith("sort!")) { - const flags = rest.slice(4).trim(); - return { kind: "sort", range: range ?? undefined, flags }; - } - if (rest === "j" || rest === "join" || rest === "j!" || rest === "join!") { - return { kind: "join", range: range ?? undefined, trimWhitespace: !rest.endsWith("!") }; - } - - if (rest.startsWith("substitute")) { - const segments = parseDelimitedSegments(rest.slice("substitute".length)); - return { - kind: "substitute", - range, - pattern: segments.pattern, - replacement: segments.replacement, - flags: segments.flags, - }; - } - - if (/^s(?:\W|$)/.test(rest)) { - const segments = parseDelimitedSegments(rest.slice(1)); - return { - kind: "substitute", - range, - pattern: segments.pattern, - replacement: segments.replacement, - flags: segments.flags, - }; - } - - if ( - rest === "d" || - rest === "del" || - rest === "delete" || - rest.startsWith("d ") || - rest.startsWith("del ") || - rest.startsWith("delete ") - ) { - return { - kind: "delete", - range, - }; - } - - if ( - rest === "y" || - rest === "ya" || - rest === "yank" || - rest.startsWith("y ") || - rest.startsWith("ya ") || - rest.startsWith("yank ") - ) { - return { - kind: "yank", - range, - }; - } - - if (rest === "pu" || rest === "put" || rest === "pu!" || rest === "put!") { - return { - kind: "put", - range, - before: rest.endsWith("!"), - }; - } - - const copyDestination = matchDestinationCommand(rest, ["copy", "co", "t"]); - if (copyDestination !== undefined) { - const destination = parseDestination(copyDestination, context); - return { kind: "copy", range, destination }; - } - - const moveDestination = matchDestinationCommand(rest, ["move", "mo", "m"]); - if (moveDestination !== undefined) { - const destination = parseDestination(moveDestination, context); - return { kind: "move", range, destination }; - } - - if (rest === "a" || rest === "append" || rest.startsWith("a ") || rest.startsWith("append ")) { - const text = rest.startsWith("append") ? rest.slice(6).trimStart() : rest.slice(1).trimStart(); - return { kind: "append", range: range === "all" ? undefined : range, text }; - } - - if (rest === "i" || rest === "insert" || rest.startsWith("i ") || rest.startsWith("insert ")) { - const text = rest.startsWith("insert") ? rest.slice(6).trimStart() : rest.slice(1).trimStart(); - return { kind: "insert-before", range: range === "all" ? undefined : range, text }; - } - - throw new VimInputError(`Unsupported ex command: ${input}.`); -} diff --git a/packages/coding-agent/src/vim/engine.ts b/packages/coding-agent/src/vim/engine.ts deleted file mode 100644 index 10951542d..000000000 --- a/packages/coding-agent/src/vim/engine.ts +++ /dev/null @@ -1,2409 +0,0 @@ -import type { FileDiagnosticsResult } from "../lsp"; -import { snapshotEqual, type VimBuffer } from "./buffer"; -import { parseExCommand } from "./commands"; -import { replayTokens } from "./parser"; -import type { - Position, - VimBufferSnapshot, - VimInputMode, - VimKeyToken, - VimLineRange, - VimLoadedFile, - VimPendingInput, - VimRegister, - VimSearchState, - VimSelection, - VimUndoEntry, -} from "./types"; -import { clonePosition, maxPosition, minPosition, toPublicMode, VimInputError as VimError } from "./types"; - -export interface VimSaveResult { - loaded: VimLoadedFile; - diagnostics?: FileDiagnosticsResult; -} - -export interface VimEngineCallbacks { - beforeMutate: (buffer: VimBuffer) => Promise; - loadBuffer: (path: string) => Promise; - saveBuffer: (buffer: VimBuffer, options?: { force?: boolean }) => Promise; -} - -interface PendingChange { - before: VimBufferSnapshot; - tokens: string[]; - moveCursorLeftOnEscape: boolean; - inserted: boolean; -} - -interface MotionResult { - nextIndex: number; - target: Position; - inclusive?: boolean; - linewise?: boolean; - range?: { start: number; end: number; linewise?: boolean }; -} - -const WORD_CHAR = /[A-Za-z0-9_]/; -const DEFAULT_VIEWPORT_HEIGHT = 10; -const BRACKET_PAIRS = new Map([ - ["(", ")"], - ["[", "]"], - ["{", "}"], - ["<", ">"], -]); -const CLOSING_BRACKETS = new Map( - Array.from(BRACKET_PAIRS.entries()).map(([open, close]) => [close, open]), -); -const NOOP_Z_COMMANDS = new Set(["a", "A", "c", "C", "m", "M", "o", "O", "r", "R", "v", "x", "X"]); - -function escapeRegex(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -function isWhitespace(char: string): boolean { - return /\s/.test(char); -} - -function isWordChar(char: string): boolean { - return WORD_CHAR.test(char); -} - -function wordCategory(char: string, bigWord: boolean): "space" | "word" | "punct" { - if (char.length === 0 || isWhitespace(char)) { - return "space"; - } - if (bigWord) { - return "word"; - } - return isWordChar(char) ? "word" : "punct"; -} - -function decodeReplacement(replacement: string): string { - return replacement.replace(/\\\//g, "/").replace(/\\\\/g, "\\"); -} - -function literalTextToReplayTokens(text: string): string[] { - const tokens: string[] = []; - for (const char of text) { - if (char === "\n") { - tokens.push("CR"); - continue; - } - if (char === "\t") { - tokens.push("Tab"); - continue; - } - tokens.push(char); - } - return tokens; -} - -// Convert a vim-style search pattern to a JavaScript RegExp. -// In vim's default ("magic") mode, (, ), {, }, |, + are literal unless backslash-escaped. -// In JS regex these are metacharacters. Swap the escaping so bare chars are literal -// and \( etc. become regex groups. -function vimPatternToJsRegex(pattern: string): string { - return pattern.replace(/\\([(){}|+])|([(){}|+])/g, (_match, escaped, bare) => { - if (escaped) return escaped; // \( -> ( (regex group) - return `\\${bare}`; // ( -> \( (literal paren) - }); -} - -function createSearchRegex(pattern: string, flags = "g"): RegExp { - try { - return new RegExp(vimPatternToJsRegex(pattern), flags); - } catch { - return new RegExp(escapeRegex(pattern), flags); - } -} - -function detectIndentUnit(lines: string[]): string { - for (const line of lines) { - if (line.startsWith("\t")) { - return "\t"; - } - if (line.startsWith(" ")) { - return " "; - } - } - return "\t"; -} - -function normalizeRange(start: number, end: number): { start: number; end: number } { - return { - start: Math.min(start, end), - end: Math.max(start, end), - }; -} - -function selectionFromAnchor(buffer: VimBuffer, anchor: Position, linewise: boolean): VimSelection { - if (linewise) { - const startLine = Math.min(anchor.line, buffer.cursor.line); - const endLine = Math.max(anchor.line, buffer.cursor.line); - return { - kind: "line", - start: { line: startLine + 1, col: 1 }, - end: { line: endLine + 1, col: buffer.getLine(endLine).length + 1 }, - }; - } - const start = minPosition(anchor, buffer.cursor); - const end = maxPosition(anchor, buffer.cursor); - return { - kind: "char", - start: { line: start.line + 1, col: start.col + 1 }, - end: { line: end.line + 1, col: end.col + 1 }, - }; -} - -function expandVisualOffsets( - buffer: VimBuffer, - anchor: Position, - linewise: boolean, -): { start: number; end: number; linewise: boolean } { - if (linewise) { - const startLine = Math.min(anchor.line, buffer.cursor.line); - const endLine = Math.max(anchor.line, buffer.cursor.line); - const startOffset = buffer.positionToOffset({ line: startLine, col: 0 }); - const endOffset = - endLine >= buffer.lastLineIndex() - ? buffer.getText().length - : buffer.positionToOffset({ line: endLine + 1, col: 0 }); - return { start: startOffset, end: endOffset, linewise: true }; - } - const anchorOffset = buffer.positionToOffset(anchor); - const cursorOffset = buffer.positionToOffset(buffer.cursor); - const { start, end } = normalizeRange(anchorOffset, cursorOffset); - return { start, end: end + 1, linewise: false }; -} - -function nextWordStart(text: string, offset: number, bigWord: boolean): number { - let index = Math.min(Math.max(offset, 0), text.length); - if (index >= text.length) { - return text.length; - } - - const currentCategory = wordCategory(text[index] ?? "", bigWord); - if (currentCategory === "space") { - while (index < text.length && wordCategory(text[index] ?? "", bigWord) === "space") { - index += 1; - } - return index; - } - - while (index < text.length && wordCategory(text[index] ?? "", bigWord) === currentCategory) { - index += 1; - } - while (index < text.length && wordCategory(text[index] ?? "", bigWord) === "space") { - index += 1; - } - return index; -} - -function previousWordStart(text: string, offset: number, bigWord: boolean): number { - let index = Math.min(Math.max(offset - 1, 0), text.length); - while (index > 0 && wordCategory(text[index] ?? "", bigWord) === "space") { - index -= 1; - } - const category = wordCategory(text[index] ?? "", bigWord); - while (index > 0 && wordCategory(text[index - 1] ?? "", bigWord) === category) { - index -= 1; - } - return index; -} - -function endOfWord(text: string, offset: number, bigWord: boolean): number { - let index = Math.min(Math.max(offset, 0), text.length); - while (index < text.length && wordCategory(text[index] ?? "", bigWord) === "space") { - index += 1; - } - const category = wordCategory(text[index] ?? "", bigWord); - while (index < text.length && wordCategory(text[index] ?? "", bigWord) === category) { - index += 1; - } - return Math.max(0, index - 1); -} - -function endOfPreviousWord(text: string, offset: number, bigWord: boolean): number { - if (text.length === 0) { - return 0; - } - - let index = Math.min(Math.max(offset - 1, 0), text.length - 1); - while (index >= 0 && wordCategory(text[index] ?? "", bigWord) === "space") { - index -= 1; - } - if (index < 0) { - return 0; - } - - const currentCategory = wordCategory(text[index] ?? "", bigWord); - while (index >= 0 && wordCategory(text[index] ?? "", bigWord) === currentCategory) { - index -= 1; - } - while (index >= 0 && wordCategory(text[index] ?? "", bigWord) === "space") { - index -= 1; - } - - return Math.max(0, index); -} - -function toggleCase(text: string): string { - let toggled = ""; - for (const char of text) { - if (char >= "a" && char <= "z") { - toggled += char.toUpperCase(); - continue; - } - if (char >= "A" && char <= "Z") { - toggled += char.toLowerCase(); - continue; - } - toggled += char; - } - return toggled; -} - -function lastNonBlankColumn(line: string): number { - for (let index = line.length - 1; index >= 0; index -= 1) { - if (!isWhitespace(line[index] ?? "")) { - return index; - } - } - return 0; -} - -function findParagraphStart(lines: string[], line: number): number { - let index = Math.max(0, line - 1); - while (index > 0 && lines[index]!.trim().length > 0) { - index -= 1; - } - while (index > 0 && lines[index - 1]!.trim().length === 0) { - index -= 1; - } - return index; -} - -function findParagraphEnd(lines: string[], line: number): number { - let index = Math.min(lines.length - 1, line + 1); - while (index < lines.length - 1 && lines[index]!.trim().length > 0) { - index += 1; - } - while (index < lines.length - 1 && lines[index + 1]!.trim().length === 0) { - index += 1; - } - return index; -} - -function cloneUndoStack(stack: VimUndoEntry[]): VimUndoEntry[] { - return stack.map(entry => ({ - before: { - ...entry.before, - lines: [...entry.before.lines], - cursor: clonePosition(entry.before.cursor), - baseFingerprint: entry.before.baseFingerprint ? { ...entry.before.baseFingerprint } : null, - }, - after: { - ...entry.after, - lines: [...entry.after.lines], - cursor: clonePosition(entry.after.cursor), - baseFingerprint: entry.after.baseFingerprint ? { ...entry.after.baseFingerprint } : null, - }, - })); -} - -export class VimEngine { - buffer: VimBuffer; - inputMode: VimInputMode = "normal"; - selectionAnchor: Position | null = null; - register: VimRegister = { kind: "char", text: "" }; - lastSearch: VimSearchState | null = null; - lastCharFind: { char: string; mode: "f" | "F" | "t" | "T" } | null = null; - lastVisual: { anchor: Position; cursor: Position; mode: VimInputMode } | null = null; - lastCommand?: string; - statusMessage?: string; - diagnostics?: FileDiagnosticsResult; - viewportStart = 1; - closed = false; - - #callbacks: VimEngineCallbacks; - #undoStack: VimUndoEntry[] = []; - #redoStack: VimUndoEntry[] = []; - #pendingInput = ""; - #lastChangeTokens: string[] | null = null; - #pendingChange: PendingChange | null = null; - #stepCallback?: () => Promise; - - constructor(buffer: VimBuffer, callbacks: VimEngineCallbacks) { - this.buffer = buffer; - this.#callbacks = callbacks; - } - - clone(callbacks?: Partial): VimEngine { - const next = new VimEngine(this.buffer.clone(), { - beforeMutate: callbacks?.beforeMutate ?? this.#callbacks.beforeMutate, - loadBuffer: callbacks?.loadBuffer ?? this.#callbacks.loadBuffer, - saveBuffer: callbacks?.saveBuffer ?? this.#callbacks.saveBuffer, - }); - next.inputMode = this.inputMode; - next.selectionAnchor = this.selectionAnchor ? clonePosition(this.selectionAnchor) : null; - next.register = { ...this.register }; - next.lastSearch = this.lastSearch ? { ...this.lastSearch } : null; - next.lastCharFind = this.lastCharFind ? { ...this.lastCharFind } : null; - next.lastVisual = this.lastVisual - ? { - anchor: clonePosition(this.lastVisual.anchor), - cursor: clonePosition(this.lastVisual.cursor), - mode: this.lastVisual.mode, - } - : null; - next.lastCommand = this.lastCommand; - next.statusMessage = this.statusMessage; - next.diagnostics = this.diagnostics; - next.viewportStart = this.viewportStart; - next.closed = this.closed; - next.#pendingInput = this.#pendingInput; - next.#lastChangeTokens = this.#lastChangeTokens ? [...this.#lastChangeTokens] : null; - next.#pendingChange = this.#pendingChange - ? { - before: { - ...this.#pendingChange.before, - lines: [...this.#pendingChange.before.lines], - cursor: clonePosition(this.#pendingChange.before.cursor), - baseFingerprint: this.#pendingChange.before.baseFingerprint - ? { ...this.#pendingChange.before.baseFingerprint } - : null, - }, - tokens: [...this.#pendingChange.tokens], - moveCursorLeftOnEscape: this.#pendingChange.moveCursorLeftOnEscape, - inserted: this.#pendingChange.inserted, - } - : null; - next.#undoStack = cloneUndoStack(this.#undoStack); - next.#redoStack = cloneUndoStack(this.#redoStack); - return next; - } - - getPublicMode() { - return toPublicMode(this.inputMode); - } - - getSelection(): VimSelection | undefined { - if (this.selectionAnchor === null) { - return undefined; - } - return selectionFromAnchor(this.buffer, this.selectionAnchor, this.inputMode === "visual-line"); - } - - getPendingInput(): VimPendingInput | undefined { - switch (this.inputMode) { - case "insert": - return { kind: "insert", text: "" }; - case "command": - case "search-forward": - case "search-backward": - return { kind: this.inputMode, text: this.#pendingInput }; - default: - return undefined; - } - } - - rollbackPendingInsert(): void { - if (this.#pendingChange) { - this.buffer.restore(this.#pendingChange.before); - this.#pendingChange = null; - } - this.inputMode = "normal"; - this.selectionAnchor = null; - this.#pendingInput = ""; - } - - setCursor(line: number, col: number): void { - this.buffer.setCursor({ line, col }); - } - - async executeTokens( - tokens: readonly VimKeyToken[], - lastCommand?: string, - onStep?: () => Promise, - ): Promise { - const previousStepCallback = this.#stepCallback; - this.#stepCallback = onStep ?? previousStepCallback; - this.lastCommand = lastCommand; - this.statusMessage = undefined; - this.diagnostics = undefined; - - try { - for (let index = 0; index < tokens.length; ) { - switch (this.inputMode) { - case "insert": - index = await this.#executeInsert(tokens, index); - break; - case "command": - case "search-forward": - case "search-backward": - index = await this.#executePrompt(tokens, index); - break; - case "visual": - case "visual-line": - index = await this.#executeVisual(tokens, index); - break; - default: - index = await this.#executeNormal(tokens, index); - break; - } - if (this.closed) { - break; - } - this.#ensureCursorVisible(); - await this.#stepCallback?.(); - } - } finally { - this.#stepCallback = previousStepCallback; - } - } - - async close(force: boolean): Promise { - if (this.buffer.modified && !force) { - throw new VimError("Unsaved changes; use force to discard"); - } - this.closed = true; - this.statusMessage = `Closed ${this.buffer.displayPath}`; - } - - #ensureCursorVisible(): void { - const line = this.buffer.cursor.line + 1; - if (line < this.viewportStart) { - this.viewportStart = line; - return; - } - const viewportEnd = this.viewportStart + DEFAULT_VIEWPORT_HEIGHT - 1; - if (line > viewportEnd) { - this.viewportStart = Math.max(1, line - DEFAULT_VIEWPORT_HEIGHT + 1); - } - } - - centerViewportOnCursor(size = DEFAULT_VIEWPORT_HEIGHT): void { - const lineCount = Math.max(this.buffer.lineCount(), 1); - const clampedSize = Math.max(1, Math.min(size, lineCount)); - const maxStart = Math.max(1, lineCount - clampedSize + 1); - this.viewportStart = Math.max(1, Math.min(this.buffer.cursor.line + 1 - Math.floor(clampedSize / 2), maxStart)); - } - - #clearSelection(): void { - if (this.selectionAnchor && (this.inputMode === "visual" || this.inputMode === "visual-line")) { - this.lastVisual = { - anchor: clonePosition(this.selectionAnchor), - cursor: clonePosition(this.buffer.cursor), - mode: this.inputMode, - }; - } - this.selectionAnchor = null; - if (this.inputMode === "visual" || this.inputMode === "visual-line") { - this.inputMode = "normal"; - } - } - - async #ensureEditable(): Promise { - await this.#callbacks.beforeMutate(this.buffer); - this.diagnostics = undefined; - this.statusMessage = undefined; - } - - #pushUndo(entry: VimUndoEntry, changeTokens?: readonly string[]): void { - if (snapshotEqual(entry.before, entry.after)) { - return; - } - this.#undoStack.push(entry); - this.#redoStack = []; - if (changeTokens && changeTokens.length > 0) { - this.#lastChangeTokens = [...changeTokens]; - } - } - - #beginPendingChange(prefixTokens: readonly string[], moveCursorLeftOnEscape: boolean): void { - this.#pendingChange = { - before: this.buffer.createSnapshot(), - tokens: [...prefixTokens], - moveCursorLeftOnEscape, - inserted: false, - }; - } - - #markPendingInserted(): void { - if (this.#pendingChange) { - this.#pendingChange.inserted = true; - } - } - - #commitPendingChange(): void { - if (!this.#pendingChange) { - return; - } - const entry: VimUndoEntry = { - before: this.#pendingChange.before, - after: this.buffer.createSnapshot(), - }; - this.#pushUndo(entry, this.#pendingChange.tokens); - this.#pendingChange = null; - } - - async #applyAtomicChange(tokens: readonly string[], mutator: () => void): Promise { - await this.#ensureEditable(); - const before = this.buffer.createSnapshot(); - mutator(); - this.buffer.modified = true; - this.#pushUndo({ before, after: this.buffer.createSnapshot() }, tokens); - } - - async #startInsertChange( - tokens: readonly string[], - mutator?: () => void, - moveCursorLeftOnEscape = true, - ): Promise { - await this.#ensureEditable(); - this.#beginPendingChange(tokens, moveCursorLeftOnEscape); - mutator?.(); - this.buffer.modified = true; - this.inputMode = "insert"; - } - - async #executePrompt(tokens: readonly VimKeyToken[], index: number): Promise { - const token = tokens[index]!; - if (token.value === "Esc") { - this.#pendingInput = ""; - this.inputMode = "normal"; - return index + 1; - } - if (token.value === "BS") { - this.#pendingInput = this.#pendingInput.slice(0, -1); - return index + 1; - } - if (token.value !== "CR") { - this.#pendingInput += token.value === "Tab" ? "\t" : token.value; - return index + 1; - } - - const input = this.#pendingInput; - this.#pendingInput = ""; - const mode = this.inputMode; - this.inputMode = "normal"; - if (mode === "command") { - await this.#executeEx(input); - } else { - await this.#runSearch(input, mode === "search-forward" ? 1 : -1, true); - } - return index + 1; - } - - #exitInsertMode(): void { - if (this.#pendingChange) { - this.#pendingChange.tokens.push("Esc"); - if (this.#pendingChange.moveCursorLeftOnEscape && this.#pendingChange.inserted && this.buffer.cursor.col > 0) { - this.buffer.setCursor({ line: this.buffer.cursor.line, col: this.buffer.cursor.col - 1 }); - } - } - this.inputMode = "normal"; - this.#commitPendingChange(); - } - - async applyLiteralInsert(text: string, exitInsertMode: boolean): Promise { - if (this.inputMode !== "insert" || !this.#pendingChange) { - throw new VimError("Insert payload requires INSERT mode."); - } - - if (text.length > 0) { - const offset = this.buffer.currentOffset(); - this.buffer.replaceOffsets(offset, offset, text, offset + text.length); - this.buffer.modified = true; - if (text.includes("\n")) { - this.buffer.trailingNewline = this.buffer.trailingNewline || text.endsWith("\n"); - } - this.#pendingChange.tokens.push(...literalTextToReplayTokens(text)); - this.#markPendingInserted(); - } - - if (exitInsertMode) { - this.#exitInsertMode(); - } - } - - async #executeInsert(tokens: readonly VimKeyToken[], index: number): Promise { - const token = tokens[index]!; - if (token.value === "Esc") { - this.#exitInsertMode(); - return index + 1; - } - if (token.value === "CR") { - const offset = this.buffer.currentOffset(); - this.buffer.replaceOffsets(offset, offset, "\n", offset + 1); - this.buffer.modified = true; - this.buffer.trailingNewline = true; - this.#pendingChange?.tokens.push(token.value); - this.#markPendingInserted(); - return index + 1; - } - if (token.value === "BS") { - const offset = this.buffer.currentOffset(); - if (offset > 0) { - this.buffer.deleteOffsets(offset - 1, offset); - this.buffer.modified = true; - this.#pendingChange?.tokens.push(token.value); - this.#markPendingInserted(); - } - return index + 1; - } - if (token.value === "Tab") { - const offset = this.buffer.currentOffset(); - this.buffer.replaceOffsets(offset, offset, "\t", offset + 1); - this.buffer.modified = true; - this.#pendingChange?.tokens.push(token.value); - this.#markPendingInserted(); - return index + 1; - } - if (token.value === "C-w") { - const offset = this.buffer.currentOffset(); - const text = this.buffer.getText(); - let start = previousWordStart(text, offset, false); - if (start === offset && start > 0) { - start -= 1; - } - this.buffer.deleteOffsets(start, offset); - this.buffer.modified = true; - this.#pendingChange?.tokens.push(token.value); - this.#markPendingInserted(); - return index + 1; - } - if (token.value === "C-u") { - const offset = this.buffer.currentOffset(); - const lineStart = this.buffer.positionToOffset({ line: this.buffer.cursor.line, col: 0 }); - if (offset > lineStart) { - this.buffer.deleteOffsets(lineStart, offset); - this.buffer.modified = true; - this.#pendingChange?.tokens.push(token.value); - this.#markPendingInserted(); - } - return index + 1; - } - if (token.value === "C-o") { - // Execute one normal-mode command, then return to insert - const nextToken = tokens[index + 1]; - if (!nextToken) { - return index + 1; - } - const savedMode = this.inputMode; - this.inputMode = "normal"; - const nextIdx = await this.#executeNormal(tokens, index + 1); - this.inputMode = savedMode; - return nextIdx; - } - - const insertText = token.value; - const offset = this.buffer.currentOffset(); - this.buffer.replaceOffsets(offset, offset, insertText, offset + insertText.length); - this.buffer.modified = true; - this.#pendingChange?.tokens.push(token.value); - this.#markPendingInserted(); - return index + 1; - } - - async #executeVisual(tokens: readonly VimKeyToken[], index: number): Promise { - const token = tokens[index]!; - if (token.value === "Esc") { - this.#clearSelection(); - return index + 1; - } - if (token.value === "v") { - if (this.inputMode === "visual") { - this.#clearSelection(); - } - return index + 1; - } - if (token.value === "V") { - this.inputMode = this.inputMode === "visual-line" ? "visual" : "visual-line"; - return index + 1; - } - if (token.value === "o") { - if (this.selectionAnchor) { - const tmp = clonePosition(this.buffer.cursor); - this.buffer.setCursor(this.selectionAnchor); - this.selectionAnchor = tmp; - } - return index + 1; - } - if (token.value === "J") { - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - const startLine = this.buffer.offsetToPosition(visual.start).line; - const endLine = this.buffer.offsetToPosition(Math.max(visual.start, visual.end - 1)).line; - await this.#applyAtomicChange(["J"], () => { - this.buffer.joinLines(startLine, endLine - startLine); - }); - this.#clearSelection(); - return index + 1; - } - if (token.value === "u" || token.value === "U") { - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - await this.#applyAtomicChange([token.value], () => { - const original = this.buffer.getText().slice(visual.start, visual.end); - const transformed = token.value === "U" ? original.toUpperCase() : original.toLowerCase(); - this.buffer.replaceOffsets(visual.start, visual.end, transformed, visual.start); - }); - this.#clearSelection(); - return index + 1; - } - if (token.value === "p" || token.value === "P") { - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - await this.#applyAtomicChange([token.value], () => { - const removed = this.buffer.getText().slice(visual.start, visual.end); - const pasteText = this.register.text; - this.buffer.replaceOffsets(visual.start, visual.end, pasteText, visual.start + pasteText.length); - this.register = { kind: visual.linewise ? "line" : "char", text: removed }; - }); - this.#clearSelection(); - return index + 1; - } - if (token.value === "g") { - const next = tokens[index + 1]; - if (!next) { - throw new VimError("g requires a second key", token); - } - if (next.value === "J") { - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - const startLine = this.buffer.offsetToPosition(visual.start).line; - const endLine = this.buffer.offsetToPosition(Math.max(visual.start, visual.end - 1)).line; - await this.#applyAtomicChange(["g", "J"], () => { - const start = this.buffer.clampLine(startLine); - const end = this.buffer.clampLine(endLine); - if (start < end) { - const joined = this.buffer.lines.slice(start, end + 1).join(""); - this.buffer.lines.splice(start, end - start + 1, joined); - this.buffer.setCursor({ line: start, col: Math.max(0, joined.length - 1) }); - } - }); - this.#clearSelection(); - return index + 2; - } - if (next.value === "u" || next.value === "U" || next.value === "~") { - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - await this.#applyAtomicChange(["g", next.value], () => { - const original = this.buffer.getText().slice(visual.start, visual.end); - const transformed = - next.value === "u" - ? original.toLowerCase() - : next.value === "U" - ? original.toUpperCase() - : toggleCase(original); - this.buffer.replaceOffsets(visual.start, visual.end, transformed, visual.start); - }); - this.#clearSelection(); - return index + 2; - } - throw new VimError(`Unsupported g command: g${next.display}`, next); - } - - const { count, hasCount, nextIndex } = this.#readCount(tokens, index); - const opToken = tokens[nextIndex]; - if (!opToken) { - return nextIndex; - } - - switch (opToken.value) { - case "d": - case "x": - case "X": - case "D": - case "y": - case "c": - case "s": - case "S": - case "C": - case ">": - case "<": - case "~": { - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - const consumeExtraIndent = - (opToken.value === ">" || opToken.value === "<") && tokens[nextIndex + 1]?.value === opToken.value; - const operatorValue = - opToken.value === "x" || opToken.value === "X" || opToken.value === "D" - ? "d" - : opToken.value === "s" || opToken.value === "S" || opToken.value === "C" - ? "c" - : opToken.value; - const visualTokens = consumeExtraIndent ? [opToken.value, opToken.value] : [opToken.value]; - await this.#applyVisualOperator(operatorValue, visual, count, visualTokens); - return nextIndex + visualTokens.length; - } - case "r": { - const replacement = tokens[nextIndex + 1]; - if (!replacement || replacement.value.length !== 1) { - throw new VimError("Visual replace requires a literal character", opToken); - } - const visual = expandVisualOffsets( - this.buffer, - this.selectionAnchor ?? this.buffer.cursor, - this.inputMode === "visual-line", - ); - await this.#applyAtomicChange(["r", replacement.value], () => { - const original = this.buffer.getText().slice(visual.start, visual.end); - let replaced = ""; - for (const char of original) { - replaced += char === "\n" ? "\n" : replacement.value; - } - this.buffer.replaceOffsets(visual.start, visual.end, replaced, visual.start); - }); - this.#clearSelection(); - return nextIndex + 2; - } - default: - break; - } - - const motion = this.#resolveMotion(tokens, nextIndex, count, hasCount); - this.buffer.setCursor(motion.target); - return motion.nextIndex; - } - - async #applyVisualOperator( - operator: string, - visual: { start: number; end: number; linewise: boolean }, - count: number, - tokens: readonly string[], - ): Promise { - switch (operator) { - case "y": { - this.register = { - kind: visual.linewise ? "line" : "char", - text: this.buffer.getText().slice(visual.start, visual.end), - }; - this.#clearSelection(); - this.statusMessage = `Yanked ${count} selection${count === 1 ? "" : "s"}`; - return; - } - case "d": { - await this.#applyAtomicChange(tokens, () => { - this.#yankAndDeleteRange(visual); - }); - this.#clearSelection(); - return; - } - case "c": { - await this.#startInsertChange(tokens, () => { - this.#yankAndDeleteRange(visual); - }); - this.#clearSelection(); - return; - } - case ">": - case "<": { - const startLine = this.buffer.offsetToPosition(visual.start).line; - const endLine = this.buffer.offsetToPosition(Math.max(visual.start, visual.end - 1)).line; - await this.#applyAtomicChange(tokens, () => { - this.buffer.indentLines( - startLine, - endLine, - detectIndentUnit(this.buffer.lines), - operator === ">" ? 1 : -1, - ); - }); - this.#clearSelection(); - return; - } - case "~": { - await this.#applyAtomicChange(tokens, () => { - const original = this.buffer.getText().slice(visual.start, visual.end); - this.buffer.replaceOffsets(visual.start, visual.end, toggleCase(original), visual.start); - }); - this.#clearSelection(); - return; - } - default: - throw new VimError(`Unsupported visual operator: ${operator}`); - } - } - - async #executeNormal(tokens: readonly VimKeyToken[], index: number): Promise { - const { count, hasCount, nextIndex } = this.#readCount(tokens, index); - const token = tokens[nextIndex]; - if (!token) { - return nextIndex; - } - - switch (token.value) { - case "h": - this.buffer.setCursor({ line: this.buffer.cursor.line, col: this.buffer.cursor.col - count }); - return nextIndex + 1; - case "j": - this.buffer.setCursor({ line: this.buffer.cursor.line + count, col: this.buffer.cursor.col }); - return nextIndex + 1; - case "k": - this.buffer.setCursor({ line: this.buffer.cursor.line - count, col: this.buffer.cursor.col }); - return nextIndex + 1; - case "l": - case " ": - this.buffer.setCursor({ line: this.buffer.cursor.line, col: this.buffer.cursor.col + count }); - return nextIndex + 1; - case "w": - case "W": - case "b": - case "B": - case "e": - case "E": - case "0": - case "$": - case "^": - case "|": - case ";": - case ",": - case "G": - case "f": - case "F": - case "t": - case "T": - case "{": - case "}": - case "%": - case "H": - case "M": - case "+": - case "-": - case "_": - case "L": { - const motion = this.#resolveMotion(tokens, nextIndex, count, hasCount); - this.buffer.setCursor(motion.target); - return motion.nextIndex; - } - case "*": - case "#": { - const text = this.buffer.getText(); - const offset = this.buffer.currentOffset(); - const cat = wordCategory(text[offset] ?? "", false); - if (cat === "space") { - throw new VimError("No word under cursor", token); - } - let start = offset; - while (start > 0 && wordCategory(text[start - 1] ?? "", false) === cat) start -= 1; - let end = offset; - while (end < text.length && wordCategory(text[end] ?? "", false) === cat) end += 1; - const word = text.slice(start, end); - const pattern = `\\b${escapeRegex(word)}\\b`; - const direction = token.value === "*" ? 1 : -1; - for (let step = 0; step < count; step += 1) { - await this.#runSearch(pattern, direction, true); - } - return nextIndex + 1; - } - case "n": - await this.#repeatSearch(this.lastSearch?.direction ?? 1, count); - return nextIndex + 1; - case "N": - await this.#repeatSearch(((this.lastSearch?.direction ?? 1) * -1) as 1 | -1, count); - return nextIndex + 1; - case "/": - this.inputMode = "search-forward"; - this.#pendingInput = ""; - return nextIndex + 1; - case "?": - this.inputMode = "search-backward"; - this.#pendingInput = ""; - return nextIndex + 1; - case ":": - this.inputMode = "command"; - this.#pendingInput = ""; - return nextIndex + 1; - case "v": - this.inputMode = "visual"; - this.selectionAnchor = clonePosition(this.buffer.cursor); - return nextIndex + 1; - case "V": - this.inputMode = "visual-line"; - this.selectionAnchor = clonePosition(this.buffer.cursor); - return nextIndex + 1; - case "i": - // When count > 1 (e.g. `2i`), interpret as `2Gi` — go to line N then insert. - // Models confuse `Ni` with `NGi`; bare `i` with a high count is almost never intended. - if (hasCount) { - this.buffer.setCursor({ line: Math.min(count, this.buffer.lineCount()) - 1, col: 0 }); - } - await this.#startInsertChange(["i"]); - return nextIndex + 1; - case "a": - this.buffer.setCursor({ line: this.buffer.cursor.line, col: this.buffer.cursor.col + 1 }); - await this.#startInsertChange(["a"]); - return nextIndex + 1; - case "I": - this.buffer.setCursor({ - line: this.buffer.cursor.line, - col: this.buffer.firstNonBlank(this.buffer.cursor.line), - }); - await this.#startInsertChange(["I"]); - return nextIndex + 1; - case "A": - this.buffer.setCursor({ - line: this.buffer.cursor.line, - col: this.buffer.getLine(this.buffer.cursor.line).length, - }); - await this.#startInsertChange(["A"]); - return nextIndex + 1; - case "o": - // When count > 1 (e.g. `13o`), interpret as `13Go` — go to line N then open below. - // Models confuse `No` with `NGo`; bare `o` with a high count is almost never intended. - if (hasCount) { - this.buffer.setCursor({ line: Math.min(count, this.buffer.lineCount()) - 1, col: 0 }); - } - await this.#startInsertChange(["o"], () => { - const line = this.buffer.cursor.line + 1; - this.buffer.insertLines(line, [""]); - }); - return nextIndex + 1; - case "O": - if (hasCount) { - this.buffer.setCursor({ line: Math.min(count, this.buffer.lineCount()) - 1, col: 0 }); - } - await this.#startInsertChange(["O"], () => { - const line = this.buffer.cursor.line; - this.buffer.insertLines(line, [""]); - }); - return nextIndex + 1; - case "s": - await this.#startInsertChange(["s"], () => { - this.#deleteCharsForward(count); - }); - return nextIndex + 1; - case "S": - await this.#changeWholeLines(count, ["S"]); - return nextIndex + 1; - case "x": - await this.#applyAtomicChange(["x"], () => { - this.#deleteCharsForward(count); - }); - return nextIndex + 1; - case "X": - await this.#applyAtomicChange(["X"], () => { - const end = this.buffer.currentOffset(); - const start = Math.max(0, end - count); - this.register = { kind: "char", text: this.buffer.deleteOffsets(start, end) }; - }); - return nextIndex + 1; - case "r": { - const replacement = tokens[nextIndex + 1]; - if (!replacement || replacement.value.length !== 1) { - throw new VimError("r requires a replacement character", token); - } - await this.#applyAtomicChange(["r", replacement.value], () => { - const start = this.buffer.currentOffset(); - this.buffer.replaceOffsets( - start, - Math.min(this.buffer.getText().length, start + count), - replacement.value.repeat(count), - start, - ); - }); - return nextIndex + 2; - } - case "~": - await this.#applyAtomicChange(["~"], () => { - const start = this.buffer.currentOffset(); - const end = Math.min(this.buffer.getText().length, start + count); - const text = this.buffer.getText().slice(start, end); - this.buffer.replaceOffsets(start, end, toggleCase(text), end); - }); - return nextIndex + 1; - case "J": - await this.#applyAtomicChange(["J"], () => { - this.buffer.joinLines(this.buffer.cursor.line, count); - }); - return nextIndex + 1; - case "p": - case "P": - await this.#applyAtomicChange([token.value], () => { - this.#paste(token.value === "p", count); - }); - return nextIndex + 1; - case "u": - await this.#undo(count); - return nextIndex + 1; - case "C-r": - await this.#redo(count); - return nextIndex + 1; - case ".": - await this.#repeatLastChange(count, token); - return nextIndex + 1; - case "d": - case "c": - case "y": - case ">": - case "<": - return this.#executeOperator(tokens, nextIndex, count, hasCount, token.value); - case "D": - await this.#applyAtomicChange(["D"], () => { - const start = this.buffer.currentOffset(); - const line = this.buffer.getLine(this.buffer.cursor.line); - const end = start + (line.length - this.buffer.cursor.col); - this.register = { kind: "char", text: this.buffer.deleteOffsets(start, end) }; - }); - return nextIndex + 1; - case "C": - await this.#startInsertChange(["C"], () => { - const start = this.buffer.currentOffset(); - const line = this.buffer.getLine(this.buffer.cursor.line); - const end = start + (line.length - this.buffer.cursor.col); - this.register = { kind: "char", text: this.buffer.deleteOffsets(start, end) }; - }); - return nextIndex + 1; - case "z": { - const zTarget = tokens[nextIndex + 1]; - if (!zTarget) { - throw new VimError("z requires a second key", token); - } - if (zTarget.value === "z") { - this.centerViewportOnCursor(); - } else if (zTarget.value === "t" || zTarget.value === "CR") { - this.viewportStart = this.buffer.cursor.line + 1; - this.buffer.setCursor({ - line: this.buffer.cursor.line, - col: this.buffer.firstNonBlank(this.buffer.cursor.line), - }); - } else if (zTarget.value === "b" || zTarget.value === "-") { - this.viewportStart = Math.max(1, this.buffer.cursor.line + 1 - (DEFAULT_VIEWPORT_HEIGHT - 1)); - this.buffer.setCursor({ - line: this.buffer.cursor.line, - col: this.buffer.firstNonBlank(this.buffer.cursor.line), - }); - } else if (zTarget.value === ".") { - this.centerViewportOnCursor(); - this.buffer.setCursor({ - line: this.buffer.cursor.line, - col: this.buffer.firstNonBlank(this.buffer.cursor.line), - }); - } else if (NOOP_Z_COMMANDS.has(zTarget.value)) { - this.statusMessage = `Ignored z${zTarget.display} (folds unsupported)`; - } else { - throw new VimError(`Unsupported z command: z${zTarget.display}`, zTarget); - } - return nextIndex + 2; - } - case "C-f": - this.buffer.setCursor({ - line: this.buffer.cursor.line + Math.max(1, (DEFAULT_VIEWPORT_HEIGHT - 2) * count), - col: this.buffer.cursor.col, - }); - return nextIndex + 1; - case "C-b": - this.buffer.setCursor({ - line: this.buffer.cursor.line - Math.max(1, (DEFAULT_VIEWPORT_HEIGHT - 2) * count), - col: this.buffer.cursor.col, - }); - return nextIndex + 1; - case "C-d": - this.buffer.setCursor({ - line: this.buffer.cursor.line + Math.max(1, Math.floor(DEFAULT_VIEWPORT_HEIGHT / 2) * count), - col: this.buffer.cursor.col, - }); - return nextIndex + 1; - case "C-u": - this.buffer.setCursor({ - line: this.buffer.cursor.line - Math.max(1, Math.floor(DEFAULT_VIEWPORT_HEIGHT / 2) * count), - col: this.buffer.cursor.col, - }); - return nextIndex + 1; - case "Esc": - return nextIndex + 1; - case "Y": { - const start = this.buffer.cursor.line; - const end = this.buffer.clampLine(start + count - 1); - this.register = { kind: "line", text: this.buffer.lines.slice(start, end + 1).join("\n") }; - this.statusMessage = `Yanked ${end - start + 1} line${end === start ? "" : "s"}`; - return nextIndex + 1; - } - case "R": - await this.#startInsertChange(["R"], undefined, false); - return nextIndex + 1; - case "g": { - const gNext = tokens[nextIndex + 1]; - if (!gNext) { - throw new VimError("g requires a second key", token); - } - if (gNext.value === "g") { - this.buffer.setCursor({ line: hasCount ? Math.max(0, count - 1) : 0, col: 0 }); - return nextIndex + 2; - } - if (gNext.value === "v") { - if (this.lastVisual) { - this.selectionAnchor = clonePosition(this.lastVisual.anchor); - this.buffer.setCursor(this.lastVisual.cursor); - this.inputMode = this.lastVisual.mode; - } - return nextIndex + 2; - } - if (gNext.value === "*" || gNext.value === "#") { - const text = this.buffer.getText(); - const offset = this.buffer.currentOffset(); - const cat = wordCategory(text[offset] ?? "", false); - if (cat === "space") { - throw new VimError("No word under cursor", gNext); - } - let start = offset; - while (start > 0 && wordCategory(text[start - 1] ?? "", false) === cat) start -= 1; - let end = offset; - while (end < text.length && wordCategory(text[end] ?? "", false) === cat) end += 1; - const word = text.slice(start, end); - const direction = gNext.value === "*" ? 1 : -1; - for (let step = 0; step < count; step += 1) { - await this.#runSearch(escapeRegex(word), direction, true); - } - return nextIndex + 2; - } - if (gNext.value === "U" || gNext.value === "u") { - const caseOp = gNext.value; - const { - count: motionCount, - hasCount: hasMotionCount, - nextIndex: motionStart, - } = this.#readCount(tokens, nextIndex + 2); - const motionToken = tokens[motionStart]; - if (!motionToken) { - throw new VimError(`g${caseOp} requires a motion`, gNext); - } - if ((motionToken.value === "U" && caseOp === "U") || (motionToken.value === "u" && caseOp === "u")) { - const effectiveCount = hasMotionCount ? count * motionCount : count; - await this.#applyAtomicChange(["g", caseOp, motionToken.value], () => { - const start = this.buffer.cursor.line; - const end = this.buffer.clampLine(start + effectiveCount - 1); - for (let line = start; line <= end; line++) { - const content = this.buffer.getLine(line); - this.buffer.replaceLine(line, caseOp === "U" ? content.toUpperCase() : content.toLowerCase()); - } - }); - return motionStart + 1; - } - const effectiveCount = hasMotionCount ? count * motionCount : count; - const motion = this.#resolveMotion(tokens, motionStart, effectiveCount, hasCount || hasMotionCount); - const range = this.#resolveMotionRange(motion); - await this.#applyAtomicChange( - tokens.slice(nextIndex, motion.nextIndex).map(tokenEntry => tokenEntry.value), - () => { - const text = this.buffer.getText(); - const slice = text.slice(range.start, range.end); - const transformed = caseOp === "U" ? slice.toUpperCase() : slice.toLowerCase(); - this.buffer.replaceOffsets(range.start, range.end, transformed, range.start); - }, - ); - return motion.nextIndex; - } - if (gNext.value === "~") { - const { - count: motionCount, - hasCount: hasMotionCount, - nextIndex: motionStart, - } = this.#readCount(tokens, nextIndex + 2); - const motionToken = tokens[motionStart]; - if (!motionToken) { - throw new VimError("g~ requires a motion", gNext); - } - if (motionToken.value === "~") { - const effectiveCount = hasMotionCount ? count * motionCount : count; - await this.#applyAtomicChange(["g", "~", motionToken.value], () => { - const start = this.buffer.cursor.line; - const end = this.buffer.clampLine(start + effectiveCount - 1); - for (let line = start; line <= end; line += 1) { - this.buffer.replaceLine(line, toggleCase(this.buffer.getLine(line))); - } - }); - return motionStart + 1; - } - const effectiveCount = hasMotionCount ? count * motionCount : count; - const motion = this.#resolveMotion(tokens, motionStart, effectiveCount, hasCount || hasMotionCount); - const range = this.#resolveMotionRange(motion); - await this.#applyAtomicChange( - tokens.slice(nextIndex, motion.nextIndex).map(tokenEntry => tokenEntry.value), - () => { - const text = this.buffer.getText(); - const slice = text.slice(range.start, range.end); - this.buffer.replaceOffsets(range.start, range.end, toggleCase(slice), range.start); - }, - ); - return motion.nextIndex; - } - if (gNext.value === "J") { - await this.#applyAtomicChange(["g", "J"], () => { - const start = this.buffer.clampLine(this.buffer.cursor.line); - const end = this.buffer.clampLine(start + Math.max(count, 1)); - if (start < end) { - const joined = this.buffer.lines.slice(start, end + 1).join(""); - this.buffer.lines.splice(start, end - start + 1, joined); - this.buffer.setCursor({ line: start, col: Math.max(0, joined.length - 1) }); - } - }); - return nextIndex + 2; - } - throw new VimError(`Unsupported g command: g${gNext.display}`, gNext); - } - case "Z": { - const zNext = tokens[nextIndex + 1]; - if (!zNext) { - throw new VimError("Z requires a second key", token); - } - if (zNext.value === "Z") { - await this.#executeEx("wq"); - return nextIndex + 2; - } - if (zNext.value === "Q") { - await this.#executeEx("q!"); - return nextIndex + 2; - } - throw new VimError(`Unsupported Z command: Z${zNext.display}`, zNext); - } - default: - throw new VimError(`Unsupported command: ${token.display}`, token); - } - } - - async #repeatLastChange(count: number, token: VimKeyToken): Promise { - if (!this.#lastChangeTokens || this.#lastChangeTokens.length === 0) { - throw new VimError("No previous change to repeat", token); - } - for (let index = 0; index < count; index += 1) { - await this.executeTokens(replayTokens(this.#lastChangeTokens), "."); - } - } - - async #undo(count: number): Promise { - await this.#ensureEditable(); - let applied = 0; - for (let index = 0; index < count; index += 1) { - const entry = this.#undoStack.pop(); - if (!entry) { - break; - } - this.#redoStack.push(entry); - this.buffer.restore(entry.before); - applied += 1; - } - this.inputMode = "normal"; - this.selectionAnchor = null; - this.#pendingChange = null; - this.statusMessage = `Undid ${applied} change${applied === 1 ? "" : "s"}`; - } - - async #redo(count: number): Promise { - await this.#ensureEditable(); - let applied = 0; - for (let index = 0; index < count; index += 1) { - const entry = this.#redoStack.pop(); - if (!entry) { - break; - } - this.#undoStack.push(entry); - this.buffer.restore(entry.after); - applied += 1; - } - this.inputMode = "normal"; - this.selectionAnchor = null; - this.#pendingChange = null; - this.statusMessage = `Redid ${applied} change${applied === 1 ? "" : "s"}`; - } - - async #executeOperator( - tokens: readonly VimKeyToken[], - operatorIndex: number, - operatorCount: number, - hasOperatorCount: boolean, - operator: string, - ): Promise { - const { count: motionCount, hasCount: hasMotionCount, nextIndex } = this.#readCount(tokens, operatorIndex + 1); - const token = tokens[nextIndex]; - if (!token) { - throw new VimError(`Operator ${operator} requires a motion`, tokens[operatorIndex]); - } - const hasAnyCount = hasOperatorCount || hasMotionCount; - const effectiveCount = hasMotionCount ? operatorCount * motionCount : operatorCount; - - if (token.value === operator) { - if (operator === "d") { - await this.#applyAtomicChange([operator, operator], () => { - const start = this.buffer.cursor.line; - const removed = this.buffer.deleteLines(start, start + Math.max(1, effectiveCount) - 1); - this.register = { kind: "line", text: removed.join("\n") }; - }); - return nextIndex + 1; - } - if (operator === "y") { - const start = this.buffer.cursor.line; - const end = this.buffer.clampLine(start + Math.max(1, effectiveCount) - 1); - this.register = { kind: "line", text: this.buffer.lines.slice(start, end + 1).join("\n") }; - this.statusMessage = `Yanked ${end - start + 1} line${end === start ? "" : "s"}`; - return nextIndex + 1; - } - if (operator === "c") { - await this.#changeWholeLines(Math.max(1, effectiveCount), [operator, operator]); - return nextIndex + 1; - } - if (operator === ">" || operator === "<") { - await this.#applyAtomicChange([operator, operator], () => { - this.buffer.indentLines( - this.buffer.cursor.line, - this.buffer.cursor.line + Math.max(1, effectiveCount) - 1, - detectIndentUnit(this.buffer.lines), - operator === ">" ? 1 : -1, - ); - }); - return nextIndex + 1; - } - } - - if (token.value === "i" || token.value === "a") { - const object = tokens[nextIndex + 1]; - if (!object) { - throw new VimError(`Missing text object after ${operator}${token.value}`, token); - } - const textObject = this.#resolveTextObject(token.value === "i", object.value, object); - await this.#applyOperatorToMotion( - operator, - { nextIndex: nextIndex + 2, target: this.buffer.cursor, range: textObject }, - [operator, token.value, object.value], - ); - return nextIndex + 2; - } - - // In vim, `cw` and `cW` act like `ce` and `cE` (don't include trailing whitespace) - const motionToken = tokens[nextIndex]; - let motion: MotionResult; - if (operator === "c" && motionToken && (motionToken.value === "w" || motionToken.value === "W")) { - const eMotionValue = motionToken.value === "w" ? "e" : "E"; - const syntheticTokens: readonly VimKeyToken[] = [ - ...tokens.slice(0, nextIndex), - { ...motionToken, value: eMotionValue }, - ...tokens.slice(nextIndex + 1), - ]; - motion = this.#resolveMotion(syntheticTokens, nextIndex, effectiveCount, hasAnyCount); - } else { - motion = this.#resolveMotion(tokens, nextIndex, effectiveCount, hasAnyCount); - } - await this.#applyOperatorToMotion( - operator, - motion, - tokens.slice(operatorIndex, motion.nextIndex).map(tokenEntry => tokenEntry.value), - ); - return motion.nextIndex; - } - - async #applyOperatorToMotion(operator: string, motion: MotionResult, tokens: readonly string[]): Promise { - if (operator === "y") { - const range = this.#resolveMotionRange(motion); - this.register = { - kind: range.linewise ? "line" : "char", - text: this.buffer.getText().slice(range.start, range.end), - }; - this.statusMessage = `Yanked ${range.linewise ? "line" : "selection"}`; - return; - } - - if (operator === ">" || operator === "<") { - const range = this.#resolveMotionRange(motion); - const startLine = this.buffer.offsetToPosition(range.start).line; - const endLine = this.buffer.offsetToPosition(Math.max(range.start, range.end - 1)).line; - await this.#applyAtomicChange(tokens, () => { - this.buffer.indentLines(startLine, endLine, detectIndentUnit(this.buffer.lines), operator === ">" ? 1 : -1); - }); - return; - } - - if (operator === "d") { - const range = this.#resolveMotionRange(motion); - await this.#applyAtomicChange(tokens, () => { - this.#yankAndDeleteRange(range); - }); - return; - } - - if (operator === "c") { - const range = this.#resolveMotionRange(motion); - await this.#startInsertChange(tokens, () => { - this.#yankAndDeleteRange(range); - }); - return; - } - } - - #yankAndDeleteRange(range: { start: number; end: number; linewise: boolean }): void { - this.register = { - kind: range.linewise ? "line" : "char", - text: this.buffer.getText().slice(range.start, range.end), - }; - this.buffer.deleteOffsets(range.start, range.end); - } - - #deleteCharsForward(count: number): void { - const start = this.buffer.currentOffset(); - this.register = { - kind: "char", - text: this.buffer.deleteOffsets(start, Math.min(this.buffer.getText().length, start + count)), - }; - } - - async #changeWholeLines(count: number, tokens: readonly string[]): Promise { - await this.#startInsertChange(tokens, () => { - const start = this.buffer.cursor.line; - const end = this.buffer.clampLine(start + count - 1); - const removed = this.buffer.lines.slice(start, end + 1); - this.register = { kind: "line", text: removed.join("\n") }; - this.buffer.lines.splice(start, end - start + 1, ""); - if (this.buffer.lines.length === 0) { - this.buffer.lines = [""]; - } - this.buffer.setCursor({ line: Math.min(start, this.buffer.lastLineIndex()), col: 0 }); - }); - } - - #resolveMotionRange(motion: MotionResult): { start: number; end: number; linewise: boolean } { - if (motion.range) { - return { - start: motion.range.start, - end: motion.range.end, - linewise: motion.range.linewise ?? false, - }; - } - - if (motion.linewise) { - const startLine = Math.min(this.buffer.cursor.line, motion.target.line); - const endLine = Math.max(this.buffer.cursor.line, motion.target.line); - const start = this.buffer.positionToOffset({ line: startLine, col: 0 }); - const end = - endLine >= this.buffer.lastLineIndex() - ? this.buffer.getText().length - : this.buffer.positionToOffset({ line: endLine + 1, col: 0 }); - return { start, end, linewise: true }; - } - - const from = this.buffer.positionToOffset(this.buffer.cursor); - const to = this.buffer.positionToOffset(motion.target); - const normalized = normalizeRange(from, to); - return { - start: normalized.start, - end: normalized.end + (motion.inclusive === false ? 0 : 1), - linewise: false, - }; - } - - #resolveMotion(tokens: readonly VimKeyToken[], index: number, count: number, hasCount = true): MotionResult { - const token = tokens[index]; - if (!token) { - throw new VimError("Missing motion"); - } - - const text = this.buffer.getText(); - switch (token.value) { - case "h": - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line, col: this.buffer.cursor.col - count }, - }; - case "j": - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line + count, col: this.buffer.cursor.col }, - linewise: true, - }; - case "k": - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line - count, col: this.buffer.cursor.col }, - linewise: true, - }; - case "l": - case " ": - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line, col: this.buffer.cursor.col + count }, - }; - case "w": - case "W": { - let offset = this.buffer.currentOffset(); - for (let step = 0; step < count; step += 1) { - offset = nextWordStart(text, step === 0 ? offset + 1 : offset, token.value === "W"); - } - return { nextIndex: index + 1, target: this.buffer.offsetToPosition(offset), inclusive: false }; - } - case "b": - case "B": { - let offset = this.buffer.currentOffset(); - for (let step = 0; step < count; step += 1) { - offset = previousWordStart(text, offset, token.value === "B"); - } - return { nextIndex: index + 1, target: this.buffer.offsetToPosition(offset) }; - } - case "e": - case "E": { - let offset = this.buffer.currentOffset(); - for (let step = 0; step < count; step += 1) { - offset = endOfWord(text, step === 0 ? offset : offset + 1, token.value === "E"); - } - return { nextIndex: index + 1, target: this.buffer.offsetToPosition(offset) }; - } - case "0": - return { nextIndex: index + 1, target: { line: this.buffer.cursor.line, col: 0 } }; - case "^": - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line, col: this.buffer.firstNonBlank(this.buffer.cursor.line) }, - }; - case "|": - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line, col: Math.max(0, count - 1) }, - }; - case "$": - return { - nextIndex: index + 1, - target: { - line: this.buffer.cursor.line, - col: Math.max(0, this.buffer.getLine(this.buffer.cursor.line).length - 1), - }, - }; - case "+": { - const targetLine = this.buffer.clampLine(this.buffer.cursor.line + count); - return { - nextIndex: index + 1, - target: { line: targetLine, col: this.buffer.firstNonBlank(targetLine) }, - linewise: true, - }; - } - case "-": { - const targetLine = this.buffer.clampLine(this.buffer.cursor.line - count); - return { - nextIndex: index + 1, - target: { line: targetLine, col: this.buffer.firstNonBlank(targetLine) }, - linewise: true, - }; - } - case "_": { - const targetLine = this.buffer.clampLine(this.buffer.cursor.line + (count - 1)); - return { - nextIndex: index + 1, - target: { line: targetLine, col: this.buffer.firstNonBlank(targetLine) }, - linewise: true, - }; - } - case "g": { - const next = tokens[index + 1]; - if (!next) { - throw new VimError("Unsupported g motion", token); - } - if (next.value === "g") { - return { - nextIndex: index + 2, - target: { line: hasCount ? Math.max(0, count - 1) : 0, col: 0 }, - linewise: true, - }; - } - if (next.value === "e" || next.value === "E") { - let offset = this.buffer.currentOffset(); - for (let step = 0; step < count; step += 1) { - offset = endOfPreviousWord(text, offset, next.value === "E"); - } - return { nextIndex: index + 2, target: this.buffer.offsetToPosition(offset) }; - } - if (next.value === "_") { - const targetLine = this.buffer.clampLine(this.buffer.cursor.line + (count - 1)); - return { - nextIndex: index + 2, - target: { line: targetLine, col: lastNonBlankColumn(this.buffer.getLine(targetLine)) }, - }; - } - throw new VimError("Unsupported g motion", token); - } - case "G": - return { - nextIndex: index + 1, - target: { line: hasCount ? count - 1 : this.buffer.lastLineIndex(), col: 0 }, - linewise: true, - }; - case "f": - case "F": - case "t": - case "T": { - const searchToken = tokens[index + 1]; - if (!searchToken || searchToken.value.length !== 1) { - throw new VimError(`${token.value} requires a literal character`, token); - } - this.lastCharFind = { char: searchToken.value, mode: token.value as "f" | "F" | "t" | "T" }; - const line = this.buffer.getLine(this.buffer.cursor.line); - const cursorCol = this.buffer.cursor.col; - let matchIndex = -1; - if (token.value === "f" || token.value === "t") { - let start = cursorCol + 1; - for (let step = 0; step < count; step += 1) { - matchIndex = line.indexOf(searchToken.value, start); - if (matchIndex === -1) break; - start = matchIndex + 1; - } - if (matchIndex === -1) { - throw new VimError(`Character not found: ${searchToken.value}`, searchToken); - } - if (token.value === "t") { - matchIndex -= 1; - } - } else { - let start = Math.max(0, cursorCol - 1); - for (let step = 0; step < count; step += 1) { - matchIndex = line.lastIndexOf(searchToken.value, start); - if (matchIndex === -1) break; - start = matchIndex - 1; - } - if (matchIndex === -1) { - throw new VimError(`Character not found: ${searchToken.value}`, searchToken); - } - if (token.value === "T") { - matchIndex += 1; - } - } - return { - nextIndex: index + 2, - target: { line: this.buffer.cursor.line, col: Math.max(0, matchIndex) }, - }; - } - case "{": - return { - nextIndex: index + 1, - target: { line: findParagraphStart(this.buffer.lines, this.buffer.cursor.line), col: 0 }, - linewise: true, - }; - case "}": - return { - nextIndex: index + 1, - target: { line: findParagraphEnd(this.buffer.lines, this.buffer.cursor.line), col: 0 }, - linewise: true, - }; - case "%": { - const match = this.#findMatchingBracket(); - return { nextIndex: index + 1, target: match }; - } - case "H": - return { - nextIndex: index + 1, - target: { line: Math.max(0, this.viewportStart - 1), col: 0 }, - linewise: true, - }; - case "M": - return { - nextIndex: index + 1, - target: { line: Math.max(0, this.viewportStart - 1 + 20), col: 0 }, - linewise: true, - }; - case "L": - return { - nextIndex: index + 1, - target: { line: Math.max(0, this.viewportStart - 1 + 39), col: 0 }, - linewise: true, - }; - case ";": - case ",": { - if (!this.lastCharFind) { - throw new VimError( - "No previous character search. If you meant an ex-command range like `:4,5d`, add the `:` prefix and `` suffix.", - token, - ); - } - let mode = this.lastCharFind.mode; - if (token.value === ",") { - const reverseMap: Record = { f: "F", F: "f", t: "T", T: "t" }; - mode = reverseMap[mode]!; - } - const line = this.buffer.getLine(this.buffer.cursor.line); - const cursorCol = this.buffer.cursor.col; - let matchIndex = -1; - if (mode === "f" || mode === "t") { - let start = cursorCol + 1; - for (let step = 0; step < count; step += 1) { - matchIndex = line.indexOf(this.lastCharFind.char, start); - if (matchIndex === -1) break; - start = matchIndex + 1; - } - if (matchIndex !== -1 && mode === "t") matchIndex -= 1; - } else { - let start = Math.max(0, cursorCol - 1); - for (let step = 0; step < count; step += 1) { - matchIndex = line.lastIndexOf(this.lastCharFind.char, start); - if (matchIndex === -1) break; - start = matchIndex - 1; - } - if (matchIndex !== -1 && mode === "T") matchIndex += 1; - } - if (matchIndex === -1) { - throw new VimError(`Character not found: ${this.lastCharFind.char}`, token); - } - return { - nextIndex: index + 1, - target: { line: this.buffer.cursor.line, col: Math.max(0, matchIndex) }, - }; - } - default: - throw new VimError(`Unsupported motion: ${token.display}`, token); - } - } - - #resolveTextObject( - inner: boolean, - objectToken: string, - sourceToken: VimKeyToken, - ): { start: number; end: number; linewise?: boolean } { - if (objectToken === "w" || objectToken === "W") { - return this.#resolveWordTextObject(inner, objectToken === "W"); - } - if (objectToken === '"' || objectToken === "'" || objectToken === "`") { - return this.#resolveQuoteTextObject(inner, objectToken, sourceToken); - } - - if (objectToken === "p") { - return this.#resolveParagraphTextObject(inner); - } - - const normalized = - objectToken === ")" - ? "(" - : objectToken === "}" - ? "{" - : objectToken === "]" - ? "[" - : objectToken === ">" - ? "<" - : objectToken; - if (!BRACKET_PAIRS.has(normalized)) { - throw new VimError(`Unsupported text object: ${objectToken}`, sourceToken); - } - return this.#resolveBracketTextObject(inner, normalized, sourceToken); - } - - #resolveWordTextObject(inner: boolean, bigWord: boolean): { start: number; end: number } { - const text = this.buffer.getText(); - const cursor = this.buffer.currentOffset(); - let start = cursor; - if (wordCategory(text[start] ?? "", bigWord) === "space") { - start = nextWordStart(text, start, bigWord); - } - const category = wordCategory(text[start] ?? "", bigWord); - while (start > 0 && wordCategory(text[start - 1] ?? "", bigWord) === category) { - start -= 1; - } - let end = start; - while (end < text.length && wordCategory(text[end] ?? "", bigWord) === category) { - end += 1; - } - if (!inner) { - while (end < text.length && wordCategory(text[end] ?? "", bigWord) === "space") { - end += 1; - } - while (start > 0 && wordCategory(text[start - 1] ?? "", bigWord) === "space") { - start -= 1; - } - } - return { start, end }; - } - - #resolveParagraphTextObject(inner: boolean): { start: number; end: number; linewise: boolean } { - const lines = this.buffer.lines; - const cursorLine = this.buffer.cursor.line; - let start = cursorLine; - let end = cursorLine; - // Find paragraph boundaries (delimited by blank lines) - if (lines[cursorLine]?.trim().length === 0) { - // On a blank line: select contiguous blank lines - while (start > 0 && lines[start - 1]!.trim().length === 0) start -= 1; - while (end < lines.length - 1 && lines[end + 1]!.trim().length === 0) end += 1; - if (!inner) { - // Include following non-blank paragraph - while (end < lines.length - 1 && lines[end + 1]!.trim().length > 0) end += 1; - } - } else { - // On a non-blank line: select contiguous non-blank lines - while (start > 0 && lines[start - 1]!.trim().length > 0) start -= 1; - while (end < lines.length - 1 && lines[end + 1]!.trim().length > 0) end += 1; - if (!inner) { - // Include trailing blank lines - while (end < lines.length - 1 && lines[end + 1]!.trim().length === 0) end += 1; - } - } - const startOffset = this.buffer.positionToOffset({ line: start, col: 0 }); - const endOffset = - end >= this.buffer.lastLineIndex() - ? this.buffer.getText().length - : this.buffer.positionToOffset({ line: end + 1, col: 0 }); - return { start: startOffset, end: endOffset, linewise: true }; - } - - #resolveQuoteTextObject(inner: boolean, quote: string, sourceToken: VimKeyToken): { start: number; end: number } { - const line = this.buffer.getLine(this.buffer.cursor.line); - const col = this.buffer.cursor.col; - const before = line.lastIndexOf(quote, col); - const after = line.indexOf(quote, col + (line[col] === quote ? 1 : 0)); - if (before === -1 || after === -1 || before === after) { - throw new VimError(`Quote text object not found for ${quote}`, sourceToken); - } - const startCol = inner ? before + 1 : before; - const endCol = inner ? after : after + 1; - return { - start: this.buffer.positionToOffset({ line: this.buffer.cursor.line, col: startCol }), - end: this.buffer.positionToOffset({ line: this.buffer.cursor.line, col: endCol }), - }; - } - - #resolveBracketTextObject(inner: boolean, open: string, sourceToken: VimKeyToken): { start: number; end: number } { - const close = BRACKET_PAIRS.get(open)!; - const text = this.buffer.getText(); - const cursor = this.buffer.currentOffset(); - let start = -1; - let depth = 0; - for (let index = cursor; index >= 0; index -= 1) { - const char = text[index] ?? ""; - if (char === close) { - depth += 1; - } else if (char === open) { - if (depth === 0) { - start = index; - break; - } - depth -= 1; - } - } - if (start === -1) { - throw new VimError(`Text object ${open}${close} not found`, sourceToken); - } - let end = -1; - depth = 0; - for (let index = start; index < text.length; index += 1) { - const char = text[index] ?? ""; - if (char === open) { - depth += 1; - } else if (char === close) { - depth -= 1; - if (depth === 0) { - end = index; - break; - } - } - } - if (end === -1) { - throw new VimError(`Text object ${open}${close} not found`, sourceToken); - } - return { - start: inner ? start + 1 : start, - end: inner ? end : end + 1, - }; - } - - #findMatchingBracket(): Position { - const text = this.buffer.getText(); - const cursor = this.buffer.currentOffset(); - let offset = cursor; - let char = text[offset] ?? ""; - if (!BRACKET_PAIRS.has(char) && !CLOSING_BRACKETS.has(char)) { - offset += 1; - char = text[offset] ?? ""; - } - if (BRACKET_PAIRS.has(char)) { - const close = BRACKET_PAIRS.get(char)!; - let depth = 0; - for (let index = offset; index < text.length; index += 1) { - const current = text[index] ?? ""; - if (current === char) depth += 1; - if (current === close) { - depth -= 1; - if (depth === 0) { - return this.buffer.offsetToPosition(index); - } - } - } - } - if (CLOSING_BRACKETS.has(char)) { - const open = CLOSING_BRACKETS.get(char)!; - let depth = 0; - for (let index = offset; index >= 0; index -= 1) { - const current = text[index] ?? ""; - if (current === char) depth += 1; - if (current === open) { - depth -= 1; - if (depth === 0) { - return this.buffer.offsetToPosition(index); - } - } - } - } - throw new VimError("Matching bracket not found"); - } - - async #runSearch(pattern: string, direction: 1 | -1, updateState: boolean): Promise { - const text = this.buffer.getText(); - const regex = createSearchRegex(pattern, "g"); - const cursor = this.buffer.currentOffset(); - let matchOffset = -1; - - if (direction > 0) { - regex.lastIndex = Math.min(text.length, cursor + 1); - const match = regex.exec(text); - if (match && match.index >= 0) { - matchOffset = match.index; - } else { - regex.lastIndex = 0; - const wrapMatch = regex.exec(text); - if (wrapMatch && wrapMatch.index >= 0) { - matchOffset = wrapMatch.index; - } - } - } else { - const matches = Array.from(text.matchAll(regex)); - for (let index = matches.length - 1; index >= 0; index -= 1) { - const match = matches[index]; - if ((match.index ?? -1) < cursor) { - matchOffset = match.index ?? -1; - break; - } - } - if (matchOffset === -1 && matches.length > 0) { - matchOffset = matches[matches.length - 1]?.index ?? -1; - } - } - - if (matchOffset === -1) { - throw new VimError(`Pattern not found: ${pattern}`); - } - - this.buffer.setCursor(this.buffer.offsetToPosition(matchOffset)); - this.statusMessage = `${direction > 0 ? "/" : "?"}${pattern}`; - if (updateState) { - this.lastSearch = { pattern, direction }; - } - } - - async #repeatSearch(direction: 1 | -1, count: number): Promise { - if (!this.lastSearch) { - throw new VimError("No previous search"); - } - for (let index = 0; index < count; index += 1) { - await this.#runSearch(this.lastSearch.pattern, direction, false); - } - this.lastSearch = { pattern: this.lastSearch.pattern, direction }; - } - - #resolveExRange( - range: VimLineRange | "all" | undefined, - defaultStart: number, - defaultEnd = defaultStart, - ): VimLineRange { - const totalLines = Math.max(1, this.buffer.lineCount()); - if (range === "all") { - return { start: 1, end: totalLines }; - } - const next = range ?? { start: defaultStart, end: defaultEnd }; - const start = Math.max(1, Math.min(next.start, totalLines)); - const end = Math.max(start, Math.min(next.end, totalLines)); - return { start, end }; - } - - async #executeEx(input: string): Promise { - const command = parseExCommand(input, { - currentLine: this.buffer.cursor.line + 1, - lastLine: this.buffer.lineCount(), - }); - switch (command.kind) { - case "goto-line": - this.buffer.setCursor({ line: Math.max(0, command.line - 1), col: 0 }); - this.statusMessage = `Line ${command.line}`; - return; - case "write": { - const result = await this.#callbacks.saveBuffer(this.buffer, { force: command.force }); - this.buffer.markSaved(result.loaded); - this.diagnostics = result.diagnostics; - this.statusMessage = result.diagnostics - ? `Wrote ${this.buffer.displayPath} (${result.diagnostics.summary})` - : `Wrote ${this.buffer.displayPath}`; - this.#undoStack = []; - this.#redoStack = []; - return; - } - case "update": - if (!this.buffer.modified) { - this.statusMessage = `${this.buffer.displayPath} unchanged`; - return; - } - await this.#executeEx(command.force ? "w!" : "w"); - return; - case "write-quit": - await this.#executeEx(command.force ? "w!" : "w"); - this.closed = true; - this.statusMessage = `Wrote and closed ${this.buffer.displayPath}`; - return; - case "quit": - if (this.buffer.modified && !command.force) { - throw new VimError("Unsaved changes; use :q! to discard"); - } - this.closed = true; - this.statusMessage = `Closed ${this.buffer.displayPath}`; - return; - case "edit": { - if (this.buffer.modified && !command.force) { - throw new VimError("Unsaved changes; use :e! to reload or force open"); - } - const next = await this.#callbacks.loadBuffer(command.path ?? this.buffer.displayPath); - this.buffer.replaceLoadedFile(next); - this.inputMode = "normal"; - this.selectionAnchor = null; - this.#pendingInput = ""; - this.#pendingChange = null; - this.#undoStack = []; - this.#redoStack = []; - this.statusMessage = command.path - ? `Opened ${this.buffer.displayPath}` - : `Reloaded ${this.buffer.displayPath}`; - return; - } - case "substitute": { - const range = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - const startLine = range.start; - const endLine = range.end; - const regexFlags = command.flags.includes("i") ? "gi" : "g"; - const regex = createSearchRegex(command.pattern, regexFlags); - let replacements = 0; - await this.#applyAtomicChange([":substitute"], () => { - for (let lineIndex = startLine - 1; lineIndex <= endLine - 1; lineIndex += 1) { - const line = this.buffer.getLine(lineIndex); - let lineReplacements = 0; - const nextLine = line.replace(regex, match => { - if (!command.flags.includes("g") && lineReplacements > 0) { - return match; - } - lineReplacements += 1; - replacements += 1; - return decodeReplacement(command.replacement).replace(/&/g, match); - }); - this.buffer.replaceLine(lineIndex, nextLine); - regex.lastIndex = 0; - } - }); - if (replacements === 0) { - throw new VimError(`Pattern not found: ${command.pattern}`); - } - this.statusMessage = `${replacements} substitution${replacements === 1 ? "" : "s"}`; - return; - } - case "delete": { - const range = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - await this.#applyAtomicChange([":delete"], () => { - const removed = this.buffer.deleteLines(range.start - 1, range.end - 1); - this.register = { kind: "line", text: removed.join("\n") }; - }); - this.statusMessage = `Deleted ${range.end - range.start + 1} line${range.end === range.start ? "" : "s"}`; - return; - } - case "yank": { - const range = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - this.register = { - kind: "line", - text: this.buffer.lines.slice(range.start - 1, range.end).join("\n"), - }; - this.statusMessage = `Yanked ${range.end - range.start + 1} line${range.end === range.start ? "" : "s"}`; - return; - } - case "put": { - if (!this.register.text) { - this.statusMessage = "Register empty"; - return; - } - const anchorRange = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - const anchorLine = command.before ? anchorRange.start : anchorRange.end; - const lines = this.register.text.split("\n"); - await this.#applyAtomicChange([":put"], () => { - const insertAt = command.before - ? Math.max(0, anchorLine - 1) - : Math.min(anchorLine, this.buffer.lineCount()); - this.buffer.insertLines(insertAt, lines); - }); - this.statusMessage = `Put ${lines.length} line${lines.length === 1 ? "" : "s"}`; - return; - } - case "copy": { - const totalLines = this.buffer.lineCount(); - const range = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - const dest = Math.max(0, Math.min(command.destination, totalLines)); - await this.#applyAtomicChange([":copy"], () => { - const lines = this.buffer.lines.slice(range.start - 1, range.end); - this.buffer.insertLines(dest, lines); - }); - this.statusMessage = `Copied ${range.end - range.start + 1} line${range.end === range.start ? "" : "s"}`; - return; - } - case "move": { - const totalLines = this.buffer.lineCount(); - const range = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - const dest = Math.max(0, Math.min(command.destination, totalLines)); - await this.#applyAtomicChange([":move"], () => { - const lines = this.buffer.lines.splice(range.start - 1, range.end - range.start + 1); - const adjustedDest = dest > range.end - 1 ? dest - lines.length : dest; - this.buffer.lines.splice(adjustedDest, 0, ...lines); - if (this.buffer.lines.length === 0) this.buffer.lines = [""]; - this.buffer.setCursor({ line: adjustedDest, col: 0 }); - }); - this.statusMessage = `Moved ${range.end - range.start + 1} line${range.end === range.start ? "" : "s"}`; - return; - } - case "sort": { - const range = this.#resolveExRange(command.range ?? "all", 1, this.buffer.lineCount()); - const startLine = range.start; - const endLine = range.end; - const reverse = command.flags.includes("!"); - const ignoreCase = command.flags.includes("i"); - await this.#applyAtomicChange([":sort"], () => { - const slice = this.buffer.lines.slice(startLine - 1, endLine); - slice.sort((a, b) => { - const left = ignoreCase ? a.toLowerCase() : a; - const right = ignoreCase ? b.toLowerCase() : b; - return left < right ? -1 : left > right ? 1 : 0; - }); - if (reverse) slice.reverse(); - for (let i = 0; i < slice.length; i++) { - this.buffer.lines[startLine - 1 + i] = slice[i]!; - } - }); - this.statusMessage = `Sorted ${endLine - startLine + 1} line${endLine === startLine ? "" : "s"}`; - return; - } - case "join": { - const currentLine = this.buffer.cursor.line + 1; - const baseRange = this.#resolveExRange( - command.range, - currentLine, - command.range ? undefined : Math.min(this.buffer.lineCount(), currentLine + 1), - ); - const startLine = baseRange.start; - const endLine = - baseRange.start === baseRange.end ? Math.min(this.buffer.lineCount(), baseRange.end + 1) : baseRange.end; - const lineCount = endLine - startLine + 1; - if (lineCount < 2) { - this.statusMessage = "Nothing to join"; - return; - } - await this.#applyAtomicChange([":join"], () => { - const startIndex = startLine - 1; - if (command.trimWhitespace) { - this.buffer.joinLines(startIndex, lineCount - 1); - return; - } - const joined = this.buffer.lines.slice(startIndex, endLine).join(""); - this.buffer.lines.splice(startIndex, lineCount, joined); - this.buffer.setCursor({ line: startIndex, col: Math.max(0, joined.length - 1) }); - }); - this.statusMessage = `Joined ${lineCount} lines`; - return; - } - case "append": { - const anchorRange = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - const anchorLine = anchorRange.end; - const lines = command.text.length > 0 ? command.text.split("\n") : [""]; - await this.#applyAtomicChange([":append"], () => { - const insertAt = Math.min(anchorLine, this.buffer.lineCount()); - this.buffer.insertLines(insertAt, lines); - }); - this.statusMessage = `Appended ${lines.length} line${lines.length === 1 ? "" : "s"}`; - return; - } - case "insert-before": { - const anchorRange = this.#resolveExRange(command.range, this.buffer.cursor.line + 1); - const anchorLine = anchorRange.start; - const lines = command.text.length > 0 ? command.text.split("\n") : [""]; - await this.#applyAtomicChange([":insert"], () => { - const insertAt = Math.max(0, anchorLine - 1); - this.buffer.insertLines(insertAt, lines); - }); - this.statusMessage = `Inserted ${lines.length} line${lines.length === 1 ? "" : "s"}`; - return; - } - case "global": { - const regex = createSearchRegex(command.pattern); - const range = this.#resolveExRange(command.range ?? "all", 1, this.buffer.lineCount()); - await this.#applyAtomicChange([":global"], () => { - const linesToProcess: number[] = []; - for (let i = range.start - 1; i <= range.end - 1; i += 1) { - const matches = regex.test(this.buffer.getLine(i)); - regex.lastIndex = 0; - if (command.invert ? !matches : matches) { - linesToProcess.push(i); - } - } - if (command.command === "d" || command.command === "delete") { - // Delete matching lines in reverse to preserve indices - for (let i = linesToProcess.length - 1; i >= 0; i--) { - this.buffer.lines.splice(linesToProcess[i]!, 1); - } - if (this.buffer.lines.length === 0) this.buffer.lines = [""]; - this.buffer.clampCursor(); - this.buffer.trailingNewline = true; - } else { - throw new VimError(`Unsupported :global sub-command: ${command.command}`); - } - }); - this.statusMessage = `Global: processed ${command.pattern}`; - return; - } - } - } - - #paste(after: boolean, count: number): void { - if (!this.register.text) { - return; - } - if (this.register.kind === "line") { - const lines = this.register.text.split("\n"); - const insertAt = after ? this.buffer.cursor.line + 1 : this.buffer.cursor.line; - for (let iteration = 0; iteration < count; iteration += 1) { - this.buffer.insertLines(insertAt + iteration * lines.length, lines); - } - return; - } - const text = this.register.text.repeat(count); - const offset = this.buffer.currentOffset() + (after ? 1 : 0); - this.buffer.replaceOffsets(offset, offset, text, offset + text.length); - } - - #readCount(tokens: readonly VimKeyToken[], index: number): { count: number; hasCount: boolean; nextIndex: number } { - let cursor = index; - let digits = ""; - while (cursor < tokens.length) { - const value = tokens[cursor]?.value ?? ""; - if (!/^\d$/.test(value)) { - break; - } - if (digits.length === 0 && value === "0") { - break; - } - digits += value; - cursor += 1; - } - return { - count: digits.length > 0 ? Number.parseInt(digits, 10) : 1, - hasCount: digits.length > 0, - nextIndex: cursor, - }; - } -} diff --git a/packages/coding-agent/src/vim/parser.ts b/packages/coding-agent/src/vim/parser.ts deleted file mode 100644 index ae1e3794f..000000000 --- a/packages/coding-agent/src/vim/parser.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { VimInputError as VimError, type VimKeyToken } from "./types"; - -const SPECIAL_KEYS = new Map([ - ["esc", "Esc"], - ["escape", "Esc"], - ["cr", "CR"], - ["enter", "CR"], - ["return", "CR"], - ["bs", "BS"], - ["backspace", "BS"], - ["tab", "Tab"], - ["c-d", "C-d"], - ["c-u", "C-u"], - ["c-r", "C-r"], - ["c-w", "C-w"], - ["c-o", "C-o"], -]); - -function normalizeSpecialKey(raw: string): string | undefined { - return SPECIAL_KEYS.get(raw.trim().toLowerCase()); -} - -function toDisplayToken(value: string): string { - switch (value) { - case " ": - return ""; - default: - return value.length === 1 ? value : `<${value}>`; - } -} - -export function parseKeySequences(sequences: string[]): VimKeyToken[] { - const tokens: VimKeyToken[] = []; - - for (let sequenceIndex = 0; sequenceIndex < sequences.length; sequenceIndex += 1) { - const sequence = sequences[sequenceIndex] ?? ""; - for (let offset = 0; offset < sequence.length; offset += 1) { - const char = sequence[offset] ?? ""; - // Handle literal escape byte (\x1b / \u001b) - if (char === "\x1b") { - tokens.push({ - value: "Esc", - display: "", - sequenceIndex, - offset, - }); - continue; - } - // Handle literal carriage return - if (char === "\r") { - tokens.push({ - value: "CR", - display: "", - sequenceIndex, - offset, - }); - continue; - } - // Handle escaped sequences: \r → CR, \e → Esc, \n → newline, \t → Tab - if (char === "\\" && offset + 1 < sequence.length) { - const next = sequence[offset + 1]; - if (next === "r") { - tokens.push({ value: "CR", display: "\\r", sequenceIndex, offset }); - offset += 1; - continue; - } - if (next === "e") { - tokens.push({ value: "Esc", display: "\\e", sequenceIndex, offset }); - offset += 1; - continue; - } - if (next === "n") { - tokens.push({ value: "\n", display: "\\n", sequenceIndex, offset }); - offset += 1; - continue; - } - if (next === "t") { - tokens.push({ value: "Tab", display: "\\t", sequenceIndex, offset }); - offset += 1; - continue; - } - } - if (char !== "<") { - tokens.push({ - value: char, - display: toDisplayToken(char), - sequenceIndex, - offset, - }); - continue; - } - - const close = sequence.indexOf(">", offset + 1); - if (close === -1) { - throw new VimError(`Unterminated special key in sequence ${sequenceIndex + 1}`, { - value: char, - display: char, - sequenceIndex, - offset, - }); - } - - const rawSpecial = sequence.slice(offset + 1, close); - const special = normalizeSpecialKey(rawSpecial); - if (!special) { - throw new VimError(`Unknown special key <${rawSpecial}> in sequence ${sequenceIndex + 1}`, { - value: rawSpecial, - display: `<${rawSpecial}>`, - sequenceIndex, - offset, - }); - } - - tokens.push({ - value: special, - display: `<${rawSpecial}>`, - sequenceIndex, - offset, - }); - offset = close; - } - } - - return tokens; -} - -export function replayTokens(values: readonly string[]): VimKeyToken[] { - return values.map((value, index) => ({ - value, - display: toDisplayToken(value), - sequenceIndex: 0, - offset: index, - })); -} diff --git a/packages/coding-agent/src/vim/render.ts b/packages/coding-agent/src/vim/render.ts deleted file mode 100644 index 7ea9dae81..000000000 --- a/packages/coding-agent/src/vim/render.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { extractSegments } from "@oh-my-pi/pi-tui"; -import { formatCodeFrameLine, truncateToWidth } from "../tools/render-utils"; -import type { - VimErrorLocation, - VimFocusLine, - VimMode, - VimPendingInput, - VimSelection, - VimToolDetails, - VimViewport, - VimViewportLine, -} from "./types"; - -export const VIM_OPEN_VIEWPORT_LINES = 80; -export const VIM_DEFAULT_VIEWPORT_LINES = 10; -const VIM_TAB_DISPLAY = "→"; -const VIM_INLINE_CURSOR = "▏"; - -const VIM_VIEWPORT_WIDTH = 140; -const VIM_FOCUS_WIDTH = 100; - -interface ViewportRenderInput { - file: string; - mode: VimMode; - cursor: { line: number; col: number }; - totalLines: number; - modified: boolean; - lines: string[]; - viewport: VimViewport; - selection?: VimSelection; - statusMessage?: string; - lastCommand?: string; - pendingInput?: VimPendingInput; - errorLocation?: VimErrorLocation; - closed?: boolean; -} - -function renderHeader(details: Pick): string { - const modified = details.modified ? "[+]" : "[ ]"; - return `${details.file} ${modified} ${details.mode} L${details.cursor.line}:${details.cursor.col} (${details.totalLines} lines)`; -} - -function selectionContainsLine(selection: VimSelection | undefined, lineNumber: number): boolean { - if (!selection) { - return false; - } - return lineNumber >= selection.start.line && lineNumber <= selection.end.line; -} - -function visibleWidthForChar(char: string): number { - return char === "\t" ? VIM_TAB_DISPLAY.length : Math.max(1, Bun.stringWidth(char)); -} - -function renderVisibleText(input: string): string { - let output = ""; - for (const char of input) { - output += char === "\t" ? VIM_TAB_DISPLAY : char; - } - return output; -} - -function renderedColumnForRawColumn(input: string, rawCol: number): number { - let column = 0; - let index = 0; - for (const char of input) { - if (index >= rawCol) { - break; - } - column += visibleWidthForChar(char); - index += 1; - } - return column; -} - -function cropVisibleText(text: string, startCol: number, width: number): { text: string; startCol: number } { - if (text.length <= width) { - return { text, startCol: 0 }; - } - - const maxStart = Math.max(0, text.length - width); - const clampedStart = Math.max(0, Math.min(startCol, maxStart)); - let window = text.slice(clampedStart, clampedStart + width); - if (clampedStart > 0 && window.length > 0) { - window = `…${window.slice(1)}`; - } - if (clampedStart + width < text.length && window.length > 0) { - window = `${window.slice(0, -1)}…`; - } - return { text: window, startCol: clampedStart }; -} - -function buildFocusLine(lineNumber: number, rawText: string, rawCursorCol: number): VimFocusLine { - const visibleText = renderVisibleText(rawText); - const caretCol = renderedColumnForRawColumn(rawText, rawCursorCol); - const desiredStart = Math.max(0, caretCol - Math.floor(VIM_FOCUS_WIDTH / 2)); - const cropped = cropVisibleText(visibleText, desiredStart, VIM_FOCUS_WIDTH); - return { - line: lineNumber, - text: cropped.text, - windowStartCol: cropped.startCol + 1, - windowEndCol: cropped.startCol + cropped.text.length, - caretCol: Math.max(0, caretCol - cropped.startCol), - }; -} - -function buildViewportLines( - input: Pick, -): VimViewportLine[] { - const lines: VimViewportLine[] = []; - for (let lineNumber = input.viewport.start; lineNumber <= input.viewport.end; lineNumber += 1) { - const rawText = input.lines[lineNumber - 1] ?? ""; - const visibleText = renderVisibleText(rawText); - const isCursor = lineNumber === input.cursor.line; - if (isCursor) { - const cursorCol = renderedColumnForRawColumn(rawText, input.cursor.col - 1); - const desiredStart = Math.max(0, cursorCol - Math.floor(VIM_VIEWPORT_WIDTH / 2)); - const cropped = cropVisibleText(visibleText, desiredStart, VIM_VIEWPORT_WIDTH); - lines.push({ - line: lineNumber, - text: cropped.text, - isCursor: true, - isSelected: selectionContainsLine(input.selection, lineNumber), - cursorCol: Math.max(0, cursorCol - cropped.startCol), - }); - continue; - } - lines.push({ - line: lineNumber, - text: truncateToWidth(visibleText, VIM_VIEWPORT_WIDTH), - isCursor: false, - isSelected: selectionContainsLine(input.selection, lineNumber), - }); - } - return lines; -} - -export function computeViewport( - cursorLine: number, - totalLines: number, - size: number, - preferredStart?: number, -): VimViewport { - const lineCount = Math.max(totalLines, 1); - const clampedSize = Math.max(1, Math.min(size, lineCount)); - const maxStart = Math.max(1, lineCount - clampedSize + 1); - const centered = Math.max(1, Math.min(cursorLine - Math.floor(clampedSize / 2), maxStart)); - let start = preferredStart ? Math.max(1, Math.min(preferredStart, maxStart)) : centered; - const end = Math.min(lineCount, start + clampedSize - 1); - if (cursorLine < start) { - start = cursorLine; - } - if (cursorLine > end) { - start = Math.max(1, cursorLine - clampedSize + 1); - } - return { - start, - end: Math.min(lineCount, start + clampedSize - 1), - }; -} - -function formatPendingInput(pending: VimPendingInput | undefined): string | undefined { - if (!pending) { - return undefined; - } - if (pending.kind === "insert") { - return "Pending: INSERT mode"; - } - const prefix = pending.kind === "command" ? ":" : pending.kind === "search-forward" ? "/" : "?"; - return `Pending: ${prefix}${truncateToWidth(renderVisibleText(pending.text), 80)}`; -} - -function renderPlainViewportCursor(line: VimViewportLine): string { - if (!line.isCursor || line.cursorCol === undefined) { - return line.text; - } - const totalWidth = Bun.stringWidth(line.text); - const cursorCol = Math.max(0, Math.min(line.cursorCol, totalWidth)); - const segments = extractSegments(line.text, cursorCol, cursorCol, Math.max(0, totalWidth - cursorCol), true); - return `${segments.before}${VIM_INLINE_CURSOR}${segments.after}`; -} - -export function renderVimDetails(details: VimToolDetails): string { - const lines: string[] = [renderHeader(details)]; - - // Explicit cursor position indicator (models miss it in header) - lines.push(`[CURSOR] Line ${details.cursor.line}, Column ${details.cursor.col} (of ${details.totalLines} lines)`); - - if (details.lastCommand) { - lines.push(`Command: ${truncateToWidth(details.lastCommand, 80)}`); - } - if (details.statusMessage) { - lines.push(`Status: ${details.statusMessage}`); - } - if (details.errorLocation) { - lines.push( - `Error location: sequence ${details.errorLocation.sequenceIndex + 1}, token ${details.errorLocation.offset + 1}`, - ); - } - - const pending = formatPendingInput(details.pendingInput); - if (pending) { - lines.push(pending); - } - - if (details.closed) { - return lines.join("\n"); - } - - if (details.focus) { - const focusPrefix = formatCodeFrameLine(">", details.focus.line, "", String(details.viewport.end).length); - const caretPrefix = `${" ".repeat(focusPrefix.length)} `; - const caretPadding = " ".repeat(Math.max(0, details.focus.caretCol)); - lines.push("Focus:"); - lines.push(`${focusPrefix}${details.focus.text}`); - lines.push(`${caretPrefix}${caretPadding}^`); - } - - if (details.viewportLines && details.viewportLines.length > 0) { - const padWidth = String(details.viewport.end).length; - lines.push("Viewport:"); - for (const line of details.viewportLines) { - const marker = line.isCursor ? ">" : line.isSelected ? "*" : " "; - lines.push(formatCodeFrameLine(marker, line.line, renderPlainViewportCursor(line), padWidth)); - } - } - - return lines.join("\n"); -} - -export function buildDetails(input: ViewportRenderInput): VimToolDetails { - const details: VimToolDetails = { - file: input.file, - mode: input.mode, - cursor: input.cursor, - totalLines: input.totalLines, - modified: input.modified, - viewport: input.viewport, - selection: input.selection, - lastCommand: input.lastCommand, - statusMessage: input.statusMessage, - pendingInput: input.pendingInput, - errorLocation: input.errorLocation, - closed: input.closed, - }; - - if (!input.closed) { - details.focus = buildFocusLine(input.cursor.line, input.lines[input.cursor.line - 1] ?? "", input.cursor.col - 1); - details.viewportLines = buildViewportLines(input); - } - - return details; -} diff --git a/packages/coding-agent/src/vim/types.ts b/packages/coding-agent/src/vim/types.ts deleted file mode 100644 index 5ca3d8940..000000000 --- a/packages/coding-agent/src/vim/types.ts +++ /dev/null @@ -1,197 +0,0 @@ -import type { FileDiagnosticsResult } from "../lsp"; -import type { OutputMeta } from "../tools/output-meta"; - -export type VimMode = "NORMAL" | "INSERT" | "VISUAL" | "VISUAL-LINE" | "COMMAND"; - -export type VimInputMode = - | "normal" - | "insert" - | "visual" - | "visual-line" - | "command" - | "search-forward" - | "search-backward"; - -export interface Position { - line: number; - col: number; -} - -export interface VimViewport { - start: number; - end: number; -} - -export interface VimSelection { - kind: "char" | "line"; - start: Position; - end: Position; -} - -export interface VimFocusLine { - line: number; - text: string; - windowStartCol: number; - windowEndCol: number; - caretCol: number; -} - -export interface VimViewportLine { - line: number; - text: string; - isCursor: boolean; - isSelected: boolean; - cursorCol?: number; -} - -export interface VimPendingInput { - kind: "insert" | "command" | "search-forward" | "search-backward"; - text: string; -} - -export interface VimErrorLocation { - sequenceIndex: number; - offset: number; -} - -export interface VimToolDetails { - file: string; - mode: VimMode; - cursor: { line: number; col: number }; - totalLines: number; - modified: boolean; - viewport: VimViewport; - focus?: VimFocusLine; - viewportLines?: VimViewportLine[]; - selection?: VimSelection; - pendingInput?: VimPendingInput; - errorLocation?: VimErrorLocation; - closed?: boolean; - meta?: OutputMeta; - lastCommand?: string; - statusMessage?: string; - diagnostics?: FileDiagnosticsResult; -} - -export interface VimFingerprint { - exists: boolean; - size: number; - mtimeMs: number; - hash: string; -} - -export interface VimLoadedFile { - absolutePath: string; - displayPath: string; - lines: string[]; - trailingNewline: boolean; - fingerprint: VimFingerprint | null; -} - -export interface VimKeyToken { - value: string; - display: string; - sequenceIndex: number; - offset: number; -} - -export interface VimRegister { - kind: "char" | "line"; - text: string; -} - -export interface VimSearchState { - pattern: string; - direction: 1 | -1; -} - -export interface VimBufferSnapshot { - displayPath: string; - filePath: string; - lines: string[]; - cursor: Position; - modified: boolean; - trailingNewline: boolean; - baseFingerprint: VimFingerprint | null; - editabilityChecked: boolean; -} - -export interface VimUndoEntry { - before: VimBufferSnapshot; - after: VimBufferSnapshot; -} - -export interface VimLineRange { - start: number; - end: number; -} - -export type VimExCommand = - | { kind: "write"; force: boolean } - | { kind: "update"; force: boolean } - | { kind: "quit"; force: boolean } - | { kind: "write-quit"; force: boolean } - | { kind: "edit"; force: boolean; path?: string } - | { kind: "goto-line"; line: number } - | { kind: "substitute"; range?: VimLineRange | "all"; pattern: string; replacement: string; flags: string } - | { kind: "delete"; range?: VimLineRange | "all" } - | { kind: "yank"; range?: VimLineRange | "all" } - | { kind: "put"; range?: VimLineRange | "all"; before: boolean } - | { kind: "copy"; range?: VimLineRange | "all"; destination: number } - | { kind: "move"; range?: VimLineRange | "all"; destination: number } - | { kind: "sort"; range?: VimLineRange | "all"; flags: string } - | { kind: "join"; range?: VimLineRange | "all"; trimWhitespace: boolean } - | { kind: "global"; range?: VimLineRange | "all"; pattern: string; command: string; invert: boolean } - | { kind: "append"; range?: VimLineRange; text: string } - | { kind: "insert-before"; range?: VimLineRange; text: string }; - -export class VimInputError extends Error { - location?: { sequenceIndex: number; offset: number }; - - constructor(message: string, token?: VimKeyToken) { - super(message); - this.name = "VimInputError"; - if (token) { - this.location = { - sequenceIndex: token.sequenceIndex, - offset: token.offset, - }; - } - } -} - -export function clonePosition(position: Position): Position { - return { line: position.line, col: position.col }; -} - -function comparePositions(left: Position, right: Position): number { - if (left.line !== right.line) { - return left.line - right.line; - } - return left.col - right.col; -} - -export function minPosition(left: Position, right: Position): Position { - return comparePositions(left, right) <= 0 ? clonePosition(left) : clonePosition(right); -} - -export function maxPosition(left: Position, right: Position): Position { - return comparePositions(left, right) >= 0 ? clonePosition(left) : clonePosition(right); -} - -export function toPublicMode(mode: VimInputMode): VimMode { - switch (mode) { - case "insert": - return "INSERT"; - case "visual": - return "VISUAL"; - case "visual-line": - return "VISUAL-LINE"; - case "command": - case "search-forward": - case "search-backward": - return "COMMAND"; - default: - return "NORMAL"; - } -} diff --git a/packages/coding-agent/test/agent-session-mcp-discovery.test.ts b/packages/coding-agent/test/agent-session-mcp-discovery.test.ts index 0d770082e..8854829d7 100644 --- a/packages/coding-agent/test/agent-session-mcp-discovery.test.ts +++ b/packages/coding-agent/test/agent-session-mcp-discovery.test.ts @@ -123,8 +123,8 @@ describe("AgentSession MCP discovery", () => { }); sessions.push(session); - const firstIndex = session.getDiscoverableMCPSearchIndex(); - const secondIndex = session.getDiscoverableMCPSearchIndex(); + const firstIndex = session.getDiscoverableToolSearchIndex(); + const secondIndex = session.getDiscoverableToolSearchIndex(); expect(secondIndex).toBe(firstIndex); expect(firstIndex.documents.map(document => document.tool.name)).toEqual(["mcp__docs_search"]); @@ -132,7 +132,7 @@ describe("AgentSession MCP discovery", () => { createMcpCustomTool("mcp__pager_list", "pager", "list", "List pager alerts", ["service"]), ]); - const refreshedIndex = session.getDiscoverableMCPSearchIndex(); + const refreshedIndex = session.getDiscoverableToolSearchIndex(); expect(refreshedIndex).not.toBe(firstIndex); expect(refreshedIndex.documents.map(document => document.tool.name)).toEqual(["mcp__pager_list"]); }); @@ -310,7 +310,7 @@ describe("AgentSession MCP discovery", () => { sessions.push(session); expect(session.getActiveToolNames()).toEqual(["read"]); - expect(session.getDiscoverableMCPTools().map(tool => tool.name)).toEqual([ + expect(session.getDiscoverableTools({ source: "mcp" }).map(tool => tool.name)).toEqual([ "mcp__docs_search", "mcp__slack_send_message", ]); @@ -816,66 +816,6 @@ describe("AgentSession MCP discovery", () => { expect(session.getActiveToolNames()).toEqual(["read"]); expect(session.systemPrompt).toEqual(["tools:read"]); }); - - // ── Findings #2: legacy MCP discovery shapes ─────────────────────────────── - it("getDiscoverableMCPTools returns the legacy MCP shape with `description` populated", () => { - const readTool = createBasicTool("read", "Read"); - const docsSearchTool = createMcpTool("mcp__docs_search", "docs", "search", "Search internal docs", ["query"]); - const toolRegistry = new Map([ - [readTool.name, readTool], - [docsSearchTool.name, docsSearchTool], - ]); - const agent = new Agent({ - initialState: { model: createModel(), systemPrompt: ["initial"], tools: [readTool], messages: [] }, - }); - const session = new AgentSession({ - agent, - sessionManager: SessionManager.inMemory(), - settings: Settings.isolated({ "mcp.discoveryMode": true }), - modelRegistry: {} as never, - toolRegistry, - mcpDiscoveryEnabled: true, - rebuildSystemPrompt: async toolNames => ({ systemPrompt: [`tools:${toolNames.join(",")}`] }), - }); - sessions.push(session); - - const discoverable = session.getDiscoverableMCPTools(); - expect(discoverable).toHaveLength(1); - const entry = discoverable[0]!; - expect(entry.name).toBe("mcp__docs_search"); - expect(entry.description).toBe("Search internal docs"); - // Legacy shape must NOT carry `summary` — back-compat callers expect `description`. - expect((entry as { summary?: string }).summary).toBeUndefined(); - }); - - it("getDiscoverableMCPSearchIndex documents expose tool.description (legacy shape)", () => { - const readTool = createBasicTool("read", "Read"); - const docsSearchTool = createMcpTool("mcp__docs_search", "docs", "search", "Search internal docs", ["query"]); - const toolRegistry = new Map([ - [readTool.name, readTool], - [docsSearchTool.name, docsSearchTool], - ]); - const agent = new Agent({ - initialState: { model: createModel(), systemPrompt: ["initial"], tools: [readTool], messages: [] }, - }); - const session = new AgentSession({ - agent, - sessionManager: SessionManager.inMemory(), - settings: Settings.isolated({ "mcp.discoveryMode": true }), - modelRegistry: {} as never, - toolRegistry, - mcpDiscoveryEnabled: true, - rebuildSystemPrompt: async toolNames => ({ systemPrompt: [`tools:${toolNames.join(",")}`] }), - }); - sessions.push(session); - - const index = session.getDiscoverableMCPSearchIndex(); - expect(index.documents).toHaveLength(1); - const doc = index.documents[0]!; - expect(doc.tool.name).toBe("mcp__docs_search"); - expect(doc.tool.description).toBe("Search internal docs"); - }); - // ── Findings #3: discovery index is invalidated on active-tool changes ───── it("setActiveToolsByName invalidates the generic discoverable tool search index", async () => { const readTool = createBasicTool("read", "Read"); diff --git a/packages/coding-agent/test/agent-session-message-pipeline.test.ts b/packages/coding-agent/test/agent-session-message-pipeline.test.ts index a1e7fdf00..8a6fd595c 100644 --- a/packages/coding-agent/test/agent-session-message-pipeline.test.ts +++ b/packages/coding-agent/test/agent-session-message-pipeline.test.ts @@ -1,9 +1,17 @@ import { afterEach, describe, expect, it, vi } from "bun:test"; import { Agent, type AgentMessage } from "@oh-my-pi/pi-agent-core"; -import type { Message, SimpleStreamOptions } from "@oh-my-pi/pi-ai"; +import { + clearCustomApis, + type Message, + type Model, + registerCustomApi, + type SimpleStreamOptions, +} from "@oh-my-pi/pi-ai"; +import { AssistantMessageEventStream } from "@oh-my-pi/pi-ai/utils/event-stream"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { AgentSession, type AgentSessionEvent } from "@oh-my-pi/pi-coding-agent/session/agent-session"; import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager"; +import { createAssistantMessage } from "./helpers/agent-session-setup"; function createAgent(): Agent { return new Agent({ @@ -20,6 +28,7 @@ describe("AgentSession message pipeline", () => { afterEach(async () => { vi.restoreAllMocks(); + clearCustomApis(); for (const session of sessions.splice(0)) { await session.dispose(); } @@ -90,6 +99,58 @@ describe("AgentSession message pipeline", () => { expect(requestOnPayload).toHaveBeenCalledWith({ original: true, session: true }, undefined); expect(result).toEqual({ original: true, session: true }); }); + it("keeps ephemeral side-channel cache key separate from provider routing", async () => { + const api = "test-ephemeral-side-channel"; + let capturedOptions: SimpleStreamOptions | undefined; + registerCustomApi(api, (_model, _context, options) => { + capturedOptions = options; + const stream = new AssistantMessageEventStream(); + queueMicrotask(() => { + const message = createAssistantMessage("Answer"); + stream.push({ type: "text_delta", contentIndex: 0, delta: "Answer", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + }); + return stream; + }); + + const model = { + id: "side-model", + name: "Side Model", + api, + provider: "test-provider", + baseUrl: "", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 4096, + maxTokens: 1024, + } satisfies Model; + const session = new AgentSession({ + agent: new Agent({ + initialState: { + model, + systemPrompt: ["system prompt"], + messages: [], + tools: [], + }, + }), + sessionManager: SessionManager.inMemory(), + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry: { + getApiKey: vi.fn(async () => "key"), + } as never, + }); + sessions.push(session); + const cacheSessionId = session.sessionId; + + const result = await session.runEphemeralTurn({ promptText: "Question?" }); + + expect(result.replyText).toBe("Answer"); + expect(capturedOptions?.promptCacheKey).toBe(cacheSessionId); + expect(capturedOptions?.sessionId).toStartWith(`${cacheSessionId}:side:`); + expect(capturedOptions?.sessionId).not.toBe(cacheSessionId); + expect(capturedOptions?.preferWebsockets).toBe(false); + }); it("records raw SSE diagnostics into the session buffer before request hooks", async () => { const requestOnSseEvent = vi.fn(); diff --git a/packages/coding-agent/test/agent-session-python-cleanup.test.ts b/packages/coding-agent/test/agent-session-python-cleanup.test.ts index 126d8051b..d47e2382d 100644 --- a/packages/coding-agent/test/agent-session-python-cleanup.test.ts +++ b/packages/coding-agent/test/agent-session-python-cleanup.test.ts @@ -681,24 +681,30 @@ describe("AgentSession python cleanup", () => { expect(executeSpy).not.toHaveBeenCalled(); }); - it("aborts every active Python execution owned by the session during dispose", async () => { + it("aborts every active concurrent Python execution owned by the session during dispose", async () => { const { tempDir, cwd } = createTempProject(); tempDirs.push(tempDir); const kernel = new FakeKernel(); const blockedExecution = Promise.withResolvers(); - const blockedExecutionStarted = Promise.withResolvers(); - kernel.blockedCode = "print('first')"; + const bothStarted = Promise.withResolvers(); + let starts = 0; + kernel.blockedCode = "print('blocked')"; kernel.blockedExecution = blockedExecution.promise; - kernel.blockedExecutionStarted = () => blockedExecutionStarted.resolve(); + kernel.blockedExecutionStarted = () => { + starts += 1; + if (starts >= 2) bothStarted.resolve(); + }; vi.spyOn(pythonKernel, "checkPythonKernelAvailability").mockResolvedValue({ ok: true }); vi.spyOn(pythonKernel.PythonKernel, "start").mockResolvedValue(kernel as unknown as PythonKernelInstance); const session = await createSession(tempDir, cwd); - const firstExecution = session.executePython("print('first')"); - await blockedExecutionStarted.promise; - const secondExecution = session.executePython("print('second')"); + // Two concurrent blocked executions on the shared kernel session: both must + // be tracked when dispose runs so abortEval cancels every signal. + const firstExecution = session.executePython("print('blocked')"); + const secondExecution = session.executePython("print('blocked')"); + await bothStarted.promise; const sleepSpy = mockPositiveSleepsImmediate(); await session.dispose(); @@ -707,7 +713,7 @@ describe("AgentSession python cleanup", () => { expect(firstResult.cancelled).toBe(true); expect(secondResult.cancelled).toBe(true); - expect(kernel.executeCalls).toEqual(["print('first')"]); + expect(kernel.executeCalls).toEqual(["print('blocked')", "print('blocked')"]); expect(kernel.shutdownCalls).toBe(1); }); }); diff --git a/packages/coding-agent/test/agent-session-resolve-reminder.test.ts b/packages/coding-agent/test/agent-session-resolve-reminder.test.ts index c23e80050..b47b35d91 100644 --- a/packages/coding-agent/test/agent-session-resolve-reminder.test.ts +++ b/packages/coding-agent/test/agent-session-resolve-reminder.test.ts @@ -4,7 +4,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { Agent } from "@oh-my-pi/pi-agent-core"; import { getBundledModel } from "@oh-my-pi/pi-ai"; -import { createMockModel, type MockModelHandle } from "@oh-my-pi/pi-ai/providers/mock"; +import { createMockModel, type MockModel } from "@oh-my-pi/pi-ai/providers/mock"; import { ModelRegistry } from "@oh-my-pi/pi-coding-agent/config/model-registry"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { AgentSession } from "@oh-my-pi/pi-coding-agent/session/agent-session"; @@ -18,7 +18,7 @@ import { Snowflake } from "@oh-my-pi/pi-utils"; describe("AgentSession resolve reminder", () => { let session: AgentSession; let tempDir: string; - let mock: MockModelHandle; + let mock: MockModel; let authStorage: AuthStorage | undefined; beforeEach(async () => { diff --git a/packages/coding-agent/test/core/hashline.test.ts b/packages/coding-agent/test/core/hashline.test.ts index 0cdf84fbd..a40bf6cef 100644 --- a/packages/coding-agent/test/core/hashline.test.ts +++ b/packages/coding-agent/test/core/hashline.test.ts @@ -6,18 +6,15 @@ import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config import { applyHashlineEdits, buildCompactHashlineDiffPreview, - computeLineHash, + computeFileHash, type ExecuteHashlineSingleOptions, executeHashlineSingle, FileReadCache, generateDiffString, getFileReadCache, HashlineMismatchError, - HL_BODY_SEP, - HL_BODY_SEP_RE_RAW, hashlineEditParamsSchema, parseHashline, - parseHashlineWithWarnings, splitHashlineInput, splitHashlineInputs, tryRecoverHashlineWithCache, @@ -30,28 +27,27 @@ beforeAll(async () => { }); const pl = (text: string): string => text; -const outputSep = HL_BODY_SEP; -const outputSepRe = HL_BODY_SEP_RE_RAW; +const outputSep = ":"; +const outputSepRe = ":"; -function tag(line: number, content: string): string { - return `${line}${computeLineHash(line, content)}`; +function tag(line: number, _content: string): string { + return `${line}`; +} + +function header(filePath: string, content: string): string { + return `¶${filePath}#${computeFileHash(content)}`; } function sameLineRange(anchor: string): string { return `${anchor}-${anchor}`; } -function mistag(line: number, content: string): string { - const hash = computeLineHash(line, content); - return `${line}${hash === "zz" ? "yy" : "zz"}`; -} - function applyDiff(content: string, diff: string): string { - return applyHashlineEdits(content, parseHashline(diff)).lines; + return applyHashlineEdits(content, parseHashline(diff).edits).lines; } function applyDiffWithPureInsertAutoDrop(content: string, diff: string): string { - return applyHashlineEdits(content, parseHashline(diff), { autoDropPureInsertDuplicates: true }).lines; + return applyHashlineEdits(content, parseHashline(diff).edits, { autoDropPureInsertDuplicates: true }).lines; } async function withTempDir(fn: (tempDir: string) => Promise): Promise { @@ -92,53 +88,44 @@ describe("hashline parser — suffix-op syntax", () => { const content = "aaa\nbbb\nccc"; it("inserts payload before/after a Lid, and at BOF/EOF", () => { - const diff = [ - `${tag(2, "bbb")}↑`, - pl("before b"), - `${tag(2, "bbb")}↓`, - pl("after b"), - "BOF↓", - pl("top"), - "EOF↓", - pl("tail"), - ].join("\n"); + const diff = [`${tag(2, "bbb")}↑before b`, `${tag(2, "bbb")}↓after b`, "BOF↓top", "EOF↓tail"].join("\n"); expect(applyDiff(content, diff)).toBe("top\naaa\nbefore b\nbbb\nafter b\nccc\ntail"); }); it("inserts after the final line via `ANCHOR↓` instead of falling off the file", () => { - const diff = [`${tag(3, "ccc")}↓`, pl("tail")].join("\n"); + const diff = `${tag(3, "ccc")}↓tail`; expect(applyDiff(content, diff)).toBe("aaa\nbbb\nccc\ntail"); }); - it("deletes one line or an inclusive range when `A-B→` has no payload", () => { - expect(applyDiff(content, `${sameLineRange(tag(2, "bbb"))}→`)).toBe("aaa\nccc"); - expect(applyDiff(content, `${tag(2, "bbb")}-${tag(3, "ccc")}→`)).toBe("aaa"); + it("deletes one line or an inclusive range with `!`", () => { + expect(applyDiff(content, `${sameLineRange(tag(2, "bbb"))}!`)).toBe("aaa\nccc"); + expect(applyDiff(content, `${tag(2, "bbb")}-${tag(3, "ccc")}!`)).toBe("aaa"); }); - it("blanks a line in place with an explicit empty payload line", () => { - const diff = `${sameLineRange(tag(2, "bbb"))}→\n\n`; - expect(applyDiff(content, diff)).toBe("aaa\n\nccc"); + it("blanks a line in place with `A:` when given an explicit empty payload", () => { + const explicit = `${sameLineRange(tag(2, "bbb"))}:`; + expect(applyDiff(content, explicit)).toBe("aaa\n\nccc"); }); it("replaces one line or an inclusive range with payload lines", () => { - const single = [`${tag(2, "bbb")}→`, pl("BBB")].join("\n"); + const single = `${tag(2, "bbb")}:BBB`; expect(applyDiff(content, single)).toBe("aaa\nBBB\nccc"); - const range = [`${tag(2, "bbb")}-${tag(3, "ccc")}→`, pl("BBB"), pl("CCC")].join("\n"); + const range = [`${tag(2, "bbb")}-${tag(3, "ccc")}:BBB`, pl("CCC")].join("\n"); expect(applyDiff(content, range)).toBe("aaa\nBBB\nCCC"); }); it("treats single-anchor replace sugar as equivalent to an explicit one-line range", () => { const anchor = tag(2, "bbb"); - expect(parseHashline(`${anchor}→\nBBB`)).toEqual(parseHashline(`${anchor}-${anchor}→\nBBB`)); - expect(applyDiff(content, `${anchor}→\nBBB`)).toBe(applyDiff(content, `${anchor}-${anchor}→\nBBB`)); + expect(parseHashline(`${anchor}:\nBBB`).edits).toEqual(parseHashline(`${anchor}-${anchor}:\nBBB`).edits); + expect(applyDiff(content, `${anchor}:\nBBB`)).toBe(applyDiff(content, `${anchor}-${anchor}:\nBBB`)); }); it("accepts an inline payload on the op line as the first/only payload line", () => { const anchor = tag(2, "bbb"); expect(applyDiff(content, `${anchor}↓NEW`)).toBe("aaa\nbbb\nNEW\nccc"); expect(applyDiff(content, `${anchor}↑NEW`)).toBe("aaa\nNEW\nbbb\nccc"); - expect(applyDiff(content, `${anchor}→NEW`)).toBe("aaa\nNEW\nccc"); + expect(applyDiff(content, `${anchor}:NEW`)).toBe("aaa\nNEW\nccc"); }); it("combines an inline payload with subsequent payload lines on insert ops", () => { @@ -149,7 +136,7 @@ describe("hashline parser — suffix-op syntax", () => { it("combines an inline payload with subsequent payload lines on the replace op", () => { const anchor = tag(2, "bbb"); - const diff = [`${anchor}→FIRST`, pl("SECOND")].join("\n"); + const diff = [`${anchor}:FIRST`, pl("SECOND")].join("\n"); expect(applyDiff(content, diff)).toBe("aaa\nFIRST\nSECOND\nccc"); }); @@ -162,28 +149,28 @@ describe("hashline parser — suffix-op syntax", () => { it("auto-absorbs duplicated multiline prefix boundaries during replacement", () => { const source = ["// one", "// two", "old();"].join("\n"); - const diff = [`${sameLineRange(tag(3, "old();"))}→`, pl("// one"), pl("// two"), pl("new();")].join("\n"); + const diff = [`${sameLineRange(tag(3, "old();"))}:// one`, pl("// two"), pl("new();")].join("\n"); expect(applyDiff(source, diff)).toBe(["// one", "// two", "new();"].join("\n")); }); it("auto-absorbs duplicated multiline suffix boundaries during replacement", () => { const source = ["old();", "// one", "// two"].join("\n"); - const diff = [`${sameLineRange(tag(1, "old();"))}→`, pl("new();"), pl("// one"), pl("// two")].join("\n"); + const diff = [`${sameLineRange(tag(1, "old();"))}:new();`, pl("// one"), pl("// two")].join("\n"); expect(applyDiff(source, diff)).toBe(["new();", "// one", "// two"].join("\n")); }); it("auto-absorbs a duplicated single structural suffix during replacement", () => { const source = ["old();", "};"].join("\n"); - const diff = [`${sameLineRange(tag(1, "old();"))}→`, pl("new();"), pl("};")].join("\n"); + const diff = [`${sameLineRange(tag(1, "old();"))}:new();`, pl("};")].join("\n"); expect(applyDiff(source, diff)).toBe(["new();", "};"].join("\n")); }); it("auto-absorbs a duplicated single structural prefix during replacement", () => { const source = ["};", "old();"].join("\n"); - const diff = [`${sameLineRange(tag(2, "old();"))}→`, pl("};"), pl("new();")].join("\n"); + const diff = [`${sameLineRange(tag(2, "old();"))}:};`, pl("new();")].join("\n"); expect(applyDiff(source, diff)).toBe(["};", "new();"].join("\n")); }); @@ -193,14 +180,14 @@ describe("hashline parser — suffix-op syntax", () => { // `}` is a legitimate part of the new block, not a duplicate of the file's // existing `}`. The single-line structural absorb must NOT fire here. const source = ["old();", "}"].join("\n"); - const diff = [`${sameLineRange(tag(1, "old();"))}→`, pl("if ok {"), pl("}")].join("\n"); + const diff = [`${sameLineRange(tag(1, "old();"))}:if ok {`, pl("}")].join("\n"); expect(applyDiff(source, diff)).toBe(["if ok {", "}", "}"].join("\n")); }); it("does not auto-absorb a single duplicated boundary line", () => { const source = ["keep", "old();"].join("\n"); - const diff = [`${sameLineRange(tag(2, "old();"))}→`, pl("keep"), pl("new();")].join("\n"); + const diff = [`${sameLineRange(tag(2, "old();"))}:keep`, pl("new();")].join("\n"); expect(applyDiff(source, diff)).toBe(["keep", "keep", "new();"].join("\n")); }); @@ -210,23 +197,16 @@ describe("hashline parser — suffix-op syntax", () => { // is also the anchor of a separate insert. Absorbing it would silently // steal that anchor and turn the insert into a replacement. const source = ["A", "B", "X", "Y", "Z"].join("\n"); - const diff = [ - `${tag(1, "A")}-${tag(2, "B")}→`, - pl("alpha"), - pl("X"), - pl("Y"), - `${tag(4, "Y")}↑`, - pl("extra"), - ].join("\n"); + const diff = [`${tag(1, "A")}-${tag(2, "B")}:alpha`, pl("X"), pl("Y"), `${tag(4, "Y")}↑extra`].join("\n"); expect(applyDiff(source, diff)).toBe(["alpha", "X", "Y", "X", "extra", "Y", "Z"].join("\n")); }); it("surfaces a warning when boundary duplicates are auto-absorbed", () => { const source = ["// one", "// two", "old();"].join("\n"); - const diff = [`${sameLineRange(tag(3, "old();"))}→`, pl("// one"), pl("// two"), pl("new();")].join("\n"); + const diff = [`${sameLineRange(tag(3, "old();"))}:// one`, pl("// two"), pl("new();")].join("\n"); - const result = applyHashlineEdits(source, parseHashline(diff)); + const result = applyHashlineEdits(source, parseHashline(diff).edits); expect(result.lines).toBe(["// one", "// two", "new();"].join("\n")); expect(result.warnings).toBeDefined(); expect(result.warnings).toEqual( @@ -239,41 +219,41 @@ describe("hashline parser — suffix-op syntax", () => { // `autoDropPureInsertDuplicates` opt-in, unlike the single-line // structural absorb covered by the test below. const source = ["aaa", "bbb", "ccc"].join("\n"); - const diff = [`${tag(2, "bbb")}↓`, pl("aaa"), pl("bbb"), pl("NEW")].join("\n"); + const diff = [`${tag(2, "bbb")}↓aaa`, pl("bbb"), pl("NEW")].join("\n"); expect(applyDiff(source, diff)).toBe("aaa\nbbb\naaa\nbbb\nNEW\nccc"); }); it("auto-drops a duplicated single structural suffix for pure insert by default", () => { const source = ["if ok {", " keep();", " }"].join("\n"); - const diff = [`${tag(3, " }")}↑`, pl(" added();"), pl(" }")].join("\n"); + const diff = [`${tag(3, " }")}↑ added();`, pl(" }")].join("\n"); expect(applyDiff(source, diff)).toBe(["if ok {", " keep();", " added();", " }"].join("\n")); }); it("auto-drops a duplicated single structural prefix for pure insert by default", () => { const source = [" });", "next();"].join("\n"); - const diff = [`${tag(1, " });")}↓`, pl(" });"), pl("added();")].join("\n"); + const diff = [`${tag(1, " });")}↓ });`, pl("added();")].join("\n"); expect(applyDiff(source, diff)).toBe([" });", "added();", "next();"].join("\n")); }); it("preserves an intentional non-structural anchor duplicate for `ANCHOR↓` by default", () => { const source = ["aaa", "bbb", "ccc"].join("\n"); - const diff = [`${tag(2, "bbb")}↓`, pl("bbb"), pl("NEW")].join("\n"); + const diff = [`${tag(2, "bbb")}↓bbb`, pl("NEW")].join("\n"); expect(applyDiff(source, diff)).toBe("aaa\nbbb\nbbb\nNEW\nccc"); }); it("preserves an intentional non-structural anchor duplicate for `ANCHOR↑` by default", () => { const source = ["aaa", "bbb", "ccc"].join("\n"); - const diff = [`${tag(2, "bbb")}↑`, pl("NEW"), pl("bbb")].join("\n"); + const diff = [`${tag(2, "bbb")}↑NEW`, pl("bbb")].join("\n"); expect(applyDiff(source, diff)).toBe("aaa\nNEW\nbbb\nbbb\nccc"); }); it("does not drop a single structural pure-insert suffix when it preserves balance", () => { const source = ["if outer {", "}"].join("\n"); - const diff = [`${tag(2, "}")}↑`, pl("if inner {"), pl("}")].join("\n"); + const diff = [`${tag(2, "}")}↑if inner {`, pl("}")].join("\n"); expect(applyDiff(source, diff)).toBe(["if outer {", "if inner {", "}", "}"].join("\n")); }); @@ -282,7 +262,7 @@ describe("hashline parser — suffix-op syntax", () => { // Payload echoes the two file lines AT/ABOVE the insertion point // (aaa, bbb), then adds NEW. The leading echo is absorbed. const source = ["aaa", "bbb", "ccc"].join("\n"); - const diff = [`${tag(2, "bbb")}↓`, pl("aaa"), pl("bbb"), pl("NEW")].join("\n"); + const diff = [`${tag(2, "bbb")}↓aaa`, pl("bbb"), pl("NEW")].join("\n"); expect(applyDiffWithPureInsertAutoDrop(source, diff)).toBe("aaa\nbbb\nNEW\nccc"); }); @@ -290,7 +270,7 @@ describe("hashline parser — suffix-op syntax", () => { // Payload wraps NEW with context above (aaa, bbb) AND below (ccc, ddd). // Both ends should be absorbed, leaving only NEW inserted after bbb. const source = ["aaa", "bbb", "ccc", "ddd"].join("\n"); - const diff = [`${tag(2, "bbb")}↓`, pl("aaa"), pl("bbb"), pl("NEW"), pl("ccc"), pl("ddd")].join("\n"); + const diff = [`${tag(2, "bbb")}↓aaa`, pl("bbb"), pl("NEW"), pl("ccc"), pl("ddd")].join("\n"); expect(applyDiffWithPureInsertAutoDrop(source, diff)).toBe("aaa\nbbb\nNEW\nccc\nddd"); }); @@ -298,35 +278,35 @@ describe("hashline parser — suffix-op syntax", () => { // Insert before line 3 ("ccc"). Trailing payload echoes the anchor and the // line after it. Drop the trailing duplicates. const source = ["aaa", "bbb", "ccc", "ddd"].join("\n"); - const diff = [`${tag(3, "ccc")}↑`, pl("NEW"), pl("ccc"), pl("ddd")].join("\n"); + const diff = [`${tag(3, "ccc")}↑NEW`, pl("ccc"), pl("ddd")].join("\n"); expect(applyDiffWithPureInsertAutoDrop(source, diff)).toBe("aaa\nbbb\nNEW\nccc\nddd"); }); it("auto-absorbs duplicated leading payload at EOF insert", () => { const source = ["aaa", "bbb", "ccc"].join("\n"); // `EOF↓` payload echoes the last two file lines, then adds NEW. - const diff = ["EOF↓", pl("bbb"), pl("ccc"), pl("NEW")].join("\n"); + const diff = ["EOF↓bbb", pl("ccc"), pl("NEW")].join("\n"); expect(applyDiffWithPureInsertAutoDrop(source, diff)).toBe("aaa\nbbb\nccc\nNEW"); }); it("auto-absorbs duplicated trailing payload at BOF insert", () => { const source = ["aaa", "bbb", "ccc"].join("\n"); // `BOF↑` payload prepends NEW but trails with the first two file lines. - const diff = ["BOF↑", pl("NEW"), pl("aaa"), pl("bbb")].join("\n"); + const diff = ["BOF↑NEW", pl("aaa"), pl("bbb")].join("\n"); expect(applyDiffWithPureInsertAutoDrop(source, diff)).toBe("NEW\naaa\nbbb\nccc"); }); it("auto-drops a single duplicated anchor line in a pure insert when generic duplicate absorption is enabled", () => { const source = ["aaa", "bbb", "ccc"].join("\n"); - const diff = [`${tag(2, "bbb")}↓`, pl("bbb"), pl("NEW")].join("\n"); + const diff = [`${tag(2, "bbb")}↓bbb`, pl("NEW")].join("\n"); expect(applyDiffWithPureInsertAutoDrop(source, diff)).toBe("aaa\nbbb\nNEW\nccc"); }); it("surfaces a warning when pure-insert duplicates are auto-dropped", () => { const source = ["aaa", "bbb", "ccc"].join("\n"); - const diff = [`${tag(2, "bbb")}↓`, pl("aaa"), pl("bbb"), pl("NEW")].join("\n"); - const result = applyHashlineEdits(source, parseHashline(diff), { autoDropPureInsertDuplicates: true }); + const diff = [`${tag(2, "bbb")}↓aaa`, pl("bbb"), pl("NEW")].join("\n"); + const result = applyHashlineEdits(source, parseHashline(diff).edits, { autoDropPureInsertDuplicates: true }); expect(result.lines).toBe("aaa\nbbb\nNEW\nccc"); expect(result.warnings).toBeDefined(); expect(result.warnings).toEqual( @@ -335,137 +315,118 @@ describe("hashline parser — suffix-op syntax", () => { }); it("preserves payload text exactly", () => { - const diff = [ - `${sameLineRange(tag(2, "bbb"))}→`, - pl(""), - pl("# not a header"), - pl("+ not an op"), - pl(" spaced"), - ].join("\n"); + const diff = [`${sameLineRange(tag(2, "bbb"))}:`, pl("# not a header"), pl("+ not an op"), pl(" spaced")].join( + "\n", + ); expect(applyDiff(content, diff)).toBe("aaa\n\n# not a header\n+ not an op\n spaced\nccc"); }); it("treats blank lines inside a payload run as empty payload lines", () => { // Truly blank lines inside an active payload run are verbatim empty // payload lines as long as more payload follows. - const diff = [`${sameLineRange(tag(2, "bbb"))}→`, pl("first"), "", "", pl("after")].join("\n"); + const diff = [`${sameLineRange(tag(2, "bbb"))}:first`, "", "", pl("after")].join("\n"); expect(applyDiff(content, diff)).toBe("aaa\nfirst\n\n\nafter\nccc"); }); - it("treats blank lines before the next op as payload", () => { - const diff = [ - `${sameLineRange(tag(1, "aaa"))}→`, - pl("AAA"), - "", - "", - `${sameLineRange(tag(3, "ccc"))}→`, - pl("CCC"), - ].join("\n"); - expect(applyDiff(content, diff)).toBe("AAA\n\n\nbbb\nCCC"); + it("drops blank lines between ops (separator, not payload)", () => { + // Blank lines immediately before a next op are visual separators, not + // payload. This prevents agents from silently inflating a payload and + // shifting downstream line numbers. + const diff = [`${sameLineRange(tag(1, "aaa"))}:AAA`, "", "", `${sameLineRange(tag(3, "ccc"))}:CCC`].join("\n"); + expect(applyDiff(content, diff)).toBe("AAA\nbbb\nCCC"); }); - it("rejects missing payloads and orphan payload lines", () => { - expect(() => parseHashline(`${tag(1, "aaa")}↓`)).toThrow(/require at least one/); - expect(() => parseHashline(pl("orphan"))).toThrow(/payload line has no preceding/); - }); - - it("leniently treats a bare blank line after ↑ / ↓ as an empty payload", () => { - const hash = computeLineHash(5, "aaa"); - const anchor = { line: 5, hash }; - expect(parseHashline(`${tag(5, "aaa")}↑\n\n`)).toEqual([ - { kind: "insert", cursor: { kind: "before_anchor", anchor }, text: "", lineNum: 1, index: 0 }, + it("treats a bare insert op as inserting one empty line", () => { + // `LINE↑` / `LINE↓` with no payload default to one empty line (same as `LINE↑\n\n`). + const upAnchor = { line: 1 }; + expect(parseHashline(`${tag(1, "aaa")}↑`).edits).toEqual([ + { kind: "insert", cursor: { kind: "before_anchor", anchor: upAnchor }, text: "", lineNum: 1, index: 0 }, ]); - expect(parseHashline(`${tag(5, "aaa")}↓\n\n`)).toEqual([ - { kind: "insert", cursor: { kind: "after_anchor", anchor }, text: "", lineNum: 1, index: 0 }, + expect(parseHashline(`${tag(1, "aaa")}↓`).edits).toEqual([ + { kind: "insert", cursor: { kind: "after_anchor", anchor: upAnchor }, text: "", lineNum: 1, index: 0 }, ]); }); - it("rejects op sigils written in prefix position (legacy syntax)", () => { - expect(() => parseHashline(`↑${tag(1, "aaa")}\nold`)).toThrow(/unrecognized op/); - expect(() => parseHashline(`↓${tag(1, "aaa")}\nold`)).toThrow(/unrecognized op/); - expect(() => parseHashline(`→${tag(1, "aaa")}\nold`)).toThrow(/unrecognized op/); + it("rejects orphan payload lines with no preceding op", () => { + expect(() => parseHashline(pl("orphan")).edits).toThrow(/payload line has no preceding/); }); - it("rejects ranges with `..` separator (legacy syntax)", () => { + it("rejects op sigils written in prefix position", () => { + expect(() => parseHashline(`↑${tag(1, "aaa")}\nold`).edits).toThrow(/unrecognized op/); + expect(() => parseHashline(`↓${tag(1, "aaa")}\nold`).edits).toThrow(/unrecognized op/); + expect(() => parseHashline(`:${tag(1, "aaa")}\nold`).edits).toThrow(/unrecognized op/); + }); + + it("rejects ranges with `..` separator", () => { // `..` is no longer the range separator; the line is treated as orphan - // payload because `2yy..3yy→` does not match the new range pattern. - expect(() => parseHashline(`${tag(2, "bbb")}..${tag(3, "ccc")}→\nBBB`)).toThrow(/payload line has no preceding/); + // payload because `2..3:` does not match the new range pattern. + expect(() => parseHashline(`${tag(2, "bbb")}..${tag(3, "ccc")}:\nBBB`).edits).toThrow( + /payload line has no preceding/, + ); }); it("describes the new sigil shape on unknown-op lines", () => { - expect(() => parseHashline(`-${sameLineRange(tag(2, "bbb"))}`)).toThrow(/Use ANCHOR↑.*ANCHOR↓.*A-B→/); + expect(() => parseHashline(`-${sameLineRange(tag(2, "bbb"))}`).edits).toThrow( + /Use LINE↑.*LINE↓.*LINE: \/ A-B:.*LINE! \/ A-B!/, + ); }); - it("leniently tolerates a trailing `|TEXT` body on anchors copied verbatim from read output", () => { + it("treats `LINE:TEXT` copied from read output as a single-line replace", () => { const anchor = tag(2, "bbb"); - // Bare trailing `|`, full `|TEXT` body, and trailing decoration after a range. - expect(applyDiff(content, [`${anchor}|→`, pl("BBB")].join("\n"))).toBe("aaa\nBBB\nccc"); - expect(applyDiff(content, [`${anchor}|bbb→`, pl("BBB")].join("\n"))).toBe("aaa\nBBB\nccc"); - expect(applyDiff(content, [`${anchor}|bbb↑`, pl("X")].join("\n"))).toBe("aaa\nX\nbbb\nccc"); - expect(applyDiff(content, [`${anchor}|bbb↓`, pl("X")].join("\n"))).toBe("aaa\nbbb\nX\nccc"); - // Trailing `|TEXT` after the full range is also tolerated. - expect(applyDiff(content, `${anchor}-${tag(3, "ccc")}|ccc→`)).toBe("aaa"); + expect(applyDiff(content, `${anchor}:BBB`)).toBe("aaa\nBBB\nccc"); + expect(applyDiff(content, `${anchor}-${tag(3, "ccc")}:BBB`)).toBe("aaa\nBBB"); }); it("leniently strips `*`/`>` line-marker decoration from anchors", () => { const anchor = tag(2, "bbb"); - expect(applyDiff(content, [`*${anchor}→`, pl("BBB")].join("\n"))).toBe("aaa\nBBB\nccc"); - expect(applyDiff(content, [`>${anchor}↑`, pl("X")].join("\n"))).toBe("aaa\nX\nbbb\nccc"); + expect(applyDiff(content, `*${anchor}:BBB`)).toBe("aaa\nBBB\nccc"); + expect(applyDiff(content, `>${anchor}↑X`)).toBe("aaa\nX\nbbb\nccc"); }); - it("anchor paste decoration `|TEXT` before the op is cosmetic; real payload comes after", () => { - // `|bbb` between the anchor and the op is just paste decoration and is - // discarded. Payload must come inline after the op or on the next lines. + it("rejects arrow replace syntax as an unrecognized payload line", () => { + expect(() => parseHashline(`2→\nBBB`).edits).toThrow(/payload line has no preceding/); + expect(() => parseHashline(`2-3→\nBBB`).edits).toThrow(/payload line has no preceding/); + }); + + it("treats `LINE:TEXT` as replace syntax even when TEXT contains ↑ / ↓", () => { const anchor = tag(2, "bbb"); - const diff = [`${anchor}|bbb↓`, pl("X"), pl("Y")].join("\n"); - expect(applyDiff(content, diff)).toBe("aaa\nbbb\nX\nY\nccc"); - // No inline payload after the op and no follow-up: error. - expect(() => parseHashline(`${anchor}|bbb↓`)).toThrow(/require at least one/); + expect(applyDiff(content, `${anchor}:bbb↓`)).toBe("aaa\nbbb↓\nccc"); + expect(applyDiff(content, `${anchor}:bbb↑\nX`)).toBe("aaa\nbbb↑\nX\nccc"); }); - it("treats `|TEXT` after BOF/EOF as cosmetic decoration", () => { - expect(applyDiff(content, `BOF|head↓HEAD`)).toBe("HEAD\naaa\nbbb\nccc"); - expect(applyDiff(content, `EOF|tail↓TAIL`)).toBe("aaa\nbbb\nccc\nTAIL"); + it("uses inline payload for BOF/EOF inserts", () => { + expect(applyDiff(content, `BOF↓HEAD`)).toBe("HEAD\naaa\nbbb\nccc"); + expect(applyDiff(content, `EOF↓TAIL`)).toBe("aaa\nbbb\nccc\nTAIL"); + expect(() => parseHashline(`2!keep`).edits).toThrow( + /deletes only\. Payload is forbidden after !; use : to replace/, + ); }); }); -describe("hashline — stale anchors", () => { - it("throws HashlineMismatchError when a Lid hash no longer matches", () => { - const diff = [`${sameLineRange(mistag(2, "bbb"))}→`, pl("BBB")].join("\n"); - expect(() => applyDiff("aaa\nbbb\nccc", diff)).toThrow(HashlineMismatchError); +describe("hashline — file hash binding", () => { + it("rejects line-hash anchors as unrecognized payload lines", () => { + expect(() => parseHashline("2ab:\nBBB").edits).toThrow(/payload line has no preceding/); }); - it("rejects when an anchor's stored line shifted (no auto-rebase)", () => { - const stale = tag(2, "bbb"); - const diff = [`${sameLineRange(stale)}→`, pl("BBB")].join("\n"); - expect(() => applyDiff("aaa\nINSERTED\nbbb\nccc", diff)).toThrow(HashlineMismatchError); - }); - - it("rejects when the line hash matches a different nearby line", () => { - // Significant-content lines hash by content alone; identical content gives - // identical hashes, so an anchor pointing at a different line with the - // same hash must not be silently relocated. - const file = ["x = 1", "y = 2", "x = 1", "z = 3", "x = 1", "w = 4"].join("\n"); - const collidingHash = computeLineHash(1, "x = 1"); - // User points at line 4 (`z = 3`) with the colliding hash; without auto- - // rebase, this is a plain mismatch. - const diff = [`${sameLineRange(`4${collidingHash}`)}→`, pl("REPLACED")].join("\n"); - expect(() => applyDiff(file, diff)).toThrow(HashlineMismatchError); + it("applies line-number edits without per-anchor hash validation", () => { + const diff = `${sameLineRange(tag(2, "bbb"))}:BBB`; + expect(applyDiff("aaa\nbbb\nccc", diff)).toBe("aaa\nBBB\nccc"); }); }); describe("splitHashlineInput — ¶ headers", () => { - it("extracts path and diff body from ¶path header", () => { - const input = [`¶src/foo.ts`, `${sameLineRange(tag(2, "bbb"))}→`, pl("BBB")].join("\n"); + it("extracts path, file hash, and diff body from ¶path#hash header", () => { + const input = [`¶src/foo.ts#1a2b`, `${sameLineRange(tag(2, "bbb"))}:`, pl("BBB")].join("\n"); expect(splitHashlineInput(input)).toEqual({ path: "src/foo.ts", - diff: `${sameLineRange(tag(2, "bbb"))}→\n${pl("BBB")}`, + fileHash: "1a2b", + diff: `${sameLineRange(tag(2, "bbb"))}:\n${pl("BBB")}`, }); }); - it("strips leading blank lines and unquotes matching path quotes", () => { - expect(splitHashlineInput(`\n¶"foo bar.ts"\nBOF↓\n${pl("x")}`)).toEqual({ - path: "foo bar.ts", + it("strips leading blank lines", () => { + expect(splitHashlineInput(`\n¶foo.ts\nBOF↓\n${pl("x")}`)).toEqual({ + path: "foo.ts", diff: `BOF↓\n${pl("x")}`, }); }); @@ -514,7 +475,7 @@ describe("splitHashlineInput — ¶ headers", () => { describe("hashline executor", () => { it("creates a missing file with a file-scoped insert", async () => { await withTempDir(async tempDir => { - const input = `¶new.ts\nBOF↓\n${pl("export const x = 1;")}\n`; + const input = `¶new.ts\nBOF↓${pl("export const x = 1;")}\n`; const result = await executeHashlineSingle(hashlineExecuteOptions(tempDir, input)); expect(result.content[0]?.type === "text" ? result.content[0].text : "").toContain("new.ts:"); expect(await Bun.file(path.join(tempDir, "new.ts")).text()).toBe("export const x = 1;"); @@ -525,7 +486,7 @@ describe("hashline executor", () => { await withTempDir(async tempDir => { const filePath = path.join(tempDir, "a.ts"); const source = ["aaa", "bbb", "ccc"].join("\n"); - const input = `¶a.ts\n${tag(2, "bbb")}↓\n${pl("aaa")}\n${pl("bbb")}\n${pl("NEW")}\n`; + const input = `${header("a.ts", source)}\n${tag(2, "bbb")}↓${pl("aaa")}\n${pl("bbb")}\n${pl("NEW")}\n`; await Bun.write(filePath, source); await executeHashlineSingle(hashlineExecuteOptions(tempDir, input)); @@ -545,17 +506,16 @@ describe("hashline executor", () => { const bPath = path.join(tempDir, "b.ts"); await Bun.write(aPath, "aaa\n"); await Bun.write(bPath, "bbb\n"); + const bHeader = "¶b.ts#0000"; const input = [ - "¶a.ts", - `${sameLineRange(tag(1, "aaa"))}→`, - pl("AAA"), - "¶b.ts", - `${sameLineRange(mistag(1, "bbb"))}→`, - pl("BBB"), + header("a.ts", "aaa\n"), + `${sameLineRange(tag(1, "aaa"))}:AAA`, + bHeader, + `${sameLineRange(tag(1, "bbb"))}:BBB`, ].join("\n"); await expect(executeHashlineSingle(hashlineExecuteOptions(tempDir, input))).rejects.toThrow( - /anchor(s)? do(es)? not match the current file/, + /file changed between read and edit|file hashes to/, ); expect(await Bun.file(aPath).text()).toBe("aaa\n"); expect(await Bun.file(bPath).text()).toBe("bbb\n"); @@ -574,9 +534,8 @@ describe("hashline executor", () => { // A naive sequential apply reads the modified disk and fails anchor // validation outright. const input = [ - "¶a.ts", - `${sameLineRange(tag(2, "L2"))}→`, - pl("L2a"), + header("a.ts", `${original}\n`), + `${sameLineRange(tag(2, "L2"))}:L2a`, pl("L2b"), pl("L2c"), pl("L2d"), @@ -585,9 +544,8 @@ describe("hashline executor", () => { pl("L2g"), pl("L2h"), pl("L2i"), - "¶a.ts", - `${tag(8, "L8")}↓`, - pl("INSERTED"), + header("a.ts", `${original}\n`), + `${tag(8, "L8")}↓INSERTED`, ].join("\n"); await executeHashlineSingle(hashlineExecuteOptions(tempDir, input)); @@ -630,45 +588,40 @@ describe("hashlineEditParamsSchema — extra-field tolerance", () => { }); }); -describe("buildCompactHashlineDiffPreview — anchors track post-edit line numbers", () => { - it("emits hashes against the new file's line numbers for context after a range expansion", () => { +describe("buildCompactHashlineDiffPreview — line numbers track post-edit positions", () => { + it("emits context lines against the new file's line numbers after a range expansion", () => { const before = ["a1", "a2", "a3", "a4", "a5", "a6", "a7"].join("\n"); const after = ["a1", "a2", "a3", "X", "Y", "Z", "a5", "a6", "a7"].join("\n"); const { diff } = generateDiffString(before, after); const preview = buildCompactHashlineDiffPreview(diff); - // Walk the preview and verify every ` LINE+HASH${outputSep}content` line matches what + // Walk the preview and verify every ` LINE:content` line matches what // the file now has at that line number. const newFileLines = after.split("\n"); for (const line of preview.preview.split("\n")) { if (!line.startsWith(" ")) continue; // Skip context-elision markers ("...") which carry no real file content. if (line.endsWith(`${outputSep}...`)) continue; - const match = new RegExp(`^\\s(\\d+)([a-z]{2})${outputSepRe}(.*)$`).exec(line); + const match = new RegExp(`^\\s(\\d+)${outputSepRe}(.*)$`).exec(line); expect(match).not.toBeNull(); if (!match) continue; const lineNum = Number(match[1]); - const hash = match[2]; - const content = match[3]; + const content = match[2]; expect(newFileLines[lineNum - 1]).toBe(content); - expect(computeLineHash(lineNum, content)).toBe(hash); } }); - it("emits + lines with hashes against new line numbers and - lines with the placeholder", () => { + it("emits + and - lines with bare line numbers", () => { const before = "alpha\nbeta\ngamma\n"; const after = "alpha\nDELTA\nEPSILON\ngamma\n"; const { diff } = generateDiffString(before, after); const preview = buildCompactHashlineDiffPreview(diff); const additions = preview.preview.split("\n").filter(line => line.startsWith("+")); - expect(additions).toEqual([ - `+2${computeLineHash(2, "DELTA")}${outputSep}DELTA`, - `+3${computeLineHash(3, "EPSILON")}${outputSep}EPSILON`, - ]); + expect(additions).toEqual([`+2${outputSep}DELTA`, `+3${outputSep}EPSILON`]); const removals = preview.preview.split("\n").filter(line => line.startsWith("-")); - expect(removals).toEqual([`-2--${outputSep}beta`]); + expect(removals).toEqual([`-2${outputSep}beta`]); }); }); @@ -677,11 +630,15 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { await withTempDir(async tempDir => { const filePath = path.join(tempDir, "a.ts"); const v0Lines = ["L1", "L2", "L3", "L4", "L5", "L6", "L7", "L8"]; - await Bun.write(filePath, `${v0Lines.join("\n")}\n`); + const v0Text = `${v0Lines.join("\n")}\n`; + await Bun.write(filePath, v0Text); const session = makeHashlineSession(tempDir); // Simulate the read tool having shown V0 to the model in this session. - getFileReadCache(session).recordContiguous(filePath, 1, v0Lines); + getFileReadCache(session).recordContiguous(filePath, 1, v0Text.split("\n"), { + fullText: v0Text, + fileHash: computeFileHash(v0Text), + }); // External actor (linter, subagent, user) prepends 7 lines. Anchors // authored against V0 no longer match V1, so the model's edit cannot @@ -691,7 +648,7 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { await Bun.write(filePath, `${v1Lines.join("\n")}\n`); // Model authors anchor against V0 — line 2 is "L2" in V0. - const input = `¶a.ts\n${sameLineRange(tag(2, "L2"))}→\n${pl("L2-MODEL")}\n`; + const input = `${header("a.ts", v0Text)}\n${sameLineRange(tag(2, "L2"))}:${pl("L2-MODEL")}\n`; const result = await executeHashlineSingle(hashlineExecuteOptions(tempDir, input, undefined, session)); const finalLines = (await Bun.file(filePath).text()).replace(/\n$/, "").split("\n"); @@ -704,7 +661,7 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { expect(finalLines).toContain("L8"); const text = result.content[0]?.type === "text" ? result.content[0].text : ""; - expect(text).toMatch(/Recovered from stale anchors using a previous read snapshot/); + expect(text).toMatch(/Recovered from a stale file hash using a previous read snapshot/); }); }); @@ -712,17 +669,21 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { await withTempDir(async tempDir => { const filePath = path.join(tempDir, "a.ts"); const v0Lines = Array.from({ length: 10 }, (_, idx) => `L${idx + 1}`); - await Bun.write(filePath, `${v0Lines.join("\n")}\n`); + const v0Text = `${v0Lines.join("\n")}\n`; + await Bun.write(filePath, v0Text); const session = makeHashlineSession(tempDir); - // Cache only covers the first three lines — but the edit targets line 6. - getFileReadCache(session).recordContiguous(filePath, 1, v0Lines.slice(0, 3)); + // Cache only covers the first three lines — enough to retain the file hash + // but not enough to synthesize the requested pre-edit snapshot. + getFileReadCache(session).recordContiguous(filePath, 1, v0Lines.slice(0, 3), { + fileHash: computeFileHash(v0Text), + }); const v1Lines = [...v0Lines]; v1Lines[5] = "L6-CHANGED"; await Bun.write(filePath, `${v1Lines.join("\n")}\n`); - const input = `¶a.ts\n${sameLineRange(tag(6, "L6"))}→\n${pl("L6-MODEL")}\n`; + const input = `${header("a.ts", v0Text)}\n${sameLineRange(tag(6, "L6"))}:${pl("L6-MODEL")}\n`; await expect( executeHashlineSingle(hashlineExecuteOptions(tempDir, input, undefined, session)), ).rejects.toThrow(HashlineMismatchError); @@ -734,18 +695,23 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { it("returns null from tryRecoverHashlineWithCache when applyPatch cannot land", () => { const cache = new FileReadCache(); const fakePath = "/tmp/__hashline-recovery-applypatch__.ts"; - cache.recordContiguous(fakePath, 1, ["alpha", "beta", "gamma", "delta", "epsilon"]); + const snapshotText = "alpha\nbeta\ngamma\ndelta\nepsilon"; + cache.recordContiguous(fakePath, 1, snapshotText.split("\n"), { + fullText: snapshotText, + fileHash: computeFileHash(snapshotText), + }); // Live file is completely different — patch context cannot match even // with fuzz tolerance. const currentText = "totally\nunrelated\ncontent\nhere\nnow\n"; - const edits = parseHashline(`${sameLineRange(tag(2, "beta"))}→\n${pl("BETA-MODEL")}`); + const edits = parseHashline(`${sameLineRange(tag(2, "beta"))}:${pl("BETA-MODEL")}`).edits; const recovered = tryRecoverHashlineWithCache({ cache, absolutePath: fakePath, currentText, edits, + fileHash: computeFileHash(snapshotText), options: {}, }); expect(recovered).toBeNull(); @@ -764,15 +730,19 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { await withTempDir(async tempDir => { const filePath = path.join(tempDir, "a.ts"); const v0Lines = ["alpha", "beta", "gamma", "delta", "epsilon"]; - await Bun.write(filePath, `${v0Lines.join("\n")}\n`); + const v0Text = `${v0Lines.join("\n")}\n`; + await Bun.write(filePath, v0Text); const session = makeHashlineSession(tempDir); // Initial read populates the cache with V0. - getFileReadCache(session).recordContiguous(filePath, 1, v0Lines); + getFileReadCache(session).recordContiguous(filePath, 1, v0Text.split("\n"), { + fullText: v0Text, + fileHash: computeFileHash(v0Text), + }); - // First edit: change line 2 → BETA. After the write, the cache should + // First edit: change line 2 : BETA. After the write, the cache should // reflect V1 (post-edit), not V0. - const firstInput = `¶a.ts\n${sameLineRange(tag(2, "beta"))}→\n${pl("BETA")}\n`; + const firstInput = `${header("a.ts", v0Text)}\n${sameLineRange(tag(2, "beta"))}:${pl("BETA")}\n`; await executeHashlineSingle(hashlineExecuteOptions(tempDir, firstInput, undefined, session)); const v1Lines = ["alpha", "BETA", "gamma", "delta", "epsilon"]; expect(await Bun.file(filePath).text()).toBe(`${v1Lines.join("\n")}\n`); @@ -788,7 +758,7 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { const v2Lines = ["H1", "H2", "H3", "H4", "H5", "H6", "H7", ...v1Lines]; await Bun.write(filePath, `${v2Lines.join("\n")}\n`); - const secondInput = `¶a.ts\n${sameLineRange(tag(3, "gamma"))}→\n${pl("GAMMA")}\n`; + const secondInput = `${header("a.ts", `${v1Lines.join("\n")}\n`)}\n${sameLineRange(tag(3, "gamma"))}:${pl("GAMMA")}\n`; const result = await executeHashlineSingle(hashlineExecuteOptions(tempDir, secondInput, undefined, session)); const finalLines = (await Bun.file(filePath).text()).replace(/\n$/, "").split("\n"); @@ -797,9 +767,52 @@ describe("hashline — anchor-stale recovery via read snapshot cache", () => { expect(finalLines).toContain("GAMMA"); expect(finalLines).not.toContain("gamma"); const text = result.content[0]?.type === "text" ? result.content[0].text : ""; - expect(text).toMatch(/Recovered from stale anchors using a previous read snapshot/); + expect(text).toMatch(/Recovered from a stale file hash using a previous read snapshot/); }); }); + it("recovers from an older in-session snapshot even if the current file advanced again", () => { + const cache = new FileReadCache(); + const fakePath = "/tmp/__hashline-cache-ring-recovery__.ts"; + const v0Text = "L1\nL2\nL3\nL4\nL5\nL6\nL7\nL8\nL9\nL10\n"; + const v1Text = "L1\nL2-EDITED\nL3\nL4\nL5\nL6\nL7\nL8\nL9\nL10\n"; + const currentText = "L1\nL2-EDITED\nL3\nL4\nL5\nL6\nL7\nL8\nL9\nL10\nTRAILER\n"; + + cache.recordContiguous(fakePath, 1, v0Text.split("\n"), { + fullText: v0Text, + fileHash: computeFileHash(v0Text), + }); + cache.recordContiguous(fakePath, 1, v1Text.split("\n"), { + fullText: v1Text, + fileHash: computeFileHash(v1Text), + }); + + const recovered = tryRecoverHashlineWithCache({ + cache, + absolutePath: fakePath, + currentText, + fileHash: computeFileHash(v0Text), + edits: parseHashline(`10:L10-EDITED`).edits, + options: {}, + }); + + expect(recovered).not.toBeNull(); + expect(recovered?.lines).toContain("L10-EDITED"); + }); + + it("retains older file hashes in the per-path snapshot ring", () => { + const cache = new FileReadCache(); + const fakePath = "/tmp/__hashline-cache-ring__.ts"; + const versions = ["one\n", "two\n", "three\n"]; + for (const version of versions) { + cache.recordContiguous(fakePath, 1, version.split("\n"), { + fullText: version, + fileHash: computeFileHash(version), + }); + } + expect(cache.get(fakePath)?.fileHash).toBe(computeFileHash("three\n")); + expect(cache.getByHash(fakePath, computeFileHash("one\n"))?.fullText).toBe("one\n"); + expect(cache.getByHash(fakePath, computeFileHash("two\n"))?.fullText).toBe("two\n"); + }); it("drops a cached entry when newly recorded lines disagree on overlap", () => { const cache = new FileReadCache(); @@ -839,8 +852,8 @@ describe("hashline *** Abort recovery sentinel (harmony-leak mitigation)", () => const sentinel = "*** Abort"; it("parser breaks at *** Abort and surfaces a warning", () => { - const diff = [`${tag(1, "alpha")}↓`, pl("HELLO"), sentinel, `${tag(99, "junk")}↓`, pl("never")].join("\n"); - const { edits, warnings } = parseHashlineWithWarnings(diff); + const diff = [`${tag(1, "alpha")}↓HELLO`, sentinel, `${tag(99, "junk")}↓never`].join("\n"); + const { edits, warnings } = parseHashline(diff); expect(edits).toHaveLength(1); expect(edits[0]).toMatchObject({ kind: "insert", text: "HELLO" }); expect(warnings.length).toBeGreaterThan(0); @@ -849,8 +862,8 @@ describe("hashline *** Abort recovery sentinel (harmony-leak mitigation)", () => it("appended sentinel from harmony-leak truncation: ops above are preserved", () => { // Mirrors the exact shape harmony-leak emits inside a single section. - const diff = `${tag(1, "alpha")}↓\n${pl("KEPT")}\n*** Abort\n`; - const { edits, warnings } = parseHashlineWithWarnings(diff); + const diff = `${tag(1, "alpha")}↓${pl("KEPT")}\n*** Abort\n`; + const { edits, warnings } = parseHashline(diff); expect(edits).toHaveLength(1); expect(edits[0]).toMatchObject({ text: "KEPT" }); expect(warnings.length).toBeGreaterThan(0); @@ -873,8 +886,98 @@ describe("hashline *** Abort recovery sentinel (harmony-leak mitigation)", () => }); it("clean input without sentinel produces no warning", () => { - const diff = `${tag(1, "alpha")}↓\n${pl("PAYLOAD")}\n`; - const { warnings } = parseHashlineWithWarnings(diff); + const diff = `${tag(1, "alpha")}↓${pl("PAYLOAD")}\n`; + const { warnings } = parseHashline(diff); expect(warnings).toEqual([]); }); }); + +describe("hashline parser — bare ':' replaces with a single blank line", () => { + it("bare A: replaces the line with a single blank line", () => { + const text = "line1\nline2\nline3\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n2:\n`); + expect(applyDiff(text, diff)).toBe("line1\n\nline3\n"); + }); + + it("bare A-B: replaces the range with a single blank line", () => { + const text = "line1\nline2\nline3\nline4\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n2-3:\n`); + expect(applyDiff(text, diff)).toBe("line1\n\nline4\n"); + }); + + it("A: with inline body still works", () => { + const text = "line1\nline2\nline3\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n2:replacement\n`); + expect(applyDiff(text, diff)).toBe("line1\nreplacement\nline3\n"); + }); + + it("bare A↑ still inserts a blank line above", () => { + const text = "line1\nline2\nline3\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n2↑\n`); + expect(applyDiff(text, diff)).toBe("line1\n\nline2\nline3\n"); + }); + + it("bare A↓ still inserts a blank line below", () => { + const text = "line1\nline2\nline3\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n2↓\n`); + expect(applyDiff(text, diff)).toBe("line1\nline2\n\nline3\n"); + }); +}); + +describe("hashline apply — brace-delete soft warning", () => { + it("deleting a line with unbalanced brace emits a warning", () => { + const text = "if (x) {\n doThing();\n} else {\n doOther();\n}\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n3!\n`); + const result = applyHashlineEdits(text, parseHashline(diff).edits); + expect(result.warnings).toBeDefined(); + expect(result.warnings![0]).toContain("structural bracket/brace boundary"); + expect(result.warnings![0]).toContain("} else {"); + }); + + it("deleting a balanced line emits no warning", () => { + const text = "line1\nline2\nline3\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n2!\n`); + const result = applyHashlineEdits(text, parseHashline(diff).edits); + expect(result.warnings).toBeUndefined(); + }); + + it("replace operation that includes a brace line does NOT warn", () => { + const text = "if (x) {\n body\n}\n"; + const { diff } = splitHashlineInput(`${header("a.ts", text)}\n3:}\n`); + const result = applyHashlineEdits(text, parseHashline(diff).edits); + expect(result.warnings).toBeUndefined(); + }); +}); + +describe("hashline parser — blank line is a separator before next op", () => { + it("blank line between ops is NOT absorbed into previous payload", () => { + const text = "a\nb\nc\nd\ne\n"; + const ops = `${header("a.ts", text)}\n1:A\n\n3:C\n`; + const { diff } = splitHashlineInput(ops); + // Both replaces land on their target lines without inflating either payload. + expect(applyDiff(text, diff)).toBe("A\nb\nC\nd\ne\n"); + }); + + it("multiple blank lines between ops are also dropped", () => { + const text = "a\nb\nc\nd\ne\n"; + const ops = `${header("a.ts", text)}\n1:A\n\n\n\n3:C\n`; + const { diff } = splitHashlineInput(ops); + expect(applyDiff(text, diff)).toBe("A\nb\nC\nd\ne\n"); + }); + + it("blank-only payload before next op blanks the line", () => { + // Agent typed `2:` then a blank separator then `4:D`. Under bare-`A:` + // blank-replace semantics, `2:` blanks line 2 and `4:D` replaces line 4. + const text = "a\nb\nc\nd\ne\n"; + const ops = `${header("a.ts", text)}\n2:\n\n4:D\n`; + const { diff } = splitHashlineInput(ops); + expect(applyDiff(text, diff)).toBe("a\n\nc\nD\ne\n"); + }); + + it("blank line inside payload between two content lines is preserved", () => { + const text = "a\nb\nc\n"; + const ops = `${header("a.ts", text)}\n2:first\n\nsecond\n`; + const { diff } = splitHashlineInput(ops); + expect(applyDiff(text, diff)).toBe("a\nfirst\n\nsecond\nc\n"); + }); +}); diff --git a/packages/coding-agent/test/core/python-executor-owner-cleanup.test.ts b/packages/coding-agent/test/core/python-executor-owner-cleanup.test.ts index ca29d11cc..ece85d4a7 100644 --- a/packages/coding-agent/test/core/python-executor-owner-cleanup.test.ts +++ b/packages/coding-agent/test/core/python-executor-owner-cleanup.test.ts @@ -316,43 +316,6 @@ describe("python executor owner cleanup", () => { expect(unownedRetainedKernel.shutdown).toHaveBeenCalledTimes(1); }); - it("rejects a queued execute when its session is disposed before the slot runs", async () => { - const kernel = new FakeKernel(); - const executeHang = Promise.withResolvers(); - const executeStarted = Promise.withResolvers(); - kernel.execute = vi.fn(async () => { - executeStarted.resolve(); - return await executeHang.promise; - }); - vi.spyOn(pythonKernel, "checkPythonKernelAvailability").mockResolvedValue({ ok: true }); - const startSpy = vi.spyOn(PythonKernel, "start").mockResolvedValue(kernel as unknown as PythonKernelInstance); - - const first = executePython("first", { - cwd: "/tmp/dispose-queue-race", - sessionId: "dispose-queue-session", - kernelMode: "session", - }); - await executeStarted.promise; - - const queued = executePython("queued", { - cwd: "/tmp/dispose-queue-race", - sessionId: "dispose-queue-session", - kernelMode: "session", - }); - await flushMicrotasks(); - - await disposeAllKernelSessions(); - executeHang.resolve(OK_RESULT); - - const firstResult = await first; - const queuedResult = await queued; - - expect(firstResult.cancelled).toBe(false); - expect(queuedResult.cancelled).toBe(true); - expect(startSpy).toHaveBeenCalledTimes(1); - expect(kernel.execute).toHaveBeenCalledTimes(1); - expect(kernel.shutdown).toHaveBeenCalledTimes(1); - }); it("retains sessions whose kernel shutdown is not confirmed so a later dispose retries", async () => { const kernel = new FakeKernel(); diff --git a/packages/coding-agent/test/core/python-tool-bridge.test.ts b/packages/coding-agent/test/core/python-tool-bridge.test.ts index c905fe922..67d158356 100644 --- a/packages/coding-agent/test/core/python-tool-bridge.test.ts +++ b/packages/coding-agent/test/core/python-tool-bridge.test.ts @@ -58,10 +58,11 @@ describe("Python tool bridge HTTP server", () => { }); const session = makeSession(new Map([["read", readTool]])); const info = await ensurePyToolBridge(); - const unregister = registerPyToolBridge("test-session-1", { toolSession: session }); + const unregister = registerPyToolBridge("test-session-1", "run-1", { toolSession: session }); try { const res = await call(info, { session: "test-session-1", + run: "run-1", name: "read", args: { path: "foo.ts", _i: "py prelude" }, }); @@ -78,7 +79,7 @@ describe("Python tool bridge HTTP server", () => { it("returns ok=false when no session is registered for the given id", async () => { const info = await ensurePyToolBridge(); - const res = await call(info, { session: "missing", name: "read", args: {} }); + const res = await call(info, { session: "missing", run: "run-missing", name: "read", args: {} }); expect(res.status).toBe(200); const body = (await res.json()) as { ok: boolean; error?: string }; expect(body.ok).toBe(false); @@ -99,9 +100,9 @@ describe("Python tool bridge HTTP server", () => { }) as unknown as AgentTool, } as unknown as ToolSession; const info = await ensurePyToolBridge(); - const unregister = registerPyToolBridge("err-session", { toolSession: session }); + const unregister = registerPyToolBridge("err-session", "run-err", { toolSession: session }); try { - const res = await call(info, { session: "err-session", name: "boom", args: {} }); + const res = await call(info, { session: "err-session", run: "run-err", name: "boom", args: {} }); expect(res.status).toBe(200); const body = await res.json(); expect(body).toEqual({ ok: false, error: "kapow" }); @@ -112,7 +113,11 @@ describe("Python tool bridge HTTP server", () => { it("rejects requests with a bad bearer token", async () => { const info = await ensurePyToolBridge(); - const res = await call(info, { session: "anything", name: "read", args: {} }, { token: "wrong" }); + const res = await call( + info, + { session: "anything", run: "run-anything", name: "read", args: {} }, + { token: "wrong" }, + ); expect(res.status).toBe(403); }); @@ -130,13 +135,14 @@ describe("Python tool bridge HTTP server", () => { const session = makeSession(new Map([["read", readTool]])); const info = await ensurePyToolBridge(); const statusEvents: Array<{ op: string }> = []; - const unregister = registerPyToolBridge("status-session", { + const unregister = registerPyToolBridge("status-session", "run-status", { toolSession: session, emitStatus: event => statusEvents.push(event), }); try { const res = await call(info, { session: "status-session", + run: "run-status", name: "read", args: { path: "foo.ts" }, }); diff --git a/packages/coding-agent/test/debug/raw-sse-pretty.test.ts b/packages/coding-agent/test/debug/raw-sse-pretty.test.ts new file mode 100644 index 000000000..2dc12e11e --- /dev/null +++ b/packages/coding-agent/test/debug/raw-sse-pretty.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "bun:test"; +import { expandPrettyDataLines } from "../../src/debug/raw-sse"; + +// Wide enough that `truncateToWidth` would clip the payload in the viewer; matches +// what real Codex `response.output_item.done` frames look like on the wire. +function wideObjectLine(): string { + const payload = { + type: "response.output_item.done", + item: { + type: "message", + id: "msg_1234567890", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "Hello there, this is a long enough message" }], + }, + }; + return `data: ${JSON.stringify(payload)}`; +} + +describe("expandPrettyDataLines", () => { + it("expands wide JSON `data:` payloads into multi-line indented `data:` entries", () => { + const input = [`: ws ← response.output_item.done`, `event: response.output_item.done`, wideObjectLine()]; + + const out = expandPrettyDataLines(input); + + // Comment + event: lines pass through. + expect(out[0]).toBe(input[0]); + expect(out[1]).toBe(input[1]); + + // Original payload spans many lines now, each prefixed with `data: `. + const dataLines = out.slice(2); + expect(dataLines.length).toBeGreaterThan(1); + for (const line of dataLines) { + expect(line.startsWith("data: ")).toBe(true); + } + + // The expanded JSON, with `data: ` stripped, round-trips to the original payload. + const rejoined = dataLines.map(line => line.slice("data: ".length)).join("\n"); + expect(JSON.parse(rejoined)).toEqual(JSON.parse(wideObjectLine().slice("data: ".length))); + }); + + it("leaves short single-line payloads alone so small deltas stay compact", () => { + const input = [ + `: ws ← response.output_text.delta`, + `event: response.output_text.delta`, + `data: {"type":"response.output_text.delta","delta":"hi"}`, + ]; + expect(expandPrettyDataLines(input)).toEqual([...input]); + }); + + it("falls back to the raw line when the payload is wide but not JSON", () => { + // Wide enough to cross the threshold but doesn't start with `{` or `[`. + const wideNonJson = `data: ${"x".repeat(200)}`; + const out = expandPrettyDataLines([wideNonJson]); + expect(out).toEqual([wideNonJson]); + }); + + it("falls back when the payload is wide JSON-looking text but parses as invalid", () => { + const wideBrokenJson = `data: {"unterminated":"${"x".repeat(200)}`; + const out = expandPrettyDataLines([wideBrokenJson]); + expect(out).toEqual([wideBrokenJson]); + }); + + it("preserves non-`data:` lines (event/comment) verbatim regardless of length", () => { + const wideComment = `: ${"x".repeat(300)}`; + const wideEvent = `event: ${"x".repeat(300)}`; + const out = expandPrettyDataLines([wideComment, wideEvent]); + expect(out).toEqual([wideComment, wideEvent]); + }); +}); diff --git a/packages/coding-agent/test/edit-diff.test.ts b/packages/coding-agent/test/edit-diff.test.ts index d80ee724c..25627d1e9 100644 --- a/packages/coding-agent/test/edit-diff.test.ts +++ b/packages/coding-agent/test/edit-diff.test.ts @@ -5,10 +5,10 @@ import * as path from "node:path"; import { adjustIndentation, computeEditDiff, + computeFileHash, computeHashlineDiff, DEFAULT_FUZZY_THRESHOLD, findMatch, - formatLineHash, } from "@oh-my-pi/pi-coding-agent/edit"; describe("findMatch", () => { @@ -236,10 +236,9 @@ describe("computeHashlineDiff", () => { const line = "unchanged content"; await Bun.write(sourcePath, `${line}\n`); - // `1→` with the same line as payload is a true no-op: the edit + // `1:` with the same line as payload is a true no-op: the edit // fires through computeHashlineDiff but produces identical content. - const anchor = formatLineHash(1, line); - const input = `¶${sourcePath}\n${anchor}→\n${line}\n`; + const input = `¶${sourcePath}#${computeFileHash(`${line}\n`)}\n1:${line}\n`; const result = await computeHashlineDiff({ input }, tempDir); expect("error" in result).toBe(true); if ("error" in result) { diff --git a/packages/coding-agent/test/eval/console-table.test.ts b/packages/coding-agent/test/eval/console-table.test.ts new file mode 100644 index 000000000..6aa9f2986 --- /dev/null +++ b/packages/coding-agent/test/eval/console-table.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; +import { JsRuntime, type RuntimeHooks } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime"; +import type { JsDisplayOutput } from "@oh-my-pi/pi-coding-agent/eval/js/shared/types"; + +function makeRuntime(): { + runtime: JsRuntime; + hooks: RuntimeHooks; + texts: string[]; + displays: JsDisplayOutput[]; +} { + const texts: string[] = []; + const displays: JsDisplayOutput[] = []; + const runtime = new JsRuntime({ + initialCwd: process.cwd(), + sessionId: "test", + }); + const hooks: RuntimeHooks = { + onText: (chunk: string) => { + texts.push(chunk); + }, + onDisplay: (output: JsDisplayOutput) => { + displays.push(output); + }, + callTool: async () => undefined, + }; + return { runtime, hooks, texts, displays }; +} + +describe("console.table bridge", () => { + it("renders an array of objects as an ASCII table on text output", async () => { + const { runtime, hooks, texts, displays } = makeRuntime(); + await runtime.run("console.table([{ name: 'Ada', age: 36 }, { name: 'Linus', age: 54 }]);", undefined, hooks); + expect(displays).toEqual([]); + expect(texts.length).toBe(1); + const out = texts[0]; + // Box-drawing frame proves we routed through node:console.Console, not util.inspect. + expect(out).toContain("┌"); + expect(out).toContain("(index)"); + expect(out).toContain("name"); + expect(out).toContain("age"); + expect(out).toContain("Ada"); + expect(out).toContain("Linus"); + expect(out.endsWith("\n")).toBe(true); + }); + + it("honors the optional columns filter", async () => { + const { runtime, hooks, texts } = makeRuntime(); + await runtime.run("console.table([{ name: 'Ada', age: 36, secret: 'hidden' }], ['name']);", undefined, hooks); + const out = texts.join(""); + expect(out).toContain("name"); + expect(out).toContain("Ada"); + expect(out).not.toContain("secret"); + expect(out).not.toContain("hidden"); + expect(out).not.toContain("age"); + }); +}); diff --git a/packages/coding-agent/test/eval/display-image-coerce.test.ts b/packages/coding-agent/test/eval/display-image-coerce.test.ts index 97b8a073e..382fc5b4a 100644 --- a/packages/coding-agent/test/eval/display-image-coerce.test.ts +++ b/packages/coding-agent/test/eval/display-image-coerce.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "bun:test"; -import { JsRuntime } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime"; +import { JsRuntime, type RuntimeHooks } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime"; import type { JsDisplayOutput } from "@oh-my-pi/pi-coding-agent/eval/js/shared/types"; function collect(): { runtime: JsRuntime; + hooks: RuntimeHooks; displays: JsDisplayOutput[]; texts: string[]; } { @@ -12,17 +13,17 @@ function collect(): { const runtime = new JsRuntime({ initialCwd: process.cwd(), sessionId: "test", - getHooks: () => ({ - onText: chunk => { - texts.push(chunk); - }, - onDisplay: output => { - displays.push(output); - }, - callTool: async () => undefined, - }), }); - return { runtime, displays, texts }; + const hooks: RuntimeHooks = { + onText: (chunk: string) => { + texts.push(chunk); + }, + onDisplay: (output: JsDisplayOutput) => { + displays.push(output); + }, + callTool: async () => undefined, + }; + return { runtime, hooks, displays, texts }; } const PNG_BYTES = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]); @@ -30,27 +31,27 @@ const PNG_BASE64 = Buffer.from(PNG_BYTES).toString("base64"); describe("JsRuntime.displayValue image coercion", () => { it("passes through strict base64 strings verbatim", () => { - const { runtime, displays } = collect(); - runtime.displayValue({ type: "image", data: PNG_BASE64, mimeType: "image/png" }); + const { runtime, hooks, displays } = collect(); + runtime.displayValue({ type: "image", data: PNG_BASE64, mimeType: "image/png" }, hooks); expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]); }); it("base64-encodes Uint8Array data", () => { - const { runtime, displays } = collect(); - runtime.displayValue({ type: "image", data: PNG_BYTES, mimeType: "image/png" }); + const { runtime, hooks, displays } = collect(); + runtime.displayValue({ type: "image", data: PNG_BYTES, mimeType: "image/png" }, hooks); expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]); }); it("base64-encodes Buffer data", () => { - const { runtime, displays } = collect(); - runtime.displayValue({ type: "image", data: Buffer.from(PNG_BYTES), mimeType: "image/png" }); + const { runtime, hooks, displays } = collect(); + runtime.displayValue({ type: "image", data: Buffer.from(PNG_BYTES), mimeType: "image/png" }, hooks); expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]); }); it("base64-encodes ArrayBuffer data", () => { - const { runtime, displays } = collect(); + const { runtime, hooks, displays } = collect(); const ab = PNG_BYTES.buffer.slice(PNG_BYTES.byteOffset, PNG_BYTES.byteOffset + PNG_BYTES.byteLength); - runtime.displayValue({ type: "image", data: ab, mimeType: "image/png" }); + runtime.displayValue({ type: "image", data: ab, mimeType: "image/png" }, hooks); expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]); }); @@ -58,26 +59,26 @@ describe("JsRuntime.displayValue image coercion", () => { // Reproduces the puppeteer footgun: page.screenshot() returns Uint8Array, and // `uint8array.toString("base64")` silently falls through to Array.toString, // yielding "137,80,78,71,...". Anthropic rejects that as invalid base64. - const { runtime, displays } = collect(); + const { runtime, hooks, displays } = collect(); const decimalCsv = Array.from(PNG_BYTES).toString(); expect(decimalCsv).toBe("137,80,78,71,13,10,26,10"); - runtime.displayValue({ type: "image", data: decimalCsv, mimeType: "image/png" }); + runtime.displayValue({ type: "image", data: decimalCsv, mimeType: "image/png" }, hooks); expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]); }); it("recovers JSON-serialized Buffer shape ({ type: 'Buffer', data: [...] })", () => { - const { runtime, displays } = collect(); + const { runtime, hooks, displays } = collect(); const jsonBuffer = JSON.parse(JSON.stringify(Buffer.from(PNG_BYTES))) as { type: string; data: number[]; }; - runtime.displayValue({ type: "image", data: jsonBuffer, mimeType: "image/png" }); + runtime.displayValue({ type: "image", data: jsonBuffer, mimeType: "image/png" }, hooks); expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]); }); it("drops images whose data is unrecognized and surfaces a diagnostic in text", () => { - const { runtime, displays, texts } = collect(); - runtime.displayValue({ type: "image", data: { not: "a buffer" }, mimeType: "image/png" }); + const { runtime, hooks, displays, texts } = collect(); + runtime.displayValue({ type: "image", data: { not: "a buffer" }, mimeType: "image/png" }, hooks); expect(displays).toHaveLength(0); expect(texts.join("")).toMatch(/image dropped/); }); @@ -85,8 +86,8 @@ describe("JsRuntime.displayValue image coercion", () => { it("rejects strings that look base64-ish but aren't strictly valid", () => { // Padding mid-string, whitespace, or URL-safe alphabet are all dropped — the // Anthropic API only honors strict base64 in image sources. - const { runtime, displays, texts } = collect(); - runtime.displayValue({ type: "image", data: "abcd=efg", mimeType: "image/png" }); + const { runtime, hooks, displays, texts } = collect(); + runtime.displayValue({ type: "image", data: "abcd=efg", mimeType: "image/png" }, hooks); expect(displays).toHaveLength(0); expect(texts.join("")).toMatch(/image dropped/); }); diff --git a/packages/coding-agent/test/issue-1401-repro.test.ts b/packages/coding-agent/test/issue-1401-repro.test.ts new file mode 100644 index 000000000..0f94739c0 --- /dev/null +++ b/packages/coding-agent/test/issue-1401-repro.test.ts @@ -0,0 +1,64 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { processFileArguments } from "../src/cli/file-processor"; + +function createPdfWithText(text: string): string { + const chunks: string[] = []; + let position = 0; + const offsets = [0]; + const add = (chunk: string) => { + chunks.push(chunk); + position += Buffer.byteLength(chunk); + }; + + add("%PDF-1.4\n"); + offsets.push(position); + add("1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"); + offsets.push(position); + add("2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n"); + offsets.push(position); + add( + "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>\nendobj\n", + ); + offsets.push(position); + add("4 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n"); + + const stream = `BT /F1 24 Tf 72 720 Td (${text}) Tj ET`; + offsets.push(position); + add(`5 0 obj\n<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream\nendobj\n`); + + const xrefPosition = position; + add("xref\n0 6\n0000000000 65535 f \n"); + for (let i = 1; i <= 5; i++) { + add(`${String(offsets[i]).padStart(10, "0")} 00000 n \n`); + } + add(`trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n${xrefPosition}\n%%EOF\n`); + + return chunks.join(""); +} + +describe("processFileArguments", () => { + let testDir: string; + + beforeEach(() => { + testDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-pdf-file-args-")); + }); + + afterEach(() => { + fs.rmSync(testDir, { recursive: true, force: true }); + }); + + it("converts PDF file arguments before adding them to the prompt", async () => { + const pdfPath = path.join(testDir, "document.pdf"); + fs.writeFileSync(pdfPath, createPdfWithText("Hello PDF from issue 1401")); + + const result = await processFileArguments([pdfPath], { autoResizeImages: false }); + + expect(result.images).toEqual([]); + expect(result.text).toContain("Hello PDF from issue 1401"); + expect(result.text).not.toContain("%PDF-1.4"); + expect(result.text).not.toContain("stream"); + }); +}); diff --git a/packages/coding-agent/test/model-registry.test.ts b/packages/coding-agent/test/model-registry.test.ts index f0c75dd06..bd9f84727 100644 --- a/packages/coding-agent/test/model-registry.test.ts +++ b/packages/coding-agent/test/model-registry.test.ts @@ -2234,4 +2234,72 @@ describe("ModelRegistry", () => { expect(registry.find("ollama-cloud", "deepseek-v4-pro")?.maxTokens).toBe(384_000); }); + + test("replaces bundled google-vertex models with authoritative Vertex project discovery", () => { + const cachedModel: Model<"openai-completions"> = { + id: "zai-org/glm-4.7-maas", + name: "GLM-4.7", + api: "openai-completions", + provider: "google-vertex", + baseUrl: "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 222_222, + maxTokens: 8_888, + }; + writeModelCache("google-vertex", Date.now(), [cachedModel], true, "", cacheDbPath); + + const registry = new ModelRegistry(authStorage, modelsJsonPath); + const vertexModels = getModelsForProvider(registry, "google-vertex"); + + expect(vertexModels.map(model => model.id)).toEqual(["zai-org/glm-4.7-maas"]); + expect(registry.find("google-vertex", "gemini-1.5-pro")).toBeUndefined(); + }); + + test("keeps bundled google-vertex fallback when cached project catalog is non-authoritative", () => { + const cachedModel: Model<"openai-completions"> = { + id: "zai-org/glm-4.7-maas", + name: "GLM-4.7", + api: "openai-completions", + provider: "google-vertex", + baseUrl: "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 222_222, + maxTokens: 8_888, + }; + writeModelCache("google-vertex", Date.now(), [cachedModel], false, "", cacheDbPath); + + const registry = new ModelRegistry(authStorage, modelsJsonPath); + const vertexModels = getModelsForProvider(registry, "google-vertex"); + + expect(vertexModels.some(model => model.id === "zai-org/glm-4.7-maas")).toBe(true); + expect(vertexModels.some(model => model.id.startsWith("gemini-"))).toBe(true); + }); + + test("keeps bundled google-vertex fallback when cached project catalog is stale", () => { + const cachedModel: Model<"openai-completions"> = { + id: "zai-org/glm-4.7-maas", + name: "GLM-4.7", + api: "openai-completions", + provider: "google-vertex", + baseUrl: "https://aiplatform.googleapis.com/v1/projects/vertex-project/locations/global/endpoints/openapi", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 222_222, + maxTokens: 8_888, + }; + // 25h old > 24h TTL → cache.fresh === false even though authoritative === true. + const staleTimestamp = Date.now() - 25 * 60 * 60 * 1000; + writeModelCache("google-vertex", staleTimestamp, [cachedModel], true, "", cacheDbPath); + + const registry = new ModelRegistry(authStorage, modelsJsonPath); + const vertexModels = getModelsForProvider(registry, "google-vertex"); + + expect(vertexModels.some(model => model.id === "zai-org/glm-4.7-maas")).toBe(true); + expect(vertexModels.some(model => model.id.startsWith("gemini-"))).toBe(true); + }); }); diff --git a/packages/coding-agent/test/modes/theme/settings-list-theme.test.ts b/packages/coding-agent/test/modes/theme/settings-list-theme.test.ts new file mode 100644 index 000000000..23e6dba86 --- /dev/null +++ b/packages/coding-agent/test/modes/theme/settings-list-theme.test.ts @@ -0,0 +1,21 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import { getSettingsListTheme, initTheme, theme } from "../../../src/modes/theme/theme"; + +beforeAll(async () => { + await initTheme(); +}); + +describe("getSettingsListTheme", () => { + it("keeps modified labels dirty while selected values use the cursor accent", () => { + const settingsTheme = getSettingsListTheme(); + + const selectedChangedValue = settingsTheme.value("changed", true, true); + const unselectedChangedValue = settingsTheme.value("changed", false, true); + const selectedChangedLabel = settingsTheme.label("Changed", true, true); + + expect(selectedChangedValue).toBe(theme.fg("accent", "changed")); + expect(unselectedChangedValue).toBe(theme.fg("statusLineGitDirty", "changed")); + expect(selectedChangedLabel).toBe(theme.fg("statusLineGitDirty", "Changed")); + expect(selectedChangedValue).not.toBe(unselectedChangedValue); + }); +}); diff --git a/packages/coding-agent/test/oauth-discovery.test.ts b/packages/coding-agent/test/oauth-discovery.test.ts index 63d261b96..4748234d7 100644 --- a/packages/coding-agent/test/oauth-discovery.test.ts +++ b/packages/coding-agent/test/oauth-discovery.test.ts @@ -50,3 +50,206 @@ describe("mcp oauth discovery", () => { expect(calls[0]).toBe("https://www.figma.com/.well-known/oauth-authorization-server"); }); }); + +describe("path-prefixed auth servers", () => { + it("discovers endpoints via relative well-known path when server URL has a sub-path", async () => { + const calls: string[] = []; + using _hook = hookFetch(input => { + const url = String(input); + calls.push(url); + + // Absolute well-known fails (at origin root) + if (url === "https://gateway.example.com/.well-known/oauth-authorization-server") { + return new Response("not found", { status: 404 }); + } + // Relative well-known succeeds (under /my-service/) + if (url === "https://gateway.example.com/my-service/.well-known/oauth-authorization-server") { + return new Response( + JSON.stringify({ + authorization_endpoint: "https://gateway.example.com/my-service/oauth/authorize", + token_endpoint: "https://gateway.example.com/my-service/oauth/token", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response("not found", { status: 404 }); + }); + + const oauth = await discoverOAuthEndpoints("https://gateway.example.com/my-service/mcp"); + + expect(oauth).toEqual({ + authorizationUrl: "https://gateway.example.com/my-service/oauth/authorize", + tokenUrl: "https://gateway.example.com/my-service/oauth/token", + }); + // Absolute well-known was tried first (existing behavior) + expect(calls[0]).toBe("https://gateway.example.com/.well-known/oauth-authorization-server"); + // Relative well-known was tried as fallback + expect(calls).toContain("https://gateway.example.com/my-service/.well-known/oauth-authorization-server"); + }); + + it("discovers endpoints via single-segment path prefix (no trailing endpoint segment)", async () => { + const calls: string[] = []; + using _hook = hookFetch(input => { + const url = String(input); + calls.push(url); + + if (url === "https://gateway.example.com/.well-known/oauth-authorization-server") { + return new Response("not found", { status: 404 }); + } + if (url === "https://gateway.example.com/my-service/.well-known/oauth-authorization-server") { + return new Response( + JSON.stringify({ + authorization_endpoint: "https://gateway.example.com/my-service/oauth/authorize", + token_endpoint: "https://gateway.example.com/my-service/oauth/token", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response("not found", { status: 404 }); + }); + + const oauth = await discoverOAuthEndpoints("https://gateway.example.com/my-service"); + + expect(oauth).toEqual({ + authorizationUrl: "https://gateway.example.com/my-service/oauth/authorize", + tokenUrl: "https://gateway.example.com/my-service/oauth/token", + }); + expect(calls[0]).toBe("https://gateway.example.com/.well-known/oauth-authorization-server"); + expect(calls).toContain("https://gateway.example.com/my-service/.well-known/oauth-authorization-server"); + }); + + it("falls back to RFC 8414 path-ful issuer form (/.well-known/oauth-authorization-server/)", async () => { + const calls: string[] = []; + using _hook = hookFetch(input => { + const url = String(input); + calls.push(url); + + if (url === "https://gateway.example.com/.well-known/oauth-authorization-server/my-service") { + return new Response( + JSON.stringify({ + authorization_endpoint: "https://gateway.example.com/my-service/oauth", + token_endpoint: "https://gateway.example.com/my-service/token", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response("not found", { status: 404 }); + }); + + const oauth = await discoverOAuthEndpoints("https://gateway.example.com/my-service"); + + expect(oauth).toEqual({ + authorizationUrl: "https://gateway.example.com/my-service/oauth", + tokenUrl: "https://gateway.example.com/my-service/token", + }); + expect(calls).toContain("https://gateway.example.com/.well-known/oauth-authorization-server/my-service"); + }); + + it("prefers absolute well-known when it succeeds (origin-root servers still work)", async () => { + const calls: string[] = []; + using _hook = hookFetch(input => { + const url = String(input); + calls.push(url); + + if (url === "https://auth.example.com/.well-known/oauth-authorization-server") { + return new Response( + JSON.stringify({ + authorization_endpoint: "https://auth.example.com/oauth", + token_endpoint: "https://auth.example.com/token", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response("not found", { status: 404 }); + }); + + const oauth = await discoverOAuthEndpoints("https://mcp.example.com", "https://auth.example.com"); + + expect(oauth).toEqual({ + authorizationUrl: "https://auth.example.com/oauth", + tokenUrl: "https://auth.example.com/token", + }); + // Only the absolute path was needed + expect(calls).toHaveLength(1); + expect(calls[0]).toBe("https://auth.example.com/.well-known/oauth-authorization-server"); + }); +}); + +describe("resource_metadata chain", () => { + it("extracts resourceMetadataUrl from error message", () => { + const error = new Error( + 'HTTP 401: WWW-Authenticate: Bearer resource_metadata="https://gateway.example.com/my-service/.well-known/oauth-protected-resource"', + ); + + const auth = analyzeAuthError(error); + expect(auth.requiresAuth).toBe(true); + expect(auth.resourceMetadataUrl).toBe( + "https://gateway.example.com/my-service/.well-known/oauth-protected-resource", + ); + }); + + it("follows resource_metadata URL to discover authorization servers", async () => { + const calls: string[] = []; + using _hook = hookFetch(input => { + const url = String(input); + calls.push(url); + + // resource_metadata URL returns authorization_servers + if (url === "https://gateway.example.com/my-service/.well-known/oauth-protected-resource") { + return new Response( + JSON.stringify({ + authorization_servers: ["https://gateway.example.com/my-service"], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + // Well-known at the discovered auth server (absolute fails, relative succeeds) + if (url === "https://gateway.example.com/.well-known/oauth-authorization-server") { + return new Response("not found", { status: 404 }); + } + if (url === "https://gateway.example.com/my-service/.well-known/oauth-authorization-server") { + return new Response( + JSON.stringify({ + authorization_endpoint: "https://gateway.example.com/my-service/oauth", + token_endpoint: "https://gateway.example.com/my-service/token", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + + return new Response("not found", { status: 404 }); + }); + + const oauth = await discoverOAuthEndpoints( + "https://gateway.example.com/my-service/mcp", + undefined, + "https://gateway.example.com/my-service/.well-known/oauth-protected-resource", + ); + + expect(oauth).toEqual({ + authorizationUrl: "https://gateway.example.com/my-service/oauth", + tokenUrl: "https://gateway.example.com/my-service/token", + }); + // resource_metadata fetched first + expect(calls[0]).toBe("https://gateway.example.com/my-service/.well-known/oauth-protected-resource"); + }); +}); + +describe("relative Mcp-Auth-Server URL", () => { + it("resolves relative Mcp-Auth-Server against server URL", () => { + const error = new Error("HTTP 401: WWW-Authenticate: Bearer; Mcp-Auth-Server: /my-service/oauth"); + + // Without serverUrl, relative URL returns undefined + expect(extractMcpAuthServerUrl(error)).toBeUndefined(); + + // With serverUrl, relative URL is resolved + expect(extractMcpAuthServerUrl(error, "https://gateway.example.com/my-service/mcp")).toBe( + "https://gateway.example.com/my-service/oauth", + ); + }); +}); diff --git a/packages/coding-agent/test/prompt-templates.test.ts b/packages/coding-agent/test/prompt-templates.test.ts index af317fbbd..461bcbadb 100644 --- a/packages/coding-agent/test/prompt-templates.test.ts +++ b/packages/coding-agent/test/prompt-templates.test.ts @@ -252,9 +252,9 @@ describe("hashline prompt helpers", () => { '{{hline 2 "const timeout = 5000;"}}\nquoted={{href 2}}\nraw={{hrefr 2}}\nlast={{hrefr}}', ); const [line, quoted, raw, last] = result.split("\n"); - const ref = line.split("|", 1)[0]; + const ref = line.split(":", 1)[0]; - expect(line).toBe(`${ref}|const timeout = 5000;`); + expect(line).toBe(`${ref}:const timeout = 5000;`); expect(quoted).toBe(`quoted="${ref}"`); expect(raw).toBe(`raw=${ref}`); expect(last).toBe(`last=${ref}`); @@ -266,11 +266,11 @@ describe("hashline prompt helpers", () => { const ref = raw.slice("raw=".length); expect(quoted).toBe(`quoted="${ref}"`); - expect(ref).toMatch(/^5[a-z]{2}$/); + expect(ref).toBe("5"); }); test("href should not reuse hline state across prompt renders", () => { - expect(expandPrompt('{{hline 1 "const x = 1;"}}\n{{hrefr}}')).toMatch(/^1[a-z]{2}\|const x = 1;\n1[a-z]{2}$/); + expect(expandPrompt('{{hline 1 "const x = 1;"}}\n{{hrefr}}')).toBe("1:const x = 1;\n1"); expect(() => expandPrompt("{{hrefr}}")).toThrow("previous {{hline}}"); }); }); diff --git a/packages/coding-agent/test/read-multi-range.test.ts b/packages/coding-agent/test/read-multi-range.test.ts index 51caf2c79..61554edff 100644 --- a/packages/coding-agent/test/read-multi-range.test.ts +++ b/packages/coding-agent/test/read-multi-range.test.ts @@ -154,6 +154,6 @@ describe("read tool multi-range selector", () => { expect(text).toContain("bridge four"); expect(text).toContain("bridge five"); expect(text).not.toContain("bridge three"); - expect(text).not.toContain("disk"); + expect(text).not.toContain("disk one"); }); }); diff --git a/packages/coding-agent/test/read-summary.test.ts b/packages/coding-agent/test/read-summary.test.ts index 13d6384b4..eac4d2b0a 100644 --- a/packages/coding-agent/test/read-summary.test.ts +++ b/packages/coding-agent/test/read-summary.test.ts @@ -183,10 +183,10 @@ describe("read summary", () => { expect(text).toContain("name: Ada"); }); - it("renders brace-pair elisions as a single anchored line with `..`", async () => { + it("renders brace-pair elisions as a single numbered line with `..`", async () => { // Regression for the read-tool format request: collapse the head / - // elided / closing-brace sandwich into one anchored line of the form - // `LINE+ID-LINE+ID|head { .. }` instead of three separate lines. + // elided / closing-brace sandwich into one numbered line of the form + // `START-END:head { .. }` instead of three separate lines. const fixture = path.join(tmpDir, "merge.ts"); await fs.writeFile( fixture, @@ -200,8 +200,8 @@ describe("read summary", () => { expect(text).toContain("export function stripNewLinePrefixes(lines: string[]): string[] { .. }"); // The plain `...` ellipsis line must NOT appear once the merge fires. expect(text).not.toContain("\n...\n"); - // The merged anchor must be a hash-line range (LINE+ID-LINE+ID|head). - expect(text).toMatch(/\b1[a-z]{2}-7[a-z]{2}\|export function stripNewLinePrefixes/); + // The merged line must use the numbered range shape. + expect(text).toMatch(/\b1-7:export function stripNewLinePrefixes/); expect(result.details?.summary?.elidedSpans).toBe(1); }); @@ -240,7 +240,7 @@ describe("read summary", () => { expect(text).not.toContain(" .. "); }); - it("appends an elision footer that names the path and `:raw` recovery selector", async () => { + it("appends an elision footer that names targeted recovery ranges", async () => { // Regression for issue #1046: summarized reads must tell the model how // to recover the elided body so it does not stall on `...` / `{ .. }` // markers and burn a turn guessing the selector. @@ -256,12 +256,13 @@ describe("read summary", () => { expect(result.details?.summary?.elidedSpans).toBe(2); expect(result.details?.summary?.elidedLines).toBeGreaterThan(0); - expect(text).toContain("elided regions"); - expect(text).toContain(`${fixture}:raw`); - expect(text).toContain(`${fixture}:1-9999`); + expect(text).toContain("lines elided"); + expect(text).toContain(`${fixture}:1-5,7-11`); + expect(text).not.toContain(`${fixture}:raw`); + expect(text).not.toContain(`${fixture}:1-9999`); // Footer must be the LAST block of output so the recovery hint sits // next to the structural summary it describes. - expect(text.trimEnd().endsWith("for verbatim content]")).toBe(true); + expect(text.trimEnd().endsWith("]")).toBe(true); }); it("does not append a footer when the file has no elision", async () => { diff --git a/packages/coding-agent/test/sdk-tool-activation.test.ts b/packages/coding-agent/test/sdk-tool-activation.test.ts index 8dff72279..3bac4d713 100644 --- a/packages/coding-agent/test/sdk-tool-activation.test.ts +++ b/packages/coding-agent/test/sdk-tool-activation.test.ts @@ -5,7 +5,6 @@ import * as path from "node:path"; import { getBundledModel } from "@oh-my-pi/pi-ai"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { createAgentSession, type ExtensionFactory } from "@oh-my-pi/pi-coding-agent/sdk"; -import { AuthStorage } from "@oh-my-pi/pi-coding-agent/session/auth-storage"; import { SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager"; import { Snowflake } from "@oh-my-pi/pi-utils"; import * as z from "zod/v4"; @@ -34,15 +33,12 @@ const toolActivationExtension: ExtensionFactory = pi => { describe("createAgentSession defaultInactive tool activation", () => { const tempDirs: string[] = []; - const authStorages: AuthStorage[] = []; afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { recursive: true, force: true }); } - for (const authStorage of authStorages.splice(0)) { - authStorage.close(); - } + vi.restoreAllMocks(); }); @@ -112,7 +108,11 @@ describe("createAgentSession defaultInactive tool activation", () => { } }); - it("keeps edit active when vim edit mode is configured", async () => { + it("activates the yield tool when requireYieldTool is set and toolNames is explicit", async () => { + // Regression for #1408: plan-mode subagents pass an explicit `toolNames` list + // (e.g. `["read", "search", "find", "lsp", "web_search"]`). Without this + // invariant, `yield` ended up registered but not active, and the model + // could not satisfy the idle-reminder contract that demands a `yield` call. const tempDir = path.join(os.tmpdir(), `pi-sdk-tool-activation-${Snowflake.next()}`); tempDirs.push(tempDir); fs.mkdirSync(tempDir, { recursive: true }); @@ -121,84 +121,21 @@ describe("createAgentSession defaultInactive tool activation", () => { cwd: tempDir, agentDir: tempDir, sessionManager: SessionManager.inMemory(), - settings: Settings.isolated({ "edit.mode": "vim" }), + settings: Settings.isolated(), model: getBundledModel("openai", "gpt-4o-mini"), disableExtensionDiscovery: true, - extensions: [], skills: [], contextFiles: [], promptTemplates: [], slashCommands: [], enableMCP: false, enableLsp: false, - toolNames: ["read", "edit"], + requireYieldTool: true, + toolNames: ["read", "search", "find", "web_search"], }); try { - expect(session.getActiveToolNames()).toContain("edit"); - expect(session.getActiveToolNames()).not.toContain("vim"); - expect(session.getAllToolNames()).toContain("edit"); - expect(session.getAllToolNames()).not.toContain("vim"); - - await session.setActiveToolsByName(["read", "edit"]); - - expect(session.getActiveToolNames()).toContain("edit"); - expect(session.getActiveToolNames()).not.toContain("vim"); - } finally { - await session.dispose(); - } - }); - - it("keeps the visible edit tool stable when the active model changes edit modes", async () => { - const tempDir = path.join(os.tmpdir(), `pi-sdk-tool-activation-${Snowflake.next()}`); - tempDirs.push(tempDir); - fs.mkdirSync(tempDir, { recursive: true }); - - const settings = Settings.isolated(); - vi.spyOn(settings, "getEditVariantForModel").mockImplementation(model => - model?.includes("mini") ? "vim" : "hashline", - ); - - const authStorage = await AuthStorage.create(path.join(tempDir, "auth.db")); - authStorages.push(authStorage); - authStorage.setRuntimeApiKey("openai", "test-key"); - - const baseModel = getBundledModel("openai", "gpt-4o"); - const vimModel = getBundledModel("openai", "gpt-4o-mini"); - if (!baseModel || !vimModel) { - throw new Error("Expected bundled OpenAI models for edit-mode switching test"); - } - - const { session } = await createAgentSession({ - cwd: tempDir, - agentDir: tempDir, - sessionManager: SessionManager.inMemory(), - settings, - authStorage, - model: baseModel, - disableExtensionDiscovery: true, - extensions: [], - skills: [], - contextFiles: [], - promptTemplates: [], - slashCommands: [], - enableMCP: false, - enableLsp: false, - toolNames: ["read", "edit"], - }); - - try { - expect(session.getActiveToolNames()).toContain("edit"); - expect(session.getActiveToolNames()).not.toContain("vim"); - expect(session.getAllToolNames()).toContain("edit"); - expect(session.getAllToolNames()).not.toContain("vim"); - - await session.setModel(vimModel); - - expect(session.getActiveToolNames()).toContain("edit"); - expect(session.getActiveToolNames()).not.toContain("vim"); - expect(session.getAllToolNames()).toContain("edit"); - expect(session.getAllToolNames()).not.toContain("vim"); + expect(session.getActiveToolNames()).toContain("yield"); } finally { await session.dispose(); } diff --git a/packages/coding-agent/test/tool-discovery/persistence.test.ts b/packages/coding-agent/test/tool-discovery/persistence.test.ts index d95fc420d..5e0ace6ff 100644 --- a/packages/coding-agent/test/tool-discovery/persistence.test.ts +++ b/packages/coding-agent/test/tool-discovery/persistence.test.ts @@ -1,48 +1,6 @@ import { describe, expect, it } from "bun:test"; import type { DiscoverableTool } from "../../src/tool-discovery/tool-index"; -import { buildDiscoverableMCPSearchIndex, buildDiscoverableToolSearchIndex } from "../../src/tool-discovery/tool-index"; - -// ─── Tests that verify the generic discovery index is compatible with -// legacy MCP-format data (legacy persistence / back-compat). -// ───────────────────────────────────────────────────────────────────────────── - -describe("persistence back-compat: buildDiscoverableMCPSearchIndex wraps generic index", () => { - const legacyMCPTools = [ - { - name: "mcp__github_create_issue", - label: "github/create_issue", - description: "Create a GitHub issue", - serverName: "github", - mcpToolName: "create_issue", - schemaKeys: ["owner", "repo", "title"], - }, - { - name: "mcp__slack_post", - label: "slack/post_message", - description: "Post a Slack message", - serverName: "slack", - mcpToolName: "post_message", - schemaKeys: ["channel", "text"], - }, - ]; - - it("maps description → summary in the index", () => { - const index = buildDiscoverableMCPSearchIndex(legacyMCPTools); - // The documents contain DiscoverableTool objects with .summary, not .description - const doc = index.documents.find(d => d.tool.name === "mcp__github_create_issue"); - expect(doc).toBeDefined(); - // summary was set from description - expect(doc!.tool.summary).toBe("Create a GitHub issue"); - }); - - it("is searchable with standard search function", () => { - const { searchDiscoverableTools } = require("../../src/tool-discovery/tool-index"); - const index = buildDiscoverableMCPSearchIndex(legacyMCPTools); - const results = searchDiscoverableTools(index, "github issue", 5); - expect(results.length).toBeGreaterThan(0); - expect(results[0]!.tool.name).toBe("mcp__github_create_issue"); - }); -}); +import { buildDiscoverableToolSearchIndex } from "../../src/tool-discovery/tool-index"; describe("generic index: DiscoverableTool round-trip", () => { const tools: DiscoverableTool[] = [ diff --git a/packages/coding-agent/test/tool-discovery/tool-index.test.ts b/packages/coding-agent/test/tool-discovery/tool-index.test.ts index 2c082de35..c8a71034e 100644 --- a/packages/coding-agent/test/tool-discovery/tool-index.test.ts +++ b/packages/coding-agent/test/tool-discovery/tool-index.test.ts @@ -300,46 +300,3 @@ describe("BM25 search", () => { expect(results).toHaveLength(0); }); }); - -// ─── Back-compat: legacy MCP functions ─────────────────────────────────────── - -describe("back-compat MCP functions via mcp/discoverable-tool-metadata", () => { - it("isMCPToolName still works", async () => { - const { isMCPToolName: legacyIsMCPToolName } = await import("../../src/mcp/discoverable-tool-metadata"); - expect(legacyIsMCPToolName("mcp__foo")).toBe(true); - expect(legacyIsMCPToolName("read")).toBe(false); - }); - - it("collectDiscoverableMCPTools still works", async () => { - const { collectDiscoverableMCPTools } = await import("../../src/mcp/discoverable-tool-metadata"); - const tools = [ - mcpAgentTool("mcp__gh_search", "github", "search", "Search repos", ["query"]), - makeAgentTool("read"), // non-MCP — should be filtered out - ]; - const result = collectDiscoverableMCPTools(tools as any); - expect(result).toHaveLength(1); - expect(result[0]!.name).toBe("mcp__gh_search"); - expect(result[0]!.description).toBe("Search repos"); - }); - - it("buildDiscoverableMCPSearchIndex still works and is searchable", async () => { - const { buildDiscoverableMCPSearchIndex, searchDiscoverableMCPTools } = await import( - "../../src/mcp/discoverable-tool-metadata" - ); - const legacyTools = [ - { - name: "mcp__test", - label: "test/tool", - description: "A test MCP tool", - serverName: "test", - mcpToolName: "tool", - schemaKeys: ["query"], - }, - ]; - const index = buildDiscoverableMCPSearchIndex(legacyTools); - expect(index.documents).toHaveLength(1); - const results = searchDiscoverableMCPTools(index, "test", 5); - expect(results).toHaveLength(1); - expect(results[0]!.tool.name).toBe("mcp__test"); - }); -}); diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index d28eb585e..3ee122623 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -1269,16 +1269,11 @@ function b() { it("should abort and recover for subsequent commands", async () => { const controller = new AbortController(); - const promise = bashTool.execute( - "test-call-10-abort", - { command: "printf 'started\\n'; sleep 60" }, - controller.signal, - update => { - if (update.content?.some(content => content.type === "text" && content.text.includes("started"))) { - controller.abort("test abort"); - } - }, - ); + const promise = bashTool.execute("test-call-10-abort", { command: "sleep 60" }, controller.signal); + // Give the native shell a beat to enter `sleep`; do not depend on chunk + // delivery timing, which is flaky on loaded CI runners. + await Bun.sleep(100); + controller.abort("test abort"); await expect(promise).rejects.toThrow(/abort|cancel|timed out/i); const result = await bashTool.execute("test-call-10-after-abort", { command: "echo ok" }); @@ -1787,6 +1782,43 @@ function b() { expect(output).not.toContain(".env.generated"); expect(elapsedMs).toBeLessThan(1000); }); + + it("should return directories alongside files with a trailing slash", async () => { + fs.mkdirSync(path.join(testDir, "pkg")); + fs.mkdirSync(path.join(testDir, "pkg", "nested")); + fs.writeFileSync(path.join(testDir, "pkg", "file.txt"), "f"); + fs.writeFileSync(path.join(testDir, "pkg", "nested", "deep.txt"), "d"); + + const result = await findTool.execute("test-call-14f", { + paths: [`${testDir}/pkg/**/*`], + }); + + const outputLines = getTextOutput(result) + .split("\n") + .map(line => line.trim()) + .filter(Boolean) + .sort(); + + expect(outputLines).toEqual(["pkg/file.txt", "pkg/nested/", "pkg/nested/deep.txt"]); + }); + + it("should match a directory by glob and emit it with trailing slash", async () => { + fs.mkdirSync(path.join(testDir, "alpha", "tests"), { recursive: true }); + fs.mkdirSync(path.join(testDir, "beta", "tests"), { recursive: true }); + fs.writeFileSync(path.join(testDir, "alpha", "tests", "a.ts"), "a"); + + const result = await findTool.execute("test-call-14g", { + paths: [`${testDir}/**/tests`], + }); + + const outputLines = getTextOutput(result) + .split("\n") + .map(line => line.trim()) + .filter(Boolean) + .sort(); + + expect(outputLines).toEqual(["alpha/tests/", "beta/tests/"]); + }); }); }); diff --git a/packages/coding-agent/test/tools/ast-edit.test.ts b/packages/coding-agent/test/tools/ast-edit.test.ts index 4e1c05446..ba2b26639 100644 --- a/packages/coding-agent/test/tools/ast-edit.test.ts +++ b/packages/coding-agent/test/tools/ast-edit.test.ts @@ -60,7 +60,7 @@ describe("ast_edit tool schema", () => { expect(strict.strict).toBe(true); }); - it("renders +/- lines with aligned hashline prefixes", async () => { + it("renders +/- lines with numbered hashline prefixes", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "ast-edit-render-")); try { const filePath = path.join(tempDir, "legacy.ts"); @@ -81,9 +81,9 @@ describe("ast_edit tool schema", () => { expect(removedLine).toBeDefined(); expect(addedLine).toBeDefined(); - expect(removedLine).toMatch(/^-\d+[a-z]{2}\|/); - expect(addedLine).toMatch(/^\+\d+[a-z]{2}\|/); - expect(removedLine?.split("|", 1)[0].length).toBe(addedLine?.split("|", 1)[0].length); + expect(removedLine).toMatch(/^-\d+:/); + expect(addedLine).toMatch(/^\+\d+:/); + expect(removedLine?.split(":", 1)[0].length).toBe(addedLine?.split(":", 1)[0].length); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -211,8 +211,9 @@ describe("ast_edit tool schema", () => { | { totalReplacements?: number; fileReplacements?: Array<{ path: string; count: number }> } | undefined; - expect(text).toContain("## root.ts (1 replacement)"); - expect(text).toContain("## child.ts (1 replacement)"); + // Tree-grouped output: `# packages/pkg-…/src/` then `## root.ts# (1 replacement)`. + expect(text).toMatch(/^## root\.ts#[0-9a-f]{4} \(\d+ replacement[s]?\)$/m); + expect(text).toMatch(/^## child\.ts#[0-9a-f]{4} \(\d+ replacement[s]?\)$/m); expect(text).not.toContain("ignore.js"); expect(text).not.toContain("outside.ts"); expect(details?.totalReplacements).toBe(2); diff --git a/packages/coding-agent/test/tools/ast-grep.test.ts b/packages/coding-agent/test/tools/ast-grep.test.ts index 49b4cbc6e..47258ea73 100644 --- a/packages/coding-agent/test/tools/ast-grep.test.ts +++ b/packages/coding-agent/test/tools/ast-grep.test.ts @@ -100,8 +100,9 @@ describe("ast_grep parse errors", () => { const text = result.content.find(content => content.type === "text")?.text ?? ""; const details = result.details as { matchCount?: number; fileCount?: number } | undefined; - expect(text).toContain("## root.ts"); - expect(text).toContain("## child.ts"); + // Directory mode uses tree-grouped `# dir/` + `## name#hash` headers. + expect(text).toMatch(/## root\.ts#[0-9a-f]+/); + expect(text).toMatch(/## child\.ts#[0-9a-f]+/); expect(text).not.toContain("ignore.js"); expect(text).not.toContain("outside.ts"); expect(details?.matchCount).toBe(2); diff --git a/packages/coding-agent/test/tools/conflict-integration.test.ts b/packages/coding-agent/test/tools/conflict-integration.test.ts index a9ac07efa..d1b5b318e 100644 --- a/packages/coding-agent/test/tools/conflict-integration.test.ts +++ b/packages/coding-agent/test/tools/conflict-integration.test.ts @@ -511,7 +511,7 @@ describe("write resolves conflicts via conflict://N", () => { await read.execute("read-hashed", { path: "hashed.ts" }); const result = await write.execute("write-hashed", { path: "conflict://1", - content: "42xy|cleanline\n", + content: "¶hashed.ts#1a2b\n42:cleanline\n", }); expect(getText(result)).toContain("auto-stripped hashline display prefixes"); const after = await Bun.file(filePath).text(); diff --git a/packages/coding-agent/test/tools/index.test.ts b/packages/coding-agent/test/tools/index.test.ts index 2eea2fd44..0ff2cabcd 100644 --- a/packages/coding-agent/test/tools/index.test.ts +++ b/packages/coding-agent/test/tools/index.test.ts @@ -44,7 +44,7 @@ function createDiscoverySessionHooks(): Partial { const selected: string[] = []; return { isMCPDiscoveryEnabled: () => true, - getDiscoverableMCPTools: () => [], + getDiscoverableTools: () => [], getSelectedMCPToolNames: () => [...selected], activateDiscoveredMCPTools: async toolNames => { const activated: string[] = []; @@ -86,19 +86,6 @@ describe("createTools", () => { expect(names).not.toContain("vim"); }); - it("keeps edit visible when vim edit mode is active", async () => { - const session = createTestSession({ - settings: createSettingsWithOverrides({ - "edit.mode": "vim", - }), - }); - const tools = await createTools(session); - const names = tools.map(t => t.name); - - expect(names).toContain("edit"); - expect(names).not.toContain("vim"); - }); - it("includes bash and eval when both eval backends are allowed", async () => { const session = createTestSession({ settings: createSettingsWithOverrides({ @@ -167,18 +154,6 @@ describe("createTools", () => { expect(names).toEqual(["read", "write", "resolve"]); }); - it("ignores vim as an unknown requested tool even when vim edit mode is active", async () => { - const session = createTestSession({ - settings: createSettingsWithOverrides({ - "edit.mode": "vim", - }), - }); - const tools = await createTools(session, ["read", "vim"]); - const names = tools.map(t => t.name); - - expect(names).toEqual(["read", "resolve"]); - }); - it("lowercases requested tool subset", async () => { const session = createTestSession(); const tools = await createTools(session, ["Read", "Write"]); diff --git a/packages/coding-agent/test/tools/irc.test.ts b/packages/coding-agent/test/tools/irc.test.ts index f489f9e99..a14c8016e 100644 --- a/packages/coding-agent/test/tools/irc.test.ts +++ b/packages/coding-agent/test/tools/irc.test.ts @@ -1,7 +1,9 @@ -import { beforeEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test"; +import { Agent } from "@oh-my-pi/pi-agent-core"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { AgentRegistry } from "@oh-my-pi/pi-coding-agent/registry/agent-registry"; -import type { AgentSession } from "@oh-my-pi/pi-coding-agent/session/agent-session"; +import { AgentSession } from "@oh-my-pi/pi-coding-agent/session/agent-session"; +import { type FileEntry, SessionManager } from "@oh-my-pi/pi-coding-agent/session/session-manager"; import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; import { IrcTool } from "@oh-my-pi/pi-coding-agent/tools/irc"; @@ -14,15 +16,22 @@ interface FakeSession { setError: (error: Error) => void; /** Resolve the next respondAsBackground call only when allowed. */ gateNextCall: () => { release: () => void }; + /** Keep the next respondAsBackground call pending until aborted. */ + hangNextCall: () => void; } - function makeFakeSession(): FakeSession { let nextReply = "auto-reply"; let nextError: Error | null = null; let gate: { promise: Promise; release: () => void } | null = null; + let hangNext = false; const calls: Array<{ from: string; message: string; awaitReply: boolean }> = []; const session = { - respondAsBackground: async (args: { from: string; message: string; awaitReply?: boolean }) => { + respondAsBackground: async (args: { + from: string; + message: string; + awaitReply?: boolean; + signal?: AbortSignal; + }) => { const awaitReply = args.awaitReply !== false; calls.push({ from: args.from, message: args.message, awaitReply }); if (gate) { @@ -30,6 +39,21 @@ function makeFakeSession(): FakeSession { gate = null; await g.promise; } + if (hangNext) { + hangNext = false; + const deferred = Promise.withResolvers(); + if (args.signal?.aborted) { + deferred.reject(args.signal.reason instanceof Error ? args.signal.reason : new Error("aborted")); + } else { + args.signal?.addEventListener( + "abort", + () => + deferred.reject(args.signal?.reason instanceof Error ? args.signal.reason : new Error("aborted")), + { once: true }, + ); + } + return await deferred.promise; + } if (nextError) { const err = nextError; nextError = null; @@ -48,12 +72,12 @@ function makeFakeSession(): FakeSession { nextError = error; }, gateNextCall: () => { - let release!: () => void; - const promise = new Promise(resolve => { - release = resolve; - }); - gate = { promise, release }; - return { release }; + const { promise, resolve } = Promise.withResolvers(); + gate = { promise, release: resolve }; + return { release: resolve }; + }, + hangNextCall: () => { + hangNext = true; }, }; } @@ -69,14 +93,37 @@ function makeToolSession(registry: AgentRegistry, agentId: string): ToolSession getAgentId: () => agentId, }; } +function createRealBackgroundSession(): { session: AgentSession; sessionManager: SessionManager } { + const sessionManager = SessionManager.inMemory("/tmp"); + const session = new AgentSession({ + agent: new Agent({ + initialState: { + systemPrompt: ["system prompt"], + messages: [], + tools: [], + }, + }), + sessionManager, + settings: Settings.isolated({ "compaction.enabled": false }), + modelRegistry: {} as never, + }); + return { session, sessionManager }; +} describe("IrcTool", () => { let registry: AgentRegistry; + const sessions: AgentSession[] = []; beforeEach(() => { AgentRegistry.resetGlobalForTests(); registry = AgentRegistry.global(); }); + afterEach(async () => { + vi.restoreAllMocks(); + for (const session of sessions.splice(0)) { + await session.dispose(); + } + }); it("createIf returns null when irc is disabled", () => { const session: ToolSession = { @@ -202,6 +249,83 @@ describe("IrcTool", () => { expect(result.details?.notFound).toEqual(["0-Ghost"]); }); + it("op=send fails a hung recipient after the configured timeout", async () => { + const main = makeFakeSession(); + const sub = makeFakeSession(); + sub.hangNextCall(); + registry.register({ id: "0-Main", displayName: "main", kind: "main", session: main.session }); + registry.register({ id: "0-Hung", displayName: "task", kind: "sub", parentId: "0-Main", session: sub.session }); + + const toolSession = makeToolSession(registry, "0-Main"); + toolSession.settings.set("irc.timeoutMs", 5); + const tool = new IrcTool(toolSession); + const result = await tool.execute("call-timeout", { op: "send", to: "0-Hung", message: "ping" }); + + expect(result.details?.delivered ?? []).toEqual([]); + expect(result.details?.failed).toEqual([{ id: "0-Hung", error: "IRC timed out waiting for 0-Hung after 5 ms" }]); + expect(sub.calls).toEqual([{ from: "0-Main", message: "ping", awaitReply: true }]); + }); + it("op=send preserves the recipient's incoming DM when the auto-reply times out", async () => { + const main = makeFakeSession(); + const { session: recipient, sessionManager } = createRealBackgroundSession(); + sessions.push(recipient); + vi.spyOn(recipient, "runEphemeralTurn").mockImplementation(async ({ signal }) => { + if (!signal) { + throw new Error("Missing abort signal"); + } + const deferred = Promise.withResolvers(); + const rejectOnAbort = () => { + deferred.reject(signal.reason instanceof Error ? signal.reason : new Error("IRC aborted")); + }; + if (signal.aborted) { + rejectOnAbort(); + } else { + signal.addEventListener("abort", rejectOnAbort, { once: true }); + } + return await deferred.promise; + }); + registry.register({ id: "0-Main", displayName: "main", kind: "main", session: main.session }); + registry.register({ + id: "0-Hung", + displayName: "task", + kind: "sub", + parentId: "0-Main", + session: recipient, + }); + + const toolSession = makeToolSession(registry, "0-Main"); + toolSession.settings.set("irc.timeoutMs", 5); + const tool = new IrcTool(toolSession); + const result = await tool.execute("call-timeout-persist", { op: "send", to: "0-Hung", message: "ping" }); + + expect(result.details?.delivered ?? []).toEqual([]); + expect(result.details?.failed).toEqual([{ id: "0-Hung", error: "IRC timed out waiting for 0-Hung after 5 ms" }]); + expect(recipient.messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + role: "custom", + customType: "irc:incoming", + content: "[IRC `0-Main` → you]\n\nping", + }), + ]), + ); + expect(recipient.messages).not.toEqual( + expect.arrayContaining([expect.objectContaining({ role: "custom", customType: "irc:autoreply" })]), + ); + const persistedEntries = sessionManager + .captureState() + .fileEntries.filter( + (entry): entry is Extract => + entry.type === "custom_message" && typeof entry.content === "string", + ); + expect(persistedEntries).toEqual([ + expect.objectContaining({ + customType: "irc:incoming", + content: "[IRC `0-Main` → you]\n\nping", + }), + ]); + }); + it("op=send surfaces recipient errors as failed", async () => { const main = makeFakeSession(); const sub = makeFakeSession(); diff --git a/packages/coding-agent/test/tools/lsp-regressions.test.ts b/packages/coding-agent/test/tools/lsp-regressions.test.ts index f0d28f680..447dc7594 100644 --- a/packages/coding-agent/test/tools/lsp-regressions.test.ts +++ b/packages/coding-agent/test/tools/lsp-regressions.test.ts @@ -7,13 +7,19 @@ import { LspTool } from "@oh-my-pi/pi-coding-agent/lsp"; import * as lspClient from "@oh-my-pi/pi-coding-agent/lsp/client"; import * as lspConfig from "@oh-my-pi/pi-coding-agent/lsp/config"; import { getServersForFile, loadConfig } from "@oh-my-pi/pi-coding-agent/lsp/config"; +import { applyWorkspaceEdit } from "@oh-my-pi/pi-coding-agent/lsp/edits"; import { renderCall, renderResult } from "@oh-my-pi/pi-coding-agent/lsp/render"; import type { CodeAction, + CreateFile, + DeleteFile, Diagnostic, LspClient, + RenameFile, ServerConfig, SymbolInformation, + TextDocumentEdit, + WorkspaceEdit, } from "@oh-my-pi/pi-coding-agent/lsp/types"; import { applyCodeAction, @@ -930,4 +936,237 @@ describe("lsp regressions", () => { tempDir.removeSync(); } }); + + it("flushes pending descendant text edits before a folder rename", async () => { + const tempDir = TempDir.createSync("@omp-lsp-folder-rename-"); + try { + const srcDir = path.join(tempDir.path(), "src"); + fs.mkdirSync(srcDir, { recursive: true }); + const childPath = path.join(srcDir, "a.ts"); + await Bun.write(childPath, "export const a = 1;\n"); + + const childUri = fileToUri(childPath); + const oldFolderUri = fileToUri(srcDir); + const newFolderUri = fileToUri(path.join(tempDir.path(), "src2")); + + const childEdit: TextDocumentEdit = { + textDocument: { uri: childUri, version: null }, + edits: [ + { + range: { + start: { line: 0, character: 13 }, + end: { line: 0, character: 14 }, + }, + newText: "renamed", + }, + ], + }; + const folderRename: RenameFile = { + kind: "rename", + oldUri: oldFolderUri, + newUri: newFolderUri, + }; + const workspaceEdit: WorkspaceEdit = { + documentChanges: [childEdit, folderRename], + }; + + const applied = await applyWorkspaceEdit(workspaceEdit, tempDir.path()); + + // Old folder is gone, new folder holds the edited child. + expect(fs.existsSync(srcDir)).toBe(false); + const renamedChildPath = path.join(tempDir.path(), "src2", "a.ts"); + expect(fs.existsSync(renamedChildPath)).toBe(true); + expect(fs.readFileSync(renamedChildPath, "utf8")).toBe("export const renamed = 1;\n"); + + // Both ops are reported in original order: edit first, then rename. + expect(applied).toHaveLength(2); + expect(applied[0]).toContain("Applied 1 edit(s)"); + expect(applied[0]).toContain("src/a.ts"); + expect(applied[1]).toContain("Renamed"); + expect(applied[1]).toContain("src"); + expect(applied[1]).toContain("src2"); + } finally { + tempDir.removeSync(); + } + }); + + it("flushes pending edits queued against a rename target before performing the rename", async () => { + // LSP §3.16.2: documentChanges run in declared order. When a TextDocumentEdit + // targets `renameOp.newUri` *before* the rename, those edits must land on the + // existing file at that location BEFORE the rename overwrites/replaces it. + // Otherwise the rename clobbers the post-edit content (or worse, the edits + // land on the moved-in file with stale offsets). + const tempDir = TempDir.createSync("@omp-lsp-rename-target-prefill-"); + try { + const oldPath = path.join(tempDir.path(), "old.ts"); + const newPath = path.join(tempDir.path(), "new.ts"); + await Bun.write(oldPath, "export const moved = 1;\n"); + // A pre-existing target file the rename is about to clobber. + await Bun.write(newPath, "export const target = 2;\n"); + + const oldUri = fileToUri(oldPath); + const newUri = fileToUri(newPath); + + // Edit the target file first, then rename onto it. Pre-edit content + // MUST be observable somewhere in the applied log — proving the flush + // ran before the rename clobbered the file. + const targetEdit: TextDocumentEdit = { + textDocument: { uri: newUri, version: null }, + edits: [ + { + range: { + start: { line: 0, character: 13 }, + end: { line: 0, character: 19 }, + }, + newText: "before", + }, + ], + }; + const renameOp: RenameFile = { + kind: "rename", + oldUri, + newUri, + }; + const workspaceEdit: WorkspaceEdit = { + documentChanges: [targetEdit, renameOp], + }; + + const applied = await applyWorkspaceEdit(workspaceEdit, tempDir.path()); + + // Three steps observable in order: edit on newUri, then rename clobbers it. + expect(applied).toHaveLength(2); + expect(applied[0]).toContain("Applied 1 edit(s)"); + expect(applied[0]).toContain("new.ts"); + expect(applied[1]).toContain("Renamed"); + + // Final state: new.ts holds the moved-in content (rename ran last and won). + expect(fs.existsSync(oldPath)).toBe(false); + expect(fs.readFileSync(newPath, "utf8")).toBe("export const moved = 1;\n"); + } finally { + tempDir.removeSync(); + } + }); + it("resolves $-prefixed identifiers past compound matches", async () => { + // Pre-fix, BARE_IDENTIFIER_RE rejected leading `$`, so requireWordBoundary + // was false and `resolveSymbolColumn(_, _, "$store")` returned the column + // inside `bar$store` rather than the standalone occurrence, feeding the + // LSP server the wrong column. The new regex `/^[$A-Za-z_][\w$]*$/` plus + // IDENTIFIER_CHAR_RE's existing `$` membership enforces the boundary. + const tempDir = TempDir.createSync("@omp-lsp-dollar-identifier-"); + try { + const filePath = path.join(tempDir.path(), "store.ts"); + // Standalone `$store` starts at column 16; compound `bar$store` + // contains the substring at column 7. Old code returned 7; new code + // returns 16. + await Bun.write(filePath, "let bar$store = $store + 1;\n"); + + const column = await resolveSymbolColumn(filePath, 1, "$store"); + expect(column).toBe(16); + + // `bar$store` is itself a valid `$`-bearing identifier and resolves + // to its own start, not into either fragment. + const compoundColumn = await resolveSymbolColumn(filePath, 1, "bar$store"); + expect(compoundColumn).toBe(4); + } finally { + tempDir.removeSync(); + } + }); + + it("applies a create op followed by a text edit for the same URI in declared order", async () => { + // LSP §3.16.2 motivating case for the rewrite: "Extract to new file" + // code actions emit `[CreateFile(newUri), TextDocumentEdit(newUri, ...)]`. + // Pre-fix, all text edits flushed first → applyTextEdits opened a + // not-yet-created file → ENOENT. The new walk processes each entry in + // order, so the create lands first and the edit reads the empty file + // the create just wrote. + const tempDir = TempDir.createSync("@omp-lsp-create-then-edit-"); + try { + const newFilePath = path.join(tempDir.path(), "extracted.ts"); + expect(fs.existsSync(newFilePath)).toBe(false); + + const newUri = fileToUri(newFilePath); + const createOp: CreateFile = { + kind: "create", + uri: newUri, + }; + const textEdit: TextDocumentEdit = { + textDocument: { uri: newUri, version: null }, + edits: [ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 0 }, + }, + newText: "export const extracted = 42;\n", + }, + ], + }; + const workspaceEdit: WorkspaceEdit = { + documentChanges: [createOp, textEdit], + }; + + const applied = await applyWorkspaceEdit(workspaceEdit, tempDir.path()); + + expect(fs.existsSync(newFilePath)).toBe(true); + expect(fs.readFileSync(newFilePath, "utf8")).toBe("export const extracted = 42;\n"); + + // Declared order observable in the applied log: create first, then edit. + expect(applied).toHaveLength(2); + expect(applied[0]).toContain("Created"); + expect(applied[0]).toContain("extracted.ts"); + expect(applied[1]).toContain("Applied 1 edit(s)"); + expect(applied[1]).toContain("extracted.ts"); + } finally { + tempDir.removeSync(); + } + }); + + it("flushes pending descendant text edits before a folder delete", async () => { + // Mirror of the folder-rename subtree-flush test for the `delete` arm: + // edits queued against a child URI must land at the original path + // BEFORE the parent folder is removed, otherwise the flush at end of + // walk would target a non-existent path and throw. + const tempDir = TempDir.createSync("@omp-lsp-folder-delete-"); + try { + const srcDir = path.join(tempDir.path(), "src"); + fs.mkdirSync(srcDir, { recursive: true }); + const childPath = path.join(srcDir, "a.ts"); + await Bun.write(childPath, "export const a = 1;\n"); + + const childUri = fileToUri(childPath); + const folderUri = fileToUri(srcDir); + + const childEdit: TextDocumentEdit = { + textDocument: { uri: childUri, version: null }, + edits: [ + { + range: { + start: { line: 0, character: 18 }, + end: { line: 0, character: 19 }, + }, + newText: "999", + }, + ], + }; + const folderDelete: DeleteFile = { + kind: "delete", + uri: folderUri, + }; + const workspaceEdit: WorkspaceEdit = { + documentChanges: [childEdit, folderDelete], + }; + + const applied = await applyWorkspaceEdit(workspaceEdit, tempDir.path()); + + // Folder is gone; "Applied" message proves the flush ran before delete. + expect(fs.existsSync(srcDir)).toBe(false); + expect(applied).toHaveLength(2); + expect(applied[0]).toContain("Applied 1 edit(s)"); + expect(applied[0]).toContain("src/a.ts"); + expect(applied[1]).toContain("Deleted"); + expect(applied[1]).toContain("src"); + } finally { + tempDir.removeSync(); + } + }); }); diff --git a/packages/coding-agent/test/tools/search-internal-urls.test.ts b/packages/coding-agent/test/tools/search-internal-urls.test.ts index 53006e3e3..672f292f0 100644 --- a/packages/coding-agent/test/tools/search-internal-urls.test.ts +++ b/packages/coding-agent/test/tools/search-internal-urls.test.ts @@ -146,8 +146,9 @@ describe("SearchTool internal URL resolution", () => { const text = getResultText(result); expect(text).toContain("needle"); - // No hashline anchors (LINE+ID|content) for immutable sources - expect(text).not.toMatch(/^\*?\s*\d+[a-z]{2}\|/m); + // No hashline section headers or numbered editable lines for immutable sources. + expect(text).not.toMatch(/^¶.*#[0-9a-f]{4}$/m); + expect(text).not.toMatch(/^\*?\s*\d+:/m); }); it("resolves local:// URLs before file-name lookup", async () => { @@ -185,8 +186,9 @@ describe("SearchTool internal URL resolution", () => { const text = getResultText(result); expect(text).toContain("needle"); - // Hashline anchor (LINE+ID|content) is kept for mutable local:// sources - expect(text).toMatch(/^\*?\s*\d+[a-z]{2}\|/m); + // Mutable local:// sources keep a hashline section header plus numbered match lines. + expect(text).toMatch(/^¶.*#[0-9a-f]{4}$/m); + expect(text).toMatch(/^\*\d+:.*needle/m); }); it("keeps hashlines on mutable files when mixed with immutable artifact:// inputs", async () => { @@ -204,8 +206,9 @@ describe("SearchTool internal URL resolution", () => { const text = getResultText(result); expect(text).toContain("needle"); - // Mutable mixed.txt keeps hashlines somewhere in the output - expect(text).toMatch(/^\*?\s*\d+[a-z]{2}\|.*mixed needle/m); + // Mutable mixed.txt keeps hashlines somewhere in the output. + expect(text).toMatch(/^# mixed\.txt#[0-9a-f]{4}/m); + expect(text).toMatch(/^\*\d+:.*mixed needle/m); }); it("throws on nonexistent artifact ID", async () => { diff --git a/packages/coding-agent/test/tools/search-path-lists.test.ts b/packages/coding-agent/test/tools/search-path-lists.test.ts index 346dd1868..5720f6be1 100644 --- a/packages/coding-agent/test/tools/search-path-lists.test.ts +++ b/packages/coding-agent/test/tools/search-path-lists.test.ts @@ -1,10 +1,21 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "bun:test"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; +import type { AgentMessage } from "@oh-my-pi/pi-agent-core"; +import { validateToolArguments } from "@oh-my-pi/pi-ai/utils/validation"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import type { RenderResultOptions } from "@oh-my-pi/pi-coding-agent/extensibility/custom-tools/types"; +import type { Theme } from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; import { ToolChoiceQueue } from "@oh-my-pi/pi-coding-agent/session/tool-choice-queue"; import { createTools, type ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; +import { searchToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/search"; +import { Text } from "@oh-my-pi/pi-tui"; +import { SessionObserverOverlayComponent } from "../../src/modes/components/session-observer-overlay"; +import { TreeSelectorComponent } from "../../src/modes/components/tree-selector"; +import type { ObservableSession, SessionObserverRegistry } from "../../src/modes/session-observer-registry"; +import { initTheme } from "../../src/modes/theme/theme"; +import type { SessionEntry, SessionTreeNode } from "../../src/session/session-manager"; function createTestSession(cwd: string, overrides: Partial = {}): ToolSession { return { @@ -17,6 +28,18 @@ function createTestSession(cwd: string, overrides: Partial = {}): T }; } +const plainTheme = { + fg: (_color: unknown, text: string) => text, + styledSymbol: () => "…", + sep: { dot: " • " }, + format: { bracketLeft: "[", bracketRight: "]" }, +} as unknown as Theme; + +const renderOptions: RenderResultOptions = { + expanded: false, + isPartial: true, +}; + function getText(result: { content: Array<{ type: string; text?: string }> }): string { return result.content .filter(entry => entry.type === "text") @@ -55,11 +78,52 @@ async function createSearchFixture(rootDir: string): Promise { "const providerOptions = {};\nlegacyWrap(otherValue, otherArg);\n", ); } +async function makeJsonlSessionFile(dirPath: string, entries: object[]): Promise { + const filePath = path.join(dirPath, "session.jsonl"); + await Bun.write(filePath, `${entries.map(entry => JSON.stringify(entry)).join("\n")}\n`); + return filePath; +} + +function makeSubagentRegistry(sessions: ObservableSession[]): SessionObserverRegistry { + return { + getSessions: () => sessions, + onChange: () => () => {}, + setMainSession: () => {}, + getActiveSubagentCount: () => sessions.filter(session => session.status === "active").length, + } as unknown as SessionObserverRegistry; +} + +let treeEntryCounter = 0; +function makeMessageNode(message: AgentMessage, parentId: string | null = null): SessionTreeNode { + const entry: SessionEntry = { + type: "message", + id: `entry-${treeEntryCounter++}`, + parentId, + timestamp: new Date().toISOString(), + message, + }; + return { entry, children: [] }; +} + +function renderTree(tree: SessionTreeNode[], currentLeafId: string): string { + const selector = new TreeSelectorComponent( + tree, + currentLeafId, + 60, + () => {}, + () => {}, + ); + return Bun.stripANSI(selector.render(120).join("\n")); +} describe("tool path arrays", () => { let tempDir: string; + beforeAll(async () => { + await initTheme(false, undefined, undefined, "dark", "light"); + }); beforeEach(async () => { + treeEntryCounter = 0; tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "search-path-lists-")); await createSearchFixture(tempDir); }); @@ -81,15 +145,169 @@ describe("tool path arrays", () => { const text = getText(result); const details = result.details as { fileCount?: number; scopePath?: string } | undefined; - expect(text).toContain("# apps"); - expect(text).toContain("# packages"); - expect(text).toContain("# phases"); - expect(text).toContain("## grep.txt"); + expect(text).toMatch(/^# apps\/\n## grep\.txt#[0-9a-f]{4}/m); + expect(text).toMatch(/^# packages\/\n## grep\.txt#[0-9a-f]{4}/m); + expect(text).toMatch(/^# phases\/\n## grep\.txt#[0-9a-f]{4}/m); + expect(text).toContain("shared-needle"); expect(text).not.toContain("# other"); expect(details?.fileCount).toBe(3); expect(details?.scopePath).toBe("apps/, packages/, phases/"); }); + it("search accepts a single string path through tool validation", async () => { + const tools = await createTools(createTestSession(tempDir)); + const tool = tools.find(entry => entry.name === "search"); + expect(tool).toBeDefined(); + if (!tool) throw new Error("Missing search tool"); + + const args = validateToolArguments(tool, { + type: "toolCall", + id: "search-single-string-path", + name: tool.name, + arguments: { + pattern: "space-needle", + paths: "folder with spaces/", + }, + }); + const result = await tool.execute("search-single-string-path", args); + const text = getText(result); + const details = result.details as { fileCount?: number; scopePath?: string } | undefined; + + expect(text).toContain("note.txt"); + expect(details?.fileCount).toBe(1); + expect(details?.scopePath).toBe("folder with spaces"); + }); + + it("search pending renderer accepts a single string path", () => { + const component = searchToolRenderer.renderCall( + { pattern: "space-needle", paths: "folder with spaces/" }, + renderOptions, + plainTheme, + ); + + expect(component).toBeInstanceOf(Text); + expect((component as Text).getText()).toContain("in folder with spaces/"); + }); + it("session observer overlay renders a single-string search path summary", async () => { + const sessionFile = await makeJsonlSessionFile(tempDir, [ + { type: "session", version: 3, id: "search-overlay-session", timestamp: new Date().toISOString() }, + { + type: "message", + id: "msg-user-1", + parentId: null, + timestamp: new Date().toISOString(), + message: { role: "user", content: "search", timestamp: 1 }, + }, + { + type: "message", + id: "msg-assistant-1", + parentId: "msg-user-1", + timestamp: new Date().toISOString(), + message: { + role: "assistant", + content: [ + { + type: "toolCall", + id: "search-call-1", + name: "search", + arguments: { pattern: "space-needle", paths: "folder with spaces/" }, + }, + ], + api: "test", + provider: "test", + model: "test", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + timestamp: 2, + }, + }, + { + type: "message", + id: "msg-tool-1", + parentId: "msg-assistant-1", + timestamp: new Date().toISOString(), + message: { + role: "toolResult", + toolName: "search", + toolCallId: "search-call-1", + content: [{ type: "text", text: "note.txt" }], + isError: false, + timestamp: 3, + }, + }, + ]); + const registry = makeSubagentRegistry([ + { + id: "search-overlay-session", + kind: "subagent", + label: "Search Overlay", + status: "active", + sessionFile, + lastUpdate: Date.now(), + }, + ]); + + const overlay = new SessionObserverOverlayComponent(registry, () => {}, ["ctrl+s"]); + const rendered = Bun.stripANSI(overlay.render(120).join("\n")); + + expect(rendered).toContain("paths: folder with spaces/"); + }); + + it("tree selector renders a single-string search path summary", () => { + const root = makeMessageNode({ role: "user", content: "search", timestamp: 1 }); + const assistant = makeMessageNode( + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "search-call-1", + name: "search", + arguments: { pattern: "space-needle", paths: "folder with spaces/" }, + }, + ], + api: "test", + provider: "test", + model: "test", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + timestamp: 2, + stopReason: "stop", + } as AgentMessage, + root.entry.id, + ); + const toolResult = makeMessageNode( + { + role: "toolResult", + toolCallId: "search-call-1", + toolName: "search", + content: [{ type: "text", text: "note.txt" }], + isError: false, + timestamp: 3, + } as AgentMessage, + assistant.entry.id, + ); + root.children.push(assistant); + assistant.children.push(toolResult); + + const rendered = renderTree([root], toolResult.entry.id); + + expect(rendered).toContain("[search: /space-needle/ in folder with spaces/]"); + expect(rendered).not.toContain("[search: /space-needle/ in .]"); + }); + it("search keeps a single path that contains spaces", async () => { const tools = await createTools(createTestSession(tempDir)); const tool = tools.find(entry => entry.name === "search"); @@ -141,8 +359,8 @@ describe("tool path arrays", () => { const text = getText(result); const details = result.details as { fileCount?: number; scopePath?: string } | undefined; - expect(text).toContain("# apps"); - expect(text).toContain("## grep.txt"); + expect(text).toMatch(/^# apps\/\n## grep\.txt#[0-9a-f]{4}/m); + expect(text).toContain("shared-needle"); expect(text).not.toContain(tempDir); expect(details?.fileCount).toBe(1); expect(details?.scopePath).toBe("apps"); @@ -198,10 +416,9 @@ describe("tool path arrays", () => { const text = getText(result); const details = result.details as { fileCount?: number; scopePath?: string } | undefined; - expect(text).toContain("# apps"); - expect(text).toContain("# packages"); - expect(text).toContain("# phases"); - expect(text).toContain("## ast.ts"); + expect(text).toMatch(/^# apps\/\n## ast\.ts#[0-9a-f]{4}/m); + expect(text).toMatch(/^# packages\/\n## ast\.ts#[0-9a-f]{4}/m); + expect(text).toMatch(/^# phases\/\n## ast\.ts#[0-9a-f]{4}/m); expect(text).not.toContain("# other"); expect(details?.fileCount).toBe(3); expect(details?.scopePath).toBe("apps/**/*.ts, packages/**/*.ts, phases/**/*.ts"); @@ -227,10 +444,9 @@ describe("tool path arrays", () => { const text = getText(preview); const details = preview.details as { totalReplacements?: number; scopePath?: string } | undefined; - expect(text).toContain("# apps"); - expect(text).toContain("# packages"); - expect(text).toContain("# phases"); - expect(text).toContain("## ast.ts (1 replacement)"); + expect(text).toMatch(/^# apps\/\n## ast\.ts#[0-9a-f]{4} \(\d+ replacement/m); + expect(text).toMatch(/^# packages\/\n## ast\.ts#[0-9a-f]{4} \(\d+ replacement/m); + expect(text).toMatch(/^# phases\/\n## ast\.ts#[0-9a-f]{4} \(\d+ replacement/m); expect(text).not.toContain("# other"); expect(details?.totalReplacements).toBe(3); expect(details?.scopePath).toBe("apps/**/*.ts, packages/**/*.ts, phases/**/*.ts"); @@ -330,9 +546,9 @@ describe("tool path arrays", () => { const text = getText(result); const details = result.details as { fileCount?: number; scopePath?: string } | undefined; - expect(text).toContain("# apps"); - expect(text).toContain("# packages"); - expect(text).toContain("# phases"); + expect(text).toMatch(/^# apps\/\n## grep\.txt#[0-9a-f]{4}/m); + expect(text).toMatch(/^# packages\/\n## grep\.txt#[0-9a-f]{4}/m); + expect(text).toMatch(/^# phases\/\n## grep\.txt#[0-9a-f]{4}/m); expect(text).not.toContain("# other"); expect(details?.fileCount).toBe(3); expect(details?.scopePath).toBe("apps, packages, phases"); @@ -357,8 +573,8 @@ describe("tool path arrays", () => { const text = getText(result); const details = result.details as { fileCount?: number; scopePath?: string } | undefined; - expect(text).toContain("# alpha.txt"); - expect(text).toContain("# beta.txt"); + expect(text).toMatch(/^# alpha\.txt#[0-9a-f]{4}/m); + expect(text).toMatch(/^# beta\.txt#[0-9a-f]{4}/m); expect(text).toContain("exact-needle alpha"); expect(text).toContain("exact-needle beta"); expect(text).not.toContain("nested"); @@ -408,8 +624,8 @@ describe("tool path arrays", () => { }); const text = getText(result); - expect(text).toMatch(/ 1(?:[a-z]{2})?\|#if FLAG/); - expect(text).toMatch(/\*2(?:[a-z]{2})?\|needle/); - expect(text).toMatch(/ 3(?:[a-z]{2})?\|#endif/); + expect(text).toMatch(/ 1:#if FLAG/); + expect(text).toMatch(/\*2:needle/); + expect(text).toMatch(/ 3:#endif/); }); }); diff --git a/packages/coding-agent/test/tools/search-tool-bm25.test.ts b/packages/coding-agent/test/tools/search-tool-bm25.test.ts index 392b639e3..135279bfb 100644 --- a/packages/coding-agent/test/tools/search-tool-bm25.test.ts +++ b/packages/coding-agent/test/tools/search-tool-bm25.test.ts @@ -1,38 +1,23 @@ import { describe, expect, it } from "bun:test"; import { Settings } from "../../src/config/settings"; -// Back-compat import check — these re-exports from mcp/discoverable-tool-metadata should still work -import { buildDiscoverableMCPSearchIndex, type DiscoverableMCPTool } from "../../src/mcp/discoverable-tool-metadata"; -import type { DiscoverableMCPSearchIndex, DiscoverableTool } from "../../src/tool-discovery/tool-index"; +import { + buildDiscoverableToolSearchIndex, + type DiscoverableTool, + type DiscoverableToolSearchIndex, +} from "../../src/tool-discovery/tool-index"; import type { ToolSession } from "../../src/tools/index"; import { SearchToolBm25Tool } from "../../src/tools/search-tool-bm25"; -type TestDiscoverableTool = DiscoverableTool; - -/** Adapt a generic discoverable tool to the legacy MCP shape (with `description`). */ -function toLegacyMCP(t: DiscoverableTool): DiscoverableMCPTool { - return { - name: t.name, - label: t.label, - description: t.summary, - serverName: t.serverName, - mcpToolName: t.mcpToolName, - schemaKeys: t.schemaKeys, - }; -} - type DiscoveryToolSession = ToolSession & { isMCPDiscoveryEnabled: () => boolean; - getDiscoverableMCPTools: () => DiscoverableMCPTool[]; - getDiscoverableMCPSearchIndex?: () => DiscoverableMCPSearchIndex; + getDiscoverableTools: (filter?: { source?: DiscoverableTool["source"] }) => DiscoverableTool[]; + getDiscoverableToolSearchIndex?: () => DiscoverableToolSearchIndex; getSelectedMCPToolNames: () => string[]; activateDiscoveredMCPTools: (toolNames: string[]) => Promise; getSelected: () => string[]; }; -function createSession( - tools: TestDiscoverableTool[], - overrides: Partial = {}, -): DiscoveryToolSession { +function createSession(tools: DiscoverableTool[], overrides: Partial = {}): DiscoveryToolSession { const selected: string[] = []; return { cwd: "/tmp/test", @@ -41,7 +26,7 @@ function createSession( getSessionSpawns: () => "*", settings: Settings.isolated({ "mcp.discoveryMode": true }), isMCPDiscoveryEnabled: () => true, - getDiscoverableMCPTools: () => tools.map(toLegacyMCP), + getDiscoverableTools: () => tools, getSelectedMCPToolNames: () => [...selected], activateDiscoveredMCPTools: async (toolNames: string[]) => { for (const name of toolNames) { @@ -56,7 +41,7 @@ function createSession( }; } -/** Helper to create a discoverable MCP tool (new unified shape) */ +/** Helper to create a discoverable MCP tool. */ function mcpTool( name: string, serverName: string, @@ -75,7 +60,7 @@ function mcpTool( }; } -/** Helper to create a discoverable built-in tool (new unified shape) */ +/** Helper to create a discoverable built-in tool. */ function builtinTool(name: string, summary: string, schemaKeys: string[] = []): DiscoverableTool { return { name, @@ -109,14 +94,13 @@ describe("SearchToolBm25Tool", () => { it("uses the session-provided cached search index during execution", async () => { let rawToolsCalls = 0; let searchIndexCalls = 0; - // Build via the legacy helper so documents expose `tool.description` (the legacy shape). - const searchIndex = buildDiscoverableMCPSearchIndex(discoverableTools.map(toLegacyMCP)); + const searchIndex = buildDiscoverableToolSearchIndex(discoverableTools); const session = createSession(discoverableTools, { - getDiscoverableMCPTools: () => { + getDiscoverableTools: () => { rawToolsCalls++; - return discoverableTools.map(toLegacyMCP); + return discoverableTools; }, - getDiscoverableMCPSearchIndex: () => { + getDiscoverableToolSearchIndex: () => { searchIndexCalls++; return searchIndex; }, @@ -216,30 +200,12 @@ describe("SearchToolBm25Tool", () => { const allTools = [...discoverableTools, ...builtinTools]; const session = createSession(discoverableTools, { settings: Settings.isolated({ "tools.discoveryMode": "all" }), - // Override to provide all tools including built-ins (legacy MCP shape). - getDiscoverableMCPTools: () => allTools.map(toLegacyMCP), + getDiscoverableTools: () => allTools, }); const tool = new SearchToolBm25Tool(session); const result = await tool.execute("call-builtin", { query: "find files" }); - // Should find built-in 'find' tool const names = result.details?.tools.map(t => t.name) ?? []; expect(names).toContain("find"); }); - - it("back-compat: buildDiscoverableMCPSearchIndex still works via mcp/discoverable-tool-metadata", () => { - // This test ensures the legacy MCP module re-exports still function correctly - const index = buildDiscoverableMCPSearchIndex([ - { - name: "mcp__test", - label: "test/tool", - description: "A test MCP tool", - serverName: "test", - mcpToolName: "tool", - schemaKeys: ["query"], - }, - ]); - expect(index.documents).toHaveLength(1); - expect(index.documents[0]?.tool.name).toBe("mcp__test"); - }); }); diff --git a/packages/coding-agent/test/tools/vim.test.ts b/packages/coding-agent/test/tools/vim.test.ts deleted file mode 100644 index 064cf01d6..000000000 --- a/packages/coding-agent/test/tools/vim.test.ts +++ /dev/null @@ -1,769 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test"; -import * as fs from "node:fs/promises"; -import * as os from "node:os"; -import * as path from "node:path"; -import { resetSettingsForTest, Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; -import { ToolExecutionComponent } from "@oh-my-pi/pi-coding-agent/modes/components/tool-execution"; -import * as themeModule from "@oh-my-pi/pi-coding-agent/modes/theme/theme"; -import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; -import { resetVimRendererStateForTest, VimTool, vimToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/vim"; -import { VimBuffer } from "@oh-my-pi/pi-coding-agent/vim/buffer"; -import { VimEngine } from "@oh-my-pi/pi-coding-agent/vim/engine"; -import { parseKeySequences } from "@oh-my-pi/pi-coding-agent/vim/parser"; -import type { TUI } from "@oh-my-pi/pi-tui"; - -function textResult(result: { content: Array<{ type: string; text?: string }> }): string { - return result.content - .filter(block => block.type === "text" && typeof block.text === "string") - .map(block => block.text) - .join("\n"); -} - -function createSession(cwd: string, overrides: Partial = {}): ToolSession { - return { - cwd, - hasUI: false, - getSessionFile: () => null, - getSessionSpawns: () => "*", - settings: Settings.isolated({ "lsp.enabled": false }), - ...overrides, - }; -} - -function createEngine(text: string): VimEngine { - return new VimEngine( - new VimBuffer({ - absolutePath: "/tmp/test.ts", - displayPath: "test.ts", - lines: text.split("\n"), - trailingNewline: false, - fingerprint: null, - }), - { - beforeMutate: async () => {}, - loadBuffer: async inputPath => ({ - absolutePath: inputPath, - displayPath: inputPath, - lines: [""], - trailingNewline: false, - fingerprint: null, - }), - saveBuffer: async buffer => ({ - loaded: { - absolutePath: buffer.filePath, - displayPath: buffer.displayPath, - lines: [...buffer.lines], - trailingNewline: buffer.trailingNewline, - fingerprint: null, - }, - }), - }, - ); -} - -function step(kbd: string[], insert?: string): { kbd: string[]; insert?: string } { - return insert === undefined ? { kbd } : { kbd, insert }; -} - -afterEach(() => { - vi.restoreAllMocks(); - resetVimRendererStateForTest(); -}); - -describe("vim parser", () => { - it("parses literal and special keys in order", () => { - const tokens = parseKeySequences(["ciwnewName", ":w"]); - expect(tokens.map(token => token.value)).toEqual([ - "c", - "i", - "w", - "n", - "e", - "w", - "N", - "a", - "m", - "e", - "Esc", - ":", - "w", - "CR", - ]); - }); - - it("handles literal escape byte and carriage return", () => { - const tokens = parseKeySequences(["itest\x1b", ":w\r"]); - expect(tokens.map(token => token.value)).toEqual(["i", "t", "e", "s", "t", "Esc", ":", "w", "CR"]); - }); - - it("handles backslash-r and backslash-e as CR and Esc", () => { - // Models often send \r as two chars (backslash + r) instead of a real CR byte - const tokens = parseKeySequences([":w\\r", "ciwnew\\e"]); - expect(tokens.map(token => token.value)).toEqual([":", "w", "CR", "c", "i", "w", "n", "e", "w", "Esc"]); - }); -}); - -describe("vim engine", () => { - it("repeats the last change with dot", async () => { - const engine = createEngine("foo foo"); - await engine.executeTokens(parseKeySequences(["ciwbar", "w", "."]), "ciwbar w ."); - expect(engine.buffer.getText()).toBe("bar bar"); - }); - - it("streams dot-repeat replays through the step callback", async () => { - const engine = createEngine("foo foo"); - await engine.executeTokens(parseKeySequences(["ciwbar", "w"]), "ciwbar w"); - - const snapshots: string[] = []; - await engine.executeTokens(parseKeySequences(["."]), ".", async () => { - snapshots.push(`${engine.getPublicMode()}|${engine.buffer.cursor.col}|${engine.buffer.getText()}`); - }); - - expect(engine.buffer.getText()).toBe("bar bar"); - expect(snapshots.length).toBeGreaterThan(1); - expect(snapshots.some(snapshot => snapshot.startsWith("INSERT|"))).toBe(true); - }); - - it("deletes lines and supports undo/redo", async () => { - const engine = createEngine("one\ntwo\nthree\nfour"); - await engine.executeTokens(parseKeySequences(["2G", "2dd"]), "2G 2dd"); - expect(engine.buffer.getText()).toBe("one\nfour"); - await engine.executeTokens(parseKeySequences(["u"]), "u"); - expect(engine.buffer.getText()).toBe("one\ntwo\nthree\nfour"); - await engine.executeTokens(parseKeySequences([""]), ""); - expect(engine.buffer.getText()).toBe("one\nfour"); - }); - - it("surfaces undo counts in the status message", async () => { - const engine = createEngine("alpha beta gamma"); - await engine.executeTokens(parseKeySequences(["dw", "dw"]), "dw dw"); - await engine.executeTokens(parseKeySequences(["2u"]), "2u"); - expect(engine.buffer.getText()).toBe("alpha beta gamma"); - expect(engine.statusMessage).toBe("Undid 2 changes"); - }); - - it("accepts doubled indent operators in visual mode", async () => { - const engine = createEngine("one\ntwo\nthree"); - await engine.executeTokens(parseKeySequences(["Vj>>"]), "Vj>>"); - expect(engine.buffer.getText()).toBe("\tone\n\ttwo\nthree"); - }); - - it("applies file-wide substitution through ex commands", async () => { - const engine = createEngine("alpha beta\nalpha gamma"); - await engine.executeTokens(parseKeySequences([":%s/alpha/delta/g"]), ":%s/alpha/delta/g"); - expect(engine.buffer.getText()).toBe("delta beta\ndelta gamma"); - expect(engine.statusMessage).toContain("2 substitution"); - }); - - it("deletes all lines with :%d", async () => { - const engine = createEngine("line one\nline two\nline three"); - await engine.executeTokens(parseKeySequences([":%d"]), ":%d"); - expect(engine.buffer.getText()).toBe(""); - expect(engine.statusMessage).toBe("Deleted 3 lines"); - }); - - it("supports explicit numeric ex ranges like :4,6d", async () => { - const engine = createEngine("one\ntwo\nthree\nfour\nfive\nsix\nseven"); - await engine.executeTokens(parseKeySequences([":4,6d"]), ":4,6d"); - expect(engine.buffer.getText()).toBe("one\ntwo\nthree\nseven"); - expect(engine.statusMessage).toBe("Deleted 3 lines"); - }); - - it("supports current and last-line ex addresses plus ranged :global", async () => { - const engine = createEngine("alpha\nkeep\nalpha\ntrim alpha\nfinal alpha"); - await engine.executeTokens(parseKeySequences(["2G", ":.,$g/alpha/d"]), "2G :.,$g/alpha/d"); - expect(engine.buffer.getText()).toBe("alpha\nkeep"); - expect(engine.statusMessage).toBe("Global: processed alpha"); - }); - - it("supports :join aliases with whitespace normalization", async () => { - const engine = createEngine("alpha \n beta\ngamma"); - await engine.executeTokens(parseKeySequences([":1,2j"]), ":1,2j"); - expect(engine.buffer.getText()).toBe("alpha beta\ngamma"); - expect(engine.statusMessage).toBe("Joined 2 lines"); - }); - - it("supports :join! aliases without whitespace normalization", async () => { - const engine = createEngine("alpha \n beta\ngamma"); - await engine.executeTokens(parseKeySequences([":1,2join!"]), ":1,2join!"); - expect(engine.buffer.getText()).toBe("alpha beta\ngamma"); - expect(engine.statusMessage).toBe("Joined 2 lines"); - }); - it("supports destination addresses for :copy", async () => { - const engine = createEngine("one\ntwo\nthree\nfour"); - await engine.executeTokens(parseKeySequences([":1,2t$"]), ":1,2t$"); - expect(engine.buffer.getText()).toBe("one\ntwo\nthree\nfour\none\ntwo"); - expect(engine.statusMessage).toBe("Copied 2 lines"); - }); - - it("yanks addressed lines and puts them before or after the anchor line", async () => { - const engine = createEngine("one\ntwo\nthree\nfour"); - await engine.executeTokens( - parseKeySequences([":2,3yank", "1G", ":put", "G", ":put!"]), - ":2,3yank 1G :put G :put!", - ); - expect(engine.buffer.getText()).toBe("one\ntwo\nthree\ntwo\nthree\ntwo\nthree\nfour"); - expect(engine.statusMessage).toBe("Put 2 lines"); - }); - - it("treats :update as a no-op for clean buffers and writes modified buffers", async () => { - const saveBuffer = vi.fn(async (buffer: VimBuffer) => ({ - loaded: { - absolutePath: buffer.filePath, - displayPath: buffer.displayPath, - lines: [...buffer.lines], - trailingNewline: buffer.trailingNewline, - fingerprint: null, - }, - })); - const engine = new VimEngine( - new VimBuffer({ - absolutePath: "/tmp/test.ts", - displayPath: "test.ts", - lines: ["alpha"], - trailingNewline: false, - fingerprint: null, - }), - { - beforeMutate: async () => {}, - loadBuffer: async inputPath => ({ - absolutePath: inputPath, - displayPath: inputPath, - lines: [""], - trailingNewline: false, - fingerprint: null, - }), - saveBuffer, - }, - ); - - await engine.executeTokens(parseKeySequences([":up"]), ":up"); - expect(saveBuffer).not.toHaveBeenCalled(); - expect(engine.statusMessage).toBe("test.ts unchanged"); - - await engine.executeTokens(parseKeySequences(["ccchanged", ":up"]), "ccchanged :up"); - expect(saveBuffer).toHaveBeenCalledTimes(1); - expect(engine.buffer.getText()).toBe("changed"); - expect(engine.statusMessage).toBe("Wrote test.ts"); - }); - - it("renders literal spaces visibly in unsupported command errors", async () => { - const engine = createEngine("alpha"); - await expect(engine.executeTokens(parseKeySequences(["z "]), "z ")).rejects.toThrow(/z/); - }); -}); - -describe("vim tool", () => { - let tmpDir: string; - - beforeEach(async () => { - resetSettingsForTest(); - tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "vim-tool-")); - await Settings.init({ inMemory: true, cwd: tmpDir }); - }); - - it("opens, edits, saves, and persists content", async () => { - const filePath = path.join(tmpDir, "sample.ts"); - await Bun.write(filePath, "foo = 1;\nfoo = foo + 1;\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "sample.ts" }); - await tool.execute("edit", { file: "sample.ts", steps: [step(["ciwbar", "j", "."])] }); - await tool.execute("save", { file: "sample.ts", steps: [step([":w"])] }); - - const saved = await Bun.file(filePath).text(); - expect(saved).toContain("bar = 1;"); - expect(saved).toContain("bar = foo + 1;"); - }); - - it("keeps the cursor line visible after large jumps", async () => { - const filePath = path.join(tmpDir, "long.ts"); - await Bun.write(filePath, Array.from({ length: 1100 }, (_, index) => `line ${index + 1};`).join("\n")); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "long.ts" }); - const moved = await tool.execute("jump", { file: "long.ts", steps: [step(["1014G"])] }); - const text = textResult(moved); - expect(text).toContain(">1014│line 1014;"); - expect(moved.details?.cursor.line).toBe(1014); - }); - - it("centers the viewport on the cursor after a large edit", async () => { - const filePath = path.join(tmpDir, "center.ts"); - await Bun.write(filePath, Array.from({ length: 500 }, (_, index) => `line ${index + 1};`).join("\n")); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "center.ts" }); - const edited = await tool.execute("edit", { - file: "center.ts", - steps: [step(["386Go"], "inserted")], - pause: true, - }); - expect(edited.details?.cursor.line).toBe(387); - expect(edited.details?.viewport.start).toBe(382); - expect(edited.details?.viewport.end).toBe(391); - expect(textResult(edited)).toContain("Diff:"); - expect(textResult(edited)).toContain("+inserted"); - }); - - it("recenters the viewport and includes a diff after edits", async () => { - const filePath = path.join(tmpDir, "long-edit.ts"); - await Bun.write(filePath, Array.from({ length: 1100 }, (_, index) => `line ${index + 1};`).join("\n")); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "long-edit.ts" }); - const edited = await tool.execute("edit", { file: "long-edit.ts", steps: [step(["1014G", "o"], "inserted")] }); - const text = textResult(edited); - expect(edited.details?.cursor.line).toBe(1015); - expect(edited.details?.viewport.start).toBe(1010); - expect(text).toContain("Diff:"); - expect(text).toContain("+inserted"); - }); - - it("supports raw insert payloads after kbd enters insert mode", async () => { - const filePath = path.join(tmpDir, "replace.ts"); - await Bun.write(filePath, "first\nsecond\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "replace.ts" }); - const replaced = await tool.execute("replace", { file: "replace.ts", steps: [step(["cc"], "alpha\nbeta")] }); - await tool.execute("save", { file: "replace.ts", steps: [step([":w"])] }); - - const saved = await Bun.file(filePath).text(); - expect(saved).toBe("alpha\nbeta\nsecond\n"); - expect(textResult(replaced)).toContain("Diff:"); - expect(textResult(replaced)).toContain("+beta"); - }); - - it("applies multi-step inserts at different locations", async () => { - const filePath = path.join(tmpDir, "multi-step.ts"); - await Bun.write(filePath, "import sys\n\ndef main():\n pass\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "multi-step.ts" }); - const edited = await tool.execute("edit", { - file: "multi-step.ts", - steps: [step(["1Go"], "import os"), step(["G", "o"], " os.path.exists('tmp')")], - }); - - const saved = await Bun.file(filePath).text(); - expect(saved).toBe("import sys\nimport os\n\ndef main():\n pass\n os.path.exists('tmp')\n"); - expect(textResult(edited)).toContain("+import os"); - expect(textResult(edited)).toContain("+ os.path.exists('tmp')"); - }); - - it("supports navigation-only steps between inserts", async () => { - const filePath = path.join(tmpDir, "multi-step-navigation.ts"); - await Bun.write(filePath, "alpha\nbeta\ngamma\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "multi-step-navigation.ts" }); - await tool.execute("edit", { - file: "multi-step-navigation.ts", - steps: [step(["1Go"], "between"), step(["/gamma"]), step(["o"], "tail")], - }); - - const saved = await Bun.file(filePath).text(); - expect(saved).toBe("alpha\nbetween\nbeta\ngamma\ntail\n"); - }); - - it("preserves earlier step changes when a later step fails", async () => { - const filePath = path.join(tmpDir, "multi-step-error.ts"); - await Bun.write(filePath, "alpha\nbeta\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "multi-step-error.ts" }); - await expect( - tool.execute("bad", { - file: "multi-step-error.ts", - steps: [step(["1Go"], "first"), step(["o", "o"])], - }), - ).rejects.toThrow(/entered INSERT mode/i); - - const viewed = await tool.execute("view", { file: "multi-step-error.ts" }); - expect(textResult(viewed)).toContain("first"); - expect(await Bun.file(filePath).text()).toBe("alpha\nbeta\n"); - - await tool.execute("save", { file: "multi-step-error.ts", steps: [step([":w"])] }); - expect(await Bun.file(filePath).text()).toBe("alpha\nfirst\nbeta\n"); - }); - - it("applies pause only to the last step of a multi-step edit", async () => { - const filePath = path.join(tmpDir, "multi-step-pause.ts"); - await Bun.write(filePath, "first\nsecond\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "multi-step-pause.ts" }); - const paused = await tool.execute("pause", { - file: "multi-step-pause.ts", - steps: [step(["1Go"], "alpha"), step(["G", "o"], "omega")], - pause: true, - }); - - expect(paused.details?.mode).toBe("INSERT"); - expect(textResult(paused)).toContain("Pending: INSERT mode"); - expect(await Bun.file(filePath).text()).toBe("first\nsecond\n"); - - await tool.execute("resume", { file: "multi-step-pause.ts", steps: [step([], "!")] }); - expect(await Bun.file(filePath).text()).toBe("first\nalpha\nsecond\nomega!\n"); - }); - - it("supports full-file rewrites when models emit a space before i", async () => { - const filePath = path.join(tmpDir, "full-rewrite.ts"); - await Bun.write(filePath, "first\nsecond\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "full-rewrite.ts" }); - const rewritten = await tool.execute("rewrite", { - file: "full-rewrite.ts", - steps: [step(["ggdG i"], "alpha\nbeta\n")], - }); - - const saved = await Bun.file(filePath).text(); - expect(saved).toBe("alpha\nbeta\n"); - expect(textResult(rewritten)).toContain("+alpha"); - expect(rewritten.details?.cursor.line).toBe(2); - }); - - it("rejects another kbd entry after entering insert mode", async () => { - const filePath = path.join(tmpDir, "ambiguous.ts"); - await Bun.write(filePath, "first\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "ambiguous.ts" }); - await expect(tool.execute("bad", { file: "ambiguous.ts", steps: [step(["o", "o"])] })).rejects.toThrow( - /entered INSERT mode/i, - ); - }); - - it("rejects additional kbd entries after entering insert mode", async () => { - const filePath = path.join(tmpDir, "insert-boundary.ts"); - await Bun.write(filePath, "alpha\nbeta\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "insert-boundary.ts" }); - await expect( - tool.execute("edit", { file: "insert-boundary.ts", steps: [step(["2G", "o", "o"])] }), - ).rejects.toThrow(/insert field|/i); - const saved = await Bun.file(filePath).text(); - expect(saved).toBe("alpha\nbeta\n"); - }); - - it("supports paused insert mode and resuming with a later insert payload", async () => { - const filePath = path.join(tmpDir, "pause.ts"); - await Bun.write(filePath, "first\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "pause.ts" }); - const paused = await tool.execute("pause", { file: "pause.ts", steps: [step(["cc"])], pause: true }); - expect(paused.details?.mode).toBe("INSERT"); - expect(textResult(paused)).toContain("Pending: INSERT mode"); - - await tool.execute("resume", { file: "pause.ts", steps: [step([], "replacement")] }); - await tool.execute("save", { file: "pause.ts", steps: [step([":w"])] }); - const saved = await Bun.file(filePath).text(); - expect(saved).toBe("replacement\n"); - }); - - it("rejects insert payloads outside insert mode with a snapshot error", async () => { - const filePath = path.join(tmpDir, "bad-insert.ts"); - await Bun.write(filePath, "first\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "bad-insert.ts" }); - await expect(tool.execute("bad", { file: "bad-insert.ts", steps: [step([], "nope")] })).rejects.toThrow( - /Insert payload requires INSERT mode/i, - ); - }); - - it("renders visible tab markers and a caret-focused snapshot", async () => { - const filePath = path.join(tmpDir, "tabs.ts"); - await Bun.write(filePath, "\treturn value;\n"); - const tool = new VimTool(createSession(tmpDir)); - - const opened = await tool.execute("open", { file: "tabs.ts" }); - const text = textResult(opened); - expect(text).toContain("Focus:"); - expect(text).toContain("→return value;"); - expect(text).toContain("^"); - }); - - it("renders the cursor inline in plain text viewport snapshots", async () => { - const filePath = path.join(tmpDir, "cursor.txt"); - await Bun.write(filePath, "alpha\n"); - const tool = new VimTool(createSession(tmpDir)); - - const opened = await tool.execute("open", { file: "cursor.txt" }); - expect(textResult(opened)).toContain(">1│▏alpha"); - }); - - it("shows paused search input in the snapshot", async () => { - const filePath = path.join(tmpDir, "search.ts"); - await Bun.write(filePath, "alpha\nbeta\n"); - const tool = new VimTool(createSession(tmpDir)); - - await tool.execute("open", { file: "search.ts" }); - const paused = await tool.execute("search", { file: "search.ts", steps: [step(["/be"])], pause: true }); - expect(paused.details?.pendingInput?.kind).toBe("search-forward"); - expect(textResult(paused)).toContain("Pending: /be"); - }); - - it("streams ex command input through onUpdate while typing", async () => { - const filePath = path.join(tmpDir, "command.ts"); - await Bun.write(filePath, "foo foo\n"); - const tool = new VimTool(createSession(tmpDir)); - const pendingInputs: string[] = []; - - await tool.execute("open", { file: "command.ts" }); - const result = await tool.execute( - "command", - { file: "command.ts", steps: [step([":%s/foo/bar/g"])] }, - undefined, - update => { - const pending = update.details?.pendingInput; - if (pending?.kind === "command") { - pendingInputs.push(pending.text); - } - }, - ); - - expect(pendingInputs).toContain(""); - expect(pendingInputs).toContain("%"); - expect(pendingInputs).toContain("%s/foo/bar/g"); - expect(textResult(result)).toContain("bar bar"); - }); - - it("streams large insert payloads through onUpdate in chunks", async () => { - const filePath = path.join(tmpDir, "stream-insert.ts"); - await Bun.write(filePath, "header\nfooter\n"); - const tool = new VimTool(createSession(tmpDir)); - const visibleMaxItems: number[] = []; - - await tool.execute("open", { file: "stream-insert.ts" }); - await tool.execute( - "insert", - { - file: "stream-insert.ts", - steps: [step(["2Go"], Array.from({ length: 60 }, (_, index) => `item ${index + 1}`).join("\n"))], - pause: true, - }, - undefined, - update => { - const viewportText = update.details?.viewportLines?.map(line => line.text).join("\n") ?? ""; - const matches = Array.from(viewportText.matchAll(/item (\d+)/g), match => Number(match[1])); - if (matches.length > 0) { - visibleMaxItems.push(Math.max(...matches)); - } - }, - ); - - expect(visibleMaxItems.length).toBeGreaterThan(1); - expect(visibleMaxItems.some(value => value < 60)).toBe(true); - expect(Math.max(...visibleMaxItems)).toBe(60); - }); - - it("streams single-line insert payloads through onUpdate in chunks", async () => { - const filePath = path.join(tmpDir, "stream-single-line.ts"); - await Bun.write(filePath, "alpha\nomega\n"); - const tool = new VimTool(createSession(tmpDir)); - const visibleLengths: number[] = []; - const insertedText = - "// Insert a new line after line 7 with a long comment that should render incrementally in the viewport."; - - await tool.execute("open", { file: "stream-single-line.ts" }); - await tool.execute( - "insert-single-line", - { - file: "stream-single-line.ts", - steps: [step(["2Go"], insertedText)], - pause: true, - }, - undefined, - update => { - const insertedLine = update.details?.viewportLines?.find(line => line.line === 3)?.text; - if (typeof insertedLine === "string" && insertedLine.length > 0) { - visibleLengths.push(insertedLine.length); - } - }, - ); - - expect(visibleLengths.length).toBeGreaterThan(1); - expect(visibleLengths.some(length => length < insertedText.length)).toBe(true); - expect(Math.max(...visibleLengths)).toBe(insertedText.length); - }); - - it("allows navigation in plan mode but blocks mutations", async () => { - const filePath = path.join(tmpDir, "plan.ts"); - await Bun.write(filePath, "one\ntwo\nthree\n"); - const tool = new VimTool( - createSession(tmpDir, { - getPlanModeState: () => ({ - enabled: true, - planFilePath: path.join(tmpDir, "PLAN.md"), - }), - }), - ); - - await tool.execute("open", { file: "plan.ts" }); - const moved = await tool.execute("move", { file: "plan.ts", steps: [step(["2G"])] }); - expect(textResult(moved)).toContain("L2:1"); - await expect(tool.execute("edit", { file: "plan.ts", steps: [step(["dd"])] })).rejects.toThrow(/Plan mode/i); - await expect(tool.execute("insert", { file: "plan.ts", steps: [step(["cc"], "blocked")] })).rejects.toThrow( - /Plan mode/i, - ); - }); -}); - -describe("vim renderer", () => { - it("reuses the last real vim snapshot while args stream", async () => { - const previewDir = await fs.mkdtemp(path.join(os.tmpdir(), "vim-render-preview-")); - const filePath = path.join(previewDir, "preview.ts"); - await Bun.write(filePath, Array.from({ length: 900 }, (_, index) => `line ${index + 1};`).join("\n")); - const tool = new VimTool(createSession(previewDir)); - const theme = await themeModule.getThemeByName("dark"); - expect(theme).toBeDefined(); - const uiTheme = theme!; - - await tool.execute("open", { file: "preview.ts" }); - - const component = vimToolRenderer.renderCall( - { file: "preview.ts", steps: [step(["643G"])] }, - { expanded: false, isPartial: true, spinnerFrame: 0 }, - uiTheme, - ); - - const rendered = Bun.stripANSI(component.render(160).join("\n")); - expect(rendered).toContain("643G"); - expect(rendered).toContain("line 1;"); - expect(rendered).not.toContain("line 643;"); - }); - - it("renders first-call streamed inserts as text instead of priming a preview buffer", async () => { - const theme = await themeModule.getThemeByName("dark"); - expect(theme).toBeDefined(); - const uiTheme = theme!; - - const component = vimToolRenderer.renderCall( - { - file: "preview.txt", - steps: [step(["ggdGi"])], - __partialJson: '{"file":"preview.txt","steps":[{"kbd":["ggdGi"],"insert":"replacement', - }, - { expanded: false, isPartial: true, spinnerFrame: 0 }, - uiTheme, - ); - - const rendered = Bun.stripANSI(component.render(140).join("\n")); - expect(rendered).toContain("ggdGi"); - expect(rendered).toContain("insert: replacement"); - expect(rendered).not.toContain(">1│replacement"); - }); - - it("updates streamed vim args without async preview priming", async () => { - const previewDir = await fs.mkdtemp(path.join(os.tmpdir(), "vim-render-growing-first-call-component-")); - const theme = await themeModule.getThemeByName("dark"); - expect(theme).toBeDefined(); - await themeModule.initTheme(false, undefined, undefined, "dark", "light"); - const uiStub = { requestRender() {} } as unknown as TUI; - - const component = new ToolExecutionComponent( - "edit", - { - file: "preview.txt", - steps: [step(["ggdGi"])], - __partialJson: '{"file":"preview.txt","steps":[{"kbd":["ggdGi"],"insert":"rep', - }, - {}, - undefined, - uiStub, - previewDir, - "growing-first-call-component", - ); - - let rendered = Bun.stripANSI(component.render(140).join("\n")); - expect(rendered).toContain("insert: rep"); - - component.updateArgs( - { - file: "preview.txt", - steps: [step(["ggdGi"])], - __partialJson: '{"file":"preview.txt","steps":[{"kbd":["ggdGi"],"insert":"replacement text', - }, - "growing-first-call-component", - ); - - rendered = Bun.stripANSI(component.render(140).join("\n")); - expect(rendered).toContain("insert: replacement text"); - }); - - it("caches repeated renders for the same viewport snapshot", async () => { - const theme = await themeModule.getThemeByName("dark"); - expect(theme).toBeDefined(); - const uiTheme = theme!; - const highlightSpy = vi.spyOn(themeModule, "highlightCode"); - - const component = vimToolRenderer.renderResult( - { - content: [{ type: "text", text: "" }], - details: { - file: "sample.ts", - mode: "NORMAL", - cursor: { line: 1, col: 1 }, - totalLines: 2, - modified: false, - viewport: { start: 1, end: 2 }, - viewportLines: [ - { line: 1, text: "const foo = 1;", isCursor: true, isSelected: false }, - { line: 2, text: "return foo;", isCursor: false, isSelected: false }, - ], - }, - }, - { expanded: false, isPartial: true, spinnerFrame: 0 }, - uiTheme, - ); - - component.render(120); - component.render(120); - - expect(highlightSpy).toHaveBeenCalledTimes(1); - }); - - it("renders an inline cursor highlight inside the viewport row", async () => { - const previewDir = await fs.mkdtemp(path.join(os.tmpdir(), "vim-render-inline-cursor-")); - const filePath = path.join(previewDir, "cursor.txt"); - await Bun.write(filePath, "Title line\n"); - const tool = new VimTool(createSession(previewDir)); - const theme = await themeModule.getThemeByName("dark"); - expect(theme).toBeDefined(); - const uiTheme = theme!; - - const opened = await tool.execute("open", { file: "cursor.txt" }); - const rendered = vimToolRenderer - .renderResult(opened, { expanded: false, isPartial: false, spinnerFrame: 0 }, uiTheme) - .render(160) - .join("\n"); - - expect(rendered).toMatch(/\x1b\[7mT/); - }); - - it("keeps long cursor rows horizontally centered around the cursor", async () => { - const previewDir = await fs.mkdtemp(path.join(os.tmpdir(), "vim-render-long-line-cursor-")); - const filePath = path.join(previewDir, "cursor.txt"); - await Bun.write(filePath, `prefix-${"x".repeat(220)};`); - const tool = new VimTool(createSession(previewDir)); - const theme = await themeModule.getThemeByName("dark"); - expect(theme).toBeDefined(); - const uiTheme = theme!; - - await tool.execute("open", { file: "cursor.txt" }); - const moved = await tool.execute("move", { file: "cursor.txt", steps: [step(["$"])] }); - expect(moved.details?.viewportLines?.[0]?.text.startsWith("…")).toBe(true); - - const rendered = vimToolRenderer - .renderResult(moved, { expanded: false, isPartial: false, spinnerFrame: 0 }, uiTheme) - .render(200) - .join("\n"); - - expect(rendered).toMatch(/\x1b\[7m;/); - }); -}); diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index db01baa2d..8b65463d8 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed bash heredocs (`<<`) and here-strings (`<<<`) deadlocking the shell on Windows past ~4 KiB and on macOS past 16-64 KiB. `brush_core::interp::setup_open_file_with_contents` wrote the entire body into an anonymous pipe synchronously before handing the reader to the next command; once the body exceeded the OS pipe buffer the writer blocked forever and the `bash` tool timed out at the hard 305 s ceiling without ever launching the consumer. The Linux fast path still uses `F_SETPIPE_SZ` to grow the pipe in-place; every other OS-threaded platform (and Linux bodies above `pipe-max-size`) now decouples the write onto a fire-and-forget thread that terminates naturally on drain or `BrokenPipe`; no-thread targets keep the upstream synchronous path so heredocs do not fail at thread spawn. + ## [15.3.2] - 2026-05-25 ### Fixed diff --git a/packages/natives/native/index.d.ts b/packages/natives/native/index.d.ts index f69afa7d1..c1f5649a6 100644 --- a/packages/natives/native/index.d.ts +++ b/packages/natives/native/index.d.ts @@ -136,7 +136,7 @@ export declare class Shell { * `packages/natives/native/index.js` (which derives the name from * `package.json#version`). */ -export declare function __piNativesV15_3_2(): void +export declare function __piNativesV15_4_3(): void /** * Apply conservative pre-execution rewrites to a bash command. diff --git a/packages/natives/native/index.js b/packages/natives/native/index.js index 13aba7fe0..ed08720f1 100644 --- a/packages/natives/native/index.js +++ b/packages/natives/native/index.js @@ -23,7 +23,7 @@ export const PtySession = nativeBindings.PtySession; export const Shell = nativeBindings.Shell; // functions -export const __piNativesV15_3_2 = nativeBindings.__piNativesV15_3_2; +export const __piNativesV15_4_3 = nativeBindings.__piNativesV15_4_3; export const applyBashFixups = nativeBindings.applyBashFixups; export const astEdit = nativeBindings.astEdit; export const astGrep = nativeBindings.astGrep; diff --git a/packages/natives/package.json b/packages/natives/package.json index 55f912968..7dcf6f74a 100644 --- a/packages/natives/package.json +++ b/packages/natives/package.json @@ -1,6 +1,6 @@ { "name": "@oh-my-pi/pi-natives", - "version": "15.3.2", + "version": "15.4.3", "description": "Native Rust bindings for grep, clipboard, image processing, syntax highlighting, PTY, and shell operations via N-API", "type": "module", "homepage": "https://omp.sh", diff --git a/packages/stats/package.json b/packages/stats/package.json index e2ca8b601..5cc511ec6 100644 --- a/packages/stats/package.json +++ b/packages/stats/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/omp-stats", - "version": "15.3.2", + "version": "15.4.3", "description": "Local observability dashboard for pi AI usage statistics", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/swarm-extension/package.json b/packages/swarm-extension/package.json index 49b725bbd..5d190fdd0 100644 --- a/packages/swarm-extension/package.json +++ b/packages/swarm-extension/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/swarm-extension", - "version": "15.3.2", + "version": "15.4.3", "description": "Swarm orchestration extension for omp", "homepage": "https://omp.sh", "author": "Derek Rynd", diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 6d0a8d527..49349b488 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -4,6 +4,12 @@ ### Fixed +- Fixed `@` file mention autocomplete stalling for seconds when the query references something outside the project root (e.g. `@../`, `@~/`, `@/abs/`). `CombinedAutocompleteProvider` now short-circuits to plain immediate-directory prefix listing in those cases instead of dispatching a recursive `fuzzyFind` walk over a sibling directory full of unrelated projects. Inside-cwd queries keep the existing fuzzy-then-prefix behavior. ([#1395](https://github.com/can1357/oh-my-pi/issues/1395)) + +## [15.4.0] - 2026-05-26 + +### Fixed + - Fixed terminal scrollback gaining duplicate copies of the welcome screen (and any other header content) when the bottom tool cell mutated across the previous viewport boundary. Once a row scrolls into terminal history it cannot be retracted, so a subsequent shrink that would re-expose that row in the repainted viewport now clears stale scrollback and replays the transcript, then suppresses one immediate suffix-scroll frame so live status/editor chrome is not deposited twice. Multiplexer panes ignore `\x1b[3J`, so the recovery is gated on `!isMultiplexerSession()`. - Fixed the IME / hardware cursor sticking to the bottom of the terminal after a resize that grew the viewport taller than the rendered transcript. `#emitViewportRepaint` always writes one row per screen line (padding empty rows past the content), so the post-write hardware cursor sits at screen row `height - 1`. The bookkeeping previously clamped the tracked cursor row to `lines.length - 1`, making `#cursorControlSequence`'s relative `rowDelta` underestimate the upward move by `(height - lines.length)` rows and pinning the cursor at the viewport bottom even though the focused component's `CURSOR_MARKER` was on a content row. diff --git a/packages/tui/package.json b/packages/tui/package.json index fd42317e7..ef01597b8 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-tui", - "version": "15.3.2", + "version": "15.4.3", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/tui/src/autocomplete.ts b/packages/tui/src/autocomplete.ts index 44f8c28e3..8e86233ae 100644 --- a/packages/tui/src/autocomplete.ts +++ b/packages/tui/src/autocomplete.ts @@ -237,6 +237,17 @@ export class CombinedAutocompleteProvider implements AutocompleteProvider { const atPrefix = this.#extractAtPrefix(textBeforeCursor); if (atPrefix) { const { rawPrefix, isQuotedPrefix } = parsePathPrefix(atPrefix); + // Recursive fuzzy walks rooted outside the project (e.g. `@../`, + // `@~/`, `@/abs`) can be huge — a parent dir full of sibling + // projects blows past several seconds of latency. Outside cwd, + // fall back to plain prefix listing of the immediate directory + // (matches Claude Code's behavior). Inside cwd we keep the + // fuzzy-then-prefix flow. + if (rawPrefix.length > 0 && this.#isOutsideCwd(rawPrefix)) { + const items = await this.#getFileSuggestions(atPrefix); + if (items.length === 0) return null; + return { items, prefix: atPrefix }; + } const suggestions = rawPrefix.length > 0 ? await this.#getFuzzyFileSuggestions(rawPrefix, { isQuotedPrefix }) @@ -479,6 +490,28 @@ export class CombinedAutocompleteProvider implements AutocompleteProvider { return filePath; } + // Resolve `rawPrefix` lexically (no I/O) and report whether it points + // somewhere outside `this.#basePath`. Used to skip recursive fuzzy walks + // rooted at parent / absolute / home paths — those routinely include + // thousands of unrelated files and stall the UI for seconds. + #isOutsideCwd(rawPrefix: string): boolean { + if (rawPrefix.length === 0) return false; + let target: string; + if (rawPrefix.startsWith("~")) { + target = this.#expandHomePath(rawPrefix); + } else if (path.isAbsolute(rawPrefix)) { + target = rawPrefix; + } else { + target = path.resolve(this.#basePath, rawPrefix); + } + const rel = path.relative(this.#basePath, target); + if (rel === "" || rel === ".") return false; + if (path.isAbsolute(rel)) return true; + const firstSep = rel.indexOf(path.sep); + const head = firstSep === -1 ? rel : rel.slice(0, firstSep); + return head === ".."; + } + async #resolveScopedFuzzyQuery( rawQuery: string, ): Promise<{ baseDir: string; query: string; displayBase: string } | null> { diff --git a/packages/tui/src/components/settings-list.ts b/packages/tui/src/components/settings-list.ts index ed8148afa..3024ddb88 100644 --- a/packages/tui/src/components/settings-list.ts +++ b/packages/tui/src/components/settings-list.ts @@ -15,11 +15,13 @@ export interface SettingItem { values?: string[]; /** If provided, Enter opens this submenu. Receives current value and done callback. */ submenu?: (currentValue: string, done: (selectedValue?: string) => void) => Component; + /** True when the displayed setting differs from its default value. */ + changed?: boolean; } export interface SettingsListTheme { - label: (text: string, selected: boolean) => string; - value: (text: string, selected: boolean) => string; + label: (text: string, selected: boolean, changed: boolean) => string; + value: (text: string, selected: boolean, changed: boolean) => string; description: (text: string) => string; cursor: string; hint: (text: string) => string; @@ -117,7 +119,7 @@ export class SettingsList implements Component { // Pad label to align values const labelPadded = item.label + padding(Math.max(0, maxLabelWidth - visibleWidth(item.label))); - const labelText = this.#theme.label(labelPadded, isSelected); + const labelText = this.#theme.label(labelPadded, isSelected, item.changed === true); // Calculate space for value const separator = " "; @@ -127,6 +129,7 @@ export class SettingsList implements Component { const valueText = this.#theme.value( truncateToWidth(item.currentValue, valueMaxWidth, Ellipsis.Omit), isSelected, + item.changed === true, ); lines.push(truncateToWidth(prefix + labelText + separator + valueText, width)); diff --git a/packages/tui/test/autocomplete.test.ts b/packages/tui/test/autocomplete.test.ts index f79714b57..0287e659a 100644 --- a/packages/tui/test/autocomplete.test.ts +++ b/packages/tui/test/autocomplete.test.ts @@ -100,7 +100,7 @@ describe("CombinedAutocompleteProvider", () => { }); }); - describe("@ fuzzy search scoped paths", () => { + describe("@ paths outside cwd", () => { let rootDir: string; let baseDir: string; let outsideDir: string; @@ -117,22 +117,41 @@ describe("CombinedAutocompleteProvider", () => { fs.rmSync(rootDir, { recursive: true, force: true }); }); - it("scopes @ fuzzy search to the typed relative path prefix", async () => { - fs.writeFileSync(path.join(baseDir, "alpha-local.ts"), "export const local = 1;\n"); - fs.mkdirSync(path.join(outsideDir, "nested", "deeper"), { recursive: true }); - fs.writeFileSync(path.join(outsideDir, "nested", "alpha.ts"), "export const alpha = 1;\n"); - fs.writeFileSync(path.join(outsideDir, "nested", "deeper", "also-alpha.ts"), "export const also = 1;\n"); - fs.writeFileSync(path.join(outsideDir, "nested", "deeper", "zzz.ts"), "export const zzz = 1;\n"); + it("uses immediate-directory prefix completion for @../ (no recursive fuzzy walk)", async () => { + // Sibling-of-cwd layout, mirroring the user-reported case: parent + // dir holds many unrelated projects, each with deep subtrees. + fs.mkdirSync(path.join(outsideDir, "workspace"), { recursive: true }); + fs.mkdirSync(path.join(outsideDir, "workflows"), { recursive: true }); + fs.mkdirSync(path.join(outsideDir, "other"), { recursive: true }); + fs.mkdirSync(path.join(outsideDir, "other", "deep", "nested"), { recursive: true }); + fs.writeFileSync(path.join(outsideDir, "other", "deep", "nested", "workspace-config.yml"), "x\n"); const provider = new CombinedAutocompleteProvider([], baseDir); - const line = "@../outside/a"; + const line = "@../outside/wor"; const result = await provider.getSuggestions([line], 0, line.length); const values = result?.items.map(item => item.value) ?? []; - expect(values).toContain("@../outside/nested/alpha.ts"); - expect(values).toContain("@../outside/nested/deeper/also-alpha.ts"); - expect(values).not.toContain("@../outside/nested/deeper/zzz.ts"); - expect(values.some(value => value.includes("alpha-local.ts"))).toBe(false); + expect(values).toContain("@../outside/workspace/"); + expect(values).toContain("@../outside/workflows/"); + // Recursive matches must NOT leak in — that's the whole point of + // the short-circuit. + expect(values.some(value => value.includes("workspace-config.yml"))).toBe(false); + expect(values.some(value => value.includes("/deep/"))).toBe(false); + }); + + it("lists entries inside an absolute @/abs/ path without walking recursively", async () => { + fs.mkdirSync(path.join(outsideDir, "alpha"), { recursive: true }); + fs.mkdirSync(path.join(outsideDir, "beta"), { recursive: true }); + fs.writeFileSync(path.join(outsideDir, "alpha", "nested.ts"), "export {};\n"); + + const provider = new CombinedAutocompleteProvider([], baseDir); + const line = `@${outsideDir}/`; + const result = await provider.getSuggestions([line], 0, line.length); + + const values = result?.items.map(item => item.value) ?? []; + expect(values).toContain(`@${outsideDir}/alpha/`); + expect(values).toContain(`@${outsideDir}/beta/`); + expect(values.some(value => value.endsWith("nested.ts"))).toBe(false); }); }); describe("dot-slash path completion", () => { diff --git a/packages/tui/test/settings-list.test.ts b/packages/tui/test/settings-list.test.ts index ad7d87134..e75d769a1 100644 --- a/packages/tui/test/settings-list.test.ts +++ b/packages/tui/test/settings-list.test.ts @@ -44,4 +44,30 @@ describe("SettingsList", () => { expect(changes).toEqual([["mode", "on"]]); }); + + it("passes changed state to item label and value renderers", () => { + const themed: SettingsListTheme = { + label: (text: string, _selected: boolean, changed: boolean) => (changed ? `[changed-label]${text}` : text), + value: (text: string, _selected: boolean, changed: boolean) => (changed ? `[changed-value]${text}` : text), + description: (text: string) => text, + cursor: "→ ", + hint: (text: string) => text, + }; + const list = new SettingsList( + [ + { id: "default", label: "Default", currentValue: "off", values: ["off", "on"] }, + { id: "changed", label: "Changed", currentValue: "on", values: ["off", "on"], changed: true }, + ], + 5, + themed, + () => {}, + () => {}, + ); + + const output = list.render(80).join("\n"); + + expect(output).toContain("[changed-label]Changed"); + expect(output).toContain("[changed-value]on"); + expect(output).not.toContain("[changed-label]Default"); + }); }); diff --git a/packages/typescript-edit-benchmark/src/in-process-client.ts b/packages/typescript-edit-benchmark/src/in-process-client.ts index d69cc7d61..9ae3024e7 100644 --- a/packages/typescript-edit-benchmark/src/in-process-client.ts +++ b/packages/typescript-edit-benchmark/src/in-process-client.ts @@ -12,7 +12,7 @@ import { type CreateAgentSessionResult, createAgentSession, discoverAuthStorage, - type ModelRegistry, + ModelRegistry, SessionManager, Settings, } from "@oh-my-pi/pi-coding-agent"; @@ -49,24 +49,28 @@ export interface DiscoverSharedInfraOptions { /** Discover shared infrastructure once for the entire benchmark run. */ export async function discoverSharedInfra(options: DiscoverSharedInfraOptions = {}): Promise { - const { ModelRegistry: MR } = await import("@oh-my-pi/pi-coding-agent"); const authStorage = await discoverAuthStorage(); - const modelRegistry = new MR(authStorage); + try { + const modelRegistry = new ModelRegistry(authStorage); - // Initialize global Settings singleton (required by code paths that use the global `settings` proxy) - const overrides: Record = {}; - if (options.editVariant && options.editVariant !== "auto") { - overrides["edit.mode"] = options.editVariant; - } - if (options.editFuzzy !== undefined && options.editFuzzy !== "auto") { - overrides["edit.fuzzyMatch"] = options.editFuzzy; - } - if (options.editFuzzyThreshold !== undefined && options.editFuzzyThreshold !== "auto") { - overrides["edit.fuzzyThreshold"] = options.editFuzzyThreshold; - } - await Settings.init({ cwd: options.cwd, overrides }); + // Initialize global Settings singleton (required by code paths that use the global `settings` proxy) + const overrides: Record = {}; + if (options.editVariant && options.editVariant !== "auto") { + overrides["edit.mode"] = options.editVariant; + } + if (options.editFuzzy !== undefined && options.editFuzzy !== "auto") { + overrides["edit.fuzzyMatch"] = options.editFuzzy; + } + if (options.editFuzzyThreshold !== undefined && options.editFuzzyThreshold !== "auto") { + overrides["edit.fuzzyThreshold"] = options.editFuzzyThreshold; + } + await Settings.init({ cwd: options.cwd, overrides }); - return { authStorage, modelRegistry }; + return { authStorage, modelRegistry }; + } catch (error) { + authStorage.close(); + throw error; + } } /** diff --git a/packages/typescript-edit-benchmark/src/index.ts b/packages/typescript-edit-benchmark/src/index.ts index 9917270ad..f2a2052ca 100755 --- a/packages/typescript-edit-benchmark/src/index.ts +++ b/packages/typescript-edit-benchmark/src/index.ts @@ -513,8 +513,12 @@ async function main(): Promise { console.log(""); console.log("Benchmark complete!"); - console.log(` Success rate: ${(result.summary.overallSuccessRate * 100).toFixed(1)}%`); - console.log(` Total tokens: ${result.summary.totalTokens.input} in / ${result.summary.totalTokens.output} out`); + console.log( + ` Task success rate (best of ${config.runsPerTask}): ${(result.summary.taskSuccessRate * 100).toFixed(1)}% (${result.summary.successfulTasks}/${result.summary.totalTasks})`, + ); + console.log( + ` Total tokens (best): ${result.summary.totalTokens.input} in / ${result.summary.totalTokens.output} out`, + ); if (result.summary.ghostRuns > 0) { console.log(` Ghost runs (0/0/0): ${result.summary.ghostRuns}`); } @@ -529,6 +533,11 @@ async function main(): Promise { if (cleanup) { await cleanup(); } + + // In-process benchmark runs can leave provider keep-alive sockets and + // background AgentSession timers alive after the report is written. Treat the + // final report as the CLI boundary so the command returns to the shell. + await postmortem.quit(0); } class LiveProgress { @@ -711,7 +720,7 @@ class LiveProgress { } } -main().catch(err => { +main().catch(async err => { console.error("Benchmark failed:", err); - process.exit(1); + await postmortem.quit(1); }); diff --git a/packages/typescript-edit-benchmark/src/report.ts b/packages/typescript-edit-benchmark/src/report.ts index d8350bc1b..9e0de939e 100644 --- a/packages/typescript-edit-benchmark/src/report.ts +++ b/packages/typescript-edit-benchmark/src/report.ts @@ -5,22 +5,28 @@ import { formatDuration, formatPercent, truncate } from "@oh-my-pi/pi-utils"; import { type BenchmarkResult, EDIT_FAILURE_CATEGORIES, type TaskResult } from "./runner"; -function getStatusEmoji(successRate: number, runsPerTask: number): string { - const passing = Math.round(successRate * runsPerTask); - if (passing === runsPerTask) return "✅"; - if (passing === 0) return "❌"; - return "⚠️"; +function formatBestStatus(task: TaskResult, runsPerTask: number): { status: string; label: string } { + const completed = task.runs.filter(run => !isCompletedGhost(run)).length; + const succeeded = task.runs.filter(run => run.success).length; + if (task.success) { + // best-of-N pass; flag flakiness when not every run succeeded. + const flaky = completed > 0 && succeeded < completed; + const status = flaky ? "⚠️" : "✅"; + const label = `PASS (${succeeded}/${completed || runsPerTask})`; + return { status, label }; + } + return { status: "❌", label: `FAIL (0/${completed || runsPerTask})` }; +} + +function isCompletedGhost(run: TaskResult["runs"][number]): boolean { + if (run.success) return false; + return run.tokens.total === 0 && run.toolCalls.read === 0 && run.toolCalls.edit === 0 && run.toolCalls.write === 0; } function formatNumber(n: number): string { return n.toLocaleString(); } -function formatPassRate(successRate: number, runsPerTask: number): string { - const passing = Math.round(successRate * runsPerTask); - return `${passing}/${runsPerTask}`; -} - function formatRate(numerator: number, denominator: number): string { if (denominator === 0) return "—"; const percent = (numerator / denominator) * 100; @@ -82,7 +88,6 @@ export function generateReport(result: BenchmarkResult): string { ); const verifiedRuns = nonGhostRuns.filter(run => run.verificationPassed).length; const editToolRuns = nonGhostRuns.filter(run => run.patchApplied).length; - const successRuns = nonGhostRuns.filter(run => run.success).length; const totalEditAttempts = nonGhostRuns.reduce((sum, run) => sum + run.toolCalls.edit, 0); const totalEditFailures = nonGhostRuns.reduce((sum, run) => sum + run.toolCalls.editFailures, 0); @@ -115,17 +120,21 @@ export function generateReport(result: BenchmarkResult): string { lines.push("## Summary"); lines.push(""); + lines.push( + "Primary metrics (tokens, duration, tool calls) are aggregated over the **best run** of each task. Diagnostic counts (ghost runs, timeouts, retries, failure categories) span every executed run.", + ); + lines.push(""); lines.push("| Metric | Value |"); lines.push("|--------|-------|"); lines.push(`| Total Tasks | ${summary.totalTasks} |`); lines.push(`| Total Runs | ${summary.totalRuns} |`); lines.push(`| Successful Runs | ${summary.successfulRuns} |`); - lines.push(`| **Task Success Rate** | **${formatRate(successRuns, summary.totalRuns)}** |`); + lines.push(`| **Task Success Rate** | **${formatRate(summary.successfulTasks, summary.totalTasks)}** |`); if (config.editVariant === "hashline") { lines.push( - `| **Autocorrect-Free Success Rate** | **${formatRate(summary.autocorrectFreeSuccessfulRuns, summary.totalRuns)}** |`, + `| **Autocorrect-Free Success Rate** | **${formatRate(summary.autocorrectFreeSuccessfulTasks, summary.totalTasks)}** |`, ); - lines.push(`| Autocorrected Runs | ${formatRate(summary.autocorrectedRuns, summary.totalRuns)} |`); + lines.push(`| Autocorrected Best Runs | ${formatRate(summary.autocorrectedBestRuns, summary.totalTasks)} |`); lines.push(`| Edit Autocorrect Rate | ${formatPercent(summary.editAutocorrectRate)} |`); } lines.push(`| Verified Rate | ${formatRate(verifiedRuns, summary.totalRuns)} |`); @@ -149,34 +158,36 @@ export function generateReport(result: BenchmarkResult): string { if (config.editVariant === "patch" || config.editVariant === "hashline") { lines.push(`| Patch Failure Rate | ${formatRate(totalEditFailures, totalEditAttempts)} |`); } - lines.push(`| Tasks All Passing | ${summary.tasksWithAllPassing} |`); - lines.push(`| Tasks Flaky/Failing | ${summary.tasksWithAnyFailing} |`); + lines.push(`| Tasks All Passing | ${summary.consistentlyPassingTasks} |`); + lines.push(`| Tasks Flaky/Failing | ${summary.totalTasks - summary.consistentlyPassingTasks} |`); lines.push(""); lines.push("### Tool Calls"); lines.push(""); - lines.push("| Tool | Total | Avg/Run |"); - lines.push("|------|-------|---------|"); - lines.push(`| Read | ${summary.totalToolCalls.read} | ${summary.avgToolCallsPerRun.read.toFixed(1)} |`); - lines.push(`| Edit | ${summary.totalToolCalls.edit} | ${summary.avgToolCallsPerRun.edit.toFixed(1)} |`); - lines.push(`| Write | ${summary.totalToolCalls.write} | ${summary.avgToolCallsPerRun.write.toFixed(1)} |`); + lines.push("| Tool | Total (best) | Avg/Task |"); + lines.push("|------|--------------|----------|"); + lines.push(`| Read | ${summary.totalToolCalls.read} | ${summary.avgToolCallsPerTask.read.toFixed(1)} |`); + lines.push(`| Edit | ${summary.totalToolCalls.edit} | ${summary.avgToolCallsPerTask.edit.toFixed(1)} |`); + lines.push(`| Write | ${summary.totalToolCalls.write} | ${summary.avgToolCallsPerTask.write.toFixed(1)} |`); lines.push( - `| **Tool Input Chars** | ${formatNumber(summary.totalToolCalls.totalInputChars)} | ${formatNumber(Math.round(summary.avgToolCallsPerRun.totalInputChars))} |`, + `| **Tool Input Chars** | ${formatNumber(summary.totalToolCalls.totalInputChars)} | ${formatNumber(Math.round(summary.avgToolCallsPerTask.totalInputChars))} |`, ); lines.push(""); lines.push("### Tokens & Time"); lines.push(""); - lines.push("| Metric | Total | Avg/Run |"); - lines.push("|--------|-------|---------|"); + lines.push("| Metric | Total (best) | Avg/Task |"); + lines.push("|--------|--------------|----------|"); lines.push( - `| Input Tokens | ${formatNumber(summary.totalTokens.input)} | ${formatNumber(summary.avgTokensPerRun.input)} |`, + `| Input Tokens | ${formatNumber(summary.totalTokens.input)} | ${formatNumber(summary.avgTokensPerTask.input)} |`, ); lines.push( - `| Output Tokens | ${formatNumber(summary.totalTokens.output)} | ${formatNumber(summary.avgTokensPerRun.output)} |`, + `| Output Tokens | ${formatNumber(summary.totalTokens.output)} | ${formatNumber(summary.avgTokensPerTask.output)} |`, ); lines.push( - `| Total Tokens | ${formatNumber(summary.totalTokens.total)} | ${formatNumber(summary.avgTokensPerRun.total)} |`, + `| Total Tokens | ${formatNumber(summary.totalTokens.total)} | ${formatNumber(summary.avgTokensPerTask.total)} |`, + ); + lines.push( + `| Duration | ${formatDuration(summary.totalDuration)} | ${formatDuration(summary.avgDurationPerTask)} |`, ); - lines.push(`| Duration | ${formatDuration(summary.totalDuration)} | ${formatDuration(summary.avgDurationPerRun)} |`); lines.push(`| **Avg Indent Score** | — | **${formatScore(summary.avgIndentScore)}** |`); lines.push(""); @@ -222,12 +233,11 @@ export function generateReport(result: BenchmarkResult): string { lines.push("|------|------|---------|----------|-------|-----------------|------|--------|"); for (const task of tasks) { - const status = getStatusEmoji(task.successRate, runsPerTask); - const passRate = formatPassRate(task.successRate, runsPerTask); + const { status, label } = formatBestStatus(task, runsPerTask); const editHitRate = formatPercent(task.editSuccessRate); - const toolCalls = `${task.avgToolCalls.read.toFixed(0)}/${task.avgToolCalls.edit.toFixed(0)}/${task.avgToolCalls.write.toFixed(0)}`; + const toolCalls = `${task.toolCalls.read.toFixed(0)}/${task.toolCalls.edit.toFixed(0)}/${task.toolCalls.write.toFixed(0)}`; lines.push( - `| ${escapeMarkdown(task.name)} | ${escapeMarkdown(formatFiles(task.files))} | ${passRate} ${status} | ${editHitRate} | ${toolCalls} | ${formatNumber(task.avgTokens.input)}/${formatNumber(task.avgTokens.output)} | ${formatDuration(task.avgDuration)} | ${formatScore(task.avgIndentScore)} |`, + `| ${escapeMarkdown(task.name)} | ${escapeMarkdown(formatFiles(task.files))} | ${label} ${status} | ${editHitRate} | ${toolCalls} | ${formatNumber(task.tokens.input)}/${formatNumber(task.tokens.output)} | ${formatDuration(task.duration)} | ${formatScore(task.indentScore)} |`, ); } lines.push(""); @@ -279,30 +289,38 @@ export function generateReport(result: BenchmarkResult): string { } } - const flakyTasks = tasks.filter(t => t.successRate > 0 && t.successRate < 1); + const flakyTasks = tasks.filter(task => { + if (!task.success) return false; + const nonGhost = task.runs.filter(run => !isCompletedGhost(run)); + return nonGhost.length > 0 && nonGhost.some(run => !run.success); + }); if (flakyTasks.length > 0) { - lines.push("## Flaky Tasks (partial passing)"); + lines.push("## Flaky Tasks (best passed; some runs failed)"); lines.push(""); for (const task of flakyTasks) { - const passing = Math.round(task.successRate * runsPerTask); - lines.push(`### ${task.name} (${formatFiles(task.files)}) — ${passing}/${runsPerTask}`); + const nonGhost = task.runs.filter(run => !isCompletedGhost(run)); + const passing = nonGhost.filter(run => run.success).length; + const denom = nonGhost.length || runsPerTask; + const bestNote = task.bestRunIndex >= 0 ? ` (best: run ${task.bestRunIndex + 1})` : ""; + lines.push(`### ${task.name} (${formatFiles(task.files)}) — ${passing}/${denom}${bestNote}`); lines.push(""); lines.push("| Run | Status | Error | Tokens (in/out) | Time |"); lines.push("|-----|--------|-------|-----------------|------|"); for (const run of task.runs) { + const marker = run.runIndex === task.bestRunIndex ? " ★" : ""; const status = run.success ? "✅" : "❌"; const error = run.error ? truncate(escapeMarkdown(run.error), 50) : "—"; lines.push( - `| ${run.runIndex + 1} | ${status} | ${error} | ${formatNumber(run.tokens.input)} / ${formatNumber(run.tokens.output)} | ${formatDuration(run.duration)} |`, + `| ${run.runIndex + 1}${marker} | ${status} | ${error} | ${formatNumber(run.tokens.input)} / ${formatNumber(run.tokens.output)} | ${formatDuration(run.duration)} |`, ); } lines.push(""); } } - const failedTasks = tasks.filter(t => t.successRate === 0); + const failedTasks = tasks.filter(task => !task.success); if (failedTasks.length > 0) { lines.push("## Failed Tasks (0% passing)"); lines.push(""); diff --git a/packages/typescript-edit-benchmark/src/runner.ts b/packages/typescript-edit-benchmark/src/runner.ts index 06dd86af5..f9e4a00b7 100644 --- a/packages/typescript-edit-benchmark/src/runner.ts +++ b/packages/typescript-edit-benchmark/src/runner.ts @@ -9,7 +9,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import type { AgentMessage, ResolvedThinkingLevel, ThinkingLevel } from "@oh-my-pi/pi-agent-core"; import type { Model } from "@oh-my-pi/pi-ai"; -import { computeLineHash, formatSessionDumpText, RpcClient } from "@oh-my-pi/pi-coding-agent"; +import { computeFileHash, formatSessionDumpText, RpcClient } from "@oh-my-pi/pi-coding-agent"; import { prompt } from "@oh-my-pi/pi-utils"; import { diffLines } from "diff"; import { formatDirectory } from "./formatter"; @@ -42,7 +42,7 @@ type ConversationDumpSessionState = { /** Common interface for both RPC and in-process clients */ interface BenchmarkClient { start(): Promise; - setThinkingLevel(level: import("@oh-my-pi/pi-agent-core").ResolvedThinkingLevel): Promise; + setThinkingLevel(level: ResolvedThinkingLevel): Promise; onEvent(listener: (event: { type: string; [key: string]: unknown }) => void): () => void; prompt(text: string): Promise; followUp(text: string): Promise; @@ -294,27 +294,30 @@ function buildMutationPreviewAgainstOriginal(original: string, current: string): const changes = diffLines(original, current); const preview: string[] = []; - let lineNum = 1; + let origLineNum = 1; + let newLineNum = 1; + // Hashline diff-preview format: `-LINE:TEXT` for removed (pre-edit line + // number), `+LINE:TEXT` for added (post-edit line number). No per-line hash. for (const change of changes) { const lines = splitLines(change.value); if (!change.added && !change.removed) { - lineNum += lines.length; + origLineNum += lines.length; + newLineNum += lines.length; continue; } if (change.removed) { for (const line of lines) { - const hash = computeLineHash(lineNum, line); - preview.push(`${lineNum}#${hash}|-${line}`); - lineNum += 1; + preview.push(`-${origLineNum}:${line}`); + origLineNum += 1; } continue; } for (const line of lines) { - const hash = computeLineHash(lineNum, line); - preview.push(`${lineNum}#${hash}|+${line}`); + preview.push(`+${newLineNum}:${line}`); + newLineNum += 1; } } @@ -524,69 +527,56 @@ async function evaluateMutationIntent( }; } -type GuidedHashlineEdit = - | { set: { ref: string; body: string[] } } - | { set_range: { beg: string; end: string; body: string[] } } - | { insert: { after: string; body: string[] } }; - -function buildGuidedHashlineEdits(actual: string, expected: string): GuidedHashlineEdit[] { +/** + * Build a textual hashline patch (with `¶path#hash` section header) that + * transforms `actual` into `expected`. Returns null when no changes are + * needed or the diff isn't expressible as straight insert/replace/delete ops. + */ +function buildGuidedHashlinePatch(file: string, actual: string, expected: string): string | null { const changes = diffLines(actual, expected); const actualLines = actual.split("\n"); + // File-trailing newline produces a phantom empty last entry that is not a + // real line; the hashline grammar's line numbers count real lines only. + const fileLineCount = + actualLines.length > 0 && actualLines[actualLines.length - 1] === "" + ? actualLines.length - 1 + : actualLines.length; + const ops: string[] = []; let line = 1; let pendingStart = 1; - let pendingRemoved: string[] = []; + let pendingRemoved = 0; let pendingAdded: string[] = []; - const edits: GuidedHashlineEdit[] = []; + + const formatPayload = (body: string[]): string => (body.length === 0 ? "" : `\n${body.join("\n")}`); const flush = () => { - if (pendingRemoved.length === 0 && pendingAdded.length === 0) { - return; - } + if (pendingRemoved === 0 && pendingAdded.length === 0) return; - if (pendingRemoved.length === 0) { - const insertLine = pendingStart; + if (pendingRemoved === 0) { + // Pure insertion at `pendingStart` (line numbers are 1-indexed and + // refer to the pre-edit file). if (pendingAdded.length === 0) return; - if (insertLine === 1) { - const firstLine = actualLines[0] ?? ""; - const firstRef = `1#${computeLineHash(1, firstLine)}`; - edits.push({ - set: { ref: firstRef, body: [...pendingAdded, firstLine] }, - }); - } else if (insertLine <= actualLines.length) { - const afterLine = actualLines[insertLine - 2] ?? ""; - const afterRef = `${insertLine - 1}#${computeLineHash(insertLine - 1, afterLine)}`; - edits.push({ - insert: { after: afterRef, body: [...pendingAdded] }, - }); - } else if (insertLine === actualLines.length + 1 && actualLines.length > 0) { - const afterLine = actualLines[actualLines.length - 1] ?? ""; - const afterRef = `${actualLines.length}#${computeLineHash(actualLines.length, afterLine)}`; - edits.push({ - insert: { after: afterRef, body: [...pendingAdded] }, - }); + if (pendingStart <= 1) { + ops.push(`BOF↓${formatPayload(pendingAdded)}`); + } else if (pendingStart > fileLineCount) { + ops.push(`EOF↓${formatPayload(pendingAdded)}`); + } else { + // Insert above `pendingStart` so the new content lands at that line. + ops.push(`${pendingStart}↑${formatPayload(pendingAdded)}`); } } else { const startLine = pendingStart; - const endLine = pendingStart + pendingRemoved.length - 1; - const startContent = actualLines[startLine - 1] ?? ""; - const startRef = `${startLine}#${computeLineHash(startLine, startContent)}`; - if (startLine === endLine) { - edits.push({ set: { ref: startRef, body: [...pendingAdded] } }); + const endLine = pendingStart + pendingRemoved - 1; + const anchor = startLine === endLine ? `${startLine}` : `${startLine}-${endLine}`; + if (pendingAdded.length === 0) { + ops.push(`${anchor}!`); } else { - const endContent = actualLines[endLine - 1] ?? ""; - const endRef = `${endLine}#${computeLineHash(endLine, endContent)}`; - edits.push({ - set_range: { - beg: startRef, - end: endRef, - body: [...pendingAdded], - }, - }); + ops.push(`${anchor}:${formatPayload(pendingAdded)}`); } } - pendingRemoved = []; + pendingRemoved = 0; pendingAdded = []; }; @@ -595,13 +585,14 @@ function buildGuidedHashlineEdits(actual: string, expected: string): GuidedHashl if (!change.added && !change.removed) { flush(); line += lines.length; + pendingStart = line; continue; } - if (pendingRemoved.length === 0 && pendingAdded.length === 0) { + if (pendingRemoved === 0 && pendingAdded.length === 0) { pendingStart = line; } if (change.removed) { - pendingRemoved.push(...lines); + pendingRemoved += lines.length; line += lines.length; } if (change.added) { @@ -610,7 +601,9 @@ function buildGuidedHashlineEdits(actual: string, expected: string): GuidedHashl } flush(); - return edits; + if (ops.length === 0) return null; + const header = `¶${file}#${computeFileHash(actual)}`; + return `${header}\n${ops.join("\n")}`; } async function buildGuidedContext( @@ -635,11 +628,13 @@ async function buildGuidedContext( .catch(() => null); if (actual === null || expected === null) return null; - const edits = buildGuidedHashlineEdits(actual, expected); - if (edits.length === 0) return null; - if (edits.length > 25) return null; + const patch = buildGuidedHashlinePatch(file, actual, expected); + if (patch === null) return null; + // Rough complexity guard: too many ops or too long → skip guidance. + const opCount = patch.split("\n").filter(l => /[↑↓→]/.test(l)).length; + if (opCount === 0 || opCount > 25) return null; - const args = { path: file, edits }; + const args = { path: file, input: patch }; const argsText = JSON.stringify(args, null, 2); if (argsText.length > 20_000) return null; const metaParts: string[] = []; @@ -836,46 +831,78 @@ export interface TaskResult { name: string; files: string[]; runs: TaskRunResult[]; - successRate: number; - avgTokens: TokenStats; - avgDuration: number; - avgIndentScore: number; - avgToolCalls: ToolCallStats; + /** Index into `runs` (ordered by runIndex) of the selected best run; -1 if no runs completed. */ + bestRunIndex: number; + /** True when the selected best run succeeded. */ + success: boolean; + /** Token usage of the best run. */ + tokens: TokenStats; + /** Duration (ms) of the best run. */ + duration: number; + /** Indent score of the best run, or 0 if unscored. */ + indentScore: number; + /** Tool call stats of the best run. */ + toolCalls: ToolCallStats; + /** Edit-tool success rate of the best run (defaults to 1 when no edit attempts). */ editSuccessRate: number; - autocorrectFreeSuccessRate: number; + /** True if the best run succeeded with zero autocorrects. */ + autocorrectFreeSuccess: boolean; + /** Fraction of completed (non-ghost) runs that succeeded — flakiness indicator. */ + flakeSuccessRate: number; } export interface BenchmarkSummary { totalTasks: number; + /** Total completed runs across all tasks (excludes ghost runs). */ totalRuns: number; + /** Successful runs across every executed run (any of N). Diagnostic. */ successfulRuns: number; - overallSuccessRate: number; - tasksWithAllPassing: number; - tasksWithAnyFailing: number; + /** Tasks whose best run succeeded (best-of-N). Primary headline metric. */ + successfulTasks: number; + /** successfulTasks / totalTasks. */ + taskSuccessRate: number; + /** Tasks where best succeeded but at least one of N failed (flakiness). */ + flakyTasks: number; + /** Tasks where every executed non-ghost run succeeded. */ + consistentlyPassingTasks: number; + /** Tokens summed over the best run of each task. */ totalTokens: TokenStats; - avgTokensPerRun: TokenStats; + /** Average tokens per task (sum of best runs / number of tasks). */ + avgTokensPerTask: TokenStats; + /** Duration summed over best runs. */ totalDuration: number; - avgDurationPerRun: number; + /** Average duration of the best run per task. */ + avgDurationPerTask: number; + /** Average indent score over best runs (only counts runs with a score). */ avgIndentScore: number; + /** Tool calls summed over best runs. */ totalToolCalls: ToolCallStats; - avgToolCallsPerRun: ToolCallStats; + /** Average tool calls per task (sum of best runs / number of tasks). */ + avgToolCallsPerTask: ToolCallStats; + /** Edit-tool success rate aggregated across best runs. */ editSuccessRate: number; - autocorrectFreeSuccessfulRuns: number; + /** Tasks where the best run succeeded without any autocorrects. */ + autocorrectFreeSuccessfulTasks: number; + /** autocorrectFreeSuccessfulTasks / totalTasks. */ autocorrectFreeSuccessRate: number; - autocorrectedRuns: number; + /** Best runs with any autocorrects. */ + autocorrectedBestRuns: number; + /** Autocorrect rate across best-run edit successes. */ editAutocorrectRate: number; + /** Diagnostic: runs (across all N) that timed out. */ timeoutRuns: number; - /** Total retry counts across all runs */ + /** Diagnostic: total retry counts across all runs. */ totalTimeoutRetries: number; totalZeroToolRetries: number; totalProviderFailureRetries: number; - /** Runs where the 0/0/0 ghost signature was detected (0 tokens, 0 tool calls) */ + /** Diagnostic: ghost runs (0 tokens, 0 tool calls) across all N. */ ghostRuns: number; - /** Runs excluded because provider/transport stalls exhausted retries (subset of ghostRuns when error matches). */ + /** Diagnostic: runs excluded because provider/transport stalls exhausted retries. */ transportFailureRuns: number; mutationIntentMatchRate?: number; + /** Edit failure categories across all runs. */ editFailureCategories: Record; - /** Hashline edit subtype totals — only when editVariant is hashline */ + /** Hashline edit subtype totals across all runs — only when editVariant is hashline. */ hashlineEditSubtypes?: Record; } @@ -1629,70 +1656,71 @@ function isGhostRun(r: TaskRunResult): boolean { return noProgress || isTransportFailure(r); } +const EMPTY_TOOL_CALL_STATS: ToolCallStats = { + read: 0, + edit: 0, + write: 0, + editSuccesses: 0, + editFailures: 0, + editWarnings: 0, + editAutocorrects: 0, + totalInputChars: 0, +}; + +/** + * Strict ordering used to pick the "best" run for a task: + * 1. Successful runs win over failed runs. + * 2. Then prefer non-ghost runs (real work over 0/0/0 stalls). + * 3. Then prefer the run with lower total token usage. + * 4. Then prefer the earlier runIndex for stability. + */ +function isBetterRun(a: TaskRunResult, b: TaskRunResult): boolean { + if (a.success !== b.success) return a.success; + const aGhost = isGhostRun(a); + const bGhost = isGhostRun(b); + if (aGhost !== bGhost) return !aGhost; + if (a.tokens.total !== b.tokens.total) return a.tokens.total < b.tokens.total; + return a.runIndex < b.runIndex; +} + +function pickBestRunIndex(orderedRuns: TaskRunResult[]): number { + if (orderedRuns.length === 0) return -1; + let bestIdx = 0; + for (let i = 1; i < orderedRuns.length; i++) { + if (isBetterRun(orderedRuns[i]!, orderedRuns[bestIdx]!)) bestIdx = i; + } + return bestIdx; +} + function summarizeTaskRuns(task: EditTask, runs: TaskRunResult[]): TaskResult { const orderedRuns = runs.slice().sort((a, b) => a.runIndex - b.runIndex); const nonGhostRuns = orderedRuns.filter(r => !isGhostRun(r)); - const effective = nonGhostRuns.length; - const successfulRuns = orderedRuns.filter(r => r.success).length; - const successRate = effective > 0 ? successfulRuns / effective : 0; + const successfulNonGhost = nonGhostRuns.filter(r => r.success).length; + const flakeSuccessRate = nonGhostRuns.length > 0 ? successfulNonGhost / nonGhostRuns.length : 0; + const bestIdx = pickBestRunIndex(orderedRuns); + const best = bestIdx === -1 ? undefined : orderedRuns[bestIdx]!; - const avgTokens: TokenStats = - effective > 0 - ? { - input: Math.round(nonGhostRuns.reduce((sum, r) => sum + r.tokens.input, 0) / effective), - output: Math.round(nonGhostRuns.reduce((sum, r) => sum + r.tokens.output, 0) / effective), - total: Math.round(nonGhostRuns.reduce((sum, r) => sum + r.tokens.total, 0) / effective), - } - : { input: 0, output: 0, total: 0 }; - - const avgDuration = effective > 0 ? Math.round(nonGhostRuns.reduce((sum, r) => sum + r.duration, 0) / effective) : 0; - const indentScores = orderedRuns - .map(run => run.indentScore) - .filter((score): score is number => typeof score === "number"); - const avgIndentScore = - indentScores.length > 0 ? indentScores.reduce((sum, score) => sum + score, 0) / indentScores.length : 0; - - const avgToolCalls: ToolCallStats = - effective > 0 - ? { - read: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.read, 0) / effective, - edit: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.edit, 0) / effective, - write: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.write, 0) / effective, - editSuccesses: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0) / effective, - editFailures: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editFailures, 0) / effective, - editWarnings: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editWarnings, 0) / effective, - editAutocorrects: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editAutocorrects, 0) / effective, - totalInputChars: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.totalInputChars, 0) / effective, - } - : { - read: 0, - edit: 0, - write: 0, - editSuccesses: 0, - editFailures: 0, - editWarnings: 0, - editAutocorrects: 0, - totalInputChars: 0, - }; - - const totalEditAttempts = nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.edit, 0); - const totalEditSuccesses = nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0); - const editSuccessRate = totalEditAttempts > 0 ? totalEditSuccesses / totalEditAttempts : 1; - const autocorrectFreeSuccesses = nonGhostRuns.filter(run => run.success && run.editAutocorrectCount === 0).length; - const autocorrectFreeSuccessRate = effective > 0 ? autocorrectFreeSuccesses / effective : 0; + const tokens: TokenStats = best ? { ...best.tokens } : { input: 0, output: 0, total: 0 }; + const duration = best?.duration ?? 0; + const indentScore = typeof best?.indentScore === "number" ? best.indentScore : 0; + const toolCalls: ToolCallStats = best ? { ...best.toolCalls } : { ...EMPTY_TOOL_CALL_STATS }; + const editSuccessRate = toolCalls.edit > 0 ? toolCalls.editSuccesses / toolCalls.edit : 1; + const autocorrectFreeSuccess = Boolean(best?.success) && (best?.editAutocorrectCount ?? 0) === 0; return { id: task.id, name: task.name, files: task.files, runs: orderedRuns, - successRate, - avgTokens, - avgDuration, - avgIndentScore, - avgToolCalls, + bestRunIndex: best?.runIndex ?? -1, + success: Boolean(best?.success), + tokens, + duration, + indentScore, + toolCalls, editSuccessRate, - autocorrectFreeSuccessRate, + autocorrectFreeSuccess, + flakeSuccessRate, }; } @@ -1754,45 +1782,14 @@ export function buildBenchmarkResult(params: { const endTime = params.endTime ?? new Date().toISOString(); + // Diagnostic aggregates run over *every* executed run (across all N) so the + // report still surfaces ghost/timeout/retry signals. const allRuns = taskResults.flatMap(t => t.runs); - const totalRuns = allRuns.length; const ghostRuns = allRuns.filter(r => isGhostRun(r)).length; const transportFailureRuns = allRuns.filter(r => isTransportFailure(r)).length; - const effectiveRuns = totalRuns - ghostRuns; const nonGhostRuns = allRuns.filter(r => !isGhostRun(r)); + const totalRuns = nonGhostRuns.length; const successfulRuns = allRuns.filter(r => r.success).length; - - const totalTokens: TokenStats = { - input: nonGhostRuns.reduce((sum, r) => sum + r.tokens.input, 0), - output: nonGhostRuns.reduce((sum, r) => sum + r.tokens.output, 0), - total: nonGhostRuns.reduce((sum, r) => sum + r.tokens.total, 0), - }; - - const totalDuration = nonGhostRuns.reduce((sum, r) => sum + r.duration, 0); - const indentScores = nonGhostRuns - .map(run => run.indentScore) - .filter((score): score is number => typeof score === "number"); - const avgIndentScore = - indentScores.length > 0 ? indentScores.reduce((sum, score) => sum + score, 0) / indentScores.length : 0; - - const totalToolCalls: ToolCallStats = { - read: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.read, 0), - edit: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.edit, 0), - write: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.write, 0), - editSuccesses: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0), - editFailures: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editFailures, 0), - editWarnings: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editWarnings, 0), - editAutocorrects: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.editAutocorrects, 0), - totalInputChars: nonGhostRuns.reduce((sum, r) => sum + r.toolCalls.totalInputChars, 0), - }; - - const editSuccessRate = totalToolCalls.edit > 0 ? totalToolCalls.editSuccesses / totalToolCalls.edit : 1; - const autocorrectFreeSuccessfulRuns = nonGhostRuns.filter( - run => run.success && run.editAutocorrectCount === 0, - ).length; - const autocorrectedRuns = nonGhostRuns.filter(run => run.editAutocorrectCount > 0).length; - const editAutocorrectRate = - totalToolCalls.editSuccesses > 0 ? totalToolCalls.editAutocorrects / totalToolCalls.editSuccesses : 0; const timeoutRuns = nonGhostRuns.filter( r => r.error?.includes("Timeout") || r.error?.includes("Timeout exhausted"), ).length; @@ -1802,13 +1799,7 @@ export function buildBenchmarkResult(params: { (sum, r) => sum + (r.retryStats?.providerFailureRetries ?? 0), 0, ); - const runsWithMutationIntent = nonGhostRuns.filter(r => typeof r.mutationIntentMatched === "boolean"); - const mutationIntentMatchRate = - runsWithMutationIntent.length > 0 - ? runsWithMutationIntent.filter(r => r.mutationIntentMatched).length / runsWithMutationIntent.length - : undefined; const editFailureCategories = countEditFailureCategories(nonGhostRuns); - const hashlineEditSubtypes: Record | undefined = params.config.editVariant === "hashline" ? Object.fromEntries( @@ -1816,38 +1807,91 @@ export function buildBenchmarkResult(params: { ) : undefined; - const denom = effectiveRuns || 1; + // Primary aggregates run over the *best* run of each completed task. + const bestRuns: TaskRunResult[] = []; + for (const task of taskResults) { + if (task.bestRunIndex < 0) continue; + const best = task.runs.find(r => r.runIndex === task.bestRunIndex); + if (best) bestRuns.push(best); + } + const tasksWithBestRun = bestRuns.length; + const totalTasks = params.tasks.length; + const denom = totalTasks || 1; + + const successfulTasks = taskResults.filter(t => t.success).length; + const consistentlyPassingTasks = taskResults.filter( + t => t.success && t.runs.filter(r => !isGhostRun(r)).every(r => r.success), + ).length; + const flakyTasks = taskResults.filter( + t => t.success && t.runs.filter(r => !isGhostRun(r)).some(r => !r.success), + ).length; + + const totalTokens: TokenStats = { + input: bestRuns.reduce((sum, r) => sum + r.tokens.input, 0), + output: bestRuns.reduce((sum, r) => sum + r.tokens.output, 0), + total: bestRuns.reduce((sum, r) => sum + r.tokens.total, 0), + }; + const totalDuration = bestRuns.reduce((sum, r) => sum + r.duration, 0); + const totalToolCalls: ToolCallStats = { + read: bestRuns.reduce((sum, r) => sum + r.toolCalls.read, 0), + edit: bestRuns.reduce((sum, r) => sum + r.toolCalls.edit, 0), + write: bestRuns.reduce((sum, r) => sum + r.toolCalls.write, 0), + editSuccesses: bestRuns.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0), + editFailures: bestRuns.reduce((sum, r) => sum + r.toolCalls.editFailures, 0), + editWarnings: bestRuns.reduce((sum, r) => sum + r.toolCalls.editWarnings, 0), + editAutocorrects: bestRuns.reduce((sum, r) => sum + r.toolCalls.editAutocorrects, 0), + totalInputChars: bestRuns.reduce((sum, r) => sum + r.toolCalls.totalInputChars, 0), + }; + const bestIndentScores = bestRuns + .map(r => r.indentScore) + .filter((score): score is number => typeof score === "number"); + const avgIndentScore = + bestIndentScores.length > 0 ? bestIndentScores.reduce((sum, s) => sum + s, 0) / bestIndentScores.length : 0; + + const editSuccessRate = totalToolCalls.edit > 0 ? totalToolCalls.editSuccesses / totalToolCalls.edit : 1; + const autocorrectFreeSuccessfulTasks = bestRuns.filter(r => r.success && r.editAutocorrectCount === 0).length; + const autocorrectedBestRuns = bestRuns.filter(r => r.editAutocorrectCount > 0).length; + const editAutocorrectRate = + totalToolCalls.editSuccesses > 0 ? totalToolCalls.editAutocorrects / totalToolCalls.editSuccesses : 0; + const bestWithMutationIntent = bestRuns.filter(r => typeof r.mutationIntentMatched === "boolean"); + const mutationIntentMatchRate = + bestWithMutationIntent.length > 0 + ? bestWithMutationIntent.filter(r => r.mutationIntentMatched).length / bestWithMutationIntent.length + : undefined; + + const taskDenom = tasksWithBestRun || 1; const summary: BenchmarkSummary = { - totalTasks: params.tasks.length, - totalRuns: effectiveRuns, + totalTasks, + totalRuns, successfulRuns, - overallSuccessRate: successfulRuns / denom, - tasksWithAllPassing: taskResults.filter(t => t.successRate === 1).length, - tasksWithAnyFailing: taskResults.filter(t => t.successRate < 1).length, + successfulTasks, + taskSuccessRate: successfulTasks / denom, + flakyTasks, + consistentlyPassingTasks, totalTokens, - avgTokensPerRun: { - input: Math.round(totalTokens.input / denom), - output: Math.round(totalTokens.output / denom), - total: Math.round(totalTokens.total / denom), + avgTokensPerTask: { + input: Math.round(totalTokens.input / taskDenom), + output: Math.round(totalTokens.output / taskDenom), + total: Math.round(totalTokens.total / taskDenom), }, totalDuration, - avgDurationPerRun: Math.round(totalDuration / denom), + avgDurationPerTask: Math.round(totalDuration / taskDenom), avgIndentScore, totalToolCalls, - avgToolCallsPerRun: { - read: totalToolCalls.read / denom, - edit: totalToolCalls.edit / denom, - write: totalToolCalls.write / denom, - editSuccesses: totalToolCalls.editSuccesses / denom, - editFailures: totalToolCalls.editFailures / denom, - editWarnings: totalToolCalls.editWarnings / denom, - editAutocorrects: totalToolCalls.editAutocorrects / denom, - totalInputChars: totalToolCalls.totalInputChars / denom, + avgToolCallsPerTask: { + read: totalToolCalls.read / taskDenom, + edit: totalToolCalls.edit / taskDenom, + write: totalToolCalls.write / taskDenom, + editSuccesses: totalToolCalls.editSuccesses / taskDenom, + editFailures: totalToolCalls.editFailures / taskDenom, + editWarnings: totalToolCalls.editWarnings / taskDenom, + editAutocorrects: totalToolCalls.editAutocorrects / taskDenom, + totalInputChars: totalToolCalls.totalInputChars / taskDenom, }, editSuccessRate, - autocorrectFreeSuccessfulRuns, - autocorrectFreeSuccessRate: autocorrectFreeSuccessfulRuns / denom, - autocorrectedRuns, + autocorrectFreeSuccessfulTasks, + autocorrectFreeSuccessRate: autocorrectFreeSuccessfulTasks / denom, + autocorrectedBestRuns, editAutocorrectRate, timeoutRuns, totalTimeoutRetries, @@ -1887,32 +1931,50 @@ export async function runBenchmark( }) : undefined; - const runItems: TaskRunItem[] = tasks.flatMap(task => - Array.from({ length: config.runsPerTask }, (_, runIndex) => ({ task, runIndex })), - ); + try { + const runsPerTask = Math.max(1, Math.floor(config.runsPerTask)); + const taskQueue = shuffle(tasks.slice()); + const resultsByTask = new Map(); + const concurrency = Math.max(1, Math.floor(config.taskConcurrency)); - const pending = shuffle(runItems); - const resultsByTask = new Map(); - const concurrency = Math.max(1, Math.floor(config.taskConcurrency)); - const running: Promise[] = []; + const recordResult = (task: EditTask, result: TaskRunResult) => { + const list = resultsByTask.get(task.id) ?? []; + list.push(result); + resultsByTask.set(task.id, list); + onResultSnapshot?.(buildBenchmarkResult({ tasks, config, resultsByTask, startTime })); + }; - const runNext = async (): Promise => { - const nextItem = pending.shift(); - if (!nextItem) return; - const { task, result } = await runConcurrentBenchmarkRun(nextItem, config, onProgress, shared); - const list = resultsByTask.get(task.id) ?? []; - list.push(result); - resultsByTask.set(task.id, list); - onResultSnapshot?.(buildBenchmarkResult({ tasks, config, resultsByTask, startTime })); - await runNext(); - }; + // Each worker takes one task at a time and launches all N runs for that + // task concurrently. The best run is chosen later via summarizeTaskRuns; + // taskConcurrency caps the number of in-flight tasks (not runs). + const runTaskAllRuns = async (task: EditTask): Promise => { + const items: TaskRunItem[] = Array.from({ length: runsPerTask }, (_, runIndex) => ({ task, runIndex })); + await Promise.all( + items.map(async item => { + const { result } = await runConcurrentBenchmarkRun(item, config, onProgress, shared); + recordResult(task, result); + }), + ); + }; - const slots = Math.min(concurrency, pending.length); - for (let i = 0; i < slots; i++) { - running.push(runNext()); + const worker = async (): Promise => { + while (true) { + const task = taskQueue.shift(); + if (!task) return; + await runTaskAllRuns(task); + } + }; + + const slots = Math.min(concurrency, taskQueue.length); + const running: Promise[] = []; + for (let i = 0; i < slots; i++) { + running.push(worker()); + } + + await Promise.all(running); + + return buildBenchmarkResult({ tasks, config, resultsByTask, startTime }); + } finally { + shared?.authStorage.close(); } - - await Promise.all(running); - - return buildBenchmarkResult({ tasks, config, resultsByTask, startTime }); } diff --git a/packages/typescript-edit-benchmark/test/runner.test.ts b/packages/typescript-edit-benchmark/test/runner.test.ts index e0acca42e..3acce9812 100644 --- a/packages/typescript-edit-benchmark/test/runner.test.ts +++ b/packages/typescript-edit-benchmark/test/runner.test.ts @@ -35,7 +35,7 @@ function createTask(id: string): EditTask { }; } -function createRun(runIndex: number, success: boolean): TaskRunResult { +function createRun(runIndex: number, success: boolean, overrides: Partial = {}): TaskRunResult { return { runIndex, success, @@ -56,6 +56,7 @@ function createRun(runIndex: number, success: boolean): TaskRunResult { editFailures: [], editWarnings: [], editAutocorrectCount: 0, + ...overrides, }; } @@ -177,6 +178,99 @@ describe("buildBenchmarkResult", () => { expect(report).toContain("| range-continuation | 1 | 100.0% |"); expect(report).toContain("- Category: range-continuation"); }); + + it("picks the successful run with the lowest tokens as the task best", () => { + const task = createTask("best"); + const losing = createRun(0, false, { tokens: { input: 5, output: 5, total: 10 } }); + const winning = createRun(1, true, { tokens: { input: 100, output: 50, total: 150 } }); + const expensive = createRun(2, true, { tokens: { input: 500, output: 250, total: 750 } }); + const result = buildBenchmarkResult({ + tasks: [task], + config: { + provider: "anthropic", + model: "claude", + runsPerTask: 3, + timeout: 1000, + taskConcurrency: 1, + }, + resultsByTask: new Map([[task.id, [losing, winning, expensive]]]), + startTime: "2026-04-28T00:00:00.000Z", + endTime: "2026-04-28T00:00:01.000Z", + }); + + const taskResult = result.tasks[0]!; + expect(taskResult.success).toBe(true); + expect(taskResult.bestRunIndex).toBe(1); + expect(taskResult.tokens.total).toBe(150); + expect(result.summary.successfulTasks).toBe(1); + expect(result.summary.successfulRuns).toBe(2); + expect(result.summary.totalTokens.total).toBe(150); + expect(result.summary.taskSuccessRate).toBe(1); + expect(result.summary.flakyTasks).toBe(1); + expect(result.summary.consistentlyPassingTasks).toBe(0); + }); + + it("falls back to the cheapest failure when no run succeeded", () => { + const task = createTask("none"); + const expensiveFail = createRun(0, false, { tokens: { input: 200, output: 100, total: 300 } }); + const cheapFail = createRun(1, false, { tokens: { input: 20, output: 10, total: 30 } }); + const result = buildBenchmarkResult({ + tasks: [task], + config: { + provider: "anthropic", + model: "claude", + runsPerTask: 2, + timeout: 1000, + taskConcurrency: 1, + }, + resultsByTask: new Map([[task.id, [expensiveFail, cheapFail]]]), + startTime: "2026-04-28T00:00:00.000Z", + endTime: "2026-04-28T00:00:01.000Z", + }); + + const taskResult = result.tasks[0]!; + expect(taskResult.success).toBe(false); + expect(taskResult.bestRunIndex).toBe(1); + expect(taskResult.tokens.total).toBe(30); + expect(result.summary.successfulTasks).toBe(0); + expect(result.summary.taskSuccessRate).toBe(0); + }); + + it("ignores ghost runs when picking the best non-successful run", () => { + const task = createTask("ghost"); + const ghostRun = createRun(0, false, { + tokens: { input: 0, output: 0, total: 0 }, + toolCalls: { + read: 0, + edit: 0, + write: 0, + editSuccesses: 0, + editFailures: 0, + editWarnings: 0, + editAutocorrects: 0, + totalInputChars: 0, + }, + }); + const realFailure = createRun(1, false, { tokens: { input: 40, output: 20, total: 60 } }); + const result = buildBenchmarkResult({ + tasks: [task], + config: { + provider: "anthropic", + model: "claude", + runsPerTask: 2, + timeout: 1000, + taskConcurrency: 1, + }, + resultsByTask: new Map([[task.id, [ghostRun, realFailure]]]), + startTime: "2026-04-28T00:00:00.000Z", + endTime: "2026-04-28T00:00:01.000Z", + }); + + const taskResult = result.tasks[0]!; + expect(taskResult.bestRunIndex).toBe(1); + expect(taskResult.tokens.total).toBe(60); + expect(result.summary.ghostRuns).toBe(1); + }); }); describe("writeConversationDump", () => { diff --git a/packages/utils/package.json b/packages/utils/package.json index 366e9f10d..33b8c1d81 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "@oh-my-pi/pi-utils", - "version": "15.3.2", + "version": "15.4.3", "description": "Shared utilities for pi packages", "homepage": "https://omp.sh", "author": "Can Boluk", diff --git a/packages/utils/src/format.ts b/packages/utils/src/format.ts index 8e6a83a22..7e294e8a6 100644 --- a/packages/utils/src/format.ts +++ b/packages/utils/src/format.ts @@ -8,6 +8,7 @@ const DAY = 24 * HOUR; * Examples: "123ms", "1.5s", "30m15s", "2h30m", "3d2h" */ export function formatDuration(ms: number): string { + if (!Number.isFinite(ms) || ms <= 0) return "0ms"; if (ms < SEC) return `${ms}ms`; if (ms < MIN) return `${(ms / SEC).toFixed(1)}s`; if (ms < HOUR) { diff --git a/packages/utils/test/format.test.ts b/packages/utils/test/format.test.ts new file mode 100644 index 000000000..ab81942a0 --- /dev/null +++ b/packages/utils/test/format.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "bun:test"; +import { formatDuration } from "../src/format"; + +describe("formatDuration", () => { + // Codex's wham/usage endpoint returns the prior window's reset_at until the + // next request opens a fresh window, so the `resetsAt - now` delta can land + // in the recent past. The util must defend against that — older builds + // rendered "-612090ms", which leaked straight into the /usage TUI. + it("clamps non-positive, NaN, and Infinity inputs to 0ms", () => { + expect(formatDuration(-612_090)).toBe("0ms"); + expect(formatDuration(-1)).toBe("0ms"); + expect(formatDuration(0)).toBe("0ms"); + expect(formatDuration(Number.NaN)).toBe("0ms"); + expect(formatDuration(Number.POSITIVE_INFINITY)).toBe("0ms"); + expect(formatDuration(Number.NEGATIVE_INFINITY)).toBe("0ms"); + }); + + it("formats sub-second, sub-minute, sub-hour, sub-day, and multi-day ranges", () => { + expect(formatDuration(500)).toBe("500ms"); + expect(formatDuration(1_500)).toBe("1.5s"); + expect(formatDuration(90_000)).toBe("1m30s"); + expect(formatDuration(3_600_000)).toBe("1h"); + expect(formatDuration(3_660_000)).toBe("1h1m"); + expect(formatDuration(2 * 86_400_000 + 3_600_000)).toBe("2d1h"); + }); +});