smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge upstream/main (force-rewritten) into feat/profiles-and-alias

Browse Source 
Upstream force-rewrote history; this branch carried old-SHA twins of the
rewritten commits. All non-goal conflicts resolved to upstream (verified
ours == old upstream tip). Goal-side reconciliation:
- cli.ts: profile bootstrap woven into the new lazy-import/resolveCliArgv
  structure; worker-host entry declaration deferred until after profile
  selection (pi-utils/env eagerly snapshots the agent dir .env); the
  floating runCli call guarded with import.meta.main || !Bun.isMainThread
  so importing runCli stays side-effect free while Worker re-entry works.
- args.ts/flag-tables.ts: kept profile/alias branches; upstream's new
  repeatable --config overlay flag moved into STRING_SETTERS.
- Changelogs: upstream-released bullets deduped out of Unreleased; profile
  entries restored under Unreleased.
- task/index.ts: removed duplicated validateTaskIds block from auto-merge.
This commit is contained in:
Ogrodev
2026-06-10 08:17:40 -03:00
parent ff77f2b3e9 bbe85b6661
commit 4d4c745bb9
711 changed files with 38926 additions and 10867 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+12 -1
View File
@@ -525,7 +525,7 @@ jobs:
needs.release_binary.result == 'success' &&
needs.release_github_verify.result == 'success' &&
!inputs.skip_npm }}
needs: [release_metadata, release_binary, release_github_verify]
needs: [release_metadata, release_binary, release_github_verify, native_artifact_lookup]
runs-on: ubuntu-22.04
# `id-token: write` lets npm mint the GitHub OIDC token it exchanges for a
# short-lived publish token (trusted publishing + provenance). When a
@@ -552,6 +552,17 @@ jobs:
path: ~/.bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('**/bun.lock') }}
- run: bun install --frozen-lockfile
# The pi-coding-agent prepack executes workspace code (bundle-dist
# imports the pi-utils barrel, which loads the pi-natives addon), so
# this job needs the linux x64 native addons just like `test` does.
# Release runs always rebuild natives in this same run, so the
# default run-id resolves the artifacts.
- name: Download native addons
uses: actions/download-artifact@v4
with:
pattern: pi-natives-linux-x64-*-h${{ needs.native_artifact_lookup.outputs.source-hash }}
path: packages/natives/native
merge-multiple: true
- name: Publish to npm
env:
# Fallback auth: setup-node wrote an .npmrc referencing
AGENTS.md
+17 -13
View File
@@ -11,6 +11,7 @@ This repo contains multiple packages, but **`packages/coding-agent/`** is the pr
| Package | Description |
| ----------------------- | ---------------------------------------------------- |
| `packages/ai` | Multi-provider LLM client with streaming support |
| `packages/catalog` | Model catalog: bundled models.json, provider descriptors, model identity/classification |
| `packages/agent` | Agent runtime with tool calling and state management |
| `packages/coding-agent` | Main CLI application (primary focus) |
| `packages/tui` | Terminal UI library with differential rendering |
@@ -19,6 +20,8 @@ This repo contains multiple packages, but **`packages/coding-agent/`** is the pr
| `packages/utils` | Shared utilities (logger, streams, temp files) |
| `crates/pi-natives` | Rust crate for performance-critical text/grep ops |
**Catalog import convention**: code in this repo imports catalog *values* (bundled models, model-thinking helpers, identity, descriptors, model manager/cache) from `@oh-my-pi/pi-catalog/<module>` — never via `@oh-my-pi/pi-ai`. The pi-ai barrel re-exports only the model/effort *types* its own signatures use (`Model`, `Api`, `ThinkingConfig`, `Effort`, …); type-only imports of those from `@oh-my-pi/pi-ai` are fine.
## Code Quality
- No `any` unless absolutely necessary.
@@ -29,16 +32,17 @@ This repo contains multiple packages, but **`packages/coding-agent/`** is the pr
- **Class privacy**: use ES `#private` fields; leave externally accessible members bare. **No `private`/`protected`/`public` keyword on fields or methods**, except on **constructor parameter properties** where TypeScript requires it (e.g. `constructor(private readonly session: ToolSession)`).
- **Promises**: use `Promise.withResolvers()` instead of `new Promise((resolve, reject) => ...)`.
- **Prompts**: never build prompts in code (no inline strings, template literals, or concatenation). Prompts live in static `.md` files; use Handlebars for dynamic content. Import them via `import content from "./prompt.md" with { type: "text" }` — not `readFile`.
- **Worker scripts**: spawn workers with the dev/compile-safe hybrid pattern. `with { type: "file" }` only copies the entry as a raw asset and does **not** bundle its imports — workers crashed silently in compiled binaries on every prior incarnation of that pattern (issues #1011, #1027). Use this shape instead:
- **Worker scripts**: workers re-enter the CLI entrypoint; never spawn separate worker entry modules. `cli.ts` declares itself as the worker host at startup (`declareWorkerHostEntry()` from `@oh-my-pi/pi-utils/env`) and dispatches hidden argv selectors (`__omp_stats_sync_worker`, `__omp_tab_worker`, `__omp_js_eval_worker`, `--tiny-worker`) before loading the command registry. Spawn sites use:
```ts
import { isCompiledBinary } from "@oh-my-pi/pi-utils";
const worker = isCompiledBinary()
? new Worker("./packages/<pkg>/src/<worker>.ts", { type: "module" })
import { workerHostEntry } from "@oh-my-pi/pi-utils";
const hostEntry = workerHostEntry();
const worker = hostEntry
? new Worker(hostEntry, { type: "module", argv: ["__omp_<name>_worker"] })
: new Worker(new URL("./<worker>.ts", import.meta.url).href, { type: "module" });
```
The literal in the compiled branch is what Bun's `--compile` static analyzer needs to discover the worker — its path is **`--root`-relative** (repo root, since `build-binary.ts` passes `--root ../..`), so it must start with `./packages/...`. The `new URL` form in the dev branch keeps spawns portable across cwds.
In addition, every worker entry **MUST** be listed as an extra `--compile` entrypoint in `packages/coding-agent/scripts/build-binary.ts`. Without that the analyzer sees the literal but the worker never gets emitted into bunfs. The three current entries (`sync-worker.ts`, `tab-worker-entry.ts`, `worker-entry.ts`) live there as the working reference.
Validate any new worker with the dedicated smoke probe: `omp --smoke-test` spawns the stats sync worker, pings it, and exits — it's wired into `ci:test:smoke` and `scripts/install-tests/run-ci.sh` so binary, source-link, and tarball installs all exercise it. Add a sibling smoke if the new worker is on a different module graph.
When the process was started from the omp CLI — source `cli.ts`, npm-bundle `dist/cli.js`, or compiled binary — `workerHostEntry()` is `Bun.main` and the worker re-enters the single entry module, so no per-worker `--compile` entrypoints or bundle entries exist. Outside a CLI host (`bun test`, SDK embedding, standalone `omp-stats`) it returns `null` and the direct-module fallback loads the worker source. New worker kinds MUST add their selector to the dispatch table in `cli.ts` and keep the fallback branch.
History: `with { type: "file" }` only copied the entry as a raw asset (workers crashed silently in compiled binaries — issues #1011, #1027), and the later literal-path + extra-entrypoint pattern required keeping spawn literals and two build scripts in sync (issue #1150). The repro tests for those issues now pin the worker-host contract instead.
Validate any new worker with the dedicated smoke probe: `omp --smoke-test` spawns the stats sync worker and the tiny-model subprocess, pings them, and exits — it's wired into `ci:test:smoke` and `scripts/install-tests/run-ci.sh` so binary, source-link, and tarball installs all exercise it. Add a sibling smoke if the new worker is on a different module graph.
## Bun Over Node
@@ -146,15 +150,15 @@ Manual reader loops only when the protocol requires it (SSE, streaming JSON-RPC)
## Generated Files
**NEVER edit `packages/ai/src/models.json` directly.** It is generated from upstream sources (models.dev, provider catalog discovery, OpenCode docs) by `packages/ai/scripts/generate-models.ts` and the descriptors/resolvers in `packages/ai/src/provider-models/`. Hand-edits get overwritten on the next regen.
**NEVER edit `packages/catalog/src/models.json` directly.** It is generated from upstream sources (models.dev, provider catalog discovery, OpenCode docs) by `packages/catalog/scripts/generate-models.ts` and the descriptors/resolvers in `packages/catalog/src/provider-models/`. Hand-edits get overwritten on the next regen.
To change an entry, fix the source:
- **Resolution rules / per-id overrides** → relevant resolver in `packages/ai/src/provider-models/openai-compat.ts` (e.g. `createOpenCodeApiResolution`'s id-override map).
- **Provider descriptors** (filtering, transforms, defaults, headers, compat overrides) → `packages/ai/src/provider-models/descriptors.ts` or the provider-specific descriptor.
- **Generator-level fixups** (premium multipliers, codex pricing fallback, fallback models, post-processing) → `packages/ai/scripts/generate-models.ts`.
- **Thinking metadata / generated policies** → `packages/ai/src/model-thinking.ts` (`applyGeneratedModelPolicies`).
- **Resolution rules / per-id overrides** → relevant resolver in `packages/catalog/src/provider-models/openai-compat.ts` (e.g. `createOpenCodeApiResolution`'s id-override map).
- **Provider catalog entries** (default model, discovery factory/flags) → the `CATALOG_PROVIDERS` table in `packages/catalog/src/provider-models/descriptors.ts`.
- **Generator-level fixups** (premium multipliers, codex pricing fallback, fallback models, post-processing) → `packages/catalog/scripts/generate-models.ts`.
- **Thinking metadata / generated policies** → `packages/catalog/src/model-thinking.ts` (`applyGeneratedModelPolicies`); model-id classification (family/version parsing) lives in `packages/catalog/src/identity/classify.ts`.
Regenerate with `bun --cwd=packages/ai run generate-models` and commit `models.json` alongside the source change. Add a regression test against the **resolver/descriptor**, not the bundled JSON, so it survives upstream metadata shifts.
Regenerate with `bun --cwd=packages/catalog run generate-models` and commit `models.json` alongside the source change. Add a regression test against the **resolver/descriptor**, not the bundled JSON, so it survives upstream metadata shifts.
## Logging
Cargo.lock
Generated
+8 -8
View File
@@ -1825,9 +1825,9 @@ dependencies = [
[[package]]
name = "napi"
version = "3.9.0"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1d395473824516f38dd1071a1a37bc57daa7be65b293ebba4ead5f7abb017a2"
checksum = "ad513ff22558f1830b595ea6eb4091da48145d09a222ce157e781896f78be0b9"
dependencies = [
"bitflags 2.13.0",
"ctor",
@@ -1874,9 +1874,9 @@ dependencies = [
[[package]]
name = "napi-sys"
version = "3.2.1"
version = "3.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb602b84d7c1edae45e50bbf1374696548f36ae179dfa667f577e384bb90c2b"
checksum = "1f5bcdf71abd3a50d00b49c1c2c75251cb3c913777d6139cd37dabc093a5e400"
dependencies = [
"libloading",
]
@@ -2330,7 +2330,7 @@ dependencies = [
[[package]]
name = "pi-ast"
version = "15.10.10"
version = "15.10.12"
dependencies = [
"anyhow",
"ast-grep-core",
@@ -2398,7 +2398,7 @@ dependencies = [
[[package]]
name = "pi-iso"
version = "15.10.10"
version = "15.10.12"
dependencies = [
"async-trait",
"libc",
@@ -2410,7 +2410,7 @@ dependencies = [
[[package]]
name = "pi-natives"
version = "15.10.10"
version = "15.10.12"
dependencies = [
"anyhow",
"arboard",
@@ -2456,7 +2456,7 @@ dependencies = [
[[package]]
name = "pi-shell"
version = "15.10.10"
version = "15.10.12"
dependencies = [
"anyhow",
"brush-builtins",
Cargo.toml
+1 -1
View File
@@ -4,7 +4,7 @@ exclude = ["crates/brush-core-vendored", "crates/brush-builtins-vendored"]
resolver = "3"
[workspace.package]
version = "15.10.10"
version = "15.10.12"
edition = "2024"
license = "MIT"
authors = ["Can Boluk"]
biome.json
+1 -1
View File
@@ -62,7 +62,7 @@
"!**/test-sessions.ts",
"!**/template.generated.ts",
"!**/docs-index.generated.ts",
"!**/gen/agent_pb.ts",
"!**/agent_pb.ts",
"!.worktrees/**/*",
"!.wt/**/*"
]
bun.lock
+40 -19
View File
@@ -15,9 +15,10 @@
},
"packages/agent": {
"name": "@oh-my-pi/pi-agent-core",
"version": "15.10.10",
"version": "15.10.12",
"dependencies": {
"@oh-my-pi/pi-ai": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-natives": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"@opentelemetry/api": "catalog:",
@@ -30,9 +31,10 @@
},
"packages/ai": {
"name": "@oh-my-pi/pi-ai",
"version": "15.10.10",
"version": "15.10.12",
"dependencies": {
"@bufbuild/protobuf": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"openai": "catalog:",
"partial-json": "catalog:",
@@ -42,9 +44,22 @@
"@types/bun": "catalog:",
},
},
"packages/catalog": {
"name": "@oh-my-pi/pi-catalog",
"version": "15.10.12",
"dependencies": {
"@bufbuild/protobuf": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"zod": "catalog:",
},
"devDependencies": {
"@oh-my-pi/pi-ai": "catalog:",
"@types/bun": "catalog:",
},
},
"packages/coding-agent": {
"name": "@oh-my-pi/pi-coding-agent",
"version": "15.10.10",
"version": "15.10.12",
"bin": {
"omp": "src/cli.ts",
},
@@ -56,6 +71,7 @@
"@oh-my-pi/omp-stats": "catalog:",
"@oh-my-pi/pi-agent-core": "catalog:",
"@oh-my-pi/pi-ai": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-mnemopi": "catalog:",
"@oh-my-pi/pi-natives": "catalog:",
"@oh-my-pi/pi-tui": "catalog:",
@@ -90,7 +106,7 @@
},
"packages/hashline": {
"name": "@oh-my-pi/hashline",
"version": "15.10.10",
"version": "15.10.12",
"dependencies": {
"diff": "catalog:",
"lru-cache": "catalog:",
@@ -101,12 +117,13 @@
},
"packages/mnemopi": {
"name": "@oh-my-pi/pi-mnemopi",
"version": "15.10.10",
"version": "15.10.12",
"bin": {
"mnemopi": "src/cli.ts",
},
"dependencies": {
"@oh-my-pi/pi-ai": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"fastembed": "catalog:",
"lru-cache": "catalog:",
@@ -118,7 +135,7 @@
},
"packages/natives": {
"name": "@oh-my-pi/pi-natives",
"version": "15.10.10",
"version": "15.10.12",
"devDependencies": {
"@napi-rs/cli": "catalog:",
"@types/bun": "catalog:",
@@ -126,12 +143,13 @@
},
"packages/stats": {
"name": "@oh-my-pi/omp-stats",
"version": "15.10.10",
"version": "15.10.12",
"bin": {
"omp-stats": "./src/index.ts",
},
"dependencies": {
"@oh-my-pi/pi-ai": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"@tailwindcss/node": "catalog:",
"chart.js": "catalog:",
@@ -151,7 +169,7 @@
},
"packages/swarm-extension": {
"name": "@oh-my-pi/swarm-extension",
"version": "15.10.10",
"version": "15.10.12",
"bin": {
"omp-swarm": "src/cli.ts",
},
@@ -167,7 +185,7 @@
},
"packages/tui": {
"name": "@oh-my-pi/pi-tui",
"version": "15.10.10",
"version": "15.10.12",
"dependencies": {
"@oh-my-pi/pi-natives": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
@@ -208,7 +226,7 @@
},
"packages/utils": {
"name": "@oh-my-pi/pi-utils",
"version": "15.10.10",
"version": "15.10.12",
"dependencies": {
"@oh-my-pi/pi-natives": "catalog:",
"beautiful-mermaid": "catalog:",
@@ -248,15 +266,16 @@
"@huggingface/transformers": "^4.2.0",
"@mozilla/readability": "^0.6.0",
"@napi-rs/cli": "3.7.0",
"@oh-my-pi/hashline": "15.10.10",
"@oh-my-pi/omp-stats": "15.10.10",
"@oh-my-pi/pi-agent-core": "15.10.10",
"@oh-my-pi/pi-ai": "15.10.10",
"@oh-my-pi/pi-coding-agent": "15.10.10",
"@oh-my-pi/pi-mnemopi": "15.10.10",
"@oh-my-pi/pi-natives": "15.10.10",
"@oh-my-pi/pi-tui": "15.10.10",
"@oh-my-pi/pi-utils": "15.10.10",
"@oh-my-pi/hashline": "15.10.12",
"@oh-my-pi/omp-stats": "15.10.12",
"@oh-my-pi/pi-agent-core": "15.10.12",
"@oh-my-pi/pi-ai": "15.10.12",
"@oh-my-pi/pi-catalog": "15.10.12",
"@oh-my-pi/pi-coding-agent": "15.10.12",
"@oh-my-pi/pi-mnemopi": "15.10.12",
"@oh-my-pi/pi-natives": "15.10.12",
"@oh-my-pi/pi-tui": "15.10.12",
"@oh-my-pi/pi-utils": "15.10.12",
"@opentelemetry/api": "^1.9.1",
"@opentelemetry/context-async-hooks": "^2.7.1",
"@opentelemetry/exporter-trace-otlp-proto": "^0.218.0",
@@ -641,6 +660,8 @@
"@oh-my-pi/pi-ai": ["@oh-my-pi/pi-ai@workspace:packages/ai"],
"@oh-my-pi/pi-catalog": ["@oh-my-pi/pi-catalog@workspace:packages/catalog"],
"@oh-my-pi/pi-coding-agent": ["@oh-my-pi/pi-coding-agent@workspace:packages/coding-agent"],
"@oh-my-pi/pi-mnemopi": ["@oh-my-pi/pi-mnemopi@workspace:packages/mnemopi"],
crates/brush-core-vendored/src/sys/tokio_process.rs
+21
View File
@@ -6,5 +6,26 @@ pub(crate) use tokio::process::Child;
pub(crate) fn spawn(command: std::process::Command) -> std::io::Result<Child> {
let mut command = tokio::process::Command::from(command);
command.kill_on_drop(true);
// Isolate every external child from the host's console:
//
// - `CREATE_NO_WINDOW` gives the child its own *invisible* console instead
// of attaching it to ours. Console-sharing children can mutate shared
// console state behind the host's back — most notably the output
// codepage (PHP >=7.1 CLI issues the equivalent of `chcp` and skips the
// restore when killed; php.net request #73716), which degraded every
// non-ASCII glyph a hosting TUI painted into CP437 mojibake (`Γöé`).
// Inherited stdio handles are unaffected (handle-routed, not
// console-routed); interactive commands belong to the PTY path, which
// provisions a dedicated ConPTY anyway.
// - `CREATE_NEW_PROCESS_GROUP` makes the child a ctrl-event group root.
// Windows cannot join an existing group, so this is applied uniformly
// here rather than per-command (`creation_flags` replaces rather than
// ORs; the `sys::windows::commands` ext traits intentionally leave
// creation flags alone).
#[cfg(windows)]
{
use windows_sys::Win32::System::Threading::{CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW};
command.creation_flags(CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW);
}
command.spawn()
}
crates/brush-core-vendored/src/sys/windows/commands.rs
+16 -11
View File
@@ -1,8 +1,12 @@
//! Command execution utilities.
//!
//! On Windows, process creation flags are applied uniformly in
//! `sys::process::spawn` (`CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW`); the
//! per-command extension methods below intentionally do not touch creation
//! flags, because `CommandExt::creation_flags` replaces rather than ORs and
//! two writers would silently clobber each other.
use std::{ffi::OsStr, os::windows::process::CommandExt as WindowsCommandExt};
use windows_sys::Win32::System::Threading::CREATE_NEW_PROCESS_GROUP;
use std::ffi::OsStr;
use crate::{ShellFd, error, openfiles};
@@ -34,10 +38,9 @@ impl CommandExt for std::process::Command {
self
}
fn process_group(&mut self, pgroup: i32) -> &mut Self {
if pgroup == 0 {
self.creation_flags(CREATE_NEW_PROCESS_GROUP);
}
fn process_group(&mut self, _pgroup: i32) -> &mut Self {
// NOTE: Windows cannot join an existing process group, and new-group
// creation is handled uniformly by `sys::process::spawn`.
self
}
}
@@ -95,19 +98,21 @@ pub trait CommandFgControlExt {
impl CommandFgControlExt for std::process::Command {
fn take_foreground(&mut self) {
self.creation_flags(CREATE_NEW_PROCESS_GROUP);
// NOTE: no terminal foregrounding on Windows; group/console flags are
// applied uniformly by `sys::process::spawn`.
}
fn lead_session(&mut self) {
self.creation_flags(CREATE_NEW_PROCESS_GROUP);
// NOTE: no sessions on Windows; group/console flags are applied
// uniformly by `sys::process::spawn`.
}
}
/// Extension trait for detaching a command from the parent's controlling terminal.
pub trait CommandSessionExt {
/// Arranges for the command to run in a new POSIX session with no controlling
/// terminal. On Windows this is a no-op; process-group behavior is handled
/// by `CommandFgControlExt` via `CREATE_NEW_PROCESS_GROUP`.
/// terminal. On Windows this is a no-op; process-group and console behavior
/// are handled uniformly by `sys::process::spawn`.
fn detach_session(&mut self);
}
crates/pi-natives/src/crash_handler.rs
+498
View File
@@ -0,0 +1,498 @@
//! Native crash diagnostics.
//!
//! Installs Rust-side panic and allocation-error hooks the first time the
//! native module loads, so any crash inside `pi-natives` writes an actionable
//! record (thread, payload, backtrace) to disk and to stderr before the host
//! process exits.
//!
//! Without these hooks, Bun receives only the bare
//! `memory allocation of N bytes failed` line and aborts with no stack —
//! see issue #2211 ("Windows crash: Rust allocator failure after tasklist.exe
//! popup"). The hooks do not change the abort behavior (the cdylib release
//! profile uses `panic = "abort"`); they make the next crash diagnosable.
//!
//! Notes:
//! - Backtraces are captured via [`Backtrace::force_capture`], so they work
//! regardless of `RUST_BACKTRACE`.
//! - The crash log path mirrors the JS side (`packages/utils/src/dirs.ts`):
//! `$XDG_STATE_HOME/omp/logs/` on Linux / macOS when the user has migrated to
//! XDG (i.e. that directory already exists and `PI_CODING_AGENT_DIR` isn't
//! pointed somewhere custom), otherwise `<home>/<PI_CONFIG_DIR>/logs/`
//! (defaulting to `~/.omp/logs/`).
//! - Hook installation is idempotent across repeated module loads.
use std::{
alloc::Layout,
backtrace::Backtrace,
ffi::OsStr,
fmt::Write as _,
fs::{self, OpenOptions},
io::Write as _,
path::{Path, PathBuf},
process,
sync::{
Once,
atomic::{AtomicBool, Ordering},
},
thread,
time::{SystemTime, UNIX_EPOCH},
};
/// Default directory name for OMP's per-user state (overridable via
/// `PI_CONFIG_DIR`, matching `packages/utils/src/dirs.ts`).
const DEFAULT_CONFIG_DIR: &str = ".omp";
/// App name used as the XDG-root subdirectory (`$XDG_STATE_HOME/omp/`),
/// matching `APP_NAME` in `packages/utils/src/dirs.ts`.
const APP_NAME: &str = "omp";
static INSTALL: Once = Once::new();
static ALLOC_HOOK_ACTIVE: AtomicBool = AtomicBool::new(false);
/// Install the panic and allocation-error hooks. Idempotent.
pub fn install() {
INSTALL.call_once(|| {
let prev_panic = std::panic::take_hook();
std::panic::set_hook(Box::new(move |info| {
let report = format_panic_report(info);
persist(&report, CrashKind::Panic);
prev_panic(info);
}));
std::alloc::set_alloc_error_hook(|layout| {
// Print the canonical line before doing anything allocation-prone.
// If this is genuine process-wide OOM, report formatting/path work may
// recursively enter this hook; the secondary entry writes the same
// stack-only fallback and aborts immediately.
write_alloc_failure_line(std::io::stderr(), layout.size());
if ALLOC_HOOK_ACTIVE.swap(true, Ordering::AcqRel) {
process::abort();
}
let report = format_alloc_report(layout);
persist(&report, CrashKind::Alloc);
process::abort();
});
});
}
#[derive(Clone, Copy)]
enum CrashKind {
Panic,
Alloc,
}
impl CrashKind {
const fn as_str(self) -> &'static str {
match self {
Self::Panic => "panic",
Self::Alloc => "alloc",
}
}
}
fn format_panic_report(info: &std::panic::PanicHookInfo<'_>) -> String {
let bt = Backtrace::force_capture();
let location = info.location().map_or_else(
|| String::from("<unknown>"),
|l| format!("{}:{}:{}", l.file(), l.line(), l.column()),
);
let mut out = report_header(CrashKind::Panic);
let _ = writeln!(out, "location: {location}");
let _ = writeln!(out, "message: {}", panic_payload(info.payload()));
let _ = writeln!(out, "backtrace:\n{bt}");
out
}
fn format_alloc_report(layout: Layout) -> String {
// Capturing a backtrace allocates. If the global allocator is in a state
// where small allocations keep failing this will recurse into the hook —
// `Backtrace::force_capture` swallows the secondary failure internally and
// returns an empty backtrace, which is still strictly more useful than the
// nothing the default handler prints.
let bt = Backtrace::force_capture();
let mut out = report_header(CrashKind::Alloc);
let _ = writeln!(out, "size: {} bytes", layout.size());
let _ = writeln!(out, "alignment: {} bytes", layout.align());
let _ = writeln!(out, "backtrace:\n{bt}");
out
}
fn report_header(kind: CrashKind) -> String {
let thread_name = thread::current().name().unwrap_or("<unnamed>").to_owned();
let now_ms = unix_millis();
format!(
"pi-natives {kind} crash\npid: {pid}\nthread: {thread_name}\ntimestamp: {now_ms} \
(unix ms)\n",
kind = kind.as_str(),
pid = process::id(),
)
}
fn write_alloc_failure_line(mut out: impl std::io::Write, size: usize) {
let _ = out.write_all(b"memory allocation of ");
let mut digits = [0u8; usize::MAX.ilog10() as usize + 1];
let mut pos = digits.len();
let mut value = size;
if value == 0 {
pos -= 1;
digits[pos] = b'0';
} else {
while value > 0 {
pos -= 1;
digits[pos] = b'0' + (value % 10) as u8;
value /= 10;
}
}
let _ = out.write_all(&digits[pos..]);
let _ = out.write_all(b" bytes failed\n");
}
fn panic_payload(payload: &(dyn std::any::Any + Send)) -> String {
if let Some(s) = payload.downcast_ref::<&'static str>() {
(*s).to_owned()
} else if let Some(s) = payload.downcast_ref::<String>() {
s.clone()
} else {
String::from("<non-string panic payload>")
}
}
fn persist(report: &str, kind: CrashKind) {
// Echo to stderr unconditionally so the user still sees something even
// when the file write fails (read-only home, missing $HOME, etc.).
let _ = writeln!(std::io::stderr(), "{report}");
let Some(path) = crash_log_path(kind) else {
return;
};
if let Some(parent) = path.parent() {
let _ = fs::create_dir_all(parent);
}
if let Ok(mut f) = OpenOptions::new().create(true).append(true).open(&path) {
let _ = f.write_all(report.as_bytes());
let _ = f.flush();
let _ = f.sync_data();
let _ = writeln!(std::io::stderr(), "pi-natives crash report written to {}", path.display());
}
}
fn crash_log_path(kind: CrashKind) -> Option<PathBuf> {
let dir = logs_dir()?;
Some(build_crash_log_path(&dir, kind, process::id(), unix_millis()))
}
fn build_crash_log_path(dir: &Path, kind: CrashKind, pid: u32, now_ms: u128) -> PathBuf {
dir.join(format!("native-{}-{pid}-{now_ms}.log", kind.as_str()))
}
fn logs_dir() -> Option<PathBuf> {
let home = home_dir()?;
let config_override = std::env::var_os("PI_CONFIG_DIR");
let xdg_logs = xdg_state_logs_from_env(&home, config_override.as_deref());
Some(resolve_logs_dir(&home, config_override.as_deref(), xdg_logs))
}
fn resolve_logs_dir(
home: &Path,
config_dir_override: Option<&OsStr>,
xdg_state_logs: Option<PathBuf>,
) -> PathBuf {
// XDG takes precedence so users who migrated to `$XDG_STATE_HOME/omp/logs/`
// see native crash reports in the same directory the JS logger rotates.
if let Some(p) = xdg_state_logs {
return p;
}
let config_dir = config_dir_override
.filter(|s| !s.is_empty())
.unwrap_or_else(|| OsStr::new(DEFAULT_CONFIG_DIR));
let base = config_root_dir(home, config_dir);
base.join("logs")
}
/// Compute the XDG-state logs dir if the runtime environment matches the
/// JS-side eligibility rules in `packages/utils/src/dirs.ts`: linux/macos,
/// `$XDG_STATE_HOME` set, `$XDG_STATE_HOME/omp` exists on disk, and
/// `PI_CODING_AGENT_DIR` is unset or pointing at the default agent dir.
#[cfg(any(target_os = "linux", target_os = "macos"))]
fn xdg_state_logs_from_env(home: &Path, config_dir_override: Option<&OsStr>) -> Option<PathBuf> {
let default_agent_dir = default_agent_dir(home, config_dir_override);
let agent_override = std::env::var_os("PI_CODING_AGENT_DIR");
let xdg_state_home = std::env::var_os("XDG_STATE_HOME");
xdg_state_logs(
xdg_state_home.as_deref(),
agent_override.as_deref(),
&default_agent_dir,
Path::exists,
)
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
#[allow(clippy::missing_const_for_fn, reason = "windows/non-xdg platforms keep the signature")]
fn xdg_state_logs_from_env(_home: &Path, _config_dir_override: Option<&OsStr>) -> Option<PathBuf> {
None
}
/// Pure XDG-eligibility computation extracted for unit testing — no env
/// reads, no fs reads. `omp_dir_exists` decides whether the candidate
/// `<xdg_state_home>/omp` actually lives on disk.
fn xdg_state_logs(
xdg_state_home: Option<&OsStr>,
agent_dir_override: Option<&OsStr>,
default_agent_dir: &Path,
omp_dir_exists: impl FnOnce(&Path) -> bool,
) -> Option<PathBuf> {
if let Some(ov) = agent_dir_override.filter(|s| !s.is_empty()) {
// `path.resolve(value)` on the JS side: make absolute against cwd
// without touching the filesystem. Anything that diverges from the
// default agent dir disables XDG, matching `isDefault === false`.
let resolved = std::path::absolute(Path::new(ov)).ok()?;
if resolved != default_agent_dir {
return None;
}
}
let xdg = xdg_state_home.filter(|s| !s.is_empty())?;
let omp_dir = Path::new(xdg).join(APP_NAME);
if !omp_dir_exists(&omp_dir) {
return None;
}
Some(omp_dir.join("logs"))
}
fn default_agent_dir(home: &Path, config_dir_override: Option<&OsStr>) -> PathBuf {
let config_dir = config_dir_override
.filter(|s| !s.is_empty())
.unwrap_or_else(|| OsStr::new(DEFAULT_CONFIG_DIR));
let base = config_root_dir(home, config_dir);
base.join("agent")
}
fn config_root_dir(home: &Path, config_dir: &OsStr) -> PathBuf {
let mut base = PathBuf::from(home);
for component in Path::new(config_dir).components() {
match component {
std::path::Component::Prefix(_) | std::path::Component::RootDir => {},
std::path::Component::CurDir => {},
std::path::Component::ParentDir => {
base.pop();
},
std::path::Component::Normal(part) => base.push(part),
}
}
base
}
fn home_dir() -> Option<PathBuf> {
#[cfg(unix)]
{
std::env::var_os("HOME").map(PathBuf::from)
}
#[cfg(windows)]
{
if let Some(profile) = std::env::var_os("USERPROFILE") {
return Some(PathBuf::from(profile));
}
let drive = std::env::var_os("HOMEDRIVE")?;
let path = std::env::var_os("HOMEPATH")?;
let mut combined = drive;
combined.push(path);
Some(PathBuf::from(combined))
}
}
fn unix_millis() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| d.as_millis())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn alloc_report_contains_size_alignment_and_backtrace() {
let layout = Layout::from_size_align(7714, 8).unwrap();
let report = format_alloc_report(layout);
assert!(report.contains("pi-natives alloc crash"), "report missing header: {report}");
assert!(report.contains("size: 7714 bytes"), "report missing size: {report}");
assert!(report.contains("alignment: 8 bytes"), "report missing alignment: {report}");
assert!(report.contains("backtrace:"), "report missing backtrace section: {report}");
assert!(
report.contains(&format!("pid: {}", process::id())),
"report missing pid: {report}"
);
assert!(report.contains("thread:"), "report missing thread: {report}");
}
#[test]
fn alloc_failure_line_matches_rust_default_text_without_heap_formatting() {
let mut buf = Vec::new();
write_alloc_failure_line(&mut buf, 7714);
assert_eq!(buf, b"memory allocation of 7714 bytes failed\n");
buf.clear();
write_alloc_failure_line(&mut buf, usize::MAX);
assert_eq!(buf, format!("memory allocation of {} bytes failed\n", usize::MAX).as_bytes());
}
#[test]
fn panic_payload_handles_str_string_and_other() {
let static_str: Box<dyn std::any::Any + Send> = Box::new("static panic");
assert_eq!(panic_payload(&*static_str), "static panic");
let owned: Box<dyn std::any::Any + Send> = Box::new(String::from("owned panic"));
assert_eq!(panic_payload(&*owned), "owned panic");
let other: Box<dyn std::any::Any + Send> = Box::new(42u32);
assert_eq!(panic_payload(&*other), "<non-string panic payload>");
}
#[test]
fn resolve_logs_dir_defaults_under_dot_omp() {
let dir = resolve_logs_dir(Path::new("/tmp/pi-natives-test-home"), None, None);
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/.omp/logs"));
}
#[test]
fn resolve_logs_dir_honors_relative_pi_config_dir() {
let dir = resolve_logs_dir(
Path::new("/tmp/pi-natives-test-home"),
Some(OsStr::new(".omp-dev")),
None,
);
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/.omp-dev/logs"));
}
#[test]
fn resolve_logs_dir_reroots_absolute_pi_config_dir_under_home() {
// JS resolves the config root via `path.join(os.homedir(),
// getConfigDirName())`, which never honors an absolute PI_CONFIG_DIR — it is
// always re-rooted under `$HOME` (and `..` components are normalized away).
let dir = resolve_logs_dir(
Path::new("/tmp/pi-natives-test-home"),
Some(OsStr::new("/var/tmp/pi-natives-state")),
None,
);
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/var/tmp/pi-natives-state/logs"));
}
#[test]
fn resolve_logs_dir_normalizes_parent_components_like_path_join() {
let dir = resolve_logs_dir(
Path::new("/tmp/pi-natives-test-home"),
Some(OsStr::new("nested/../.omp-dev")),
None,
);
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/.omp-dev/logs"));
}
#[test]
fn xdg_state_logs_ignores_empty_agent_dir_override() {
// An empty PI_CODING_AGENT_DIR is "unset", not a divergent override; it
// must not disable XDG resolution.
let dir = xdg_state_logs(
Some(OsStr::new("/xdg/state")),
Some(OsStr::new("")),
Path::new("/tmp/pi-natives-test-home/.omp/agent"),
|_p| true,
);
assert_eq!(dir, Some(PathBuf::from("/xdg/state/omp/logs")));
}
#[test]
fn resolve_logs_dir_ignores_empty_pi_config_dir() {
let dir =
resolve_logs_dir(Path::new("/tmp/pi-natives-test-home"), Some(OsStr::new("")), None);
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/.omp/logs"));
}
#[test]
fn resolve_logs_dir_prefers_xdg_when_provided() {
let dir = resolve_logs_dir(
Path::new("/tmp/pi-natives-test-home"),
None,
Some(PathBuf::from("/xdg/state/omp/logs")),
);
assert_eq!(dir, PathBuf::from("/xdg/state/omp/logs"));
}
#[test]
fn xdg_state_logs_resolves_when_dir_exists_and_no_agent_override() {
let dir = xdg_state_logs(
Some(OsStr::new("/xdg/state")),
None,
Path::new("/tmp/pi-natives-test-home/.omp/agent"),
|_p| true,
);
assert_eq!(dir, Some(PathBuf::from("/xdg/state/omp/logs")));
}
#[test]
fn xdg_state_logs_skipped_when_omp_dir_missing() {
let dir = xdg_state_logs(
Some(OsStr::new("/xdg/state")),
None,
Path::new("/tmp/pi-natives-test-home/.omp/agent"),
|_p| false,
);
assert_eq!(dir, None);
}
#[test]
fn xdg_state_logs_skipped_when_xdg_state_home_unset_or_empty() {
let default_agent = Path::new("/tmp/pi-natives-test-home/.omp/agent");
assert_eq!(xdg_state_logs(None, None, default_agent, |_p| true), None);
assert_eq!(xdg_state_logs(Some(OsStr::new("")), None, default_agent, |_p| true), None);
}
#[test]
fn xdg_state_logs_skipped_when_agent_dir_overridden() {
// `PI_CODING_AGENT_DIR` pointing elsewhere mirrors the JS `isDefault === false`
// branch in `packages/utils/src/dirs.ts` and must disable XDG.
let dir = xdg_state_logs(
Some(OsStr::new("/xdg/state")),
Some(OsStr::new("/some/custom/agent")),
Path::new("/tmp/pi-natives-test-home/.omp/agent"),
|_p| true,
);
assert_eq!(dir, None);
}
#[test]
fn xdg_state_logs_honored_when_agent_override_matches_default() {
let default_agent = std::path::absolute(Path::new("./.omp/agent")).unwrap();
let dir = xdg_state_logs(
Some(OsStr::new("/xdg/state")),
Some(OsStr::new("./.omp/agent")),
&default_agent,
|_p| true,
);
assert_eq!(dir, Some(PathBuf::from("/xdg/state/omp/logs")));
}
#[test]
fn default_agent_dir_uses_dot_omp_by_default() {
let dir = default_agent_dir(Path::new("/tmp/pi-natives-test-home"), None);
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/.omp/agent"));
}
#[test]
fn default_agent_dir_respects_pi_config_dir() {
let dir =
default_agent_dir(Path::new("/tmp/pi-natives-test-home"), Some(OsStr::new(".omp-dev")));
assert_eq!(dir, PathBuf::from("/tmp/pi-natives-test-home/.omp-dev/agent"));
}
#[test]
fn build_crash_log_path_tags_kind_and_pid() {
let dir = Path::new("/tmp/pi-natives-test-home/.omp/logs");
let panic_log = build_crash_log_path(dir, CrashKind::Panic, 4242, 1_700_000_000_000);
assert_eq!(
panic_log,
PathBuf::from("/tmp/pi-natives-test-home/.omp/logs/native-panic-4242-1700000000000.log")
);
let alloc_log = build_crash_log_path(dir, CrashKind::Alloc, 99, 1);
assert_eq!(
alloc_log,
PathBuf::from("/tmp/pi-natives-test-home/.omp/logs/native-alloc-99-1.log")
);
}
}
crates/pi-natives/src/lib.rs
+11 -2
View File
@@ -20,11 +20,13 @@
#![allow(clippy::trailing_empty_array, reason = "generated by napi macro")]
#![allow(clippy::trivially_copy_pass_by_ref, reason = "napi env idiom")]
#![feature(alloc_error_hook)]
pub mod appearance;
pub mod ast;
pub mod block;
pub mod clipboard;
pub mod crash_handler;
pub mod fd;
pub mod fs_cache;
pub mod glob;
@@ -50,7 +52,7 @@ pub mod tokens;
pub(crate) mod utils;
pub mod workspace;
use napi_derive::napi;
use napi_derive::{module_init, napi};
/// Version sentinel — exists solely so the JS loader can prove at load time
/// that the `.node` file on disk is from the same package release as the
@@ -68,5 +70,12 @@ use napi_derive::napi;
/// MUST stay in sync with `VERSION_SENTINEL_EXPORT` in
/// `packages/natives/native/index.js` (which derives the name from
/// `package.json#version`).
#[napi(js_name = "__piNativesV15_10_10")]
#[napi(js_name = "__piNativesV15_10_12")]
pub const fn pi_natives_version_sentinel() {}
/// Native module entry point: install crash diagnostics before any tool can
/// invoke a panicking or allocating native call. Runs once at `.node` load.
#[module_init]
fn install_native_crash_handler() {
crash_handler::install();
}
crates/pi-natives/src/shell.rs
+22 -12
View File
@@ -25,33 +25,43 @@ use crate::task;
#[derive(Debug, Clone, Default)]
pub struct MinimizerOptions {
/// Master switch. Absent / false = disabled.
pub enabled: Option<bool>,
pub enabled: Option<bool>,
/// Optional path to a TOML settings file whose values override
/// field-level defaults. `~` is expanded.
pub settings_path: Option<String>,
pub settings_path: Option<String>,
/// Optional xxHash64 digest (hex) of the settings file contents. When
/// supplied, the engine refuses to honor a settings file whose hash does
/// not match — a lightweight trust gate for agent-controllable paths.
pub settings_hash: Option<String>,
pub settings_hash: Option<String>,
/// Opt-in allowlist of program names (e.g. `"git"`). When empty or
/// absent, all built-in filters are active.
pub only: Option<Vec<String>>,
pub only: Option<Vec<String>>,
/// Program names explicitly excluded from minimization.
pub except: Option<Vec<String>>,
pub except: Option<Vec<String>>,
/// Maximum captured bytes per command before the engine falls back to
/// the raw, un-minimized output. Default 4 MiB.
pub max_capture_bytes: Option<u32>,
pub max_capture_bytes: Option<u32>,
/// Source-outline level for `cat <source-file>` minimization. Accepts
/// `"default"` (current behavior) or `"aggressive"` (strip function bodies).
pub source_outline_level: Option<String>,
/// Kill-switch to fall back to the pre-PR (legacy) filter behavior for
/// grep / find / pytest. When `Some(true)`, filters that opted into the
/// always-shrink Tier 1 / Tier 2 behavior skip the new code path. When
/// `None`, defers to the `OMP_MINIMIZER_LEGACY_FILTERS` env var.
pub legacy_filters: Option<bool>,
}
impl From<MinimizerOptions> for minimizer::MinimizerOptions {
fn from(value: MinimizerOptions) -> Self {
Self {
enabled: value.enabled,
settings_path: value.settings_path,
settings_hash: value.settings_hash,
only: value.only,
except: value.except,
max_capture_bytes: value.max_capture_bytes,
enabled: value.enabled,
settings_path: value.settings_path,
settings_hash: value.settings_hash,
only: value.only,
except: value.except,
max_capture_bytes: value.max_capture_bytes,
source_outline_level: value.source_outline_level,
legacy_filters: value.legacy_filters,
}
}
}
crates/pi-shell/ATTRIBUTION-RTK.md
+46
View File
@@ -0,0 +1,46 @@
# Third-party attribution — RTK
Portions of the shell-output minimizer adapt algorithms from **RTK**
(`rtk-ai/rtk`), used under the MIT License, which is compatible with this
workspace's MIT License.
## Ported component
- **Upstream:** [`rtk-ai/rtk`](https://github.com/rtk-ai/rtk) @ commit
`878af7de99e0ba71da2e8fd996f6b52a1836e06c`
- **Upstream path:** `src/cmds/python/pytest_cmd.rs`
- **Local path:** `crates/pi-shell/src/minimizer/filters/python.rs`
- **What was adapted:** the `build_pytest_summary` algorithm — re-implemented
here as the pytest state machine (`filter_pytest`, `pytest_success`,
`is_pytest_*`, `looks_like_pytest_summary_part`). It preserves failures,
errors, and the final summary line; strips header framing, progress dots, and
verbose `PASSED` rows; and falls through unchanged on unknown-state lines
(RTK's defensive default) so xdist `[gwN]` prefixes and custom reporters never
cause data loss.
## License (MIT)
RTK is distributed under the MIT License. A copy of the upstream license text
is reproduced below for the pinned revision above.
```
MIT License
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
```
crates/pi-shell/Cargo.toml
+2
View File
@@ -5,6 +5,8 @@ edition.workspace = true
license.workspace = true
authors.workspace = true
repository.workspace = true
# Portions of the minimizer adapt MIT-licensed algorithms from rtk-ai/rtk.
# See ATTRIBUTION-RTK.md at this crate root (packaged on publish).
[lints]
workspace = true
crates/pi-shell/src/minimizer.rs
+28 -4
View File
@@ -44,8 +44,11 @@ pub struct MinimizerOutput {
/// Byte length of `text` after minimization.
#[allow(dead_code, reason = "test-only API surface")]
pub output_bytes: usize,
/// Name of the dispatch path that produced this output (e.g. `"git"`,
/// `"pipeline:gradle"`, or `"passthrough"`). Useful for telemetry.
/// Label for the dispatch path that produced this output (e.g. `"git"`,
/// `"pipeline:gradle"`, or `"passthrough"`). For non-rewrite misses, this
/// carries the reason label (e.g. `"compound"`, `"piped"`, `"parse-error"`,
/// `"too-large"`, `"disabled"`, `"unknown"`, `"unsupported"`,
/// `"pipeline-noop"`).
pub filter: &'static str,
/// Original (un-minimized) capture, surfaced only when the filter
/// actually rewrote the output. The caller (JS session layer) is expected
@@ -79,7 +82,7 @@ impl MinimizerOutput {
}
/// Attach a `filter` label (e.g. `"git"`, `"pipeline:gradle"`) to an
/// output for telemetry. No-op on passthrough outputs.
/// output for telemetry, including non-rewrite miss reasons.
#[must_use]
pub const fn labeled(mut self, filter: &'static str) -> Self {
self.filter = filter;
@@ -113,8 +116,29 @@ impl MinimizerOutput {
}
}
/// Aggregate output for a segmented chain.
#[allow(
clippy::missing_const_for_fn,
reason = "kept non-const because this constructs owned output used only at runtime"
)]
pub(crate) fn chain_output(
text: String,
original_text: String,
input_bytes: usize,
changed: bool,
) -> MinimizerOutput {
let filter = if changed { "chain" } else { "chain-noop" };
let output_bytes = text.len();
MinimizerOutput {
text,
changed,
input_bytes,
output_bytes,
filter,
original_text: Some(original_text),
}
}
/// Apply the configured filter pipeline to a captured buffer.
///
/// Returns the original text unchanged when minimization is disabled, no
/// filter matches, or a filter panics.
pub fn apply(
crates/pi-shell/src/minimizer/config.rs
+204 -24
View File
@@ -18,50 +18,91 @@ use crate::minimizer::pipeline::{self, PipelineRegistry, SUPPORTED_SCHEMA_VERSIO
const DEFAULT_MAX_CAPTURE_BYTES: u32 = 4 * 1024 * 1024;
/// Source-outline aggressiveness for `cat <source-file>` minimization.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum OutlineLevel {
/// Current behavior: only outline when input is large enough to warrant it.
#[default]
Default,
/// Strip function/method bodies regardless of size for supported source
/// languages (`ts`, `tsx`, `js`, `jsx`, `py`, `rs`, `go`).
Aggressive,
}
impl OutlineLevel {
fn parse(raw: &str) -> Option<Self> {
match raw.trim().to_ascii_lowercase().as_str() {
"default" | "" => Some(Self::Default),
"aggressive" => Some(Self::Aggressive),
_ => None,
}
}
}
/// N-API opt-in handle for the minimizer.
#[derive(Debug, Clone, Default)]
pub struct MinimizerOptions {
/// Master switch. Absent / false = disabled.
pub enabled: Option<bool>,
pub enabled: Option<bool>,
/// Optional path to a TOML settings file whose values override
/// field-level defaults. `~` is expanded.
pub settings_path: Option<String>,
pub settings_path: Option<String>,
/// Optional xxHash64 digest (hex) of the settings file contents. When
/// supplied, the engine refuses to honor a settings file whose hash does
/// not match — a lightweight trust gate for agent-controllable paths.
pub settings_hash: Option<String>,
pub settings_hash: Option<String>,
/// Opt-in allowlist of program names (e.g. `"git"`). When empty or
/// absent, all built-in filters are active.
pub only: Option<Vec<String>>,
pub only: Option<Vec<String>>,
/// Program names explicitly excluded from minimization.
pub except: Option<Vec<String>>,
pub except: Option<Vec<String>>,
/// Maximum captured bytes per command before the engine falls back to
/// the raw, un-minimized output. Default 4 MiB.
pub max_capture_bytes: Option<u32>,
pub max_capture_bytes: Option<u32>,
/// Source-outline level for `cat <source-file>` minimization. Accepts
/// `"default"` (current behavior) or `"aggressive"` (strip function bodies).
pub source_outline_level: Option<String>,
/// Kill-switch to fall back to the pre-PR (legacy) filter behavior for
/// grep / find / pytest. When `Some(true)`, filters that opted into the
/// always-shrink Tier 1 / Tier 2 behavior skip the new code path and
/// return the legacy passthrough. When `None`, defers to the
/// `OMP_MINIMIZER_LEGACY_FILTERS` environment variable (truthy = "1",
/// "true", or "yes", case-insensitive); default `false`.
pub legacy_filters: Option<bool>,
}
/// Resolved minimizer configuration used by the engine.
#[derive(Debug, Clone)]
pub struct MinimizerConfig {
pub enabled: bool,
pub only: HashSet<String>,
pub except: HashSet<String>,
pub max_capture_bytes: u32,
pub per_command: HashMap<String, toml::Value>,
pub enabled: bool,
pub only: HashSet<String>,
pub except: HashSet<String>,
pub max_capture_bytes: u32,
pub per_command: HashMap<String, toml::Value>,
/// Compiled user-defined pipelines parsed from `settings_path`. Searched
/// before the built-in pipelines so user filters win.
pub user_pipelines: Option<Arc<PipelineRegistry>>,
pub user_pipelines: Option<Arc<PipelineRegistry>>,
/// Aggressiveness for source-outline body stripping in `compact_cat_output`.
pub source_outline_level: OutlineLevel,
/// Resolved kill-switch: when true, opted-in filters (Tier 1 grep/find,
/// Tier 2 pytest) return the pre-PR legacy behavior. Resolved at
/// `from_options()` time from caller-supplied
/// `MinimizerOptions.legacy_filters` or the `OMP_MINIMIZER_LEGACY_FILTERS`
/// env var; default `false`.
pub legacy_filters_active: bool,
}
impl Default for MinimizerConfig {
fn default() -> Self {
Self {
enabled: false,
only: HashSet::new(),
except: HashSet::new(),
max_capture_bytes: DEFAULT_MAX_CAPTURE_BYTES,
per_command: HashMap::new(),
user_pipelines: None,
enabled: false,
only: HashSet::new(),
except: HashSet::new(),
max_capture_bytes: DEFAULT_MAX_CAPTURE_BYTES,
per_command: HashMap::new(),
user_pipelines: None,
source_outline_level: OutlineLevel::Default,
legacy_filters_active: false,
}
}
}
@@ -83,6 +124,18 @@ impl MinimizerConfig {
if let Some(n) = opts.max_capture_bytes {
cfg.max_capture_bytes = n.max(1024);
}
if let Some(raw) = opts.source_outline_level.as_deref()
&& let Some(level) = OutlineLevel::parse(raw)
{
cfg.source_outline_level = level;
}
let legacy_requested = resolve_legacy_filters(
opts.legacy_filters,
std::env::var("OMP_MINIMIZER_LEGACY_FILTERS")
.ok()
.as_deref(),
);
cfg.legacy_filters_active = legacy_requested;
if let Some(path) = opts.settings_path.as_deref()
&& !path.is_empty()
{
@@ -106,6 +159,15 @@ impl MinimizerConfig {
}
if let Ok(file) = toml::from_str::<SettingsFile>(&contents) {
file.merge_into(&mut cfg);
if opts.enabled == Some(false) {
cfg.enabled = false;
}
if legacy_requested {
cfg.legacy_filters_active = true;
}
if opts.legacy_filters == Some(false) {
cfg.legacy_filters_active = false;
}
}
match pipeline::parse_file(&contents, "user") {
Ok((pipelines, tests)) => {
@@ -141,18 +203,25 @@ impl MinimizerConfig {
pub fn per_command(&self, program: &str) -> Option<&toml::Value> {
self.per_command.get(&program.to_lowercase())
}
/// Whether opted-in filters should fall back to pre-PR legacy behavior.
pub const fn legacy_filters_active(&self) -> bool {
self.legacy_filters_active
}
}
#[derive(Debug, Default, Deserialize)]
struct SettingsFile {
#[serde(default)]
schema_version: Option<u32>,
enabled: Option<bool>,
only: Option<Vec<String>>,
except: Option<Vec<String>>,
max_capture_bytes: Option<u32>,
schema_version: Option<u32>,
enabled: Option<bool>,
only: Option<Vec<String>>,
except: Option<Vec<String>>,
max_capture_bytes: Option<u32>,
source_outline_level: Option<String>,
legacy_filters: Option<bool>,
#[serde(flatten)]
tables: HashMap<String, toml::Value>,
tables: HashMap<String, toml::Value>,
}
impl SettingsFile {
@@ -178,6 +247,14 @@ impl SettingsFile {
if let Some(n) = self.max_capture_bytes {
cfg.max_capture_bytes = n.max(1024);
}
if let Some(raw) = self.source_outline_level.as_deref()
&& let Some(level) = OutlineLevel::parse(raw)
{
cfg.source_outline_level = level;
}
if let Some(v) = self.legacy_filters {
cfg.legacy_filters_active = resolve_legacy_filters(Some(v), None);
}
for (k, v) in self.tables {
if v.is_table() && k != "filters" && k != "tests" {
cfg.per_command.insert(k.to_lowercase(), v);
@@ -186,6 +263,22 @@ impl SettingsFile {
}
}
/// Resolve the effective `legacy_filters_active` flag from the caller option
/// and the raw `OMP_MINIMIZER_LEGACY_FILTERS` env value.
///
/// Pure so it can be unit-tested without mutating the process-global
/// environment (the test harness runs tests in one process in parallel). An
/// explicit option always wins; otherwise a truthy env value enables the
/// legacy path.
fn resolve_legacy_filters(option: Option<bool>, env_value: Option<&str>) -> bool {
match option {
Some(v) => v,
None => env_value.is_some_and(|raw| {
matches!(raw.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes")
}),
}
}
fn expand_tilde(path: &str) -> PathBuf {
if let Some(rest) = path.strip_prefix("~/")
&& let Some(home) = home_dir()
@@ -265,4 +358,91 @@ mod tests {
});
assert!(cfg.enabled);
}
#[test]
fn legacy_filters_option_some_true_sets_active() {
// Explicit caller-supplied `Some(true)` must result in
// `legacy_filters_active == true` regardless of env. This is the
// test-friendly invocation path that avoids env-var mutation.
let cfg = MinimizerConfig::from_options(&MinimizerOptions {
enabled: Some(true),
legacy_filters: Some(true),
..Default::default()
});
assert!(cfg.legacy_filters_active());
}
#[test]
fn legacy_filters_option_some_false_overrides_env() {
// Explicit `Some(false)` must override any env var (verified by
// inspecting the resolver — `Some(_)` arm short-circuits before
// reading the env). We assert behavior via a default-constructed
// `MinimizerOptions { legacy_filters: Some(false), .. }`.
let cfg = MinimizerConfig::from_options(&MinimizerOptions {
enabled: Some(true),
legacy_filters: Some(false),
..Default::default()
});
assert!(!cfg.legacy_filters_active());
}
#[test]
fn resolve_legacy_filters_prefers_option_over_env() {
// The pure resolver lets us exercise every branch without mutating the
// process-global env var (which would race the parallel test harness).
assert!(super::resolve_legacy_filters(Some(true), Some("0")));
assert!(!super::resolve_legacy_filters(Some(false), Some("1")));
}
#[test]
fn resolve_legacy_filters_defaults_false_without_env() {
assert!(!super::resolve_legacy_filters(None, None));
}
#[test]
fn resolve_legacy_filters_honors_truthy_env_when_option_absent() {
for raw in ["1", "true", "yes", " TRUE ", "Yes"] {
assert!(super::resolve_legacy_filters(None, Some(raw)), "{raw:?} should enable");
}
for raw in ["0", "false", "no", ""] {
assert!(!super::resolve_legacy_filters(None, Some(raw)), "{raw:?} should not enable");
}
}
#[test]
fn settings_file_parses_legacy_filters_switch() {
let file: SettingsFile = toml::from_str("legacy_filters = true\n").unwrap();
let mut cfg = MinimizerConfig::default();
file.merge_into(&mut cfg);
assert!(cfg.legacy_filters_active());
}
#[test]
fn explicit_disabled_option_overrides_enabled_settings_file() {
let path = std::env::temp_dir()
.join(format!("omp-minimizer-config-disabled-{}.toml", std::process::id()));
std::fs::write(&path, "enabled = true\n").unwrap();
let cfg = MinimizerConfig::from_options(&MinimizerOptions {
enabled: Some(false),
settings_path: Some(path.display().to_string()),
..Default::default()
});
let _ = std::fs::remove_file(&path);
assert!(!cfg.enabled);
}
#[test]
fn explicit_legacy_true_overrides_disabled_settings_file() {
let path = std::env::temp_dir()
.join(format!("omp-minimizer-config-legacy-{}.toml", std::process::id()));
std::fs::write(&path, "legacy_filters = false\n").unwrap();
let cfg = MinimizerConfig::from_options(&MinimizerOptions {
enabled: Some(true),
settings_path: Some(path.display().to_string()),
legacy_filters: Some(true),
..Default::default()
});
let _ = std::fs::remove_file(&path);
assert!(cfg.legacy_filters_active());
}
}
crates/pi-shell/src/minimizer/defs/apt.toml
+52
View File
@@ -0,0 +1,52 @@
[filters.apt]
description = "Compact apt/apt-get/yum/dnf/apk package manager output — strip progress lines, keep errors and final result"
match_command = "^(apt|apt-get|yum|dnf|apk)$"
strip_ansi = true
strip_lines_matching = [
"^\\s*$",
"^\\s*(Get:|Hit:|Ign:)",
"^\\s*(Fetched|Processing|Selecting|Preparing|Unpacking|Setting up|Reading|Building)\\s",
"^\\s*\\[\\d+%\\]",
"^\\(Reading database",
"^Loaded plugins:",
"^Loading mirror",
"^\\s*-->",
"^\\s*Package\\s",
"^\\s*Marking\\s",
"^fetch\\s",
"^\\(\\d+/\\d+\\)",
"^OK:",
"^\\s*\\d+/\\d+:",
]
max_lines = 60
on_empty = "ok"
[[tests.apt]]
name = "successful apt-get install strips noise and keeps summary"
input = """
Reading package lists... Done
Building dependency tree... Done
Reading state information... Done
The following NEW packages will be installed:
curl
0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 curl amd64 7.68.0-1ubuntu2 [161 kB]
Fetched 161 kB in 1s (189 kB/s)
Selecting previously unselected package curl.
(Reading database ... 112300 files and directories currently installed.)
Preparing to unpack .../curl_7.68.0-1ubuntu2_amd64.deb ...
Unpacking curl (7.68.0-1ubuntu2) ...
Setting up curl (7.68.0-1ubuntu2) ...
Processing triggers for man-db (2.9.1-1) ...
"""
expected = "The following NEW packages will be installed:\n curl\n0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\n"
[[tests.apt]]
name = "failed install passes through error block"
input = """
Reading package lists... Done
Building dependency tree... Done
Reading state information... Done
E: Unable to locate package nonexistent-pkg
"""
expected = "E: Unable to locate package nonexistent-pkg\n"
crates/pi-shell/src/minimizer/defs/conda.toml
+51
View File
@@ -0,0 +1,51 @@
[filters.conda]
description = "Compact conda install/create/update output — strip download progress and transaction noise"
match_command = "^conda$"
strip_ansi = true
strip_lines_matching = [
"^\\s*$",
"^(Downloading|Extracting|Preparing transaction|Verifying transaction|Executing transaction)",
"^\\s*##",
"^\\s*\\d+%",
"^\\s*[#=]{3,}",
]
max_lines = 50
on_empty = "ok"
[[tests.conda]]
name = "successful install strips progress and keeps package list"
input = """
Collecting package metadata (current_repodata.json): done
Solving environment: done
## Package Plan ##
environment location: /opt/conda
added / updated specs:
- numpy
The following packages will be downloaded:
package | build
---------------------------|-----------------
numpy-1.24.3 | py310h8e6c1ab_0 5.5 MB
Preparing transaction: done
Verifying transaction: done
Executing transaction: done
"""
expected = "Collecting package metadata (current_repodata.json): done\nSolving environment: done\n environment location: /opt/conda\n added / updated specs:\n - numpy\nThe following packages will be downloaded:\n package | build\n ---------------------------|-----------------\n numpy-1.24.3 | py310h8e6c1ab_0 5.5 MB\n"
[[tests.conda]]
name = "failed install passes through error"
input = """
Collecting package metadata (current_repodata.json): done
Solving environment: failed
PackagesNotFoundError: The following packages are not available from current channels:
- fake-package-xyz
"""
expected = "Collecting package metadata (current_repodata.json): done\nSolving environment: failed\nPackagesNotFoundError: The following packages are not available from current channels:\n - fake-package-xyz\n"
crates/pi-shell/src/minimizer/defs/gcc.toml
+22 -4
View File
@@ -3,15 +3,13 @@
[filters.gcc]
description = "Compact gcc/g++ compiler output — strip notes, keep errors and warnings"
match_command = "^(gcc|g\\+\\+)$"
match_command = "^(gcc|g\\+\\+|clang|clang\\+\\+)$"
strip_ansi = true
strip_lines_matching = [
"^\\s*$",
"^\\s+\\|\\s*$",
"^In file included from",
"^\\s+from\\s",
"^\\d+ warnings? generated",
"^\\d+ errors? generated",
]
max_lines = 50
on_empty = "gcc: ok"
@@ -30,7 +28,7 @@ main.c:15:12: warning: unused variable 'x' [-Wunused-variable]
2 warnings generated.
1 error generated.
"""
expected = "main.c:10:5: error: use of undeclared identifier 'foo'\n foo();\n ^\nmain.c:15:12: warning: unused variable 'x' [-Wunused-variable]\n int x = 42;\n ^\n"
expected = "main.c:10:5: error: use of undeclared identifier 'foo'\n foo();\n ^\nmain.c:15:12: warning: unused variable 'x' [-Wunused-variable]\n int x = 42;\n ^\n2 warnings generated.\n1 error generated.\n"
[[tests.gcc]]
name = "clean compilation"
@@ -50,3 +48,23 @@ expected = "/usr/bin/ld: /tmp/main.o: undefined reference to 'missing_func'\ncol
name = "empty input returns on_empty message"
input = ""
expected = "gcc: ok"
[[tests.gcc]]
name = "clang variant errors"
input = """
foo.c:3:10: fatal error: 'missing.h' file not found
#include "missing.h"
^~~~~~~~~~~
1 error generated.
"""
expected = "foo.c:3:10: fatal error: 'missing.h' file not found\n#include \"missing.h\"\n ^~~~~~~~~~~\n1 error generated.\n"
[[tests.gcc]]
name = "clang++ variant errors"
input = """
foo.cpp:8:5: error: unknown type name 'Widget'
Widget w;
^
1 error generated.
"""
expected = "foo.cpp:8:5: error: unknown type name 'Widget'\n Widget w;\n ^\n1 error generated.\n"
crates/pi-shell/src/minimizer/detect.rs
+214
View File
@@ -168,6 +168,71 @@ fn skip_time_options(tokens: &[String], mut index: usize) -> Option<usize> {
Some(index)
}
fn skip_aws_global_options(args: &[String]) -> Option<usize> {
const VALUE_FLAGS: &[&str] = &[
"--profile",
"--region",
"--endpoint-url",
"--cli-binary-format",
"--output",
"--cli-read-timeout",
"--cli-connect-timeout",
"--ca-bundle",
"--color",
"--query",
"--cli-input-json",
"--cli-input-yaml",
];
const BOOL_FLAGS: &[&str] = &[
"--no-cli-pager",
"--debug",
"--no-verify-ssl",
"--no-paginate",
"--no-sign-request",
"--cli-auto-prompt",
"--no-cli-auto-prompt",
];
let mut index = 0;
while let Some(arg) = args.get(index) {
if arg == "--" {
return args.get(index + 1).map(|_| index + 1);
}
if BOOL_FLAGS.contains(&arg.as_str()) {
index += 1;
continue;
}
if arg == "--generate-cli-skeleton" {
index += 1;
if args
.get(index)
.is_some_and(|value| matches!(value.as_str(), "input" | "output" | "yaml-input"))
{
index += 1;
}
continue;
}
if arg.starts_with("--generate-cli-skeleton=") {
index += 1;
continue;
}
if option_consumes_value(arg, VALUE_FLAGS) {
index = if option_has_inline_value(arg, VALUE_FLAGS) {
index + 1
} else {
index + 2
};
continue;
}
break;
}
if index > args.len() {
None
} else {
Some(index)
}
}
fn skip_option_value(tokens: &[String], index: usize) -> Option<usize> {
let token = tokens.get(index)?;
if token.starts_with("--") && token.contains('=') {
@@ -254,6 +319,24 @@ fn detect_subcommand(program: &str, args: &[String]) -> Option<String> {
],
&[],
),
"npx" => first_non_global_arg(
args,
&[
"--workspace",
"-w",
"--package",
"-p",
"--prefix",
"--cache",
"--registry",
"--userconfig",
"--call",
"--shell",
"--node-arg",
],
&["--yes", "--no", "--no-install", "--quiet", "--silent", "--verbose"],
&[],
),
"pnpm" => first_non_global_arg(
args,
&["--dir", "-C", "--filter", "-F", "--workspace", "--config", "--store-dir"],
@@ -292,6 +375,48 @@ fn detect_subcommand(program: &str, args: &[String]) -> Option<String> {
&["--verbose", "--quiet", "--no-color"],
&[],
),
"aws" => skip_aws_global_options(args)
.and_then(|index| args.get(index))
.map(|arg| arg.to_lowercase()),
"uv" | "uvx" => first_non_global_arg(
args,
&[
"--directory",
"-C",
"--project",
"-p",
"--cache-dir",
"--config-file",
"--config-setting",
"--python",
"--python-preference",
"--exclude-newer",
"--color",
"--allow-insecure-host",
"--no-binary",
"--only-binary",
],
&[
"--offline",
"--no-cache",
"--no-cache-dir",
"--no-progress",
"--native-tls",
"--no-native-tls",
"--quiet",
"-q",
"--verbose",
"-v",
"--upgrade",
"--no-upgrade",
"--require-hashes",
"--verify-hashes",
"--no-verify-hashes",
"--no-build",
"--reinstall",
],
&[],
),
"jest" | "vitest" => first_non_global_arg(args, &[], &[], &[]),
_ => args
.iter()
@@ -462,6 +587,17 @@ mod tests {
assert!(detect("env -S 'git status'").is_none());
}
#[test]
fn detects_direct_lint_tools() {
let command = detect("eslint src/foo.ts").expect("eslint command is detected");
assert_eq!(command.program, "eslint");
assert_eq!(command.subcommand.as_deref(), Some("src/foo.ts"));
let command = detect("tsc --project tsconfig.json").expect("tsc command is detected");
assert_eq!(command.program, "tsc");
assert_eq!(command.subcommand.as_deref(), Some("tsconfig.json"));
}
#[test]
fn detects_gt_through_wrappers_and_globals() {
let command = detect("env GRAPHITE_TOKEN=x command gt --repo owner/repo submit --stack")
@@ -476,6 +612,63 @@ mod tests {
assert_eq!(command.program, "gt");
assert_eq!(command.subcommand.as_deref(), Some("sync"));
}
#[test]
fn skips_aws_global_options() {
let command = detect(
"aws --profile foo --region=us-east-1 --endpoint-url http://localhost:4566 \
--no-cli-pager --generate-cli-skeleton output s3 ls",
)
.expect("aws command is detected");
assert_eq!(command.program, "aws");
assert_eq!(command.subcommand.as_deref(), Some("s3"));
}
#[test]
fn aws_double_dash_terminates_global_options() {
let command = detect("aws -- --literal-service op").expect("aws command is detected");
assert_eq!(command.program, "aws");
assert_eq!(command.subcommand.as_deref(), Some("--literal-service"));
}
#[test]
fn aws_global_option_permutations_keep_service_subcommand() {
let flags = [
"--profile dev",
"--region us-east-1",
"--endpoint-url=http://localhost:4566",
"--cli-binary-format raw-in-base64-out",
"--output json",
"--cli-read-timeout=5",
"--cli-connect-timeout 5",
"--ca-bundle /tmp/ca.pem",
"--color off",
"--query Buckets[].Name",
"--cli-input-json file://input.json",
"--cli-input-yaml file://input.yaml",
"--no-cli-pager",
"--debug",
"--no-verify-ssl",
"--no-paginate",
"--no-sign-request",
"--cli-auto-prompt",
"--no-cli-auto-prompt",
"--generate-cli-skeleton",
"--generate-cli-skeleton=output",
];
for idx in 0..128 {
let mut command = String::from("aws");
for (bit, flag) in flags.iter().enumerate() {
if idx & (1 << (bit % 7)) != 0 && (idx + bit) % 3 == 0 {
command.push(' ');
command.push_str(flag);
}
}
command.push_str(" lambda list-functions");
let detected = detect(&command).expect("aws command is detected");
assert_eq!(detected.subcommand.as_deref(), Some("lambda"), "{command}");
}
}
}
#[test]
@@ -488,3 +681,24 @@ fn detects_bun_globals_and_subcommands() {
assert_eq!(command.program, "bun");
assert_eq!(command.subcommand.as_deref(), Some("test"));
}
#[test]
fn npx_workspace_value_is_skipped_in_subcommand_detection() {
// `npx -w <workspace>` is a value-taking option; the workspace name must
// not be returned as the subcommand. The actual tool name follows after
// the option value.
let command = detect("npx -w vitest echo PASS").expect("npx command is detected");
assert_eq!(command.program, "npx");
assert_eq!(command.subcommand.as_deref(), Some("echo"));
// plain npx invocation with an actual tool still resolves correctly
let command = detect("npx vitest").expect("npx vitest is detected");
assert_eq!(command.program, "npx");
assert_eq!(command.subcommand.as_deref(), Some("vitest"));
// workspace value that happens to be a tool name is skipped; the next
// token is the actual tool
let command = detect("npx -w my-workspace vitest").expect("npx with workspace is detected");
assert_eq!(command.program, "npx");
assert_eq!(command.subcommand.as_deref(), Some("vitest"));
}
crates/pi-shell/src/minimizer/engine.rs
+554 -18
View File
@@ -21,6 +21,8 @@ pub enum MinimizerMode {
None,
/// Capture the whole command and apply one filter to the whole buffer.
WholeCommand,
/// Execute a safe `&&` / `;` chain segment-by-segment.
SegmentedChain,
}
/// Return the minimization mode for a command.
@@ -36,6 +38,22 @@ pub fn mode_for(command: &str, config: &MinimizerConfig) -> MinimizerMode {
MinimizerMode::None
}
},
plan::CommandPlan::Chain { segments } => {
// Only route a chain through the segmented runner when the minimizer is
// enabled, the legacy kill-switch is off, at least one segment is
// eligible, and no segment can permanently rewire the shell's own file
// descriptors (`exec >out`). Any failed guard restores the pre-PR
// single-exec passthrough behaviour.
if config.enabled
&& !config.legacy_filters_active()
&& chain_has_eligible_segment(&segments, config)
&& !chain_mutates_shell_fds(&segments)
{
MinimizerMode::SegmentedChain
} else {
MinimizerMode::None
}
},
plan::CommandPlan::Compound | plan::CommandPlan::Piped | plan::CommandPlan::Unsupported => {
MinimizerMode::None
},
@@ -72,11 +90,15 @@ pub fn apply(
}
// Structural guard: this whole-buffer path only handles single simple
// commands. Compound commands and pipes can feed downstream parsers
// (awk, jq, rg, …), so rewriting their combined output is a correctness
// bug.
// commands. Safe chains are intentionally kept opaque here so the engine
// can only segment them when the shell executes each piece separately.
// Pipes can feed downstream parsers (awk, jq, rg, …), so rewriting their
// combined output is a correctness bug.
match plan::analyze(command) {
plan::CommandPlan::Single { .. } => {},
plan::CommandPlan::Chain { segments } => {
return apply_chain(command, &segments, captured, exit_code, config);
},
plan::CommandPlan::Piped => {
return MinimizerOutput::passthrough(captured).labeled("piped");
},
@@ -95,6 +117,41 @@ pub fn apply(
apply_identity(&identity, command, captured, exit_code, config)
}
/// Apply the whole-buffer dispatch path for a `Chain { segments }` plan.
///
/// The FFI whole-buffer entry point sees the entire chain's captured stdout
/// (interleaved across segments) — it cannot split it back into per-segment
/// slices. That makes the whole-buffer path fundamentally unable to minimize a
/// chain safely: every git renderer that condenses output (`condense_status`,
/// `compact_diff_output`, `condense_stash`, …) parses the buffer and rebuilds a
/// single synthetic result, so feeding it two segments' interleaved captures
/// produces output that never existed for any one command.
///
/// Concretely, `git -C a status && git -C b status` would let `condense_status`
/// overwrite `summary.branch` with the *last* repo and sum both repos'
/// clean/dirty counts into one fabricated status. The same multi-capture merge
/// corrupts same-subcommand `diff`/`stash`/`log`/… chains: none of these
/// renderers is associative over concatenated captures, and the whole-buffer
/// path has no way to attribute lines back to their originating segment.
///
/// Per-segment minimization (where each segment is captured in isolation and is
/// safe to route through its own filter) is handled separately by the segmented
/// chain runner. The whole-buffer path therefore stays opaque for every chain:
/// it preserves the captured bytes verbatim and labels the result `compound`.
///
/// Kill-switch parity (M2): `legacy_filters_active` also returns the opaque
/// passthrough so callers can rollback without recompile.
fn apply_chain(
command: &str,
segments: &[plan::ChainSegment],
captured: &str,
_exit_code: i32,
_config: &MinimizerConfig,
) -> MinimizerOutput {
let _ = (command, segments);
MinimizerOutput::passthrough(captured).labeled("compound")
}
fn identity_has_filter(identity: &detect::CommandIdentity, config: &MinimizerConfig) -> bool {
if !config.is_program_enabled(&identity.program) {
return false;
@@ -105,6 +162,133 @@ fn identity_has_filter(identity: &detect::CommandIdentity, config: &MinimizerCon
|| resolve_pipeline(config, &identity.program, subcommand).is_some()
}
fn chain_has_eligible_segment(segments: &[plan::ChainSegment], config: &MinimizerConfig) -> bool {
segments.iter().any(|segment| {
detect::detect(&segment.command)
.is_some_and(|identity| identity_has_filter(&identity, config))
|| is_common_chain_utility(&segment.program)
})
}
/// True when any segment can permanently rewire the shell's own file
/// descriptors. The segmented chain runner executes each segment in a fresh
/// capture context with its own stdout/stderr pipe, so fd mutations made by one
/// segment (e.g. `exec >out`, `exec 2>err`) are not honored by the segments
/// that follow: output the user redirected to a file would instead be captured
/// and returned to the caller. When such a segment is present we refuse to
/// segment and leave the chain opaque (passthrough), preserving the original
/// redirection semantics.
fn chain_mutates_shell_fds(segments: &[plan::ChainSegment]) -> bool {
segments.iter().any(is_shell_fd_mutating_segment)
}
/// True when a segment's effective command can mutate the shell parse/runtime
/// environment in a way that segmented execution cannot preserve.
///
/// `exec` rewires fds; `eval` / `source` / `.` can introduce that opaquely;
/// `alias` / `unalias` change how later words in separate `run_string` calls
/// are expanded. Resolves the simple direct case from the parsed program word
/// first so quoted assignments such as `FOO="a b" exec >out` cannot fool the
/// fallback whitespace scan.
fn is_shell_fd_mutating_segment(segment: &plan::ChainSegment) -> bool {
if is_shell_state_mutating_program(&segment.program) {
return true;
}
if matches!(segment.program.as_str(), "command" | "builtin")
&& command_wrapper_invokes_mutator(segment)
{
return true;
}
false
}
fn is_shell_state_mutating_program(program: &str) -> bool {
matches!(program, "exec" | "eval" | "source" | "." | "alias" | "unalias")
}
fn command_wrapper_invokes_mutator(segment: &plan::ChainSegment) -> bool {
for word in segment.command.split_whitespace() {
if is_shell_state_mutating_program(word) {
return true;
}
// A split quoted assignment means we are no longer looking at real shell
// words. Stay opaque rather than proving safety from corrupted tokens.
if is_ambiguous_assignment_fragment(word) {
return true;
}
if word == "command" || word == "builtin" || word.starts_with('-') || is_env_assignment(word)
{
continue;
}
return false;
}
false
}
fn is_ambiguous_assignment_fragment(word: &str) -> bool {
is_env_assignment(word) && (word.contains('"') || word.contains('\''))
}
/// True for a leading `KEY=value` environment assignment (a prefix that does
/// not change which command word ultimately runs).
fn is_env_assignment(word: &str) -> bool {
word.split_once('=').is_some_and(|(key, _)| {
!key.is_empty() && key.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_')
})
}
/// Common shell utilities that on their own would not warrant whole-command
/// minimization, but whose presence in a `&&` / `;` chain alongside other
/// segments is enough to fire the segmented chain runner. Each such segment
/// is captured and passes through `minimizer::apply` which will treat it as
/// `Single` with no matching filter and stream the text unchanged.
fn is_common_chain_utility(program: &str) -> bool {
matches!(
program,
"echo"
| "printf"
| "head"
| "tail"
| "file"
| "which"
| "type"
| "sed"
| "awk"
| "sleep"
| "seq"
| "cp" | "mv"
| "rm" | "mkdir"
| "rmdir"
| "touch"
| "basename"
| "dirname"
| "realpath"
| "readlink"
| "true"
| "false"
| "yes"
| "tr" | "tee"
| "sort"
| "uniq"
| "cut"
| "paste"
| "rev"
| "split"
| "comm"
| "patch"
| "xargs"
| "unzip"
| "zip"
| "tar"
| "gzip"
| "gunzip"
| "cd" | "pwd"
| "export"
| "env"
| "test"
)
}
fn apply_identity(
identity: &detect::CommandIdentity,
command: &str,
@@ -120,13 +304,22 @@ fn apply_identity(
if filters::supports(&identity.program, subcommand) {
let ctx = MinimizerCtx { program: &identity.program, subcommand, command, config };
let rust_output =
match catch_unwind(AssertUnwindSafe(|| filters::filter(&ctx, captured, exit_code))) {
Ok(out) => out,
Err(_) => MinimizerOutput::passthrough(captured),
};
let Ok(rust_output) =
catch_unwind(AssertUnwindSafe(|| filters::filter(&ctx, captured, exit_code)))
else {
return MinimizerOutput::passthrough(captured)
.labeled(program_label(&identity.program))
.with_original(captured);
};
let label = program_label(&identity.program);
let overlaid = apply_pipeline_overlay(config, &identity.program, rust_output, label);
let overlaid = apply_pipeline_overlay(
config,
&identity.program,
subcommand,
exit_code,
rust_output,
label,
);
return ensure_success_visible(overlaid, exit_code).with_original(captured);
}
@@ -190,6 +383,10 @@ fn program_label(program: &str) -> &'static str {
"rake" => "rake",
"rails" => "rails",
"rubocop" => "rubocop",
"rustfmt" => "rustfmt",
"xxd" => "xxd",
"strings" => "strings",
"od" => "od",
"tsc" => "tsc",
"eslint" => "eslint",
"biome" => "biome",
@@ -232,12 +429,17 @@ fn program_label(program: &str) -> &'static str {
fn apply_pipeline_overlay(
config: &MinimizerConfig,
program: &str,
subcommand: Option<&str>,
exit_code: i32,
inner: MinimizerOutput,
primary_label: &'static str,
) -> MinimizerOutput {
let Some(pipeline) = resolve_pipeline(config, program, None) else {
let Some(pipeline) = resolve_pipeline(config, program, subcommand) else {
return inner.labeled(primary_label);
};
if pipeline.skipped_by_exit(exit_code) {
return inner.labeled(primary_label);
}
let text = catch_unwind(AssertUnwindSafe(|| pipeline.apply(&inner.text).into_owned()))
.unwrap_or_else(|_| inner.text.clone());
if text == inner.text {
@@ -312,8 +514,28 @@ pub fn verify_builtin_filters() -> Vec<pipeline::TestOutcome> {
#[cfg(test)]
mod tests {
use super::*;
use std::{
fs,
sync::atomic::{AtomicUsize, Ordering},
};
static CONFIG_COUNTER: AtomicUsize = AtomicUsize::new(0);
use super::*;
use crate::minimizer::MinimizerOptions;
fn config_from_settings(contents: &str) -> MinimizerConfig {
let nonce = CONFIG_COUNTER.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir()
.join(format!("pi-shell-minimizer-engine-{}-{nonce}.toml", std::process::id()));
fs::write(&path, contents).expect("write minimizer settings");
let cfg = MinimizerConfig::from_options(&MinimizerOptions {
enabled: Some(true),
settings_path: Some(path.to_string_lossy().into_owned()),
..Default::default()
});
let _ = fs::remove_file(path);
cfg
}
#[test]
fn disabled_config_does_not_minimize() {
let cfg = MinimizerConfig::default();
@@ -322,6 +544,55 @@ mod tests {
assert!(!out.changed);
}
#[test]
fn disabled_minimizer_and_disabled_program_do_not_transform_supported_command() {
let input = "diff --git a/file.rs b/file.rs\n@@\n-old\n+new\n";
let disabled = MinimizerConfig::default();
assert!(!should_minimize("git diff", &disabled));
let out = apply("git diff", input, 0, &disabled);
assert!(!out.changed);
assert_eq!(out.text, input);
assert_eq!(out.filter, "disabled");
let except_git = MinimizerConfig {
enabled: true,
except: std::iter::once("git".to_string()).collect(),
..Default::default()
};
assert!(!should_minimize("git diff", &except_git));
let out = apply("git diff", input, 0, &except_git);
assert!(!out.changed);
assert_eq!(out.text, input);
assert_eq!(out.filter, "disabled");
}
#[test]
fn pipeline_overlay_honors_subcommand_and_exit_gates() {
let cfg = config_from_settings(
r#"
schema_version = 1
[filters.git_diff_overlay]
match_command = "^git$"
match_subcommand = "^diff$"
strip_lines_matching = [".*"]
on_empty = "OVERLAY"
only_on_exit = [0]
"#,
);
let diff_input = "diff --git a/file.rs b/file.rs\n@@\n-old\n+new\n";
let diff = apply("git diff", diff_input, 0, &cfg);
assert_eq!(diff.filter, "pipeline+builtin");
assert_eq!(diff.text, "OVERLAY");
let status = apply("git status", "## main\n M file.rs\n", 0, &cfg);
assert_ne!(status.filter, "pipeline+builtin");
assert!(status.text.contains("unstaged 1"));
let failed = apply("git diff", diff_input, 1, &cfg);
assert_ne!(failed.filter, "pipeline+builtin");
assert!(failed.text.contains("file changed"));
}
#[test]
fn enabled_known_filter_minimizes() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
@@ -332,13 +603,14 @@ mod tests {
}
#[test]
fn enabled_config_does_not_minimize_git_status() {
fn enabled_config_minimizes_git_status() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
assert!(!should_minimize("git status", &cfg));
assert!(should_minimize("git status", &cfg));
let input = "## main\n M file.rs\n";
let out = apply("git status", input, 0, &cfg);
assert!(!out.changed);
assert_eq!(out.text, input);
assert!(out.changed);
assert!(out.text.contains("unstaged 1"));
assert_eq!(out.filter, "git");
}
#[test]
@@ -358,6 +630,27 @@ mod tests {
assert!(out.original_text.is_some());
}
#[test]
fn successful_user_pipeline_empty_output_returns_visible_ok() {
let cfg = config_from_settings(
r#"
schema_version = 1
[filters.empty_ok]
match_command = "^printf$"
strip_lines_matching = [".*"]
"#,
);
assert!(should_minimize("printf done", &cfg));
let out = apply("printf done", "drop me\n", 0, &cfg);
assert!(out.changed);
assert_eq!(out.text, "OK\n");
assert_eq!(out.filter, "pipeline");
assert_eq!(out.output_bytes, out.text.len());
assert_eq!(out.original_text.as_deref(), Some("drop me\n"));
}
#[test]
fn failed_minimization_does_not_invent_ok_for_empty_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
@@ -378,11 +671,44 @@ mod tests {
}
#[test]
fn compound_and_piped_commands_do_not_minimize() {
fn segmented_chain_mode_is_only_for_eligible_safe_chains() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
assert_eq!(mode_for("echo start ; git status", &cfg), MinimizerMode::None);
assert_eq!(mode_for("false && git status", &cfg), MinimizerMode::None);
assert_eq!(
mode_for("git diff --stat && git diff --name-only", &cfg),
MinimizerMode::SegmentedChain
);
assert_eq!(mode_for("git diff ; printf done", &cfg), MinimizerMode::SegmentedChain);
// Common shell utilities make a chain eligible for the segmented runner
// even when no segment has a dedicated filter — segments stream through
// per-segment passthrough so the chain itself is captured for telemetry.
assert_eq!(mode_for("false && echo no ; echo yes", &cfg), MinimizerMode::SegmentedChain);
assert_eq!(mode_for("foo || bar", &cfg), MinimizerMode::None);
assert_eq!(mode_for("git status | cat", &cfg), MinimizerMode::None);
assert_eq!(mode_for("sleep 1 &", &cfg), MinimizerMode::None);
assert_eq!(mode_for("(cd foo && make)", &cfg), MinimizerMode::None);
}
#[test]
fn segmented_chain_supported_command_does_not_record_unknown() {
// Phase 7 (Mode α resolution): supported chains route through
// filters::dispatch via the chain decomposer instead of falling
// back to passthrough. The unknown-command counter must remain
// stable — the chain entry point is structurally known.
reset_unknown_command_count();
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = "diff --git a/file.rs b/file.rs\n@@\n-old\n+new\n";
let before = unknown_command_count();
assert_eq!(mode_for("git diff ; printf done", &cfg), MinimizerMode::SegmentedChain);
let out = apply("git diff ; printf done", input, 0, &cfg);
// Whole-buffer entry: a mixed chain (`git diff` + `printf`) stays opaque
// rather than running the git filter over the interleaved capture. The
// chain entry point is still structurally known, so no unknown-command is
// recorded (per-segment minimization is the segmented runner's job).
assert!(!out.changed, "mixed chain must stay passthrough in whole-buffer minimization");
assert_eq!(out.filter, "compound");
assert_eq!(unknown_command_count(), before);
}
#[test]
@@ -415,6 +741,216 @@ mod tests {
assert!(!gtest.text.contains("Foo.Pass"));
assert!(gtest.text.contains("foo_test.cc:42: Failure"));
}
#[test]
fn git_status_chain_stays_opaque() {
// `condense_status` rebuilds a single synthetic status from the whole
// buffer: it keeps only the last `On branch …` it sees and sums every
// segment's clean/dirty counts. For `git -C a status && git -C b status`
// that fabricates one status that never existed for either repo, so the
// whole-buffer path must stay opaque and preserve the captured bytes.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = "On branch feature-a\n M a.rs\nOn branch feature-b\n M b.rs\n";
let out = apply("git -C a status && git -C b status", input, 0, &cfg);
assert!(!out.changed, "same-subcommand status chain must stay passthrough");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input, "captured output must be preserved verbatim");
// Both repos' branch headers survive — no synthetic merged status.
assert!(out.text.contains("feature-a") && out.text.contains("feature-b"));
}
#[test]
fn git_commit_chain_differing_actions_stays_opaque() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = "On branch main\nChanges to be committed:\n modified: src/lib.rs\n[main \
abc1234] init\n 1 file changed, 1 insertion(+)\n";
let out = apply("git commit --dry-run && git commit -m init", input, 0, &cfg);
assert!(!out.changed, "commit actions share a subcommand but not an output contract");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input);
}
#[test]
fn git_only_chain_differing_subcommands_stays_opaque() {
// `git status && git log` must NOT route the whole buffer through one
// subcommand filter: `condense_status` rebuilds output from its own parse
// and would silently drop the `git log` segment's lines. Stay opaque and
// preserve the captured output verbatim.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = "## main\n M file.rs\n";
let out = apply("git status && git log -1", input, 0, &cfg);
assert!(!out.changed, "differing-subcommand git chain must stay passthrough");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input, "captured output must be preserved verbatim");
}
#[test]
fn git_diff_chain_differing_formats_stays_opaque() {
// `git diff --name-only && git diff --stat` share the `diff` subcommand but
// select incompatible renderers. Routing the combined buffer through one
// (the whole-chain command carries BOTH `--name-only` and `--stat`, so the
// diff filter would treat it as a stat buffer) corrupts the listing
// segment's output. Diverging diff formats must stay opaque.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input =
"src/a.rs\nsrc/b.rs\n src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)\n";
let out = apply("git diff --name-only && git diff --stat", input, 0, &cfg);
assert!(!out.changed, "differing diff formats must stay passthrough");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input, "captured output must be preserved verbatim");
}
#[test]
fn git_diff_chain_same_format_stays_opaque() {
// Even same-format diff chains stay opaque on the whole-buffer path: the
// renderer parses the combined buffer and rebuilds one summary, with no
// way to attribute files back to each segment's repo/ref. `git -C a diff
// && git -C b diff` would merge both repos into one fabricated stat, so
// the captured bytes must be preserved verbatim.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let mut listing = String::new();
for i in 0..30 {
use std::fmt::Write as _;
let _ = writeln!(listing, "src/file{i}.rs");
}
let out = apply("git diff --name-only && git diff --name-only HEAD~1", &listing, 0, &cfg);
assert!(!out.changed, "same-format diff chain must stay passthrough");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, listing, "captured output must be preserved verbatim");
}
#[test]
fn git_diff_raw_and_default_diff_stays_opaque() {
// `git diff --raw && git diff` share the `diff` subcommand but have
// incompatible output formats (raw vs unified). They MUST get distinct
// format keys so the chain stays opaque.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = ":100644 100644 12345... abcde... M\tsrc/a.rs\n diff --git a/src/a.rs \
b/src/a.rs\nindex abc..def 100644\n--- a/src/a.rs\n+++ b/src/a.rs\n@@ -1 +1 \
@@\n-old\n+new\n";
let out = apply("git diff --raw && git diff", input, 0, &cfg);
assert!(!out.changed, "raw+unified diff must stay opaque");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input, "captured output must be preserved verbatim");
}
#[test]
fn git_diff_summary_and_default_diff_stays_opaque() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = " create mode 100644 src/a.rs\n delete mode 100644 src/b.rs\n";
let out = apply("git diff --summary && git diff", input, 0, &cfg);
assert!(!out.changed, "summary+unified diff must stay opaque");
assert_eq!(out.filter, "compound");
}
#[test]
fn git_diff_check_and_default_diff_stays_opaque() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = "src/a.rs:1: leftover conflict marker\n";
let out = apply("git diff --check && git diff", input, 0, &cfg);
assert!(!out.changed, "check+unified diff must stay opaque");
assert_eq!(out.filter, "compound");
}
#[test]
fn git_diff_same_raw_format_stays_opaque() {
// Same subcommand AND same raw format still stays opaque on the
// whole-buffer path: like every git chain here, the combined capture
// cannot be attributed back to each segment, so it is preserved verbatim.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = ":100644 100644 12345... abcde... M\tsrc/a.rs\n:100644 100644 67890... fghij... \
M\tsrc/b.rs\n";
let out = apply("git diff --raw && git diff --raw HEAD~1", input, 0, &cfg);
assert!(!out.changed, "same-raw-format diff chain must stay passthrough");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input, "captured output must be preserved verbatim");
}
#[test]
fn mixed_chain_stays_opaque_in_whole_buffer_minimization() {
// A mixed chain (`git status` + unrelated `echo`) must NOT route the whole
// interleaved capture through the first segment's filter: `condense_status`
// rebuilds from its own parse and would drop the `echo` segment's output.
// Stay opaque and preserve the captured bytes verbatim.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = "## main\n M file.rs\nIMPORTANT side-effect line\n";
let out = apply("git status && echo IMPORTANT side-effect line", input, 0, &cfg);
assert!(!out.changed, "mixed chain must stay passthrough");
assert_eq!(out.filter, "compound");
assert_eq!(out.text, input, "captured output must be preserved verbatim");
assert!(out.text.contains("IMPORTANT side-effect line"));
}
#[test]
fn unsupported_first_segment_chain_is_passthrough() {
// Phase 7: chains whose first segment has no filter fall back to
// passthrough labeled "compound" (preserves legacy behavior).
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let out = apply("zzzobscure && zzznever", "noise\n", 0, &cfg);
assert!(!out.changed);
assert_eq!(out.filter, "compound");
}
#[test]
fn chain_legacy_filters_active_passes_through() {
// Phase 7 kill-switch parity (M2): legacy_filters_active=true returns
// passthrough.labeled("compound") regardless of segment shape.
let cfg =
MinimizerConfig { enabled: true, legacy_filters_active: true, ..Default::default() };
let input = "## main\n M file.rs\n";
let out = apply("git status && git log -1", input, 0, &cfg);
assert!(!out.changed);
assert_eq!(out.filter, "compound");
}
#[test]
fn legacy_filters_active_disables_segmented_chain() {
// Kill-switch parity: with the legacy filters flag set, an otherwise
// eligible safe chain must NOT route through the segmented runner so
// pre-segmentation single-exec behavior is restored.
let mut cfg = MinimizerConfig { enabled: true, ..Default::default() };
cfg.legacy_filters_active = true;
assert_eq!(mode_for("git diff --stat && git diff --name-only", &cfg), MinimizerMode::None);
assert_eq!(mode_for("git diff ; printf done", &cfg), MinimizerMode::None);
}
#[test]
fn disabled_config_does_not_segment_chain() {
// With the master switch off, no chain is segmented even when a segment
// would otherwise be eligible.
let cfg = MinimizerConfig::default();
assert!(!cfg.enabled);
assert_eq!(mode_for("git diff ; printf done", &cfg), MinimizerMode::None);
}
#[test]
fn chains_with_exec_fd_mutation_are_not_segmented() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
// `exec >out` rewires the shell's stdout; segmenting would run the
// following segment with a fresh capture pipe and lose the redirection,
// returning output to the caller that should have gone to the file.
assert_eq!(mode_for("exec >out ; echo hi", &cfg), MinimizerMode::None);
assert_eq!(mode_for("exec 2>err ; git status", &cfg), MinimizerMode::None);
// The fd-mutating segment poisons the chain even when it is not first.
assert_eq!(mode_for("git status ; exec >out", &cfg), MinimizerMode::None);
// `exec` wrapped by `command`/`builtin` (with flags or env assignments)
// mutates the same fds and must also block segmentation.
assert_eq!(mode_for("command exec >out ; echo hi", &cfg), MinimizerMode::None);
assert_eq!(mode_for("builtin exec >out ; echo hi", &cfg), MinimizerMode::None);
assert_eq!(mode_for("git diff ; command -p exec 2>err", &cfg), MinimizerMode::None);
assert_eq!(mode_for("FOO=\"a b\" exec >out ; echo hi", &cfg), MinimizerMode::None);
assert_eq!(mode_for("FOO=\"a b\" command exec >out ; echo hi", &cfg), MinimizerMode::None);
// Alias mutations affect later words when segments are parsed in separate
// calls, so they must stay on the original single-parse path too.
assert_eq!(mode_for("alias cat='printf hacked' ; cat file", &cfg), MinimizerMode::None);
assert_eq!(mode_for("unalias cat ; cat file", &cfg), MinimizerMode::None);
// A real command merely named with `exec` as an argument is not the
// builtin and must NOT block segmentation.
assert_eq!(mode_for("echo exec ; printf done", &cfg), MinimizerMode::SegmentedChain);
// Such chains pass through untouched.
let out = apply("exec >out ; echo hi", "hi\n", 0, &cfg);
assert_eq!(out.text, "hi\n");
assert!(!out.changed);
}
}
#[cfg(test)]
crates/pi-shell/src/minimizer/filters/binary_tools.rs
+121
View File
@@ -0,0 +1,121 @@
//! Binary-inspection tool filters (Tier 3b): `xxd`, `strings`, `od`.
//!
//! These tools all share the same failure mode in the minimizer's
//! `unknown` bucket: very long head-or-tail-or-elide outputs (5000+
//! lines) on multi-megabyte binaries dwarf the 64 KB capture budget and
//! waste the agent's context window on repetitive hex/string dumps. We
//! preserve the first 50 lines and last 20 lines and elide the middle
//! with a count marker — diagnostic anchors (magic bytes at the head,
//! footer/trailer bytes at the tail) survive intact while the bulk
//! middle is dropped.
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
const HEAD_LINES: usize = 50;
const TAIL_LINES: usize = 20;
pub fn supports(program: &str, _subcommand: Option<&str>) -> bool {
matches!(program, "xxd" | "strings" | "od")
}
pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
// Kill-switch parity (M2): legacy_filters_active=true skips this
// filter so callers can rollback without recompile.
if ctx.config.legacy_filters_active() {
return MinimizerOutput::passthrough(input);
}
let cleaned = primitives::strip_ansi(input);
let total_lines = cleaned.lines().count();
if total_lines <= HEAD_LINES + TAIL_LINES {
// Short dump — passthrough. Even errored runs are tiny enough
// here that the head/tail cap would not help.
let _ = exit_code;
return MinimizerOutput::passthrough(input);
}
let text = primitives::head_tail_lines(&cleaned, HEAD_LINES, TAIL_LINES);
if text == input {
MinimizerOutput::passthrough(input)
} else {
MinimizerOutput::transformed(text, input.len())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::minimizer::MinimizerConfig;
fn ctx<'a>(program: &'a str, command: &'a str, config: &'a MinimizerConfig) -> MinimizerCtx<'a> {
MinimizerCtx { program, subcommand: None, command, config }
}
fn build_lines(prefix: &str, count: usize) -> String {
let mut s = String::new();
for i in 0..count {
s.push_str(&format!("{prefix}{i:08x}\n"));
}
s
}
#[test]
fn xxd_long_dump_compacts_with_head_tail_marker() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = build_lines("00000000: ", 5000);
let context = ctx("xxd", "xxd /bin/ls", &cfg);
let out = filter(&context, &input, 0);
assert!(out.changed);
// 50 head + 20 tail + 1 marker = 71 lines
let line_count = out.text.lines().count();
assert_eq!(line_count, HEAD_LINES + TAIL_LINES + 1, "got {line_count} lines: {out:?}");
assert!(out.text.contains("lines omitted"));
// Head anchor preserved.
assert!(out.text.starts_with("00000000: 00000000"));
}
#[test]
fn xxd_short_dump_passthrough() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = build_lines("00000000: ", 60);
let context = ctx("xxd", "xxd small", &cfg);
let out = filter(&context, &input, 0);
assert!(!out.changed);
assert_eq!(out.text, input);
}
#[test]
fn strings_long_output_compacts() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = build_lines("symbol_", 2000);
let context = ctx("strings", "strings /bin/ls", &cfg);
let out = filter(&context, &input, 0);
assert!(out.changed);
assert!(out.text.contains("lines omitted"));
}
#[test]
fn od_long_output_compacts() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = build_lines("0000000 ", 1500);
let context = ctx("od", "od -c /bin/ls", &cfg);
let out = filter(&context, &input, 0);
assert!(out.changed);
assert!(out.text.contains("lines omitted"));
}
#[test]
fn binary_tools_legacy_filters_active_passes_through() {
// Kill-switch parity (M2).
let mut cfg = MinimizerConfig::default();
cfg.enabled = true;
cfg.legacy_filters_active = true;
let input = build_lines("00000000: ", 5000);
for prog in ["xxd", "strings", "od"] {
let context = ctx(prog, "binary-tool", &cfg);
let out = filter(&context, &input, 0);
assert!(!out.changed, "{prog} should passthrough with kill-switch");
assert_eq!(out.text, input);
}
}
}
crates/pi-shell/src/minimizer/filters/bun.rs
+500 -14
View File
@@ -5,7 +5,7 @@ use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
const BUN_PACKAGE_SUBCOMMANDS: &[&str] = &[
"install", "i", "add", "update", "up", "upgrade", "remove", "rm", "outdated", "pm", "audit",
"run", "exec",
"run", "exec", "check",
];
const BUN_TEST_SUBCOMMANDS: &[&str] = &["test"];
const BUN_BUILD_SUBCOMMANDS: &[&str] = &["build"];
@@ -36,6 +36,9 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
{
return pkg::filter(ctx, input, exit_code);
}
if is_check_invocation(ctx.program, subcommand, ctx.command) {
return filter_bun_check(ctx, input, exit_code);
}
if is_test_invocation(ctx.program, subcommand, ctx.command) {
return node_tests::filter(ctx, input, exit_code);
}
@@ -49,6 +52,7 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
return js_tools::filter(ctx, input, exit_code);
}
match (ctx.program, subcommand) {
("bun", Some("check")) => filter_bun_check(ctx, input, exit_code),
("bun", Some(subcommand)) if BUN_PACKAGE_SUBCOMMANDS.contains(&subcommand) => {
pkg::filter(ctx, input, exit_code)
},
@@ -58,7 +62,7 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
}
fn is_non_exec_package_subcommand(subcommand: &str) -> bool {
BUN_PACKAGE_SUBCOMMANDS.contains(&subcommand) && !matches!(subcommand, "run" | "exec")
BUN_PACKAGE_SUBCOMMANDS.contains(&subcommand) && !matches!(subcommand, "run" | "exec" | "check")
}
fn is_test_invocation(program: &str, subcommand: Option<&str>, command: &str) -> bool {
@@ -66,33 +70,227 @@ fn is_test_invocation(program: &str, subcommand: Option<&str>, command: &str) ->
(program, subcommand),
("bun", Some("test")) | ("bunx", Some("jest" | "vitest" | "playwright"))
) || is_exec_package_subcommand(program, subcommand)
&& command_contains_tool(command, &["jest", "vitest", "playwright"])
&& command_invoked_word(command).is_some_and(|token| {
["jest", "vitest", "playwright"].contains(&token) || is_test_script_token(token)
})
}
fn command_invoked_word(command: &str) -> Option<&str> {
let mut after_marker = false;
let mut skip_option_value = false;
for raw in command.split(|ch: char| ch.is_whitespace() || matches!(ch, ';' | '|' | '&')) {
let token = trim_command_token(raw);
if token.is_empty() {
continue;
}
if !after_marker {
if matches!(token, "run" | "exec") {
after_marker = true;
}
continue;
}
if skip_option_value {
skip_option_value = false;
continue;
}
if token.starts_with('-') {
if bun_wrapper_option_takes_value(token) && !token.contains('=') {
skip_option_value = true;
}
continue;
}
return Some(token);
}
None
}
fn trim_command_token(token: &str) -> &str {
token.trim_matches(|ch| matches!(ch, '\'' | '"' | '`'))
}
fn bun_wrapper_option_takes_value(token: &str) -> bool {
matches!(token, "--filter" | "--cwd" | "--env-file" | "--preload" | "-F" | "-C" | "-r")
}
fn is_test_script_token(token: &str) -> bool {
let token = trim_command_token(token);
matches!(token, "test" | "t" | "e2e" | "spec") || token.starts_with("test:")
}
fn is_exec_package_subcommand(program: &str, subcommand: Option<&str>) -> bool {
matches!((program, subcommand), ("bun", Some("run" | "exec")))
}
fn is_check_invocation(program: &str, subcommand: Option<&str>, command: &str) -> bool {
is_exec_package_subcommand(program, subcommand)
&& command_invoked_word(command).is_some_and(is_check_script_token)
}
fn is_check_script_token(token: &str) -> bool {
let token = trim_command_token(token);
matches!(token, "check") || token.starts_with("check:")
}
fn is_lint_script_token(token: &str) -> bool {
let token = trim_command_token(token);
matches!(token, "lint" | "typecheck" | "type-check")
|| token.starts_with("lint:")
|| token.starts_with("typecheck:")
|| token.starts_with("type-check:")
}
fn is_lint_invocation(program: &str, subcommand: Option<&str>, command: &str) -> bool {
matches!((program, subcommand), ("bun" | "bunx", Some("tsc" | "eslint" | "biome")))
|| is_exec_package_subcommand(program, subcommand)
&& command_contains_tool(command, &["tsc", "eslint", "biome"])
&& command_invoked_word(command).is_some_and(|token| {
["tsc", "eslint", "biome"].contains(&token) || is_lint_script_token(token)
})
}
fn is_js_tool_invocation(program: &str, subcommand: Option<&str>, command: &str) -> bool {
matches!((program, subcommand), ("bun" | "bunx", Some("next" | "prettier" | "prisma")))
|| is_exec_package_subcommand(program, subcommand)
&& command_contains_tool(command, &["next", "prettier", "prisma"])
}
fn is_cpp_invocation(program: &str, subcommand: Option<&str>, command: &str) -> bool {
matches!((program, subcommand), ("bunx", Some(subcommand)) if BUN_CPP_TOOL_SUBCOMMANDS.contains(&subcommand))
|| is_exec_package_subcommand(program, subcommand) && cpp::supports_invocation(command)
&& command_invoked_word(command)
.is_some_and(|token| ["next", "prettier", "prisma"].contains(&token))
}
fn command_contains_tool(command: &str, tools: &[&str]) -> bool {
command
.split(|ch: char| ch.is_whitespace() || matches!(ch, ';' | '|' | '&'))
.any(|token| tools.contains(&token))
fn is_cpp_invocation(program: &str, subcommand: Option<&str>, command: &str) -> bool {
matches!((program, subcommand), ("bunx", Some(subcommand)) if BUN_CPP_TOOL_SUBCOMMANDS.contains(&subcommand))
|| is_exec_package_subcommand(program, subcommand)
&& command_invoked_word(command).is_some_and(|token| {
BUN_CPP_TOOL_SUBCOMMANDS.contains(&token) || cpp::supports_invocation(token)
})
}
fn filter_bun_check(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
let cleaned = primitives::strip_ansi(input);
let text = compact_bun_check_output(ctx, &cleaned, exit_code)
.unwrap_or_else(|| lint::condense_lint_output(ctx.program, &cleaned, exit_code));
if text == input {
MinimizerOutput::passthrough(input)
} else {
MinimizerOutput::transformed(text, input.len())
}
}
fn compact_bun_check_output(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> Option<String> {
let mut root_checked = false;
let mut packages: Vec<&str> = Vec::new();
let mut diagnostics: Vec<&str> = Vec::new();
let mut nonzero_exits: Vec<&str> = Vec::new();
let mut timeout: Option<&str> = None;
for line in input.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let lower = trimmed.to_ascii_lowercase();
if lower.contains("timeout") || lower.contains("timed out") {
timeout = Some(trimmed);
continue;
}
if trimmed.starts_with("$ ") || lower.contains(" check: $ ") {
continue;
}
if let Some(package) = parse_checked_package(trimmed) {
if !packages.contains(&package) {
packages.push(package);
}
continue;
}
if lower.starts_with("checked ") && lower.contains("no fixes applied") {
root_checked = true;
continue;
}
if let Some(code) = parse_exited_code(trimmed) {
if code != "0" {
nonzero_exits.push(trimmed);
}
continue;
}
if is_bun_check_noise(trimmed, &lower) {
continue;
}
if exit_code != 0 && is_important(trimmed) {
diagnostics.push(trimmed);
}
}
if !root_checked && packages.is_empty() && diagnostics.is_empty() && nonzero_exits.is_empty() {
return None;
}
let mut out = String::new();
out.push_str(command_summary(ctx.command));
out.push_str(": ");
if !nonzero_exits.is_empty() || !diagnostics.is_empty() {
out.push_str("failed\n");
} else if timeout.is_some() {
out.push_str("visible checks passed; wrapper timed out\n");
} else if exit_code == 0 {
out.push_str("passed\n");
} else {
out.push_str("incomplete\n");
}
if root_checked {
out.push_str("root biome: ok\n");
}
if !packages.is_empty() {
out.push_str("packages checked: ");
out.push_str(&packages.join(", "));
out.push('\n');
}
if let Some(timeout) = timeout {
out.push_str("timeout: ");
out.push_str(trim_notice_brackets(timeout));
out.push('\n');
}
for line in nonzero_exits.iter().chain(diagnostics.iter()).take(40) {
out.push_str(line);
out.push('\n');
}
let omitted = nonzero_exits.len() + diagnostics.len();
if omitted > 40 {
out.push_str("… ");
out.push_str(&(omitted - 40).to_string());
out.push_str(" diagnostic lines omitted\n");
}
Some(out)
}
fn command_summary(command: &str) -> &str {
let mut parts = command.split_whitespace();
match (parts.next(), parts.next(), parts.next()) {
(Some("bun"), Some("run"), Some(script)) => {
script.trim_matches(|ch| matches!(ch, '\'' | '"' | '`'))
},
_ => "bun check",
}
}
fn parse_checked_package(line: &str) -> Option<&str> {
let (package, rest) = line.split_once(" check: Checked ")?;
if rest.contains("No fixes applied") {
Some(package)
} else {
None
}
}
fn parse_exited_code(line: &str) -> Option<&str> {
let (_, code) = line.rsplit_once("Exited with code ")?;
Some(code.trim())
}
fn is_bun_check_noise(line: &str, lower: &str) -> bool {
line.starts_with("$ ")
|| lower.contains(" check: $ ")
|| lower.starts_with("checked ")
|| lower.ends_with("no fixes applied.")
}
fn trim_notice_brackets(line: &str) -> &str {
line.trim_matches(|ch| matches!(ch, '[' | ']' | '⟦' | '⟧'))
}
fn filter_bun_build(input: &str, exit_code: i32) -> MinimizerOutput {
@@ -155,7 +353,8 @@ mod tests {
#[test]
fn supports_bun_package_test_and_tool_subcommands() {
for subcommand in ["install", "add", "run", "test", "build", "tsc", "next", "ctest"] {
for subcommand in ["install", "add", "run", "test", "build", "tsc", "next", "ctest", "check"]
{
assert!(supports("bun", Some(subcommand)), "{subcommand} should be supported");
}
assert!(supports("bunx", Some("vitest")));
@@ -163,6 +362,22 @@ mod tests {
assert!(!supports("bun", Some("unknown")));
}
#[test]
fn bun_check_direct_subcommand_is_supported_and_routes_to_check_filter() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
// supports() must admit "check" as a subcommand
assert!(supports("bun", Some("check")), "bun check should be supported");
// filter() must route directly to filter_bun_check
let ctx = ctx("bun", Some("check"), "bun check", &cfg);
let biome_output = "packages/coding-agent/src/foo.ts:1:1 lint/suspicious/noExplicitAny \
━━━━━━━━━\n\n ✖ Unexpected any.\n\nChecked 127 files in 234ms. 1 error \
found.\n";
let out = filter(&ctx, biome_output, 1);
assert!(out.changed, "bun check output should be changed/compressed");
// should not route to pkg::filter (which would strip the error details)
assert!(out.text.contains("error"), "check filter must preserve error output");
}
#[test]
fn bun_install_uses_package_noise_filter() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
@@ -246,4 +461,275 @@ mod tests {
assert!(!out.text.contains("Bundled 12 modules"));
assert!(out.text.contains("error: missing export"));
}
#[test]
fn bun_run_test_routes_to_node_tests() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run test", &cfg);
let out = filter(&ctx, "✓ pass 1\n✓ pass 2\nFAIL app.test.ts\nTests 1 failed, 2 passed\n", 1);
assert!(!out.text.contains("✓ pass 1"));
assert!(out.text.contains("FAIL app.test.ts"));
assert!(out.text.contains("Tests 1 failed, 2 passed"));
}
#[test]
fn bun_run_lint_and_typecheck_route_to_lint_filter() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let input = concat!(
"src/app.ts:1:1: error TS2322: Type 'string' is not assignable to type 'number'.\n",
"src/app.ts:2:1: error TS7006: Parameter 'x' implicitly has an 'any' type.\n",
);
for command in
["bun run lint", "bun run lint:ci", "bun run typecheck", "bun run typecheck:ci"]
{
let ctx = ctx("bun", Some("run"), command, &cfg);
let routed = filter(&ctx, input, 1).text;
let expected = lint::filter(&ctx, input, 1).text;
assert_eq!(routed, expected, "{command} should use lint filter");
assert!(
routed.contains("2 diagnostics in 1 files"),
"{command} should condense lint output"
);
}
}
#[test]
fn quoted_bun_run_test_routes_to_node_tests() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run 'test'", &cfg);
let out = filter(&ctx, "✓ pass 1\nFAIL app.test.ts\nTests 1 failed, 1 passed\n", 1);
assert!(!out.text.contains("✓ pass 1"));
assert!(out.text.contains("FAIL app.test.ts"));
}
#[test]
fn bun_run_test_colon_routes_to_node_tests() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run test:unit", &cfg);
let out = filter(&ctx, "✓ passes\nFAIL src/example.test.ts\nTests 1 failed, 1 passed\n", 1);
assert!(!out.text.contains("✓ passes"));
assert!(out.text.contains("FAIL src/example.test.ts"));
}
#[test]
fn bun_run_e2e_routes_to_node_tests() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run e2e", &cfg);
let out = filter(&ctx, "✓ passes\nFAIL e2e/spec.ts\nTests 1 failed, 1 passed\n", 1);
assert!(!out.text.contains("✓ passes"));
assert!(out.text.contains("FAIL e2e/spec.ts"));
}
#[test]
fn bun_run_check_colon_compacts_workspace_success_noise() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run 'check:ts'", &cfg);
let out = filter(
&ctx,
"$ bun run check:tools && bun run --workspaces --if-present check\n$ biome check . \
--no-errors-on-unmatched\nChecked 1690 files in 371ms. No fixes \
applied.\n@oh-my-pi/pi-utils check: Checked 40 files in 11ms. No fixes \
applied.\n@oh-my-pi/pi-utils check: $ tsgo -p tsconfig.json \
--noEmit\n@oh-my-pi/pi-utils check: Exited with code 0\n@oh-my-pi/pi-coding-agent \
check: Checked 1178 files in 287ms. No fixes applied.\n@oh-my-pi/pi-coding-agent check: \
$ tsgo -p tsconfig.json --noEmit\n@oh-my-pi/pi-coding-agent check: Exited with code 0\n",
0,
);
assert!(out.text.contains("check:ts: passed"));
assert!(out.text.contains("root biome: ok"));
assert!(out.text.contains("@oh-my-pi/pi-utils"));
assert!(out.text.contains("@oh-my-pi/pi-coding-agent"));
assert!(!out.text.contains("No fixes applied"));
assert!(!out.text.contains("tsgo -p"));
assert!(!out.text.contains("Exited with code 0"));
}
#[test]
fn bun_run_check_timeout_preserves_ambiguous_success() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run check:ts", &cfg);
let out = filter(
&ctx,
"@oh-my-pi/pi-utils check: Checked 40 files in 11ms. No fixes \
applied.\n@oh-my-pi/pi-utils check: Exited with code 0\n[Command timed out after 300 \
seconds]\n",
1,
);
assert!(
out.text
.contains("visible checks passed; wrapper timed out")
);
assert!(
out.text
.contains("timeout: Command timed out after 300 seconds")
);
assert!(!out.text.contains("failed"));
}
#[test]
fn bun_run_build_still_uses_pkg_filter() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run build", &cfg);
let out = filter(&ctx, "Resolving dependencies\nDownloaded foo\nerror: failed\n", 1);
assert!(!out.text.contains("Resolving dependencies"));
assert!(out.text.contains("error: failed"));
}
#[test]
fn bun_run_build_argument_named_test_stays_on_package_filter() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("run"), "bun run build -- test", &cfg);
let out = filter(&ctx, "PASS emitted by build\n✓ emitted by build\nerror: failed\n", 1);
assert!(out.text.contains("PASS emitted by build"));
assert!(out.text.contains("✓ emitted by build"));
assert!(out.text.contains("error: failed"));
}
// --- bun test failure — failure lines and summary survive ---
#[test]
fn bun_test_failure_keeps_fail_file_and_summary() {
// `bun test` failure: FAIL lines, error text, and the totals line
// must survive. Passing checkmarks must be stripped.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let bun_ctx = ctx("bun", Some("test"), "bun test", &cfg);
let input = concat!(
"✓ auth.test.ts > login passes (12ms)\n",
"✓ auth.test.ts > logout ok (8ms)\n",
"FAIL auth.test.ts\n",
"● register fails when email taken\n",
" Error: expected status 409, got 200\n",
" at auth.test.ts:88:5\n",
"Tests 1 failed, 2 passed (33ms)\n",
);
let out = filter(&bun_ctx, input, 1);
assert!(
!out.text.contains("✓ auth.test.ts > login"),
"passing lines must be stripped: {:?}",
out.text
);
assert!(out.text.contains("FAIL auth.test.ts"), "FAIL line must survive: {:?}", out.text);
assert!(
out.text.contains("Error: expected status 409"),
"error body must survive: {:?}",
out.text
);
assert!(out.text.contains("Tests 1 failed"), "summary line must survive: {:?}", out.text);
assert!(out.text.contains("2 passed"), "passed count must survive: {:?}", out.text);
}
#[test]
fn bun_test_success_strips_all_pass_lines() {
// On success all ✓ lines are noise — the agent only needs the summary.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let bun_ctx = ctx("bun", Some("test"), "bun test", &cfg);
let input = concat!(
"✓ foo.test.ts > passes (5ms)\n",
"✓ bar.test.ts > also passes (3ms)\n",
"Tests 2 passed (8ms)\n",
);
let out = filter(&bun_ctx, input, 0);
assert!(
!out.text.contains("✓ foo.test.ts"),
"passing lines must be stripped: {:?}",
out.text
);
assert!(
!out.text.contains("✓ bar.test.ts"),
"passing lines must be stripped: {:?}",
out.text
);
// Summary or some indication of passing must survive.
assert!(
out.text.contains("passed") || !out.changed,
"summary must survive or output unchanged"
);
}
// --- bun check failure — diagnostic lines survive, noise stripped ---
#[test]
fn bun_check_failure_keeps_diagnostic_and_emits_failed_status() {
// `bun check` (routed via `bun run check:ts`) with real type errors
// must surface the diagnostic lines and emit a `failed` verdict.
// Package-manager download noise and `Exited with code 0` lines
// must not appear.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let bun_ctx = ctx("bun", Some("run"), "bun run check:ts", &cfg);
let input = concat!(
"$ bun run --workspaces check\n",
"@oh-my-pi/pi-utils check: $ tsgo -p tsconfig.json --noEmit\n",
"@oh-my-pi/pi-utils check: Exited with code 0\n",
"@oh-my-pi/pi-coding-agent check: $ tsgo -p tsconfig.json --noEmit\n",
"src/tools/bash.ts(42,7): error TS2322: Type 'string' is not assignable to type \
'number'.\n",
"@oh-my-pi/pi-coding-agent check: Exited with code 1\n",
);
let out = filter(&bun_ctx, input, 1);
assert!(out.text.contains("failed"), "failed verdict must appear: {:?}", out.text);
assert!(out.text.contains("error TS2322"), "diagnostic must survive: {:?}", out.text);
assert!(
!out.text.contains("tsgo -p"),
"internal command lines must be stripped: {:?}",
out.text
);
// Nonzero exit lines are preserved as evidence (code 0 exits are stripped).
assert!(
out.text.contains("Exited with code 1"),
"nonzero exit line must survive as evidence: {:?}",
out.text
);
assert!(
!out.text.contains("Exited with code 0"),
"zero exit noise must be stripped: {:?}",
out.text
);
}
#[test]
fn bun_check_success_emits_passed_status_and_no_noise() {
// Clean `bun run check:ts` (all packages exit 0) must compact to a
// single `passed` summary line without biome/tsgo details.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let bun_ctx = ctx("bun", Some("run"), "bun run check:ts", &cfg);
let input = concat!(
"$ bun run --workspaces check\n",
"Checked 1690 files in 371ms. No fixes applied.\n",
"@oh-my-pi/pi-utils check: Checked 40 files in 11ms. No fixes applied.\n",
"@oh-my-pi/pi-utils check: $ tsgo -p tsconfig.json --noEmit\n",
"@oh-my-pi/pi-utils check: Exited with code 0\n",
"@oh-my-pi/pi-coding-agent check: Checked 1178 files in 287ms. No fixes applied.\n",
"@oh-my-pi/pi-coding-agent check: $ tsgo -p tsconfig.json --noEmit\n",
"@oh-my-pi/pi-coding-agent check: Exited with code 0\n",
);
let out = filter(&bun_ctx, input, 0);
assert!(out.changed, "clean check must be compacted");
assert!(out.text.contains("passed"), "passed verdict must appear: {:?}", out.text);
assert!(
!out.text.contains("No fixes applied"),
"biome noise must be stripped: {:?}",
out.text
);
assert!(
!out.text.contains("tsgo -p"),
"internal command lines must be stripped: {:?}",
out.text
);
assert!(
!out.text.contains("Exited with code"),
"exit noise must be stripped: {:?}",
out.text
);
}
}
crates/pi-shell/src/minimizer/filters/cargo.rs
+461 -15
View File
@@ -1,5 +1,7 @@
//! Cargo build/test output filters.
use std::{collections::BTreeMap, fmt::Write as _};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(subcommand: Option<&str>) -> bool {
@@ -26,9 +28,11 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
Some("metadata") => input.to_string(),
Some("test" | "bench") => failures_only(&cleaned, exit_code),
Some("nextest") => filter_nextest(&cleaned),
Some("build" | "check" | "clippy" | "doc" | "run") => condense_build(&cleaned),
Some("clippy") => filter_clippy(&cleaned, exit_code),
Some("build" | "check" | "doc" | "run") => condense_build(&cleaned),
Some("fmt") => condense_fmt(&cleaned),
Some("tree" | "update" | "install" | "publish") => compact_general(&cleaned),
Some("install") => filter_install(&cleaned, exit_code),
Some("tree" | "update" | "publish") => compact_general(&cleaned),
_ => cleaned,
};
if text == input {
@@ -289,6 +293,182 @@ fn is_general_cargo_noise(line: &str) -> bool {
|| trimmed.starts_with("Checking ")
|| trimmed.starts_with("Fresh ")
}
/// Filter `cargo install` output: strip compilation/download noise, keep
/// install/error summaries.
fn filter_install(input: &str, exit_code: i32) -> String {
let stripped = primitives::strip_lines(input, &[is_compiling_noise]);
if exit_code != 0 {
return primitives::head_tail_lines(&stripped, 100, 40);
}
let mut summaries = String::new();
for line in stripped.lines() {
let trimmed = line.trim_start();
if is_install_summary(trimmed) || trimmed.starts_with("WARNING:") {
summaries.push_str(line);
summaries.push('\n');
}
}
if summaries.is_empty() {
let deduped = primitives::dedup_consecutive_lines(&stripped);
primitives::head_tail_lines(&deduped, 60, 20)
} else {
primitives::dedup_consecutive_lines(&summaries)
}
}
fn is_install_summary(line: &str) -> bool {
line.starts_with("Installed ")
|| line.starts_with("Replaced ")
|| line.starts_with("Replacing ")
|| line.starts_with("Ignored ")
}
#[derive(Debug)]
struct ClippyWarning {
location: String,
message: String,
lint_rule: Option<String>,
}
/// Filter `cargo clippy`: group warnings by lint rule; keep errors verbatim.
fn filter_clippy(input: &str, exit_code: i32) -> String {
let no_noise = primitives::strip_lines(input, &[is_compiling_noise]);
let has_compile_error = no_noise.lines().any(|l| {
let t = l.trim_start();
(t.starts_with("error:")
&& !t.starts_with("error: could not compile")
&& !t.starts_with("error: aborting"))
|| t.starts_with("error[")
});
if has_compile_error {
let grouped = primitives::group_by_file(&no_noise, 20);
return primitives::head_tail_lines(&grouped, 120, 60);
}
let warnings = parse_clippy_warnings(&no_noise);
if warnings.is_empty() {
let deduped = primitives::dedup_consecutive_lines(&no_noise);
return primitives::head_tail_lines(&deduped, 80, 40);
}
format_clippy_grouped(&warnings, exit_code)
}
fn parse_clippy_warnings(input: &str) -> Vec<ClippyWarning> {
let mut warnings = Vec::new();
let lines: Vec<&str> = input.lines().collect();
let mut i = 0;
while i < lines.len() {
let trimmed = lines[i].trim();
if !trimmed.starts_with("warning: ") {
i += 1;
continue;
}
let msg = trimmed.strip_prefix("warning: ").unwrap_or("");
// Skip summary lines like "warning: `crate` (lib) generated N warning(s)"
if msg.contains(" generated ") && (msg.ends_with(" warnings") || msg.ends_with(" warning")) {
i += 1;
continue;
}
let message = msg.to_string();
let mut location = String::new();
let mut lint_rule = None;
i += 1;
while i < lines.len() {
let t = lines[i].trim();
if t.starts_with("--> ") {
location = t.strip_prefix("--> ").unwrap_or("").to_string();
}
if let Some(rule) = extract_lint_rule(t) {
lint_rule = Some(rule);
}
i += 1;
if i >= lines.len() {
break;
}
let next = lines[i].trim();
if next.starts_with("warning: ")
|| next.starts_with("error:")
|| next.starts_with("error[")
{
break;
}
}
if !message.is_empty() {
warnings.push(ClippyWarning { location, message, lint_rule });
}
}
warnings
}
fn extract_lint_rule(line: &str) -> Option<String> {
let line = line.trim();
if !line.starts_with("= note:") {
return None;
}
let after_note = line.strip_prefix("= note:")?.trim();
let rest = after_note
.strip_prefix("`#[warn(")
.or_else(|| after_note.strip_prefix("`#[deny("))
.or_else(|| after_note.strip_prefix("`#[allow("))?;
Some(rest.split(")]`").next()?.to_string())
}
fn format_clippy_grouped(warnings: &[ClippyWarning], exit_code: i32) -> String {
let mut groups: BTreeMap<String, Vec<&ClippyWarning>> = BTreeMap::new();
let mut ungrouped = Vec::new();
for w in warnings {
if let Some(ref rule) = w.lint_rule {
groups.entry(rule.clone()).or_default().push(w);
} else {
ungrouped.push(w);
}
}
let mut out = String::new();
for (rule, warns) in &groups {
if warns.len() == 1 {
let loc = if warns[0].location.is_empty() {
String::new()
} else {
format!("{} ", warns[0].location)
};
let _ = writeln!(out, "clippy: {} — {}{}", rule, loc, warns[0].message);
} else {
let _ = writeln!(out, "clippy: {} ({} warnings)", rule, warns.len());
for w in warns {
let _ = writeln!(out, " {} {}", w.location, w.message);
}
}
}
for w in &ungrouped {
let _ = writeln!(out, "clippy warning: {} {}", w.location, w.message);
}
if exit_code != 0 {
out.push_str("(clippy found issues)\n");
}
if out.is_empty() {
"cargo clippy: ok\n".to_string()
} else {
out
}
}
#[cfg(test)]
mod tests {
@@ -337,21 +517,170 @@ mod tests {
assert!(out.contains("stdout text"));
assert!(out.contains("Summary [0.2s] 2 tests run: 1 passed, 1 failed"));
}
#[test]
fn install_strips_noise_keeps_summary() {
assert!(supports(Some("install")));
let input = concat!(
" Updating crates.io index\n",
" Downloaded foo v1.0.0\n",
" Compiling bar v0.1.0\n",
" Compiling tool v3.0.0\n",
" Finished release [optimized] target(s) in 45.2s\n",
" Installing /home/user/.cargo/bin/tool\n",
" Installed package `tool v3.0.0` (executable `tool`)\n",
);
let out = filter_install(input, 0);
assert!(!out.contains("Compiling"));
assert!(!out.contains("Downloaded"));
assert!(!out.contains("Updating"));
assert!(!out.contains("Finished"));
assert!(out.contains("Installed package `tool v3.0.0`"));
}
#[test]
fn install_uses_general_head_tail_dedup_strategy() {
assert!(supports(Some("install")));
let mut input = "Downloading crate\n".repeat(2);
input.push_str("Installed package `tool v1.0.0`\n");
for i in 0..130 {
input.push_str("line ");
input.push_str(&i.to_string());
input.push('\n');
}
let out = compact_general(&input);
assert!(!out.contains("Downloading crate"));
assert!(out.contains("Installed package `tool v1.0.0`"));
assert!(out.contains("lines omitted"));
fn install_already_installed() {
let input = concat!(
" Updating crates.io index\n",
" Ignored package `tool v1.0.0` is already installed, use --force to override\n",
);
let out = filter_install(input, 0);
assert!(!out.contains("Updating"));
assert!(out.contains("Ignored package `tool v1.0.0`"));
}
#[test]
fn install_error_preserves_context() {
let input = concat!(
" Updating crates.io index\n",
" Compiling foo v0.1.0\n",
"error[E0425]: cannot find value `x` in this scope\n",
" --> src/main.rs:5:9\n",
" |\n",
"5 | let y = x;\n",
" | ^ not found in this scope\n",
"error: could not compile `foo` due to 1 previous error\n",
);
let out = filter_install(input, 1);
assert!(!out.contains("Compiling"));
assert!(!out.contains("Updating"));
assert!(out.contains("error[E0425]"));
assert!(out.contains("cannot find value `x`"));
}
#[test]
fn clippy_groups_warnings_by_lint_rule() {
assert!(supports(Some("clippy")));
let input = concat!(
" Checking foo v0.1.0\n",
"warning: unused variable: `x`\n",
" --> src/lib.rs:2:9\n",
" |\n",
"2 | let x = 1;\n",
" | ^ help: if this is intentional, prefix with an underscore: `_x`\n",
" |\n",
" = note: `#[warn(unused_variables)]` on by default\n",
"\n",
"warning: unused variable: `y`\n",
" --> src/lib.rs:5:9\n",
" |\n",
"5 | let y = 2;\n",
" | ^ help: if this is intentional, prefix with an underscore: `_y`\n",
" |\n",
" = note: `#[warn(unused_variables)]` on by default\n",
"\n",
"warning: `foo` (lib) generated 2 warnings\n",
);
let out = filter_clippy(input, 0);
assert!(!out.contains("Checking"));
assert!(!out.contains("generated"));
assert!(out.contains("unused_variables"));
assert!(out.contains("2 warnings"));
assert!(out.contains("src/lib.rs:2:9"));
assert!(out.contains("src/lib.rs:5:9"));
}
#[test]
fn clippy_single_warning_compact() {
let input = concat!(
"warning: redundant clone\n",
" --> src/main.rs:10:3\n",
" |\n",
"10| foo.clone()\n",
" | ^^^^^^^^^^^^ help: remove this\n",
" |\n",
" = note: `#[warn(clippy::redundant_clone)]` on by default\n",
"\n",
"warning: `foo` (bin \"foo\") generated 1 warning\n",
);
let out = filter_clippy(input, 0);
assert!(!out.contains("generated"));
assert!(out.contains("clippy::redundant_clone"));
assert!(out.contains("src/main.rs:10:3"));
assert!(out.contains("redundant clone"));
}
#[test]
fn clippy_multiple_rules_grouped_separately() {
let input = concat!(
"warning: unused variable: `x`\n",
" --> src/lib.rs:2:9\n",
" |\n",
"2 | let x = 1;\n",
" | ^\n",
" |\n",
" = note: `#[warn(unused_variables)]` on by default\n",
"\n",
"warning: redundant clone\n",
" --> src/main.rs:10:3\n",
" |\n",
"10| foo.clone()\n",
" | ^^^^^^^^^^^^ help: remove this\n",
" |\n",
" = note: `#[warn(clippy::redundant_clone)]` on by default\n",
"\n",
"warning: `foo` (lib) generated 2 warnings\n",
);
let out = filter_clippy(input, 0);
assert!(out.contains("unused_variables"));
assert!(out.contains("clippy::redundant_clone"));
// Two separate groups, not merged
let unused_pos = out.find("unused_variables").unwrap();
let clone_pos = out.find("clippy::redundant_clone").unwrap();
assert!(unused_pos != clone_pos);
}
#[test]
fn clippy_compile_error_falls_back_to_build_style() {
let input = concat!(
" Compiling foo v0.1.0\n",
"error[E0425]: cannot find value `x` in this scope\n",
" --> src/lib.rs:5:9\n",
" |\n",
"5 | let y = x;\n",
" | ^ not found in this scope\n",
"error: could not compile `foo` due to 1 previous error\n",
);
let out = filter_clippy(input, 1);
assert!(!out.contains("Compiling"));
assert!(out.contains("error[E0425]"));
assert!(out.contains("cannot find value `x`"));
// Should NOT have clippy: prefix since it fell back to build style
assert!(!out.contains("clippy:"));
}
#[test]
fn clippy_exit_code_signals_issues() {
let input = concat!(
"warning: unused variable: `x`\n",
" --> src/lib.rs:2:9\n",
" |\n",
"2 | let x = 1;\n",
" | ^\n",
" |\n",
" = note: `#[deny(unused_variables)]` on by default\n",
);
let out = filter_clippy(input, 1);
assert!(out.contains("(clippy found issues)"));
}
#[test]
@@ -368,4 +697,121 @@ mod tests {
assert_eq!(out.text, input);
assert!(!out.changed);
}
// --- cargo test failure — failure block and panic line survive ---
#[test]
fn cargo_test_failure_keeps_thread_panic_and_failures_block() {
// `cargo test` with exit 101 must surface the thread panic message,
// the `failures:` block listing the failing test names, and the
// `test result: FAILED` summary line. Passing test lines and
// `Compiling` noise must not appear.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = MinimizerCtx {
program: "cargo",
subcommand: Some("test"),
command: "cargo test",
config: &cfg,
};
let input = concat!(
" Compiling pi-shell v0.1.0\n",
"running 3 tests\n",
"test ok_one ... ok\n",
"test ok_two ... ok\n",
"test bad_parse ... FAILED\n",
"\n",
"---- bad_parse stdout ----\n",
"thread 'bad_parse' panicked at 'assertion failed: result.is_ok()', src/lib.rs:42:5\n",
"note: run with RUST_BACKTRACE=1 for a backtrace.\n",
"\n",
"failures:\n",
" bad_parse\n",
"\n",
"test result: FAILED. 2 passed; 1 failed; 0 ignored; 0 measured\n",
);
let out = filter(&ctx, input, 101);
// Failure evidence must survive.
assert!(
out.text.contains("thread 'bad_parse' panicked"),
"panic line must survive: {:?}",
out.text
);
assert!(out.text.contains("failures:\n"), "failures block must survive: {:?}", out.text);
assert!(out.text.contains("bad_parse"), "failing test name must survive: {:?}", out.text);
assert!(out.text.contains("test result: FAILED"), "result line must survive: {:?}", out.text);
// Noise must be stripped.
assert!(!out.text.contains("Compiling"), "Compiling noise must be stripped");
assert!(!out.text.contains("test ok_one"), "passing test lines must be stripped");
assert!(!out.text.contains("test ok_two"), "passing test lines must be stripped");
}
#[test]
fn cargo_test_success_via_filter_produces_one_line_summary() {
// The token-savings contract: a full passing run must collapse to a
// single `cargo test: N passed (M suite[s])` line through filter(),
// not through the helper directly.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = MinimizerCtx {
program: "cargo",
subcommand: Some("test"),
command: "cargo test --workspace",
config: &cfg,
};
let input = concat!(
" Compiling pi-shell v0.1.0\n",
"running 42 tests\n",
"test a ... ok\n",
"test b ... ok\n",
"test result: ok. 42 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n",
"running 18 tests\n",
"test c ... ok\n",
"test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n",
"warning: `pi-shell` (test \"integration\") generated 2 warnings\n",
);
let out = filter(&ctx, input, 0);
assert!(out.changed, "successful run must be compacted");
// One-line summary: total passed, suite count, warnings.
assert!(out.text.contains("60 passed"), "total across suites must be summed: {:?}", out.text);
assert!(out.text.contains("2 suites"), "suite count must appear: {:?}", out.text);
assert!(out.text.contains("2 warnings"), "warning count must appear: {:?}", out.text);
// No per-test lines.
assert!(!out.text.contains("test a"), "individual test lines must be stripped");
assert!(!out.text.contains("Compiling"), "Compiling noise must be stripped");
}
#[test]
fn cargo_test_failure_exit_code_non_zero_is_not_summarized() {
// A run that reports `test result: ok` but then exits non-zero
// (e.g. a post-test hook failing) must not be falsely summarized
// as a clean pass — failures_only should fall through to condense_build
// rather than fabricating a `cargo test: N passed` line.
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = MinimizerCtx {
program: "cargo",
subcommand: Some("test"),
command: "cargo test",
config: &cfg,
};
// The test suite itself says ok, but a subsequent build step failed.
let input = concat!(
"running 1 tests\n",
"test it_works ... ok\n",
"test result: ok. 1 passed; 0 failed\n",
"error: could not compile `pi-shell` due to 1 previous error\n",
);
let out = filter(&ctx, input, 1);
// Must not emit a clean "cargo test: N passed" summary because exit was
// non-zero.
assert!(
!out.text.starts_with("cargo test:"),
"must not fabricate a pass summary on non-zero exit: {:?}",
out.text
);
}
}
crates/pi-shell/src/minimizer/filters/cloud.rs
+1191 -6
View File
File diff suppressed because it is too large Load Diff
crates/pi-shell/src/minimizer/filters/docker.rs
+1003 -13
View File
File diff suppressed because it is too large Load Diff
crates/pi-shell/src/minimizer/filters/generic.rs
+2 -2
View File
@@ -5,8 +5,8 @@ use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn filter(_ctx: &MinimizerCtx<'_>, input: &str, _exit_code: i32) -> MinimizerOutput {
let stripped = primitives::strip_ansi(input);
let deduped = primitives::dedup_consecutive_lines(&stripped);
let text = if deduped.lines().count() > 200 {
primitives::head_tail_lines(&deduped, 100, 60)
let text = if deduped.lines().count() > primitives::CapClass::Errors.lines() {
primitives::head_tail_cap(&deduped, primitives::CapClass::Errors)
} else {
deduped
};
crates/pi-shell/src/minimizer/filters/git.rs
+2031 -41
View File
File diff suppressed because it is too large Load Diff
crates/pi-shell/src/minimizer/filters/js_tools.rs
+3 -3
View File
@@ -40,7 +40,7 @@ pub fn effective_tool<'a>(program: &'a str, subcommand: Option<&'a str>) -> Opti
{
return Some(tool);
}
if is_npx_like(program) {
if is_npx_like(program, subcommand) {
let tool = subcommand?;
if NPX_ROUTABLE_TOOLS.contains(&tool) {
return Some(tool);
@@ -62,8 +62,8 @@ fn effective_tool_from_command<'a>(
.find(|token| SUPPORTED_TOOLS.contains(token))
}
fn is_npx_like(program: &str) -> bool {
matches!(program, "npx" | "bunx" | "pnpm dlx")
fn is_npx_like(program: &str, subcommand: Option<&str>) -> bool {
matches!(program, "npx" | "bunx") || matches!((program, subcommand), ("pnpm", Some("dlx")))
}
fn filter_next(input: &str, exit_code: i32) -> String {
crates/pi-shell/src/minimizer/filters/lint.rs
+61 -1
View File
@@ -9,7 +9,7 @@ pub fn supports(subcommand: Option<&str>) -> bool {
}
pub fn supports_program(program: &str, subcommand: Option<&str>) -> bool {
matches!(program, "ruff" | "mypy" | "rubocop")
matches!(program, "ruff" | "mypy" | "rubocop" | "pyright" | "basedpyright")
|| matches!(
subcommand,
None | Some("check" | "lint" | "run" | "format" | "fmt" | "typecheck")
@@ -17,6 +17,10 @@ pub fn supports_program(program: &str, subcommand: Option<&str>) -> bool {
}
pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
if preserves_machine_readable_output(ctx) {
return MinimizerOutput::passthrough(input);
}
let text = condense_lint_output(ctx.program, input, exit_code);
if text == input {
MinimizerOutput::passthrough(input)
@@ -45,6 +49,14 @@ fn strip_lint_noise(program: &str, input: &str, exit_code: i32) -> String {
out
}
fn preserves_machine_readable_output(ctx: &MinimizerCtx<'_>) -> bool {
matches!(ctx.program, "pyright" | "basedpyright")
&& ctx
.command
.split_whitespace()
.any(|part| part == "--outputjson" || part.starts_with("--outputjson="))
}
fn is_lint_noise(program: &str, line: &str, exit_code: i32) -> bool {
if exit_code != 0 && contains_diagnostic_signal(line) {
return false;
@@ -58,6 +70,7 @@ fn is_lint_noise(program: &str, line: &str, exit_code: i32) -> bool {
|| matches!(program, "eslint" | "biome") && lower.starts_with("warning: react version")
|| matches!(program, "ruff") && lower.starts_with("all checks passed")
|| matches!(program, "mypy") && lower.starts_with("success: no issues found")
|| matches!(program, "pyright" | "basedpyright") && lower.starts_with("0 errors, 0 warnings")
|| matches!(program, "rubocop")
&& (lower.starts_with("inspecting ")
|| lower == "offenses:"
@@ -219,6 +232,33 @@ fn contains_diagnostic_signal(line: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::minimizer::MinimizerConfig;
#[test]
fn pyright_outputjson_passes_through_untouched() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let json = "{\"version\": \"1.1.0\", \"generalDiagnostics\": []}\n";
for command in ["pyright --outputjson src", "basedpyright --outputjson=true src"] {
let ctx = MinimizerCtx {
program: command.split_whitespace().next().unwrap(),
subcommand: None,
command,
config: &cfg,
};
let out = filter(&ctx, json, 1);
assert!(!out.changed, "{command} output must not be rewritten");
assert_eq!(out.text, json);
}
// Plain (non-JSON) runs still condense.
let ctx = MinimizerCtx {
program: "pyright",
subcommand: None,
command: "pyright src",
config: &cfg,
};
let plain = "src/app.py:4:7 - error: bad\nsrc/app.py:9:3 - error: worse\n";
assert!(filter(&ctx, plain, 1).changed);
}
#[test]
fn supports_common_lint_subcommands_for_future_dispatch() {
@@ -251,4 +291,24 @@ mod tests {
assert!(out.contains("src/main.rs (20 diagnostics)"));
assert!(out.contains("… 8 more"));
}
#[test]
fn direct_pyright_support_and_grouping_work() {
assert!(supports_program("pyright", None));
let input = "0 errors, 0 warnings, 0 informations\nsrc/app.ts:4:7 - error TS2322: Type \
'string' is not assignable to type 'number'.\nsrc/app.ts:9:3 - error TS7006: \
Parameter 'x' implicitly has an 'any' type.\n";
let out = condense_lint_output("pyright", input, 1);
assert!(out.contains("2 diagnostics in 1 files"));
assert!(out.contains("src/app.ts (2 diagnostics)"));
assert!(out.contains("TS2322"));
assert!(out.contains("TS7006"));
}
#[test]
fn direct_basedpyright_success_noise_is_stripped() {
assert!(supports_program("basedpyright", None));
let out = condense_lint_output("basedpyright", "0 errors, 0 warnings, 0 notes\n", 0);
assert_eq!(out, "");
}
}
crates/pi-shell/src/minimizer/filters/listing.rs
+916 -15
View File
File diff suppressed because it is too large Load Diff
crates/pi-shell/src/minimizer/filters/mod.rs
+693 -10
View File
@@ -5,6 +5,7 @@ use crate::minimizer::{MinimizerCtx, MinimizerOutput};
pub mod cloud;
pub mod cpp;
pub mod binary_tools;
pub mod bun;
pub mod cargo;
@@ -29,6 +30,7 @@ pub mod pkg;
pub mod python;
pub mod ruby;
pub mod rust_tools;
pub mod system;
pub fn supports(program: &str, subcommand: Option<&str>) -> bool {
@@ -52,6 +54,8 @@ pub fn supports(program: &str, subcommand: Option<&str>) -> bool {
python::supports(program, subcommand)
},
"rspec" | "rake" | "rails" | "rubocop" => ruby::supports(program, subcommand),
"rustfmt" => rust_tools::supports(program, subcommand),
"xxd" | "strings" | "od" => binary_tools::supports(program, subcommand),
"tsc" | "eslint" | "biome" | "shellcheck" | "markdownlint" | "hadolint" | "yamllint"
| "oxlint" | "pyright" | "basedpyright" => {
lint::supports(subcommand) || lint::supports_program(program, subcommand)
@@ -63,14 +67,68 @@ pub fn supports(program: &str, subcommand: Option<&str>) -> bool {
|| js_tools::supports(program, subcommand)
},
"pnpm" if matches!(subcommand, Some("dlx")) => true,
"npm" | "pnpm" | "yarn" | "pip" | "pip3" | "bundle" | "brew" | "composer" | "uv"
| "poetry" => pkg::supports(subcommand),
"uv" if matches!(subcommand, Some("run")) => true,
"npm" | "pnpm" | "yarn" | "pip" | "pip3" | "bundle" | "brew" | "composer" | "poetry" => {
pkg::supports(subcommand)
},
"uv" => {
// uv dispatch coverage (B1 / m4): admit additional subcommand forms
// that wrap a known tool. `uv run` is already handled above; this
// arm covers `uv pytest`, `uv -m pytest`, `uv ruff`, `uv mypy`,
// and other wrapped-tool forms that pre-PR fell through to the
// package-manager filter.
matches!(subcommand, Some("pytest" | "ruff" | "mypy" | "-m")) || pkg::supports(subcommand)
},
"env" | "log" | "deps" | "summary" | "err" | "test" | "diff" | "format" | "pipe" | "ps"
| "ping" | "ssh" | "sops" => system::supports(program),
_ => false,
}
}
fn is_test_script_token(token: &str) -> bool {
let token = token.trim_matches(|ch| matches!(ch, '\'' | '"' | '`'));
matches!(token, "test" | "t" | "e2e" | "spec") || token.starts_with("test:")
}
/// The script/command word a `run`-style invocation targets: the first
/// non-flag token after the `run`/`-m`/`--module` marker. Returns `None` when
/// no marker (or no following word) is present.
///
/// Selecting only this word — instead of scanning the entire command line —
/// keeps tool/script names that appear merely as later arguments from
/// mis-routing output through a test/lint/wrapped-tool filter. Examples that
/// must NOT route as tests: `npm run build -- test`, `uv run echo pytest`.
fn run_invoked_word(command: &str) -> Option<&str> {
let mut tokens = command
.split(|ch: char| ch.is_whitespace() || matches!(ch, ';' | '|' | '&'))
.filter(|tok| !tok.is_empty());
tokens
.by_ref()
.find(|tok| matches!(*tok, "run" | "-m" | "--module"))?;
tokens.find(|tok| !tok.starts_with('-'))
}
fn is_pkg_test_invocation(ctx: &MinimizerCtx<'_>) -> bool {
matches!(ctx.subcommand, Some("test" | "t"))
|| (matches!(ctx.subcommand, Some("run"))
&& run_invoked_word(ctx.command).is_some_and(is_test_script_token))
}
fn is_pkg_lint_invocation(ctx: &MinimizerCtx<'_>) -> bool {
matches!(ctx.subcommand, Some("run"))
&& run_invoked_word(ctx.command).is_some_and(|word| {
is_lint_script_token(word) || matches!(word, "tsc" | "eslint" | "biome")
})
}
fn is_lint_script_token(token: &str) -> bool {
let token = token.trim_matches(|ch| matches!(ch, '\'' | '"' | '`'));
matches!(token, "lint" | "typecheck" | "type-check")
|| token.starts_with("lint:")
|| token.starts_with("typecheck:")
|| token.starts_with("type-check:")
}
/// Apply the matching built-in filter.
pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
let _ = ctx.command;
@@ -95,14 +153,29 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
python::filter(ctx, input, exit_code)
},
"rspec" | "rake" | "rails" | "rubocop" => ruby::filter(ctx, input, exit_code),
"rustfmt" => rust_tools::filter(ctx, input, exit_code),
"xxd" | "strings" | "od" => binary_tools::filter(ctx, input, exit_code),
"tsc" | "eslint" | "biome" | "shellcheck" | "markdownlint" | "hadolint" | "yamllint"
| "oxlint" | "pyright" | "basedpyright" => lint::filter(ctx, input, exit_code),
"jest" | "vitest" | "playwright" => node_tests::filter(ctx, input, exit_code),
"next" | "prettier" | "prisma" => js_tools::filter(ctx, input, exit_code),
"npx" => filter_js_wrapper(ctx, input, exit_code),
"pnpm" if matches!(ctx.subcommand, Some("dlx")) => filter_js_wrapper(ctx, input, exit_code),
"npm" | "pnpm" | "yarn" | "pip" | "pip3" | "bundle" | "brew" | "composer" | "uv"
| "poetry" => pkg::filter(ctx, input, exit_code),
"uv" if matches!(ctx.subcommand, Some("run" | "pytest" | "ruff" | "mypy" | "-m")) => {
filter_uv_wrapper(ctx, input, exit_code)
},
"npm" | "pnpm" | "yarn" => {
if is_pkg_test_invocation(ctx) {
node_tests::filter(ctx, input, exit_code)
} else if is_pkg_lint_invocation(ctx) {
lint::filter(ctx, input, exit_code)
} else {
pkg::filter(ctx, input, exit_code)
}
},
"pip" | "pip3" | "bundle" | "brew" | "composer" | "uv" | "poetry" => {
pkg::filter(ctx, input, exit_code)
},
"env" | "log" | "deps" | "summary" | "err" | "test" | "diff" | "format" | "pipe" | "ps"
| "ping" | "ssh" | "sops" => system::filter(ctx, input, exit_code),
_ => generic::filter(ctx, input, exit_code),
@@ -121,13 +194,227 @@ fn filter_js_wrapper(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> Min
}
}
fn filter_uv_wrapper(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
// uv dispatch normalization (B1 / m4): admit `uv pytest`, `uv -m pytest`,
// `uv ruff`, `uv mypy` in addition to the pre-existing `uv run …` path.
if let Some(tool) = normalize_uv_form(ctx.subcommand, ctx.command) {
let routed = MinimizerCtx {
program: tool,
subcommand: Some(tool),
command: ctx.command,
config: ctx.config,
};
return match tool {
"pytest" | "ruff" | "mypy" => python::filter(&routed, input, exit_code),
_ => MinimizerOutput::passthrough(input),
};
}
match uv_wrapper_tool(ctx) {
Some("pytest") => {
let routed = MinimizerCtx {
program: "pytest",
subcommand: Some("pytest"),
command: ctx.command,
config: ctx.config,
};
python::filter(&routed, input, exit_code)
},
Some("ruff") => {
let subcommand = if ctx.command.split_whitespace().any(|part| part == "format") {
Some("format")
} else {
Some("ruff")
};
let routed =
MinimizerCtx { program: "ruff", subcommand, command: ctx.command, config: ctx.config };
python::filter(&routed, input, exit_code)
},
Some("mypy") => {
let routed = MinimizerCtx {
program: "mypy",
subcommand: Some("mypy"),
command: ctx.command,
config: ctx.config,
};
python::filter(&routed, input, exit_code)
},
Some(tool @ ("tsc" | "eslint" | "biome" | "pyright" | "basedpyright" | "oxlint")) => {
let routed = MinimizerCtx {
program: tool,
subcommand: Some(tool),
command: ctx.command,
config: ctx.config,
};
lint::filter(&routed, input, exit_code)
},
Some("jest" | "vitest" | "playwright") => node_tests::filter(ctx, input, exit_code),
_ => MinimizerOutput::passthrough(input),
}
}
/// Normalize uv invocation forms into a routable tool name (B1 / m4).
///
/// Resolution order:
/// 1. If `subcommand` is itself a known python tool name (pytest, ruff,
/// mypy), return `Some(<tool>)`.
/// 2. If `subcommand` is `"-m"`, scan `command` tokens for the first non-flag
/// word matching the python-tool allowlist; return `Some(<tool>)`.
/// 3. If `subcommand` is `"run"`, return `None` so the caller falls through
/// to the existing `uv_wrapper_tool` path (regression guard).
/// 4. Otherwise return `None`.
///
/// The returned `&'static str` is one of `"pytest"`, `"ruff"`, `"mypy"`;
/// the caller is expected to route via the python filter.
fn normalize_uv_form(subcommand: Option<&str>, command: &str) -> Option<&'static str> {
const ALLOWLIST: &[&str] = &["pytest", "ruff", "mypy"];
let sub = subcommand?;
if let Some(&tool) = ALLOWLIST.iter().find(|&&tool| tool == sub) {
return Some(tool);
}
if sub == "-m" {
// Only the immediate next non-flag token after `-m` may select a tool;
// scanning all subsequent tokens would pick up positional arguments
// (e.g. `uv -m my_module pytest` where `pytest` is an arg to `my_module`).
let mut tokens = command.split_whitespace().skip_while(|t| t != &"-m");
tokens.next(); // consume `-m` itself
let next = tokens.next().filter(|tok| !tok.starts_with('-'))?;
ALLOWLIST.iter().find(|&&tool| tool == next).copied()
} else {
None
}
}
fn uv_wrapper_tool<'a>(ctx: &'a MinimizerCtx<'_>) -> Option<&'a str> {
wrapper_invoked_tool(ctx, &[
"pytest",
"ruff",
"mypy",
"tsc",
"eslint",
"biome",
"pyright",
"basedpyright",
"oxlint",
"jest",
"vitest",
"playwright",
])
}
/// Wrapper options whose value is the *following* token (`--with pytest`),
/// rather than being self-contained (`--with=pytest`). When skipping flags to
/// find the invoked command word we must also skip these options' values, or
/// the value (`pytest`) is mistaken for the command and routes arbitrary output
/// through that tool's filter. Covers the value-taking options of the wrappers
/// routed here — `uv run`, `npx`, `pnpm dlx`, `bun x`. The `--opt=value` form
/// is already a single flag token and needs no entry here.
const WRAPPER_VALUE_OPTIONS: &[&str] = &[
// uv run
"--with",
"--with-requirements",
"--with-editable",
"--python",
"-p",
"--from",
"--directory",
"--project",
"--index",
"--default-index",
"--index-url",
"--extra-index-url",
"--find-links",
"-f",
"--cache-dir",
"--config-file",
"--refresh-package",
"--resolution",
"--prerelease",
"--exclude-newer",
"--link-mode",
"--color",
"--python-preference",
// npx / pnpm dlx
"--package",
"-c",
"--call",
"--workspace",
"-w",
"--node-arg",
];
/// Advance `tokens` to the next invoked-command word, skipping flag tokens and
/// the space-separated values of value-taking options (see
/// [`WRAPPER_VALUE_OPTIONS`]). Inline `--opt=value` flags are skipped whole.
fn next_command_word<'a>(tokens: &mut impl Iterator<Item = &'a str>) -> Option<&'a str> {
while let Some(tok) = tokens.next() {
if !tok.starts_with('-') {
return Some(tok);
}
if !tok.contains('=') && WRAPPER_VALUE_OPTIONS.contains(&tok) {
tokens.next(); // consume the option's value
}
}
None
}
/// The command/tool word a wrapper invocation actually executes: the first
/// non-flag token after a single wrapper keyword (`run`/`dlx`/`exec`), or —
/// when none is present — the first non-flag token after the program.
/// Value-taking options (`--with pytest`) have their value skipped so it is not
/// mistaken for the command. A leading `python`/`python3`/`py` interpreter is
/// descended through its `-m`/`--module` argument so `uv run python -m pytest`
/// resolves to `pytest`. Tool names that appear only as later arguments
/// (`uv run build -- pytest`, `uv run echo pytest`, `uv run --with pytest
/// echo`) are never returned.
fn wrapper_command_word(command: &str) -> Option<&str> {
let mut tokens = command
.split(|ch: char| ch.is_whitespace() || matches!(ch, ';' | '|' | '&'))
.filter(|tok| !tok.is_empty());
tokens.next()?; // drop the program token
let mut word = next_command_word(&mut tokens)?;
if matches!(word, "run" | "dlx" | "exec") {
word = next_command_word(&mut tokens)?;
}
if matches!(word, "python" | "python3" | "py") {
while let Some(tok) = tokens.next() {
if tok == "--" {
return Some(word);
}
if matches!(tok, "-c" | "--command") {
return Some(word);
}
if matches!(tok, "-m" | "--module") {
return tokens
.find(|candidate| !candidate.starts_with('-'))
.or(Some(word));
}
if tok.starts_with('-') {
continue;
}
return Some(tok);
}
}
Some(word)
}
fn wrapper_invokes(ctx: &MinimizerCtx<'_>, tools: &[&str]) -> bool {
ctx.subcommand
.is_some_and(|subcommand| tools.contains(&subcommand))
|| ctx
.command
.split(|ch: char| ch.is_whitespace() || matches!(ch, ';' | '|' | '&'))
.any(|token| tools.contains(&token))
wrapper_invoked_tool(ctx, tools).is_some()
}
fn wrapper_invoked_tool<'a>(ctx: &'a MinimizerCtx<'_>, tools: &[&'a str]) -> Option<&'a str> {
// Prefer wrapper_command_word over ctx.subcommand: it properly skips
// value-taking option values (e.g. -w, --workspace, --with) that
// detect_subcommand may mistake for the invoked tool name.
let word = wrapper_command_word(ctx.command)?;
match tools.iter().copied().find(|&tool| tool == word) {
Some(tool) => Some(tool),
None => {
// Fallback: detect_subcommand may have normalized case or
// resolved through program-specific logic.
ctx.subcommand
.and_then(|subcommand| tools.iter().copied().find(|tool| *tool == subcommand))
},
}
}
#[cfg(test)]
@@ -166,9 +453,405 @@ mod tests {
assert!(!out.changed);
}
#[test]
fn run_invoked_word_picks_script_not_arguments() {
assert_eq!(run_invoked_word("npm run build -- test"), Some("build"));
assert_eq!(run_invoked_word("npm run test"), Some("test"));
assert_eq!(run_invoked_word("npm run --silent test:unit"), Some("test:unit"));
assert_eq!(run_invoked_word("uv run echo pytest"), Some("echo"));
assert_eq!(run_invoked_word("uv run build -- pytest"), Some("build"));
assert_eq!(run_invoked_word("uv run -- pytest"), Some("pytest"));
assert_eq!(run_invoked_word("uv run python -m pytest"), Some("python"));
assert_eq!(run_invoked_word("npm ci"), None);
}
#[test]
fn pkg_test_routing_ignores_test_as_argument() {
let config = MinimizerConfig::default();
// a non-test script that merely passes `test` as an argument must not route as
// a test
assert!(!is_pkg_test_invocation(&ctx("npm", Some("run"), "npm run build -- test", &config)));
assert!(is_pkg_test_invocation(&ctx("npm", Some("run"), "npm run test", &config)));
assert!(is_pkg_test_invocation(&ctx("npm", Some("test"), "npm test", &config)));
}
#[test]
fn pkg_lint_routing_ignores_tool_as_argument() {
let config = MinimizerConfig::default();
assert!(!is_pkg_lint_invocation(&ctx(
"pnpm",
Some("run"),
"pnpm run build -- eslint",
&config
)));
assert!(is_pkg_lint_invocation(&ctx("pnpm", Some("run"), "pnpm run lint", &config)));
assert!(is_pkg_lint_invocation(&ctx("pnpm", Some("run"), "pnpm run tsc", &config)));
}
#[test]
fn uv_wrapper_ignores_tool_as_argument() {
let config = MinimizerConfig::default();
assert_eq!(
uv_wrapper_tool(&ctx("uv", Some("run"), "uv run pytest", &config)),
Some("pytest")
);
assert_eq!(uv_wrapper_tool(&ctx("uv", Some("run"), "uv run echo pytest", &config)), None);
assert_eq!(uv_wrapper_tool(&ctx("uv", Some("run"), "uv run build -- pytest", &config)), None);
}
#[test]
fn uv_wrapper_skips_value_taking_option_values() {
let config = MinimizerConfig::default();
// `--with <pkg>` consumes the following token as its value; that value must
// not be mistaken for the invoked command and route output through it.
assert_eq!(
uv_wrapper_tool(&ctx("uv", Some("run"), "uv run --with pytest echo hi", &config)),
None
);
assert_eq!(
uv_wrapper_tool(&ctx("uv", Some("run"), "uv run --with pytest build", &config)),
None
);
// the genuinely invoked tool still routes when preceded by a value option
assert_eq!(
uv_wrapper_tool(&ctx("uv", Some("run"), "uv run --python 3.12 pytest", &config)),
Some("pytest")
);
// inline `--opt=value` is a single token; the command word follows it
assert_eq!(
uv_wrapper_tool(&ctx("uv", Some("run"), "uv run --with=pytest echo hi", &config)),
None
);
// `python -m <module>` descent still resolves through a value option
assert_eq!(
uv_wrapper_tool(&ctx("uv", Some("run"), "uv run --with foo python -m pytest", &config)),
Some("pytest")
);
}
#[test]
fn uv_run_with_option_value_is_left_opaque() {
let config = MinimizerConfig::default();
// `pytest` is the value of `--with`, the invoked command is `echo` — output
// (including PASS/✓-style lines) must pass through untouched.
let context = ctx("uv", Some("run"), "uv run --with pytest echo PASS", &config);
let input = "collected 2 items\nPASS\n";
let out = filter(&context, input, 0);
assert_eq!(out.text, input);
assert!(!out.changed);
}
#[test]
fn uv_run_echo_pytest_is_left_opaque() {
let config = MinimizerConfig::default();
// `pytest` is an argument to `echo`, not the invoked command — output must pass
// through
let context = ctx("uv", Some("run"), "uv run echo pytest", &config);
let input = "collected 2 items\npytest\n";
let out = filter(&context, input, 0);
assert_eq!(out.text, input);
assert!(!out.changed);
}
#[test]
fn uv_run_pytest_routes_to_python_filter() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run pytest", &config);
let input = "============================= test session starts \
==============================\ncollected 2 items\n\na.py .\nb.py \
F\n\n=================================== FAILURES \
===================================\nFAILED b.py::test_fail - AssertionError: \
expected 2 == 1\n=========================== short test summary info \
============================\nFAILED b.py::test_fail - AssertionError: \
expected 2 == 1\n========================= 1 failed, 1 passed in 0.12s \
=========================\n";
let out = filter(&context, input, 1).text;
assert!(out.contains("FAILED b.py::test_fail"));
assert!(!out.contains("collected 2 items"));
assert!(out.contains("pytest: 1 failed, 1 passed"));
}
#[test]
fn uv_run_ruff_routes_to_python_filter() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run ruff check .", &config);
let input = "src/app.py:1:1: F401 imported but unused\nFound 1 error.\n";
let out = filter(&context, input, 1).text;
assert!(out.contains("F401"));
}
#[test]
fn uv_run_python_module_pytest_routes_to_python_filter() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run python -m pytest", &config);
let input = "============================= test session starts \
==============================\ncollected 1 item\n\na.py \
F\n\n=================================== FAILURES \
===================================\nFAILED a.py::test_fail - \
AssertionError\n========================= 1 failed in 0.03s \
=========================\n";
let out = filter(&context, input, 1).text;
assert!(out.contains("FAILED a.py::test_fail"));
assert!(!out.contains("collected 1 item"));
}
#[test]
fn uv_run_pyright_routes_to_lint_filter() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run pyright", &config);
let input = "0 errors, 0 warnings, 0 informations\nsrc/app.ts:4:7 - error TS2322: Type \
'string' is not assignable to type 'number'.\n";
let out = filter(&context, input, 1).text;
assert!(out.contains("TS2322"));
}
#[test]
fn uv_run_basedpyright_routes_to_lint_filter() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run basedpyright", &config);
let input = "0 errors, 0 warnings, 0 notes\nsrc/app.ts:4:7 - error TS2322: Type 'string' is \
not assignable to type 'number'.\n";
let out = filter(&context, input, 1).text;
assert!(out.contains("TS2322"));
}
#[test]
fn uv_run_unknown_tool_is_passthrough() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run custom-tool", &config);
let input = "line 1\nline 2\n";
let out = filter(&context, input, 0);
assert_eq!(out.text, input);
assert!(!out.changed);
}
#[test]
fn npm_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("npm", Some("test"), "npm test", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn npm_run_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("npm", Some("run"), "npm run test", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn npm_run_quoted_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("npm", Some("run"), "npm run \"test\"", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn pnpm_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("pnpm", Some("test"), "pnpm test", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn pnpm_run_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("pnpm", Some("run"), "pnpm run test", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn yarn_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("yarn", Some("test"), "yarn test", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn yarn_run_test_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("yarn", Some("run"), "yarn run test", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn npm_run_build_still_uses_pkg_filter() {
let config = MinimizerConfig::default();
let context = ctx("npm", Some("run"), "npm run build", &config);
let out = filter(&context, "Resolving dependencies\nDownloaded foo\nerror: failed\n", 1).text;
assert!(!out.contains("Resolving dependencies"));
assert!(out.contains("error: failed"));
}
#[test]
fn package_manager_lint_scripts_route_to_lint_filter() {
let config = MinimizerConfig::default();
let input = concat!(
"src/app.ts:1:1: error TS2322: Type 'string' is not assignable to type 'number'.\n",
"src/app.ts:2:1: error TS7006: Parameter 'x' implicitly has an 'any' type.\n",
);
for (program, command) in [
("npm", "npm run lint"),
("npm", "npm run typecheck"),
("pnpm", "pnpm run lint:ci"),
("yarn", "yarn run typecheck:ci"),
] {
let context = ctx(program, Some("run"), command, &config);
let routed = filter(&context, input, 1).text;
let expected = lint::filter(&context, input, 1).text;
assert_eq!(routed, expected, "{command} should use lint filter");
assert!(
routed.contains("2 diagnostics in 1 files"),
"{command} should condense lint output"
);
}
}
#[test]
fn npm_t_routes_to_node_tests() {
let config = MinimizerConfig::default();
let context = ctx("npm", Some("t"), "npm t", &config);
let out = filter(&context, "✓ ok\nFAIL app.test.ts\nTests 1 failed\n", 1).text;
assert!(!out.contains("✓ ok"));
assert!(out.contains("FAIL app.test.ts"));
}
#[test]
fn pi_cli_names_are_not_supported() {
assert!(!supports("rtk", None));
assert!(!supports("pi", None));
}
// ---------------------------------------------------------------
// Tier 2a: uv dispatch coverage tests (m4)
// ---------------------------------------------------------------
const PYTEST_FAILURE_INPUT: &str = "============================= test session starts \
==============================\ncollected 2 \
items\n\nFAILED tests/test_x.py::test_fail - \
AssertionError\n========================= 1 failed, 1 \
passed in 0.05s =========================\n";
#[test]
fn uv_pytest_routes_to_python_filter() {
// B1 fix: `uv pytest <args>` now routes to the python filter.
let config = MinimizerConfig::default();
let context = ctx("uv", Some("pytest"), "uv pytest tests/", &config);
assert!(supports("uv", Some("pytest")));
let out = filter(&context, PYTEST_FAILURE_INPUT, 1).text;
assert!(out.contains("FAILED tests/test_x.py::test_fail"));
assert!(out.contains("pytest: 1 failed, 1 passed"));
}
#[test]
fn uv_dash_m_pytest_routes_to_python_filter() {
// B1 fix: `uv -m pytest <args>` now routes via -m token scan.
let config = MinimizerConfig::default();
let context = ctx("uv", Some("-m"), "uv -m pytest tests/", &config);
assert!(supports("uv", Some("-m")));
let out = filter(&context, PYTEST_FAILURE_INPUT, 1).text;
assert!(out.contains("FAILED tests/test_x.py::test_fail"));
assert!(out.contains("pytest: 1 failed, 1 passed"));
}
#[test]
fn uv_ruff_routes_to_python_filter() {
// B1 fix: `uv ruff <args>` now routes to the python filter.
let config = MinimizerConfig::default();
let context = ctx("uv", Some("ruff"), "uv ruff check .", &config);
assert!(supports("uv", Some("ruff")));
let out =
filter(&context, "src/a.py:1:1: F401 imported but unused\nFound 1 error.\n", 1).text;
assert!(out.contains("F401"));
}
#[test]
fn uv_mypy_routes_to_python_filter() {
// B1 fix: `uv mypy <args>` now routes to the python filter.
let config = MinimizerConfig::default();
let context = ctx("uv", Some("mypy"), "uv mypy src/", &config);
assert!(supports("uv", Some("mypy")));
// mypy filter routes through lint::condense_lint_output; smoke-check
// it does not crash and produces a string output.
let _ = filter(&context, "src/a.py:1: error: foo\n", 1).text;
}
#[test]
fn uv_run_pytest_still_routes_regression_guard() {
// Regression guard for the pre-existing `uv run pytest` path.
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run pytest tests/", &config);
let out = filter(&context, PYTEST_FAILURE_INPUT, 1).text;
assert!(out.contains("FAILED tests/test_x.py::test_fail"));
assert!(out.contains("pytest: 1 failed, 1 passed"));
}
#[test]
fn uv_run_python_dash_m_pytest_still_routes() {
// Regression guard: `uv run python -m pytest` was supported pre-PR.
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run python -m pytest tests/", &config);
let out = filter(&context, PYTEST_FAILURE_INPUT, 1).text;
assert!(out.contains("FAILED tests/test_x.py::test_fail"));
assert!(out.contains("pytest: 1 failed, 1 passed"));
}
#[test]
fn uv_run_python_script_with_pytest_argument_stays_opaque() {
let config = MinimizerConfig::default();
let context = ctx("uv", Some("run"), "uv run python scripts/report.py -m pytest", &config);
let out = filter(&context, PYTEST_FAILURE_INPUT, 1);
assert_eq!(out.text, PYTEST_FAILURE_INPUT);
assert!(!out.changed);
}
#[test]
fn normalize_uv_form_unit_pytest_subcommand() {
assert_eq!(super::normalize_uv_form(Some("pytest"), "uv pytest"), Some("pytest"));
}
#[test]
fn normalize_uv_form_unit_dash_m_pytest() {
assert_eq!(super::normalize_uv_form(Some("-m"), "uv -m pytest tests/"), Some("pytest"));
}
#[test]
fn normalize_uv_form_unit_run_returns_none() {
// `uv run` is handled by the pre-existing path; normalize returns None.
assert_eq!(super::normalize_uv_form(Some("run"), "uv run pytest"), None);
}
#[test]
fn normalize_uv_form_unit_unknown_returns_none() {
assert_eq!(super::normalize_uv_form(Some("unknown"), "uv unknown"), None);
assert_eq!(super::normalize_uv_form(None, "uv"), None);
}
#[test]
fn pytest_legacy_filters_active_passes_through() {
// Kill-switch parity (M2): legacy_filters_active=true skips the
// pytest state machine even when invoked via `uv pytest`.
let mut config = MinimizerConfig::default();
config.enabled = true;
config.legacy_filters_active = true;
let context = ctx("uv", Some("pytest"), "uv pytest tests/", &config);
let out = filter(&context, PYTEST_FAILURE_INPUT, 1);
assert_eq!(out.text, PYTEST_FAILURE_INPUT);
assert!(!out.changed);
}
}
crates/pi-shell/src/minimizer/filters/node_tests.rs
+109 -2
View File
@@ -65,7 +65,7 @@ fn failures_only(input: &str) -> String {
}
if keeping_block {
if is_pass_noise(trimmed) && !is_error_context_line(trimmed) {
if is_pass_noise(trimmed) {
keeping_block = false;
trailing_context = 0;
continue;
@@ -112,6 +112,7 @@ fn is_summary_line(trimmed: &str) -> bool {
|| trimmed.starts_with("% ")
|| trimmed.starts_with("Failed Tests")
|| trimmed.starts_with("Playwright Test Report")
|| (trimmed.starts_with("Ran ") && trimmed.contains("tests across"))
|| starts_count_summary(trimmed)
}
@@ -123,7 +124,7 @@ fn starts_count_summary(trimmed: &str) -> bool {
if !count.chars().all(|ch| ch.is_ascii_digit()) {
return false;
}
matches!(parts.next(), Some("failed" | "passed" | "skipped" | "flaky"))
matches!(parts.next(), Some("failed" | "passed" | "skipped" | "flaky" | "pass" | "fail"))
}
fn is_pass_noise(trimmed: &str) -> bool {
@@ -134,6 +135,13 @@ fn is_pass_noise(trimmed: &str) -> bool {
|| trimmed.starts_with("○")
|| trimmed.starts_with(" RUN ")
|| trimmed.starts_with("DEV ")
|| trimmed.starts_with("bun test ")
|| trimmed.ends_with(".test.ts:")
|| trimmed.ends_with(".test.js:")
|| trimmed.ends_with(".test.tsx:")
|| trimmed.ends_with(".test.jsx:")
|| trimmed.ends_with(".spec.ts:")
|| trimmed.ends_with(".spec.js:")
}
fn starts_failure_block(trimmed: &str) -> bool {
@@ -160,6 +168,7 @@ fn is_error_context_line(trimmed: &str) -> bool {
|| trimmed.starts_with("Expected")
|| trimmed.starts_with("Received")
|| trimmed.starts_with("Error:")
|| trimmed.starts_with("error:")
|| trimmed.starts_with("AssertionError")
|| trimmed.starts_with("TimeoutError")
|| trimmed.contains(" › ")
@@ -235,4 +244,102 @@ mod tests {
let filtered = drop_passed_lines("✓ one passed\n✓ two passed\n3 passed (1.2s)\n");
assert_eq!(filtered, "3 passed (1.2s)\n");
}
#[test]
fn bun_pass_only_collapses_to_counts() {
let input = "\
✓ a.test.ts > add works [0.50ms]
✓ a.test.ts > subtract works [0.30ms]
✓ b.test.ts > multiply works [0.40ms]
✓ b.test.ts > divide works [0.60ms]
✓ c.test.ts > negate works [0.20ms]
5 pass
0 fail
7 expect() calls
Ran 5 tests across 3 files. [102.00ms]
";
let filtered = drop_passed_lines(input);
assert!(!filtered.contains("add works"));
assert!(!filtered.contains("subtract works"));
assert!(!filtered.contains("multiply works"));
assert!(filtered.contains("5 pass"));
assert!(filtered.contains("0 fail"));
assert!(filtered.contains("7 expect() calls"));
assert!(filtered.contains("Ran 5 tests across 3 files"));
}
#[test]
fn bun_failure_keeps_error_and_counts() {
let input = "\
✗ a.test.ts > bad test [0.40ms]
error: expect(received).toBe(expected)
Expected: 2
Received: 3
at a.test.ts:5:7
✓ b.test.ts > another good [0.60ms]
2 pass
1 fail
Ran 3 tests across 2 files. [150.00ms]
";
let filtered = failures_only(input);
assert!(!filtered.contains("another good"));
assert!(filtered.contains("✗ a.test.ts > bad test"));
assert!(filtered.contains("error: expect(received).toBe(expected)"));
assert!(filtered.contains("Expected: 2"));
assert!(filtered.contains("Received: 3"));
assert!(filtered.contains("at a.test.ts:5:7"));
assert!(filtered.contains("2 pass"));
assert!(filtered.contains("1 fail"));
}
#[test]
fn vitest_many_passes_collapses_to_summary() {
let input = "\
✓ src/a.test.ts > suite > test1 (2ms)
✓ src/a.test.ts > suite > test2 (1ms)
✓ src/a.test.ts > other > test3 (3ms)
✓ src/b.test.ts > feature > test4 (1ms)
✓ src/b.test.ts > feature > test5 (2ms)
✓ src/b.test.ts > edge > test6 (5ms)
Test Files 2 passed (2)
Tests 6 passed (6)
Start at 12:00:00
Duration 1.23s
";
let filtered = drop_passed_lines(input);
assert!(!filtered.contains("test1"));
assert!(!filtered.contains("test6"));
assert!(filtered.contains("Test Files 2 passed (2)"));
assert!(filtered.contains("Tests 6 passed (6)"));
assert!(filtered.contains("Duration 1.23s"));
}
#[test]
fn jest_many_passes_collapses_to_summary() {
let input = "\
PASS src/a.test.ts
PASS src/b.test.ts
PASS src/c.test.ts
PASS src/d.test.ts
PASS src/e.test.ts
Test Suites: 5 passed, 5 total
Tests: 32 passed, 32 total
Snapshots: 0 total
Time: 2.345s
";
let filtered = drop_passed_lines(input);
assert!(!filtered.contains("src/a.test.ts"));
assert!(!filtered.contains("src/e.test.ts"));
assert!(filtered.contains("Test Suites: 5 passed, 5 total"));
assert!(filtered.contains("Tests: 32 passed, 32 total"));
assert!(filtered.contains("Time: 2.345s"));
}
}
crates/pi-shell/src/minimizer/filters/pkg.rs
+531 -25
View File
@@ -1,6 +1,9 @@
//! Package manager output filters.
use std::{collections::HashSet, fmt::Write as _};
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
const PACKAGE_TREE_HEAD_LINES: usize = 80;
pub fn supports(subcommand: Option<&str>) -> bool {
matches!(
@@ -13,6 +16,7 @@ pub fn supports(subcommand: Option<&str>) -> bool {
| "remove"
| "rm" | "uninstall"
| "list" | "ls"
| "tree" | "pip"
| "outdated"
| "sync" | "lock"
| "run" | "exec"
@@ -30,19 +34,42 @@ pub fn supports(subcommand: Option<&str>) -> bool {
| "dedupe"
| "publish"
| "pack" | "link"
| "why"
| "why" | "export"
)
)
}
pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
if exit_code == 0
&& (command_contains_any(ctx.command, &["--json"])
|| ctx.program == "uv"
&& matches!(ctx.subcommand, Some("pip"))
&& command_contains_any(ctx.command, &["freeze"]))
{
return MinimizerOutput::passthrough(input);
}
let cleaned = primitives::strip_ansi(input);
let stripped = strip_package_noise(ctx.program, &cleaned, exit_code);
let deduped = primitives::dedup_consecutive_lines(&stripped);
let text = if contains_audit_or_security_summary(&deduped) {
deduped
let text = if exit_code == 0 && is_package_lock_command(ctx) {
compact_package_lock_output(ctx, &cleaned)
} else {
primitives::head_tail_lines(&deduped, 120, 80)
let stripped = strip_package_noise(ctx, &cleaned, exit_code);
let deduped = primitives::dedup_consecutive_lines(&stripped);
if contains_audit_or_security_summary(&deduped) {
deduped
} else if exit_code == 0
&& (is_package_tree_command(ctx) || is_package_export_command(ctx))
&& !command_contains_any(ctx.command, &["--json"])
{
compact_package_tree_output(&deduped)
} else {
let cap = if exit_code == 0 {
primitives::CapClass::Inventory
} else {
primitives::CapClass::Errors
};
primitives::head_tail_cap(&deduped, cap)
}
};
if text == input {
@@ -52,7 +79,7 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
}
}
fn strip_package_noise(program: &str, input: &str, exit_code: i32) -> String {
fn strip_package_noise(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> String {
let mut out = String::new();
let mut previous_blank = false;
for line in input.lines() {
@@ -66,7 +93,7 @@ fn strip_package_noise(program: &str, input: &str, exit_code: i32) -> String {
}
previous_blank = false;
if is_noise_line(program, trimmed, exit_code) {
if is_noise_line(ctx, trimmed, exit_code) {
continue;
}
out.push_str(line.trim_end());
@@ -75,19 +102,248 @@ fn strip_package_noise(program: &str, input: &str, exit_code: i32) -> String {
out
}
fn is_noise_line(program: &str, line: &str, exit_code: i32) -> bool {
if is_audit_or_security_summary(line) {
fn is_package_tree_command(ctx: &MinimizerCtx<'_>) -> bool {
match ctx.program {
"npm" | "pnpm" | "yarn" => {
matches!(ctx.subcommand, Some("list" | "ls" | "tree" | "why" | "explain"))
},
"bun" => {
matches!(ctx.subcommand, Some("list" | "ls" | "tree" | "why" | "explain"))
|| matches!(ctx.subcommand, Some("pm"))
&& command_contains_any(ctx.command, &["list", "ls", "tree", "why"])
},
"uv" => {
matches!(ctx.subcommand, Some("list" | "ls" | "tree"))
|| matches!(ctx.subcommand, Some("pip"))
&& command_contains_any(ctx.command, &["list", "ls", "tree"])
},
"poetry" => {
matches!(ctx.subcommand, Some("tree"))
|| matches!(ctx.subcommand, Some("show"))
&& command_contains_any(ctx.command, &["--tree"])
},
_ => false,
}
}
fn is_package_export_command(ctx: &MinimizerCtx<'_>) -> bool {
match ctx.program {
"uv" | "poetry" => ctx.subcommand == Some("export"),
_ => false,
}
}
fn is_package_lock_command(ctx: &MinimizerCtx<'_>) -> bool {
matches!((ctx.program, ctx.subcommand), ("uv" | "poetry", Some("lock")))
}
fn command_contains_any(command: &str, words: &[&str]) -> bool {
command.split_whitespace().any(|part| words.contains(&part))
}
fn compact_package_tree_output(input: &str) -> String {
if let Some(summary) = compact_package_tree_json_output(input) {
return summary;
}
if let Some(summary) = compact_package_tree_ndjson_output(input) {
return summary;
}
let lines: Vec<&str> = input
.lines()
.map(str::trim_end)
.filter(|line| !line.trim().is_empty())
.collect();
if lines.len() <= PACKAGE_TREE_HEAD_LINES {
return input.to_string();
}
let mut out = format!("package tree/list: {} entries\n", lines.len());
for line in lines.iter().take(PACKAGE_TREE_HEAD_LINES) {
out.push_str(line);
out.push('\n');
}
let _ = writeln!(out, "… {} package entries omitted …", lines.len() - PACKAGE_TREE_HEAD_LINES);
out
}
fn compact_package_tree_json_output(input: &str) -> Option<String> {
let value: serde_json::Value = serde_json::from_str(input).ok()?;
let mut rows = Vec::new();
let mut seen = HashSet::new();
collect_package_tree_json_rows(&value, &mut rows, &mut seen);
summarize_package_rows(rows)
}
fn compact_package_tree_ndjson_output(input: &str) -> Option<String> {
let mut rows = Vec::new();
let mut seen = HashSet::new();
for line in input.lines().map(str::trim).filter(|line| !line.is_empty()) {
let value: serde_json::Value = serde_json::from_str(line).ok()?;
collect_package_tree_json_rows(&value, &mut rows, &mut seen);
if let Some(data) = value.get("data").and_then(serde_json::Value::as_str) {
for row in data
.lines()
.map(str::trim_end)
.filter(|row| !row.trim().is_empty())
{
push_unique_row(&mut rows, &mut seen, row.to_string());
}
}
}
summarize_package_rows(rows)
}
fn summarize_package_rows(rows: Vec<String>) -> Option<String> {
if rows.is_empty() {
return None;
}
let mut out = format!("package tree/list: {} entries\n", rows.len());
for row in rows.iter().take(PACKAGE_TREE_HEAD_LINES) {
out.push_str(row);
out.push('\n');
}
if rows.len() > PACKAGE_TREE_HEAD_LINES {
let _ = writeln!(out, "… {} package entries omitted …", rows.len() - PACKAGE_TREE_HEAD_LINES);
}
Some(out)
}
fn collect_package_tree_json_rows(
value: &serde_json::Value,
rows: &mut Vec<String>,
seen: &mut HashSet<String>,
) {
match value {
serde_json::Value::Object(map) => {
if let Some(name) = map.get("name").and_then(serde_json::Value::as_str) {
let version = map
.get("version")
.and_then(serde_json::Value::as_str)
.unwrap_or("");
push_unique_row(
rows,
seen,
if version.is_empty() {
name.to_string()
} else {
format!("{name} {version}")
},
);
}
if let Some(dependencies) = map
.get("dependencies")
.and_then(serde_json::Value::as_object)
{
for (name, child) in dependencies {
push_json_dependency_row(rows, seen, name, child);
}
}
for value in map.values() {
if value.is_array() || value.is_object() {
collect_package_tree_json_rows(value, rows, seen);
}
}
},
serde_json::Value::Array(items) => {
for item in items {
collect_package_tree_json_rows(item, rows, seen);
}
},
_ => {},
}
}
fn push_json_dependency_row(
rows: &mut Vec<String>,
seen: &mut HashSet<String>,
name: &str,
child: &serde_json::Value,
) {
let version = child
.get("version")
.and_then(serde_json::Value::as_str)
.unwrap_or("");
push_unique_row(
rows,
seen,
if version.is_empty() {
name.to_string()
} else {
format!("{name} {version}")
},
);
}
fn push_unique_row(rows: &mut Vec<String>, seen: &mut HashSet<String>, row: String) {
if seen.insert(row.clone()) {
rows.push(row);
}
}
fn is_noise_line(ctx: &MinimizerCtx<'_>, line: &str, exit_code: i32) -> bool {
let lower = line.to_ascii_lowercase();
// Strip: "found 0 vulnerabilities" (non-actionable success noise)
if lower.contains("found 0 vulnerabilities") {
return true;
}
// Strip: "audited X packages" timing summaries (non-actionable)
if lower.contains("audited") && lower.contains("package") {
return true;
}
// Keep: vulnerability mentions (actionable — real findings)
if lower.contains("vulnerab") {
return false;
}
if exit_code != 0 && is_error_or_summary(line) {
return false;
}
let lower = line.to_ascii_lowercase();
if is_package_lock_command(ctx) && is_lock_summary_line(&lower) {
return false;
}
is_generic_progress(line, &lower)
|| is_js_package_noise(program, line, &lower)
|| is_python_package_noise(program, line, &lower)
|| is_ruby_php_brew_noise(program, line, &lower)
|| is_js_package_noise(ctx.program, line, &lower)
|| is_python_package_noise(ctx.program, line, &lower)
|| is_ruby_php_brew_noise(ctx.program, line, &lower)
}
fn compact_package_lock_output(ctx: &MinimizerCtx<'_>, input: &str) -> String {
let mut out = String::new();
for line in input.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let lower = trimmed.to_ascii_lowercase();
if is_lock_summary_line(&lower) {
out.push_str(trimmed);
out.push('\n');
continue;
}
if is_generic_progress(trimmed, &lower)
|| is_python_package_noise(ctx.program, trimmed, &lower)
|| is_js_package_noise(ctx.program, trimmed, &lower)
{
continue;
}
out.push_str(trimmed);
out.push('\n');
}
if out.trim().is_empty() {
primitives::head_tail_cap(input, primitives::CapClass::Inventory)
} else {
primitives::head_tail_cap(&out, primitives::CapClass::Inventory)
}
}
fn is_lock_summary_line(lower: &str) -> bool {
lower.starts_with("writing lock file")
|| lower.starts_with("updated lockfile")
|| lower.starts_with("resolved ")
|| lower.starts_with("installing dependencies from lock file")
|| lower == "no changes."
|| lower.starts_with("no dependencies to install or update")
}
fn is_generic_progress(line: &str, lower: &str) -> bool {
@@ -110,7 +366,6 @@ fn is_js_package_noise(program: &str, line: &str, lower: &str) -> bool {
}
line.starts_with('>') && line.contains('@')
|| lower.starts_with("npm notice")
|| lower.starts_with("npm warn deprecated")
|| lower.starts_with("npm http fetch")
|| lower.starts_with("pnpm: progress")
|| lower.starts_with("packages:")
@@ -119,6 +374,7 @@ fn is_js_package_noise(program: &str, line: &str, lower: &str) -> bool {
|| lower.starts_with("added ") && lower.contains("packages")
|| lower.starts_with("done in ")
|| lower.contains("already up-to-date")
|| lower.contains("up to date")
}
fn is_python_package_noise(program: &str, _line: &str, lower: &str) -> bool {
@@ -133,6 +389,17 @@ fn is_python_package_noise(program: &str, _line: &str, lower: &str) -> bool {
|| lower.starts_with("resolving dependencies")
|| lower.starts_with("writing lock file")
|| lower.starts_with("package operations:")
|| program == "uv" && is_uv_progress_noise(lower)
}
fn is_uv_progress_noise(lower: &str) -> bool {
lower.starts_with("resolved ")
|| lower.starts_with("prepared ")
|| lower.starts_with("installed ")
|| lower.starts_with("uninstalled ")
|| lower.starts_with("updated ")
|| lower.starts_with("built ")
|| lower.starts_with("downloaded ")
}
fn is_ruby_php_brew_noise(program: &str, _line: &str, lower: &str) -> bool {
@@ -177,12 +444,15 @@ fn is_error_or_summary(line: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::minimizer::MinimizerConfig;
#[test]
fn strips_progress_but_keeps_package_errors() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("npm", Some("install"), "npm install", &cfg);
let input = "Resolving: total 10\nDownloading: left-pad\nERROR failed to install \
left-pad\nfound 1 vulnerability\n";
let out = strip_package_noise("npm", input, 1);
let out = strip_package_noise(&ctx, input, 1);
assert!(!out.contains("Resolving:"));
assert!(!out.contains("Downloading:"));
assert!(out.contains("ERROR failed"));
@@ -190,22 +460,34 @@ mod tests {
}
#[test]
fn preserves_successful_install_audit_and_security_summaries() {
fn strips_success_noise_audited_and_zero_vulnerabilities() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("npm", Some("install"), "npm install", &cfg);
let input = "Resolving: total 10\nadded 3 packages, and audited 4 packages in 1s\n2 \
packages are looking for funding\nfound 0 vulnerabilities\n";
let out = strip_package_noise("npm", input, 0);
let out = strip_package_noise(&ctx, input, 0);
assert!(!out.contains("Resolving:"));
assert!(out.contains("added 3 packages, and audited 4 packages in 1s"));
assert!(out.contains("2 packages are looking for funding"));
assert!(out.contains("found 0 vulnerabilities"));
assert!(!out.contains("audited 4 packages"));
assert!(!out.contains("found 0 vulnerabilities"));
}
#[test]
fn preserves_deprecation_warnings() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("npm", Some("install"), "npm install", &cfg);
let input = "npm warn deprecated left-pad@1.0.0: Please upgrade to left-pad@2.0.0\nnpm warn \
deprecated old-lib@2.0.0: Use new-lib instead\n";
let out = strip_package_noise(&ctx, input, 0);
assert!(out.contains("npm warn deprecated left-pad@1.0.0: Please upgrade to left-pad@2.0.0"));
assert!(out.contains("npm warn deprecated old-lib@2.0.0: Use new-lib instead"));
}
#[test]
fn supports_common_package_subcommands_for_future_dispatch() {
for subcommand in [
"ci", "add", "outdated", "sync", "audit", "why", "view", "fund", "explain", "test", "t",
"start", "stop", "restart", "config", "cache", "prune", "dedupe", "publish", "pack",
"link",
"ci", "add", "outdated", "sync", "audit", "why", "tree", "pip", "view", "fund", "explain",
"test", "t", "start", "stop", "restart", "config", "cache", "prune", "dedupe", "publish",
"pack", "link",
] {
assert!(supports(Some(subcommand)), "{subcommand} should be supported");
}
@@ -213,10 +495,234 @@ mod tests {
#[test]
fn bun_install_noise_uses_js_package_rules() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("bun", Some("install"), "bun install", &cfg);
let input = "Resolving dependencies\nDownloaded foo\nerror: failed\n";
let out = strip_package_noise("bun", input, 1);
let out = strip_package_noise(&ctx, input, 1);
assert!(!out.contains("Resolving dependencies"));
assert!(!out.contains("Downloaded foo"));
assert!(out.contains("error: failed"));
}
fn ctx<'a>(
program: &'a str,
subcommand: Option<&'a str>,
command: &'a str,
config: &'a MinimizerConfig,
) -> MinimizerCtx<'a> {
MinimizerCtx { program, subcommand, command, config }
}
#[test]
fn compacts_large_js_package_tree() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("npm", Some("list"), "npm list --all", &cfg);
let mut input = String::from("app@1.0.0\n");
for idx in 0..90 {
input.push_str(&format!("├── dep{idx:03}@1.0.0\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("├── dep000@1.0.0"));
assert!(out.text.contains("├── dep078@1.0.0"));
assert!(!out.text.contains("├── dep089@1.0.0"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn compacts_depth_limited_package_tree_commands() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("npm", Some("ls"), "npm ls --depth=0", &cfg);
let mut input = String::from("app@1.0.0\n");
for idx in 0..90 {
input.push_str(&format!("├── dep{idx:03}@1.0.0\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("dep000"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn compacts_pnpm_why_style_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("pnpm", Some("why"), "pnpm why react", &cfg);
let mut input =
String::from("Legend: production dependency, optional only, dev only\nreact 19.0.0\n");
for idx in 0..90 {
input.push_str(&format!("└─ dependent{idx:03}\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 92 entries\n"));
assert!(out.text.contains("react 19.0.0"));
assert!(out.text.contains("└─ dependent000"));
assert!(out.text.contains("… 12 package entries omitted …"));
}
#[test]
fn passes_through_npm_json_dependency_tree() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("npm", Some("ls"), "npm ls --json", &cfg);
let input = r#"{"name":"app","version":"1.0.0","dependencies":{"react":{"version":"19.0.0","dependencies":{"scheduler":{"version":"0.25.0"}}},"zod":{"version":"4.0.0"}}}"#;
let out = filter(&context, input, 0);
assert!(!out.changed);
assert_eq!(out.text, input);
}
#[test]
fn passes_through_pnpm_why_json_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("pnpm", Some("why"), "pnpm why react --json", &cfg);
let input = r#"[{"name":"react","version":"19.0.0","dependents":[{"name":"app","version":"1.0.0"},{"name":"docs","version":"1.0.0"}]}]"#;
let out = filter(&context, input, 0);
assert!(!out.changed);
assert_eq!(out.text, input);
}
#[test]
fn passes_through_yarn_why_ndjson_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("yarn", Some("why"), "yarn why react --json", &cfg);
let input = "{\"type\":\"info\",\"data\":\"=> Found \
\\\"react@npm:19.0.0\\\"\"}\n{\"type\":\"tree\",\"data\":\"react@npm:19.0.0\\\
n└─ app@workspace:.\"}\n";
let out = filter(&context, input, 0);
assert!(!out.changed);
assert_eq!(out.text, input);
}
#[test]
fn passes_through_npm_explain_json_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("npm", Some("explain"), "npm explain react --json", &cfg);
let input = r#"{"name":"react","version":"19.0.0","dependents":[{"name":"app","version":"1.0.0","location":"."}]}"#;
let out = filter(&context, input, 0);
assert!(!out.changed);
assert_eq!(out.text, input);
}
#[test]
fn compacts_uv_pip_list_and_strips_progress_noise() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("uv", Some("pip"), "uv pip list", &cfg);
let mut input = String::from(
"Resolved 91 packages in 12ms\nPrepared 2 packages in 3ms\nPackage Version\n",
);
for idx in 0..90 {
input.push_str(&format!("pkg{idx:03} 1.0.{idx}\n"));
}
let out = filter(&context, &input, 0);
assert!(!out.text.contains("Resolved 91 packages"));
assert!(!out.text.contains("Prepared 2 packages"));
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("Package Version"));
assert!(out.text.contains("pkg000 1.0.0"));
assert!(out.text.contains("pkg078 1.0.78"));
assert!(!out.text.contains("pkg089 1.0.89"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn compacts_uv_tree_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("uv", Some("tree"), "uv tree", &cfg);
let mut input = String::from("project v1.0.0\n");
for idx in 0..90 {
input.push_str(&format!("├── pkg{idx:03} v1.0.{idx}\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("project v1.0.0"));
assert!(out.text.contains("pkg000"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn compacts_poetry_show_tree_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("poetry", Some("show"), "poetry show --tree", &cfg);
let mut input = String::from("requests 2.32.0 Python HTTP for Humans.\n");
for idx in 0..90 {
input.push_str(&format!("├── dep{idx:03} 1.0.{idx}\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("requests 2.32.0"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn passes_through_uv_pip_freeze_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("uv", Some("pip"), "uv pip freeze", &cfg);
let mut input = String::new();
for idx in 0..90 {
input.push_str(&format!("pkg{idx:03}==1.0.{idx}\n"));
}
let out = filter(&context, &input, 0);
assert!(!out.changed);
assert_eq!(out.text, input);
assert!(out.text.contains("pkg089==1.0.89"));
assert!(!out.text.starts_with("package tree/list:"));
}
#[test]
fn compacts_uv_export_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("uv", Some("export"), "uv export -f requirements-txt", &cfg);
let mut input = String::from("# generated by uv\n");
for idx in 0..90 {
input.push_str(&format!("pkg{idx:03}==1.0.{idx}\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("pkg000==1.0.0"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn compacts_poetry_export_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("poetry", Some("export"), "poetry export -f requirements.txt", &cfg);
let mut input = String::from("# generated by poetry\n");
for idx in 0..90 {
input.push_str(&format!("dep{idx:03}==2.0.{idx}\n"));
}
let out = filter(&context, &input, 0);
assert!(out.text.starts_with("package tree/list: 91 entries\n"));
assert!(out.text.contains("dep000==2.0.0"));
assert!(out.text.contains("… 11 package entries omitted …"));
}
#[test]
fn compacts_uv_lock_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("uv", Some("lock"), "uv lock", &cfg);
let input =
"Resolved 42 packages in 7ms\nDownloading requests\nUpdated lockfile at uv.lock\n";
let out = filter(&context, input, 0);
assert!(out.text.contains("Resolved 42 packages in 7ms"));
assert!(out.text.contains("Updated lockfile at uv.lock"));
assert!(!out.text.contains("Downloading requests"));
}
#[test]
fn compacts_poetry_lock_output() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("poetry", Some("lock"), "poetry lock", &cfg);
let input =
"Resolving dependencies...\nInstalling dependencies from lock file\nWriting lock file\n";
let out = filter(&context, input, 0);
assert!(out.text.contains("Installing dependencies from lock file"));
assert!(out.text.contains("Writing lock file"));
}
}
crates/pi-shell/src/minimizer/filters/python.rs
+81 -5
View File
@@ -1,4 +1,17 @@
//! Python test, type-check, and lint output filters.
//!
//! Ported from rtk-ai/rtk@878af7de99e0ba71da2e8fd996f6b52a1836e06c
//! Path: `src/cmds/python/pytest_cmd.rs`
//! License: MIT (compatible with workspace MIT). See `ATTRIBUTION-RTK.md` at
//! the `pi-shell` crate root.
//!
//! The pytest state machine (`filter_pytest`, `pytest_success`,
//! `is_pytest_*`, `looks_like_pytest_summary_part`) adapts the
//! `build_pytest_summary` algorithm from RTK at the pinned SHA above:
//! preserve failures, errors, and the final summary line; strip header
//! framing, progress dots, and verbose PASSED rows. Unknown-state lines
//! fall through unchanged (RTK's defensive default), so xdist `[gwN]`
//! prefixes and custom reporters never cause data loss.
use super::lint;
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
@@ -12,6 +25,12 @@ pub fn supports(program: &str, subcommand: Option<&str>) -> bool {
}
pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
// Kill-switch parity (M2): when `legacy_filters_active`, fall back to
// the pre-PR passthrough so callers can rollback an RTK-port regression
// without recompile.
if ctx.config.legacy_filters_active() {
return MinimizerOutput::passthrough(input);
}
let tool = python_tool(ctx.program, ctx.subcommand);
let cleaned = primitives::strip_ansi(input);
let text = match tool {
@@ -50,11 +69,16 @@ fn filter_pytest(input: &str, exit_code: i32) -> String {
for line in input.lines() {
let trimmed = line.trim();
if is_pytest_summary_header(trimmed) || is_pytest_summary_line(trimmed) {
if is_pytest_summary_header(trimmed) {
in_failure = false;
push_line(&mut out, line);
continue;
}
if is_pytest_summary_line(trimmed) {
in_failure = false;
push_pytest_summary_line(&mut out, trimmed);
continue;
}
if starts_pytest_failure(trimmed) {
in_failure = true;
@@ -91,11 +115,16 @@ fn pytest_success(input: &str) -> String {
for line in input.lines() {
let trimmed = line.trim();
if is_pytest_summary_line(trimmed) || is_pytest_summary_header(trimmed) {
if is_pytest_summary_header(trimmed) {
push_line(&mut summary, line);
push_line(&mut out, line);
continue;
}
if is_pytest_summary_line(trimmed) {
push_pytest_summary_line(&mut summary, trimmed);
push_pytest_summary_line(&mut out, trimmed);
continue;
}
if is_pytest_pass_noise(trimmed) {
continue;
}
@@ -162,6 +191,14 @@ fn looks_like_pytest_summary_part(part: &str) -> bool {
false
}
fn compact_pytest_summary_line(trimmed: &str) -> &str {
if trimmed.starts_with('=') {
trimmed.trim_matches('=').trim()
} else {
trimmed
}
}
fn is_pytest_section_delimiter(trimmed: &str) -> bool {
trimmed.len() >= 6
&& trimmed
@@ -180,6 +217,7 @@ fn is_pytest_pass_noise(trimmed: &str) -> bool {
|| trimmed.starts_with("platform ")
|| trimmed.starts_with("cachedir:")
|| is_pytest_verbose_pass_line(trimmed)
|| is_pytest_progress_line(trimmed)
|| trimmed
.chars()
.all(|ch| matches!(ch, '.' | 's' | 'S' | 'x' | 'X' | 'f' | 'F' | 'E'))
@@ -193,6 +231,19 @@ fn is_pytest_verbose_pass_line(trimmed: &str) -> bool {
parts.any(|part| matches!(part, "PASSED" | "SKIPPED" | "XPASS" | "XFAIL"))
}
fn is_pytest_progress_line(trimmed: &str) -> bool {
let Some((path, statuses)) = trimmed.split_once(char::is_whitespace) else {
return false;
};
std::path::Path::new(path)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("py"))
&& statuses
.trim()
.chars()
.all(|ch| matches!(ch, '.' | 's' | 'S' | 'x' | 'X' | 'f' | 'F' | 'E'))
}
fn is_ruff_format(ctx: &MinimizerCtx<'_>) -> bool {
ctx.subcommand == Some("format") || ctx.command.split_whitespace().any(|part| part == "format")
}
@@ -236,6 +287,12 @@ fn push_line(out: &mut String, line: &str) {
out.push('\n');
}
fn push_pytest_summary_line(out: &mut String, trimmed: &str) {
out.push_str("pytest: ");
out.push_str(compact_pytest_summary_line(trimmed));
out.push('\n');
}
fn has_content(text: &str) -> bool {
text.lines().any(|line| !line.trim().is_empty())
}
@@ -269,7 +326,7 @@ mod tests {
assert!(!out.contains("test session starts"));
assert!(out.contains("test_adds_badly"));
assert!(out.contains("AssertionError"));
assert!(out.contains("1 failed, 1 passed"));
assert!(out.contains("pytest: 1 failed, 1 passed"));
}
#[test]
@@ -298,7 +355,7 @@ mod tests {
let out = filter_pytest(input, 1);
assert!(!out.contains("................................................................"));
assert!(out.contains("5 failed, 1698 passed, 2 skipped in 108.89s"));
assert!(out.contains("pytest: 5 failed, 1698 passed, 2 skipped in 108.89s"));
}
#[test]
@@ -309,7 +366,26 @@ mod tests {
PASSED [ 3%]\ntest_utils.py::TestListOps::test_flatten PASSED \
[100%]\n\n====== 33 passed in 0.05s ======\n";
let out = filter_pytest(input, 0);
assert_eq!(out, "====== 33 passed in 0.05s ======\n");
assert_eq!(out, "pytest: 33 passed in 0.05s\n");
}
#[test]
fn direct_pytest_success_routes_to_compact_summary() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = MinimizerCtx {
program: "pytest",
subcommand: None,
command: "pytest",
config: &cfg,
};
let out = filter(
&context,
"===== test session starts =====\ncollected 2 items\n\ntests/test_a.py ..\n===== 2 \
passed in 0.01s =====\n",
0,
);
assert_eq!(out.text, "pytest: 2 passed in 0.01s\n");
}
#[test]
crates/pi-shell/src/minimizer/filters/rust_tools.rs
+170
View File
@@ -0,0 +1,170 @@
//! Rust toolchain filters that are not `cargo` subcommands (Tier 3a).
//!
//! Today this module hosts the `rustfmt` filter — real-data evidence (~6
//! invocations / 7d, 38 KB average, ~0.23 MB total) showed rustfmt landing
//! in the minimizer's `unknown` bucket. The filter groups diff-style
//! output by file and elides per-file unified-diff chunks for `--check`
//! mode while letting silent runs (no diffs / formatter no-op) pass
//! through unchanged.
use std::fmt::Write;
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(program: &str, _subcommand: Option<&str>) -> bool {
matches!(program, "rustfmt")
}
pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerOutput {
// Kill-switch parity (M2): legacy_filters_active=true skips this
// filter so callers can rollback without recompile.
if ctx.config.legacy_filters_active() {
return MinimizerOutput::passthrough(input);
}
let cleaned = primitives::strip_ansi(input);
let text = match ctx.program {
"rustfmt" => condense_rustfmt(&cleaned, exit_code),
_ => cleaned,
};
if text == input {
MinimizerOutput::passthrough(input)
} else {
MinimizerOutput::transformed(text, input.len())
}
}
/// Condense `rustfmt`/`rustfmt --check` output.
///
/// Two main shapes are handled:
///
/// - **Check mode** emits `Diff in <path> at line <N>:` headers followed by
/// per-hunk `+`/`-` lines. We collect the set of affected files, print a
/// one-line header (`N files reformatted (M with diffs):`), the first 3 file
/// paths, and elide every diff body — the agent rarely needs the full diff
/// inline; the artifact reference carries the original.
/// - **Silent runs** (rustfmt formatted in place, no `--check`) emit no stdout.
/// The empty buffer passes through; no transformation.
///
/// On compile errors / panics rustfmt prints to stderr in tens-of-lines
/// form, well under the head/tail cap below — we keep it as-is.
fn condense_rustfmt(input: &str, exit_code: i32) -> String {
if input.trim().is_empty() {
return input.to_string();
}
let files: Vec<&str> = collect_diff_files(input);
if files.is_empty() {
// No `Diff in ` markers — likely a panic / parse error / usage
// message. Cap with the standard error head/tail budget.
if exit_code != 0 {
return primitives::head_tail_lines(input, 80, 40);
}
return input.to_string();
}
let mut out = String::new();
let unique: Vec<&&str> = {
let mut seen = std::collections::BTreeSet::new();
files.iter().filter(|f| seen.insert(**f)).collect()
};
let total = unique.len();
let _ = writeln!(out, "{total} files reformatted:");
for file in unique.iter().take(3) {
out.push_str(" ");
out.push_str(file);
out.push('\n');
}
if total > 3 {
let _ = writeln!(out, " … {} more", total - 3);
}
out
}
fn collect_diff_files(input: &str) -> Vec<&str> {
let mut files = Vec::new();
for line in input.lines() {
if let Some(rest) = line.strip_prefix("Diff in ") {
// rest looks like: `<path> at line <N>:`
let path = rest
.split(" at line ")
.next()
.unwrap_or(rest)
.trim_end_matches(':');
files.push(path);
}
}
files
}
#[cfg(test)]
mod tests {
use super::*;
use crate::minimizer::MinimizerConfig;
fn ctx<'a>(program: &'a str, command: &'a str, config: &'a MinimizerConfig) -> MinimizerCtx<'a> {
MinimizerCtx { program, subcommand: None, command, config }
}
#[test]
fn rustfmt_diff_output_compacts() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let mut input = String::new();
for i in 0..50 {
input.push_str(&format!("Diff in src/file_{i}.rs at line 10:\n"));
for _ in 0..8 {
input.push_str("- old line\n");
input.push_str("+ new line\n");
}
}
let context = ctx("rustfmt", "rustfmt --check src/", &cfg);
let out = filter(&context, &input, 1);
assert!(out.changed);
assert!(out.text.contains("50 files reformatted"));
assert!(out.text.contains("src/file_0.rs"));
assert!(out.text.contains("… 47 more"));
// Diff bodies must be elided.
assert!(!out.text.contains("old line"));
// Savings ratio ≥ 0.7
let saved_ratio = 1.0 - (out.text.len() as f64 / input.len() as f64);
assert!(saved_ratio >= 0.7, "expected ≥0.7 savings, got {saved_ratio}");
}
#[test]
fn rustfmt_silent_output_passthrough() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("rustfmt", "rustfmt src/lib.rs", &cfg);
let out = filter(&context, "", 0);
assert!(!out.changed);
assert_eq!(out.text, "");
}
#[test]
fn rustfmt_error_output_capped() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let context = ctx("rustfmt", "rustfmt --check missing.rs", &cfg);
// Simulate a long usage/error dump with no `Diff in ` headers.
let mut input = String::new();
for i in 0..400 {
input.push_str(&format!("error: usage line {i}\n"));
}
let out = filter(&context, &input, 1);
assert!(out.changed);
// head_tail_lines(input, 80, 40) keeps 120 lines + marker.
assert!(out.text.contains("lines omitted"));
}
#[test]
fn rustfmt_legacy_filters_active_passes_through() {
// Kill-switch parity (M2).
let mut cfg = MinimizerConfig::default();
cfg.enabled = true;
cfg.legacy_filters_active = true;
let context = ctx("rustfmt", "rustfmt --check src/", &cfg);
let input = "Diff in src/a.rs at line 1:\n-old\n+new\n";
let out = filter(&context, input, 1);
assert!(!out.changed);
assert_eq!(out.text, input);
}
}
crates/pi-shell/src/minimizer/filters/system.rs
+403 -22
View File
@@ -2,6 +2,7 @@
use std::collections::HashMap;
use super::git;
use crate::minimizer::{MinimizerCtx, MinimizerOutput, primitives};
pub fn supports(program: &str) -> bool {
@@ -26,13 +27,23 @@ pub fn filter(ctx: &MinimizerCtx<'_>, input: &str, exit_code: i32) -> MinimizerO
let cleaned = primitives::strip_ansi(input);
let command = ctx.program;
let text = match command {
"env" => compact_env(&cleaned),
"env" => {
if ctx
.command
.split_whitespace()
.any(|t| t == "-0" || t == "--null")
{
cleaned
} else {
compact_env(&cleaned)
}
},
"log" => compact_log(&cleaned),
"deps" => compact_dependency_output(&cleaned),
"summary" => compact_summary_output(&cleaned, exit_code),
"err" => cleaned,
"err" => compact_err_output(&cleaned),
"test" => compact_test_output(&cleaned),
"diff" => cleaned,
"diff" => git::compact_diff_output(&cleaned),
"format" => compact_format_output(&cleaned),
"pipe" => compact_pipe_like_output(&cleaned, exit_code),
"ps" => compact_ps_output(&cleaned),
@@ -215,17 +226,13 @@ struct LogLine {
fn normalize_log_line(line: &str) -> String {
let without_timestamp = strip_leading_timestamp(line.trim());
let mut out = String::new();
let mut digits = String::new();
for ch in without_timestamp.chars() {
if ch.is_ascii_digit() {
digits.push(ch);
continue;
for token in without_timestamp.split_whitespace() {
if !out.is_empty() {
out.push(' ');
}
flush_digits(&mut out, &mut digits);
out.push(ch);
push_normalized_token(&mut out, token);
}
flush_digits(&mut out, &mut digits);
out.split_whitespace().collect::<Vec<_>>().join(" ")
out
}
fn strip_leading_timestamp(line: &str) -> &str {
@@ -243,6 +250,54 @@ fn strip_leading_timestamp(line: &str) -> &str {
line
}
fn push_normalized_token(out: &mut String, token: &str) {
let core = token.trim_matches(|ch: char| ch.is_ascii_punctuation() && ch != '/' && ch != '.');
if is_uuid_like(core) {
out.push_str("<uuid>");
return;
}
if is_hex_like(core) {
out.push_str("<hex>");
return;
}
if is_path_like(core) {
out.push_str("<path>");
return;
}
let mut digits = String::new();
for ch in token.chars() {
if ch.is_ascii_digit() {
digits.push(ch);
continue;
}
flush_digits(out, &mut digits);
out.push(ch);
}
flush_digits(out, &mut digits);
}
fn is_uuid_like(token: &str) -> bool {
token.len() == 36
&& token.bytes().enumerate().all(|(idx, byte)| {
if matches!(idx, 8 | 13 | 18 | 23) {
byte == b'-'
} else {
byte.is_ascii_hexdigit()
}
})
}
fn is_hex_like(token: &str) -> bool {
let token = token.strip_prefix("0x").unwrap_or(token);
token.len() >= 8 && token.bytes().all(|byte| byte.is_ascii_hexdigit())
}
fn is_path_like(token: &str) -> bool {
(token.starts_with('/') || token.starts_with("./") || token.starts_with("../"))
&& token.len() > 1
}
fn flush_digits(out: &mut String, digits: &mut String) {
if digits.is_empty() {
return;
@@ -324,15 +379,265 @@ fn compact_summary_output(input: &str, exit_code: i32) -> String {
out
}
fn compact_err_output(input: &str) -> String {
compact_failure_output(
input,
2,
12,
is_err_signal_line,
is_err_summary_line,
is_err_noise,
is_err_relevant_line,
)
}
fn compact_test_output(input: &str) -> String {
compact_failure_output(
input,
1,
12,
is_test_signal_line,
is_test_summary_line,
is_test_noise,
is_test_relevant_line,
)
}
fn compact_failure_output(
input: &str,
keep_before: usize,
keep_after: usize,
is_signal_line: fn(&str) -> bool,
is_summary_line: fn(&str) -> bool,
is_noise_line: fn(&str) -> bool,
is_relevant_line: fn(&str) -> bool,
) -> String {
let lines: Vec<&str> = input.lines().collect();
if lines.len() <= 120 {
return primitives::dedup_consecutive_lines(input);
if lines.is_empty() {
return input.to_string();
}
let mut out = format!("test output: {} lines\n", lines.len());
push_important_lines(&mut out, input, 80);
out.push_str(&primitives::head_tail_lines(input, 35, 35));
out
let mut keep = vec![false; lines.len()];
let mut saw_relevant = false;
for (idx, line) in lines.iter().enumerate() {
let trimmed = line.trim_start();
if is_relevant_line(trimmed) {
saw_relevant = true;
}
if is_summary_line(trimmed) {
keep[idx] = true;
continue;
}
if is_signal_line(trimmed) {
let start = idx.saturating_sub(keep_before);
let end = idx
.saturating_add(keep_after)
.min(lines.len().saturating_sub(1));
for slot in keep.iter_mut().take(end + 1).skip(start) {
*slot = true;
}
}
}
if !saw_relevant {
return input.to_string();
}
let mut out = String::new();
for (idx, line) in lines.iter().enumerate() {
if !keep[idx] {
continue;
}
let trimmed = line.trim_start();
if is_noise_line(trimmed) {
continue;
}
out.push_str(line);
out.push('\n');
}
primitives::dedup_consecutive_lines(&out)
}
fn is_err_signal_line(trimmed: &str) -> bool {
let lower = trimmed.to_ascii_lowercase();
lower.starts_with("error:")
|| lower.starts_with("fatal:")
|| lower.starts_with("failed:")
|| lower.starts_with("panic:")
|| lower.starts_with("exception:")
|| lower.starts_with("traceback ")
|| lower.starts_with("assertionerror")
|| lower.starts_with("timeouterror")
|| lower.starts_with("caused by:")
|| lower.starts_with("warning:")
|| lower.starts_with("warn:")
|| lower.contains(": error:")
|| lower.contains(": warning:")
|| lower.contains(" fatal error")
|| lower.contains(" failed")
|| lower.contains(" panic")
|| lower.contains(" exception")
}
fn is_err_summary_line(trimmed: &str) -> bool {
let lower = trimmed.to_ascii_lowercase();
lower.starts_with("build failed")
|| lower.starts_with("failures!")
|| lower.starts_with("failed")
|| lower.starts_with("errors:")
|| lower.starts_with("warnings:")
|| lower.starts_with("play recap")
|| lower.starts_with("summary")
|| lower.starts_with("test result")
|| lower.starts_with("test files")
|| lower.starts_with("tests:")
|| is_count_summary(trimmed)
}
fn is_err_noise(trimmed: &str) -> bool {
let lower = trimmed.to_ascii_lowercase();
lower.starts_with("compiling ")
|| lower.starts_with("building ")
|| lower.starts_with("checking ")
|| lower.starts_with("running ")
|| lower.starts_with("executing ")
|| lower.starts_with("fetching ")
|| lower.starts_with("resolving ")
|| lower.starts_with("downloading ")
|| lower.starts_with("installing ")
|| lower.starts_with("finished ")
|| lower.starts_with("done ")
|| lower.starts_with("pass ")
|| lower.starts_with("✓")
|| lower.starts_with("✔")
|| lower.starts_with("√")
|| lower.starts_with("○")
|| lower.starts_with("ok ")
|| lower.contains(" ... ok")
}
fn is_test_signal_line(trimmed: &str) -> bool {
let lower = trimmed.to_ascii_lowercase();
trimmed.starts_with("FAIL ")
|| trimmed.starts_with("FAILURES")
|| trimmed.starts_with("Failed Tests")
|| trimmed.starts_with("● ")
|| trimmed.starts_with("✕")
|| trimmed.starts_with("×")
|| trimmed.starts_with("✗")
|| trimmed.starts_with("❯")
|| lower.starts_with("error:")
|| lower.starts_with("assertionerror")
|| lower.starts_with("timeouterror")
|| lower.starts_with("panic:")
|| lower.starts_with("failed ")
}
fn is_test_summary_line(trimmed: &str) -> bool {
let lower = trimmed.to_ascii_lowercase();
trimmed.starts_with("Test Suites:")
|| trimmed.starts_with("Test Suites")
|| trimmed.starts_with("Tests:")
|| trimmed.starts_with("Tests")
|| trimmed.starts_with("Test Files")
|| trimmed.starts_with("Snapshots:")
|| trimmed.starts_with("Snapshots")
|| trimmed.starts_with("Time:")
|| trimmed.starts_with("Duration")
|| trimmed.starts_with("Start at")
|| trimmed.starts_with("Ran all test suites")
|| trimmed.starts_with("Ran ")
|| trimmed.starts_with("Failed Tests")
|| trimmed.starts_with("FAILURES")
|| trimmed.starts_with("Summary")
|| lower.starts_with("build failed")
|| lower.starts_with("test run failed")
|| lower.starts_with("test result")
|| is_count_summary(trimmed)
}
fn is_test_noise(trimmed: &str) -> bool {
let lower = trimmed.to_ascii_lowercase();
lower.starts_with("pass ")
|| trimmed.starts_with("✓")
|| trimmed.starts_with("✔")
|| trimmed.starts_with("√")
|| trimmed.starts_with("○")
|| lower.starts_with("running ")
|| lower.starts_with("run ")
|| lower.starts_with("dev ")
|| lower.starts_with("ok ")
|| lower.contains(" ... ok")
}
fn is_err_relevant_line(trimmed: &str) -> bool {
is_err_signal_line(trimmed) || is_err_summary_line(trimmed)
}
fn is_test_relevant_line(trimmed: &str) -> bool {
is_test_signal_line(trimmed) || is_test_summary_line(trimmed) || is_test_noise(trimmed)
}
fn is_count_summary(trimmed: &str) -> bool {
let mut parts = trimmed.split_whitespace();
let Some(count) = parts.next() else {
return false;
};
if !count.chars().all(|ch| ch.is_ascii_digit()) {
return false;
}
let Some(kind) = parts
.next()
.map(|word| word.trim_matches(|ch: char| ch.is_ascii_punctuation()))
else {
return false;
};
if matches!(
kind,
"failed"
| "passed"
| "skipped"
| "flaky"
| "pass"
| "fail"
| "error"
| "errors"
| "warning"
| "warnings"
| "information"
| "informations"
) {
return true;
}
if kind == "of" {
let Some(total) = parts.next() else {
return false;
};
if !total.chars().all(|ch| ch.is_ascii_digit()) {
return false;
}
return parts
.next()
.map(|word| word.trim_matches(|ch: char| ch.is_ascii_punctuation()))
.is_some_and(|kind| {
matches!(
kind,
"failed"
| "passed" | "skipped"
| "flaky" | "pass"
| "fail" | "error"
| "errors" | "warning"
| "warnings" | "information"
| "informations"
)
});
}
false
}
fn push_important_lines(out: &mut String, input: &str, max: usize) {
@@ -613,6 +918,18 @@ mod tests {
assert!(out.text.contains("(×2)"));
}
#[test]
fn log_dedups_normalized_uuid_hex_and_paths() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("log", &cfg);
let input = "2026-01-01T10:00:00 ERROR request 550e8400-e29b-41d4-a716-446655440000 file \
/tmp/a.rs hash deadbeef failed\n2026-01-01T10:00:01 ERROR request \
123e4567-e89b-12d3-a456-426614174000 file /tmp/b.rs hash cafebabe failed\n";
let out = filter(&ctx, input, 1);
assert!(out.text.contains("2 lines, 1 unique"));
assert!(out.text.contains("(×2)"));
}
#[test]
fn env_masks_secrets_and_compacts_long_values() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
@@ -625,11 +942,39 @@ mod tests {
}
#[test]
fn diff_output_passthrough_is_lossless() {
fn err_output_keeps_diagnostics_and_context() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("err", &cfg);
let input = "\
Compiling app v0.1.0
running 1 test
test pass ... ok
src/main.rs:10:5: error: cannot find value `foo` in this scope
|
10 | foo();
| ^^^
note: required by a bound in `bar`
warning: unused import: `baz`
";
let out = filter(&ctx, input, 1);
assert!(out.changed);
assert!(!out.text.contains("Compiling app v0.1.0"));
assert!(!out.text.contains("running 1 test"));
assert!(!out.text.contains("test pass ... ok"));
assert!(
out.text
.contains("src/main.rs:10:5: error: cannot find value `foo` in this scope")
);
assert!(out.text.contains("10 | foo();"));
assert!(out.text.contains("note: required by a bound in `bar`"));
assert!(out.text.contains("warning: unused import: `baz`"));
}
#[test]
fn diff_output_reuses_unified_diff_compaction() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("diff", &cfg);
let mut input =
String::from("diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1,140 +1,140 @@\n");
let mut input = String::from("--- a/a.rs\n+++ b/a.rs\n@@ -1,140 +1,140 @@\n");
for idx in 0..140 {
input.push_str("-old ");
input.push_str(&idx.to_string());
@@ -638,7 +983,43 @@ mod tests {
input.push('\n');
}
let out = filter(&ctx, &input, 0);
assert_eq!(out.text, input);
assert!(out.changed);
assert!(out.text.contains("a.rs | 280"));
assert!(
out.text
.contains("1 file changed, 140 insertions(+), 140 deletions(-)")
);
assert!(out.text.contains("--- Changes ---"));
assert!(out.text.contains("-old 0"));
assert!(out.text.contains("+new 0"));
assert_ne!(out.text, input);
}
#[test]
fn test_output_drops_pass_chatter_and_keeps_failure_summary() {
let cfg = MinimizerConfig { enabled: true, ..Default::default() };
let ctx = ctx("test", &cfg);
let input = "\
PASS src/pass.test.ts
✓ src/ok.test.ts (3ms)
FAIL src/fail.test.ts
suite > breaks
Error: expected 1 to equal 2
at src/fail.test.ts:12:3
Test Files 1 failed | 1 passed (2)
Tests 1 failed | 3 passed (4)
Time 0.42s
";
let out = filter(&ctx, input, 1);
assert!(out.changed);
assert!(!out.text.contains("PASS src/pass.test.ts"));
assert!(!out.text.contains("✓ src/ok.test.ts"));
assert!(out.text.contains("FAIL src/fail.test.ts"));
assert!(out.text.contains("Error: expected 1 to equal 2"));
assert!(out.text.contains("Test Files 1 failed | 1 passed (2)"));
assert!(out.text.contains("Tests 1 failed | 3 passed (4)"));
assert!(out.text.contains("Time 0.42s"));
}
#[test]
crates/pi-shell/src/minimizer/plan.rs
+287 -41
View File
@@ -11,9 +11,13 @@
//! regardless of what `bar` is. A user piping through `awk`, `jq`, `rg`, or
//! any other consumer is almost certainly parsing the output; rewriting it
//! would be a correctness bug. The engine falls back to passthrough.
//! - **Compound commands are opaque.** `a && b`, `a ; b`, and `a || b` cannot
//! be minimized as one combined buffer without risking semantic corruption,
//! so they are left unchanged.
//! - **Safe chains are segmented, not rewritten whole.** Top-level simple
//! commands joined only by `&&` and `;` may be split into `ChainSegment`s for
//! the segmented engine path, but the whole-buffer minimizer still treats the
//! combined chain as opaque.
//! - **Other compound commands are opaque.** `a || b`, background jobs, and
//! compound shell syntax such as subshells or function definitions are left
//! unchanged.
//! - **Single simple commands** are safe for the whole-buffer path; the engine
//! dispatches them through `detect.rs` as before.
//!
@@ -22,9 +26,21 @@
use brush_parser::{
ParserOptions, SourceInfo,
ast::{AndOrList, Command, CompoundListItem, Pipeline, Program, SeparatorOperator},
ast::{
AndOr, Command, CommandPrefixOrSuffixItem, CompoundListItem, IoFileRedirectTarget,
IoRedirect, Pipeline, Program, SeparatorOperator, Word,
},
};
/// One segment of a safe `&&` / `;` chain.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChainSegment {
pub command: String,
pub program: String,
pub run_if_previous_succeeded: bool,
pub suppress_errexit: bool,
}
/// Outcome of analyzing a raw command string.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CommandPlan {
@@ -35,9 +51,12 @@ pub enum CommandPlan {
/// NOT identify upstream / downstream programs here — any pipe defeats
/// safe minimization for this engine.
Piped,
/// The command has multiple segments joined by `&&`, `||`, `;`, or `&`.
/// This shape is left unchanged; the minimizer only rewrites whole simple
/// command output.
/// Top-level simple commands joined by `&&` and/or `;`. These can be
/// minimized segment-by-segment, but not as one combined buffer.
Chain { segments: Vec<ChainSegment> },
/// The command has multiple segments joined by `||`, `&`, or other
/// unsupported shell syntax. This shape is left unchanged; the minimizer
/// only rewrites whole simple command output.
Compound,
/// Parse failed, a compound shell construct (for loops, subshells, etc.)
/// was encountered, or the command was empty.
@@ -52,9 +71,9 @@ pub fn analyze(command: &str) -> CommandPlan {
}
let options = ParserOptions::default();
let source = SourceInfo::default();
let source_info = SourceInfo::default();
let reader = std::io::Cursor::new(command.as_bytes());
let mut parser = brush_parser::Parser::new(reader, &options, &source);
let mut parser = brush_parser::Parser::new(reader, &options, &source_info);
let Ok(program) = parser.parse_program() else {
return CommandPlan::Unsupported;
@@ -64,6 +83,10 @@ pub fn analyze(command: &str) -> CommandPlan {
}
fn classify(program: &Program) -> CommandPlan {
if let Some(chain) = classify_chain(program) {
return chain;
}
// Count separator-separated top-level items across all complete_commands.
let items: Vec<&CompoundListItem> = program
.complete_commands
@@ -96,7 +119,143 @@ fn classify(program: &Program) -> CommandPlan {
}
// Only a single pipeline at this point.
classify_pipeline(&and_or.first).unwrap_or_else(|| classify_andorlist(and_or))
classify_pipeline(&and_or.first).unwrap_or(CommandPlan::Unsupported)
}
fn classify_chain(program: &Program) -> Option<CommandPlan> {
let items: Vec<&CompoundListItem> = program
.complete_commands
.iter()
.flat_map(|cl| cl.0.iter())
.collect();
if items.is_empty() {
return None;
}
let mut segments = Vec::new();
let mut run_if_previous_succeeded = false;
for (item_index, item) in items.iter().enumerate() {
if matches!(item.1, SeparatorOperator::Async) {
return None;
}
let is_last_item = item_index + 1 == items.len();
let mut pipeline = &item.0.first;
let mut additional = item.0.additional.iter().peekable();
loop {
let (command, program) = simple_segment(pipeline)?;
let suppress_errexit = additional
.peek()
.is_some_and(|and_or| matches!(and_or, AndOr::And(_)));
segments.push(ChainSegment {
command,
program,
run_if_previous_succeeded,
suppress_errexit,
});
let Some(and_or) = additional.next() else {
run_if_previous_succeeded = false;
break;
};
match and_or {
AndOr::And(next_pipeline) => {
run_if_previous_succeeded = true;
pipeline = next_pipeline;
},
AndOr::Or(_) => return None,
}
}
if !is_last_item {
run_if_previous_succeeded = false;
}
}
(segments.len() >= 2).then_some(CommandPlan::Chain { segments })
}
fn word_has_command_substitution(word: &Word) -> bool {
word.value.contains("$(") || word.value.contains('`')
}
fn command_prefix_or_suffix_item_is_safe(item: &CommandPrefixOrSuffixItem) -> bool {
match item {
CommandPrefixOrSuffixItem::IoRedirect(io) => io_redirect_is_safe(io),
CommandPrefixOrSuffixItem::Word(word) => !word_has_command_substitution(word),
CommandPrefixOrSuffixItem::AssignmentWord(_, word) => !word_has_command_substitution(word),
CommandPrefixOrSuffixItem::ProcessSubstitution(..) => false,
}
}
fn io_redirect_is_safe(io: &IoRedirect) -> bool {
match io {
IoRedirect::File(_, _, target) => match target {
IoFileRedirectTarget::Filename(word) | IoFileRedirectTarget::Duplicate(word) => {
!word_has_command_substitution(word)
},
IoFileRedirectTarget::Fd(_) => true,
IoFileRedirectTarget::ProcessSubstitution(..) => false,
},
IoRedirect::HereDocument(_, here_doc) => {
!word_has_command_substitution(&here_doc.here_end)
&& !word_has_command_substitution(&here_doc.doc)
},
IoRedirect::HereString(_, word) => !word_has_command_substitution(word),
IoRedirect::OutputAndError(word, _) => !word_has_command_substitution(word),
}
}
fn simple_segment(pipeline: &Pipeline) -> Option<(String, String)> {
if pipeline.timed.is_some() || pipeline.bang || pipeline.seq.is_empty() {
return None;
}
// For multi-stage pipes inside a chain segment, identify the segment by its
// first stage's program. The downstream per-segment minimizer::apply will
// detect the pipeline at runtime via plan::CommandPlan::Piped and pass it
// through unchanged — so a piped segment is safely captured but never
// rewritten. This keeps the chain decomposable when even one inner stage
// uses a pipe (e.g. `ls | head -10 && git status`).
let first = pipeline.seq.first()?;
match first {
Command::Simple(simple) => {
if simple.prefix.as_ref().is_some_and(|prefix| {
prefix
.0
.iter()
.any(|item| !command_prefix_or_suffix_item_is_safe(item))
}) {
return None;
}
if simple.suffix.as_ref().is_some_and(|suffix| {
suffix
.0
.iter()
.any(|item| !command_prefix_or_suffix_item_is_safe(item))
}) {
return None;
}
let program_word = simple.word_or_name.as_ref()?;
if word_has_command_substitution(program_word) {
return None;
}
let program = program_word.to_string();
if program.trim().is_empty() {
return None;
}
Some((pipeline.to_string(), program))
},
// Compound shell syntax (if / for / while / subshell / { ... }) is
// not something the minimizer should touch.
Command::Compound(..) | Command::Function(_) | Command::ExtendedTest(..) => None,
}
}
fn classify_pipeline(pipeline: &Pipeline) -> Option<CommandPlan> {
@@ -115,16 +274,12 @@ fn classify_pipeline(pipeline: &Pipeline) -> Option<CommandPlan> {
},
// Compound shell syntax (if / for / while / subshell / { ... }) is
// not something the minimizer should touch.
Command::Compound(..) | Command::Function(_) | Command::ExtendedTest(_) => {
Command::Compound(..) | Command::Function(_) | Command::ExtendedTest(..) => {
Some(CommandPlan::Compound)
},
}
}
const fn classify_andorlist(_and_or: &AndOrList) -> CommandPlan {
CommandPlan::Unsupported
}
#[cfg(test)]
mod tests {
use super::*;
@@ -136,6 +291,20 @@ mod tests {
}
}
fn chain_of(plan: CommandPlan) -> Option<Vec<ChainSegment>> {
match plan {
CommandPlan::Chain { segments } => Some(segments),
_ => None,
}
}
fn assert_not_chain(command: &str) {
assert!(
!matches!(analyze(command), CommandPlan::Chain { .. }),
"{command:?} unexpectedly classified as Chain"
);
}
#[test]
fn single_simple_command() {
let plan = analyze("git status --short");
@@ -150,25 +319,114 @@ mod tests {
}
#[test]
fn pipe_is_piped() {
assert_eq!(analyze("git status | cat"), CommandPlan::Piped);
assert_eq!(analyze("ls -la | awk '{print $1}'"), CommandPlan::Piped);
fn safe_and_chain_is_segmented() {
let plan = analyze("git diff --stat && git diff --name-only");
assert_eq!(
chain_of(plan),
Some(vec![
ChainSegment {
command: "git diff --stat".to_string(),
program: "git".to_string(),
run_if_previous_succeeded: false,
suppress_errexit: true,
},
ChainSegment {
command: "git diff --name-only".to_string(),
program: "git".to_string(),
run_if_previous_succeeded: true,
suppress_errexit: false,
},
])
);
}
#[test]
fn and_or_is_compound() {
assert_eq!(analyze("cd foo && cargo test"), CommandPlan::Compound);
fn safe_sequence_chain_is_segmented() {
let plan = analyze("git status ; bun test");
assert_eq!(
chain_of(plan),
Some(vec![
ChainSegment {
command: "git status".to_string(),
program: "git".to_string(),
run_if_previous_succeeded: false,
suppress_errexit: false,
},
ChainSegment {
command: "bun test".to_string(),
program: "bun".to_string(),
run_if_previous_succeeded: false,
suppress_errexit: false,
},
])
);
}
#[test]
fn mixed_chain_is_segmented() {
let plan = analyze("false && echo no ; echo yes");
assert_eq!(
chain_of(plan),
Some(vec![
ChainSegment {
command: "false".to_string(),
program: "false".to_string(),
run_if_previous_succeeded: false,
suppress_errexit: true,
},
ChainSegment {
command: "echo no".to_string(),
program: "echo".to_string(),
run_if_previous_succeeded: true,
suppress_errexit: false,
},
ChainSegment {
command: "echo yes".to_string(),
program: "echo".to_string(),
run_if_previous_succeeded: false,
suppress_errexit: false,
},
])
);
}
#[test]
fn chain_with_piped_segment_is_segmented() {
// A chain that contains a piped segment (`ls | head -5`) must still be
// classified as Chain so the segmented runner can decompose it. The
// piped segment is identified by its first stage's program; the
// per-segment minimizer::apply will treat that segment as Piped at
// runtime and pass it through unchanged.
let plan = analyze("ls -lh *.txt | head -5 && git status --short");
let segments = chain_of(plan).expect("expected Chain");
assert_eq!(segments.len(), 2);
assert_eq!(segments[0].program, "ls");
assert_eq!(segments[1].program, "git");
}
#[test]
fn rejects_unsafe_chain_segments() {
for command in [
"echo $(pwd) ; git status",
"echo `pwd` ; git status",
"cat <(printf hi) ; git status",
"git status > >(cat) ; bun test",
"! git status ; bun test",
] {
assert_not_chain(command);
}
}
#[test]
fn rejects_legacy_opaque_shapes() {
assert_eq!(analyze("foo || bar"), CommandPlan::Compound);
}
#[test]
fn sequence_is_compound() {
assert_eq!(analyze("echo a ; echo b"), CommandPlan::Compound);
}
#[test]
fn async_is_compound() {
assert_eq!(analyze("git status | cat"), CommandPlan::Piped);
assert_eq!(analyze("sleep 1 &"), CommandPlan::Compound);
assert_eq!(analyze("(cd foo && make)"), CommandPlan::Compound);
assert_eq!(analyze("{ echo hi; }"), CommandPlan::Compound);
assert_eq!(analyze("f() { echo hi; }"), CommandPlan::Compound);
assert_eq!(analyze("[[ -f foo ]]"), CommandPlan::Compound);
assert_eq!(analyze("a && && b"), CommandPlan::Unsupported);
}
#[test]
@@ -176,16 +434,4 @@ mod tests {
assert_eq!(analyze(""), CommandPlan::Unsupported);
assert_eq!(analyze(" "), CommandPlan::Unsupported);
}
#[test]
fn subshell_is_compound_not_single() {
// `(cmd)` is a compound-command variant, not Simple.
let plan = analyze("(cd foo && make)");
assert!(matches!(plan, CommandPlan::Compound | CommandPlan::Unsupported));
}
#[test]
fn malformed_is_unsupported() {
assert_eq!(analyze("a && && b"), CommandPlan::Unsupported);
}
}
crates/pi-shell/src/minimizer/primitives.rs
+56 -5
View File
@@ -2,7 +2,31 @@
use std::collections::BTreeMap;
/// Remove ANSI CSI escape sequences and carriage-return progress frames.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum CapClass {
Errors,
Warnings,
List,
Inventory,
}
impl CapClass {
pub const fn lines(self) -> usize {
match self {
Self::Errors => 160,
Self::Warnings => 120,
Self::List => 80,
Self::Inventory => 40,
}
}
}
pub const fn reduced(cap: usize, by: usize) -> usize {
let reduced = cap.saturating_sub(by);
if reduced == 0 && cap > 0 { 1 } else { reduced }
}
/// Remove ANSI CSI escape sequences while preserving line endings verbatim.
pub fn strip_ansi(input: &str) -> String {
let mut out = String::with_capacity(input.len());
let mut chars = input.chars().peekable();
@@ -16,10 +40,6 @@ pub fn strip_ansi(input: &str) -> String {
}
continue;
}
if ch == '\r' {
out.push('\n');
continue;
}
out.push(ch);
}
out
@@ -78,6 +98,14 @@ pub fn head_tail_lines(input: &str, head: usize, tail: usize) -> String {
out
}
/// Keep head/tail lines using a named cap class.
pub fn head_tail_cap(input: &str, class: CapClass) -> String {
let cap = class.lines();
let head = reduced(cap, cap / 3);
let tail = cap - head;
head_tail_lines(input, head, tail)
}
/// Drop lines matching any of the supplied predicates.
pub fn strip_lines(input: &str, predicates: &[fn(&str) -> bool]) -> String {
let mut out = String::new();
@@ -287,6 +315,11 @@ mod tests {
assert_eq!(strip_ansi("\x1b[31mred\x1b[0m"), "red");
}
#[test]
fn strip_ansi_preserves_carriage_returns() {
assert_eq!(strip_ansi("a\r\nb\rc"), "a\r\nb\rc");
}
#[test]
fn dedups_consecutive_lines() {
assert_eq!(dedup_consecutive_lines("a\na\nb\n"), "a (×2)\nb\n");
@@ -298,6 +331,24 @@ mod tests {
assert_eq!(out, "1\n2\n… 2 lines omitted …\n5\n");
}
#[test]
fn named_caps_have_nonzero_reductions() {
assert_eq!(CapClass::Errors.lines(), 160);
assert_eq!(reduced(1, 10), 1);
assert_eq!(reduced(0, 10), 0);
}
#[test]
fn head_tail_cap_uses_named_budget() {
let input = (0..100)
.map(|idx| idx.to_string())
.collect::<Vec<_>>()
.join("\n");
let out = head_tail_cap(&input, CapClass::List);
assert!(out.contains("lines omitted"));
assert!(out.lines().count() <= CapClass::List.lines() + 1);
}
#[test]
fn groups_file_diagnostics() {
let out = group_by_file("src/a.ts:1:2 error one\nsrc/a.ts:2:3 error two\n", 10);
crates/pi-shell/src/shell.rs
+588 -72
View File
@@ -12,9 +12,9 @@ use std::{
use anyhow::{Error, Result};
use brush_builtins::{BuiltinSet, default_builtins};
use brush_core::{
ExecutionContext, ExecutionControlFlow, ExecutionExitCode, ExecutionResult, ProcessGroupPolicy,
ProfileLoadBehavior, RcLoadBehavior, Shell as BrushShell, ShellValue, ShellVariable, SourceInfo,
builtins,
ExecutionContext, ExecutionControlFlow, ExecutionExitCode, ExecutionParameters, ExecutionResult,
ProcessGroupPolicy, ProfileLoadBehavior, RcLoadBehavior, Shell as BrushShell, ShellValue,
ShellVariable, SourceInfo, builtins,
env::EnvironmentScope,
openfiles::{self, OpenFile, OpenFiles},
};
@@ -571,6 +571,42 @@ async fn source_snapshot(shell: &mut BrushShell, snapshot_path: &str) -> Result<
Ok(())
}
#[derive(Clone, Copy)]
enum CommandCaptureMode {
Streaming,
Buffered { max_capture_bytes: usize },
}
struct CommandRunOutput {
result: ExecutionResult,
buffered: Option<BufferedOutput>,
}
struct ChainCapture {
original_text: String,
text: String,
input_bytes: usize,
changed: bool,
}
impl ChainCapture {
const fn new() -> Self {
Self {
original_text: String::new(),
text: String::new(),
input_bytes: 0,
changed: false,
}
}
fn push(&mut self, original: &str, original_input_bytes: usize, minimized: &str, changed: bool) {
self.original_text.push_str(original);
self.text.push_str(minimized);
self.input_bytes = self.input_bytes.saturating_add(original_input_bytes);
self.changed |= changed;
}
}
async fn run_shell_command(
session: &mut ShellSessionCore,
options: &ShellRunConfig,
@@ -591,13 +627,252 @@ async fn run_shell_command(
} else {
minimizer::engine::MinimizerMode::None
};
let should_minimize = !matches!(minimizer_mode, minimizer::engine::MinimizerMode::None);
let max_capture_bytes = if let Some(config) = options.minimizer.as_ref() {
config.max_capture_bytes as usize
} else {
0
let result = match minimizer_mode {
minimizer::engine::MinimizerMode::SegmentedChain => {
run_shell_command_segmented_chain(session, options, on_chunk, cancel_token).await
},
minimizer::engine::MinimizerMode::WholeCommand | minimizer::engine::MinimizerMode::None => {
run_shell_command_single(session, options, on_chunk, cancel_token, minimizer_mode).await
},
};
if env_scope_pushed {
session
.shell
.env_mut()
.pop_scope(EnvironmentScope::Command)
.map_err(|err| Error::msg(format!("Failed to pop env scope: {err}")))?;
}
result
}
async fn run_shell_command_single(
session: &mut ShellSessionCore,
options: &ShellRunConfig,
on_chunk: Option<mpsc::UnboundedSender<String>>,
cancel_token: CancellationToken,
minimizer_mode: minimizer::engine::MinimizerMode,
) -> Result<(ExecutionResult, Option<MinimizerResult>)> {
debug_assert!(!matches!(minimizer_mode, minimizer::engine::MinimizerMode::SegmentedChain));
let params = session.shell.default_exec_params();
let capture_mode = match minimizer_mode {
minimizer::engine::MinimizerMode::WholeCommand => {
let Some(config) = options.minimizer.as_ref() else {
return Err(Error::msg("Missing minimizer config for whole-command mode"));
};
CommandCaptureMode::Buffered { max_capture_bytes: config.max_capture_bytes as usize }
},
minimizer::engine::MinimizerMode::None => CommandCaptureMode::Streaming,
minimizer::engine::MinimizerMode::SegmentedChain => CommandCaptureMode::Streaming,
};
let command_run = run_shell_command_once(
session,
options.command.clone(),
params,
on_chunk,
cancel_token,
capture_mode,
)
.await?;
let mut minimized_out = None;
if let Some(buffered) = command_run.buffered
&& let Some(config) = options.minimizer.as_ref()
{
// When the capture cap is exceeded the output was streamed raw and never
// buffered, so nothing was minimized — leave `minimized` absent, matching
// every other passthrough path and `apply_shell_minimizer`. Previously a
// `too-large` result with empty `text`/`original_text` was emitted, which a
// consumer keying off `minimized` presence could mistake for a real rewrite
// that produced empty output.
if !buffered.exceeded {
let minimized = match minimizer_mode {
minimizer::engine::MinimizerMode::WholeCommand => minimizer::apply(
&options.command,
&buffered.text,
exit_code(&command_run.result),
config,
),
minimizer::engine::MinimizerMode::None => {
minimizer::MinimizerOutput::passthrough(&buffered.text)
},
minimizer::engine::MinimizerMode::SegmentedChain => {
minimizer::MinimizerOutput::passthrough(&buffered.text)
},
};
// Surface telemetry only when the filter actually rewrote the output
// and kept the original buffer — same contract as `apply_shell_minimizer`
// in `pi-natives`. A supported filter that runs but leaves the output
// unchanged (e.g. a short `git diff --name-only`) reports `changed:
// false` with no `original_text` and must NOT set `minimized`, or API
// consumers keying off `result.minimized` are misled. The separate
// `too-large` reason path above is unaffected.
if minimized.changed
&& let Some(original_text) = minimized.original_text
{
let output_bytes = u32::try_from(minimized.text.len()).unwrap_or(u32::MAX);
minimized_out = Some(MinimizerResult {
filter: minimized.filter.to_string(),
text: minimized.text,
original_text,
input_bytes: u32::try_from(minimized.input_bytes).unwrap_or(u32::MAX),
output_bytes,
});
}
}
}
Ok((command_run.result, minimized_out))
}
async fn run_shell_command_segmented_chain(
session: &mut ShellSessionCore,
options: &ShellRunConfig,
on_chunk: Option<mpsc::UnboundedSender<String>>,
cancel_token: CancellationToken,
) -> Result<(ExecutionResult, Option<MinimizerResult>)> {
let Some(config) = options.minimizer.as_ref() else {
return run_shell_command_single(
session,
options,
on_chunk,
cancel_token,
minimizer::engine::MinimizerMode::None,
)
.await;
};
// When minimizer is disabled, don't segment — stream the original single path.
if !config.enabled {
return run_shell_command_single(
session,
options,
on_chunk,
cancel_token,
minimizer::engine::MinimizerMode::None,
)
.await;
}
let minimizer::plan::CommandPlan::Chain { segments } =
minimizer::plan::analyze(&options.command)
else {
return run_shell_command_single(
session,
options,
on_chunk,
cancel_token,
minimizer::engine::MinimizerMode::None,
)
.await;
};
let params = session.shell.default_exec_params();
let mut aggregate = Some(ChainCapture::new());
let mut previous_succeeded = true;
let mut last_result = None;
let max_capture_bytes = config.max_capture_bytes as usize;
for segment in segments {
if segment.run_if_previous_succeeded && !previous_succeeded {
continue;
}
let mut segment_params = params.clone();
segment_params.suppress_errexit = segment.suppress_errexit;
let capture_mode = if aggregate.is_some() {
CommandCaptureMode::Buffered { max_capture_bytes }
} else {
CommandCaptureMode::Streaming
};
let command_run = run_shell_command_once(
session,
segment.command.clone(),
segment_params,
on_chunk.clone(),
cancel_token.clone(),
capture_mode,
)
.await?;
let exit = exit_code(&command_run.result);
previous_succeeded = exit == 0;
if let Some(buffered) = command_run.buffered {
if buffered.exceeded {
// Cap exceeded mid-chain: output streamed raw, drop the buffered
// aggregate so the remaining segments stream too. No minimization
// happened, so we emit no `minimized` telemetry (see below).
aggregate = None;
} else if let Some(capture) = aggregate.as_mut() {
let next_input_bytes = capture.input_bytes.saturating_add(buffered.input_bytes);
if next_input_bytes > max_capture_bytes {
aggregate = None;
} else {
let minimized = minimizer::apply(&segment.command, &buffered.text, exit, config);
capture.push(
&buffered.text,
buffered.input_bytes,
&minimized.text,
minimized.changed,
);
}
}
} else if aggregate.is_some() {
aggregate = None;
}
let keep_running = session_keepalive(&command_run.result) && !cancel_token.is_cancelled();
last_result = Some(command_run.result);
if !keep_running {
break;
}
}
let Some(result) = last_result else {
return Err(Error::msg("Segmented chain executed no segments"));
};
let minimized_out = aggregate
// Only surface telemetry when the segmented chain actually rewrote the
// output; a `chain-noop` capture (`changed == false`) must yield `None`,
// matching the public `ShellRunResult.minimized` contract.
.filter(|capture| capture.changed)
.map(|capture| {
let minimized = minimizer::chain_output(
capture.text,
capture.original_text,
capture.input_bytes,
capture.changed,
);
MinimizerResult {
filter: minimized.filter.to_string(),
text: minimized.text,
original_text: minimized.original_text.unwrap_or_default(),
input_bytes: u32::try_from(minimized.input_bytes).unwrap_or(u32::MAX),
output_bytes: u32::try_from(minimized.output_bytes).unwrap_or(u32::MAX),
}
});
// A chain that overflowed the aggregate cap streamed its output raw and was
// not minimized — `minimized_out` stays `None`, matching the whole-command
// path and `apply_shell_minimizer`. (Previously a `too-large` result with
// empty `text` was emitted, a footgun for consumers keying off presence.)
Ok((result, minimized_out))
}
async fn run_shell_command_once(
session: &mut ShellSessionCore,
command: String,
mut params: ExecutionParameters,
on_chunk: Option<mpsc::UnboundedSender<String>>,
cancel_token: CancellationToken,
capture_mode: CommandCaptureMode,
) -> Result<CommandRunOutput> {
let (reader_file, writer_file) = pipe_to_files("output")?;
let stdout_file = OpenFile::from(
@@ -607,7 +882,6 @@ async fn run_shell_command(
);
let stderr_file = OpenFile::from(writer_file);
let mut params = session.shell.default_exec_params();
params.set_fd(OpenFiles::STDIN_FD, null_file()?);
params.set_fd(OpenFiles::STDOUT_FD, stdout_file);
params.set_fd(OpenFiles::STDERR_FD, stderr_file);
@@ -616,28 +890,27 @@ async fn run_shell_command(
let baseline_descendants = process::current_descendant_pids();
let reader_cancel = CancellationToken::new();
let (activity_tx, mut activity_rx) = mpsc::channel::<()>(1);
// Stream every raw chunk to the caller live, regardless of whether
// minimization is enabled. When minimization actually transforms the
// output, we propagate the replacement text via `MinimizerResult.text`
// so the caller can swap their accumulated buffer for the minimized
// version without losing intermediate progress updates.
let reader_callback = on_chunk;
let mut reader_handle = tokio::spawn({
let reader_cancel = reader_cancel.clone();
async move {
if should_minimize {
let output = read_output_buffered(
reader_file,
reader_callback,
reader_cancel,
activity_tx,
max_capture_bytes,
)
.await;
Result::<OutputRead>::Ok(OutputRead::Buffered(output))
} else {
Box::pin(read_output(reader_file, reader_callback, reader_cancel, activity_tx)).await;
Result::<OutputRead>::Ok(OutputRead::Streaming)
match capture_mode {
CommandCaptureMode::Buffered { max_capture_bytes } => {
let output = read_output_buffered(
reader_file,
reader_callback,
reader_cancel,
activity_tx,
max_capture_bytes,
)
.await;
Result::<OutputRead>::Ok(OutputRead::Buffered(output))
},
CommandCaptureMode::Streaming => {
Box::pin(read_output(reader_file, reader_callback, reader_cancel, activity_tx))
.await;
Result::<OutputRead>::Ok(OutputRead::Streaming)
},
}
}
});
@@ -660,21 +933,13 @@ async fn run_shell_command(
let source_info = SourceInfo::from("pi-natives:command");
let result = session
.shell
.run_string(options.command.clone(), &source_info, &params)
.run_string(command, &source_info, &params)
.await;
if cancel_token.is_cancelled() {
terminate_background_jobs(&mut session.shell);
}
if env_scope_pushed {
session
.shell
.env_mut()
.pop_scope(EnvironmentScope::Command)
.map_err(|err| Error::msg(format!("Failed to pop env scope: {err}")))?;
}
drop(params);
// The foreground command can complete while background jobs keep the
@@ -737,33 +1002,11 @@ async fn run_shell_command(
}
let result = result.map_err(|err| Error::msg(format!("Shell execution failed: {err}")))?;
let mut minimized_out: Option<MinimizerResult> = None;
if let Some(OutputRead::Buffered(output)) = reader_output
&& let Some(config) = options.minimizer.as_ref()
&& !output.exceeded
{
let minimized = match minimizer_mode {
minimizer::engine::MinimizerMode::WholeCommand => {
minimizer::apply(&options.command, &output.text, exit_code(&result), config)
},
minimizer::engine::MinimizerMode::None => {
minimizer::MinimizerOutput::passthrough(&output.text)
},
};
if minimized.changed
&& let Some(original) = minimized.original_text
{
let output_bytes = u32::try_from(minimized.text.len()).unwrap_or(u32::MAX);
minimized_out = Some(MinimizerResult {
filter: minimized.filter.to_string(),
text: minimized.text,
original_text: original,
input_bytes: u32::try_from(minimized.input_bytes).unwrap_or(u32::MAX),
output_bytes,
});
}
}
Ok((result, minimized_out))
let buffered = match reader_output {
Some(OutputRead::Buffered(output)) => Some(output),
Some(OutputRead::Streaming) | None => None,
};
Ok(CommandRunOutput { result, buffered })
}
async fn run_shell_command_streams(
@@ -824,7 +1067,28 @@ async fn run_shell_command_streams(
let baseline_descendants = baseline_descendants.clone();
async move {
cancel_token.cancelled().await;
terminate_new_descendants(&baseline_descendants).await;
const WAVES: u32 = 3;
for wave in 0..WAVES {
let mut targets = process::TerminationTargets::new();
process::add_new_descendants(&mut targets, &baseline_descendants);
if targets.is_empty() {
return;
}
let signal = if wave == 0 {
process::TERM_SIGNAL
} else {
process::KILL_SIGNAL
};
targets.signal(signal);
if wave + 1 < WAVES {
let pause = if wave == 0 {
Duration::from_millis(75)
} else {
Duration::from_millis(150)
};
time::sleep(pause).await;
}
}
}
});
let source_info = SourceInfo::from("pi-shell:streams");
@@ -1150,8 +1414,9 @@ enum OutputRead {
}
struct BufferedOutput {
text: String,
exceeded: bool,
text: String,
input_bytes: usize,
exceeded: bool,
}
async fn read_output(
@@ -1272,6 +1537,7 @@ async fn read_output_buffered(
const REPLACEMENT: &str = "\u{FFFD}";
const BUF: usize = 65536;
let mut buf = vec![0u8; BUF];
let mut input_bytes = 0usize;
let mut captured = Vec::new();
let mut exceeded = false;
// Pending bytes from a prior read that ended mid-UTF-8 sequence. We hold
@@ -1281,7 +1547,7 @@ async fn read_output_buffered(
#[cfg(unix)]
let Ok(reader) = register_nonblocking_pipe(reader) else {
return BufferedOutput { text: String::new(), exceeded: true };
return BufferedOutput { text: String::new(), input_bytes: 0, exceeded: true };
};
#[cfg(not(unix))]
let reader = tokio::fs::File::from_std(reader);
@@ -1321,6 +1587,7 @@ async fn read_output_buffered(
};
if n > 0 {
let _ = activity.try_send(());
input_bytes = input_bytes.saturating_add(n);
}
// Once `exceeded`, the post-process minimizer is bypassed (see the
// `!output.exceeded` gate at the call site), so further appends just
@@ -1379,7 +1646,7 @@ async fn read_output_buffered(
}
}
BufferedOutput { text: String::from_utf8_lossy(&captured).into_owned(), exceeded }
BufferedOutput { text: String::from_utf8_lossy(&captured).into_owned(), input_bytes, exceeded }
}
#[cfg(unix)]
@@ -1692,9 +1959,9 @@ mod tests {
/// Brush leading a new pgroup with non-terminal stdin always detaches —
/// including the first stage of a pipeline. `setsid()` keeps the child
/// off the host's controlling tty; the spawn path skips
/// `process_group(...)` for detached children, so later stages no
/// longer try to `setpgid`-join a leader that has moved sessions (the
/// historical EPERM hazard).
/// `process_group(...)` for detached children, so later stages no longer
/// try to `setpgid`-join a leader that has moved sessions (the historical
/// EPERM hazard).
#[test]
fn non_terminal_stdin_detaches_regardless_of_pipeline() {
assert_eq!(child_session_action(true, false, false), ChildSessionAction::DetachSession,);
@@ -1736,6 +2003,256 @@ mod tests {
}
}
#[cfg(unix)]
fn shell_test_lock() -> &'static TokioMutex<()> {
static LOCK: std::sync::OnceLock<TokioMutex<()>> = std::sync::OnceLock::new();
LOCK.get_or_init(|| TokioMutex::new(()))
}
#[cfg(unix)]
async fn run_command_capture(
command: &str,
cwd: Option<&std::path::Path>,
minimizer: Option<minimizer::MinimizerOptions>,
cancel_token: CancelToken,
) -> (ShellExecuteResult, String) {
let _guard = shell_test_lock().lock().await;
let (tx, mut rx) = mpsc::unbounded_channel::<String>();
let options = ShellExecuteOptions {
command: command.to_string(),
cwd: cwd.map(|path| path.to_string_lossy().into_owned()),
minimizer,
..Default::default()
};
let result = execute_shell(options, Some(tx), cancel_token)
.await
.expect("execute_shell");
let mut output = String::new();
while let Some(chunk) = rx.recv().await {
output.push_str(&chunk);
}
(result, output)
}
#[cfg(unix)]
fn unique_temp_dir(prefix: &str) -> std::path::PathBuf {
let mut path = std::env::temp_dir();
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time")
.as_nanos();
path.push(format!("pi-shell-{prefix}-{}-{nonce}", std::process::id()));
std::fs::create_dir_all(&path).expect("create temp dir");
path
}
#[cfg(unix)]
fn printf_minimizer(
settings_path: &std::path::Path,
max_capture_bytes: Option<u32>,
) -> minimizer::MinimizerOptions {
std::fs::write(
settings_path,
r#"
schema_version = 1
[filters.printf]
match_command = "^printf$"
replace = [{ pattern = "hello", replacement = "HI" }]
"#,
)
.expect("write settings");
minimizer::MinimizerOptions {
enabled: Some(true),
settings_path: Some(settings_path.to_string_lossy().into_owned()),
max_capture_bytes,
..Default::default()
}
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_false_and_printf_skips_second_and_returns_nonzero() {
let root = unique_temp_dir("false-and");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), None);
let (result, output) = run_command_capture(
"false && printf skipped",
None,
Some(minimizer),
CancelToken::default(),
)
.await;
let _ = std::fs::remove_dir_all(&root);
assert_eq!(result.exit_code, Some(1));
assert!(!result.cancelled);
assert!(!result.timed_out);
assert_eq!(output, "");
// `false && printf` short-circuits: nothing is rewritten, so a no-op chain
// must surface no minimizer telemetry (None).
assert!(result.minimized.is_none(), "chain noop must not surface telemetry");
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_false_semicolon_printf_continues_and_returns_last_code() {
let root = unique_temp_dir("false-semi");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), None);
let (result, output) = run_command_capture(
"false ; printf 'hello\n'",
None,
Some(minimizer),
CancelToken::default(),
)
.await;
let _ = std::fs::remove_dir_all(&root);
let minimized = result.minimized.expect("minimized result");
assert_eq!(result.exit_code, Some(0));
assert_eq!(output, "hello\n");
assert_eq!(minimized.filter, "chain");
assert_eq!(minimized.original_text, "hello\n");
assert_eq!(minimized.text, "HI\n");
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_cd_tmp_and_pwd_persists_state_across_segments() {
let root = unique_temp_dir("cwd");
let tmp_dir = root.join("tmp");
std::fs::create_dir_all(&tmp_dir).expect("create nested tmp dir");
let settings_path = root.join("minimizer.toml");
std::fs::write(
&settings_path,
r#"
schema_version = 1
[filters.pwd]
match_command = "^pwd$"
replace = [{ pattern = "^.+$", replacement = "PWD" }]
"#,
)
.expect("write settings");
let minimizer = minimizer::MinimizerOptions {
enabled: Some(true),
settings_path: Some(settings_path.to_string_lossy().into_owned()),
..Default::default()
};
let expected = format!("{}\n", tmp_dir.display());
let (result, output) =
run_command_capture("cd tmp && pwd", Some(&root), Some(minimizer), CancelToken::default())
.await;
let _ = std::fs::remove_dir_all(&root);
let minimized = result.minimized.expect("minimized result");
assert_eq!(result.exit_code, Some(0));
assert_eq!(output, expected);
assert_eq!(minimized.filter, "chain");
assert_eq!(minimized.text, "PWD\n");
assert_eq!(minimized.original_text, expected);
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn whole_command_exceeding_capture_cap_streams_raw_without_minimized() {
let root = unique_temp_dir("whole-cap");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), Some(1024));
let (result, output) =
run_command_capture("printf '%1200s' x", None, Some(minimizer), CancelToken::default())
.await;
let _ = std::fs::remove_dir_all(&root);
assert_eq!(result.exit_code, Some(0));
assert_eq!(output.len(), 1200);
assert!(output.ends_with('x'));
// Output exceeded the capture cap: streamed raw and never buffered, so
// nothing was minimized. `minimized` must be absent (not a `too-large`
// result with empty `text`, which would mislead presence-keyed consumers).
assert!(result.minimized.is_none());
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_printf_chain_preserves_raw_original_text() {
let root = unique_temp_dir("minimizer");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), None);
let (result, output) = run_command_capture(
"printf 'hello\n' ; printf 'world\n'",
None,
Some(minimizer),
CancelToken::default(),
)
.await;
let _ = std::fs::remove_dir_all(&root);
let minimized = result.minimized.expect("minimized result");
assert_eq!(result.exit_code, Some(0));
assert_eq!(output, "hello\nworld\n");
assert_eq!(minimized.filter, "chain");
assert_eq!(minimized.original_text, "hello\nworld\n");
assert_eq!(minimized.text, "HI\nworld\n");
assert_eq!(minimized.input_bytes, 12);
assert_eq!(minimized.output_bytes, 9);
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_chain_exceeding_aggregate_capture_cap_stays_raw() {
let root = unique_temp_dir("aggregate-cap");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), Some(1024));
let (result, output) = run_command_capture(
"printf '%600s' x ; printf '%600s' y",
None,
Some(minimizer),
CancelToken::default(),
)
.await;
let _ = std::fs::remove_dir_all(&root);
assert_eq!(result.exit_code, Some(0));
assert_eq!(output.len(), 1200);
assert!(output.ends_with('y'));
// Aggregate cap exceeded: the chain streamed its output raw and was not
// minimized, so `minimized` is absent (not an empty-text `too-large`).
assert!(result.minimized.is_none());
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_timeout_in_first_segment_prevents_later_segments() {
let root = unique_temp_dir("timeout");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), None);
let (result, output) = run_command_capture(
"sleep 1 && printf later",
None,
Some(minimizer),
CancelToken::new(Some(10)),
)
.await;
let _ = std::fs::remove_dir_all(&root);
assert!(result.exit_code.is_none());
assert!(!result.cancelled);
assert!(result.timed_out);
assert!(result.minimized.is_none());
assert!(!output.contains("later"));
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn segmented_cancel_in_first_segment_prevents_later_segments() {
let root = unique_temp_dir("cancel");
let minimizer = printf_minimizer(&root.join("minimizer.toml"), None);
let mut cancel_token = CancelToken::default();
let abort_token = cancel_token.emplace_abort_token();
let cancel_task = tokio::spawn(async move {
time::sleep(Duration::from_millis(10)).await;
abort_token.abort(AbortReason::Signal);
});
let (result, output) =
run_command_capture("sleep 1 && printf later", None, Some(minimizer), cancel_token).await;
let _ = cancel_task.await;
let _ = std::fs::remove_dir_all(&root);
assert!(result.exit_code.is_none());
assert!(result.cancelled);
assert!(!result.timed_out);
assert!(result.minimized.is_none());
assert!(!output.contains("later"));
}
/// End-to-end verification that brush, when embedded as a non-interactive
/// library (`interactive: false`, exactly what `create_session` produces),
/// spawns external commands in a **separate session** from the host.
@@ -2024,7 +2541,6 @@ mod tests {
assert!(!result.cancelled);
assert!(!result.timed_out);
}
#[tokio::test]
async fn abort_state_signals_cancel_token() {
let abort_state = ShellAbortState::default();
docs/adding-a-provider.md
+48 -25
View File
@@ -1,26 +1,42 @@
# Adding a provider
Providers in `packages/ai` are described by a single declarative
`ProviderDefinition` and collected in one registry. Every scattered structure —
the `KnownProvider` / `OAuthProvider` type unions, `PROVIDER_DESCRIPTORS`,
`DEFAULT_MODEL_PER_PROVIDER`, the `serviceProviderMap` env-key fallbacks, the
`/login` provider list, the `refreshOAuthToken` / `AuthStorage.login` dispatch,
and the coding-agent callback maps — is **derived** from that registry.
A provider is described in two halves:
- **Catalog half** (`packages/catalog`): one entry in the `CATALOG_PROVIDERS`
table (`packages/catalog/src/provider-models/descriptors.ts`) carrying the
`id`, `defaultModel`, runtime model-discovery factory, and catalog-generation
wiring. `KnownProvider`, `PROVIDER_DESCRIPTORS`, and
`DEFAULT_MODEL_PER_PROVIDER` are derived from this table.
- **Auth half** (`packages/ai`): one declarative `ProviderDefinition` in the
registry carrying env-key fallbacks and login/refresh flows. The
`OAuthProvider` union, the env-key map, the `/login` provider list, the
`refreshOAuthToken` / `AuthStorage.login` dispatch, and the coding-agent
callback maps are derived from the registry.
**Scope.** This is for a provider that reuses an existing wire API
(`openai-completions`, `anthropic-messages`, `google-generative-ai`, …) — the
common case for gateways and API-key providers, since stream dispatch keys on
`model.api`, not `model.provider`. Adding a *new wire protocol* (a new
`KnownApi`) is a separate task that also touches `stream.ts` dispatch,
`api-registry.ts`, and `types.ts`.
`api-registry.ts`, and the catalog `types.ts`.
## Shape
For the common case, a provider is still **one new def file + one registry line**:
For the common case, a provider is **one catalog entry + one def file + one registry line**:
1. **Create `packages/ai/src/registry/<id>.ts`** exporting one
`export const <camelId>Provider = { … } as const satisfies ProviderDefinition;`.
2. **Add it to the `ALL` array** in `packages/ai/src/registry/registry.ts`
1. **Add an entry to `CATALOG_PROVIDERS`** in
`packages/catalog/src/provider-models/descriptors.ts` with the `id`,
`defaultModel`, the plain API-key env var(s) as `envVars`, and (usually) a
`createModelManagerOptions` factory. For a
simple OpenAI-compatible gateway, build the factory in
`packages/catalog/src/provider-models/openai-compat.ts` or inline with the
exported `createSimpleOpenAICompletionsOptions(providerId, baseUrl, config)`.
2. **Create `packages/ai/src/registry/<id>.ts`** exporting one
`export const <camelId>Provider = { … } as const satisfies ProviderDefinition;`
with the auth fields (`login`, …). Plain env-var names live in the catalog
entry's `envVars`; set `envKeys` only for computed resolvers (Foundry/ADC/
Bedrock-style probes).
3. **Add it to the `ALL` array** in `packages/ai/src/registry/registry.ts`
(one import + one array entry). `ALL` order is the `/login` list order for
loginable providers.
@@ -34,26 +50,33 @@ For a **non-trivial provider-local OAuth flow**, put the implementation in
file. The shared OAuth flow infrastructure it builds on lives in the same
`registry/oauth/` directory.
Either way, descriptors, default-model map, env-key map, login list, and refresh
dispatch all update automatically, and the `KnownProvider` / `OAuthProvider`
unions gain the new id by derivation.
Descriptors, the default-model map, env-key map, login list, and refresh
dispatch all update automatically; the `KnownProvider` union gains the new id
from the catalog table and `OAuthProvider` from the registry.
## `ProviderDefinition` fields
## Field reference
See `packages/ai/src/registry/types.ts` for the authoritative,
JSDoc-annotated interface. Presence of a field opts the provider into a derived
structure:
**Catalog table entry** (`ProviderCatalogEntry`, see
`packages/catalog/src/provider-models/descriptor-types.ts` for JSDoc):
| Field | Effect |
|---|---|
| `id` | Required. Member of `KnownProvider`. |
| `defaultModel` | Required. Preferred model when no explicit selection is made. |
| `envVars` | Env var name(s), in order, for the runtime API-key fallback (`getEnvApiKey`). |
| `createModelManagerOptions` | Runtime model-discovery factory. Present (and not `specialModelManager`) ⇒ appears in `PROVIDER_DESCRIPTORS`. |
| `allowUnauthenticated` | Runtime creates a model manager even without a key. |
| `dynamicModelsAuthoritative` | Successful discovery replaces bundled models. |
| `catalogDiscovery` | `{ label, envVars?, oauthProvider?, allowUnauthenticated? }` for offline catalog generation (`generate-models.ts`). `envVars` here overrides the entry-level list when generation uses different credentials (e.g. `cursor`). |
| `specialModelManager` | Bespoke runtime factory (`google-antigravity` / `google-gemini-cli` / `openai-codex`); excluded from `PROVIDER_DESCRIPTORS`. |
**Registry definition** (`ProviderDefinition`, see
`packages/ai/src/registry/types.ts`):
| Field | Effect |
|---|---|
| `id`, `name` | Required. `name` shows in the `/login` list. |
| `defaultModel` | Present ⇒ member of `KnownProvider` (a chat-model provider). |
| `createModelManagerOptions` | Runtime model-discovery factory. Present (and not `specialModelManager`) ⇒ appears in `PROVIDER_DESCRIPTORS`. |
| `allowUnauthenticated` | Runtime creates a model manager even without a key. |
| `dynamicModelsAuthoritative` | Successful discovery replaces bundled models. |
| `catalogDiscovery` | `{ label, envVars, oauthProvider?, allowUnauthenticated? }` for offline catalog generation (`generate-models.ts`). |
| `specialModelManager` | Bespoke runtime factory (`google-antigravity` / `google-gemini-cli` / `openai-codex`); excluded from `PROVIDER_DESCRIPTORS`. |
| `envKeys` | Env-var fallback for `getEnvApiKey`: a var name string or a `() => string \| undefined` resolver. |
| `envKeys` | Computed env fallback for `getEnvApiKey`, overriding the catalog entry's `envVars`: a var name string or a `() => string \| undefined` resolver. Omit when `envVars` covers it. |
| `login` | Interactive login. Present ⇒ member of `OAuthProvider`, shown in `/login`, dispatchable via `AuthStorage.login`. Returns an api-key `string` or `OAuthCredentials`. |
| `refreshToken` | OAuth refresher; omit for static-token providers (the dispatch returns credentials unchanged). |
| `storeCredentialsAs` | Store credentials under a different provider id (e.g. `openai-codex-device` ⇒ `openai-codex`). |
docs/config-usage.md
+19 -4
View File
@@ -146,12 +146,13 @@ The runtime settings model is layered:
1. Global settings: `~/.omp/agent/config.yml`
2. Project settings: discovered via settings capability (`settings.json` and `config.yml` from providers)
3. Runtime overrides: in-memory, non-persistent
4. Schema defaults: from `SETTINGS_SCHEMA`
3. CLI config overlays: `omp --config <path>` / repeated `--config` files, loaded as `config.yml`-style YAML for this process only
4. Runtime overrides: in-memory, non-persistent
5. Schema defaults: from `SETTINGS_SCHEMA`
Effective read path:
Effective precedence:
`defaults <- global <- project <- overrides`
`defaults <- global <- project <- CLI config overlays <- overrides`
Write behavior:
@@ -251,6 +252,20 @@ Native provider (`id: native`) reads native config from:
- `Settings.init()` loads global `config.yml` + discovered project settings capability items.
- Only capability items with `level === "project"` are merged into project layer.
### Session title prompt override
Create `TITLE_SYSTEM.md` in the same config locations as `SYSTEM.md` / `APPEND_SYSTEM.md`:
```text
# ~/.omp/agent/TITLE_SYSTEM.md
Generate a session name using lowercase `<type>:<primary-objective>`.
```
- Missing `TITLE_SYSTEM.md` keeps the bundled title prompts.
- Discovery uses the same project-then-user config directory pattern as `SYSTEM.md`: project `.omp/TITLE_SYSTEM.md` first, then user `~/.omp/agent/TITLE_SYSTEM.md` and the other supported config bases.
- The override replaces only the automatic session-title generation system prompt; normal `SYSTEM.md` / `APPEND_SYSTEM.md` prompt customization is unaffected.
- The online path still forces the `set_title` tool call. The local tiny-title path keeps the `<title>...</title>` prefill/stop wrapper and uses this file as its system turn.
## Skills subsystem
- `extensibility/skills.ts` loads via `loadCapability(skillCapability.id, { cwd })`.
docs/models.md
+28
View File
@@ -137,6 +137,20 @@ Must define at least one of:
- `id` required
- `contextWindow` and `maxTokens` must be positive if provided
### Command-resolved secrets
Provider `apiKey` values and provider/model `headers` values may start with `!` to read a secret from command stdout. The command is run with a 10 s timeout, stdout is trimmed, and empty/failing commands are omitted:
```yaml
providers:
openai:
apiKey: "!op read op://dev/openai/api-key"
headers:
X-Team-Key: "!bw get password omp-team-key"
```
Successful command outputs are cached for the process lifetime so the command is not re-run for every model.
## Merge and override order
ModelRegistry pipeline (on refresh):
@@ -272,6 +286,8 @@ If `lm-studio` is not explicitly configured, registry adds an implicit discovera
Runtime discovery fetches models (`GET /models`) and synthesizes model entries with local defaults.
This path also works for local OpenAI-compatible servers that are not LM Studio. For example, if oMLX is bound to Ollama's usual port, set `LM_STUDIO_BASE_URL=http://127.0.0.1:11434/v1` to discover it through the existing `/v1/models` flow. Running oMLX and Ollama side by side requires assigning a different port to one of them. Do not configure oMLX as `ollama`: Ollama discovery uses native `/api/tags` and `/api/show` endpoints, not OpenAI `/v1/models`.
### Explicit provider discovery
You can configure discovery yourself:
@@ -606,6 +622,18 @@ providers:
name: Qwen 2.5 Coder 32B (local)
```
For oMLX or another local OpenAI-compatible server with a discoverable `/v1/models` endpoint, prefer discovery instead of listing models by hand. Set `api` to the endpoint family your server actually exposes: `openai-completions` uses `/v1/chat/completions`; servers that expose `/v1/responses` need `openai-responses` instead.
```yaml
providers:
omlx:
baseUrl: http://127.0.0.1:11434/v1
auth: none
api: openai-completions
discovery:
type: openai-models-list
```
### Hosted proxy with env-based key
```yaml
docs/non-compaction-retry-policy.md
+13 -9
View File
@@ -62,8 +62,8 @@ Flow (`#handleRetryableError`):
3. Increment `#retryAttempt`.
4. Create `#retryPromise` once (first attempt in a chain).
5. If attempt exceeded `retry.maxRetries`, emit final failure event and stop.
6. Compute base delay: `retry.baseDelayMs * 2^(attempt-1)`.
7. For usage-limit errors, parse retry hints and call auth storage (`markUsageLimitReached(...)`); if credential switching succeeds, force delay to `0`, otherwise use a larger retry-after/backoff hint when present.
6. Compute capped jittered local delay: `min(retry.baseDelayMs * 2^(attempt-1), 8000ms) * (75–100% jitter)`.
7. For usage-limit errors, parse retry hints and call auth storage (`markUsageLimitReached(...)`); if credential switching succeeds, force delay to `0`. Otherwise wait for whichever comes first — the provider's retry-after/backoff hint, or the earliest moment a temporarily blocked sibling credential frees up (`retryAtMs` + 1s buffer) so the next attempt can pick it up.
8. If no credential switch occurred, suppress the current model selector for cooldown, try configured retry model fallback chains, and force delay to `0` on model switch.
9. If the final delay exceeds `retry.maxDelayMs` and no credential/model switch happened, emit final failure and do not sleep.
10. Emit `auto_retry_start`.
@@ -87,8 +87,8 @@ Flow (`#handleRetryableError`):
Settings:
- `retry.enabled` (default `true`)
- `retry.maxRetries` (default `3`)
- `retry.baseDelayMs` (default `2000`)
- `retry.maxRetries` (default `10`)
- `retry.baseDelayMs` (default `500`)
- `retry.maxDelayMs` (default `300000`, 5 minutes; `<= 0` disables the fail-fast cap)
Attempt numbering:
@@ -97,13 +97,17 @@ Attempt numbering:
- start events use current attempt (1-based)
- max-exceeded end event reports `attempt: this.#retryAttempt - 1` (last attempted retry count)
Backoff sequence with default settings:
Backoff sequence with default settings, before jitter:
- attempt 1: 2000 ms
- attempt 2: 4000 ms
- attempt 3: 8000 ms
- attempt 1: 500 ms
- attempt 2: 1000 ms
- attempt 3: 2000 ms
- attempt 4: 4000 ms
- attempt 5+: 8000 ms
Delay override inputs can come from parsed retry headers (`retry-after-ms`, `retry-after`, `x-ratelimit-reset-ms`, `x-ratelimit-reset`) or usage-limit backoff. Credential/model fallback switches set delay to `0`; otherwise parsed hints can extend the exponential local delay. If the computed delay is greater than `retry.maxDelayMs` and no switch succeeded, retry ends immediately with a final error instead of sleeping.
The actual local sleep is 75–100% of the nominal value, matching Anthropic-style retry jitter so concurrent sessions do not retry in lockstep.
Delay override inputs can come from parsed retry headers (`retry-after-ms`, `retry-after`, `x-ratelimit-reset-ms`, `x-ratelimit-reset`) or usage-limit backoff. Credential/model fallback switches set delay to `0`; otherwise parsed hints can extend the capped local delay. If the computed delay is greater than `retry.maxDelayMs` and no switch succeeded, retry ends immediately with a final error instead of sleeping.
## Abort mechanics
docs/sdk.md
+2 -2
View File
@@ -318,9 +318,9 @@ Use `setToolUIContext(...)` only if your embedder provides UI capabilities that
- **Conditional LSP warmup.** Startup LSP servers (those returned by `discoverStartupLspServers(cwd)`) are only warmed when **all** of these hold:
- `enableLsp !== false` on the session options, **and**
- `options.hasUI === true` (interactive TUI), **and**
- the `lsp.diagnosticsOnWrite` setting is enabled.
- the `lsp.lazy` setting is disabled (it defaults to `true`).
Print / script / RPC / ACP invocations (`hasUI=false`) skip the warmup entirely: they don't render the warmup status indicator and typically finish before the language servers would stabilize, so warming them just spends CPU parsing big `initialize` responses concurrently with the LLM stream consumer and jitters perceived latency. Tools that actually need an LSP server still spin one up on demand through `getOrCreateClient()` — only the _startup_ warmup is skipped. The returned `lspServers` field in `CreateAgentSessionResult` is therefore `undefined` (not an empty array) whenever the warmup branch was bypassed.
With `lsp.lazy` enabled — the default — no language servers are launched at startup at all; each server cold-starts on first use, i.e. when the agent invokes the `lsp` tool or an edit/write touches a file whose extension matches the server's `fileTypes`. Print / script / RPC / ACP invocations (`hasUI=false`) skip the warmup regardless of the setting: they don't render the warmup status indicator and typically finish before the language servers would stabilize, so warming them just spends CPU parsing big `initialize` responses concurrently with the LLM stream consumer and jitters perceived latency. Tools that actually need an LSP server still spin one up on demand through `getOrCreateClient()` — only the _startup_ warmup is skipped. The returned `lspServers` field in `CreateAgentSessionResult` is still populated for UI sessions in lazy mode — recognized servers are discovered (no processes spawned) and reported with status `"available"` so the welcome screen and `/status` can list them; it is `undefined` only when `enableLsp === false` or `hasUI === false`.
## Minimal controlled embed example
docs/system-prompt-customization.md
+13
View File
@@ -124,6 +124,18 @@ The dynamic project/environment footer that remains after `SYSTEM.md` is only bl
There is currently no supported CLI mode for "replace the stable default instructions but keep the generated skills/rules/tool guidance." If you need automatic skills loading, keep the default block and add your customization via `APPEND_SYSTEM.md`. If you fully replace with `SYSTEM.md`, you must hard-code any skill names/instructions you want the model to know about, and those will not track discovery automatically.
### "Customize automatic session titles"
`SYSTEM.md` and `APPEND_SYSTEM.md` do not affect the model call that names a new session. Create the title-specific prompt file instead:
```text
# ~/.omp/agent/TITLE_SYSTEM.md
Generate a session name using lowercase `<type>:<primary-objective>`.
If the message carries no concrete task, output exactly `none`.
```
`TITLE_SYSTEM.md` is discovered with the same project-then-user config-directory pattern as `SYSTEM.md` / `APPEND_SYSTEM.md`. When absent, OMP uses the bundled `title-system.md` / `tiny-title-system.md` prompts. When present, the online title path still forces the `set_title` tool call, and the local tiny-model path keeps the `<title>...</title>` wrapper while using this file as the system turn.
### "Replace everything, including project context" — SDK-only
The normal CLI file/flag path intentionally preserves `defaultPrompt.slice(1)`. Code using `CreateAgentSessionOptions.systemPrompt` directly can return a full replacement array and omit the project footer, but that is not what `.omp/SYSTEM.md`, `~/.omp/agent/SYSTEM.md`, or `--system-prompt` do.
@@ -163,6 +175,7 @@ Net effect for CLI users: put `SYSTEM.md` / `APPEND_SYSTEM.md` directly under `<
| Add an instruction on top of the full default prompt | `APPEND_SYSTEM.md` or `--append-system-prompt` |
| Replace the stable default instructions but keep project/environment context | `SYSTEM.md` or `--system-prompt` |
| Preserve generated skills/rules/tool guidance while customizing | `APPEND_SYSTEM.md`; `SYSTEM.md` replaces that generated block |
| Customize automatic session titles | `TITLE_SYSTEM.md`; chat-turn `SYSTEM.md` / `APPEND_SYSTEM.md` do not affect title generation |
| Use `{{cwd}}` / `{{date}}` / other internals in my file | Not supported. Files are inserted verbatim. |
| Inherit specific sections from `system-prompt.md` | Not supported; use append, or copy what you need into `SYSTEM.md`. |
| Override at a per-repo level | Project `.omp/SYSTEM.md` under the cwd you launch `omp` from |
docs/tools/lsp.md
+1 -1
View File
@@ -310,5 +310,5 @@ Same as `definition`, but sends `textDocument/implementation` and reports `imple
- `reload` does not recreate a client immediately after killing it; the next request triggers reinitialization.
- `workspace/applyEdit` can apply edits initiated by the server outside the direct tool action result path.
- `detectLspmux()` can be disabled with `PI_DISABLE_LSPMUX=1`; only `rust-analyzer` is in `DEFAULT_SUPPORTED_SERVERS`.
- Startup LSP warmup (`discoverStartupLspServers(cwd)` in `sdk.ts`) is gated on `enableLsp && options.hasUI && settings.get("lsp.diagnosticsOnWrite")` — print/RPC/ACP/script sessions skip it and let `getOrCreateClient()` cold-start servers on demand. See `docs/sdk.md` § Startup performance.
- Startup LSP discovery (`discoverStartupLspServers(cwd)` in `sdk.ts`) runs for `enableLsp && options.hasUI`; the background warmup additionally requires `!settings.get("lsp.lazy")`. `lsp.lazy` defaults to `true`, so by default discovered servers are surfaced with status `"available"` (gray dot in the welcome screen) and cold-start through `getOrCreateClient()` on first use (lsp tool call or edit/write on a matching file type). Print/RPC/ACP/script sessions skip discovery and warmup entirely. See `docs/sdk.md` § Startup performance.
- `configCache` is per-process and never auto-invalidated; config changes require a fresh process to be observed by `getConfig()` callers.
docs/tools/read.md
+1 -1
View File
@@ -10,7 +10,7 @@
- `packages/coding-agent/src/tools/archive-reader.ts` — detect `archive.ext:inner/path`, index archives, list/read entries.
- `packages/coding-agent/src/tools/sqlite-reader.ts` — detect SQLite targets, parse selectors, render tables.
- `packages/coding-agent/src/tools/fetch.ts` — URL parsing, fetch/render pipeline, URL cache/artifacts.
- `packages/coding-agent/src/internal-urls/router.ts` — resolve `agent://`, `artifact://`, `local://`, `mcp://`, `memory://`, `omp://`, `rule://`, `skill://`.
- `packages/coding-agent/src/internal-urls/router.ts` — resolve `agent://`, `artifact://`, `issue://`, `local://`, `mcp://`, `memory://`, `omp://`, `pr://`, `rule://`, `skill://`, and `vault://`.
- `packages/coding-agent/src/edit/notebook.ts` — convert `.ipynb` to editable `# %% [...] cell:N` text.
- `packages/coding-agent/src/utils/file-display-mode.ts` — decide hashline vs line-number vs raw display.
- `packages/coding-agent/src/workspace-tree.ts` — render directory trees.
docs/tui.md
+5 -3
View File
@@ -25,13 +25,15 @@ If your extension/tool can run in non-interactive mode, guard with `ctx.hasUI` /
```ts
export interface Component {
render(width: number): string[];
render(width: number): readonly string[];
handleInput?(data: string): void;
wantsKeyRelease?: boolean;
invalidate?(): void;
}
```
Render results are component-owned and immutable to callers; a component that did not change should return the **same array reference** it returned last time (reference equality is what enables the renderer's memoization and row virtualization), and must return a new array whenever its content changed.
`Focusable` is separate:
```ts
@@ -56,7 +58,7 @@ Minimal pattern:
```ts
import { replaceTabs, truncateToWidth } from "@oh-my-pi/pi-tui";
render(width: number): string[] {
render(width: number): readonly string[] {
return this.lines.map(line => truncateToWidth(replaceTabs(line), width));
}
```
@@ -218,7 +220,7 @@ class Picker implements Component {
this.list.handleInput(data);
}
render(width: number): string[] {
render(width: number): readonly string[] {
return this.list
.render(width)
.map((line) => truncateToWidth(replaceTabs(line), width));
package.json
+12 -11
View File
@@ -20,15 +20,16 @@
"@huggingface/transformers": "^4.2.0",
"@mozilla/readability": "^0.6.0",
"@napi-rs/cli": "3.7.0",
"@oh-my-pi/hashline": "15.10.10",
"@oh-my-pi/omp-stats": "15.10.10",
"@oh-my-pi/pi-agent-core": "15.10.10",
"@oh-my-pi/pi-ai": "15.10.10",
"@oh-my-pi/pi-coding-agent": "15.10.10",
"@oh-my-pi/pi-mnemopi": "15.10.10",
"@oh-my-pi/pi-natives": "15.10.10",
"@oh-my-pi/pi-tui": "15.10.10",
"@oh-my-pi/pi-utils": "15.10.10",
"@oh-my-pi/hashline": "15.10.12",
"@oh-my-pi/omp-stats": "15.10.12",
"@oh-my-pi/pi-agent-core": "15.10.12",
"@oh-my-pi/pi-ai": "15.10.12",
"@oh-my-pi/pi-catalog": "15.10.12",
"@oh-my-pi/pi-coding-agent": "15.10.12",
"@oh-my-pi/pi-mnemopi": "15.10.12",
"@oh-my-pi/pi-natives": "15.10.12",
"@oh-my-pi/pi-tui": "15.10.12",
"@oh-my-pi/pi-utils": "15.10.12",
"@opentelemetry/api": "^1.9.1",
"@opentelemetry/context-async-hooks": "^2.7.1",
"@opentelemetry/exporter-trace-otlp-proto": "^0.218.0",
@@ -85,7 +86,7 @@
},
"overrides": {},
"scripts": {
"install:dev": "bun install && bun --cwd=packages/coding-agent link && ln -sfn \"$(pwd)/packages/coding-agent/scripts/dev-launch\" \"$(bun pm -g bin)/omp\"",
"install:dev": "bun install && bun --cwd=packages/coding-agent link && ln -sfn \"$(pwd)/packages/coding-agent/scripts/omp\" \"$(bun pm -g bin)/omp\"",
"dev": "bun --cwd=packages/coding-agent src/cli.ts",
"dev:timing": "PI_TIMING=x bun --cwd=packages/coding-agent --preload ../utils/src/module-timer.ts src/cli.ts",
"stats": "bun --cwd=packages/coding-agent src/cli.ts stats",
@@ -152,7 +153,7 @@
"publish": "bun run prepublishOnly && npm publish -ws --access public",
"publish:dry": "bun run prepublishOnly && npm publish -ws --access public --dry-run",
"release": "bun scripts/release.ts",
"generate-models": "bun --cwd=packages/ai run generate-models",
"generate-models": "bun --cwd=packages/catalog run generate-models",
"generate-docs-index": "bun --cwd=packages/coding-agent run generate-docs-index",
"generate-template": "bun --cwd=packages/coding-agent run generate-template",
"check-spoofed-versions": "bun scripts/check-spoofed-versions.ts"
packages/agent/CHANGELOG.md
+18 -2
View File
@@ -2,11 +2,26 @@
## [Unreleased]
## [15.10.12] - 2026-06-10
### Added
- Added `AgentLoopConfig.getDisableReasoning` so callers can override `disableReasoning` per LLM call, mirroring `getReasoning`.
- Added `transformProviderContext` to `AgentOptions`/`AgentLoopConfig`: an optional hook applied to the assembled provider context after conversion, normalization, and append-only handling, but before telemetry capture and provider send.
### Fixed
- Fixed `Agent` runs so explicit reasoning disablement is forwarded to provider stream options and re-resolved per continuation, keeping mid-run thinking-off changes in sync with the next provider request.
## [15.10.11] - 2026-06-10
### Changed
- Editorial pass over the compaction prompts: fixed garbled grammar and missing articles, RFC-keyed prohibitions, deduped restated instructions; parsed markers (`<read-files>`/`<modified-files>`/`<previous-summary>`) and all output-format headings left byte-identical
- Catalog imports moved to the new `@oh-my-pi/pi-catalog` package: subpath imports (`calculateCost`, Codex wire constants) plus catalog values previously taken from the `@oh-my-pi/pi-ai` root (`getBundledModel`, `clampThinkingLevelForModel`), which pi-ai no longer re-exports; type-only `Model`/`Api`/`Effort` imports from pi-ai are unchanged
## [15.10.8] - 2026-06-09
### Added
- Added optional `fetch` overrides to `SummaryOptions` and `compact`/`generateSummary` so remote compaction can use custom HTTP clients
@@ -15,6 +30,7 @@
- Added the upstream provider that served a request (`AssistantMessage.upstreamProvider`, e.g. OpenRouter's routed provider) as a `pi.gen_ai.response.upstream_provider` chat-span telemetry attribute, alongside the existing response id and time-to-first-chunk.
## [15.10.5] - 2026-06-08
### Removed
- Removed the `maxToolCallsPerTurn` option from `AgentOptions` and `AgentLoopConfig`, so assistant turns are no longer capped after a configured number of completed tool calls
@@ -52,7 +68,6 @@
- Removed stale synthetic user-message tag filters from OpenAI remote compaction output preservation; developer messages are now dropped by role instead.
- Tool executions now receive the active turn `AbortSignal` unconditionally.
## [15.10.2] - 2026-06-08
### Fixed
@@ -84,6 +99,7 @@
- Surfaced Anthropic stream failures whose message starts with `Output blocked by conten` as normal assistant error lifecycle events, so interactive clients render content-filter blocks instead of silently dropping the streaming bubble at `agent_end`.
## [15.8.3] - 2026-06-03
### Added
- Added `getReadToolPath(context)` to `@oh-my-pi/pi-agent-core/compaction/tool-protection` to extract a paired `read` tool call's `path` for embedders building read-targeted protection matchers
@@ -648,4 +664,4 @@ Initial release under @oh-my-pi scope. See previous releases at [badlogic/pi-mon
- `Agent` constructor now has all options optional (empty options use defaults).
- `queueMessage()` is now synchronous (no longer returns a Promise).
- `queueMessage()` is now synchronous (no longer returns a Promise).
packages/agent/package.json
+2 -1
View File
@@ -1,7 +1,7 @@
{
"type": "module",
"name": "@oh-my-pi/pi-agent-core",
"version": "15.10.10",
"version": "15.10.12",
"description": "General-purpose agent with transport abstraction, state management, and attachment support",
"homepage": "https://omp.sh",
"author": "Can Boluk",
@@ -36,6 +36,7 @@
},
"dependencies": {
"@oh-my-pi/pi-ai": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-natives": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"@opentelemetry/api": "catalog:"
packages/agent/src/agent-loop.ts
+6
View File
@@ -829,6 +829,9 @@ async function streamAssistantResponse(
tools: normalizeTools(context.tools, !!config.intentTracing),
};
}
if (config.transformProviderContext) {
llmContext = config.transformProviderContext(llmContext);
}
const streamFunction = streamFn || streamSimple;
@@ -845,6 +848,7 @@ async function streamAssistantResponse(
const dynamicToolChoice = config.getToolChoice?.();
const dynamicReasoning = config.getReasoning?.();
const dynamicDisableReasoning = config.getDisableReasoning?.();
const harmonyMitigationEnabled = isHarmonyLeakMitigationTarget(config.model);
const harmonyAbortController = harmonyMitigationEnabled ? new AbortController() : undefined;
const requestSignal = harmonyAbortController
@@ -856,6 +860,7 @@ async function streamAssistantResponse(
harmonyRetryAttempt > 0 && config.temperature !== undefined ? config.temperature + 0.05 : config.temperature;
const effectiveToolChoice = dynamicToolChoice ?? config.toolChoice;
const effectiveReasoning = dynamicReasoning ?? config.reasoning;
const effectiveDisableReasoning = dynamicDisableReasoning ?? config.disableReasoning;
const chatStepNumber = stepCounter.count;
stepCounter.count += 1;
@@ -916,6 +921,7 @@ async function streamAssistantResponse(
metadata: resolvedMetadata,
toolChoice: effectiveToolChoice,
reasoning: effectiveReasoning,
disableReasoning: effectiveDisableReasoning,
temperature: effectiveTemperature,
signal: requestSignal,
onResponse: captureOnResponse,
packages/agent/src/agent.ts
+18 -1
View File
@@ -6,10 +6,10 @@ import {
type ApiKeyResolveContext,
type AssistantMessage,
type AssistantMessageEvent,
type Context,
type CursorExecHandlers,
type CursorToolResultHandler,
type Effort,
getBundledModel,
type ImageContent,
type Message,
type Model,
@@ -22,6 +22,7 @@ import {
type ToolChoice,
type ToolResultMessage,
} from "@oh-my-pi/pi-ai";
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
import { abortReasonText, agentLoop, agentLoopContinue } from "./agent-loop";
import type { AppendOnlyContextManager } from "./append-only-context";
import type { HarmonyAuditEvent } from "./harmony-leak";
@@ -93,6 +94,12 @@ export interface AgentOptions {
*/
transformContext?: (messages: AgentMessage[], signal?: AbortSignal) => Promise<AgentMessage[]>;
/**
* Optional transform applied after provider context assembly and before
* telemetry capture/provider send.
*/
transformProviderContext?: (context: Context) => Context;
/**
* Steering mode: "all" = send all steering messages at once, "one-at-a-time" = one per turn
*/
@@ -265,6 +272,7 @@ export class Agent {
systemPrompt: [],
model: getBundledModel("google", "gemini-2.5-flash-lite-preview-06-17"),
thinkingLevel: undefined,
disableReasoning: false,
tools: [],
messages: [],
isStreaming: false,
@@ -277,6 +285,7 @@ export class Agent {
#abortController?: AbortController;
#convertToLlm: (messages: AgentMessage[]) => Message[] | Promise<Message[]>;
#transformContext?: (messages: AgentMessage[], signal?: AbortSignal) => Promise<AgentMessage[]>;
#transformProviderContext?: (context: Context) => Context;
#steeringQueue: AgentMessage[] = [];
#followUpQueue: AgentMessage[] = [];
#steeringMode: "all" | "one-at-a-time";
@@ -375,6 +384,7 @@ export class Agent {
this.afterToolCall = opts.afterToolCall;
this.#telemetry = opts.telemetry;
this.#appendOnlyContext = opts.appendOnlyContext;
this.#transformProviderContext = opts.transformProviderContext;
}
/**
@@ -658,6 +668,10 @@ export class Agent {
this.#state.thinkingLevel = l;
}
setDisableReasoning(disabled: boolean) {
this.#state.disableReasoning = disabled;
}
setSteeringMode(mode: "all" | "one-at-a-time") {
this.#steeringMode = mode;
}
@@ -942,6 +956,7 @@ export class Agent {
const config: AgentLoopConfig = {
model,
reasoning,
disableReasoning: this.#state.disableReasoning,
temperature: this.#temperature,
topP: this.#topP,
topK: this.#topK,
@@ -961,6 +976,7 @@ export class Agent {
kimiApiFormat: this.#kimiApiFormat,
preferWebsockets: this.#preferWebsockets,
convertToLlm: this.#convertToLlm,
transformProviderContext: this.#transformProviderContext,
transformContext: this.#transformContext,
onPayload: this.#onPayload,
onResponse: this.#onResponse,
@@ -985,6 +1001,7 @@ export class Agent {
onHarmonyLeak: this.#onHarmonyLeak,
getToolChoice,
getReasoning: () => this.#state.thinkingLevel,
getDisableReasoning: () => this.#state.disableReasoning,
getSteeringMessages: async () => {
if (skipInitialSteeringPoll) {
skipInitialSteeringPoll = false;
packages/agent/src/compaction/compaction.ts
+9 -7
View File
@@ -7,19 +7,20 @@
import {
type AssistantMessage,
clampThinkingLevelForModel,
Effort,
type FetchImpl,
type Message,
type MessageAttribution,
type Model,
type Tool,
type Usage,
} from "@oh-my-pi/pi-ai";
import { clampThinkingLevelForModel } from "@oh-my-pi/pi-catalog/model-thinking";
import { countTokens } from "@oh-my-pi/pi-natives";
import { logger, prompt } from "@oh-my-pi/pi-utils";
import { type AgentTelemetry, instrumentedCompleteSimple } from "../telemetry";
import { ThinkingLevel } from "../thinking";
import type { AgentMessage, AgentTool } from "../types";
import type { AgentMessage } from "../types";
import type { CompactionEntry, SessionEntry } from "./entries";
import { type ConvertToLlm, convertToLlm, createBranchSummaryMessage, createCustomMessage } from "./messages";
import {
@@ -539,10 +540,11 @@ function effortFromThinkingLevel(level: ThinkingLevel): Effort {
* - Explicit effort → respect user choice → clamped per model.
*
* The clamp routes through `clampThinkingLevelForModel`, which returns
* `undefined` for models with `compat.supportsReasoningEffort: false`
* (e.g. `xai-oauth/grok-build`). That `undefined` then flows through to the
* openai-responses mapper where `modelOmitsReasoningEffort` short-circuits
* the wire param — no `requireSupportedEffort` throw.
* `undefined` for reasoning models without a thinking config — the build-time
* encoding of `compat.supportsReasoningEffort: false` (e.g.
* `xai-oauth/grok-build`). That `undefined` then flows through to the
* openai-responses mapper, which omits the wire param — no
* `requireSupportedEffort` throw.
*/
function resolveCompactionEffort(model: Model, level: ThinkingLevel | undefined): Effort | undefined {
if (level === ThinkingLevel.Off) return undefined;
@@ -689,7 +691,7 @@ export interface HandoffOptions {
/** Live agent system prompt — passed verbatim so providers hit the cached prefix. */
systemPrompt: string[];
/** Live agent tool list — same purpose. Forced to `toolChoice: "none"`. */
tools?: AgentTool<any>[];
tools?: Tool[];
customInstructions?: string;
convertToLlm?: ConvertToLlm;
initiatorOverride?: MessageAttribution;
packages/agent/src/compaction/openai.ts
+6 -6
View File
@@ -12,12 +12,6 @@
* with `{ summary, shortSummary? }`.
*/
import {
CODEX_BASE_URL,
getCodexAccountId,
OPENAI_HEADER_VALUES,
OPENAI_HEADERS,
} from "@oh-my-pi/pi-ai/providers/openai-codex/constants";
import { parseTextSignature } from "@oh-my-pi/pi-ai/providers/openai-responses-shared";
import { transformMessages } from "@oh-my-pi/pi-ai/providers/transform-messages";
import type { AssistantMessage, FetchImpl, Message, Model } from "@oh-my-pi/pi-ai/types";
@@ -26,6 +20,12 @@ import {
getOpenAIResponsesHistoryPayload,
normalizeResponsesToolCallId,
} from "@oh-my-pi/pi-ai/utils";
import {
CODEX_BASE_URL,
getCodexAccountId,
OPENAI_HEADER_VALUES,
OPENAI_HEADERS,
} from "@oh-my-pi/pi-catalog/wire/codex";
import { logger } from "@oh-my-pi/pi-utils";
// ============================================================================
packages/agent/src/proxy.ts
+1 -1
View File
@@ -13,8 +13,8 @@ import {
type StopReason,
type ToolCall,
} from "@oh-my-pi/pi-ai";
import { calculateCost } from "@oh-my-pi/pi-ai/models";
import { parseStreamingJson } from "@oh-my-pi/pi-ai/utils/json-parse";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import { readSseJson } from "@oh-my-pi/pi-utils";
// Event stream adapter for proxy SSE events
packages/agent/src/types.ts
+18
View File
@@ -3,6 +3,7 @@ import type {
AssistantMessage,
AssistantMessageEvent,
AssistantMessageEventStream,
Context,
Effort,
ImageContent,
Message,
@@ -107,6 +108,13 @@ export interface AgentLoopConfig extends SimpleStreamOptions {
*/
transformContext?: (messages: AgentMessage[], signal?: AbortSignal) => Promise<AgentMessage[]>;
/**
* Optional transform applied to the final provider context after conversion,
* normalization, and append-only context handling, but before telemetry capture
* and provider send.
*/
transformProviderContext?: (context: Context) => Context;
/**
* Resolves an API key dynamically for each LLM call.
*
@@ -210,6 +218,15 @@ export interface AgentLoopConfig extends SimpleStreamOptions {
*/
getReasoning?: () => Effort | undefined;
/**
* Dynamic reasoning-disable override, resolved per LLM call. When set,
* its return value overrides the static `disableReasoning` from
* `SimpleStreamOptions` for that request. Pair with `getReasoning` so
* mid-run transitions into and out of the explicit `off` state propagate
* to the next provider call.
*/
getDisableReasoning?: () => boolean | undefined;
/**
* Called after a tool call has been validated and is about to execute.
*
@@ -358,6 +375,7 @@ export interface AgentState {
systemPrompt: string[];
model: Model;
thinkingLevel?: Effort;
disableReasoning?: boolean;
tools: AgentTool<any>[];
messages: AgentMessage[]; // Can include attachments + custom message types
isStreaming: boolean;
packages/agent/test/agent.test.ts
+63
View File
@@ -354,6 +354,69 @@ describe("Agent", () => {
expect(reasoningPerCall).toEqual([ThinkingLevel.Low, ThinkingLevel.High]);
});
it("forwards explicit reasoning disablement to the stream", async () => {
const mock = createMockModel({ responses: [{ content: ["ok"] }] });
const agent = new Agent({
initialState: {
model: mock.model,
messages: [],
disableReasoning: true,
},
streamFn: mock.stream,
});
await agent.prompt("run");
expect(mock.calls[0]?.options?.disableReasoning).toBe(true);
});
it("re-reads disableReasoning for each model call within a run", async () => {
const toolSchema = z.object({ value: z.string() });
type Details = { value: string };
const alphaTool: AgentTool<typeof toolSchema, Details> = {
name: "alpha",
label: "Alpha",
description: "Alpha tool",
parameters: toolSchema,
async execute(_toolCallId, params) {
return { content: [{ type: "text", text: `alpha:${params.value}` }], details: { value: params.value } };
},
};
const mock = createMockModel({
responses: [
{ content: [{ type: "toolCall", id: "tool-1", name: "alpha", arguments: { value: "hello" } }] },
{ content: ["done"] },
],
});
const agent = new Agent({
initialState: {
model: mock.model,
thinkingLevel: ThinkingLevel.High,
disableReasoning: false,
tools: [alphaTool],
messages: [],
},
streamFn: mock.stream,
});
// Flip thinking off mid-run after the first assistant turn produces the
// tool call but before the continuation request is sent.
const unsubscribe = agent.subscribe(event => {
if (event.type === "message_end" && event.message.role === "toolResult") {
agent.setThinkingLevel(undefined);
agent.setDisableReasoning(true);
}
});
await agent.prompt("run");
unsubscribe();
const disablePerCall = mock.calls.map(call => call.options?.disableReasoning);
expect(disablePerCall).toEqual([false, true]);
});
it("forwards distinct provider session id and prompt cache key to the stream", async () => {
const mock = createMockModel({ responses: [{ content: ["ok"] }] });
const agent = new Agent({
packages/agent/test/compaction-error-status.test.ts
+1 -1
View File
@@ -9,7 +9,7 @@ import {
} from "@oh-my-pi/pi-agent-core/compaction";
import type { AssistantMessage, Model } from "@oh-my-pi/pi-ai";
import * as ai from "@oh-my-pi/pi-ai";
import { getBundledModel } from "@oh-my-pi/pi-ai/models";
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
// Pins the fix for the "raw 401 surfaced as Compaction failed:" bug.
//
packages/agent/test/compaction-telemetry.test.ts
+3 -2
View File
@@ -27,6 +27,7 @@ import {
import type { AgentMessage } from "@oh-my-pi/pi-agent-core/types";
import type { AssistantMessage, Model, Usage } from "@oh-my-pi/pi-ai";
import * as ai from "@oh-my-pi/pi-ai";
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import { SpanStatusCode } from "@opentelemetry/api";
import {
BasicTracerProvider,
@@ -35,7 +36,7 @@ import {
SimpleSpanProcessor,
} from "@opentelemetry/sdk-trace-base";
const MODEL: Model = {
const MODEL: Model = buildModel({
id: "mock-model",
name: "mock-model",
api: "mock",
@@ -46,7 +47,7 @@ const MODEL: Model = {
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200_000,
maxTokens: 32_768,
};
});
let exporter: InMemorySpanExporter;
let provider: BasicTracerProvider;
packages/agent/test/compaction-thinking-level.test.ts
+1 -1
View File
@@ -10,7 +10,7 @@ import {
import { ThinkingLevel } from "@oh-my-pi/pi-agent-core/thinking";
import type { AssistantMessage, Model } from "@oh-my-pi/pi-ai";
import * as ai from "@oh-my-pi/pi-ai";
import { getBundledModel } from "@oh-my-pi/pi-ai/models";
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
// Pins fix #1 of the compaction effort-override bug. Before this fix,
// `generateHandoff` (and the three other compaction summarizers) hardcoded
packages/agent/test/handoff.test.ts
+1 -1
View File
@@ -4,7 +4,7 @@ import { AUTO_HANDOFF_THRESHOLD_FOCUS, generateHandoff, renderHandoffPrompt } fr
import type { AssistantMessage, Model, ToolCall } from "@oh-my-pi/pi-ai";
import * as ai from "@oh-my-pi/pi-ai";
import { Effort } from "@oh-my-pi/pi-ai";
import { getBundledModel } from "@oh-my-pi/pi-ai/models";
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
function createAssistantMessage(content: AssistantMessage["content"]): AssistantMessage {
return {
packages/agent/test/harmony-leak.test.ts
+1 -1
View File
@@ -9,7 +9,7 @@ import {
signalListLabel,
} from "@oh-my-pi/pi-agent-core/harmony-leak";
import type { AssistantMessage, Model, ToolCall } from "@oh-my-pi/pi-ai";
import { getBundledModel } from "@oh-my-pi/pi-ai";
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
import corpus from "./fixtures/harmony-leak-corpus.json" with { type: "json" };
import { createAssistantMessage } from "./helpers";
packages/agent/test/proxy-stream-disconnect.test.ts
+3 -2
View File
@@ -10,8 +10,9 @@ import { describe, expect, it } from "bun:test";
import type { ProxyAssistantMessageEvent } from "@oh-my-pi/pi-agent-core/proxy";
import { type ProxyMessageEventStream, streamProxy } from "@oh-my-pi/pi-agent-core/proxy";
import type { AssistantMessageEvent, Context, FetchImpl, Model } from "@oh-my-pi/pi-ai";
import { buildModel } from "@oh-my-pi/pi-catalog/build";
const mockModel: Model = {
const mockModel: Model = buildModel({
id: "test-model",
name: "Test Model",
api: "openai",
@@ -22,7 +23,7 @@ const mockModel: Model = {
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 4096,
maxTokens: 1024,
};
});
const mockContext: Context = {
messages: [{ role: "user", content: "hello", timestamp: Date.now() }],
packages/agent/test/remote-compaction.test.ts
+5 -3
View File
@@ -1,9 +1,11 @@
import { describe, expect, test } from "bun:test";
import { buildOpenAiNativeHistory, requestOpenAiRemoteCompaction } from "@oh-my-pi/pi-agent-core/compaction/openai";
import type { AssistantMessage, FetchImpl, Model, ToolResultMessage } from "@oh-my-pi/pi-ai/types";
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import type { ModelSpec } from "@oh-my-pi/pi-catalog/types";
function makeOpenAiModel(overrides: Partial<Model<"openai-responses">> = {}): Model<"openai-responses"> {
return {
function makeOpenAiModel(overrides: Partial<ModelSpec<"openai-responses">> = {}): Model<"openai-responses"> {
return buildModel({
id: "gpt-5",
name: "GPT-5",
api: "openai-responses",
@@ -15,7 +17,7 @@ function makeOpenAiModel(overrides: Partial<Model<"openai-responses">> = {}): Mo
contextWindow: 400000,
maxTokens: 128000,
...overrides,
};
});
}
describe("buildOpenAiNativeHistory custom tool calls", () => {
packages/ai/CHANGELOG.md
+40 -13
View File
@@ -2,20 +2,56 @@
## [Unreleased]
## [15.10.12] - 2026-06-10
### Added
- Added `antigravityRankingStrategy` and registered it for `google-antigravity` in `DEFAULT_RANKING_STRATEGIES`, so new sessions are routed to OAuth credentials with quota headroom for the requested model backend (lowest relevant `remainingFraction` counter as the sole ranked window, 24h `windowDefaults` matching `daily-cloudcode-pa.googleapis.com` resets). Without it, the existing `antigravityUsageProvider` data never reached credential selection. ([#2198](https://github.com/can1357/oh-my-pi/issues/2198))
### Changed
- Updated MiniMax and MiniMax Token Plan defaults to `MiniMax-M3` and refreshed Token Plan login copy/links ([#1725](https://github.com/can1357/oh-my-pi/issues/1725)).
### Fixed
- Fixed OpenAI Responses and Azure OpenAI Responses streams silently surfacing incomplete output as successful when a custom/proxy provider drops the connection without sending a terminal `response.completed`/`response.incomplete` event. Both providers now detect premature stream closure and throw with `stopReason: "error"` ([#2184](https://github.com/can1357/oh-my-pi/pull/2184))
- Fixed `isUsageLimitError` missing Antigravity / Cloud Code Assist's `Individual quota reached` 429 phrasing. The `USAGE_LIMIT_PATTERN` only knew `quota.?exceeded` / `limit_reached`, so `auth-retry` and `AuthStorage.markUsageLimitReached` treated the response as a terminal provider error and pinned sessions to the exhausted OAuth account instead of rotating to a sibling credential. The pattern now also matches `quota.?reached`. ([#2198](https://github.com/can1357/oh-my-pi/issues/2198))
- Scoped Antigravity usage blocking and ranking by model family (`gemini-*`/`gemma-*` → Google, `claude-*` → Anthropic, `gpt-*`/`openai/*` → OpenAI), so an exhausted Gemini counter no longer makes a healthy Claude/OpenAI Antigravity credential unavailable until reset. ([#2198](https://github.com/can1357/oh-my-pi/issues/2198))
- Fixed no-model Antigravity credential lookups (e.g. image-provider discovery) inheriting provider-wide exhaustion: `scopeLimits` now returns no limits without a concrete backend counter, and `blockScope` always returns a counter scope so missing model context can never fall through to AuthStorage's provider-wide block bucket. ([#2198](https://github.com/can1357/oh-my-pi/issues/2198))
## [15.10.11] - 2026-06-10
### Breaking Changes
- The model catalog moved to the new `@oh-my-pi/pi-catalog` package. Deep subpath exports `@oh-my-pi/pi-ai/models.json`, `/models`, `/model-cache`, `/model-manager`, `/model-thinking`, `/effort`, `/provider-models*`, `/utils/discovery*`, `/providers/openai-codex/constants`, `/providers/google-gemini-headers`, and `/providers/openai-completions-compat` are gone — import the `@oh-my-pi/pi-catalog` equivalents (`/models.json`, `/models`, `/model-cache`, `/model-manager`, `/model-thinking`, `/effort`, `/provider-models*`, `/discovery*`, `/wire/codex`, `/wire/gemini-headers`, `/compat/openai`). The pi-ai root barrel re-exports only the model/effort *types* its own signatures use (`Model`, `Api`, `ThinkingConfig`, `Effort`, `Usage`, compat interfaces) — catalog *values* (`getBundledModel(s)`, `calculateCost`, `modelsAreEqual`, `clampThinkingLevelForModel`, `DEFAULT_MODEL_PER_PROVIDER`, …) must be imported from `@oh-my-pi/pi-catalog`.
- `ProviderDefinition` is now auth-only: `defaultModel`, `createModelManagerOptions`, `catalogDiscovery`, `dynamicModelsAuthoritative`, `allowUnauthenticated`, and `specialModelManager` moved to pi-catalog's `CATALOG_PROVIDERS` table, and `KnownProviderId` was replaced by pi-catalog's `KnownProvider` (registry completeness is enforced by a compile-time check against that union). The pure GitHub Copilot key/endpoint helpers moved from `registry/oauth/github-copilot` to `@oh-my-pi/pi-catalog/wire/github-copilot`.
### Added
- Exported `wrapFetchForCch` so non-streaming OAuth callers (e.g. the web-search provider) can patch the Claude Code billing-header `cch` attestation into their request bodies instead of shipping the `cch=00000` placeholder.
### Changed
- Reduced idle-watchdog churn on the token hot path: the abort promise/listener is created once per stream instead of per yielded item, the deadline uses a persistent re-armed timer instead of a `setTimeout` create/destroy pair per delta, and the persistent race promises are re-minted every 1024 items so per-race reaction records cannot accumulate for the stream's whole life.
- Memoized Anthropic many-image downscaling by content-block identity, so long sessions with stable message objects no longer re-decode and re-encode every oversized image on each request and retry.
- Tool-argument validation errors now truncate embedded argument strings at 256 chars per field — a failed `write`-class call no longer echoes hundreds of KB of payload back to the model as the error message.
- Auth storage no longer issues per-boot no-op writes: the schema-version row is only rewritten when the recorded version actually changes, and the credential identity-key backfill skips rows whose derived identity is null — reopening a current-schema database now performs zero write transactions
- Plain provider env-var names moved to the catalog table: registry defs dropped their 48 `envKeys` literals (including the pure `$pickenv` pickers for `huggingface`/`qwen-portal`/`xai-oauth`), `getEnvApiKey` now derives those fallbacks from `CATALOG_PROVIDERS[].envVars`, and `envKeys` remains only for computed resolvers (Anthropic Foundry, Vertex ADC, Bedrock credential chains) and non-catalog providers (`kagi`, `tavily`, `parallel`, `perplexity`)
- Protocol handlers are now pure `model.compat` readers — the per-request `resolve*Compat`/`detect*Compat` calls (anthropic ×11, responses ×3, completions wrappers), inline `strictResponsesPairing` host detection, the OpenCode `reasoning_content` mutation block, and all `resolvedBaseUrl` threading are gone. Compat is materialized once at model build time (`@oh-my-pi/pi-catalog` `buildModel`); the OpenCode thinking-mode quirk is a precomputed `compat.whenThinking` pointer swap, and request-time base-URL overrides only feed the HTTP client. Behavior is unchanged (the Anthropic `supportsLongCacheRetention` official-endpoint gate is folded into detection).
- Providers now read baked thinking/wire metadata instead of re-parsing model ids per request: the Anthropic handler gates sampling params on `model.compat.supportsSamplingParams` and adaptive `display` on `model.thinking.supportsDisplay` (Bedrock too), adaptive effort tiers come from the baked `thinking.effortMap`, the Google `thinkingLevel` map is static, and effort-dial-less reasoners (`thinking: undefined`, e.g. `xai-oauth/grok-build`) short-circuit `resolveOpenAiReasoningEffort` without the removed `modelOmitsReasoningEffort` predicate.
- Anthropic streaming retries now use a 10-retry budget with the Anthropic-compatible 0.5s exponential backoff capped at 8s with jitter; server `retry-after` hints still win, and retryable pre-content failures such as 502s no longer stop after three tries.
### Fixed
- Fixed Ollama chat requests honoring `omitMaxOutputTokens`, sending `think: false` when reasoning is explicitly disabled, and preserving HTTP 400 response bodies in surfaced errors.
- Fixed `AuthStorage.markUsageLimitReached` collapsing "every sibling is momentarily blocked" into "no sibling exists": it now returns `UsageLimitMarkResult` with the earliest sibling block expiry (`retryAtMs`), so retry layers can wait out a short-lived block (60s post-401, 5-min usage-probe) instead of adopting the provider's multi-hour retry-after. `rotateSessionCredential` and the auth-gateway adapt to the new shape.
- Fixed Gemini streaming silently presenting truncated or blocked output as a successful `stop`: in-band `{"error":{...}}` events and `promptFeedback.blockReason` chunks were never inspected, and a stream ending without any `finishReason` kept the initialized `stop` — all three now surface as errors (both the API-key and gemini-cli/Antigravity consumers), and the `toolUse` stop-reason override no longer masks `SAFETY`/`MALFORMED_FUNCTION_CALL` finishes that arrive after a valid tool call.
- Fixed Gemini/Bedrock error finishes reporting "An unknown error occurred": the raw finish/stop reason (`MALFORMED_FUNCTION_CALL`, `RECITATION`, `guardrail_intervened`, …) is now recorded into the surfaced error message.
- Fixed the Anthropic provider retry loop ignoring server `retry-after` on 429/529 — it now waits `max(headerDelay, backoff)` instead of hammering a rate-limited endpoint three times within ~14s of guaranteed failures.
- Fixed in-stream Anthropic SSE `error` events being thrown as raw JSON envelopes; the structured `error.type`/`message` is parsed out, keeping retry classification on the typed token instead of accidental regex hits.
- Fixed transparent-reconnect tolerance duplicating content behind replaying proxies: after a duplicate `message_start`, replayed `content_block_start` events for already-closed indexes are now consumed silently instead of appending duplicate text/tool calls.
- Fixed the Anthropic gateway accepting malformed known-type content blocks (e.g. `{type:"text", text:123}`) through the unknown-block catch-all, corrupting history and surfacing later as an opaque TypeError — they now fail validation with a clean 400. The gateway's encode stream also emits `ping` keepalives every 15s and a complete `message_start`/`message_delta`/`message_stop` envelope when the inner stream ends without a terminal event, so strict clients no longer classify slow or empty streams as protocol errors.
- Fixed dotted-version Claude ids (`claude-opus-4.7`/`4.8` on GitHub Copilot, Vercel AI Gateway, Zenmux) missing adaptive thinking `display` support — streamed reasoning stayed hidden on those entries because the display predicate only matched dash-form ids (same failure class as #1373).
- Fixed the Mistral `requiresThinkingAsText` replay path calling `.unshift()` on string assistant content — an unconditional TypeError that failed any same-model history turn carrying both thinking and text.
- Fixed the Responses gateway stripping `encrypted_content` from inbound reasoning items (strip-mode schema), which broke codex-style stateless replay; the schema is now loose, restoring the symmetry the outbound encoder already preserved. Composite internal `callId|itemId` ids are also split before hitting the wire so third-party clients that validate `call_id` charsets no longer reject them.
- Ported the shared unfinished-tool-call sweep to the codex `response.completed` handler, so a lost `output_item.done` can no longer persist a tool call with stale `{}` arguments and transient parser fields into session history.
@@ -33,19 +69,6 @@
- Fixed Gemini <3 multimodal tool results breaking the single-function-response-turn invariant for parallel tool calls (image turns are buffered and flushed after the merged functionResponse turn), and the gemini-cli consumer now defaults missing `functionCall.args` to `{}` like the shared consumer.
- Fixed Bedrock dropping `toolConfig` entirely when `toolChoice` is `"none"` while history still contains tool blocks — the Converse API rejects such requests, so tool specs are kept and only the choice is omitted.
- Fixed AWS credential handling serving expired credentials until process restart: cache entries are invalidated on 401/403, file-sourced session-token credentials get a 5-minute TTL, and concurrent first requests single-flight instead of spawning duplicate `credential_process`/SSO fetches — the shared resolution is detached from the first caller's abort signal (one cancelled request no longer fails every waiter) and bounded by its own 30s timeout. The eventstream reader also cancels the response body on abnormal exit instead of leaving the HTTP connection draining.
### Removed
- Removed the dead `iterateUntilAbort` helper (superseded by `iterateWithIdleTimeout`); it leaked the upstream iterator when the consumer abandoned mid-yield and had no production call sites.
## [15.10.10] - 2026-06-09
### Added
- Exported `wrapFetchForCch` so non-streaming OAuth callers (e.g. the web-search provider) can patch the Claude Code billing-header `cch` attestation into their request bodies instead of shipping the `cch=00000` placeholder.
### Fixed
- Fixed an unbounded, zero-backoff Codex WebSocket reconnect loop on `websocket_connection_limit_reached`: the no-content reconnect path never consulted the retry budget and never waited, hammering the endpoint forever when the limit is account-scoped. Reconnects are now budgeted and delayed like every other WS retry path, falling back to a single SSE replay when exhausted.
- Fixed the Codex whitespace-loop breaker not observing degenerate frames that arrive after their item closed (or before it opened) — those frames count as stream progress, so the idle watchdogs never fired and the turn hung forever, which is exactly the failure mode the breaker exists for. Whitespace-loop recovery now also refuses to replay the turn once a `toolcall_end` was delivered, surfacing the error instead of re-emitting the same tool calls.
- Fixed the two remaining Codex retry paths (WS mid-stream reconnect and the empty-content SSE fallback) leaking blockless native output items (e.g. `web_search_call`) from the failed attempt into the replayed turn's `providerPayload` and append baseline.
@@ -95,6 +118,10 @@
- Fixed `mergeHeaders` merging case-sensitively on the Copilot/client-options path, where a miscased user-configured header (e.g. `authorization` next to the synthesized `Authorization`) survived as two keys that the `Headers` constructor joins comma-separated on the wire.
- Hardened the Anthropic stream lifecycle: prologue failures (e.g. a malformed Copilot credential in `buildCopilotDynamicHeaders`) and error-finalization failures now surface as an `error` event instead of an unhandled rejection that left `stream.result()` hanging forever; the spurious "cch billing placeholder not patched" warning no longer fires when the placeholder only appears in user content.
### Removed
- Removed the dead `iterateUntilAbort` helper (superseded by `iterateWithIdleTimeout`); it leaked the upstream iterator when the consumer abandoned mid-yield and had no production call sites.
## [15.10.9] - 2026-06-09
### Added
packages/ai/README.md
+1 -1
View File
@@ -68,7 +68,7 @@ Unified LLM API with automatic model discovery, provider configuration, token an
- **Kilo Gateway** (supports OAuth `/login kilo` or `KILO_API_KEY`)
- **LiteLLM** (requires `LITELLM_API_KEY`)
- **zAI** (requires `ZAI_API_KEY`)
- **MiniMax Coding Plan** (requires `MINIMAX_CODE_API_KEY` or `MINIMAX_CODE_CN_API_KEY`)
- **MiniMax Token Plan** (requires `MINIMAX_CODE_API_KEY` or `MINIMAX_CODE_CN_API_KEY`)
- **Xiaomi MiMo** (requires `XIAOMI_API_KEY`)
- **ZenMux** (requires `ZENMUX_API_KEY`)
- **Qwen Portal** (supports `QWEN_OAUTH_TOKEN` or `QWEN_PORTAL_API_KEY`)
packages/ai/package.json
+3 -27
View File
@@ -1,7 +1,7 @@
{
"type": "module",
"name": "@oh-my-pi/pi-ai",
"version": "15.10.10",
"version": "15.10.12",
"description": "Unified LLM API with automatic model discovery and provider configuration",
"homepage": "https://omp.sh",
"author": "Can Boluk",
@@ -34,11 +34,11 @@
"lint": "biome lint .",
"test": "bun test --parallel",
"fix": "biome check --write --unsafe .",
"fmt": "biome format --write .",
"generate-models": "bun scripts/generate-models.ts"
"fmt": "biome format --write ."
},
"dependencies": {
"@bufbuild/protobuf": "catalog:",
"@oh-my-pi/pi-catalog": "catalog:",
"@oh-my-pi/pi-utils": "catalog:",
"openai": "catalog:",
"partial-json": "catalog:",
@@ -80,26 +80,10 @@
"types": "./src/auth-gateway/*.ts",
"import": "./src/auth-gateway/*.ts"
},
"./models.json": {
"types": "./src/models.json.d.ts",
"import": "./src/models.json"
},
"./provider-models": {
"types": "./src/provider-models/index.ts",
"import": "./src/provider-models/index.ts"
},
"./provider-models/*": {
"types": "./src/provider-models/*.ts",
"import": "./src/provider-models/*.ts"
},
"./providers/*": {
"types": "./src/providers/*.ts",
"import": "./src/providers/*.ts"
},
"./providers/cursor/gen/*": {
"types": "./src/providers/cursor/gen/*.ts",
"import": "./src/providers/cursor/gen/*.ts"
},
"./providers/openai-codex/*": {
"types": "./src/providers/openai-codex/*.ts",
"import": "./src/providers/openai-codex/*.ts"
@@ -112,14 +96,6 @@
"types": "./src/utils/*.ts",
"import": "./src/utils/*.ts"
},
"./utils/discovery": {
"types": "./src/utils/discovery/index.ts",
"import": "./src/utils/discovery/index.ts"
},
"./utils/discovery/*": {
"types": "./src/utils/discovery/*.ts",
"import": "./src/utils/discovery/*.ts"
},
"./oauth": {
"types": "./src/registry/oauth/index.ts",
"import": "./src/registry/oauth/index.ts"
packages/ai/src/auth-gateway/http.ts
+1 -1
View File
@@ -74,7 +74,7 @@ const PASSTHROUGH_HEADER_NAMES: Record<string, true> = {
"openai-organization": true,
"openai-project": true,
"openai-beta": true,
// Codex / ChatGPT-OAuth backend headers (see openai-codex/constants.ts).
// Codex / ChatGPT-OAuth backend headers (see @oh-my-pi/pi-catalog/wire/codex).
// `session_id` and `conversation_id` thread the upstream session so prompt
// caching and per-conversation rate limiting work; `chatgpt-account-id` and
// `originator` identify the calling account and client surface.
packages/ai/src/auth-gateway/server.ts
+5 -2
View File
@@ -17,10 +17,11 @@
* POST /v1/messages → Anthropic messages in/out
* POST /v1/responses → OpenAI Responses in/out
*/
import { Effort } from "@oh-my-pi/pi-catalog/effort";
import { extractRetryHint, logger } from "@oh-my-pi/pi-utils";
import type { ApiKeyResolver } from "../auth-retry";
import type { AuthStorage } from "../auth-storage";
import { Effort } from "../effort";
import * as anthropicMessages from "../providers/anthropic-messages-server";
import * as openaiChat from "../providers/openai-chat-server";
import * as openaiResponses from "../providers/openai-responses-server";
@@ -315,9 +316,10 @@ async function refreshGatewayApiKeyAfterAuthError(
const message = error instanceof Error ? error.message : String(error);
if (isUsageLimitError(message)) {
const retryAfterMs = extractRetryHint(undefined, message);
const switched = await storage.markUsageLimitReached(provider, sessionId, {
const { switched, retryAtMs } = await storage.markUsageLimitReached(provider, sessionId, {
retryAfterMs,
baseUrl: model.baseUrl,
modelId: model.id,
signal,
});
logger.debug("auth-gateway retrying provider request after usage-limit block", {
@@ -326,6 +328,7 @@ async function refreshGatewayApiKeyAfterAuthError(
peer,
switched,
retryAfterMs,
retryAtMs,
error: message,
});
if (!switched) return undefined;
packages/ai/src/auth-gateway/types.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import type { Effort } from "../effort";
import type { Effort } from "@oh-my-pi/pi-catalog/effort";
import type {
AssistantMessage,
AssistantMessageEventStream,
packages/ai/src/auth-storage.ts
+151 -52
View File
@@ -19,6 +19,7 @@ import type { OAuthController, OAuthCredentials, OAuthProvider, OAuthProviderId
import { getEnvApiKey, getEnvApiKeyName } from "./stream";
import type { Provider } from "./types";
import type {
CredentialRankingContext,
CredentialRankingStrategy,
UsageCredential,
UsageFetchContext,
@@ -539,6 +540,23 @@ export function isDefinitiveOAuthFailure(errorMsg: string): boolean {
return false;
}
/**
* Outcome of {@link AuthStorage.markUsageLimitReached}.
*
* `switched` is `true` when an unblocked same-type sibling credential is
* available right now, so the caller can retry immediately and the next
* `getApiKey` will hand it out. When `false`, `retryAtMs` (epoch ms) carries
* the earliest moment any same-type sibling's temporary block expires —
* callers should prefer waiting until then over the provider's (often
* multi-hour) retry-after when it is sooner. `retryAtMs` is `undefined` when
* no sibling credentials exist at all, or when the session has no tracked
* credential to rotate away from.
*/
export interface UsageLimitMarkResult {
switched: boolean;
retryAtMs?: number;
}
type UsageCacheEntry<T> = {
value: T;
expiresAt: number;
@@ -1168,33 +1186,58 @@ export class AuthStorage {
return order;
}
/** Returns block expiry timestamp for a credential, cleaning up expired entries. */
#getCredentialBlockedUntil(providerKey: string, credentialIndex: number): number | undefined {
const backoffMap = this.#credentialBackoff.get(providerKey);
#toScopedBackoffKey(providerKey: string, blockScope: string | undefined): string {
return blockScope ? `${providerKey}\0${blockScope}` : providerKey;
}
/** Returns block expiry timestamp for a credential/key pair, cleaning up expired entries. */
#getCredentialBlockedUntilForKey(backoffKey: string, credentialIndex: number): number | undefined {
const backoffMap = this.#credentialBackoff.get(backoffKey);
if (!backoffMap) return undefined;
const blockedUntil = backoffMap.get(credentialIndex);
if (!blockedUntil) return undefined;
if (blockedUntil <= Date.now()) {
backoffMap.delete(credentialIndex);
if (backoffMap.size === 0) {
this.#credentialBackoff.delete(providerKey);
this.#credentialBackoff.delete(backoffKey);
}
return undefined;
}
return blockedUntil;
}
/** Returns block expiry timestamp for a credential, checking global then scoped blocks. */
#getCredentialBlockedUntil(
providerKey: string,
credentialIndex: number,
blockScope: string | undefined = undefined,
): number | undefined {
const globalBlockedUntil = this.#getCredentialBlockedUntilForKey(providerKey, credentialIndex);
if (globalBlockedUntil !== undefined || !blockScope) return globalBlockedUntil;
return this.#getCredentialBlockedUntilForKey(this.#toScopedBackoffKey(providerKey, blockScope), credentialIndex);
}
/** Checks if a credential is temporarily blocked due to usage limits. */
#isCredentialBlocked(providerKey: string, credentialIndex: number): boolean {
return this.#getCredentialBlockedUntil(providerKey, credentialIndex) !== undefined;
#isCredentialBlocked(
providerKey: string,
credentialIndex: number,
blockScope: string | undefined = undefined,
): boolean {
return this.#getCredentialBlockedUntil(providerKey, credentialIndex, blockScope) !== undefined;
}
/** Marks a credential as blocked until the specified time. */
#markCredentialBlocked(providerKey: string, credentialIndex: number, blockedUntilMs: number): void {
const backoffMap = this.#credentialBackoff.get(providerKey) ?? new Map<number, number>();
#markCredentialBlocked(
providerKey: string,
credentialIndex: number,
blockedUntilMs: number,
blockScope: string | undefined = undefined,
): void {
const backoffKey = this.#toScopedBackoffKey(providerKey, blockScope);
const backoffMap = this.#credentialBackoff.get(backoffKey) ?? new Map<number, number>();
const existing = backoffMap.get(credentialIndex) ?? 0;
backoffMap.set(credentialIndex, Math.max(existing, blockedUntilMs));
this.#credentialBackoff.set(providerKey, backoffMap);
this.#credentialBackoff.set(backoffKey, backoffMap);
}
/** Records which credential was used for a session (for rate-limit switching). */
@@ -2157,15 +2200,24 @@ export class AuthStorage {
return false;
}
/** Return the usage limits that apply to the requested model for this strategy. */
#getScopedUsageLimits(
strategy: CredentialRankingStrategy,
report: UsageReport,
context: CredentialRankingContext,
): UsageLimit[] {
return strategy.scopeLimits?.(report, context) ?? report.limits;
}
/** Returns true if usage indicates rate limit has been reached. */
#isUsageLimitReached(report: UsageReport): boolean {
return report.limits.some(limit => this.#isUsageLimitExhausted(limit));
#isUsageLimitReached(limits: UsageLimit[]): boolean {
return limits.some(limit => this.#isUsageLimitExhausted(limit));
}
/** Extracts the earliest reset timestamp from exhausted windows (in ms). */
#getUsageResetAtMs(report: UsageReport, nowMs: number): number | undefined {
#getUsageResetAtMs(limits: UsageLimit[], nowMs: number): number | undefined {
const candidates: number[] = [];
for (const limit of report.limits) {
for (const limit of limits) {
if (!this.#isUsageLimitExhausted(limit)) continue;
const window = limit.window;
if (window?.resetsAt && window.resetsAt > nowMs) {
@@ -2451,34 +2503,42 @@ export class AuthStorage {
/**
* Marks the current session's credential as temporarily blocked due to usage limits.
* Uses usage reports to determine accurate reset time when available.
* Returns true if a credential was blocked, enabling automatic fallback to the next credential.
* Returns whether a sibling credential is available now; when none is, also
* reports the earliest time a blocked sibling becomes available again so
* callers can wait for the sibling instead of the provider's full window.
*/
async markUsageLimitReached(
provider: string,
sessionId: string | undefined,
options?: { retryAfterMs?: number; baseUrl?: string; signal?: AbortSignal },
): Promise<boolean> {
options?: { retryAfterMs?: number; baseUrl?: string; modelId?: string; signal?: AbortSignal },
): Promise<UsageLimitMarkResult> {
const sessionCredential = this.#getSessionCredential(provider, sessionId);
if (!sessionCredential) return false;
if (!sessionCredential) return { switched: false };
const providerKey = this.#getProviderTypeKey(provider, sessionCredential.type);
const strategy = this.#rankingStrategyResolver?.(provider);
const rankingContext: CredentialRankingContext = { modelId: options?.modelId };
const blockScope = strategy?.blockScope?.(rankingContext);
const now = Date.now();
let blockedUntil = now + (options?.retryAfterMs ?? AuthStorage.#defaultBackoffMs);
if (sessionCredential.type === "oauth" && this.#rankingStrategyResolver?.(provider)) {
if (sessionCredential.type === "oauth" && strategy) {
const credential = this.#getCredentialsForProvider(provider)[sessionCredential.index];
if (credential?.type === "oauth") {
const report = await this.#getUsageReport(provider, credential, options);
if (report && this.#isUsageLimitReached(report)) {
const resetAtMs = this.#getUsageResetAtMs(report, Date.now());
if (resetAtMs && resetAtMs > blockedUntil) {
blockedUntil = resetAtMs;
if (report) {
const scopedLimits = this.#getScopedUsageLimits(strategy, report, rankingContext);
if (this.#isUsageLimitReached(scopedLimits)) {
const resetAtMs = this.#getUsageResetAtMs(scopedLimits, Date.now());
if (resetAtMs && resetAtMs > blockedUntil) {
blockedUntil = resetAtMs;
}
}
}
}
}
this.#markCredentialBlocked(providerKey, sessionCredential.index, blockedUntil);
this.#markCredentialBlocked(providerKey, sessionCredential.index, blockedUntil, blockScope);
const remainingCredentials = this.#getCredentialsForProvider(provider)
.map((credential, index) => ({ credential, index }))
@@ -2487,7 +2547,13 @@ export class AuthStorage {
entry.credential.type === sessionCredential.type && entry.index !== sessionCredential.index,
);
return remainingCredentials.some(candidate => !this.#isCredentialBlocked(providerKey, candidate.index));
let retryAtMs: number | undefined;
for (const candidate of remainingCredentials) {
const candidateBlockedUntil = this.#getCredentialBlockedUntil(providerKey, candidate.index, blockScope);
if (candidateBlockedUntil === undefined) return { switched: true };
if (retryAtMs === undefined || candidateBlockedUntil < retryAtMs) retryAtMs = candidateBlockedUntil;
}
return { switched: false, retryAtMs };
}
#resolveWindowResetAt(window: UsageLimit["window"]): number | undefined {
@@ -2648,6 +2714,8 @@ export class AuthStorage {
options?: AuthApiKeyOptions;
sessionId?: string;
strategy: CredentialRankingStrategy;
rankingContext: CredentialRankingContext;
blockScope?: string;
}): Promise<OAuthCandidate[]> {
const nowMs = Date.now();
const { strategy } = args;
@@ -2661,7 +2729,7 @@ export class AuthStorage {
args.order.map(async idx => {
const selection = args.credentials[idx];
if (!selection) return null;
const blockedUntil = this.#getCredentialBlockedUntil(args.providerKey, selection.index);
const blockedUntil = this.#getCredentialBlockedUntil(args.providerKey, selection.index, args.blockScope);
if (blockedUntil !== undefined) return { selection, usage: null, usageChecked: false, blockedUntil };
const usage = await this.#getUsageReport(args.provider, selection.credential, {
...args.options,
@@ -2694,13 +2762,14 @@ export class AuthStorage {
const { selection, usage, usageChecked } = result;
let { blockedUntil } = result;
let blocked = blockedUntil !== undefined;
if (!blocked && usage && this.#isUsageLimitReached(usage)) {
const resetAtMs = this.#getUsageResetAtMs(usage, nowMs);
const scopedLimits = usage ? this.#getScopedUsageLimits(strategy, usage, args.rankingContext) : undefined;
if (!blocked && scopedLimits && this.#isUsageLimitReached(scopedLimits)) {
const resetAtMs = this.#getUsageResetAtMs(scopedLimits, nowMs);
blockedUntil = resetAtMs ?? Date.now() + AuthStorage.#defaultBackoffMs;
this.#markCredentialBlocked(args.providerKey, selection.index, blockedUntil);
this.#markCredentialBlocked(args.providerKey, selection.index, blockedUntil, args.blockScope);
blocked = true;
}
const windows = usage ? strategy.findWindowLimits(usage) : undefined;
const windows = usage ? strategy.findWindowLimits(usage, args.rankingContext) : undefined;
const primary = windows?.primary;
const secondary = windows?.secondary;
const secondaryTarget = secondary ?? primary;
@@ -2749,6 +2818,8 @@ export class AuthStorage {
const providerKey = this.#getProviderTypeKey(provider, "oauth");
const order = this.#getCredentialOrder(providerKey, sessionId, credentials.length);
const strategy = this.#rankingStrategyResolver?.(provider);
const rankingContext: CredentialRankingContext = { modelId: options?.modelId };
const blockScope = strategy?.blockScope?.(rankingContext);
const requiresProModel = requiresOpenAICodexProModel(provider, options?.modelId);
const checkUsage = strategy !== undefined && (credentials.length > 1 || requiresProModel);
const sessionCredential = this.#getSessionCredential(provider, sessionId);
@@ -2758,7 +2829,8 @@ export class AuthStorage {
// (no preference) and sessions whose preferred is blocked still rank, so we pick the account
// with the most headroom proactively and fall back intelligently when rate-limited.
const sessionPreferredIsAvailable =
sessionPreferredIndex !== undefined && !this.#isCredentialBlocked(providerKey, sessionPreferredIndex);
sessionPreferredIndex !== undefined &&
!this.#isCredentialBlocked(providerKey, sessionPreferredIndex, blockScope);
const shouldRank = checkUsage && (!sessionPreferredIsAvailable || requiresProModel);
const rankingOrder = shouldRank && sessionId ? credentials.map((_credential, index) => index) : order;
const candidates = shouldRank
@@ -2770,6 +2842,8 @@ export class AuthStorage {
options,
sessionId,
strategy: strategy!,
rankingContext,
blockScope,
})
: order
.map(idx => credentials[idx])
@@ -2779,7 +2853,7 @@ export class AuthStorage {
if (sessionPreferredIndex !== undefined && !requiresProModel) {
const sessionPreferredCandidate = candidates.findIndex(
candidate =>
!this.#isCredentialBlocked(providerKey, candidate.selection.index) &&
!this.#isCredentialBlocked(providerKey, candidate.selection.index, blockScope) &&
candidate.selection.index === sessionPreferredIndex,
);
if (sessionPreferredCandidate > 0) {
@@ -2853,18 +2927,24 @@ export class AuthStorage {
prefetchedUsage: candidate.usage,
usagePrechecked: candidate.usageChecked,
enforceProRequirement,
strategy,
rankingContext,
blockScope,
},
);
if (resolved) return resolved;
}
if (fallback && this.#isCredentialBlocked(providerKey, fallback.selection.index)) {
if (fallback && this.#isCredentialBlocked(providerKey, fallback.selection.index, blockScope)) {
return this.#tryOAuthCredential(provider, fallback.selection, providerKey, sessionId, options, {
checkUsage,
allowBlocked: true,
prefetchedUsage: fallback.usage,
usagePrechecked: fallback.usageChecked,
enforceProRequirement,
strategy,
rankingContext,
blockScope,
});
}
@@ -2983,6 +3063,9 @@ export class AuthStorage {
prefetchedUsage?: UsageReport | null;
usagePrechecked?: boolean;
enforceProRequirement?: boolean;
strategy?: CredentialRankingStrategy;
rankingContext?: CredentialRankingContext;
blockScope?: string;
},
): Promise<OAuthResolutionResult | undefined> {
const {
@@ -2991,8 +3074,11 @@ export class AuthStorage {
prefetchedUsage = null,
usagePrechecked = false,
enforceProRequirement,
strategy,
rankingContext,
blockScope,
} = usageOptions;
if (!allowBlocked && this.#isCredentialBlocked(providerKey, selection.index)) {
if (!allowBlocked && this.#isCredentialBlocked(providerKey, selection.index, blockScope)) {
return undefined;
}
@@ -3019,14 +3105,18 @@ export class AuthStorage {
if (applyProFilter && !hasOpenAICodexProPlan(usage)) {
return undefined;
}
if (checkUsage && !allowBlocked && usage && this.#isUsageLimitReached(usage)) {
const resetAtMs = this.#getUsageResetAtMs(usage, Date.now());
this.#markCredentialBlocked(
providerKey,
selection.index,
resetAtMs ?? Date.now() + AuthStorage.#defaultBackoffMs,
);
return undefined;
if (checkUsage && !allowBlocked && usage && strategy && rankingContext) {
const scopedLimits = this.#getScopedUsageLimits(strategy, usage, rankingContext);
if (this.#isUsageLimitReached(scopedLimits)) {
const resetAtMs = this.#getUsageResetAtMs(scopedLimits, Date.now());
this.#markCredentialBlocked(
providerKey,
selection.index,
resetAtMs ?? Date.now() + AuthStorage.#defaultBackoffMs,
blockScope,
);
return undefined;
}
}
}
@@ -3085,14 +3175,18 @@ export class AuthStorage {
if (applyProFilter && !hasOpenAICodexProPlan(usage)) {
return undefined;
}
if (checkUsage && !allowBlocked && usage && this.#isUsageLimitReached(usage)) {
const resetAtMs = this.#getUsageResetAtMs(usage, Date.now());
this.#markCredentialBlocked(
providerKey,
selection.index,
resetAtMs ?? Date.now() + AuthStorage.#defaultBackoffMs,
);
return undefined;
if (checkUsage && !allowBlocked && usage && strategy && rankingContext) {
const scopedLimits = this.#getScopedUsageLimits(strategy, usage, rankingContext);
if (this.#isUsageLimitReached(scopedLimits)) {
const resetAtMs = this.#getUsageResetAtMs(scopedLimits, Date.now());
this.#markCredentialBlocked(
providerKey,
selection.index,
resetAtMs ?? Date.now() + AuthStorage.#defaultBackoffMs,
blockScope,
);
return undefined;
}
}
}
this.#recordSessionCredential(provider, sessionId, "oauth", selection.index);
@@ -3448,7 +3542,7 @@ export class AuthStorage {
async rotateSessionCredential(
provider: string,
sessionId: string | undefined,
options?: { error?: unknown; signal?: AbortSignal },
options?: { error?: unknown; modelId?: string; signal?: AbortSignal },
): Promise<boolean> {
const sessionCredential = this.#getSessionCredential(provider, sessionId);
if (!sessionCredential) return false;
@@ -3456,7 +3550,12 @@ export class AuthStorage {
const error = options?.error;
const message = error instanceof Error ? error.message : typeof error === "string" ? error : undefined;
if (message && isUsageLimitError(message)) {
return this.markUsageLimitReached(provider, sessionId, { signal: options?.signal });
return (
await this.markUsageLimitReached(provider, sessionId, {
modelId: options?.modelId,
signal: options?.signal,
})
).switched;
}
const providerKey = this.#getProviderTypeKey(provider, sessionCredential.type);
@@ -3507,7 +3606,7 @@ export class AuthStorage {
return this.getApiKey(provider, sessionId, { baseUrl, modelId, signal });
}
if (lastChance) {
await this.rotateSessionCredential(provider, sessionId, { error, signal });
await this.rotateSessionCredential(provider, sessionId, { error, modelId, signal });
return this.getApiKey(provider, sessionId, { baseUrl, modelId, signal });
}
return this.getApiKey(provider, sessionId, { baseUrl, modelId, forceRefresh: true, signal });
packages/ai/src/index.ts
-8
View File
@@ -5,13 +5,7 @@ export { type AuthGatewayBootOptions, type ModelResolver, startAuthGateway } fro
export * from "./auth-gateway/types";
export * from "./auth-retry";
export * from "./auth-storage";
export * from "./effort";
export * from "./model-cache";
export * from "./model-manager";
export * from "./model-thinking";
export * from "./models";
export * from "./provider-details";
export * from "./provider-models";
export * from "./providers/anthropic";
export * from "./providers/anthropic-client";
export * from "./providers/azure-openai-responses";
@@ -19,7 +13,6 @@ export type * from "./providers/cursor";
export * from "./providers/gitlab-duo";
export type * from "./providers/google";
export type * from "./providers/google-gemini-cli";
export * from "./providers/google-gemini-headers";
export type * from "./providers/google-vertex";
export * from "./providers/kimi";
export * from "./providers/mock";
@@ -42,7 +35,6 @@ export * from "./usage/minimax-code";
export * from "./usage/openai-codex";
export * from "./usage/zai";
export * from "./utils/anthropic-auth";
export * from "./utils/discovery";
export * from "./utils/event-stream";
export * from "./utils/overflow";
export * from "./utils/retry";
packages/ai/src/model-thinking.ts
-770
View File
@@ -1,770 +0,0 @@
import { Effort, THINKING_EFFORTS } from "./effort";
import { resolveOpenAICompat } from "./providers/openai-completions-compat";
import type { Api, Model as ApiModel, ThinkingConfig } from "./types";
const DEFAULT_REASONING_EFFORTS: readonly Effort[] = [Effort.Minimal, Effort.Low, Effort.Medium, Effort.High];
const DEFAULT_REASONING_EFFORTS_WITH_XHIGH: readonly Effort[] = [
Effort.Minimal,
Effort.Low,
Effort.Medium,
Effort.High,
Effort.XHigh,
];
const GEMINI_3_PRO_EFFORTS: readonly Effort[] = [Effort.Low, Effort.High];
const GEMINI_3_FLASH_EFFORTS: readonly Effort[] = [Effort.Minimal, Effort.Low, Effort.Medium, Effort.High];
const GPT_5_2_PLUS_EFFORTS: readonly Effort[] = [Effort.Low, Effort.Medium, Effort.High, Effort.XHigh];
const GPT_5_1_CODEX_MINI_EFFORTS: readonly Effort[] = [Effort.Medium, Effort.High];
const CLOUDFLARE_AI_GATEWAY_BASE_URL = "https://gateway.ai.cloudflare.com/v1/<account>/<gateway>/anthropic";
type SemVer = {
major: number;
minor: number;
patch: number;
};
type GeminiKind = "pro" | "flash";
type AnthropicKind = "opus" | "sonnet" | "fable" | "mythos";
type OpenAIVariant = "base" | "codex" | "codex-max" | "codex-mini" | "codex-spark" | "mini" | "max" | "nano";
const CODEX_GPT_5_4_PRIORITY_BY_VARIANT: Partial<Record<OpenAIVariant, number>> = {
base: 0,
mini: 1,
nano: 2,
};
const COPILOT_GENERATED_LIMITS: Record<string, { contextWindow: number; maxTokens: number }> = {
"claude-opus-4.6": { contextWindow: 168000, maxTokens: 32000 },
"gpt-5.2": { contextWindow: 272000, maxTokens: 128000 },
"gpt-5.4": { contextWindow: 272000, maxTokens: 128000 },
"gpt-5.4-mini": { contextWindow: 272000, maxTokens: 128000 },
"grok-code-fast-1": { contextWindow: 192000, maxTokens: 64000 },
};
interface GeminiModel {
family: "gemini";
kind: GeminiKind;
version: SemVer;
}
interface AnthropicModel {
family: "anthropic";
kind: AnthropicKind;
version: SemVer;
}
interface OpenAIModel {
family: "openai";
variant: OpenAIVariant;
version: SemVer;
}
interface UnknownModel {
family: "unknown";
id: string;
}
type ParsedModel = GeminiModel | AnthropicModel | OpenAIModel | UnknownModel;
/**
* Static fallback model injected when Cloudflare AI Gateway discovery
* returns no results. Ensures the provider always has at least one usable
* model entry in the catalog.
*/
export const CLOUDFLARE_FALLBACK_MODEL: ApiModel<"anthropic-messages"> = {
id: "claude-sonnet-4-5",
name: "Claude Sonnet 4.5",
api: "anthropic-messages",
provider: "cloudflare-ai-gateway",
baseUrl: CLOUDFLARE_AI_GATEWAY_BASE_URL,
reasoning: true,
input: ["text", "image"],
cost: {
input: 3,
output: 15,
cacheRead: 0.3,
cacheWrite: 3.75,
},
contextWindow: 200000,
maxTokens: 64000,
};
const kEnrichedModel = Symbol("model-thinking.enrichedModel");
type ModelWithEnriched = ApiModel<Api> & { [kEnrichedModel]?: ApiModel<Api> };
/**
* Returns a copy of the model with canonical thinking metadata attached.
*
* This helper belongs to catalog enrichment only. Runtime consumers should
* trust `model.thinking` and avoid inferring capabilities on demand.
*/
export function enrichModelThinking<TApi extends Api>(model: ApiModel<TApi>): ApiModel<TApi> {
const tagged = model as ModelWithEnriched;
const cached = tagged[kEnrichedModel];
if (cached !== undefined) {
return cached as ApiModel<TApi>;
}
const normalizedThinking = normalizeThinkingConfig(model.thinking);
let result: ApiModel<TApi>;
if (!model.reasoning) {
result =
normalizedThinking === undefined && model.thinking === undefined ? model : { ...model, thinking: undefined };
} else {
const thinking = normalizedThinking ?? inferModelThinking(model);
result = thinkingsEqual(normalizedThinking, thinking) ? model : { ...model, thinking };
}
// Stash the enriched copy on a non-enumerable slot so callers that hand us
// the same reference twice skip the work. `enumerable: false` is critical:
// many call sites build derived models via `{ ...model, ...overrides }`,
// which would otherwise copy this cache slot and trick us into returning
// the *original* enriched model — silently discarding the overrides.
Object.defineProperty(tagged, kEnrichedModel, {
value: result,
enumerable: false,
configurable: true,
writable: true,
});
return result;
}
/**
* Returns a copy of the model with thinking metadata recomputed from the
* canonical rules, replacing any existing `thinking`.
*/
export function refreshModelThinking<TApi extends Api>(model: ApiModel<TApi>): ApiModel<TApi> {
if (!model.reasoning) {
const normalizedThinking = normalizeThinkingConfig(model.thinking);
return normalizedThinking === undefined && model.thinking === undefined
? model
: { ...model, thinking: undefined };
}
return { ...model, thinking: inferModelThinking(model) };
}
/**
* Apply upstream metadata corrections to a mutable array of models.
*
* Each model is first normalized through `refreshModelThinking()` so generated
* catalogs keep canonical thinking metadata and policy fixes in one pass.
*/
export function applyGeneratedModelPolicies(models: ApiModel<Api>[]): void {
for (let index = 0; index < models.length; index++) {
const model = refreshModelThinking(models[index]!);
applyGeneratedModelPolicy(model);
models[index] = model;
}
}
/**
* Link OpenAI model variants to their context promotion targets.
*
* When a model's context is exhausted, the agent can promote to a sibling
* model with a larger context window on the same provider:
* - `codex-spark` variants promote to `gpt-5.5`.
* - `gpt-5.5` (270K input) promotes to `gpt-5.4` (1M input).
*/
export function linkOpenAIPromotionTargets(models: ApiModel<Api>[]): void {
for (const candidate of models) {
const parsedCandidate = parseKnownModel(candidate.id);
if (parsedCandidate.family !== "openai") continue;
let targetId: string | undefined;
if (parsedCandidate.variant === "codex-spark") {
targetId = "gpt-5.5";
} else if (parsedCandidate.variant === "base" && semverEqual(parsedCandidate.version, "5.5")) {
targetId = "gpt-5.4";
} else {
continue;
}
const fallback = models.find(
model => model.provider === candidate.provider && model.api === candidate.api && model.id === targetId,
);
if (!fallback) continue;
candidate.contextPromotionTarget = `${fallback.provider}/${fallback.id}`;
}
}
/**
* True when the model reasons natively but rejects the wire `reasoning.effort`
* param (compat.supportsReasoningEffort: false on openai-responses*). Callers
* are expected to omit the effort field; the wire-side omitReasoningEffort
* gate (providers/xai-responses.ts:78) is the actual strip, and this
* predicate is the upstream check that prevents a redundant
* requireSupportedEffort throw from defeating that gate.
*
* Scoped to openai-responses* because that's the only API surface where
* `compat.supportsReasoningEffort: false` is meaningful today. The
* `in`-narrowed access is necessary because Model.compat is
* `AnthropicCompat | OpenAICompat` and the api gate doesn't narrow the
* union for TS.
*/
export function modelOmitsReasoningEffort<TApi extends Api>(model: ApiModel<TApi>): boolean {
if (model.api !== "openai-responses" && model.api !== "openai-codex-responses") {
return false;
}
const compat = model.compat;
return Boolean(compat && "supportsReasoningEffort" in compat && compat.supportsReasoningEffort === false);
}
/**
* Returns the supported thinking efforts declared on the model metadata.
*
* Catalog enrichment is responsible for normalizing bundled model metadata up front.
* Runtime callers must treat explicit `model.thinking` on custom models as authoritative
* so proxy-specific overrides from `models.yml` survive request construction.
*
* @throws Error when a reasoning-capable model is missing thinking metadata
*/
export function getSupportedEfforts<TApi extends Api>(model: ApiModel<TApi>): readonly Effort[] {
if (!model.reasoning) {
return [];
}
// Models that reason natively but reject the `reasoning.effort` wire param
// (xAI Grok off the GROK_EFFORT_CAPABLE_PREFIXES allowlist in
// providers/xai-responses.ts: grok-build, grok-4.20-0309-reasoning) hide the
// picker's effort dial. Scoped to openai-responses* by
// `modelOmitsReasoningEffort` — openai-completions has its own
// supportsReasoningEffort consultation at inferFallbackEfforts L536 and
// changing that path's semantics is out-of-scope.
if (modelOmitsReasoningEffort(model)) {
return [];
}
if (!model.thinking) {
throw new Error(`Model ${model.provider}/${model.id} is missing thinking metadata`);
}
return expandEffortRange(model.thinking);
}
/**
* Clamps a requested thinking level against explicit model metadata.
*
* Non-reasoning models always resolve to `undefined`.
*/
export function clampThinkingLevelForModel<TApi extends Api>(
model: ApiModel<TApi> | undefined,
requested: Effort | undefined,
): Effort | undefined {
if (!model) {
return requested;
}
if (!model.reasoning || requested === undefined) {
return undefined;
}
const levels = getSupportedEfforts(model);
if (levels.includes(requested)) {
return requested;
}
const requestedIndex = THINKING_EFFORTS.indexOf(requested);
if (requestedIndex === -1) {
return undefined;
}
let clamped: Effort | undefined;
for (const effort of levels) {
if (THINKING_EFFORTS.indexOf(effort) > requestedIndex) {
break;
}
clamped = effort;
}
return clamped ?? levels[0];
}
export function requireSupportedEffort<TApi extends Api>(model: ApiModel<TApi>, effort: Effort): Effort {
if (!model.reasoning) {
throw new Error(`Model ${model.provider}/${model.id} does not support thinking`);
}
const levels = getSupportedEfforts(model);
if (!levels.includes(effort)) {
throw new Error(
`Thinking effort ${effort} is not supported by ${model.provider}/${model.id}. Supported efforts: ${levels.join(", ")}`,
);
}
return effort;
}
/** Maps a normalized thinking effort to Google's `thinkingLevel` enum values. */
export function mapEffortToGoogleThinkingLevel<TApi extends Api>(
model: ApiModel<TApi>,
effort: Effort,
): "MINIMAL" | "LOW" | "MEDIUM" | "HIGH" {
switch (requireSupportedEffort(model, effort)) {
case Effort.Minimal:
return "MINIMAL";
case Effort.Low:
return "LOW";
case Effort.Medium:
return "MEDIUM";
case Effort.High:
case Effort.XHigh:
return "HIGH";
}
}
/** Maps a normalized thinking effort to Anthropic adaptive effort values. */
export function mapEffortToAnthropicAdaptiveEffort<TApi extends Api>(
model: ApiModel<TApi>,
effort: Effort,
): "low" | "medium" | "high" | "xhigh" | "max" {
const supported = requireSupportedEffort(model, effort);
if (anthropicModelHasRealXHighEffort(model)) {
// Opus 4.7+ and Fable/Mythos 5 on the Messages API expose the full
// five-tier adaptive scale
// (low/medium/high/xhigh/max). Shift our user-facing efforts up one notch so
// the top tier reaches the genuine "max" and "high" lands on Anthropic's
// recommended "xhigh" coding/agentic default.
switch (supported) {
case Effort.Minimal:
return "low";
case Effort.Low:
return "medium";
case Effort.Medium:
return "high";
case Effort.High:
return "xhigh";
case Effort.XHigh:
return "max";
}
}
// Older adaptive models (Opus 4.6) and Bedrock Converse expose only four tiers
// with no real "xhigh"; XHigh is a legacy alias for the top "max" tier there.
switch (supported) {
case Effort.Minimal:
case Effort.Low:
return "low";
case Effort.Medium:
return "medium";
case Effort.High:
return "high";
case Effort.XHigh:
return "max";
}
}
/**
* Returns true for Anthropic models with Opus 4.7+/Fable/Mythos API restrictions:
* - Sampling parameters (temperature/top_p/top_k) return 400 error
* - Thinking content is omitted by default (needs display: "summarized")
*/
export function hasOpus47ApiRestrictions(modelId: string): boolean {
const parsed = parseAnthropicModel(getCanonicalModelId(modelId));
if (!parsed) return false;
return (parsed.kind === "opus" && semverGte(parsed.version, "4.7")) || isFableOrMythos(parsed.kind);
}
/**
* Mid-conversation `role: "system"` messages (system instructions appended at
* non-first positions in the `messages` array) are supported starting with
* Claude Opus 4.8 and the Claude Fable/Mythos 5 generation. Earlier Claude
* models reject the role.
* @see https://platform.claude.com/docs/en/build-with-claude/mid-conversation-system-messages
*/
export function supportsMidConversationSystemMessages(modelId: string): boolean {
const parsed = parseAnthropicModel(getCanonicalModelId(modelId));
if (!parsed) return false;
return (parsed.kind === "opus" && semverGte(parsed.version, "4.8")) || isFableOrMythos(parsed.kind);
}
export function isAnthropicFableOrMythosModel(modelId: string): boolean {
const parsed = parseAnthropicModel(getCanonicalModelId(modelId));
return parsed !== null && isFableOrMythos(parsed.kind);
}
function isFableOrMythos(kind: AnthropicKind): boolean {
return kind === "fable" || kind === "mythos";
}
function isOpenRouterAnthropicAdaptiveReasoningModel<TApi extends Api>(
parsedModel: AnthropicModel,
model: ApiModel<TApi>,
): boolean {
if (model.api !== "openai-completions") return false;
if (model.provider !== "openrouter" && !model.baseUrl.includes("openrouter.ai")) return false;
return isFableOrMythos(parsedModel.kind) || (parsedModel.kind === "opus" && semverGte(parsedModel.version, "4.6"));
}
function anthropicModelHasRealXHighEffort<TApi extends Api>(model: ApiModel<TApi>): boolean {
if (model.api !== "anthropic-messages") return false;
const parsedModel = parseKnownModel(model.id);
if (parsedModel.family !== "anthropic") return false;
if (isFableOrMythos(parsedModel.kind)) return true;
return parsedModel.kind === "opus" && semverGte(parsedModel.version, "4.7");
}
function applyGeneratedModelPolicy(model: ApiModel<Api>): void {
const copilotLimits = model.provider === "github-copilot" ? COPILOT_GENERATED_LIMITS[model.id] : undefined;
if (copilotLimits) {
model.contextWindow = copilotLimits.contextWindow;
model.maxTokens = copilotLimits.maxTokens;
}
if (
model.api === "openai-completions" &&
(model.provider === "minimax-code" || model.provider === "minimax-code-cn")
) {
model.compat = {
...(model.compat ?? {}),
supportsStore: false,
supportsDeveloperRole: false,
supportsReasoningEffort: false,
reasoningContentField: "reasoning_content",
};
delete model.compat.thinkingFormat;
}
if (
model.api === "openai-completions" &&
model.provider === "opencode-go" &&
(model.id === "deepseek-v4-flash" || model.id === "deepseek-v4-pro")
) {
model.compat = {
...(model.compat ?? {}),
supportsToolChoice: false,
reasoningContentField: "reasoning_content",
requiresReasoningContentForToolCalls: true,
};
}
const parsedModel = parseKnownModel(model.id);
const applyPatchToolType = inferGeneratedApplyPatchToolType(model, parsedModel);
if (applyPatchToolType) {
model.applyPatchToolType = applyPatchToolType;
} else {
delete model.applyPatchToolType;
}
if (parsedModel.family === "anthropic") {
applyAnthropicCatalogPolicy(model, parsedModel);
}
if (parsedModel.family === "openai") {
applyOpenAICatalogPolicy(model, parsedModel);
}
}
function applyAnthropicCatalogPolicy(model: ApiModel<Api>, parsedModel: AnthropicModel): void {
// Claude Opus 4.5: models.dev reports 3x the correct cache pricing.
if (model.provider === "anthropic" && parsedModel.kind === "opus" && semverEqual(parsedModel.version, "4.5")) {
model.cost.cacheRead = 0.5;
model.cost.cacheWrite = 6.25;
}
// Bedrock Opus 4.6: upstream metadata is stale for cache pricing and context.
if (model.provider === "amazon-bedrock" && parsedModel.kind === "opus" && semverEqual(parsedModel.version, "4.6")) {
model.cost.cacheRead = 0.5;
model.cost.cacheWrite = 6.25;
model.contextWindow = 1000000;
model.maxTokens = 128000;
}
// Claude Fable/Mythos 5: Anthropic's /v1/models omits token limits and
// pricing, and models.dev lags new releases. Pin authoritative values from
// the model card (1M context / 128k output) and pricing docs ($10 in / $50
// out per MTok).
if (model.provider === "anthropic" && isFableOrMythos(parsedModel.kind)) {
model.contextWindow = 1_000_000;
model.maxTokens = 128_000;
model.cost.input = 10;
model.cost.output = 50;
model.cost.cacheRead = 1;
model.cost.cacheWrite = 12.5;
}
}
function inferGeneratedApplyPatchToolType(
model: ApiModel<Api>,
parsedModel: ParsedModel,
): ApiModel<Api>["applyPatchToolType"] {
if (parsedModel.family !== "openai" || parsedModel.version.major !== 5) {
return undefined;
}
if (model.provider === "openai" && model.api === "openai-responses") {
return "freeform";
}
if (model.provider === "openai-codex" && model.api === "openai-codex-responses") {
return "freeform";
}
return undefined;
}
function applyOpenAICatalogPolicy(model: ApiModel<Api>, parsedModel: OpenAIModel): void {
// Codex models: 400K figure includes output budget; input window is 272K.
if (parsedModel.variant.startsWith("codex") && parsedModel.variant !== "codex-spark") {
model.contextWindow = 272000;
return;
}
// GPT-5.4 mini/nano use plain OpenAI IDs on the Codex transport, but Codex still
// enforces the lower prompt budget for these variants. Codex discovery can also
// report inconsistent priorities for the GPT-5.4 family, so normalize by parsed
// variant instead of special-casing raw model ids.
if (model.api === "openai-codex-responses" && semverEqual(parsedModel.version, "5.4")) {
const normalizedPriority = CODEX_GPT_5_4_PRIORITY_BY_VARIANT[parsedModel.variant];
if (normalizedPriority !== undefined) {
model.priority = normalizedPriority;
}
if (parsedModel.variant === "mini" || parsedModel.variant === "nano") {
model.contextWindow = 272000;
}
}
}
function inferModelThinking<TApi extends Api>(model: ApiModel<TApi>): ThinkingConfig {
const parsedModel = parseKnownModel(model.id);
const efforts = inferSupportedEfforts(parsedModel, model);
const minLevel = efforts[0];
const maxLevel = efforts.at(-1);
if (!minLevel || !maxLevel) {
throw new Error(`Model ${model.provider}/${model.id} resolved to an empty thinking range`);
}
const config: ThinkingConfig = {
mode: inferThinkingControlMode(model, parsedModel),
minLevel,
maxLevel,
};
// Encode explicit levels only when the inferred set has gaps the min..max range cannot represent.
const minIndex = THINKING_EFFORTS.indexOf(minLevel);
const maxIndex = THINKING_EFFORTS.indexOf(maxLevel);
const expandedRange = THINKING_EFFORTS.slice(minIndex, maxIndex + 1);
if (expandedRange.length !== efforts.length) {
config.levels = efforts;
}
return config;
}
function normalizeThinkingConfig(thinking: ThinkingConfig | undefined): ThinkingConfig | undefined {
if (!thinking || expandEffortRange(thinking).length === 0) {
return undefined;
}
return thinking;
}
function thinkingsEqual(left: ThinkingConfig | undefined, right: ThinkingConfig | undefined): boolean {
if (left === right) return true;
if (!left || !right) return false;
if (left.mode !== right.mode || left.minLevel !== right.minLevel || left.maxLevel !== right.maxLevel) return false;
const leftLevels = left.levels;
const rightLevels = right.levels;
if (leftLevels === rightLevels) return true;
if (!leftLevels || !rightLevels) return false;
if (leftLevels.length !== rightLevels.length) return false;
return leftLevels.every((level, index) => level === rightLevels[index]);
}
function expandEffortRange(thinking: ThinkingConfig): readonly Effort[] {
if (thinking.levels && thinking.levels.length > 0) {
return thinking.levels;
}
const minIndex = THINKING_EFFORTS.indexOf(thinking.minLevel);
const maxIndex = THINKING_EFFORTS.indexOf(thinking.maxLevel);
if (minIndex === -1 || maxIndex === -1 || minIndex > maxIndex) {
return [];
}
return THINKING_EFFORTS.slice(minIndex, maxIndex + 1);
}
function inferSupportedEfforts<TApi extends Api>(parsedModel: ParsedModel, model: ApiModel<TApi>): readonly Effort[] {
switch (parsedModel.family) {
case "openai":
return inferOpenAISupportedEfforts(parsedModel);
case "gemini":
return inferGeminiSupportedEfforts(parsedModel);
case "anthropic":
return inferAnthropicSupportedEfforts(parsedModel, model);
case "unknown":
return inferFallbackEfforts(model);
}
}
function inferOpenAISupportedEfforts(model: OpenAIModel): readonly Effort[] {
if (model.variant === "codex-mini" && semverEqual(model.version, "5.1")) {
return GPT_5_1_CODEX_MINI_EFFORTS;
}
if (semverGte(model.version, "5.2")) {
return GPT_5_2_PLUS_EFFORTS;
}
return DEFAULT_REASONING_EFFORTS;
}
function inferGeminiSupportedEfforts(model: GeminiModel): readonly Effort[] {
if (!semverGte(model.version, "3.0")) {
return DEFAULT_REASONING_EFFORTS;
}
return model.kind === "pro" ? GEMINI_3_PRO_EFFORTS : GEMINI_3_FLASH_EFFORTS;
}
function inferAnthropicSupportedEfforts<TApi extends Api>(
parsedModel: AnthropicModel,
model: ApiModel<TApi>,
): readonly Effort[] {
if (
(model.api === "anthropic-messages" || model.api === "bedrock-converse-stream") &&
semverGte(parsedModel.version, "4.6")
) {
return parsedModel.kind === "opus" || isFableOrMythos(parsedModel.kind)
? DEFAULT_REASONING_EFFORTS_WITH_XHIGH
: DEFAULT_REASONING_EFFORTS;
}
if (isOpenRouterAnthropicAdaptiveReasoningModel(parsedModel, model)) {
return DEFAULT_REASONING_EFFORTS_WITH_XHIGH;
}
return inferFallbackEfforts(model);
}
function inferFallbackEfforts<TApi extends Api>(model: ApiModel<TApi>): readonly Effort[] {
if (model.api === "anthropic-messages") {
return DEFAULT_REASONING_EFFORTS_WITH_XHIGH;
}
if (model.name.includes("deepseek-v4")) {
return DEFAULT_REASONING_EFFORTS_WITH_XHIGH;
}
if (model.api === "bedrock-converse-stream") {
return DEFAULT_REASONING_EFFORTS;
}
if (model.api === "openai-completions") {
const compat = resolveOpenAICompat(model as ApiModel<"openai-completions">);
if (compat.thinkingFormat === "openai" && compat.supportsReasoningEffort) {
return DEFAULT_REASONING_EFFORTS_WITH_XHIGH;
}
return DEFAULT_REASONING_EFFORTS;
}
// OpenAI Responses APIs encode discrete effort levels, including xhigh.
if (model.api === "openai-responses" || model.api === "openai-codex-responses") {
return DEFAULT_REASONING_EFFORTS_WITH_XHIGH;
}
return DEFAULT_REASONING_EFFORTS;
}
function inferThinkingControlMode<TApi extends Api>(
model: ApiModel<TApi>,
parsedModel: ParsedModel,
): ThinkingConfig["mode"] {
switch (model.api) {
case "google-generative-ai":
case "google-gemini-cli":
case "google-vertex":
return parsedModel.family === "gemini" &&
semverGte(parsedModel.version, "3.0") &&
parsedModel.version.major === 3
? "google-level"
: "budget";
case "anthropic-messages":
if (parsedModel.family === "anthropic") {
if (semverGte(parsedModel.version, "4.6")) {
return "anthropic-adaptive";
}
if (semverGte(parsedModel.version, "4.5")) {
return "anthropic-budget-effort";
}
}
return "budget";
case "bedrock-converse-stream":
if (parsedModel.family === "anthropic") {
if (
semverGte(parsedModel.version, "4.6") &&
(parsedModel.kind === "opus" || isFableOrMythos(parsedModel.kind))
) {
return "anthropic-adaptive";
}
if (semverGte(parsedModel.version, "4.5")) {
return "anthropic-budget-effort";
}
}
return "budget";
default:
return "effort";
}
}
function parseKnownModel(modelId: string): ParsedModel {
const canonicalId = getCanonicalModelId(modelId);
return (
parseGeminiModel(canonicalId) ??
parseAnthropicModel(canonicalId) ??
parseOpenAIModel(canonicalId) ?? { family: "unknown", id: canonicalId }
);
}
const GEMINI_SUFFIX = "-preview";
function parseGeminiModel(modelId: string): GeminiModel | null {
if (modelId.endsWith(GEMINI_SUFFIX)) {
modelId = modelId.slice(0, -GEMINI_SUFFIX.length);
}
const match = /gemini-(\d+(?:\.\d+){0,2})-(pro|flash)\b/.exec(modelId);
if (!match) {
return null;
}
const version = parseSemVer(match[1]);
if (!version) {
return null;
}
return { family: "gemini", kind: match[2] as GeminiKind, version };
}
function parseAnthropicModel(modelId: string): AnthropicModel | null {
const match = /claude-(opus|sonnet|fable|mythos)-(\d{1,2}(?:[.-]\d{1,2}){0,2})\b/.exec(modelId);
if (!match) {
return null;
}
const version = parseSemVer(match[2]);
if (!version) {
return null;
}
return { family: "anthropic", kind: match[1] as AnthropicKind, version };
}
function parseOpenAIModel(modelId: string): OpenAIModel | null {
const match = /gpt-(\d+(?:\.\d+){0,2})(?:-(codex-spark|codex-mini|codex-max|codex|mini|max|nano))?\b/.exec(modelId);
if (!match) {
return null;
}
const version = parseSemVer(match[1]);
if (!version) {
return null;
}
return { family: "openai", variant: (match[2] as OpenAIVariant | undefined) ?? "base", version };
}
function createSemVer(major: number, minor: number, patch = 0): SemVer {
return { major, minor, patch };
}
// extend this table if we need anything more than 9.10
const precomputeTable: Record<string, SemVer> = {};
for (let major = 0; major <= 9; major++) {
for (let minor = 0; minor <= 10; minor++) {
const version = createSemVer(major, minor, 0);
precomputeTable[`${major}.${minor}`] = version;
precomputeTable[`${major}-${minor}`] = version;
}
precomputeTable[`${major}`] = createSemVer(major, 0, 0);
}
function parseSemVer(version: string): SemVer | null {
return precomputeTable[version] ?? null;
}
function semverGte(left: SemVer | string, right: SemVer | string): boolean {
return compareSemVer(left, right) >= 0;
}
function semverEqual(left: SemVer | string, right: SemVer | string): boolean {
return compareSemVer(left, right) === 0;
}
function compareSemVer(left: SemVer | string | null, right: SemVer | string | null): number {
left = typeof left === "string" ? parseSemVer(left) : left;
right = typeof right === "string" ? parseSemVer(right) : right;
if (!left || !right) return (left ? 1 : 0) - (right ? 1 : 0);
if (left.major !== right.major) {
return left.major - right.major;
}
if (left.minor !== right.minor) {
return left.minor - right.minor;
}
return left.patch - right.patch;
}
function getCanonicalModelId(modelId: string): string {
const p = modelId.lastIndexOf("/");
return p !== -1 ? modelId.slice(p + 1) : modelId;
}
packages/ai/src/provider-models/bundled-references.ts
-38
View File
@@ -1,38 +0,0 @@
import { getBundledModels, getBundledProviders } from "../models";
import type { Api, Model } from "../types";
export function createBundledReferenceMap<TApi extends Api>(
provider: Parameters<typeof getBundledModels>[0],
): Map<string, Model<TApi>> {
const references = new Map<string, Model<TApi>>();
for (const model of getBundledModels(provider)) {
references.set(model.id, model as Model<TApi>);
}
return references;
}
export function createReferenceResolver<TApi extends Api>(
providerRefs: Map<string, Model<TApi>>,
): (modelId: string) => Model<TApi> | undefined {
const globalRefs = new Map<string, Model<Api>>();
for (const provider of getBundledProviders()) {
for (const model of getBundledModels(provider as Parameters<typeof getBundledModels>[0])) {
const candidate = model as Model<Api>;
const existing = globalRefs.get(candidate.id);
if (!existing) {
globalRefs.set(candidate.id, candidate);
} else if (candidate.contextWindow !== existing.contextWindow) {
if (candidate.contextWindow > existing.contextWindow) {
globalRefs.set(candidate.id, candidate);
}
} else if (candidate.maxTokens !== existing.maxTokens) {
if (candidate.maxTokens > existing.maxTokens) {
globalRefs.set(candidate.id, candidate);
}
} else if (existing.provider !== "openai" && candidate.provider === "openai") {
globalRefs.set(candidate.id, candidate);
}
}
}
return (modelId: string) => providerRefs.get(modelId) ?? (globalRefs.get(modelId) as Model<TApi> | undefined);
}
packages/ai/src/provider-models/descriptors.ts
-43
View File
@@ -1,43 +0,0 @@
/**
* Provider descriptors and the default-model map, derived from the single-source
* provider registry (`../registry`).
*
* The descriptor/catalog types and guards now live in the registry; they are
* re-exported here for back-compat with `generate-models.ts` and existing
* `@oh-my-pi/pi-ai/provider-models` consumers.
*/
import { PROVIDER_REGISTRY } from "../registry";
import type { ProviderDescriptor } from "../registry/types";
import type { KnownProvider } from "../types";
export * from "../registry/types";
/**
* Runtime model-discovery descriptors: every registry provider that exposes a
* standard model-manager factory. Special-managed providers
* (`google-antigravity`/`google-gemini-cli`/`openai-codex`) are built bespoke in
* the coding-agent runtime and are excluded here.
*/
export const PROVIDER_DESCRIPTORS: readonly ProviderDescriptor[] = PROVIDER_REGISTRY.flatMap(provider => {
const { createModelManagerOptions } = provider;
if (!createModelManagerOptions || provider.specialModelManager) {
return [];
}
return [
{
providerId: provider.id,
defaultModel: provider.defaultModel ?? "",
createModelManagerOptions,
allowUnauthenticated: provider.allowUnauthenticated,
dynamicModelsAuthoritative: provider.dynamicModelsAuthoritative,
catalogDiscovery: provider.catalogDiscovery,
},
];
});
/** Default model IDs for all known providers, derived from the registry. */
export const DEFAULT_MODEL_PER_PROVIDER: Record<KnownProvider, string> = Object.fromEntries(
PROVIDER_REGISTRY.filter(provider => provider.defaultModel != null).map(
provider => [provider.id, provider.defaultModel] as [string, string],
),
) as Record<KnownProvider, string>;
packages/ai/src/providers/amazon-bedrock.ts
+4 -20
View File
@@ -7,10 +7,10 @@
* Bun's native `HTTPS_PROXY` support.
*/
import type { Effort } from "@oh-my-pi/pi-catalog/effort";
import { mapEffortToAnthropicAdaptiveEffort, requireSupportedEffort } from "@oh-my-pi/pi-catalog/model-thinking";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import { $env, $flag, extractHttpStatusFromError, fetchWithRetry } from "@oh-my-pi/pi-utils";
import type { Effort } from "../effort";
import { mapEffortToAnthropicAdaptiveEffort, requireSupportedEffort } from "../model-thinking";
import { calculateCost } from "../models";
import type {
Api,
AssistantMessage,
@@ -818,7 +818,7 @@ function buildAdditionalModelRequestFields(
// runs (issue #1373). Opt back into "summarized" by default on models that
// accept the field.
const adaptive: { type: "adaptive"; display?: BedrockThinkingDisplay } = { type: "adaptive" };
if (supportsAdaptiveThinkingDisplay(model.id)) {
if (model.thinking?.supportsDisplay) {
adaptive.display = options.thinkingDisplay ?? "summarized";
}
return {
@@ -852,22 +852,6 @@ function buildAdditionalModelRequestFields(
return result;
}
/**
* Adaptive thinking `display` is supported starting with Claude Opus 4.7 and
* Claude Fable/Mythos 5. Older adaptive-thinking models (Opus 4.6, Sonnet
* 4.6+) reject the field. Bedrock model ids are prefixed with region/inference-
* profile slugs (e.g. `eu.anthropic.claude-opus-4-7-...`); the regex matches
* the Claude model fragment regardless of prefix.
*/
function supportsAdaptiveThinkingDisplay(modelId: string): boolean {
if (/claude-(?:fable|mythos)-5\b/.test(modelId)) return true;
const match = /claude-opus-(\d+)-(\d+)/.exec(modelId);
if (!match) return false;
const major = Number(match[1]);
const minor = Number(match[2]);
return major > 4 || (major === 4 && minor >= 7);
}
/**
* Bedrock's wire format expects the image as `{ source: { bytes: <base64-string> }, format }`.
* The caller already passes base64-encoded data, so no decode/re-encode round-trip is needed.
packages/ai/src/providers/anthropic-client.ts
+2 -2
View File
@@ -140,7 +140,7 @@ export function retryDelayFromHeaders(headers: Headers | undefined): number | un
return undefined;
}
function defaultRetryDelayMs(attempt: number): number {
export function calculateAnthropicRetryDelayMs(attempt: number): number {
const sleepSeconds = Math.min(INITIAL_RETRY_DELAY_S * 2 ** attempt, MAX_RETRY_DELAY_S);
const jitter = 1 - Math.random() * 0.25;
return sleepSeconds * jitter * 1000;
@@ -310,7 +310,7 @@ export class AnthropicMessagesClient implements AnthropicMessagesClientLike {
responseHeaders: Headers | undefined,
signal: AbortSignal | undefined,
): Promise<void> {
const delayMs = retryDelayFromHeaders(responseHeaders) ?? defaultRetryDelayMs(attempt);
const delayMs = retryDelayFromHeaders(responseHeaders) ?? calculateAnthropicRetryDelayMs(attempt);
try {
await scheduler.wait(delayMs, { signal });
} catch {
packages/ai/src/providers/anthropic.ts
+31 -139
View File
@@ -2,6 +2,11 @@ import * as nodeCrypto from "node:crypto";
import * as fs from "node:fs";
import { scheduler } from "node:timers/promises";
import * as tls from "node:tls";
import { isOfficialAnthropicApiUrl } from "@oh-my-pi/pi-catalog/compat/anthropic";
import { mapEffortToAnthropicAdaptiveEffort } from "@oh-my-pi/pi-catalog/model-thinking";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import { isAnthropicOAuthToken } from "@oh-my-pi/pi-catalog/utils";
import { parseGitHubCopilotApiKey } from "@oh-my-pi/pi-catalog/wire/github-copilot";
import {
$env,
extractHttpStatusFromError,
@@ -12,15 +17,7 @@ import {
logger,
readSseEvents,
} from "@oh-my-pi/pi-utils";
import {
hasOpus47ApiRestrictions,
isAnthropicFableOrMythosModel,
mapEffortToAnthropicAdaptiveEffort,
supportsMidConversationSystemMessages,
} from "../model-thinking";
import { calculateCost } from "../models";
import { isUsageLimitError } from "../rate-limit-utils";
import { parseGitHubCopilotApiKey } from "../registry/oauth/github-copilot";
import { getEnvApiKey, OUTPUT_FALLBACK_BUFFER } from "../stream";
import type {
Api,
@@ -47,13 +44,7 @@ import type {
Usage,
} from "../types";
import { resolveServiceTier } from "../types";
import {
isAnthropicOAuthToken,
isRecord,
normalizeSystemPrompts,
normalizeToolCallId,
resolveCacheRetention,
} from "../utils";
import { isRecord, normalizeSystemPrompts, normalizeToolCallId, resolveCacheRetention } from "../utils";
import { createAbortSourceTracker } from "../utils/abort";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { isFoundryEnabled } from "../utils/foundry";
@@ -72,6 +63,7 @@ import {
type AnthropicFetchOptions,
AnthropicMessagesClient,
type AnthropicMessagesClientLike,
calculateAnthropicRetryDelayMs,
retryDelayFromHeaders,
} from "./anthropic-client";
import type {
@@ -186,16 +178,6 @@ function isClaudeCodeClientUserAgent(userAgent: string | undefined): userAgent i
return userAgent.toLowerCase().startsWith("claude-cli");
}
export function isAnthropicApiBaseUrl(baseUrl?: string): boolean {
if (!baseUrl) return true;
try {
const url = new URL(baseUrl);
return url.protocol.toLowerCase() === "https:" && url.hostname.toLowerCase() === "api.anthropic.com";
} catch {
return false;
}
}
const sharedHeaders = {
"Accept-Encoding": "gzip, deflate, br, zstd",
Connection: "keep-alive",
@@ -268,7 +250,7 @@ export function buildAnthropicHeaders(options: AnthropicHeaderOptions): Record<s
"x-client-request-id": nodeCrypto.randomUUID(),
"User-Agent": userAgent,
};
} else if (!isAnthropicApiBaseUrl(options.baseUrl)) {
} else if (!isOfficialAnthropicApiUrl(options.baseUrl)) {
return {
...modelHeaders,
Accept: acceptHeader,
@@ -315,22 +297,6 @@ type AnthropicOutputConfig = NonNullable<MessageCreateParamsStreaming["output_co
const ANTHROPIC_STOP_SEQUENCES_MAX = 4;
let warnedStopSequencesTrim = false;
/**
* Adaptive thinking `display` is supported starting with Claude Opus 4.7 and
* Claude Fable/Mythos 5. Older adaptive-thinking models (Opus 4.6, Sonnet
* 4.6+) reject the field.
*/
function supportsAdaptiveThinkingDisplay(modelId: string): boolean {
if (/claude-(?:fable|mythos)-5\b/.test(modelId)) return true;
// Bound the minor to non-date digits: bare dated ids like
// `claude-opus-4-20250514` (Opus 4.0) must not parse as minor=20250514.
const match = /claude-opus-(\d+)-(\d{1,2})(?!\d)/.exec(modelId);
if (!match) return false;
const major = Number(match[1]);
const minor = Number(match[2]);
return major > 4 || (major === 4 && minor >= 7);
}
const ANTHROPIC_PROVIDER_SESSION_STATE_KEY = "anthropic-messages";
type AnthropicProviderSessionState = ProviderSessionState & {
@@ -446,7 +412,6 @@ function dropAnthropicStrictTools(params: MessageCreateParamsStreaming): void {
function getCacheControl(
model: Model<"anthropic-messages">,
baseUrl: string,
cacheRetention: CacheRetention | undefined,
isOAuthToken: boolean,
): { retention: CacheRetention; cacheControl?: AnthropicCacheControl } {
@@ -454,10 +419,7 @@ function getCacheControl(
if (retention === "none") {
return { retention };
}
const ttl =
retention === "long" && isAnthropicApiBaseUrl(baseUrl) && getAnthropicCompat(model).supportsLongCacheRetention
? "1h"
: undefined;
const ttl = retention === "long" && model.compat.supportsLongCacheRetention ? "1h" : undefined;
return {
retention,
cacheControl: { type: "ephemeral", ...(ttl && { ttl }) },
@@ -1151,7 +1113,7 @@ function parseAnthropicCustomHeaders(rawHeaders: string | undefined): Record<str
export function resolveAnthropicCustomHeadersForBaseUrl(
baseUrl: string | undefined,
): Record<string, string> | undefined {
if (!isFoundryEnabled() && isAnthropicApiBaseUrl(baseUrl)) return undefined;
if (!isFoundryEnabled() && isOfficialAnthropicApiUrl(baseUrl)) return undefined;
return parseAnthropicCustomHeaders($env.ANTHROPIC_CUSTOM_HEADERS);
}
@@ -1409,26 +1371,7 @@ async function* observeDecodedAnthropicSdkEvents(
}
}
function getAnthropicCompat(
model: Model<"anthropic-messages">,
): Required<NonNullable<Model<"anthropic-messages">["compat"]>> {
return {
disableStrictTools: model.compat?.disableStrictTools ?? false,
disableAdaptiveThinking: model.compat?.disableAdaptiveThinking ?? false,
supportsEagerToolInputStreaming: model.compat?.supportsEagerToolInputStreaming ?? true,
supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true,
supportsMidConversationSystem:
model.compat?.supportsMidConversationSystem ??
// First-party Claude API only. Bedrock/Vertex/Foundry and other
// Anthropic-compatible proxies reject the role; gate auto-detection on
// the canonical api.anthropic.com host plus a supported model id.
(isAnthropicApiBaseUrl(model.baseUrl) && supportsMidConversationSystemMessages(model.id)),
supportsForcedToolChoice: model.compat?.supportsForcedToolChoice ?? !isAnthropicFableOrMythosModel(model.id),
};
}
const PROVIDER_MAX_RETRIES = 3;
const PROVIDER_BASE_DELAY_MS = 2000;
const PROVIDER_MAX_RETRIES = 10;
/** Transient stream corruption errors where the response was truncated mid-JSON. */
function isTransientStreamParseError(error: unknown): boolean {
@@ -1631,7 +1574,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
const sendsAdaptiveEffortPin =
options?.thinkingEnabled === false &&
model.thinking?.mode === "anthropic-adaptive" &&
!getAnthropicCompat(model).disableAdaptiveThinking;
!model.compat.disableAdaptiveThinking;
if (
model.reasoning &&
(options?.thinkingEnabled || sendsAdaptiveEffortPin) &&
@@ -1639,10 +1582,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
) {
extraBetas.push(effortBeta);
}
if (
getAnthropicCompat(model).supportsMidConversationSystem &&
!extraBetas.includes(midConversationSystemBeta)
) {
if (model.compat.supportsMidConversationSystem && !extraBetas.includes(midConversationSystemBeta)) {
// convertAnthropicMessages may upgrade developer turns to the
// mid-conversation `system` role on these models; API-key requests
// need the beta alongside the role (OAuth agent requests already
@@ -1670,7 +1610,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
}
const preparedContext = await prepareAnthropicManyImageContext(context, model.input.includes("image"));
const prepareParams = async (): Promise<MessageCreateParamsStreaming> => {
let nextParams = buildParams(model, baseUrl, preparedContext, isOAuthToken, options, disableStrictTools);
let nextParams = buildParams(model, preparedContext, isOAuthToken, options, disableStrictTools);
if (disableStrictTools) {
dropAnthropicStrictTools(nextParams);
}
@@ -1740,8 +1680,8 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
const { requestSignal } = activeAbortTracker;
// The provider loop owns retries: pin the client's internal retry loop
// to zero even when no watchdog timeout is configured (the helper only
// pins it alongside a timeout; the client default of 5 would otherwise
// multiply with PROVIDER_MAX_RETRIES into up to 24 wire attempts).
// pins it alongside a timeout; a client retry budget of 5 would otherwise
// multiply with PROVIDER_MAX_RETRIES into up to 66 wire attempts).
const requestOptions = { ...createSdkStreamRequestOptions(requestSignal, requestTimeoutMs), maxRetries: 0 };
const anthropicRequest: unknown =
isOAuthToken && client.beta
@@ -2196,7 +2136,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages"> = (
throw streamFailure;
}
providerRetryAttempt++;
const backoffDelayMs = PROVIDER_BASE_DELAY_MS * 2 ** (providerRetryAttempt - 1);
const backoffDelayMs = calculateAnthropicRetryDelayMs(providerRetryAttempt - 1);
// Honor the server's retry hint (`retry-after-ms`/`retry-after`) on
// 429/529-style failures: retrying sooner than the server asked is a
// guaranteed failure that just burns the retry budget.
@@ -2337,8 +2277,8 @@ export function buildAnthropicClientOptions(args: AnthropicClientOptionsArgs): A
isOAuth,
claudeCodeSessionId,
} = args;
const compat = getAnthropicCompat(model);
const needsInterleavedBeta = interleavedThinking && !supportsAdaptiveThinkingDisplay(model.id);
const compat = model.compat;
const needsInterleavedBeta = interleavedThinking && !model.thinking?.supportsDisplay;
const needsFineGrainedToolStreamingBeta = hasTools && !compat.supportsEagerToolInputStreaming;
const oauthToken = isOAuth ?? isAnthropicOAuthToken(apiKey);
const baseUrl = resolveAnthropicBaseUrl(model, apiKey);
@@ -2448,7 +2388,7 @@ export function buildAnthropicClientOptions(args: AnthropicClientOptionsArgs): A
const authorizationHeader = getHeaderCaseInsensitive(defaultHeaders, "Authorization");
const shouldSuppressClientApiKey =
!oauthToken &&
!isAnthropicApiBaseUrl(baseUrl) &&
!model.compat.officialEndpoint &&
typeof authorizationHeader === "string" &&
/^Bearer\s+/i.test(authorizationHeader);
@@ -2757,13 +2697,12 @@ function extractClaudeCodeFirstUserMessageText(messages: readonly Message[]): st
function buildParams(
model: Model<"anthropic-messages">,
baseUrl: string,
context: Context,
isOAuthToken: boolean,
options?: AnthropicOptions,
disableStrictTools = false,
): MessageCreateParamsStreaming {
const { cacheControl } = getCacheControl(model, baseUrl, options?.cacheRetention, isOAuthToken);
const { cacheControl } = getCacheControl(model, options?.cacheRetention, isOAuthToken);
// Pre-compute system blocks so they occupy the right slot in the serialized body.
const shouldInjectClaudeCodeInstruction = isOAuthToken && !model.id.startsWith("claude-3-5-haiku");
@@ -2782,7 +2721,7 @@ function buildParams(
context.tools,
isOAuthToken,
disableStrictTools || model.provider === "github-copilot",
getAnthropicCompat(model).supportsEagerToolInputStreaming,
model.compat.supportsEagerToolInputStreaming,
);
} else if (isOAuthToken) {
tools = [];
@@ -2805,7 +2744,7 @@ function buildParams(
if (options?.thinkingEnabled) {
const mode = model.thinking?.mode;
const effort = resolveAnthropicAdaptiveEffort(model, options);
const compat = getAnthropicCompat(model);
const compat = model.compat;
if (mode === "anthropic-adaptive" && !compat.disableAdaptiveThinking) {
const adaptive: { type: "adaptive"; display?: AnthropicThinkingDisplay } = { type: "adaptive" };
// Starting with Claude Opus 4.7 and Claude Fable/Mythos 5, adaptive thinking
@@ -2814,7 +2753,7 @@ function buildParams(
// callers that rely on it. The `display` field is gated strictly on model
// support: Opus 4.6 / Sonnet 4.6+ reject it with a 400, so an explicit
// `thinkingDisplay` MUST NOT force it onto a model that can't accept it.
if (supportsAdaptiveThinkingDisplay(model.id)) {
if (model.thinking?.supportsDisplay) {
adaptive.display = options.thinkingDisplay ?? "summarized";
}
thinking = adaptive;
@@ -2828,7 +2767,7 @@ function buildParams(
if (mode === "anthropic-budget-effort" && effort) outputConfigEffort = effort;
}
} else if (options?.thinkingEnabled === false) {
const compat = getAnthropicCompat(model);
const compat = model.compat;
if (model.thinking?.mode === "anthropic-adaptive" && !compat.disableAdaptiveThinking) {
// Adaptive-only Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5) reject
// `thinking.type: "disabled"` — adaptive thinking cannot be switched off.
@@ -2863,7 +2802,7 @@ function buildParams(
// metadata → max_tokens → thinking → context_management → output_config → stream.
const params: MessageCreateParamsStreaming = {
model: model.id,
messages: convertAnthropicMessages(context.messages, model, isOAuthToken, baseUrl),
messages: convertAnthropicMessages(context.messages, model, isOAuthToken),
...(systemBlocks && { system: systemBlocks }),
...(tools !== undefined && { tools }),
...(metadata && { metadata }),
@@ -2877,7 +2816,7 @@ function buildParams(
// Opus 4.7+ and Fable/Mythos 5 reject non-default sampling parameters with 400 error.
const thinkingType = params.thinking?.type;
const allowSamplingParams =
!hasOpus47ApiRestrictions(model.id) && (thinkingType === undefined || thinkingType === "disabled");
model.compat.supportsSamplingParams && (thinkingType === undefined || thinkingType === "disabled");
if (allowSamplingParams && options?.temperature !== undefined) {
params.temperature = options.temperature;
}
@@ -2917,7 +2856,7 @@ function buildParams(
// request succeeds; the tool stays available and the caller's prompt steers
// the model toward it.
const choiceType = params.tool_choice?.type;
if ((choiceType === "any" || choiceType === "tool") && !getAnthropicCompat(model).supportsForcedToolChoice) {
if ((choiceType === "any" || choiceType === "tool") && !model.compat.supportsForcedToolChoice) {
params.tool_choice = { type: "auto" };
}
}
@@ -2931,52 +2870,6 @@ function buildParams(
return params;
}
/**
* Z.AI's Anthropic-compatible proxy at `api.z.ai/api/anthropic` deserializes
* tool_result blocks into a Python class that accesses `.id`, even though
* Anthropic's standard tool_result schema only carries `tool_use_id`. Detect
* that endpoint so we can emit the non-standard alias for it without
* polluting requests to api.anthropic.com or other compatible proxies.
* See: https://github.com/can1357/oh-my-pi/issues/814
*/
function isZaiAnthropicEndpoint(model: Model<"anthropic-messages">): boolean {
if (model.provider === "zai") return true;
const baseUrl = model.baseUrl;
if (!baseUrl) return false;
try {
return new URL(baseUrl).hostname.toLowerCase() === "api.z.ai";
} catch {
return false;
}
}
/**
* Returns true when unsigned `thinking` blocks from prior assistant turns should
* be replayed as Anthropic-native thinking instead of demoted to text.
*
* Official Anthropic (matched via `isAnthropicApiBaseUrl`, which intentionally
* treats a missing baseUrl as official since `resolveAnthropicBaseUrl` routes
* it to `https://api.anthropic.com`) enforces signature-based thinking-chain
* integrity, so unsigned blocks must remain text there. Anthropic-compatible
* reasoning endpoints commonly emit unsigned thinking blocks while still
* expecting them back as `type: "thinking"` on continuation; demoting them
* loses the model's reasoning chain and can destabilize the next tool-call
* arguments (#2005). Known non-signing hosts are also preserved for
* compatibility.
*/
function shouldReplayUnsignedThinking(model: Model<"anthropic-messages">, baseUrl: string | undefined): boolean {
if (model.provider === "zai" || model.provider === "deepseek") return true;
if (baseUrl) {
try {
const hostname = new URL(baseUrl).hostname.toLowerCase();
if (hostname === "api.deepseek.com" || hostname.endsWith(".deepseek.com")) return true;
} catch {
// Fall through to the protocol-level reasoning rule below.
}
}
return model.reasoning && !isAnthropicApiBaseUrl(baseUrl);
}
function buildToolResultBlock(model: Model<"anthropic-messages">, msg: ToolResultMessage): ContentBlockParam {
const block: ContentBlockParam = {
type: "tool_result",
@@ -2984,7 +2877,7 @@ function buildToolResultBlock(model: Model<"anthropic-messages">, msg: ToolResul
content: convertContentBlocks(msg.content, model.input.includes("image")),
is_error: msg.isError,
};
if (isZaiAnthropicEndpoint(model)) {
if (model.compat.requiresToolResultId) {
// Z.AI workaround (issue #814): include `id` aliased to `tool_use_id`.
(block as unknown as Record<string, unknown>).id = msg.toolCallId;
}
@@ -3032,7 +2925,6 @@ export function convertAnthropicMessages(
messages: Message[],
model: Model<"anthropic-messages">,
isOAuthToken: boolean,
baseUrl = resolveAnthropicBaseUrl(model),
): AnthropicMessageParam[] {
// Indices of params emitted from `developer` messages. After the main pass,
// the ones whose placement satisfies Anthropic's mid-conversation rules are
@@ -3097,7 +2989,7 @@ export function convertAnthropicMessages(
}
if (block.thinking.trim().length === 0) continue;
if (!block.thinkingSignature || block.thinkingSignature.trim().length === 0) {
if (shouldReplayUnsignedThinking(model, baseUrl)) {
if (model.compat.replayUnsignedThinking) {
blocks.push({
type: "thinking",
thinking: block.thinking.toWellFormed(),
@@ -3175,7 +3067,7 @@ export function convertAnthropicMessages(
// never consecutive. Requiring the next param to be `assistant` (or absent)
// covers both the "followed by assistant / last" and "no consecutive system"
// constraints. Anything that does not qualify stays a `user` message.
if (developerParamIndices.length > 0 && getAnthropicCompat(model).supportsMidConversationSystem) {
if (developerParamIndices.length > 0 && model.compat.supportsMidConversationSystem) {
for (const idx of developerParamIndices) {
const followsUser = idx > 0 && params[idx - 1]?.role === "user";
const next = params[idx + 1];
packages/ai/src/providers/azure-openai-responses.ts
+12 -6
View File
@@ -31,7 +31,7 @@ import { sanitizeSchemaForOpenAIResponses, toolWireSchema } from "../utils/schem
import { createSdkStreamRequestOptions } from "../utils/sdk-stream-timeout";
import { notifyRawSseEvent } from "../utils/sse-debug";
import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice";
import { getOpenAIResponsesCacheSessionId, supportsDeveloperRole } from "./openai-responses";
import { getOpenAIResponsesCacheSessionId } from "./openai-responses";
import {
appendResponsesToolResultMessages,
applyCommonResponsesSamplingParams,
@@ -136,7 +136,7 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
const apiKey = options?.apiKey || getEnvApiKey(model.provider) || "";
const client = createClient(model, apiKey, options);
const { baseUrl } = resolveAzureConfig(model, options);
const params = buildParams(model, context, options, deploymentName, baseUrl);
const params = buildParams(model, context, options, deploymentName);
options?.onPayload?.(params);
const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs();
const firstEventTimeoutMs =
@@ -179,6 +179,7 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
abortSignal: options?.signal,
isProgressItem: isOpenAIResponsesProgressEvent,
});
let sawCompleted = false;
const observedOpenaiStream = rawSseObserver
? observeDecodedAzureResponsesEvents(timedOpenaiStream, rawSseObserver)
: timedOpenaiStream;
@@ -186,6 +187,9 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
onFirstToken: () => {
if (!firstTokenTime) firstTokenTime = Date.now();
},
onCompleted: () => {
sawCompleted = true;
},
});
const firstEventTimeoutError = abortTracker.getLocalAbortReason();
@@ -197,6 +201,10 @@ export const streamAzureOpenAIResponses: StreamFunction<"azure-openai-responses"
throw new Error("Request was aborted");
}
if (!sawCompleted) {
throw new Error("Azure OpenAI responses stream closed before response.completed was received");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error(output.errorMessage ?? "An unknown error occurred");
}
@@ -296,9 +304,8 @@ function buildParams(
context: Context,
options: AzureOpenAIResponsesOptions | undefined,
deploymentName: string,
resolvedBaseUrl?: string,
) {
const messages = convertMessages(model, context, true, resolvedBaseUrl);
const messages = convertMessages(model, context, true);
const params: AzureOpenAIResponsesSamplingParams = {
model: deploymentName,
@@ -328,7 +335,6 @@ function convertMessages(
model: Model<"azure-openai-responses">,
context: Context,
strictResponsesPairing: boolean,
resolvedBaseUrl?: string,
): ResponseInput {
const messages: ResponseInput = [];
const transformedMessages = transformMessages(context.messages, model, normalizeResponsesToolCallIdForTransform);
@@ -337,7 +343,7 @@ function convertMessages(
const systemPrompts = normalizeSystemPrompts(context.systemPrompt);
if (systemPrompts.length > 0) {
const role = model.reasoning && supportsDeveloperRole(resolvedBaseUrl ?? model) ? "developer" : "system";
const role = model.reasoning && model.compat.supportsDeveloperRole ? "developer" : "system";
for (const systemPrompt of systemPrompts) {
messages.push({ role, content: systemPrompt });
}
packages/ai/src/providers/cursor.ts
+30 -30
View File
@@ -3,35 +3,7 @@ import * as fs from "node:fs/promises";
import http2 from "node:http2";
import { create, fromBinary, fromJson, type JsonValue, toBinary, toJson } from "@bufbuild/protobuf";
import { ValueSchema } from "@bufbuild/protobuf/wkt";
import { $env, extractHttpStatusFromError, sanitizeText } from "@oh-my-pi/pi-utils";
import { calculateCost } from "../models";
import type {
Api,
AssistantMessage,
Context,
CursorExecHandlerResult,
CursorExecHandlers,
CursorMcpCall,
CursorShellStreamCallbacks,
CursorToolResultHandler,
ImageContent,
Message,
Model,
StreamFunction,
StreamOptions,
TextContent,
ThinkingContent,
Tool,
ToolCall,
ToolResultMessage,
} from "../types";
import { normalizeSystemPrompts } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { parseStreamingJson } from "../utils/json-parse";
import { createRequestDebugSession, isRequestDebugEnabled, type RequestDebugResponseLog } from "../utils/request-debug";
import { formatErrorMessageWithRetryAfter } from "../utils/retry-after";
import { toolWireSchema } from "../utils/schema/wire";
import type { McpToolDefinition } from "./cursor/gen/agent_pb";
import type { McpToolDefinition } from "@oh-my-pi/pi-catalog/discovery/cursor-gen/agent_pb";
import {
AgentClientMessageSchema,
AgentConversationTurnStructureSchema,
@@ -128,7 +100,35 @@ import {
WriteShellStdinErrorSchema,
WriteShellStdinResultSchema,
WriteSuccessSchema,
} from "./cursor/gen/agent_pb";
} from "@oh-my-pi/pi-catalog/discovery/cursor-gen/agent_pb";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import { $env, extractHttpStatusFromError, sanitizeText } from "@oh-my-pi/pi-utils";
import type {
Api,
AssistantMessage,
Context,
CursorExecHandlerResult,
CursorExecHandlers,
CursorMcpCall,
CursorShellStreamCallbacks,
CursorToolResultHandler,
ImageContent,
Message,
Model,
StreamFunction,
StreamOptions,
TextContent,
ThinkingContent,
Tool,
ToolCall,
ToolResultMessage,
} from "../types";
import { normalizeSystemPrompts } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { parseStreamingJson } from "../utils/json-parse";
import { createRequestDebugSession, isRequestDebugEnabled, type RequestDebugResponseLog } from "../utils/request-debug";
import { formatErrorMessageWithRetryAfter } from "../utils/retry-after";
import { toolWireSchema } from "../utils/schema/wire";
export const CURSOR_API_URL = "https://api2.cursor.sh";
export const CURSOR_CLIENT_VERSION = "cli-2026.01.09-231024f";
packages/ai/src/providers/github-copilot-headers.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { getGitHubCopilotBaseUrl, parseGitHubCopilotApiKey } from "../registry/oauth/github-copilot";
import { getGitHubCopilotBaseUrl, parseGitHubCopilotApiKey } from "@oh-my-pi/pi-catalog/wire/github-copilot";
import type { Message } from "../types";
/**
* Infer whether the current request to Copilot is user-initiated or agent-initiated.
packages/ai/src/providers/gitlab-duo.ts
+30 -24
View File
@@ -1,5 +1,6 @@
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import { ANTHROPIC_THINKING, mapAnthropicToolChoice } from "../stream";
import type { Api, Context, FetchImpl, Model, SimpleStreamOptions } from "../types";
import type { Api, Context, FetchImpl, Model, ModelSpec, SimpleStreamOptions } from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { createProviderErrorMessage } from "./error-message";
import type { OpenAICompletionsOptions } from "./openai-completions";
@@ -145,23 +146,25 @@ export function getModelMapping(modelId: string): GitLabModelMapping | undefined
}
export function getGitLabDuoModels(): Model<Api>[] {
return Object.entries(MODEL_MAPPINGS).map(([id, mapping]) => ({
id,
name: mapping.name,
api:
mapping.provider === "anthropic"
? "anthropic-messages"
: mapping.openaiApiType === "responses"
? "openai-responses"
: "openai-completions",
provider: "gitlab-duo",
baseUrl: mapping.provider === "anthropic" ? ANTHROPIC_PROXY_URL : OPENAI_PROXY_URL,
reasoning: mapping.reasoning,
input: [...mapping.input],
cost: { ...mapping.cost },
contextWindow: mapping.contextWindow,
maxTokens: mapping.maxTokens,
}));
return Object.entries(MODEL_MAPPINGS).map(([id, mapping]) =>
buildModel({
id,
name: mapping.name,
api:
mapping.provider === "anthropic"
? "anthropic-messages"
: mapping.openaiApiType === "responses"
? "openai-responses"
: "openai-completions",
provider: "gitlab-duo",
baseUrl: mapping.provider === "anthropic" ? ANTHROPIC_PROXY_URL : OPENAI_PROXY_URL,
reasoning: mapping.reasoning,
input: [...mapping.input],
cost: { ...mapping.cost },
contextWindow: mapping.contextWindow,
maxTokens: mapping.maxTokens,
} as ModelSpec<Api>),
);
}
interface DirectAccessToken {
@@ -255,12 +258,13 @@ export function streamGitLabDuo(
const inner =
mapping.provider === "anthropic"
? streamAnthropic(
{
buildModel({
...model,
id: mapping.model,
api: "anthropic-messages",
baseUrl: ANTHROPIC_PROXY_URL,
} as Model<"anthropic-messages">,
compat: model.compatConfig,
} as ModelSpec<"anthropic-messages">),
context,
{
apiKey: directAccess.token,
@@ -293,12 +297,13 @@ export function streamGitLabDuo(
)
: mapping.openaiApiType === "responses"
? streamOpenAIResponses(
{
buildModel({
...model,
id: mapping.model,
api: "openai-responses",
baseUrl: OPENAI_PROXY_URL,
} as Model<"openai-responses">,
compat: model.compatConfig,
} as ModelSpec<"openai-responses">),
context,
{
apiKey: directAccess.token,
@@ -325,12 +330,13 @@ export function streamGitLabDuo(
} satisfies OpenAIResponsesOptions,
)
: streamOpenAICompletions(
{
buildModel({
...model,
id: mapping.model,
api: "openai-completions",
baseUrl: OPENAI_PROXY_URL,
} as Model<"openai-completions">,
compat: model.compatConfig,
} as ModelSpec<"openai-completions">),
context,
{
apiKey: directAccess.token,
packages/ai/src/providers/google-gemini-cli.ts
+7 -3
View File
@@ -5,8 +5,13 @@
*/
import { createHash, randomBytes, randomUUID } from "node:crypto";
import { scheduler } from "node:timers/promises";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import {
ANTIGRAVITY_SYSTEM_INSTRUCTION,
getAntigravityUserAgent,
getGeminiCliHeaders,
} from "@oh-my-pi/pi-catalog/wire/gemini-headers";
import { extractHttpStatusFromError, fetchWithRetry, readSseJson } from "@oh-my-pi/pi-utils";
import { calculateCost } from "../models";
import type {
Api,
AssistantMessage,
@@ -24,7 +29,6 @@ import { appendRawHttpRequestDumpFor400, type RawHttpRequestDump, withHttpStatus
// Refresh is the sole responsibility of AuthStorage (broker-aware, single-flighted);
// the stream provider trusts the access token threaded through `options.apiKey`.
import { normalizeSchemaForCCA } from "../utils/schema";
import { ANTIGRAVITY_SYSTEM_INSTRUCTION, getAntigravityUserAgent, getGeminiCliHeaders } from "./google-gemini-headers";
import type { Content, FunctionCallingConfigMode, ThinkingConfig } from "./google-shared";
import {
convertMessages,
@@ -80,7 +84,7 @@ export {
getAntigravityUserAgent,
getGeminiCliHeaders,
getGeminiCliUserAgent,
} from "./google-gemini-headers";
} from "@oh-my-pi/pi-catalog/wire/gemini-headers";
// Retry configuration
const MAX_RETRIES = 3;
packages/ai/src/providers/google-shared.ts
+1 -1
View File
@@ -2,8 +2,8 @@
* Shared utilities for Google Generative AI and Google Cloud Code Assist providers.
*/
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import { extractHttpStatusFromError, readSseJson } from "@oh-my-pi/pi-utils";
import { calculateCost } from "../models";
import type {
Api,
AssistantMessage,
packages/ai/src/providers/mock.ts
+1
View File
@@ -168,6 +168,7 @@ export class MockModel implements Model<MockApi> {
readonly cost: Model["cost"];
readonly contextWindow: number;
readonly maxTokens: number;
readonly compat = undefined;
/** Recorded calls in invocation order. */
readonly calls: MockCall[] = [];
packages/ai/src/providers/ollama.ts
+42 -6
View File
@@ -16,7 +16,12 @@ import type {
} from "../types";
import { normalizeSystemPrompts } from "../utils";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
import {
type CapturedHttpErrorResponse,
finalizeErrorMessage,
type RawHttpRequestDump,
withHttpStatus,
} from "../utils/http-inspector";
import { parseStreamingJson } from "../utils/json-parse";
import { toolWireSchema } from "../utils/schema/wire";
import {
@@ -29,6 +34,7 @@ import { transformMessages } from "./transform-messages";
export interface OllamaChatOptions extends StreamOptions {
reasoning?: "minimal" | "low" | "medium" | "high" | "xhigh";
disableReasoning?: boolean;
toolChoice?: ToolChoice;
}
@@ -91,7 +97,14 @@ function normalizeBaseUrl(baseUrl?: string): string {
return trimmed.endsWith("/api") ? trimmed.slice(0, -4) : trimmed;
}
function mapReasoning(reasoning: OllamaChatOptions["reasoning"]): boolean | "low" | "medium" | "high" | undefined {
function mapReasoning(
reasoning: OllamaChatOptions["reasoning"],
disableReasoning: boolean | undefined,
modelReasoning: boolean,
): boolean | "low" | "medium" | "high" | undefined {
if (disableReasoning && modelReasoning) {
return false;
}
switch (reasoning) {
case "minimal":
case "low":
@@ -258,7 +271,7 @@ function convertTools(tools: Tool[] | undefined): OllamaFunctionTool[] | undefin
}
function createChatBody(model: Model<"ollama-chat">, context: Context, options: OllamaChatOptions | undefined) {
const think = mapReasoning(options?.reasoning);
const think = mapReasoning(options?.reasoning, options?.disableReasoning, model.reasoning);
const toolChoice = mapToolChoice(options?.toolChoice);
const selectedTools = selectToolsForToolChoice(context.tools, options?.toolChoice);
const tools = convertTools(selectedTools);
@@ -268,11 +281,32 @@ function createChatBody(model: Model<"ollama-chat">, context: Context, options:
...(tools ? { tools } : {}),
...(think !== undefined ? { think } : {}),
...(toolChoice !== undefined ? { tool_choice: toolChoice } : {}),
...(options?.maxTokens !== undefined ? { options: { num_predict: options.maxTokens } } : {}),
...(options?.maxTokens !== undefined && !model.omitMaxOutputTokens
? { options: { num_predict: options.maxTokens } }
: {}),
stream: true,
};
}
async function captureHttpErrorResponse(response: Response): Promise<CapturedHttpErrorResponse> {
let bodyText: string | undefined;
let bodyJson: unknown;
try {
bodyText = await response.text();
if (bodyText.trim()) {
try {
bodyJson = JSON.parse(bodyText) as unknown;
} catch {}
}
} catch {}
return {
status: response.status,
headers: response.headers,
bodyText,
bodyJson,
};
}
async function* iterateNdjson(stream: ReadableStream<Uint8Array>): AsyncGenerator<OllamaChatChunk> {
const reader = stream.getReader();
const decoder = new TextDecoder();
@@ -376,6 +410,7 @@ export const streamOllama: StreamFunction<"ollama-chat"> = (
let firstTokenTime: number | undefined;
const output = createEmptyOutput(model);
let rawRequestDump: RawHttpRequestDump | undefined;
let capturedErrorResponse: CapturedHttpErrorResponse | undefined;
let activeThinkingIndex: number | undefined;
let activeTextIndex: number | undefined;
const activeToolIndices = new Set<number>();
@@ -503,7 +538,8 @@ export const streamOllama: StreamFunction<"ollama-chat"> = (
fetch: options.fetch,
});
if (!response.ok) {
throw new Error(`HTTP ${response.status} from ${baseUrl}/api/chat`);
capturedErrorResponse = await captureHttpErrorResponse(response);
throw withHttpStatus(new Error(`HTTP ${response.status} from ${baseUrl}/api/chat`), response.status);
}
if (!response.body) {
throw new Error("Ollama returned an empty response body");
@@ -631,7 +667,7 @@ export const streamOllama: StreamFunction<"ollama-chat"> = (
}
output.stopReason = options.signal?.aborted ? "aborted" : "error";
output.errorStatus = extractHttpStatusFromError(error);
output.errorMessage = await finalizeErrorMessage(error, rawRequestDump);
output.errorMessage = await finalizeErrorMessage(error, rawRequestDump, capturedErrorResponse);
output.duration = Date.now() - startTime;
if (firstTokenTime) {
output.ttft = firstTokenTime - startTime;
packages/ai/src/providers/openai-anthropic-shim.ts
+10 -4
View File
@@ -8,8 +8,9 @@
* here once.
*/
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import { ANTHROPIC_THINKING } from "../stream";
import type { Context, Model, SimpleStreamOptions } from "../types";
import type { Context, Model, ModelSpec, SimpleStreamOptions } from "../types";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { createProviderErrorMessage } from "./error-message";
import { streamAnthropic, streamOpenAICompletions } from "./register-builtins";
@@ -56,7 +57,7 @@ export function streamOpenAIAnthropicShim(
};
if (format === "anthropic") {
const anthropicModel: Model<"anthropic-messages"> = {
const anthropicModel = buildModel({
id: model.id,
name: model.name,
api: "anthropic-messages",
@@ -68,7 +69,7 @@ export function streamOpenAIAnthropicShim(
reasoning: model.reasoning,
input: model.input,
cost: model.cost,
};
} as ModelSpec<"anthropic-messages">);
const reasoningEffort = options?.reasoning;
const thinkingEnabled = !!reasoningEffort && model.reasoning;
@@ -101,7 +102,12 @@ export function streamOpenAIAnthropicShim(
}
} else {
const openaiModel: Model<"openai-completions"> = config.openaiBaseUrl
? { ...model, baseUrl: config.openaiBaseUrl, headers: mergedHeaders }
? buildModel({
...model,
baseUrl: config.openaiBaseUrl,
headers: mergedHeaders,
compat: model.compatConfig,
} as ModelSpec<"openai-completions">)
: model;
const reasoningEffort = options?.reasoning;
packages/ai/src/providers/openai-codex-responses.ts
+7 -2
View File
@@ -1,5 +1,12 @@
import * as os from "node:os";
import { scheduler } from "node:timers/promises";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import {
CODEX_BASE_URL,
getCodexAccountId,
OPENAI_HEADER_VALUES,
OPENAI_HEADERS,
} from "@oh-my-pi/pi-catalog/wire/codex";
import {
$env,
$flag,
@@ -20,7 +27,6 @@ import type {
ResponseReasoningItem,
} from "openai/resources/responses/responses";
import packageJson from "../../package.json" with { type: "json" };
import { calculateCost } from "../models";
import { getEnvApiKey } from "../stream";
import {
type Api,
@@ -58,7 +64,6 @@ import { createRequestDebugSession, isRequestDebugEnabled, type RequestDebugResp
import { adaptSchemaForStrict, NO_STRICT, sanitizeSchemaForOpenAIResponses, toolWireSchema } from "../utils/schema";
import { notifyRawSseEvent } from "../utils/sse-debug";
import { compactGrammarDefinition } from "./grammar";
import { CODEX_BASE_URL, getCodexAccountId, OPENAI_HEADER_VALUES, OPENAI_HEADERS } from "./openai-codex/constants";
import {
type CodexRequestOptions,
type InputItem,
packages/ai/src/providers/openai-codex/request-transformer.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import type { Effort } from "../../effort";
import { requireSupportedEffort } from "../../model-thinking";
import type { Effort } from "@oh-my-pi/pi-catalog/effort";
import { requireSupportedEffort } from "@oh-my-pi/pi-catalog/model-thinking";
import type { Api, Model } from "../../types";
export interface ReasoningConfig {
packages/ai/src/providers/openai-codex/response-handler.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { toNumber } from "../../utils";
import { toNumber } from "@oh-my-pi/pi-catalog/utils";
export type CodexRateLimit = {
used_percent?: number;
packages/ai/src/providers/openai-completions-compat.ts
-354
View File
@@ -1,354 +0,0 @@
import type { Model, OpenAICompat } from "../types";
type OpenAIReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh";
type ResolvedToolStrictMode = NonNullable<OpenAICompat["toolStrictMode"]> | "mixed";
export type ResolvedOpenAICompat = Required<
Omit<
OpenAICompat,
| "openRouterRouting"
| "vercelGatewayRouting"
| "extraBody"
| "toolStrictMode"
| "cacheControlFormat"
| "thinkingKeep"
>
> & {
openRouterRouting?: OpenAICompat["openRouterRouting"];
vercelGatewayRouting?: OpenAICompat["vercelGatewayRouting"];
extraBody?: OpenAICompat["extraBody"];
cacheControlFormat?: OpenAICompat["cacheControlFormat"];
thinkingKeep?: OpenAICompat["thinkingKeep"];
toolStrictMode: ResolvedToolStrictMode;
};
function detectStrictModeSupport(provider: string, baseUrl: string): boolean {
if (
provider === "openai" ||
provider === "openrouter" ||
provider === "cerebras" ||
provider === "together" ||
provider === "github-copilot" ||
provider === "zenmux"
) {
return true;
}
const normalizedBaseUrl = baseUrl.toLowerCase();
return (
normalizedBaseUrl.includes("api.openai.com") ||
normalizedBaseUrl.includes(".openai.azure.com") ||
normalizedBaseUrl.includes("models.inference.ai.azure.com") ||
normalizedBaseUrl.includes("api.cerebras.ai") ||
normalizedBaseUrl.includes("api.together.xyz") ||
normalizedBaseUrl.includes("openrouter.ai") ||
normalizedBaseUrl.includes("api.deepseek.com") ||
normalizedBaseUrl.includes("deepseek.com")
);
}
function getOpenRouterAnthropicReasoningEffortMap(
modelId: string,
): Partial<Record<OpenAIReasoningEffort, string>> | undefined {
const match = /(?:^|\/)claude-(opus|fable|mythos)-(\d{1,2})(?:[.-](\d{1,2}))?/.exec(modelId);
if (!match) return undefined;
const kind = match[1];
const major = Number(match[2]);
const minor = Number(match[3] ?? 0);
const isFableOrMythos = kind === "fable" || kind === "mythos";
const isOpusAdaptive = kind === "opus" && (major > 4 || (major === 4 && minor >= 6));
if (!isFableOrMythos && !isOpusAdaptive) return undefined;
const hasRealXHigh = isFableOrMythos || major > 4 || (major === 4 && minor >= 7);
if (hasRealXHigh) {
return {
minimal: "low",
low: "medium",
medium: "high",
high: "xhigh",
xhigh: "max",
};
}
return {
minimal: "low",
xhigh: "max",
};
}
/**
* Detect compatibility settings from provider and baseUrl for known providers.
* Provider takes precedence over URL-based detection since it's explicitly configured.
* @param model - The model configuration
* @param resolvedBaseUrl - Optional resolved base URL (e.g., after GitHub Copilot proxy-ep resolution).
* If provided, this takes precedence over model.baseUrl for URL-based checks.
*/
export function detectOpenAICompat(model: Model<"openai-completions">, resolvedBaseUrl?: string): ResolvedOpenAICompat {
const provider = model.provider;
// Use resolvedBaseUrl if provided (e.g., after GitHub Copilot proxy-ep resolution)
const baseUrl = resolvedBaseUrl ?? model.baseUrl;
const isCerebras = provider === "cerebras" || baseUrl.includes("cerebras.ai");
const isZai = provider === "zai" || baseUrl.includes("api.z.ai");
const isZhipu = provider === "zhipu-coding-plan" || baseUrl.includes("open.bigmodel.cn");
const isKilo = provider === "kilo" || baseUrl.includes("api.kilo.ai");
const isKimiModel = model.id.includes("moonshotai/kimi") || /(^|\/)kimi[-.]/i.test(model.id);
const isMoonshotNativeHost =
provider === "moonshot" || provider === "kimi-code" || /api\.moonshot\.ai|api\.kimi\.com/i.test(baseUrl);
const isMoonshotKimi = isKimiModel && isMoonshotNativeHost;
const usesMoonshotKimiPreservedThinking = isMoonshotKimi && /(^|\/)kimi-k2\.6(?:[-:]|$)/i.test(model.id);
const isAnthropicModel =
provider === "anthropic" ||
baseUrl.includes("api.anthropic.com") ||
/(^|\/)claude[-.]/i.test(model.id) ||
/(^|\/)anthropic\//i.test(model.id);
const isAlibaba = provider === "alibaba-coding-plan" || baseUrl.includes("dashscope");
const isQwen = model.id.toLowerCase().includes("qwen");
// DeepSeek V4 (and other reasoning-capable DeepSeek models) reject follow-up requests in
// thinking mode unless prior assistant tool-call turns include `reasoning_content`. The
// upstream model is reachable through many OpenAI-compat hosts (api.deepseek.com, Deepinfra,
// Kilo, NVIDIA NIM, Zenmux, OpenRouter, …), so we match by model id/name as well as by
// provider/baseUrl. The flag is gated by `model.reasoning` because the invariant only
// applies when thinking mode is actually engaged.
const lowerId = model.id.toLowerCase();
const lowerName = (model.name ?? "").toLowerCase();
const isXiaomiHost =
provider === "xiaomi" || provider.startsWith("xiaomi-token-plan-") || baseUrl.includes("xiaomimimo.com");
const isMimoModel = lowerId.includes("mimo") || lowerName.includes("mimo");
const isXiaomiMimo = isXiaomiHost && isMimoModel;
// OpenCode Zen's `big-pickle` is a DeepSeek reasoning alias; the upstream
// 400s come from DeepSeek and require exact reasoning_content replay.
const isOpenCodeDeepseekAlias =
provider === "opencode-zen" && (lowerId === "big-pickle" || lowerName === "big pickle");
const isDeepseekFamily =
provider === "deepseek" ||
baseUrl.includes("deepseek.com") ||
lowerId.includes("deepseek") ||
lowerName.includes("deepseek") ||
isOpenCodeDeepseekAlias;
const isDirectDeepseekApi = provider === "deepseek" || baseUrl.includes("api.deepseek.com");
const isDirectDeepseekReasoning = isDirectDeepseekApi && isDeepseekFamily && Boolean(model.reasoning);
const isNonStandard =
isCerebras ||
provider === "xai" ||
baseUrl.includes("api.x.ai") ||
provider === "mistral" ||
baseUrl.includes("mistral.ai") ||
baseUrl.includes("chutes.ai") ||
baseUrl.includes("deepseek.com") ||
baseUrl.includes("fireworks.ai") ||
isAlibaba ||
isZai ||
isZhipu ||
isKilo ||
isQwen ||
isXiaomiHost ||
provider === "opencode-zen" ||
provider === "opencode-go" ||
baseUrl.includes("opencode.ai");
const isOpenCodeProvider = provider === "opencode-go" || provider === "opencode-zen";
const useMaxTokens =
provider === "mistral" ||
baseUrl.includes("mistral.ai") ||
baseUrl.includes("chutes.ai") ||
baseUrl.includes("fireworks.ai") ||
isDirectDeepseekApi;
const isGrok = provider === "xai" || baseUrl.includes("api.x.ai");
const isMistral = provider === "mistral" || baseUrl.includes("mistral.ai");
// Hosts whose chat-completions endpoints are known to accept multiple
// leading `system`/`developer` messages (preferred for KV-cache reuse).
// Anything outside this allowlist defaults to coalescing because
// strict chat templates (Qwen 3.5+ via vLLM, MiniMax, etc.) reject
// follow-up system messages with a 400.
const isOpenAIHost = provider === "openai" || baseUrl.includes("api.openai.com");
const isAzureHost =
provider === "azure" ||
baseUrl.includes(".openai.azure.com") ||
baseUrl.includes("models.inference.ai.azure.com") ||
baseUrl.includes("azure.com/openai");
const isOpenRouter = provider === "openrouter" || baseUrl.includes("openrouter.ai");
const isTogether = provider === "together" || baseUrl.includes("api.together.xyz");
const isFireworks = baseUrl.includes("fireworks.ai");
const isGroqHost = provider === "groq" || baseUrl.includes("api.groq.com");
const isCopilotHost = provider === "github-copilot";
const isZenmuxHost = provider === "zenmux";
// Endpoints that MUST receive a single system block. MiniMax's OpenAI
// endpoint returns error 2013 on multiple system messages; Alibaba's
// Dashscope and Qwen Portal serve Qwen models whose chat template
// raises "System message must be at the beginning" if any system
// message appears past index 0.
const isMiniMaxHost =
provider === "minimax-code" ||
provider === "minimax-code-cn" ||
baseUrl.includes("api.minimax.io") ||
baseUrl.includes("api.minimaxi.com");
const isQwenPortal = provider === "qwen-portal" || baseUrl.includes("portal.qwen.ai");
const supportsMultipleSystemMessagesDefault =
!isMiniMaxHost &&
!isAlibaba &&
!isQwenPortal &&
(isOpenAIHost ||
isAzureHost ||
isOpenRouter ||
isCerebras ||
isTogether ||
isFireworks ||
isGroqHost ||
isDeepseekFamily ||
isMistral ||
isGrok ||
isZai ||
isZhipu ||
isCopilotHost ||
isZenmuxHost);
const openRouterAnthropicReasoningEffortMap = isOpenRouter
? getOpenRouterAnthropicReasoningEffortMap(lowerId)
: undefined;
const reasoningEffortMap: NonNullable<OpenAICompat["reasoningEffortMap"]> =
provider === "groq" && model.id === "qwen/qwen3-32b"
? ({
minimal: "default",
low: "default",
medium: "default",
high: "default",
xhigh: "default",
} satisfies Partial<Record<OpenAIReasoningEffort, string>>)
: isDeepseekFamily && model.reasoning
? ({
minimal: "high",
low: "high",
medium: "high",
high: "high",
xhigh: "max",
} satisfies Partial<Record<OpenAIReasoningEffort, string>>)
: openRouterAnthropicReasoningEffortMap
? openRouterAnthropicReasoningEffortMap
: isFireworks
? ({
// Fireworks' OpenAI-compatible endpoint rejects OpenAI's
// `minimal` literal but accepts `none` for the lowest setting.
minimal: "none",
} satisfies Partial<Record<OpenAIReasoningEffort, string>>)
: {};
return {
supportsStore: !isNonStandard,
// `developer` is an OpenAI-Responses-era extension to the chat-completions schema. Almost
// every OpenAI-compatible host other than OpenAI itself (and Azure OpenAI, which mirrors
// the schema exactly) treats it as an unknown role: Moonshot returns a 400 "tokenization
// failed", Groq/Cerebras/etc. error or silently misroute. Default to `system` and require
// callers to opt in via `compat.supportsDeveloperRole: true` for hosts known to mirror
// OpenAI's reasoning-API surface.
supportsDeveloperRole: isOpenAIHost || isAzureHost,
supportsMultipleSystemMessages: supportsMultipleSystemMessagesDefault,
supportsReasoningEffort: !isGrok && !isZai && !isZhipu && !isXiaomiMimo,
reasoningEffortMap,
supportsUsageInStreaming: !isCerebras,
disableReasoningOnForcedToolChoice: isKimiModel || isAnthropicModel,
disableReasoningOnToolChoice: isDeepseekFamily && Boolean(model.reasoning) && !isOpenRouter,
supportsToolChoice: !isDirectDeepseekReasoning,
maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens",
requiresToolResultName: isMistral,
requiresAssistantAfterToolResult: false,
requiresThinkingAsText: isMistral,
requiresMistralToolIds: isMistral,
// Only Kimi's native hosts (Moonshot / Kimi-code, matched by `isMoonshotKimi`)
// speak the z.ai binary `thinking: { type }` field. Kimi reached through
// OpenAI-compatible proxies — Fireworks' Fire Pass router, OpenCode's gateway,
// etc. — drives reasoning via OpenAI-style `reasoning_effort`
// (low|medium|high|xhigh|max|none), so those stay on the "openai" path.
thinkingFormat:
isZai || isZhipu || isMoonshotKimi || isXiaomiMimo
? "zai"
: provider === "openrouter" || baseUrl.includes("openrouter.ai")
? "openrouter"
: isAlibaba || isQwen
? "qwen"
: "openai",
thinkingKeep: usesMoonshotKimiPreservedThinking ? "all" : undefined,
reasoningContentField: "reasoning_content",
// Backends that 400 follow-up requests when prior assistant tool-call turns lack `reasoning_content`:
// - Kimi: documented invariant on its native API.
// - DeepSeek-family reasoning models, including aliased OpenCode Zen models
// like `big-pickle`, validate exact thinking-mode replay.
// - Xiaomi MiMo models require exact `reasoning_content` replay on
// thinking-mode tool-call continuations across standard and Token Plan hosts.
// - Any reasoning-capable model reached through OpenRouter can enforce this
// server-side whenever the request is in thinking mode. We can't translate
// Anthropic's redacted/encrypted reasoning into provider-native plaintext,
// so cross-provider continuations rely on a placeholder.
// OpenCode Kimi aliases handle reasoning content internally and reject
// client-sent `reasoning_content`, so exclude only that Kimi-on-OpenCode path.
requiresReasoningContentForToolCalls:
(isKimiModel && !isOpenCodeProvider) ||
(isDeepseekFamily && Boolean(model.reasoning)) ||
isXiaomiMimo ||
((provider === "openrouter" || baseUrl.includes("openrouter.ai")) && Boolean(model.reasoning)),
// DeepSeek V4 and Xiaomi MiMo reject synthetic reasoning_content placeholders (".") on tool-call turns.
// Kimi and OpenRouter accept them when actual reasoning is unavailable.
allowsSyntheticReasoningContentForToolCalls: (!isDeepseekFamily || !model.reasoning) && !isXiaomiMimo,
requiresAssistantContentForToolCalls: isKimiModel || isDirectDeepseekReasoning,
cacheControlFormat: isOpenRouter && model.id.startsWith("anthropic/") ? "anthropic" : undefined,
openRouterRouting: undefined,
vercelGatewayRouting: undefined,
supportsStrictMode: detectStrictModeSupport(provider, baseUrl),
extraBody: isDirectDeepseekReasoning ? { thinking: { type: "enabled" } } : undefined,
toolStrictMode: isCerebras ? "all_strict" : "mixed",
};
}
/**
* Resolve compatibility settings by layering explicit model.compat overrides onto
* the detected defaults. This is the canonical compat view for both metadata and transport.
* @param model - The model configuration
* @param resolvedBaseUrl - Optional resolved base URL (e.g., after GitHub Copilot proxy-ep resolution).
* If provided, this takes precedence over model.baseUrl for URL-based checks.
*/
export function resolveOpenAICompat(
model: Model<"openai-completions">,
resolvedBaseUrl?: string,
): ResolvedOpenAICompat {
const detected = detectOpenAICompat(model, resolvedBaseUrl);
if (!model.compat) {
return detected;
}
return {
supportsStore: model.compat.supportsStore ?? detected.supportsStore,
supportsDeveloperRole: model.compat.supportsDeveloperRole ?? detected.supportsDeveloperRole,
supportsMultipleSystemMessages:
model.compat.supportsMultipleSystemMessages ?? detected.supportsMultipleSystemMessages,
supportsReasoningEffort: model.compat.supportsReasoningEffort ?? detected.supportsReasoningEffort,
reasoningEffortMap: { ...detected.reasoningEffortMap, ...(model.compat.reasoningEffortMap ?? {}) },
supportsUsageInStreaming: model.compat.supportsUsageInStreaming ?? detected.supportsUsageInStreaming,
supportsToolChoice: model.compat.supportsToolChoice ?? detected.supportsToolChoice,
maxTokensField: model.compat.maxTokensField ?? detected.maxTokensField,
requiresToolResultName: model.compat.requiresToolResultName ?? detected.requiresToolResultName,
requiresAssistantAfterToolResult:
model.compat.requiresAssistantAfterToolResult ?? detected.requiresAssistantAfterToolResult,
requiresThinkingAsText: model.compat.requiresThinkingAsText ?? detected.requiresThinkingAsText,
requiresMistralToolIds: model.compat.requiresMistralToolIds ?? detected.requiresMistralToolIds,
thinkingFormat: model.compat.thinkingFormat ?? detected.thinkingFormat,
thinkingKeep: model.compat.thinkingKeep ?? detected.thinkingKeep,
reasoningContentField: model.compat.reasoningContentField ?? detected.reasoningContentField,
requiresReasoningContentForToolCalls:
model.compat.requiresReasoningContentForToolCalls ?? detected.requiresReasoningContentForToolCalls,
allowsSyntheticReasoningContentForToolCalls:
model.compat.allowsSyntheticReasoningContentForToolCalls ??
detected.allowsSyntheticReasoningContentForToolCalls,
requiresAssistantContentForToolCalls:
model.compat.requiresAssistantContentForToolCalls ?? detected.requiresAssistantContentForToolCalls,
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
disableReasoningOnForcedToolChoice:
model.compat.disableReasoningOnForcedToolChoice ?? detected.disableReasoningOnForcedToolChoice,
disableReasoningOnToolChoice: model.compat.disableReasoningOnToolChoice ?? detected.disableReasoningOnToolChoice,
openRouterRouting: model.compat.openRouterRouting ?? detected.openRouterRouting,
vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting,
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
extraBody: model.compat.extraBody ?? detected.extraBody,
toolStrictMode: model.compat.toolStrictMode ?? detected.toolStrictMode,
};
}
packages/ai/src/providers/openai-completions.ts
+24 -118
View File
@@ -1,3 +1,10 @@
import type { Effort } from "@oh-my-pi/pi-catalog/effort";
import { toFirepassWireModelId, toFireworksWireModelId } from "@oh-my-pi/pi-catalog/fireworks-model-id";
import { isDeepseekModelIdOrName } from "@oh-my-pi/pi-catalog/identity";
import { getSupportedEfforts } from "@oh-my-pi/pi-catalog/model-thinking";
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import type { ResolvedOpenAICompat } from "@oh-my-pi/pi-catalog/types";
import { parseGitHubCopilotApiKey } from "@oh-my-pi/pi-catalog/wire/github-copilot";
import { $env, extractHttpStatusFromError } from "@oh-my-pi/pi-utils";
import OpenAI, { APIConnectionTimeoutError as OpenAIConnectionTimeoutError } from "openai";
import type {
@@ -10,10 +17,6 @@ import type {
ChatCompletionToolMessageParam,
} from "openai/resources/chat/completions";
import packageJson from "../../package.json" with { type: "json" };
import type { Effort } from "../effort";
import { getSupportedEfforts } from "../model-thinking";
import { calculateCost } from "../models";
import { parseGitHubCopilotApiKey } from "../registry/oauth/github-copilot";
import { getKimiCommonHeaders } from "../registry/oauth/kimi";
import { getEnvApiKey } from "../stream";
import {
@@ -43,7 +46,6 @@ import {
import { normalizeSystemPrompts } from "../utils";
import { createAbortSourceTracker } from "../utils/abort";
import { AssistantMessageEventStream } from "../utils/event-stream";
import { toFirepassWireModelId, toFireworksWireModelId } from "../utils/fireworks-model-id";
import {
type CapturedHttpErrorResponse,
finalizeErrorMessage,
@@ -73,7 +75,6 @@ import {
hasCopilotVisionInput,
resolveGitHubCopilotBaseUrl,
} from "./github-copilot-headers";
import { detectOpenAICompat, type ResolvedOpenAICompat, resolveOpenAICompat } from "./openai-completions-compat";
import { createInitialResponsesAssistantMessage } from "./openai-responses-shared";
import { transformMessages } from "./transform-messages";
import {
@@ -390,44 +391,6 @@ function getTrailingPartialDeepseekToken(text: string): string {
const OPENAI_COMPLETIONS_FIRST_EVENT_TIMEOUT_MESSAGE =
"OpenAI completions stream timed out while waiting for the first event";
const GLM_CODING_PLAN_STREAM_IDLE_TIMEOUT_MS = 600_000;
const GLM_CODING_PLAN_MODEL_PATTERN = /^glm-5(?:[.-]|$)/i;
// DeepSeek V4 reasoning models on the official api.deepseek.com emit no SSE
// bytes while the model finishes its private chain-of-thought, which routinely
// takes longer than the generic 100s first-event floor under load (issue
// #2177). Mirror the GLM coding-plan widening: a 5-minute idle floor lifts the
// first-event watchdog (it floors at idle) without changing the runtime
// streaming behavior, so reasoning warm-ups stop aborting and retrying.
const DEEPSEEK_REASONING_STREAM_IDLE_TIMEOUT_MS = 300_000;
function isDirectDeepseekReasoningModel(model: Model<"openai-completions">): boolean {
if (!model.reasoning) return false;
if (model.provider === "deepseek") return true;
return model.baseUrl.toLowerCase().includes("api.deepseek.com");
}
/** Returns the widened OpenAI stream watchdog floor for slow reasoning models hosted on OpenAI-compatible endpoints. */
export function getOpenAICompletionsStreamIdleTimeoutFallbackMs(
model: Model<"openai-completions">,
): number | undefined {
if (GLM_CODING_PLAN_MODEL_PATTERN.test(model.id)) {
if (model.provider === "zhipu-coding-plan" || model.provider === "zai")
return GLM_CODING_PLAN_STREAM_IDLE_TIMEOUT_MS;
const baseUrl = model.baseUrl.toLowerCase();
if (baseUrl.includes("open.bigmodel.cn") || baseUrl.includes("api.z.ai")) {
return GLM_CODING_PLAN_STREAM_IDLE_TIMEOUT_MS;
}
}
if (isDirectDeepseekReasoningModel(model)) {
return DEEPSEEK_REASONING_STREAM_IDLE_TIMEOUT_MS;
}
return undefined;
}
async function* observeDecodedOpenAICompletionChunks(
chunks: AsyncIterable<ChatCompletionChunk>,
observer: (event: RawSseEvent) => void,
@@ -468,7 +431,7 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = (
try {
const apiKey = options?.apiKey || getEnvApiKey(model.provider) || "";
const idleTimeoutFallbackMs = getOpenAICompletionsStreamIdleTimeoutFallbackMs(model);
const idleTimeoutFallbackMs = model.compat.streamIdleTimeoutMs;
const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(idleTimeoutFallbackMs);
const firstEventTimeoutMs =
options?.streamFirstEventTimeoutMs ?? getOpenAIStreamFirstEventTimeoutMs(idleTimeoutMs);
@@ -495,13 +458,7 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = (
const createCompletionsStream = async (toolStrictModeOverride?: ToolStrictModeOverride) => {
clearCapturedErrorResponse();
const effectiveToolStrictModeOverride = disableStrictTools ? "none" : toolStrictModeOverride;
const { params, toolStrictMode } = buildParams(
model,
context,
options,
baseUrl,
effectiveToolStrictModeOverride,
);
const { params, toolStrictMode } = buildParams(model, context, options, effectiveToolStrictModeOverride);
appliedToolStrictMode = toolStrictMode;
options?.onPayload?.(params);
rawRequestDump = {
@@ -576,7 +533,7 @@ export const streamOpenAICompletions: StreamFunction<"openai-completions"> = (
// though tool calls are also surfaced structurally. Strip the leaked markers
// so users don't see raw `<｜...｜>` tokens.
const stripDeepseekChatTemplateTokens =
/deepseek/i.test(model.id) && (model.provider === "nvidia" || model.provider === "deepseek");
isDeepseekModelIdOrName(model.id) && (model.provider === "nvidia" || model.provider === "deepseek");
type ToolCallStreamBlock = ToolCall & {
partialArgs?: string | Record<string, unknown>;
streamIndex?: number;
@@ -1216,64 +1173,32 @@ function buildParams(
model: Model<"openai-completions">,
context: Context,
options: OpenAICompletionsOptions | undefined,
resolvedBaseUrl?: string,
toolStrictModeOverride?: ToolStrictModeOverride,
): { params: OpenAICompletionsParams; toolStrictMode: AppliedToolStrictMode } {
const compat = getCompat(model, resolvedBaseUrl);
// Opencode Zen's gateway (https://opencode.ai/zen/go/v1) gates
// `reasoning_content` on the request's thinking state for every model it
// fronts (Kimi K2.x, DeepSeek V4, GLM-5.x, Qwen3.x, MiMo, MiniMax, …): it
// 400s with `Extra inputs are not permitted` when thinking is off but the
// field is supplied (#1071), and 400s with `thinking is enabled but
// reasoning_content is missing in assistant tool call message at index N`
// (#1484) when thinking is on and the field is absent. `detectOpenAICompat`
// only set `requiresReasoningContentForToolCalls` for the DeepSeek family
// (and previously for Kimi until #1071 carved out opencode); reactivate it
// per request for every opencode model whenever this turn is in thinking
// mode so prior tool-call turns replay reasoning_content. Forced-tool
// turns are excluded because the later `disableReasoningOnForcedToolChoice`
// guard at the bottom of `buildParams` strips thinking from the wire body
// for Kimi-style models — keeping the replay on under those conditions
// would resurrect the #1071 failure.
//
// `allowsSyntheticReasoningContentForToolCalls` is forced to `false` on
// the same path: the gateway specifically requires `reasoning_content`,
// and the default synthetic-friendly behavior would echo whichever field
// the upstream streamed (e.g. `reasoning` for many opencode turns),
// landing the replay in the wrong key and re-triggering the 400.
const isOpenCodeProvider = model.provider === "opencode-go" || model.provider === "opencode-zen";
let compat = model.compat;
const thinkingEnabledForRequest =
Boolean(options?.reasoning) && !options?.disableReasoning && Boolean(model.reasoning);
const forcedToolChoiceSuppressesThinking =
compat.disableReasoningOnForcedToolChoice &&
isForcedToolChoice(mapToOpenAICompletionsToolChoice(options?.toolChoice));
if (isOpenCodeProvider && thinkingEnabledForRequest && !forcedToolChoiceSuppressesThinking) {
compat.requiresReasoningContentForToolCalls = true;
compat.allowsSyntheticReasoningContentForToolCalls = false;
compat.reasoningContentField = "reasoning_content";
if (compat.whenThinking && thinkingEnabledForRequest && !forcedToolChoiceSuppressesThinking) {
compat = compat.whenThinking; // precomputed at model build — pointer swap, no allocation
}
const isKimiModelId = model.id.includes("moonshotai/kimi") || /(^|\/)kimi[-.]/i.test(model.id);
const isOpenRouter = model.baseUrl.includes("openrouter.ai");
const messages = convertMessages(model, context, compat);
maybeAddAnthropicCacheControl(compat, messages);
const supportsReasoningParams = model.provider !== "github-copilot";
const supportsReasoningParams = compat.supportsReasoningParams;
// Kimi (including via OpenRouter and Fireworks router-form IDs such as
// `accounts/fireworks/routers/kimi-*`) calculates TPM rate limits based on
// max_tokens, not actual output. The official Kimi K2 model guidance
// (https://docs.fireworks.ai/models/kimi-k2) also requires `max_tokens` for
// every call since the family can otherwise emit very long reasoning traces
// before the final answer. Always send max_tokens — match the same
// Kimi-family regex used by the compat detector.
// Note: Direct kimi-code provider is handled by the dedicated Kimi provider in kimi.ts.
const requestedMaxTokens = options?.maxTokens ?? (isKimiModelId ? model.maxTokens : undefined);
// Kimi-family models calculate TPM rate limits from max_tokens (not actual
// output) and the official guidance requires sending it on every call —
// `compat.alwaysSendMaxTokens` carries that detection.
const requestedMaxTokens = options?.maxTokens ?? (compat.alwaysSendMaxTokens ? model.maxTokens : undefined);
// OpenRouter fans out to upstreams whose output caps differ from the catalog
// value (which tracks the highest-cap provider). A max_tokens above the routed
// upstream's cap makes OpenRouter silently skip that provider (e.g. Cerebras
// GLM-4.7, ~40k) for a higher-cap one, defeating `provider.order`/`only`. Omit
// it for OpenRouter so each upstream self-caps and routing is honored. Kimi is
// exempt — it derives TPM rate limits from max_tokens (see above).
const omitMaxTokensForRouting = isOpenRouter && !isKimiModelId;
// it for OpenRouter so each upstream self-caps and routing is honored — unless
// the model always requires max_tokens (Kimi TPM accounting, see above).
const omitMaxTokensForRouting = compat.isOpenRouterHost && !compat.alwaysSendMaxTokens;
const effectiveMaxTokens =
requestedMaxTokens === undefined || omitMaxTokensForRouting
? undefined
@@ -1442,13 +1367,13 @@ function buildParams(
}
// OpenRouter provider routing preferences
if (model.baseUrl.includes("openrouter.ai") && compat.openRouterRouting) {
if (compat.isOpenRouterHost && compat.openRouterRouting) {
params.provider = compat.openRouterRouting;
}
// Vercel AI Gateway provider routing preferences
if (model.baseUrl.includes("ai-gateway.vercel.sh") && model.compat?.vercelGatewayRouting) {
const routing = model.compat.vercelGatewayRouting;
if (compat.isVercelGatewayHost && compat.vercelGatewayRouting) {
const routing = compat.vercelGatewayRouting;
if (routing.only || routing.order) {
const gatewayOptions: Record<string, string[]> = {};
if (routing.only) gatewayOptions.only = routing.only;
@@ -2121,22 +2046,3 @@ function mapStopReason(reason: ChatCompletionChunk.Choice["finish_reason"] | str
};
}
}
/**
* Detect compatibility settings from provider and baseUrl for known providers.
* Provider takes precedence over URL-based detection since it's explicitly configured.
* Returns a fully resolved OpenAICompat object with all fields set.
*/
export function detectCompat(model: Model<"openai-completions">): ResolvedOpenAICompat {
return detectOpenAICompat(model);
}
/**
* Get resolved compatibility settings for a model.
* Uses explicit model.compat if provided, otherwise auto-detects from provider/URL.
* @param model - The model configuration
* @param resolvedBaseUrl - Optional resolved base URL (e.g., after GitHub Copilot proxy-ep resolution).
*/
function getCompat(model: Model<"openai-completions">, resolvedBaseUrl?: string): ResolvedOpenAICompat {
return resolveOpenAICompat(model, resolvedBaseUrl);
}
packages/ai/src/providers/openai-responses-shared.ts
+10 -1
View File
@@ -1,3 +1,4 @@
import { calculateCost } from "@oh-my-pi/pi-catalog/models";
import { logger, structuredCloneJSON } from "@oh-my-pi/pi-utils";
import type OpenAI from "openai";
import type {
@@ -11,7 +12,6 @@ import type {
ResponseOutputMessage,
ResponseReasoningItem,
} from "openai/resources/responses/responses";
import { calculateCost } from "../models";
import {
type Api,
type AssistantMessage,
@@ -459,6 +459,14 @@ export function appendResponsesToolResultMessages<TApi extends Api>(
export interface ProcessResponsesStreamOptions {
onFirstToken?: () => void;
onOutputItemDone?: (item: ResponseOutputItem) => void;
/**
* Called when a terminal `response.completed` or `response.incomplete` event
* is successfully processed. Only invoked on the successful-completion path;
* thrown failure (`response.failed`) and cancellation paths never call this.
* Used by callers to detect premature stream closure (i.e. the stream ended
* without a recognized terminal event).
*/
onCompleted?: () => void;
}
export async function processResponsesStream<TApi extends Api>(
@@ -905,6 +913,7 @@ export async function processResponsesStream<TApi extends Api>(
if (output.content.some(block => block.type === "toolCall") && output.stopReason === "stop") {
output.stopReason = "toolUse";
}
options?.onCompleted?.();
} else if (event.type === "error") {
throw new Error(`Error Code ${event.code}: ${event.message}`);
} else if (event.type === "response.failed") {
packages/ai/src/providers/openai-responses.ts
+22 -54
View File
@@ -1,3 +1,4 @@
import { parseGitHubCopilotApiKey } from "@oh-my-pi/pi-catalog/wire/github-copilot";
import { $env, extractHttpStatusFromError } from "@oh-my-pi/pi-utils";
import OpenAI, { APIConnectionTimeoutError as OpenAIConnectionTimeoutError } from "openai";
import type {
@@ -6,11 +7,9 @@ import type {
ResponseInput,
ResponseStreamEvent,
} from "openai/resources/responses/responses";
import { parseGitHubCopilotApiKey } from "../registry/oauth/github-copilot";
import { getEnvApiKey } from "../stream";
import type {
AssistantMessage,
CacheRetention,
Context,
FetchImpl,
MessageAttribution,
@@ -69,20 +68,6 @@ import {
} from "./openai-responses-shared";
import { transformMessages } from "./transform-messages";
/**
* Get prompt cache retention based on cacheRetention and base URL.
* Only applies to direct OpenAI API calls (api.openai.com).
*/
function getPromptCacheRetention(baseUrl: string, cacheRetention: CacheRetention): "24h" | undefined {
if (cacheRetention !== "long") {
return undefined;
}
if (baseUrl.includes("api.openai.com")) {
return "24h";
}
return undefined;
}
export function normalizeOpenAIResponsesPromptCacheKey(sessionId: string | undefined): string | undefined {
if (!sessionId || sessionId.length === 0) return undefined;
const wellFormed = sessionId.toWellFormed();
@@ -242,7 +227,7 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = (
);
const premiumRequestsTotal = copilotPremiumRequests;
const providerSessionState = getOpenAIResponsesProviderSessionState(model, options?.providerSessionState);
const params = buildParams(model, context, options, providerSessionState, baseUrl);
const params = buildParams(model, context, options, providerSessionState);
const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs();
const firstEventTimeoutMs =
options?.streamFirstEventTimeoutMs ?? getOpenAIStreamFirstEventTimeoutMs(idleTimeoutMs);
@@ -294,6 +279,7 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = (
stream.push({ type: "start", partial: output });
const nativeOutputItems: Array<Record<string, unknown>> = [];
let sawCompleted = false;
const timedOpenaiStream = iterateWithIdleTimeout(openaiStream, {
idleTimeoutMs,
firstItemTimeoutMs: firstEventTimeoutMs,
@@ -316,6 +302,9 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = (
// second deep copy needed (reasoning items carry multi-KB blobs).
nativeOutputItems.push(item as unknown as Record<string, unknown>);
},
onCompleted: () => {
sawCompleted = true;
},
});
const firstEventTimeoutError = abortTracker.getLocalAbortReason();
@@ -326,6 +315,14 @@ export const streamOpenAIResponses: StreamFunction<"openai-responses"> = (
throw new Error("Request was aborted");
}
// Detect premature stream closure: the HTTP stream ended without the
// provider sending `response.completed`. Custom/proxy providers may
// drop the connection mid-stream; without this guard the incomplete
// output is silently surfaced as a successful "stop".
if (!sawCompleted) {
throw new Error("OpenAI responses stream closed before response.completed was received");
}
if (output.stopReason === "aborted" || output.stopReason === "error") {
throw new Error(output.errorMessage ?? "An unknown error occurred");
}
@@ -395,7 +392,7 @@ function createClient(
copilotPremiumRequests = copilot.premiumRequests;
baseUrl = resolveGitHubCopilotBaseUrl(model.baseUrl, rawApiKey) ?? model.baseUrl;
}
if (sessionId && model.provider === "openai" && (baseUrl ?? "").toLowerCase().includes("api.openai.com")) {
if (sessionId && model.provider === "openai") {
headers.session_id ??= sessionId;
headers["x-client-request-id"] ??= sessionId;
}
@@ -438,17 +435,14 @@ function buildParams(
context: Context,
options: OpenAIResponsesOptions | undefined,
providerSessionState: OpenAIResponsesProviderSessionState | undefined,
resolvedBaseUrl?: string,
): OpenAIResponsesSamplingParams {
const strictResponsesPairing =
options?.strictResponsesPairing ??
(isAzureOpenAIBaseUrl(model.baseUrl ?? "") || model.provider === "github-copilot");
const strictResponsesPairing = options?.strictResponsesPairing ?? model.compat.strictResponsesPairing;
const messages = convertConversationMessages(model, context, strictResponsesPairing, providerSessionState, options);
const systemPrompts = normalizeSystemPrompts(context.systemPrompt);
let systemInstructions: string | undefined;
if (systemPrompts.length > 0) {
const needsDeveloperRole = model.reasoning && supportsDeveloperRole(resolvedBaseUrl ?? model);
const needsDeveloperRole = model.reasoning && model.compat.supportsDeveloperRole;
if (needsDeveloperRole) {
// Reasoning models on known OpenAI-compatible endpoints require the
// `developer` role. Send all system prompts inline in `input`.
@@ -472,7 +466,9 @@ function buildParams(
stream: true,
prompt_cache_key: promptCacheKey,
prompt_cache_retention: promptCacheKey
? getPromptCacheRetention(resolvedBaseUrl ?? model.baseUrl, cacheRetention)
? cacheRetention === "long" && model.compat.supportsLongPromptCacheRetention
? "24h"
: undefined
: undefined,
store: false,
stream_options: model.provider === "openai" ? { include_obfuscation: false } : undefined,
@@ -485,7 +481,7 @@ function buildParams(
// `StreamOptions.frequencyPenalty` is intentionally dropped for this provider.
if (context.tools) {
params.tools = convertTools(context.tools, supportsStrictMode(model), model);
params.tools = convertTools(context.tools, model.compat.supportsStrictMode, model);
if (options?.toolChoice) {
params.tool_choice = mapOpenAIResponsesToolChoiceForTools(options.toolChoice, context.tools, model);
}
@@ -508,7 +504,7 @@ function buildParams(
effort =>
mapReasoningEffort(
effort as NonNullable<OpenAIResponsesOptions["reasoning"]>,
model.compat?.reasoningEffortMap,
model.compat.reasoningEffortMap,
),
options?.includeEncryptedReasoning ?? true,
options?.omitReasoningEffort ?? false,
@@ -528,34 +524,6 @@ function mapReasoningEffort(
return reasoningEffortMap?.[effort] ?? effort;
}
function isAzureOpenAIBaseUrl(baseUrl: string): boolean {
return baseUrl.includes(".openai.azure.com") || baseUrl.includes("azure.com/openai");
}
function supportsStrictMode(model: Model<"openai-responses">): boolean {
if (model.provider === "openai" || model.provider === "azure" || model.provider === "github-copilot") return true;
const baseUrl = model.baseUrl.toLowerCase();
return (
baseUrl.includes("api.openai.com") ||
baseUrl.includes(".openai.azure.com") ||
baseUrl.includes("models.inference.ai.azure.com")
);
}
export function supportsDeveloperRole(modelOrBaseUrl: Pick<Model, "provider" | "baseUrl"> | string): boolean {
const baseUrl =
typeof modelOrBaseUrl === "string" ? modelOrBaseUrl.toLowerCase() : (modelOrBaseUrl.baseUrl ?? "").toLowerCase();
return (
baseUrl.includes("api.openai.com") ||
baseUrl.includes(".openai.azure.com") ||
baseUrl.includes("azure.com/openai") ||
baseUrl.includes("models.inference.ai.azure.com") ||
baseUrl.includes("githubcopilot.com") ||
baseUrl.includes("copilot-api.")
);
}
function convertConversationMessages(
model: Model<"openai-responses">,
context: Context,
packages/ai/src/providers/vision-guard.ts
+5 -3
View File
@@ -1,3 +1,6 @@
import { isDashscopeCompatibleModeUrl } from "@oh-my-pi/pi-catalog/hosts";
import { isQwenModelId } from "@oh-my-pi/pi-catalog/identity";
import type { ImageContent, Model, TextContent } from "../types";
export const NON_VISION_IMAGE_PLACEHOLDER = "[image omitted: model does not support vision]";
@@ -42,11 +45,10 @@ export function joinTextWithImagePlaceholder(text: string, omittedImages: boolea
* provider (issue #1859) can't drive the request into an unrecoverable 400.
*/
export function isDashscopeCompatibleModeTextOnlyQwen(model: Model<"openai-completions">): boolean {
const baseUrl = model.baseUrl.toLowerCase();
if (!baseUrl.includes("dashscope") || !baseUrl.includes("aliyuncs.com") || !baseUrl.includes("/compatible-mode")) {
if (!isDashscopeCompatibleModeUrl(model.baseUrl)) {
return false;
}
const id = model.id.toLowerCase();
if (!id.includes("qwen")) return false;
if (!isQwenModelId(model.id)) return false;
return /\bqwen(?:[\d.]+)?-max\b/.test(id) || /\bqwen(?:[\d.]+)?-coder\b/.test(id);
}
packages/ai/src/rate-limit-utils.ts
+1 -1
View File
@@ -94,7 +94,7 @@ export function calculateRateLimitBackoffMs(reason: RateLimitReason): number {
/** Detect usage/quota limit errors in error messages (persistent, requires credential switch). */
const USAGE_LIMIT_PATTERN =
/usage.?limit|usage_limit_reached|usage_not_included|limit_reached|quota.?exceeded|resource.?exhausted|exhausted your capacity|quota will reset/i;
/usage.?limit|usage_limit_reached|usage_not_included|limit_reached|quota.?exceeded|quota.?reached|resource.?exhausted|exhausted your capacity|quota will reset/i;
export function isUsageLimitError(errorMessage: string): boolean {
return USAGE_LIMIT_PATTERN.test(errorMessage) || ACCOUNT_RATE_LIMIT_PATTERN.test(errorMessage);
packages/ai/src/registry/aimlapi.ts
+1 -7
View File
@@ -1,12 +1,6 @@
import { aimlApiModelManagerOptions } from "../provider-models/openai-compat";
import type { ModelManagerConfig, ProviderDefinition } from "./types";
import type { ProviderDefinition } from "./types";
export const aimlApiProvider = {
id: "aimlapi",
name: "AIML API",
defaultModel: "gpt-4o",
createModelManagerOptions: (config: ModelManagerConfig) => aimlApiModelManagerOptions(config),
dynamicModelsAuthoritative: true,
catalogDiscovery: { label: "AIML API", envVars: ["AIMLAPI_API_KEY"] },
envKeys: "AIMLAPI_API_KEY",
} as const satisfies ProviderDefinition;
packages/ai/src/registry/alibaba-coding-plan.ts
+1 -6
View File
@@ -1,7 +1,6 @@
import { alibabaCodingPlanModelManagerOptions } from "../provider-models/openai-compat";
import { validateOpenAICompatibleApiKey } from "./api-key-validation";
import type { OAuthController, OAuthLoginCallbacks } from "./oauth/types";
import type { ModelManagerConfig, ProviderDefinition } from "./types";
import type { ProviderDefinition } from "./types";
const AUTH_URL = "https://modelstudio.console.alibabacloud.com/";
const API_BASE_URL = "https://coding-intl.dashscope.aliyuncs.com/v1";
@@ -46,9 +45,5 @@ export async function loginAlibabaCodingPlan(options: OAuthController): Promise<
export const alibabaCodingPlanProvider = {
id: "alibaba-coding-plan",
name: "Alibaba Coding Plan",
defaultModel: "qwen3.5-plus",
createModelManagerOptions: (config: ModelManagerConfig) => alibabaCodingPlanModelManagerOptions(config),
catalogDiscovery: { label: "Alibaba Coding Plan", envVars: ["ALIBABA_CODING_PLAN_API_KEY"] },
envKeys: "ALIBABA_CODING_PLAN_API_KEY",
login: (cb: OAuthLoginCallbacks) => loginAlibabaCodingPlan(cb),
} as const satisfies ProviderDefinition;
packages/ai/src/registry/amazon-bedrock.ts
-1
View File
@@ -4,7 +4,6 @@ import type { ProviderDefinition } from "./types";
export const amazonBedrockProvider = {
id: "amazon-bedrock",
name: "Amazon Bedrock",
defaultModel: "us.anthropic.claude-opus-4-6-v1",
// Amazon Bedrock accepts bearer tokens, IAM keys, profiles, ECS/IRSA credential chains.
envKeys: () => {
const hasEcsCredentials =

Some files were not shown because too many files have changed in this diff Show More