From 7fd70eb7a95c03df73bcb7fdffd71d5ee59957ff Mon Sep 17 00:00:00 2001 From: metaphorics <152830360+metaphorics@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:46:23 +0900 Subject: [PATCH] test(pi-shell): add minimizer fixture harness with savings gate Walks tests/fixtures/minimizer//.{cmd,raw,exit,min}: fixtures >=500 bytes must minimize to <=40% of raw size; smaller ones pin exact snapshots. Seeded with three synthetic fixtures. Donor parity fixtures from rtk/snip land on this harness in later commits. Op: extend --- .../fixtures/minimizer/cargo/test-pass.cmd | 1 + .../fixtures/minimizer/cargo/test-pass.raw | 48 ++++ .../tests/fixtures/minimizer/go/test-pass.cmd | 1 + .../tests/fixtures/minimizer/go/test-pass.raw | 61 +++++ .../tests/fixtures/minimizer/npm/install.cmd | 1 + .../tests/fixtures/minimizer/npm/install.raw | 29 ++ crates/pi-shell/tests/minimizer_fixtures.rs | 248 ++++++++++++++++++ 7 files changed, 389 insertions(+) create mode 100644 crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.cmd create mode 100644 crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.raw create mode 100644 crates/pi-shell/tests/fixtures/minimizer/go/test-pass.cmd create mode 100644 crates/pi-shell/tests/fixtures/minimizer/go/test-pass.raw create mode 100644 crates/pi-shell/tests/fixtures/minimizer/npm/install.cmd create mode 100644 crates/pi-shell/tests/fixtures/minimizer/npm/install.raw create mode 100644 crates/pi-shell/tests/minimizer_fixtures.rs diff --git a/crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.cmd b/crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.cmd new file mode 100644 index 000000000..0967aec68 --- /dev/null +++ b/crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.cmd @@ -0,0 +1 @@ +cargo test --workspace diff --git a/crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.raw b/crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.raw new file mode 100644 index 000000000..ec8afa85c --- /dev/null +++ b/crates/pi-shell/tests/fixtures/minimizer/cargo/test-pass.raw @@ -0,0 +1,48 @@ + Compiling proc-macro2 v1.0.86 + Compiling unicode-ident v1.0.12 + Compiling libc v0.2.155 + Compiling autocfg v1.3.0 + Compiling quote v1.0.36 + Compiling syn v2.0.72 + Compiling serde_derive v1.0.204 + Compiling serde v1.0.204 + Compiling thiserror-impl v1.0.63 + Compiling thiserror v1.0.63 + Compiling pi-shell v0.1.0 (/home/alpha/odin/omp/crates/pi-shell) +warning: unused variable: `ctx` + --> crates/pi-shell/src/minimizer/filters/mod.rs:134:30 + | +134 | pub fn filter(ctx: &MinimizerCtx, input: &str) -> MinimizerOutput { + | ^^^ help: if this is intentional, prefix it with an underscore: `_ctx` + | + = note: `#[warn(unused_variables)]` on by default + Finished `test` profile [unoptimized + debuginfo] target(s) in 12.84s + Running unittests src/lib.rs (target/debug/deps/pi_shell-9a1b2c3d4e5f6a7b) + +running 42 tests +test minimizer::detect::tests::detects_git_status ... ok +test minimizer::detect::tests::detects_cargo_test ... ok +test minimizer::engine::tests::enabled_known_filter_minimizes ... ok +test minimizer::engine::tests::unknown_command_is_passthrough ... ok +test minimizer::filters::cargo::tests::passing_run_summarizes ... ok +test minimizer::filters::go::tests::go_test_pass_condenses ... ok +test minimizer::plan::tests::single_command_is_single ... ok +test minimizer::plan::tests::chain_splits_on_and ... ok + +test result: ok. 42 passed; 0 failed; 0 ignored; 0 measured; 0 filtered; finished in 0.31s + + Running tests/minimizer_fixtures.rs (target/debug/deps/minimizer_fixtures-1122334455667788) + +running 8 tests +test fixture_cargo_test_pass ... ok +test fixture_npm_install ... ok +test fixture_go_test_pass ... ok + +test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered; finished in 0.02s + + Doc-tests pi-shell + +running 3 tests +test crates/pi-shell/src/minimizer.rs - minimizer (line 12) ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered; finished in 0.45s diff --git a/crates/pi-shell/tests/fixtures/minimizer/go/test-pass.cmd b/crates/pi-shell/tests/fixtures/minimizer/go/test-pass.cmd new file mode 100644 index 000000000..09c17af6f --- /dev/null +++ b/crates/pi-shell/tests/fixtures/minimizer/go/test-pass.cmd @@ -0,0 +1 @@ +go test -v ./... diff --git a/crates/pi-shell/tests/fixtures/minimizer/go/test-pass.raw b/crates/pi-shell/tests/fixtures/minimizer/go/test-pass.raw new file mode 100644 index 000000000..90983b4d4 --- /dev/null +++ b/crates/pi-shell/tests/fixtures/minimizer/go/test-pass.raw @@ -0,0 +1,61 @@ +=== RUN TestParseConfig +=== PAUSE TestParseConfig +=== CONT TestParseConfig +=== RUN TestParseConfig/empty_input +=== PAUSE TestParseConfig/empty_input +=== CONT TestParseConfig/empty_input +=== RUN TestParseConfig/with_defaults +=== PAUSE TestParseConfig/with_defaults +=== CONT TestParseConfig/with_defaults +=== RUN TestParseConfig/with_overrides +=== PAUSE TestParseConfig/with_overrides +=== CONT TestParseConfig/with_overrides +--- PASS: TestParseConfig (0.00s) +=== RUN TestEncodeDecode +=== PAUSE TestEncodeDecode +=== CONT TestEncodeDecode +=== RUN TestEncodeDecode/utf8 +=== PAUSE TestEncodeDecode/utf8 +=== CONT TestEncodeDecode/utf8 +=== RUN TestEncodeDecode/binary +=== PAUSE TestEncodeDecode/binary +=== CONT TestEncodeDecode/binary +--- PASS: TestEncodeDecode (0.01s) +=== RUN TestRoundTrip +=== PAUSE TestRoundTrip +=== CONT TestRoundTrip +--- PASS: TestRoundTrip (0.02s) +PASS +ok github.com/example/project/internal/config 0.031s +=== RUN TestHandlerGet +=== PAUSE TestHandlerGet +=== CONT TestHandlerGet +=== RUN TestHandlerPost +=== PAUSE TestHandlerPost +=== CONT TestHandlerPost +=== RUN TestHandlerDelete +=== PAUSE TestHandlerDelete +=== CONT TestHandlerDelete +--- PASS: TestHandlerGet (0.00s) +--- PASS: TestHandlerPost (0.00s) +--- PASS: TestHandlerDelete (0.00s) +PASS +ok github.com/example/project/internal/server 0.044s +=== RUN TestRouterMatch +=== PAUSE TestRouterMatch +=== CONT TestRouterMatch +=== RUN TestRouterFallback +=== PAUSE TestRouterFallback +=== CONT TestRouterFallback +--- PASS: TestRouterMatch (0.00s) +--- PASS: TestRouterFallback (0.00s) +PASS +ok github.com/example/project/internal/router 0.012s +=== RUN TestMiddlewareChain +=== PAUSE TestMiddlewareChain +=== CONT TestMiddlewareChain +--- PASS: TestMiddlewareChain (0.00s) +PASS +ok github.com/example/project/internal/middleware 0.009s +? github.com/example/project/cmd/cli [no test files] +ok github.com/example/project/pkg/util 0.008s diff --git a/crates/pi-shell/tests/fixtures/minimizer/npm/install.cmd b/crates/pi-shell/tests/fixtures/minimizer/npm/install.cmd new file mode 100644 index 000000000..5fdd18ffd --- /dev/null +++ b/crates/pi-shell/tests/fixtures/minimizer/npm/install.cmd @@ -0,0 +1 @@ +npm install diff --git a/crates/pi-shell/tests/fixtures/minimizer/npm/install.raw b/crates/pi-shell/tests/fixtures/minimizer/npm/install.raw new file mode 100644 index 000000000..807d836f7 --- /dev/null +++ b/crates/pi-shell/tests/fixtures/minimizer/npm/install.raw @@ -0,0 +1,29 @@ +npm http fetch GET 200 https://registry.npmjs.org/react 142ms +npm http fetch GET 200 https://registry.npmjs.org/react-dom 138ms +npm http fetch GET 200 https://registry.npmjs.org/lodash 95ms +npm http fetch GET 200 https://registry.npmjs.org/webpack 210ms +npm http fetch GET 200 https://registry.npmjs.org/typescript 188ms +npm http fetch GET 200 https://registry.npmjs.org/eslint 176ms +npm http fetch GET 200 https://registry.npmjs.org/jest 201ms +npm http fetch GET 200 https://registry.npmjs.org/babel-core 165ms +Progress: resolved 1 / 1248, reused 0, downloaded 1, added 0 +Progress: resolved 312 / 1248, reused 120, downloaded 192, added 64 +Progress: resolved 640 / 1248, reused 280, downloaded 360, added 210 +Progress: resolved 980 / 1248, reused 510, downloaded 470, added 540 +Progress: resolved 1248 / 1248, reused 800, downloaded 448, added 1248 +npm warn deprecated inflight@1.0.6: This module is not supported, and leaks memory. Do not use it. +npm warn deprecated rimraf@2.7.1: Rimraf versions prior to v4 are no longer supported. +npm warn deprecated glob@7.2.3: Glob versions prior to v9 are no longer supported. +npm notice +npm notice New major version of npm available! 9.8.1 -> 10.2.4 +npm notice Changelog: https://github.com/npm/cli/releases/tag/v10.2.4 +npm notice Run npm install -g npm@10.2.4 to update! +npm notice +resolved 1248 packages in 8s +reused 800, downloaded 448, added 1248 +added 1248 packages, and audited 1249 packages in 14s + +142 packages are looking for funding + run `npm fund` for details + +found 0 vulnerabilities diff --git a/crates/pi-shell/tests/minimizer_fixtures.rs b/crates/pi-shell/tests/minimizer_fixtures.rs new file mode 100644 index 000000000..1c5021ea2 --- /dev/null +++ b/crates/pi-shell/tests/minimizer_fixtures.rs @@ -0,0 +1,248 @@ +//! End-to-end fixture harness for the output minimizer with a token-savings +//! gate. +//! +//! Each fixture is a `/` pair on disk under +//! `tests/fixtures/minimizer/`: +//! +//! - `.cmd` — the full command line (one line; required). +//! - `.raw` — the raw captured output fed to the minimizer (required). +//! - `.exit` — integer exit code (optional; defaults to 0). +//! - `.min` — expected minimized snapshot (optional, see gate below). +//! +//! Gate per fixture (`raw` measured in bytes): +//! +//! - `raw.len() >= 500`: assert `minimized.len() <= 0.40 * raw.len()` — the +//! savings gate. A short output is not worth a filter round-trip, so the gate +//! only applies to buffers large enough to matter. +//! - `raw.len() < 500`: a `.min` snapshot is REQUIRED and must match exactly. +//! Small buffers cannot meaningfully clear a ratio gate, so they are pinned +//! by an exact snapshot instead. +//! - `.min` present alongside a `>= 500`-byte raw: assert BOTH the savings gate +//! and the exact snapshot. +//! +//! All fixture failures are collected before the harness panics so a single run +//! reports every regression, not just the first. + +use std::{fs, path::Path}; + +use pi_shell::minimizer::{self, MinimizerConfig}; + +/// Byte-savings gate: minimized output must be at most this fraction of the raw +/// input for buffers large enough to be worth filtering. +const SAVINGS_RATIO: f64 = 0.40; + +/// Raw buffers below this byte length are pinned by an exact `.min` snapshot +/// instead of the ratio gate. +const GATE_MIN_BYTES: usize = 500; + +/// A single discovered fixture: the `.cmd`/`.raw` pair plus its optional +/// `.exit` and `.min` companions. +struct Fixture { + /// `/` — used only for human-readable failure reports. + name: String, + command: String, + raw: String, + exit: i32, + expected: Option, +} + +#[test] +fn minimizer_fixtures_clear_savings_gate() { + let root = fixtures_root(); + let fixtures = discover_fixtures(&root); + + // A silently-empty harness is worse than none: if the fixtures tree is + // missing or holds no `.cmd` cases, fail loudly rather than passing on zero + // assertions. + assert!( + !fixtures.is_empty(), + "no minimizer fixtures discovered under {}: the harness must run against at least one \ + fixture (a silently-empty gate is worse than none)", + root.display() + ); + + let cfg = MinimizerConfig { enabled: true, ..Default::default() }; + let mut failures: Vec = Vec::new(); + + for fixture in &fixtures { + if let Err(report) = check_fixture(fixture, &cfg) { + failures.push(report); + } + } + + assert!( + failures.is_empty(), + "{} of {} minimizer fixture(s) failed:\n\n{}", + failures.len(), + fixtures.len(), + failures.join("\n\n") + ); +} + +/// Run the minimizer on one fixture and apply the gate. Returns `Err(report)` +/// with a human-readable failure description (fixture name, ratio, diff +/// excerpt) on any violation so the caller can collect every failure. +fn check_fixture(fixture: &Fixture, cfg: &MinimizerConfig) -> Result<(), String> { + let out = minimizer::apply(&fixture.command, &fixture.raw, fixture.exit, cfg); + let minimized = out.text.as_str(); + + let raw_len = fixture.raw.len(); + let min_len = minimized.len(); + let ratio = if raw_len == 0 { + 0.0 + } else { + min_len as f64 / raw_len as f64 + }; + + let mut problems: Vec = Vec::new(); + + if raw_len >= GATE_MIN_BYTES { + let budget = (SAVINGS_RATIO * raw_len as f64).floor() as usize; + if min_len > budget { + problems.push(format!( + "savings gate: minimized {min_len} B > {budget} B budget ({:.1}% of {raw_len} B raw, \ + limit {:.0}%)", + ratio * 100.0, + SAVINGS_RATIO * 100.0 + )); + } + // A `.min` alongside a large raw pins the exact shape too. + if let Some(expected) = &fixture.expected + && expected != minimized + { + problems.push(format!("snapshot mismatch:\n{}", diff_excerpt(expected, minimized))); + } + } else { + // Small buffers cannot meaningfully clear a ratio gate; require an exact + // snapshot instead. + match &fixture.expected { + None => problems.push(format!( + "raw is {raw_len} B (< {GATE_MIN_BYTES} B): a `.min` snapshot is required for \ + sub-threshold fixtures" + )), + Some(expected) if expected != minimized => { + problems.push(format!("snapshot mismatch:\n{}", diff_excerpt(expected, minimized))); + }, + Some(_) => {}, + } + } + + if problems.is_empty() { + Ok(()) + } else { + Err(format!( + "[{}] cmd={:?} exit={} raw={} B min={} B ratio={:.1}%\n - {}", + fixture.name, + fixture.command, + fixture.exit, + raw_len, + min_len, + ratio * 100.0, + problems.join("\n - ") + )) + } +} + +/// Render a short, line-oriented diff excerpt (first divergence plus a little +/// context) so a snapshot failure is legible without dumping both buffers. +fn diff_excerpt(expected: &str, actual: &str) -> String { + let expected_lines: Vec<&str> = expected.lines().collect(); + let actual_lines: Vec<&str> = actual.lines().collect(); + let max = expected_lines.len().max(actual_lines.len()); + + let first_diff = (0..max).find(|&i| expected_lines.get(i) != actual_lines.get(i)); + let Some(start) = first_diff else { + // Lines all match: the difference is a trailing newline / final fragment. + return format!(" expected {expected:?}\n actual {actual:?}"); + }; + + let mut excerpt = String::new(); + for i in start..(start + 3).min(max) { + excerpt.push_str(&format!( + " line {}:\n expected {:?}\n actual {:?}\n", + i + 1, + expected_lines.get(i).copied().unwrap_or(""), + actual_lines.get(i).copied().unwrap_or("") + )); + } + excerpt.trim_end().to_string() +} + +/// Absolute path to the fixtures tree, anchored at the crate manifest dir so +/// the harness loads identically whether run as an integration test or under a +/// different working directory. +fn fixtures_root() -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/minimizer") +} + +/// Walk `//.cmd` and assemble each fixture with its +/// `.raw`/`.exit`/`.min` companions. A missing `.raw` for a discovered `.cmd` +/// is a hard error: the fixture is malformed, not absent. +fn discover_fixtures(root: &Path) -> Vec { + let mut fixtures = Vec::new(); + + let Ok(families) = fs::read_dir(root) else { + // Missing tree → return empty; the test body turns this into a loud + // failure with the full path. + return fixtures; + }; + + let mut family_dirs: Vec<_> = families + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect(); + family_dirs.sort(); + + for family_dir in family_dirs { + let family = family_dir + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("?") + .to_string(); + + let Ok(entries) = fs::read_dir(&family_dir) else { + continue; + }; + let mut cmd_paths: Vec<_> = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "cmd")) + .collect(); + cmd_paths.sort(); + + for cmd_path in cmd_paths { + let case = cmd_path + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or("?") + .to_string(); + let raw_path = cmd_path.with_extension("raw"); + let raw = fs::read_to_string(&raw_path).unwrap_or_else(|err| { + panic!( + "fixture {family}/{case}: missing or unreadable .raw ({}): {err}", + raw_path.display() + ) + }); + + let command = fs::read_to_string(&cmd_path) + .unwrap_or_else(|err| panic!("fixture {family}/{case}: unreadable .cmd: {err}")) + .trim_end_matches(['\n', '\r']) + .to_string(); + + let exit = fs::read_to_string(cmd_path.with_extension("exit")) + .ok() + .map_or(0, |text| { + text.trim().parse::().unwrap_or_else(|err| { + panic!("fixture {family}/{case}: invalid .exit ({:?}): {err}", text.trim()) + }) + }); + + let expected = fs::read_to_string(cmd_path.with_extension("min")).ok(); + + fixtures.push(Fixture { name: format!("{family}/{case}"), command, raw, exit, expected }); + } + } + + fixtures +}