From ce112bf9ffecdbf850584df06681004229904bc1 Mon Sep 17 00:00:00 2001 From: can1357 Date: Fri, 12 Jun 2026 08:21:25 +0200 Subject: [PATCH] fix(coding-agent): fixed omp bench cached OpenRouter replays and Codex instruction 400s - Sent `X-OpenRouter-Cache: false` on bench requests in `bench-cli.ts`: pi-ai opts every OpenRouter request into 1h response caching, so repeated byte-identical runs replayed a cached generation with zeroed usage as "tokens 0, TPS 0.0" successes. - Added a minimal default `systemPrompt` to bench's request context, matching eval's completion-bridge guard against Codex's HTTP 400 `{"detail":"Instructions are required"}`. - Checked in `src/prompts/bench.md`, the default bench prompt `bench-cli.ts` already imports (left untracked by 300c1ada30). - Added both bench changelog entries. --- packages/coding-agent/CHANGELOG.md | 2 ++ packages/coding-agent/src/cli/bench-cli.ts | 9 +++++++++ packages/coding-agent/src/prompts/bench.md | 7 +++++++ 3 files changed, 18 insertions(+) create mode 100644 packages/coding-agent/src/prompts/bench.md diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 3c428ab5a..a40232970 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -18,6 +18,8 @@ ### Fixed - Fixed tool calls taller than the viewport reading as cut off while streaming (the head reappeared only once the result landed): the 15.11.6 stranded-preview fix marked every collapsed pending tool preview commit-unstable, which also blocked durable top-anchored streams — e.g. a task call's context/assignment markdown — from reaching native scrollback mid-run. Commit stability is now classified per renderer (`ToolRenderer.provisionalPendingPreview`): only the tail-window previews the result render re-anchors (edit/apply_patch streamed-diff tails, bash/ssh command caps, eval cells with interleaved outputs) stay provisional; every other pending preview commits its settled head mid-stream again +- Fixed `omp bench` reporting "tokens 0, TPS 0.0" successes on repeated OpenRouter runs: pi-ai opts every OpenRouter request into response caching, so bench's byte-identical request replayed a cached generation with zeroed usage at edge speed. Bench now sends `X-OpenRouter-Cache: false` so every run measures a fresh generation +- Fixed `omp bench` failing with HTTP 400 `{"detail":"Instructions are required"}` against `openai-codex` models: bench requests now carry a minimal default system prompt (same guard as eval's completion bridge) ## [15.11.6] - 2026-06-12 diff --git a/packages/coding-agent/src/cli/bench-cli.ts b/packages/coding-agent/src/cli/bench-cli.ts index 76f0a01fe..6642568d6 100644 --- a/packages/coding-agent/src/cli/bench-cli.ts +++ b/packages/coding-agent/src/cli/bench-cli.ts @@ -171,6 +171,9 @@ async function runBenchRequest( let firstTokenAt: number | undefined; try { const context: Context = { + // Codex's Responses endpoint 400s with "Instructions are required" when no + // system prompt is present — same guard as eval's completion bridge. + systemPrompt: ["You are a helpful assistant."], messages: [{ role: "user", content: options.prompt, timestamp: Date.now(), attribution: "user" }], }; const stream = streamFn(model, context, { @@ -182,6 +185,12 @@ async function runBenchRequest( : options.maxTokens, reasoning: options.reasoning, disableReasoning: options.disableReasoning, + // pi-ai opts every OpenRouter request into response caching (1h TTL). + // Bench sends a byte-identical request each run, so within the TTL + // OpenRouter replays the cached generation with zeroed usage — the run + // shows "tokens 0, TPS 0.0" at line speed. Opt back out so every run + // measures a fresh generation. + headers: model.provider === "openrouter" ? { "X-OpenRouter-Cache": "false" } : undefined, }); let message: AssistantMessage | undefined; for await (const event of stream) { diff --git a/packages/coding-agent/src/prompts/bench.md b/packages/coding-agent/src/prompts/bench.md new file mode 100644 index 000000000..c7e640052 --- /dev/null +++ b/packages/coding-agent/src/prompts/bench.md @@ -0,0 +1,7 @@ +Write a continuous, plain-prose technical explanation of how a relational database executes a SQL query: lexing and parsing, semantic analysis, logical plan construction, cost-based optimization, physical operator selection, and row-by-row execution through the iterator model. + +Form: +- Plain paragraphs only: no headings, no lists, no code fences, no preamble. +- Do not wrap up early or summarize; keep writing until you are cut off. + +Output only the explanation.