fix(coding-agent): fixed omp bench cached OpenRouter replays and Codex instruction 400s
- Sent `X-OpenRouter-Cache: false` on bench requests in `bench-cli.ts`: pi-ai opts every OpenRouter request into 1h response caching, so repeated byte-identical runs replayed a cached generation with zeroed usage as "tokens 0, TPS 0.0" successes.
- Added a minimal default `systemPrompt` to bench's request context, matching eval's completion-bridge guard against Codex's HTTP 400 `{"detail":"Instructions are required"}`.
- Checked in `src/prompts/bench.md`, the default bench prompt `bench-cli.ts` already imports (left untracked by 300c1ada30).
- Added both bench changelog entries.
This commit is contained in:
@@ -171,6 +171,9 @@ async function runBenchRequest(
|
||||
let firstTokenAt: number | undefined;
|
||||
try {
|
||||
const context: Context = {
|
||||
// Codex's Responses endpoint 400s with "Instructions are required" when no
|
||||
// system prompt is present — same guard as eval's completion bridge.
|
||||
systemPrompt: ["You are a helpful assistant."],
|
||||
messages: [{ role: "user", content: options.prompt, timestamp: Date.now(), attribution: "user" }],
|
||||
};
|
||||
const stream = streamFn(model, context, {
|
||||
@@ -182,6 +185,12 @@ async function runBenchRequest(
|
||||
: options.maxTokens,
|
||||
reasoning: options.reasoning,
|
||||
disableReasoning: options.disableReasoning,
|
||||
// pi-ai opts every OpenRouter request into response caching (1h TTL).
|
||||
// Bench sends a byte-identical request each run, so within the TTL
|
||||
// OpenRouter replays the cached generation with zeroed usage — the run
|
||||
// shows "tokens 0, TPS 0.0" at line speed. Opt back out so every run
|
||||
// measures a fresh generation.
|
||||
headers: model.provider === "openrouter" ? { "X-OpenRouter-Cache": "false" } : undefined,
|
||||
});
|
||||
let message: AssistantMessage | undefined;
|
||||
for await (const event of stream) {
|
||||
|
||||
Reference in New Issue
Block a user