From 51fe5c935cf37ca0a70db45fc7f19b8fb670a97d Mon Sep 17 00:00:00 2001 From: roboomp Date: Sat, 1 Aug 2026 23:15:31 +0000 Subject: [PATCH] fix(ai): preserved cache slot for real assistant turns Exclude the synthetic trailing Continue. user pad from Anthropic's message cache window and anchor selection on the preceding assistant instead. This preserves the only remaining message breakpoint when three system breakpoints consume the rest of Anthropic's four-slot budget. Fixes #7324 --- packages/ai/CHANGELOG.md | 2 +- packages/ai/src/providers/anthropic.ts | 16 +++++- packages/ai/test/anthropic-alignment.test.ts | 55 +++++++++++++++++--- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 5c0ae9590..42d8b0929 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,7 +4,7 @@ ### Fixed -- Fixed Anthropic prompt caching writing a fresh entry for the entire system prefix whenever the project footer (cwd, date, workspace tree) changed. `applyPromptCaching` placed its only system breakpoint on the last block — normally the volatile footer — so starting omp in a new directory or crossing midnight re-wrote the whole cached system prefix instead of reusing it (issue [#7324](https://github.com/can1357/oh-my-pi/issues/7324)). System caching now marks up to the last three eligible blocks, covering both `[stable prefix, project footer]` and `[stable prefix, project footer, active-repo context]` layouts while skipping the OAuth cloak blocks (billing header + Claude Code identity). This does not address open-weight chat templates that render tool schemas after the system block; keeping those cached requires relocating the per-request footer out of the system message. +- Fixed Anthropic prompt caching writing a fresh entry for the entire system prefix whenever the project footer (cwd, date, workspace tree) changed. `applyPromptCaching` placed its only system breakpoint on the last block — normally the volatile footer — so starting omp in a new directory or crossing midnight re-wrote the whole cached system prefix instead of reusing it (issue [#7324](https://github.com/can1357/oh-my-pi/issues/7324)). System caching now marks up to the last three eligible blocks, covering both `[stable prefix, project footer]` and `[stable prefix, project footer, active-repo context]` layouts while skipping the OAuth cloak blocks (billing header + Claude Code identity). Message caching also skips the synthetic trailing `Continue.` pad and anchors on the preceding real assistant turn when the four-breakpoint budget is tight. This does not address open-weight chat templates that render tool schemas after the system block; keeping those cached requires relocating the per-request footer out of the system message. ## [17.2.4] - 2026-08-01 diff --git a/packages/ai/src/providers/anthropic.ts b/packages/ai/src/providers/anthropic.ts index feceabb73..1fc346ccc 100644 --- a/packages/ai/src/providers/anthropic.ts +++ b/packages/ai/src/providers/anthropic.ts @@ -3161,8 +3161,20 @@ function applyPromptCaching(params: MessageCreateParamsStreaming, cacheControl?: if (cacheBreakpointsUsed >= MAX_CACHE_BREAKPOINTS) return; - const start = isCCLayout ? Math.max(0, params.messages.length - 1) : Math.max(0, params.messages.length - 2); - for (let i = params.messages.length - 1; i >= start; i--) { + // `convertAnthropicMessages` appends this neutral pad after a trailing + // assistant because Anthropic rejects assistant-prefill endings. It is absent + // from the next normal turn, so caching it wastes a scarce breakpoint; anchor + // the cache window on the preceding real assistant instead. + const trailingIndex = params.messages.length - 1; + const trailingMessage = params.messages[trailingIndex]; + const hasTrailingAssistantPad = + trailingMessage?.role === "user" && + trailingMessage.content === "Continue." && + params.messages[trailingIndex - 1]?.role === "assistant"; + const messageEnd = hasTrailingAssistantPad ? trailingIndex - 1 : trailingIndex; + const messageWindowSize = isCCLayout ? 1 : 2; + const start = Math.max(0, messageEnd - messageWindowSize + 1); + for (let i = messageEnd; i >= start; i--) { if (cacheBreakpointsUsed >= MAX_CACHE_BREAKPOINTS) break; const message = params.messages[i]; if (!message) continue; diff --git a/packages/ai/test/anthropic-alignment.test.ts b/packages/ai/test/anthropic-alignment.test.ts index c39f912ec..cd468958e 100644 --- a/packages/ai/test/anthropic-alignment.test.ts +++ b/packages/ai/test/anthropic-alignment.test.ts @@ -486,9 +486,9 @@ describe("Anthropic request fingerprint alignment", () => { { isOAuth: false }, )) as { messages?: Array<{ role: string; content: string | Array<{ type: string; cache_control?: unknown }> }> }; - // The thinking-only assistant turn sits inside the trailing two-message - // cache window (the Continue. pad is appended after it) but must not get - // a breakpoint — Anthropic rejects cache_control on thinking blocks. + // The thinking-only assistant cannot accept cache_control, so the + // preceding real user turn gets the fallback breakpoint. The synthetic + // trailing Continue. pad must never consume it. const assistant = payload.messages?.find(message => message.role === "assistant"); expect(assistant).toBeDefined(); const assistantContent = assistant?.content; @@ -496,11 +496,52 @@ describe("Anthropic request fingerprint alignment", () => { for (const block of (assistantContent ?? []) as Array<{ type: string; cache_control?: unknown }>) { expect(block.cache_control).toBeUndefined(); } + const user = payload.messages?.[0]; + const userContent = user?.content; + expect(Array.isArray(userContent)).toBe(true); + expect(Array.isArray(userContent) ? userContent[0]?.cache_control : undefined).toBeDefined(); const last = payload.messages?.at(-1); - expect(last).toBeDefined(); - const lastContent = last?.content; - expect(Array.isArray(lastContent)).toBe(true); - expect((lastContent as Array<{ cache_control?: unknown }>)[0]?.cache_control).toBeDefined(); + expect(last?.content).toBe("Continue."); + }); + + it("caches the real assistant before a synthetic Continue pad when the breakpoint budget is tight", async () => { + const assistant: AssistantMessage = { + role: "assistant", + content: [{ type: "text", text: "real assistant answer" }], + api: "anthropic-messages", + provider: "anthropic", + model: ANTHROPIC_MODEL.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; + const payload = (await captureAnthropicPayload( + ANTHROPIC_MODEL, + { + systemPrompt: ["stable system", "volatile project footer", "active repo context"], + messages: [{ role: "user", content: "question", timestamp: Date.now() }, assistant], + }, + { isOAuth: false }, + )) as { + system?: Array<{ cache_control?: unknown }>; + messages?: Array<{ role: string; content: string | Array<{ cache_control?: unknown }> }>; + }; + + expect(payload.system?.filter(block => block.cache_control != null)).toHaveLength(3); + const assistantContent = payload.messages?.find(message => message.role === "assistant")?.content; + expect(Array.isArray(assistantContent) ? assistantContent[0]?.cache_control : undefined).toEqual({ + type: "ephemeral", + ttl: "1h", + }); + const pad = payload.messages?.at(-1); + expect(pad?.content).toBe("Continue."); }); it("adds effort and mid-conversation betas to API-key requests that use those features", async () => {