smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai): defaulted API-key requests to 1h prompt-cache retention

Browse Source 
- Defaults API-key requests to 1h `cache_control` TTL where the endpoint supports long retention, matching OAuth behavior.
- Automatically injects `extended-cache-ttl-2025-04-11` beta on API-key requests using 1h TTL; endpoints without long-cache support keep the 5m breakpoint.
- `PI_CACHE_RETENTION` env var overrides the new default in either direction.
This commit is contained in:
can1357
2026-07-18 19:18:00 +02:00
parent 37456c5f4b
commit b16d316aa9
5 changed files with 114 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/CHANGELOG.md
+4
View File
@@ -2,6 +2,10 @@
## [Unreleased]
### Changed
- Anthropic API-key requests to the canonical API now default to 1h prompt-cache retention (`cache_control: { ttl: "1h" }` plus the `extended-cache-ttl-2025-04-11` beta), matching the OAuth default. The previous 5m default cold-missed the entire prompt prefix whenever a session idled past 5 minutes — e.g. waiting on long-running background jobs. `PI_CACHE_RETENTION` now accepts `short` and `none` to override the default in either direction; endpoints without `compat.supportsLongCacheRetention` keep the 5m breakpoint.
## [17.0.4] - 2026-07-18
### Fixed
packages/ai/src/providers/anthropic.ts
+22 -2
View File
@@ -144,7 +144,8 @@ const claudeCodeAgentBetaDefaults = [
midConversationSystemBeta,
"advanced-tool-use-2025-11-20",
] as const;
const claudeCodeAgentPostEffortBetas = ["extended-cache-ttl-2025-04-11"] as const;
const extendedCacheTtlBeta = "extended-cache-ttl-2025-04-11";
const claudeCodeAgentPostEffortBetas = [extendedCacheTtlBeta] as const;
const fineGrainedToolStreamingBeta = "fine-grained-tool-streaming-2025-05-14";
const interleavedThinkingBeta = "interleaved-thinking-2025-05-14";
// Asks the API to redact thinking blocks from responses. Only sent when the
@@ -424,7 +425,15 @@ function getCacheControl(
cacheRetention: CacheRetention | undefined,
isOAuthToken: boolean,
): { retention: CacheRetention; cacheControl?: AnthropicCacheControl } {
const retention = cacheRetention ?? (isOAuthToken ? "long" : resolveCacheRetention(undefined));
// OAuth mirrors Claude Code and always defaults to 1h retention. API-key
// requests also default to 1h where the endpoint supports it (canonical
// Anthropic API, `compat.supportsLongCacheRetention`): agent sessions
// routinely idle past 5 minutes waiting on background jobs, and a 5m
// breakpoint cold-misses the entire prefix on resume. PI_CACHE_RETENTION
// still overrides the API-key default in either direction.
const retention = isOAuthToken
? (cacheRetention ?? "long")
: resolveCacheRetention(cacheRetention, model.compat.supportsLongCacheRetention ? "long" : "short");
if (retention === "none") {
return { retention };
}
@@ -1853,6 +1862,17 @@ const streamAnthropicOnce = (
) {
extraBetas.push(contextManagementBeta);
}
// `ttl: "1h"` requires the extended-cache-ttl beta on API-key
// requests. OAuth requests never add it here: agent requests
// already carry it in the Claude Code beta list, and utility
// requests must not deviate from CC's header fingerprint.
if (
!(options?.isOAuth ?? isAnthropicOAuthToken(apiKey)) &&
getCacheControl(model, options?.cacheRetention, false).cacheControl?.ttl === "1h" &&
!extraBetas.includes(extendedCacheTtlBeta)
) {
extraBetas.push(extendedCacheTtlBeta);
}
// Server-side fallback beta chain: opt-in via `options.fallbacks`.
// Nested overrides (`speed`, `output_config.effort`,
// `output_config.task_budget`) reuse the same top-level betas
packages/ai/src/utils.ts
+10 -5
View File
@@ -287,11 +287,16 @@ export function getOpenAIResponsesHistoryItems(
}
/**
* Resolve cache retention preference.
* Defaults to "short" and uses PI_CACHE_RETENTION for backward compatibility.
* Resolve cache retention preference: explicit request option first, then the
* `PI_CACHE_RETENTION` env override (`long` | `short` | `none`), then the
* provider-supplied fallback.
*/
export function resolveCacheRetention(cacheRetention?: CacheRetention): CacheRetention {
export function resolveCacheRetention(
cacheRetention?: CacheRetention,
fallback: CacheRetention = "short",
): CacheRetention {
if (cacheRetention) return cacheRetention;
if ($env.PI_CACHE_RETENTION === "long") return "long";
return "short";
const env = $env.PI_CACHE_RETENTION;
if (env === "long" || env === "short" || env === "none") return env;
return fallback;
}
packages/ai/test/anthropic-alignment.test.ts
+36 -1
View File
@@ -427,6 +427,40 @@ describe("Anthropic request fingerprint alignment", () => {
expect(capturedBeta).toContain("mid-conversation-system-2026-04-07");
});
it("adds the extended-cache-ttl beta to API-key requests that default to 1h caching", async () => {
const captureBeta = () => {
let captured: string | undefined;
const fetchMock = (async (_input: string | URL | Request, init?: RequestInit) => {
captured = (init?.headers as Record<string, string> | undefined)?.["anthropic-beta"];
return new Response(
JSON.stringify({ type: "error", error: { type: "invalid_request_error", message: "captured" } }),
{ status: 400, headers: { "Content-Type": "application/json" } },
);
}) as typeof fetch;
return { fetchMock, beta: () => captured ?? "" };
};
const cacheContext: Context = {
systemPrompt: ["Stay concise."],
messages: [{ role: "user", content: "Hi", timestamp: Date.now() }],
};
const canonical = captureBeta();
await streamAnthropic(ANTHROPIC_MODEL, cacheContext, {
apiKey: "sk-ant-api-test",
fetch: canonical.fetchMock,
}).result();
expect(canonical.beta()).toContain("extended-cache-ttl-2025-04-11");
// Endpoints without long-cache support never send `ttl: "1h"`, so the
// companion beta must stay off the wire too.
const proxy = captureBeta();
await streamAnthropic(UMANS_ANTHROPIC_MODEL, cacheContext, {
apiKey: "sk-umans-test",
fetch: proxy.fetchMock,
}).result();
expect(proxy.beta()).not.toContain("extended-cache-ttl-2025-04-11");
});
it("gates the effort beta and field off google-vertex requests (#5614)", async () => {
let capturedBeta: string | undefined;
let capturedBody:
@@ -564,7 +598,8 @@ describe("Anthropic request fingerprint alignment", () => {
expect(payload.system).toEqual([
{ type: "text", text: "stable system" },
{ type: "text", text: "stable durable context", cache_control: { type: "ephemeral" } },
// Canonical Anthropic API-key requests default to the 1h breakpoint.
{ type: "text", text: "stable durable context", cache_control: { type: "ephemeral", ttl: "1h" } },
]);
});
packages/ai/test/anthropic-stream-envelope.test.ts
+42
View File
@@ -8,6 +8,7 @@ import {
} from "@oh-my-pi/pi-ai/providers/anthropic-client";
import type { AssistantMessageEvent, Context, Model, ModelSpec, ProviderSessionState } from "@oh-my-pi/pi-ai/types";
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import { withEnv } from "./helpers";
const model: Model<"anthropic-messages"> = buildModel({
id: "claude-sonnet-4-5",
@@ -1380,4 +1381,45 @@ describe("anthropic stream envelope handling", () => {
expect(cacheControls[1]).toEqual({ type: "ephemeral" });
expect(cacheControls[2]).toEqual({ type: "ephemeral" });
});
it("defaults API-key requests to 1h cache TTL where long retention is supported", async () => {
type CapturedParams = { messages: Array<{ content: unknown }> };
const payloads: CapturedParams[] = [];
vi.spyOn(AnthropicMessages.prototype, "create").mockImplementation((params: unknown) => {
// Params captured verbatim at the mocked SDK boundary.
const captured = params as CapturedParams;
payloads.push(captured);
return createMockRequest(createTextSuccessEvents("ok")) as never;
});
const proxyModel = buildModel({
...model,
compat: { ...model.compatConfig, supportsLongCacheRetention: false },
} as ModelSpec<"anthropic-messages">);
const drain = async (testModel: Model<"anthropic-messages">): Promise<void> => {
const stream = streamAnthropic(testModel, context, { apiKey: "sk-ant-test" });
for await (const _ of stream) {
// drain stream
}
await stream.result();
};
await drain(model);
await drain(proxyModel);
await withEnv({ PI_CACHE_RETENTION: "short" }, () => drain(model));
const cacheControls = payloads.map(payload => {
const content = payload.messages.at(-1)?.content;
if (!Array.isArray(content)) return undefined;
const lastBlock: { cache_control?: { ttl?: string; type: string } } | undefined = content.at(-1);
return lastBlock?.cache_control;
});
// Agent sessions idle past 5 minutes on background jobs; the canonical
// Anthropic API defaults to the 1h breakpoint so resume doesn't cold-miss
// the whole prefix.
expect(cacheControls[0]).toEqual({ type: "ephemeral", ttl: "1h" });
// Endpoints without long-cache support keep the plain 5m breakpoint.
expect(cacheControls[1]).toEqual({ type: "ephemeral" });
// PI_CACHE_RETENTION=short opts back out of the 1h default.
expect(cacheControls[2]).toEqual({ type: "ephemeral" });
});
});