diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index baf6fe21a..7107d0b40 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -1029,7 +1029,7 @@ export function buildParams( } if (model.compat.isVercelGatewayHost) { - applyVercelResponsesCacheControls(params, model.compat, cacheRetention !== "none"); + applyVercelResponsesCacheControls(params, model.compat, cacheRetention); } else { applyOpenAIGatewayRouting(params, model.compat); } diff --git a/packages/ai/src/providers/openai-shared.ts b/packages/ai/src/providers/openai-shared.ts index 9cc12e899..d19a15630 100644 --- a/packages/ai/src/providers/openai-shared.ts +++ b/packages/ai/src/providers/openai-shared.ts @@ -637,6 +637,7 @@ export interface VercelResponsesCacheParams { caching?: "auto"; cache_anchor_items?: number; cache_ttl?: "5m" | "1h"; + providerOptions?: { gateway?: Pick }; } export interface VercelResponsesCacheCompat { @@ -645,20 +646,32 @@ export interface VercelResponsesCacheCompat { } /** - * Apply Vercel AI Gateway's Responses-only automatic cache controls. Chat - * Completions uses the distinct `providerOptions.gateway` shape above. + * Apply Vercel AI Gateway's Responses-only automatic cache controls and + * provider routing. Cache settings are top-level Responses fields, while + * `only` and `order` remain under `providerOptions.gateway`. */ export function applyVercelResponsesCacheControls( params: VercelResponsesCacheParams, compat: VercelResponsesCacheCompat, - cacheEnabled = true, + cacheRetention: CacheRetention = "short", ): void { const routing = compat.vercelGatewayRouting; - if (!cacheEnabled || !compat.isVercelGatewayHost || routing?.caching !== "auto") return; + if (!compat.isVercelGatewayHost) return; + + if (routing?.only || routing?.order) { + const gateway: Pick = {}; + if (routing.only) gateway.only = routing.only; + if (routing.order) gateway.order = routing.order; + params.providerOptions = { gateway }; + } + + if (cacheRetention === "none" || routing?.caching !== "auto") return; params.caching = "auto"; if (routing.cacheAnchorItems !== undefined) params.cache_anchor_items = routing.cacheAnchorItems; - if (routing.cacheTtl !== undefined) params.cache_ttl = routing.cacheTtl; + if (routing.cacheTtl !== undefined && (routing.cacheTtl !== "1h" || cacheRetention === "long")) { + params.cache_ttl = routing.cacheTtl; + } } export interface OpenAIExtraBodyOptions { diff --git a/packages/ai/test/vercel-gateway-cache-controls.test.ts b/packages/ai/test/vercel-gateway-cache-controls.test.ts index 7126d1308..243ee97b0 100644 --- a/packages/ai/test/vercel-gateway-cache-controls.test.ts +++ b/packages/ai/test/vercel-gateway-cache-controls.test.ts @@ -51,7 +51,7 @@ function responsesModel(provider: string, baseUrl: string, routing?: VercelGatew function captureChatPayload( model: Model<"openai-completions">, - options: { cacheRetention?: "none" } = {}, + options: { cacheRetention?: "long" | "short" | "none" } = {}, ): Promise { const { promise, resolve } = Promise.withResolvers(); streamOpenAICompletions(model, context, { @@ -65,7 +65,7 @@ function captureChatPayload( function captureResponsesPayload( model: Model<"openai-responses">, - options: { cacheRetention?: "none" } = {}, + options: { cacheRetention?: "long" | "short" | "none" } = {}, ): Promise { const { promise, resolve } = Promise.withResolvers(); streamOpenAIResponses(model, context, { @@ -78,7 +78,7 @@ function captureResponsesPayload( } describe("Vercel AI Gateway automatic cache controls", () => { - it("maps cache fields to their documented Chat and Responses request shapes", async () => { + it("maps cache fields and routing to their documented Chat and Responses request shapes", async () => { const routing: VercelGatewayRouting = { only: ["anthropic"], order: ["anthropic", "bedrock"], @@ -88,7 +88,9 @@ describe("Vercel AI Gateway automatic cache controls", () => { }; const [chat, responses] = await Promise.all([ captureChatPayload(vercelChatModel(routing)), - captureResponsesPayload(responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing)), + captureResponsesPayload(responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing), { + cacheRetention: "long", + }), ]); expect(chat.providerOptions).toEqual({ @@ -98,10 +100,44 @@ describe("Vercel AI Gateway automatic cache controls", () => { expect(chat.cache_anchor_items).toBeUndefined(); expect(chat.cache_ttl).toBeUndefined(); + expect(responses.providerOptions).toEqual({ + gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] }, + }); expect(responses.caching).toBe("auto"); expect(responses.cache_anchor_items).toBe(1); expect(responses.cache_ttl).toBe("1h"); - expect(responses.providerOptions).toBeUndefined(); + }); + + it("uses the Vercel default TTL for default, explicit, and environment short retention", async () => { + const routing: VercelGatewayRouting = { + only: ["anthropic"], + order: ["anthropic", "bedrock"], + caching: "auto", + cacheAnchorItems: 1, + cacheTtl: "1h", + }; + const defaultRetention = await captureResponsesPayload( + responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing), + ); + const explicitShortRetention = await captureResponsesPayload( + responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing), + { cacheRetention: "short" }, + ); + let environmentShortRetention!: Payload; + await withEnv({ PI_CACHE_RETENTION: "short" }, async () => { + environmentShortRetention = await captureResponsesPayload( + responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing), + ); + }); + + for (const payload of [defaultRetention, explicitShortRetention, environmentShortRetention]) { + expect(payload.providerOptions).toEqual({ + gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] }, + }); + expect(payload.caching).toBe("auto"); + expect(payload.cache_anchor_items).toBe(1); + expect(payload.cache_ttl).toBeUndefined(); + } }); it("omits Chat and Responses automatic cache controls when cache retention is none", async () => { @@ -124,6 +160,9 @@ describe("Vercel AI Gateway automatic cache controls", () => { expect(chat.cache_anchor_items).toBeUndefined(); expect(chat.cache_ttl).toBeUndefined(); + expect(responses.providerOptions).toEqual({ + gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] }, + }); expect(responses.caching).toBeUndefined(); expect(responses.cache_anchor_items).toBeUndefined(); expect(responses.cache_ttl).toBeUndefined(); @@ -148,6 +187,9 @@ describe("Vercel AI Gateway automatic cache controls", () => { expect(chat.cache_anchor_items).toBeUndefined(); expect(chat.cache_ttl).toBeUndefined(); + expect(responses.providerOptions).toEqual({ + gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] }, + }); expect(responses.caching).toBeUndefined(); expect(responses.cache_anchor_items).toBeUndefined(); expect(responses.cache_ttl).toBeUndefined();