smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gateway): preserve Vercel Responses routing and TTL policy

Browse Source
This commit is contained in:
Alexander Kirilin
2026-07-23 15:38:50 -04:00
parent ed0f5d6e29
commit 9bc02be758
3 changed files with 66 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/providers/openai-responses.ts
+1 -1
View File
@@ -1029,7 +1029,7 @@ export function buildParams(
}
if (model.compat.isVercelGatewayHost) {
applyVercelResponsesCacheControls(params, model.compat, cacheRetention !== "none");
applyVercelResponsesCacheControls(params, model.compat, cacheRetention);
} else {
applyOpenAIGatewayRouting(params, model.compat);
}
packages/ai/src/providers/openai-shared.ts
+18 -5
View File
@@ -637,6 +637,7 @@ export interface VercelResponsesCacheParams {
caching?: "auto";
cache_anchor_items?: number;
cache_ttl?: "5m" | "1h";
providerOptions?: { gateway?: Pick<VercelGatewayRouting, "only" | "order"> };
}
export interface VercelResponsesCacheCompat {
@@ -645,20 +646,32 @@ export interface VercelResponsesCacheCompat {
}
/**
* Apply Vercel AI Gateway's Responses-only automatic cache controls. Chat
* Completions uses the distinct `providerOptions.gateway` shape above.
* Apply Vercel AI Gateway's Responses-only automatic cache controls and
* provider routing. Cache settings are top-level Responses fields, while
* `only` and `order` remain under `providerOptions.gateway`.
*/
export function applyVercelResponsesCacheControls(
params: VercelResponsesCacheParams,
compat: VercelResponsesCacheCompat,
cacheEnabled = true,
cacheRetention: CacheRetention = "short",
): void {
const routing = compat.vercelGatewayRouting;
if (!cacheEnabled || !compat.isVercelGatewayHost || routing?.caching !== "auto") return;
if (!compat.isVercelGatewayHost) return;
if (routing?.only || routing?.order) {
const gateway: Pick<VercelGatewayRouting, "only" | "order"> = {};
if (routing.only) gateway.only = routing.only;
if (routing.order) gateway.order = routing.order;
params.providerOptions = { gateway };
}
if (cacheRetention === "none" || routing?.caching !== "auto") return;
params.caching = "auto";
if (routing.cacheAnchorItems !== undefined) params.cache_anchor_items = routing.cacheAnchorItems;
if (routing.cacheTtl !== undefined) params.cache_ttl = routing.cacheTtl;
if (routing.cacheTtl !== undefined && (routing.cacheTtl !== "1h" || cacheRetention === "long")) {
params.cache_ttl = routing.cacheTtl;
}
}
export interface OpenAIExtraBodyOptions {
packages/ai/test/vercel-gateway-cache-controls.test.ts
+47 -5
View File
@@ -51,7 +51,7 @@ function responsesModel(provider: string, baseUrl: string, routing?: VercelGatew
function captureChatPayload(
model: Model<"openai-completions">,
options: { cacheRetention?: "none" } = {},
options: { cacheRetention?: "long" | "short" | "none" } = {},
): Promise<Payload> {
const { promise, resolve } = Promise.withResolvers<Payload>();
streamOpenAICompletions(model, context, {
@@ -65,7 +65,7 @@ function captureChatPayload(
function captureResponsesPayload(
model: Model<"openai-responses">,
options: { cacheRetention?: "none" } = {},
options: { cacheRetention?: "long" | "short" | "none" } = {},
): Promise<Payload> {
const { promise, resolve } = Promise.withResolvers<Payload>();
streamOpenAIResponses(model, context, {
@@ -78,7 +78,7 @@ function captureResponsesPayload(
}
describe("Vercel AI Gateway automatic cache controls", () => {
it("maps cache fields to their documented Chat and Responses request shapes", async () => {
it("maps cache fields and routing to their documented Chat and Responses request shapes", async () => {
const routing: VercelGatewayRouting = {
only: ["anthropic"],
order: ["anthropic", "bedrock"],
@@ -88,7 +88,9 @@ describe("Vercel AI Gateway automatic cache controls", () => {
};
const [chat, responses] = await Promise.all([
captureChatPayload(vercelChatModel(routing)),
captureResponsesPayload(responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing)),
captureResponsesPayload(responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing), {
cacheRetention: "long",
}),
]);
expect(chat.providerOptions).toEqual({
@@ -98,10 +100,44 @@ describe("Vercel AI Gateway automatic cache controls", () => {
expect(chat.cache_anchor_items).toBeUndefined();
expect(chat.cache_ttl).toBeUndefined();
expect(responses.providerOptions).toEqual({
gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] },
});
expect(responses.caching).toBe("auto");
expect(responses.cache_anchor_items).toBe(1);
expect(responses.cache_ttl).toBe("1h");
expect(responses.providerOptions).toBeUndefined();
});
it("uses the Vercel default TTL for default, explicit, and environment short retention", async () => {
const routing: VercelGatewayRouting = {
only: ["anthropic"],
order: ["anthropic", "bedrock"],
caching: "auto",
cacheAnchorItems: 1,
cacheTtl: "1h",
};
const defaultRetention = await captureResponsesPayload(
responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing),
);
const explicitShortRetention = await captureResponsesPayload(
responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing),
{ cacheRetention: "short" },
);
let environmentShortRetention!: Payload;
await withEnv({ PI_CACHE_RETENTION: "short" }, async () => {
environmentShortRetention = await captureResponsesPayload(
responsesModel("vercel-ai-gateway", "https://ai-gateway.vercel.sh/v1", routing),
);
});
for (const payload of [defaultRetention, explicitShortRetention, environmentShortRetention]) {
expect(payload.providerOptions).toEqual({
gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] },
});
expect(payload.caching).toBe("auto");
expect(payload.cache_anchor_items).toBe(1);
expect(payload.cache_ttl).toBeUndefined();
}
});
it("omits Chat and Responses automatic cache controls when cache retention is none", async () => {
@@ -124,6 +160,9 @@ describe("Vercel AI Gateway automatic cache controls", () => {
expect(chat.cache_anchor_items).toBeUndefined();
expect(chat.cache_ttl).toBeUndefined();
expect(responses.providerOptions).toEqual({
gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] },
});
expect(responses.caching).toBeUndefined();
expect(responses.cache_anchor_items).toBeUndefined();
expect(responses.cache_ttl).toBeUndefined();
@@ -148,6 +187,9 @@ describe("Vercel AI Gateway automatic cache controls", () => {
expect(chat.cache_anchor_items).toBeUndefined();
expect(chat.cache_ttl).toBeUndefined();
expect(responses.providerOptions).toEqual({
gateway: { only: ["anthropic"], order: ["anthropic", "bedrock"] },
});
expect(responses.caching).toBeUndefined();
expect(responses.cache_anchor_items).toBeUndefined();
expect(responses.cache_ttl).toBeUndefined();