diff --git a/packages/ai/src/providers/openai-shared.ts b/packages/ai/src/providers/openai-shared.ts index afb15ba97..ca6e67771 100644 --- a/packages/ai/src/providers/openai-shared.ts +++ b/packages/ai/src/providers/openai-shared.ts @@ -1356,48 +1356,61 @@ export function convertResponsesInputContent( return normalizedContent.length > 0 ? normalizedContent : undefined; } -interface ResponsesReplayCompatibilityOptions { - supportsCustomToolCalls: boolean; - tools: readonly Tool[] | undefined; -} - -function resolveReplayCustomToolName(wireName: string, tools: readonly Tool[] | undefined): string { - if (tools) { - for (const tool of tools) { - if (tool.customWireName === wireName) return tool.name; - } +/** + * Map freeform custom-tool wire names back to the internal tool name for + * providers that only accept function_call / function_call_output. + * Built once per request; `apply_patch` → `edit` is the OMP default. + */ +function buildCustomToolWireNameMap(tools: readonly Tool[] | undefined): ReadonlyMap | undefined { + if (!tools?.length) return undefined; + const map = new Map(); + for (const tool of tools) { + if (tool.customWireName) map.set(tool.customWireName, tool.name); } - if (wireName === "apply_patch") return "edit"; - return wireName; + return map.size > 0 ? map : undefined; } +function resolveReplayCustomToolName(wireName: string, wireNameMap: ReadonlyMap | undefined): string { + return wireNameMap?.get(wireName) ?? (wireName === "apply_patch" ? "edit" : wireName); +} + +/** + * Downgrade OpenAI-only custom tool items when the target model does not + * advertise freeform custom tools (`applyPatchToolType === "freeform"`). + * No-op (returns the same array reference) when freeform is supported. + */ function adaptResponsesReplayItemsForModel( input: ResponseInput, - options: ResponsesReplayCompatibilityOptions, + supportsCustomToolCalls: boolean, + wireNameMap: ReadonlyMap | undefined, ): ResponseInput { + if (supportsCustomToolCalls) return input; + let changed = false; const adapted: ResponseInput = []; for (const item of input) { - let next = item; - if (!options.supportsCustomToolCalls && item.type === "custom_tool_call") { + if (item.type === "custom_tool_call") { changed = true; - next = { + adapted.push({ type: "function_call", ...(item.id ? { id: item.id } : {}), call_id: item.call_id, - name: resolveReplayCustomToolName(item.name, options.tools), + name: resolveReplayCustomToolName(item.name, wireNameMap), arguments: JSON.stringify({ input: item.input }), ...(item.namespace ? { namespace: item.namespace } : {}), - }; - } else if (!options.supportsCustomToolCalls && item.type === "custom_tool_call_output") { + }); + continue; + } + if (item.type === "custom_tool_call_output") { changed = true; - next = { + adapted.push({ type: "function_call_output", call_id: item.call_id, output: item.output, - }; + }); + continue; } - adapted.push(next); + adapted.push(item); } return changed ? adapted : input; } @@ -1426,13 +1439,15 @@ export function buildResponsesInput(options: BuildResponsesInp messages.push({ role: options.systemRole as "system" | "developer", content: systemPrompt }); } - const supportsImageDetailOriginal = - options.model.provider === "xai-oauth" ? false : options.supportsImageDetailOriginal; + // Compat is resolved by the catalog (e.g. Copilot / xai-oauth reject + // `detail: "original"`). Do not re-branch on provider id here. + const supportsImageDetailOriginal = options.supportsImageDetailOriginal; + // Freeform custom tools (`custom_tool_call`) only when the catalog says so; + // same gate as tool conversion (`applyPatchToolType === "freeform"`). const supportsCustomToolCalls = options.model.applyPatchToolType === "freeform"; - const replayCompatibility: ResponsesReplayCompatibilityOptions = { - supportsCustomToolCalls, - tools: options.context.tools, - }; + const customToolWireNameMap = supportsCustomToolCalls + ? undefined + : buildCustomToolWireNameMap(options.context.tools); let knownCallIds = new Set(); const customCallIds = new Set(); const transformedMessages = transformMessages( @@ -1463,7 +1478,9 @@ export function buildResponsesInput(options: BuildResponsesInp const sanitizedItems = sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems), { supportsImageDetailOriginal, }); - messages.push(...adaptResponsesReplayItemsForModel(sanitizedItems, replayCompatibility)); + messages.push( + ...adaptResponsesReplayItemsForModel(sanitizedItems, supportsCustomToolCalls, customToolWireNameMap), + ); knownCallIds = collectKnownCallIds(messages); for (const id of collectCustomCallIds(messages)) customCallIds.add(id); msgIndex++; @@ -1505,7 +1522,11 @@ export function buildResponsesInput(options: BuildResponsesInp { supportsImageDetailOriginal }, ); const sanitizedHistoryItems = rawSanitizedHistoryItems - ? adaptResponsesReplayItemsForModel(rawSanitizedHistoryItems, replayCompatibility) + ? adaptResponsesReplayItemsForModel( + rawSanitizedHistoryItems, + supportsCustomToolCalls, + customToolWireNameMap, + ) : undefined; if (nativeReplayEnabled && sanitizedHistoryItems) { if (providerPayload?.dt) { @@ -1530,7 +1551,7 @@ export function buildResponsesInput(options: BuildResponsesInp customCallIds, options.preserveAssistantMessageIds, supportsCustomToolCalls, - options.context.tools, + customToolWireNameMap, ); const outputItems = suppressHiddenEmptyFallback ? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems) @@ -1580,7 +1601,7 @@ export function convertResponsesAssistantMessage( customCallIds?: Set, preserveMessageIds = false, supportsCustomToolCalls = true, - tools?: readonly Tool[], + customToolWireNameMap?: ReadonlyMap, ): ResponseInput { const outputItems: ResponseInput = []; let unsignedTextBlocks = 0; @@ -1666,7 +1687,7 @@ export function convertResponsesAssistantMessage( } const functionName = block.customWireName && !supportsCustomToolCalls - ? resolveReplayCustomToolName(block.customWireName, tools) + ? resolveReplayCustomToolName(block.customWireName, customToolWireNameMap) : block.name; outputItems.push({ type: "function_call", diff --git a/packages/ai/src/utils.ts b/packages/ai/src/utils.ts index 988c16aa5..6e9cc5885 100644 --- a/packages/ai/src/utils.ts +++ b/packages/ai/src/utils.ts @@ -69,37 +69,32 @@ interface OpenAIResponsesReplaySanitizeOptions { supportsImageDetailOriginal?: boolean; } -function isReplayRecord(value: unknown): value is Record { - if (!value || typeof value !== "object") return false; - return !Array.isArray(value); -} +/** + * Clamp `detail: "original"` only where Responses input_image parts live — + * top-level items and `message.content[]`. Avoids a deep tree walk/clone of + * every history node on providers that reject native-resolution images. + */ +function clampReplayItemImageDetail( + item: Record, + supportsImageDetailOriginal: boolean, +): Record { + if (supportsImageDetailOriginal) return item; -function sanitizeReplayValueForCompatibility(value: unknown, options: OpenAIResponsesReplaySanitizeOptions): unknown { - if (options.supportsImageDetailOriginal !== false) return value; - if (Array.isArray(value)) { - let changed = false; - const sanitized = value.map(item => { - const next = sanitizeReplayValueForCompatibility(item, options); - if (next !== item) changed = true; - return next; - }); - return changed ? sanitized : value; + if (item.type === "input_image" && item.detail === "original") { + return { ...item, detail: "auto" }; } - if (!isReplayRecord(value)) return value; + + if (item.type !== "message" || !Array.isArray(item.content)) return item; let changed = false; - const sanitized: Record = {}; - for (const key in value) { - const child = value[key]; - const next = sanitizeReplayValueForCompatibility(child, options); - if (next !== child) changed = true; - sanitized[key] = next; - } - if (value.type === "input_image" && value.detail === "original") { - sanitized.detail = "auto"; + const content = item.content.map(part => { + if (!part || typeof part !== "object" || Array.isArray(part)) return part; + const record = part as Record; + if (record.type !== "input_image" || record.detail !== "original") return part; changed = true; - } - return changed ? sanitized : value; + return { ...record, detail: "auto" }; + }); + return changed ? { ...item, content } : item; } export function sanitizeOpenAIResponsesHistoryItemsForReplay( @@ -107,8 +102,13 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay( options: OpenAIResponsesReplaySanitizeOptions = {}, ): ResponseInput { const normalizedCallIds = new Map(); + const supportsImageDetailOriginal = options.supportsImageDetailOriginal !== false; return items.flatMap(item => { - const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds, options); + const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay( + item, + normalizedCallIds, + supportsImageDetailOriginal, + ); return sanitized ? [sanitized] : []; }); } @@ -194,7 +194,7 @@ export function sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(items: Re function sanitizeOpenAIResponsesHistoryItemForReplay( item: Record, normalizedCallIds: Map, - options: OpenAIResponsesReplaySanitizeOptions, + supportsImageDetailOriginal: boolean, ): OpenAIResponsesReplayItem | undefined { if (item.type === "item_reference") return undefined; if (item.type === "image_generation_call") return sanitizeOpenAIResponsesImageGenerationCallForReplay(item); @@ -206,8 +206,7 @@ function sanitizeOpenAIResponsesHistoryItemForReplay( sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds); } - const compatibleItem = sanitizeReplayValueForCompatibility(sanitizedItem, options); - return compatibleItem as unknown as OpenAIResponsesReplayItem; + return clampReplayItemImageDetail(sanitizedItem, supportsImageDetailOriginal) as unknown as OpenAIResponsesReplayItem; } function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record): OpenAIResponsesReplayItem { diff --git a/packages/catalog/src/compat/openai.ts b/packages/catalog/src/compat/openai.ts index a01430b33..1ced5e25c 100644 --- a/packages/catalog/src/compat/openai.ts +++ b/packages/catalog/src/compat/openai.ts @@ -595,12 +595,14 @@ export function buildOpenAIResponsesCompat(spec: OpenAIResponsesSpecLike): Resol // Azure OpenAI and GitHub Copilot Responses paths require tool results // to strictly match prior tool calls when building Responses inputs. strictResponsesPairing: isAzure || spec.provider === "github-copilot", - // GitHub Copilot's Responses endpoint rejects the `detail: "original"` - // image hint with a 400; every other host preserves native-resolution - // frames (snapcompact relies on `original`). Detect Copilot by provider id - // or base-URL host (mirroring the Anthropic compat builder) so a model - // pointed at the Copilot host under a different provider id still clamps. - supportsImageDetailOriginal: !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"), + // GitHub Copilot and xAI OAuth reject `detail: "original"` (400 / 422). + // Every other host preserves native-resolution frames (snapcompact relies + // on `original`). Detect Copilot by provider id or base-URL host so a + // model pointed at the Copilot host under a different provider id still + // clamps; xai-oauth is provider-id only (same host family as paid `xai`). + supportsImageDetailOriginal: + spec.provider !== "xai-oauth" && + !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"), reasoningEffortMap: {}, supportsReasoningParams: true, thinkingFormat,