From 9b4dcaa1140bfc9eea44cf4e1057fed3257b2b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Ara=C3=BAjo?= Date: Fri, 10 Jul 2026 15:24:27 -0300 Subject: [PATCH 1/6] fix(ai): adapted xai responses replay shapes Convert freeform custom_tool_call history to function_call pairs and clamp input_image.detail original to auto when replaying into xAI OAuth Responses, so session continuations stop 422ing. Fixes #5002 --- packages/ai/CHANGELOG.md | 3 + packages/ai/src/providers/openai-shared.ts | 84 +++++- packages/ai/src/utils.ts | 51 +++- .../openai-responses-history-payload.test.ts | 257 +++++++++++++++++- 4 files changed, 375 insertions(+), 20 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 4dbaf5b00..1f71c05aa 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -7,6 +7,9 @@ ### Changed - Enforced `all_turns` reasoning context for all Responses Lite requests +### Fixed + +- Fixed xAI OAuth Responses continuations replaying OpenAI-only `custom_tool_call`/`custom_tool_call_output` history and `input_image.detail: "original"` frames; replay now downgrades those to xAI-compatible function calls and `detail: "auto"`. ([#5002](https://github.com/can1357/oh-my-pi/issues/5002)) ## [16.4.0] - 2026-07-10 diff --git a/packages/ai/src/providers/openai-shared.ts b/packages/ai/src/providers/openai-shared.ts index f70989560..afb15ba97 100644 --- a/packages/ai/src/providers/openai-shared.ts +++ b/packages/ai/src/providers/openai-shared.ts @@ -1356,6 +1356,52 @@ export function convertResponsesInputContent( return normalizedContent.length > 0 ? normalizedContent : undefined; } +interface ResponsesReplayCompatibilityOptions { + supportsCustomToolCalls: boolean; + tools: readonly Tool[] | undefined; +} + +function resolveReplayCustomToolName(wireName: string, tools: readonly Tool[] | undefined): string { + if (tools) { + for (const tool of tools) { + if (tool.customWireName === wireName) return tool.name; + } + } + if (wireName === "apply_patch") return "edit"; + return wireName; +} + +function adaptResponsesReplayItemsForModel( + input: ResponseInput, + options: ResponsesReplayCompatibilityOptions, +): ResponseInput { + let changed = false; + const adapted: ResponseInput = []; + for (const item of input) { + let next = item; + if (!options.supportsCustomToolCalls && item.type === "custom_tool_call") { + changed = true; + next = { + type: "function_call", + ...(item.id ? { id: item.id } : {}), + call_id: item.call_id, + name: resolveReplayCustomToolName(item.name, options.tools), + arguments: JSON.stringify({ input: item.input }), + ...(item.namespace ? { namespace: item.namespace } : {}), + }; + } else if (!options.supportsCustomToolCalls && item.type === "custom_tool_call_output") { + changed = true; + next = { + type: "function_call_output", + call_id: item.call_id, + output: item.output, + }; + } + adapted.push(next); + } + return changed ? adapted : input; +} + export interface BuildResponsesInputOptions { model: Model; context: Context; @@ -1380,6 +1426,13 @@ export function buildResponsesInput(options: BuildResponsesInp messages.push({ role: options.systemRole as "system" | "developer", content: systemPrompt }); } + const supportsImageDetailOriginal = + options.model.provider === "xai-oauth" ? false : options.supportsImageDetailOriginal; + const supportsCustomToolCalls = options.model.applyPatchToolType === "freeform"; + const replayCompatibility: ResponsesReplayCompatibilityOptions = { + supportsCustomToolCalls, + tools: options.context.tools, + }; let knownCallIds = new Set(); const customCallIds = new Set(); const transformedMessages = transformMessages( @@ -1407,7 +1460,10 @@ export function buildResponsesInput(options: BuildResponsesInp }) ?? false); if (historyItems && shouldReplayPayloadItems) { - messages.push(...sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems))); + const sanitizedItems = sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems), { + supportsImageDetailOriginal, + }); + messages.push(...adaptResponsesReplayItemsForModel(sanitizedItems, replayCompatibility)); knownCallIds = collectKnownCallIds(messages); for (const id of collectCustomCallIds(messages)) customCallIds.add(id); msgIndex++; @@ -1416,7 +1472,7 @@ export function buildResponsesInput(options: BuildResponsesInp const content = convertResponsesInputContent( msg.content, options.model.input.includes("image"), - options.supportsImageDetailOriginal, + supportsImageDetailOriginal, ); if (!content) continue; messages.push({ @@ -1444,9 +1500,13 @@ export function buildResponsesInput(options: BuildResponsesInp const historyItems = providerPayload?.items; let suppressHiddenEmptyFallback = false; if (historyItems) { - const sanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay( + const rawSanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay( filterReasoning(historyItems), + { supportsImageDetailOriginal }, ); + const sanitizedHistoryItems = rawSanitizedHistoryItems + ? adaptResponsesReplayItemsForModel(rawSanitizedHistoryItems, replayCompatibility) + : undefined; if (nativeReplayEnabled && sanitizedHistoryItems) { if (providerPayload?.dt) { messages.push(...sanitizedHistoryItems); @@ -1469,6 +1529,8 @@ export function buildResponsesInput(options: BuildResponsesInp suppressHiddenEmptyFallback ? false : includeThinkingSignatures, customCallIds, options.preserveAssistantMessageIds, + supportsCustomToolCalls, + options.context.tools, ); const outputItems = suppressHiddenEmptyFallback ? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems) @@ -1481,9 +1543,10 @@ export function buildResponsesInput(options: BuildResponsesInp msg, options.model, options.strictResponsesPairing, - options.supportsImageDetailOriginal, + supportsImageDetailOriginal, knownCallIds, customCallIds, + supportsCustomToolCalls, ); } msgIndex++; @@ -1516,6 +1579,8 @@ export function convertResponsesAssistantMessage( includeThinkingSignatures = true, customCallIds?: Set, preserveMessageIds = false, + supportsCustomToolCalls = true, + tools?: readonly Tool[], ): ResponseInput { const outputItems: ResponseInput = []; let unsignedTextBlocks = 0; @@ -1587,7 +1652,7 @@ export function convertResponsesAssistantMessage( itemId = undefined; } knownCallIds.add(normalized.callId); - if (block.customWireName) { + if (block.customWireName && supportsCustomToolCalls) { const rawInput = typeof block.arguments?.input === "string" ? block.arguments.input : ""; customCallIds?.add(normalized.callId); outputItems.push({ @@ -1599,11 +1664,15 @@ export function convertResponsesAssistantMessage( } as ResponseInput[number]); continue; } + const functionName = + block.customWireName && !supportsCustomToolCalls + ? resolveReplayCustomToolName(block.customWireName, tools) + : block.name; outputItems.push({ type: "function_call", ...(itemId ? { id: itemId } : {}), call_id: normalized.callId, - name: block.name, + name: functionName, arguments: JSON.stringify(block.arguments), }); } @@ -1619,6 +1688,7 @@ export function appendResponsesToolResultMessages( supportsImageDetailOriginal: boolean, knownCallIds: ReadonlySet, customCallIds?: ReadonlySet, + supportsCustomToolCalls = true, ): void { const supportsImages = model.input.includes("image"); const textResult = toolResult.content @@ -1648,7 +1718,7 @@ export function appendResponsesToolResultMessages( } as ResponseInput[number]); return; } - if (customCallIds?.has(normalized.callId)) { + if (supportsCustomToolCalls && customCallIds?.has(normalized.callId)) { messages.push({ type: "custom_tool_call_output", call_id: normalized.callId, diff --git a/packages/ai/src/utils.ts b/packages/ai/src/utils.ts index 0abc70cb1..988c16aa5 100644 --- a/packages/ai/src/utils.ts +++ b/packages/ai/src/utils.ts @@ -65,10 +65,50 @@ export function truncateResponseItemId(id: string, prefix: string): string { return `${prefix}_${Bun.hash(id).toString(36)}`; } -export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array>): ResponseInput { +interface OpenAIResponsesReplaySanitizeOptions { + supportsImageDetailOriginal?: boolean; +} + +function isReplayRecord(value: unknown): value is Record { + if (!value || typeof value !== "object") return false; + return !Array.isArray(value); +} + +function sanitizeReplayValueForCompatibility(value: unknown, options: OpenAIResponsesReplaySanitizeOptions): unknown { + if (options.supportsImageDetailOriginal !== false) return value; + if (Array.isArray(value)) { + let changed = false; + const sanitized = value.map(item => { + const next = sanitizeReplayValueForCompatibility(item, options); + if (next !== item) changed = true; + return next; + }); + return changed ? sanitized : value; + } + if (!isReplayRecord(value)) return value; + + let changed = false; + const sanitized: Record = {}; + for (const key in value) { + const child = value[key]; + const next = sanitizeReplayValueForCompatibility(child, options); + if (next !== child) changed = true; + sanitized[key] = next; + } + if (value.type === "input_image" && value.detail === "original") { + sanitized.detail = "auto"; + changed = true; + } + return changed ? sanitized : value; +} + +export function sanitizeOpenAIResponsesHistoryItemsForReplay( + items: Array>, + options: OpenAIResponsesReplaySanitizeOptions = {}, +): ResponseInput { const normalizedCallIds = new Map(); return items.flatMap(item => { - const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds); + const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds, options); return sanitized ? [sanitized] : []; }); } @@ -82,8 +122,9 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array>, + options: OpenAIResponsesReplaySanitizeOptions = {}, ): ResponseInput | undefined { - const sanitized = sanitizeOpenAIResponsesHistoryItemsForReplay(items); + const sanitized = sanitizeOpenAIResponsesHistoryItemsForReplay(items, options); let hasReplayableAssistantOutput = false; for (const item of sanitized) { @@ -153,6 +194,7 @@ export function sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(items: Re function sanitizeOpenAIResponsesHistoryItemForReplay( item: Record, normalizedCallIds: Map, + options: OpenAIResponsesReplaySanitizeOptions, ): OpenAIResponsesReplayItem | undefined { if (item.type === "item_reference") return undefined; if (item.type === "image_generation_call") return sanitizeOpenAIResponsesImageGenerationCallForReplay(item); @@ -164,7 +206,8 @@ function sanitizeOpenAIResponsesHistoryItemForReplay( sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds); } - return sanitizedItem as unknown as OpenAIResponsesReplayItem; + const compatibleItem = sanitizeReplayValueForCompatibility(sanitizedItem, options); + return compatibleItem as unknown as OpenAIResponsesReplayItem; } function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record): OpenAIResponsesReplayItem { diff --git a/packages/ai/test/openai-responses-history-payload.test.ts b/packages/ai/test/openai-responses-history-payload.test.ts index 9520fabf0..a1a4dc84e 100644 --- a/packages/ai/test/openai-responses-history-payload.test.ts +++ b/packages/ai/test/openai-responses-history-payload.test.ts @@ -5,11 +5,12 @@ import { } from "@oh-my-pi/pi-ai/providers/openai-codex-responses"; import { type OpenAIResponsesOptions, streamOpenAIResponses } from "@oh-my-pi/pi-ai/providers/openai-responses"; import { buildResponsesInput } from "@oh-my-pi/pi-ai/providers/openai-shared"; -import type { Context, Model, ModelSpec, ProviderSessionState } from "@oh-my-pi/pi-ai/types"; +import type { Context, Model, ModelSpec, ProviderSessionState, Tool } from "@oh-my-pi/pi-ai/types"; import { createOpenAIResponsesHistoryPayload, truncateResponseItemId } from "@oh-my-pi/pi-ai/utils"; import { buildModel } from "@oh-my-pi/pi-catalog/build"; -import { getBundledModel } from "@oh-my-pi/pi-catalog/models"; +import { type GeneratedProvider, getBundledModel } from "@oh-my-pi/pi-catalog/models"; import * as piUtils from "@oh-my-pi/pi-utils"; +import { type } from "arktype"; const TEST_INSTALLATION_ID = "00000000-0000-4000-8000-000000000001"; @@ -35,13 +36,42 @@ function createCodexToken(accountId: string): string { return `${header}.${payload}.signature`; } -function getOpenAIReasoningModel( - provider: Parameters[0], - id: string, -): Model<"openai-responses"> { - return getBundledModel(provider, id) as Model<"openai-responses">; +function getOpenAIReasoningModel(provider: GeneratedProvider, id: string): Model<"openai-responses"> { + const model = getBundledModel<"openai-responses">(provider, id); + return model; } +const ISSUE_5002_PATCH = "*** Begin Patch\n*** End Patch\n"; +const ISSUE_5002_TOOL_OUTPUT = "patch applied"; +const issue5002XaiOAuthModel = buildModel({ + id: "grok-build", + name: "Grok Build", + api: "openai-responses", + provider: "xai-oauth", + baseUrl: "https://api.x.ai/v1", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 256000, + maxTokens: 64000, +} satisfies ModelSpec<"openai-responses">); + +const issue5002ZeroUsage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; +const issue5002EditTool: Tool = { + name: "edit", + customWireName: "apply_patch", + description: "Apply a hashline patch", + parameters: type({ input: "string" }), + customFormat: { syntax: "lark", definition: 'start: "*** Begin Patch" LF\nLF: /\\n/' }, +}; + const preservedHistoryItems = [ { type: "message", role: "user", content: [{ type: "input_text", text: "Preserved user" }] }, { type: "compaction", encrypted_content: "enc_123" }, @@ -298,6 +328,38 @@ function findResponsesInputItem(input: unknown[] | undefined, type: string): Rec }) as Record | undefined; } +function isIssue5002Record(value: unknown): value is Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + return true; +} + +function findResponsesInputItemByCallId( + input: unknown[], + type: string, + callId: string, +): Record | undefined { + for (const item of input) { + if (!isIssue5002Record(item)) continue; + if (item.type === type && item.call_id === callId) return item; + } + return undefined; +} + +function collectResponsesInputImageDetails(input: unknown): string[] { + const details: string[] = []; + const visit = (node: unknown): void => { + if (Array.isArray(node)) { + for (const child of node) visit(child); + return; + } + if (!isIssue5002Record(node)) return; + if (node.type === "input_image" && typeof node.detail === "string") details.push(node.detail); + for (const key in node) visit(node[key]); + }; + visit(input); + return details; +} + function containsUserInputText(input: unknown[] | undefined, text: string): boolean { return (input ?? []).some(item => { if (!item || typeof item !== "object") return false; @@ -366,11 +428,188 @@ describe("OpenAI responses history payload", () => { }); assertWireOrder(openaiItems); - const codexModel = getBundledModel("openai-codex", "gpt-5.2-codex") as Model<"openai-codex-responses">; + const codexModel = getBundledModel<"openai-codex-responses">("openai-codex", "gpt-5.2-codex"); const codexItems = convertCodexResponsesMessages(codexModel, makeContext("openai-codex")); assertWireOrder(codexItems); }); + it("adapts reconstructed apply_patch replay for xai-oauth while preserving OpenAI custom replay", () => { + const context: Context = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "previous frame" }, + { type: "image", mimeType: "image/png", data: "ZmFrZQ==", detail: "original" }, + ], + timestamp: Date.now(), + }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_apply", + name: "apply_patch", + arguments: { input: ISSUE_5002_PATCH }, + customWireName: "apply_patch", + }, + ], + api: "openai-responses", + provider: "openai", + model: "gpt-5-mini", + usage: issue5002ZeroUsage, + stopReason: "toolUse", + timestamp: Date.now(), + }, + { + role: "toolResult", + toolCallId: "call_apply", + toolName: "edit", + content: [{ type: "text", text: ISSUE_5002_TOOL_OUTPUT }], + isError: false, + timestamp: Date.now(), + }, + ], + tools: [issue5002EditTool], + }; + + const xaiInput = buildResponsesInput({ + model: issue5002XaiOAuthModel, + context, + strictResponsesPairing: false, + supportsImageDetailOriginal: issue5002XaiOAuthModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: issue5002XaiOAuthModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call", "call_apply")).toEqual({ + type: "function_call", + call_id: "call_apply", + name: "edit", + arguments: JSON.stringify({ input: ISSUE_5002_PATCH }), + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call_output", "call_apply")).toEqual({ + type: "function_call_output", + call_id: "call_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(JSON.stringify(xaiInput)).not.toContain("custom_tool_call"); + expect(collectResponsesInputImageDetails(xaiInput)).toEqual(["auto"]); + + const openaiModel = getOpenAIReasoningModel("openai", "gpt-5-mini"); + const openaiInput = buildResponsesInput({ + model: openaiModel, + context, + strictResponsesPairing: false, + supportsImageDetailOriginal: openaiModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: openaiModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call", "call_apply")).toEqual({ + type: "custom_tool_call", + call_id: "call_apply", + name: "apply_patch", + input: ISSUE_5002_PATCH, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call_output", "call_apply")).toEqual({ + type: "custom_tool_call_output", + call_id: "call_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(collectResponsesInputImageDetails(openaiInput)).toEqual(["original"]); + }); + + it("adapts persisted native apply_patch Responses items for xai-oauth continuations", () => { + const nativeHistoryItems = [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "previous native frame" }, + { type: "input_image", detail: "original", image_url: "data:image/png;base64,ZmFrZQ==" }, + ], + }, + { type: "custom_tool_call", call_id: "call_native_apply", name: "apply_patch", input: ISSUE_5002_PATCH }, + { + type: "custom_tool_call_output", + call_id: "call_native_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }, + ]; + const xaiContext: Context = { + messages: [ + { + role: "assistant", + content: [{ type: "text", text: "fallback should not be replayed" }], + api: "openai-responses", + provider: "xai-oauth", + model: issue5002XaiOAuthModel.id, + usage: issue5002ZeroUsage, + stopReason: "stop", + providerPayload: createOpenAIResponsesHistoryPayload("xai-oauth", nativeHistoryItems), + timestamp: Date.now(), + }, + { role: "user", content: "continue", timestamp: Date.now() }, + ], + }; + + const xaiInput = buildResponsesInput({ + model: issue5002XaiOAuthModel, + context: xaiContext, + strictResponsesPairing: false, + supportsImageDetailOriginal: issue5002XaiOAuthModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: issue5002XaiOAuthModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call", "call_native_apply")).toEqual({ + type: "function_call", + call_id: "call_native_apply", + name: "edit", + arguments: JSON.stringify({ input: ISSUE_5002_PATCH }), + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call_output", "call_native_apply")).toEqual({ + type: "function_call_output", + call_id: "call_native_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(JSON.stringify(xaiInput)).not.toContain("custom_tool_call"); + expect(collectResponsesInputImageDetails(xaiInput)).toEqual(["auto"]); + + const openaiModel = getOpenAIReasoningModel("openai", "gpt-5-mini"); + const openaiContext: Context = { + messages: [ + { + role: "assistant", + content: [{ type: "text", text: "fallback should not be replayed" }], + api: "openai-responses", + provider: "openai", + model: openaiModel.id, + usage: issue5002ZeroUsage, + stopReason: "stop", + providerPayload: createOpenAIResponsesHistoryPayload("openai", nativeHistoryItems), + timestamp: Date.now(), + }, + { role: "user", content: "continue", timestamp: Date.now() }, + ], + }; + const openaiInput = buildResponsesInput({ + model: openaiModel, + context: openaiContext, + strictResponsesPairing: false, + supportsImageDetailOriginal: openaiModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: openaiModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call", "call_native_apply")).toEqual({ + type: "custom_tool_call", + call_id: "call_native_apply", + name: "apply_patch", + input: ISSUE_5002_PATCH, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call_output", "call_native_apply")).toEqual({ + type: "custom_tool_call_output", + call_id: "call_native_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(collectResponsesInputImageDetails(openaiInput)).toEqual(["original"]); + }); + it("prepends multiple OpenAI developer instructions in order without changing prompt cache key routing", async () => { const model = getOpenAIReasoningModel("openai", "gpt-5-mini"); const payload = (await captureResponsesPayload( @@ -1063,7 +1302,7 @@ describe("OpenAI responses history payload", () => { { role: "user", content: "Resume", timestamp: Date.now() }, ], }; - const model = getBundledModel("openai-codex", "gpt-5.2-codex") as Model<"openai-codex-responses">; + const model = getBundledModel<"openai-codex-responses">("openai-codex", "gpt-5.2-codex"); const payload = (await captureCodexPayload(model, context)) as { input?: unknown[] }; const functionCallItem = findResponsesInputItem(payload.input, "function_call"); const functionCallOutputItem = findResponsesInputItem(payload.input, "function_call_output"); From 619519343595a10bca67fac076f09c41c37caa80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Ara=C3=BAjo?= Date: Fri, 10 Jul 2026 15:24:27 -0300 Subject: [PATCH 2/6] fix(catalog): marked xai-oauth models without image detail original Seed supportsImageDetailOriginal=false for curated and dynamic xai-oauth compat, and set the same flag on the eight bundled models.json entries without regenerating the rest of the catalog. Fixes #5002 --- packages/catalog/src/models.json | 24 ++++++++++++------- .../src/provider-models/openai-compat.ts | 2 ++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/catalog/src/models.json b/packages/catalog/src/models.json index 80bb13350..43840585a 100644 --- a/packages/catalog/src/models.json +++ b/packages/catalog/src/models.json @@ -87250,7 +87250,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": true, - "supportsReasoningEffort": false + "supportsReasoningEffort": false, + "supportsImageDetailOriginal": false } }, "grok-4.20-0309-reasoning": { @@ -87279,7 +87280,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": true, - "supportsReasoningEffort": false + "supportsReasoningEffort": false, + "supportsImageDetailOriginal": false } }, "grok-4.20-multi-agent-0309": { @@ -87320,7 +87322,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": false, - "supportsReasoningEffort": true + "supportsReasoningEffort": true, + "supportsImageDetailOriginal": false } }, "grok-4.3": { @@ -87362,7 +87365,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": false, - "supportsReasoningEffort": true + "supportsReasoningEffort": true, + "supportsImageDetailOriginal": false } }, "grok-4.5": { @@ -87404,7 +87408,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": false, - "supportsReasoningEffort": true + "supportsReasoningEffort": true, + "supportsImageDetailOriginal": false } }, "grok-build": { @@ -87433,7 +87438,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": true, - "supportsReasoningEffort": false + "supportsReasoningEffort": false, + "supportsImageDetailOriginal": false } }, "grok-build-0.1": { @@ -87462,7 +87468,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": true, - "supportsReasoningEffort": false + "supportsReasoningEffort": false, + "supportsImageDetailOriginal": false } }, "grok-composer-2.5-fast": { @@ -87490,7 +87497,8 @@ "includeEncryptedReasoning": false, "filterReasoningHistory": true, "omitReasoningEffort": true, - "supportsReasoningEffort": false + "supportsReasoningEffort": false, + "supportsImageDetailOriginal": false } } }, diff --git a/packages/catalog/src/provider-models/openai-compat.ts b/packages/catalog/src/provider-models/openai-compat.ts index ef7f7eafa..40ab74cd8 100644 --- a/packages/catalog/src/provider-models/openai-compat.ts +++ b/packages/catalog/src/provider-models/openai-compat.ts @@ -1176,6 +1176,7 @@ function withXaiOAuthCompatDefaults(model: ModelSpec<"openai-responses">): Model ...(model.compat ?? {}), includeEncryptedReasoning: model.compat?.includeEncryptedReasoning ?? false, filterReasoningHistory: model.compat?.filterReasoningHistory ?? true, + supportsImageDetailOriginal: model.compat?.supportsImageDetailOriginal ?? false, omitReasoningEffort: model.compat?.omitReasoningEffort ?? !isGrokReasoningEffortCapable(model.id), }; return { ...model, compat }; @@ -1218,6 +1219,7 @@ function mergeCuratedIntoModel( reasoningEffortMap: { ...XAI_REASONING_EFFORT_MAP, ...(base.compat?.reasoningEffortMap ?? {}) }, includeEncryptedReasoning: base.compat?.includeEncryptedReasoning ?? false, filterReasoningHistory: base.compat?.filterReasoningHistory ?? true, + supportsImageDetailOriginal: base.compat?.supportsImageDetailOriginal ?? false, omitReasoningEffort: !effortCapable, supportsReasoningEffort: effortCapable, }; From 0e3e5ab72a5f5ebac1d03339ca2707bc005bd617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Ara=C3=BAjo?= Date: Fri, 10 Jul 2026 15:27:24 -0300 Subject: [PATCH 3/6] fix(ai): tightened xai replay adaptation to catalog compat Drive image-detail clamping from resolved model compat instead of a provider hardcode, clamp input_image only on known paths, and skip custom-tool adaptation when freeform is supported. --- packages/ai/src/providers/openai-shared.ts | 87 ++++++++++++++-------- packages/ai/src/utils.ts | 59 ++++++++------- packages/catalog/src/compat/openai.ts | 14 ++-- 3 files changed, 91 insertions(+), 69 deletions(-) diff --git a/packages/ai/src/providers/openai-shared.ts b/packages/ai/src/providers/openai-shared.ts index afb15ba97..ca6e67771 100644 --- a/packages/ai/src/providers/openai-shared.ts +++ b/packages/ai/src/providers/openai-shared.ts @@ -1356,48 +1356,61 @@ export function convertResponsesInputContent( return normalizedContent.length > 0 ? normalizedContent : undefined; } -interface ResponsesReplayCompatibilityOptions { - supportsCustomToolCalls: boolean; - tools: readonly Tool[] | undefined; -} - -function resolveReplayCustomToolName(wireName: string, tools: readonly Tool[] | undefined): string { - if (tools) { - for (const tool of tools) { - if (tool.customWireName === wireName) return tool.name; - } +/** + * Map freeform custom-tool wire names back to the internal tool name for + * providers that only accept function_call / function_call_output. + * Built once per request; `apply_patch` → `edit` is the OMP default. + */ +function buildCustomToolWireNameMap(tools: readonly Tool[] | undefined): ReadonlyMap | undefined { + if (!tools?.length) return undefined; + const map = new Map(); + for (const tool of tools) { + if (tool.customWireName) map.set(tool.customWireName, tool.name); } - if (wireName === "apply_patch") return "edit"; - return wireName; + return map.size > 0 ? map : undefined; } +function resolveReplayCustomToolName(wireName: string, wireNameMap: ReadonlyMap | undefined): string { + return wireNameMap?.get(wireName) ?? (wireName === "apply_patch" ? "edit" : wireName); +} + +/** + * Downgrade OpenAI-only custom tool items when the target model does not + * advertise freeform custom tools (`applyPatchToolType === "freeform"`). + * No-op (returns the same array reference) when freeform is supported. + */ function adaptResponsesReplayItemsForModel( input: ResponseInput, - options: ResponsesReplayCompatibilityOptions, + supportsCustomToolCalls: boolean, + wireNameMap: ReadonlyMap | undefined, ): ResponseInput { + if (supportsCustomToolCalls) return input; + let changed = false; const adapted: ResponseInput = []; for (const item of input) { - let next = item; - if (!options.supportsCustomToolCalls && item.type === "custom_tool_call") { + if (item.type === "custom_tool_call") { changed = true; - next = { + adapted.push({ type: "function_call", ...(item.id ? { id: item.id } : {}), call_id: item.call_id, - name: resolveReplayCustomToolName(item.name, options.tools), + name: resolveReplayCustomToolName(item.name, wireNameMap), arguments: JSON.stringify({ input: item.input }), ...(item.namespace ? { namespace: item.namespace } : {}), - }; - } else if (!options.supportsCustomToolCalls && item.type === "custom_tool_call_output") { + }); + continue; + } + if (item.type === "custom_tool_call_output") { changed = true; - next = { + adapted.push({ type: "function_call_output", call_id: item.call_id, output: item.output, - }; + }); + continue; } - adapted.push(next); + adapted.push(item); } return changed ? adapted : input; } @@ -1426,13 +1439,15 @@ export function buildResponsesInput(options: BuildResponsesInp messages.push({ role: options.systemRole as "system" | "developer", content: systemPrompt }); } - const supportsImageDetailOriginal = - options.model.provider === "xai-oauth" ? false : options.supportsImageDetailOriginal; + // Compat is resolved by the catalog (e.g. Copilot / xai-oauth reject + // `detail: "original"`). Do not re-branch on provider id here. + const supportsImageDetailOriginal = options.supportsImageDetailOriginal; + // Freeform custom tools (`custom_tool_call`) only when the catalog says so; + // same gate as tool conversion (`applyPatchToolType === "freeform"`). const supportsCustomToolCalls = options.model.applyPatchToolType === "freeform"; - const replayCompatibility: ResponsesReplayCompatibilityOptions = { - supportsCustomToolCalls, - tools: options.context.tools, - }; + const customToolWireNameMap = supportsCustomToolCalls + ? undefined + : buildCustomToolWireNameMap(options.context.tools); let knownCallIds = new Set(); const customCallIds = new Set(); const transformedMessages = transformMessages( @@ -1463,7 +1478,9 @@ export function buildResponsesInput(options: BuildResponsesInp const sanitizedItems = sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems), { supportsImageDetailOriginal, }); - messages.push(...adaptResponsesReplayItemsForModel(sanitizedItems, replayCompatibility)); + messages.push( + ...adaptResponsesReplayItemsForModel(sanitizedItems, supportsCustomToolCalls, customToolWireNameMap), + ); knownCallIds = collectKnownCallIds(messages); for (const id of collectCustomCallIds(messages)) customCallIds.add(id); msgIndex++; @@ -1505,7 +1522,11 @@ export function buildResponsesInput(options: BuildResponsesInp { supportsImageDetailOriginal }, ); const sanitizedHistoryItems = rawSanitizedHistoryItems - ? adaptResponsesReplayItemsForModel(rawSanitizedHistoryItems, replayCompatibility) + ? adaptResponsesReplayItemsForModel( + rawSanitizedHistoryItems, + supportsCustomToolCalls, + customToolWireNameMap, + ) : undefined; if (nativeReplayEnabled && sanitizedHistoryItems) { if (providerPayload?.dt) { @@ -1530,7 +1551,7 @@ export function buildResponsesInput(options: BuildResponsesInp customCallIds, options.preserveAssistantMessageIds, supportsCustomToolCalls, - options.context.tools, + customToolWireNameMap, ); const outputItems = suppressHiddenEmptyFallback ? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems) @@ -1580,7 +1601,7 @@ export function convertResponsesAssistantMessage( customCallIds?: Set, preserveMessageIds = false, supportsCustomToolCalls = true, - tools?: readonly Tool[], + customToolWireNameMap?: ReadonlyMap, ): ResponseInput { const outputItems: ResponseInput = []; let unsignedTextBlocks = 0; @@ -1666,7 +1687,7 @@ export function convertResponsesAssistantMessage( } const functionName = block.customWireName && !supportsCustomToolCalls - ? resolveReplayCustomToolName(block.customWireName, tools) + ? resolveReplayCustomToolName(block.customWireName, customToolWireNameMap) : block.name; outputItems.push({ type: "function_call", diff --git a/packages/ai/src/utils.ts b/packages/ai/src/utils.ts index 988c16aa5..6e9cc5885 100644 --- a/packages/ai/src/utils.ts +++ b/packages/ai/src/utils.ts @@ -69,37 +69,32 @@ interface OpenAIResponsesReplaySanitizeOptions { supportsImageDetailOriginal?: boolean; } -function isReplayRecord(value: unknown): value is Record { - if (!value || typeof value !== "object") return false; - return !Array.isArray(value); -} +/** + * Clamp `detail: "original"` only where Responses input_image parts live — + * top-level items and `message.content[]`. Avoids a deep tree walk/clone of + * every history node on providers that reject native-resolution images. + */ +function clampReplayItemImageDetail( + item: Record, + supportsImageDetailOriginal: boolean, +): Record { + if (supportsImageDetailOriginal) return item; -function sanitizeReplayValueForCompatibility(value: unknown, options: OpenAIResponsesReplaySanitizeOptions): unknown { - if (options.supportsImageDetailOriginal !== false) return value; - if (Array.isArray(value)) { - let changed = false; - const sanitized = value.map(item => { - const next = sanitizeReplayValueForCompatibility(item, options); - if (next !== item) changed = true; - return next; - }); - return changed ? sanitized : value; + if (item.type === "input_image" && item.detail === "original") { + return { ...item, detail: "auto" }; } - if (!isReplayRecord(value)) return value; + + if (item.type !== "message" || !Array.isArray(item.content)) return item; let changed = false; - const sanitized: Record = {}; - for (const key in value) { - const child = value[key]; - const next = sanitizeReplayValueForCompatibility(child, options); - if (next !== child) changed = true; - sanitized[key] = next; - } - if (value.type === "input_image" && value.detail === "original") { - sanitized.detail = "auto"; + const content = item.content.map(part => { + if (!part || typeof part !== "object" || Array.isArray(part)) return part; + const record = part as Record; + if (record.type !== "input_image" || record.detail !== "original") return part; changed = true; - } - return changed ? sanitized : value; + return { ...record, detail: "auto" }; + }); + return changed ? { ...item, content } : item; } export function sanitizeOpenAIResponsesHistoryItemsForReplay( @@ -107,8 +102,13 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay( options: OpenAIResponsesReplaySanitizeOptions = {}, ): ResponseInput { const normalizedCallIds = new Map(); + const supportsImageDetailOriginal = options.supportsImageDetailOriginal !== false; return items.flatMap(item => { - const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds, options); + const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay( + item, + normalizedCallIds, + supportsImageDetailOriginal, + ); return sanitized ? [sanitized] : []; }); } @@ -194,7 +194,7 @@ export function sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(items: Re function sanitizeOpenAIResponsesHistoryItemForReplay( item: Record, normalizedCallIds: Map, - options: OpenAIResponsesReplaySanitizeOptions, + supportsImageDetailOriginal: boolean, ): OpenAIResponsesReplayItem | undefined { if (item.type === "item_reference") return undefined; if (item.type === "image_generation_call") return sanitizeOpenAIResponsesImageGenerationCallForReplay(item); @@ -206,8 +206,7 @@ function sanitizeOpenAIResponsesHistoryItemForReplay( sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds); } - const compatibleItem = sanitizeReplayValueForCompatibility(sanitizedItem, options); - return compatibleItem as unknown as OpenAIResponsesReplayItem; + return clampReplayItemImageDetail(sanitizedItem, supportsImageDetailOriginal) as unknown as OpenAIResponsesReplayItem; } function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record): OpenAIResponsesReplayItem { diff --git a/packages/catalog/src/compat/openai.ts b/packages/catalog/src/compat/openai.ts index a01430b33..1ced5e25c 100644 --- a/packages/catalog/src/compat/openai.ts +++ b/packages/catalog/src/compat/openai.ts @@ -595,12 +595,14 @@ export function buildOpenAIResponsesCompat(spec: OpenAIResponsesSpecLike): Resol // Azure OpenAI and GitHub Copilot Responses paths require tool results // to strictly match prior tool calls when building Responses inputs. strictResponsesPairing: isAzure || spec.provider === "github-copilot", - // GitHub Copilot's Responses endpoint rejects the `detail: "original"` - // image hint with a 400; every other host preserves native-resolution - // frames (snapcompact relies on `original`). Detect Copilot by provider id - // or base-URL host (mirroring the Anthropic compat builder) so a model - // pointed at the Copilot host under a different provider id still clamps. - supportsImageDetailOriginal: !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"), + // GitHub Copilot and xAI OAuth reject `detail: "original"` (400 / 422). + // Every other host preserves native-resolution frames (snapcompact relies + // on `original`). Detect Copilot by provider id or base-URL host so a + // model pointed at the Copilot host under a different provider id still + // clamps; xai-oauth is provider-id only (same host family as paid `xai`). + supportsImageDetailOriginal: + spec.provider !== "xai-oauth" && + !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"), reasoningEffortMap: {}, supportsReasoningParams: true, thinkingFormat, From bd216f4394c5edbf3e6336429c228032b32063fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Ara=C3=BAjo?= Date: Fri, 10 Jul 2026 15:29:36 -0300 Subject: [PATCH 4/6] style(ai): applied biome format to xai replay changes --- packages/ai/src/utils.ts | 5 ++++- packages/catalog/src/compat/openai.ts | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/ai/src/utils.ts b/packages/ai/src/utils.ts index 6e9cc5885..0445cd3ac 100644 --- a/packages/ai/src/utils.ts +++ b/packages/ai/src/utils.ts @@ -206,7 +206,10 @@ function sanitizeOpenAIResponsesHistoryItemForReplay( sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds); } - return clampReplayItemImageDetail(sanitizedItem, supportsImageDetailOriginal) as unknown as OpenAIResponsesReplayItem; + return clampReplayItemImageDetail( + sanitizedItem, + supportsImageDetailOriginal, + ) as unknown as OpenAIResponsesReplayItem; } function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record): OpenAIResponsesReplayItem { diff --git a/packages/catalog/src/compat/openai.ts b/packages/catalog/src/compat/openai.ts index 1ced5e25c..2425a4744 100644 --- a/packages/catalog/src/compat/openai.ts +++ b/packages/catalog/src/compat/openai.ts @@ -601,8 +601,7 @@ export function buildOpenAIResponsesCompat(spec: OpenAIResponsesSpecLike): Resol // model pointed at the Copilot host under a different provider id still // clamps; xai-oauth is provider-id only (same host family as paid `xai`). supportsImageDetailOriginal: - spec.provider !== "xai-oauth" && - !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"), + spec.provider !== "xai-oauth" && !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"), reasoningEffortMap: {}, supportsReasoningParams: true, thinkingFormat, From e22ee0b6688b26f8453d54aef98d10f238fa4cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Ara=C3=BAjo?= Date: Fri, 10 Jul 2026 16:26:19 -0300 Subject: [PATCH 5/6] docs(ai): moved xAI replay fix changelog to Unreleased MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase onto 16.4.1 landed the entry under the released section; keep release intent under Unreleased. Dropped the broad models.json regen that conflicted — surgical xai-oauth-only catalog edits remain. --- packages/ai/CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 1f71c05aa..000fda5c4 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,14 +2,15 @@ ## [Unreleased] +### Fixed + +- Fixed xAI OAuth Responses continuations replaying OpenAI-only `custom_tool_call`/`custom_tool_call_output` history and `input_image.detail: "original"` frames; replay now downgrades those to xAI-compatible function calls and `detail: "auto"`. ([#5002](https://github.com/can1357/oh-my-pi/issues/5002)) + ## [16.4.1] - 2026-07-10 ### Changed - Enforced `all_turns` reasoning context for all Responses Lite requests -### Fixed - -- Fixed xAI OAuth Responses continuations replaying OpenAI-only `custom_tool_call`/`custom_tool_call_output` history and `input_image.detail: "original"` frames; replay now downgrades those to xAI-compatible function calls and `detail: "auto"`. ([#5002](https://github.com/can1357/oh-my-pi/issues/5002)) ## [16.4.0] - 2026-07-10 From 2b46d6711b33fdef3d5e379f04476d1803998631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Ara=C3=BAjo?= Date: Fri, 10 Jul 2026 16:28:40 -0300 Subject: [PATCH 6/6] chore: re-trigger CI after unrelated workspace-fast flake Prior run failed streamPiNative under parallel load; xAI replay and scoped models.json changes are orthogonal. Empty commit to re-run checks.