diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 4dbaf5b00..1f71c05aa 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -7,6 +7,9 @@ ### Changed - Enforced `all_turns` reasoning context for all Responses Lite requests +### Fixed + +- Fixed xAI OAuth Responses continuations replaying OpenAI-only `custom_tool_call`/`custom_tool_call_output` history and `input_image.detail: "original"` frames; replay now downgrades those to xAI-compatible function calls and `detail: "auto"`. ([#5002](https://github.com/can1357/oh-my-pi/issues/5002)) ## [16.4.0] - 2026-07-10 diff --git a/packages/ai/src/providers/openai-shared.ts b/packages/ai/src/providers/openai-shared.ts index f70989560..afb15ba97 100644 --- a/packages/ai/src/providers/openai-shared.ts +++ b/packages/ai/src/providers/openai-shared.ts @@ -1356,6 +1356,52 @@ export function convertResponsesInputContent( return normalizedContent.length > 0 ? normalizedContent : undefined; } +interface ResponsesReplayCompatibilityOptions { + supportsCustomToolCalls: boolean; + tools: readonly Tool[] | undefined; +} + +function resolveReplayCustomToolName(wireName: string, tools: readonly Tool[] | undefined): string { + if (tools) { + for (const tool of tools) { + if (tool.customWireName === wireName) return tool.name; + } + } + if (wireName === "apply_patch") return "edit"; + return wireName; +} + +function adaptResponsesReplayItemsForModel( + input: ResponseInput, + options: ResponsesReplayCompatibilityOptions, +): ResponseInput { + let changed = false; + const adapted: ResponseInput = []; + for (const item of input) { + let next = item; + if (!options.supportsCustomToolCalls && item.type === "custom_tool_call") { + changed = true; + next = { + type: "function_call", + ...(item.id ? { id: item.id } : {}), + call_id: item.call_id, + name: resolveReplayCustomToolName(item.name, options.tools), + arguments: JSON.stringify({ input: item.input }), + ...(item.namespace ? { namespace: item.namespace } : {}), + }; + } else if (!options.supportsCustomToolCalls && item.type === "custom_tool_call_output") { + changed = true; + next = { + type: "function_call_output", + call_id: item.call_id, + output: item.output, + }; + } + adapted.push(next); + } + return changed ? adapted : input; +} + export interface BuildResponsesInputOptions { model: Model; context: Context; @@ -1380,6 +1426,13 @@ export function buildResponsesInput(options: BuildResponsesInp messages.push({ role: options.systemRole as "system" | "developer", content: systemPrompt }); } + const supportsImageDetailOriginal = + options.model.provider === "xai-oauth" ? false : options.supportsImageDetailOriginal; + const supportsCustomToolCalls = options.model.applyPatchToolType === "freeform"; + const replayCompatibility: ResponsesReplayCompatibilityOptions = { + supportsCustomToolCalls, + tools: options.context.tools, + }; let knownCallIds = new Set(); const customCallIds = new Set(); const transformedMessages = transformMessages( @@ -1407,7 +1460,10 @@ export function buildResponsesInput(options: BuildResponsesInp }) ?? false); if (historyItems && shouldReplayPayloadItems) { - messages.push(...sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems))); + const sanitizedItems = sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems), { + supportsImageDetailOriginal, + }); + messages.push(...adaptResponsesReplayItemsForModel(sanitizedItems, replayCompatibility)); knownCallIds = collectKnownCallIds(messages); for (const id of collectCustomCallIds(messages)) customCallIds.add(id); msgIndex++; @@ -1416,7 +1472,7 @@ export function buildResponsesInput(options: BuildResponsesInp const content = convertResponsesInputContent( msg.content, options.model.input.includes("image"), - options.supportsImageDetailOriginal, + supportsImageDetailOriginal, ); if (!content) continue; messages.push({ @@ -1444,9 +1500,13 @@ export function buildResponsesInput(options: BuildResponsesInp const historyItems = providerPayload?.items; let suppressHiddenEmptyFallback = false; if (historyItems) { - const sanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay( + const rawSanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay( filterReasoning(historyItems), + { supportsImageDetailOriginal }, ); + const sanitizedHistoryItems = rawSanitizedHistoryItems + ? adaptResponsesReplayItemsForModel(rawSanitizedHistoryItems, replayCompatibility) + : undefined; if (nativeReplayEnabled && sanitizedHistoryItems) { if (providerPayload?.dt) { messages.push(...sanitizedHistoryItems); @@ -1469,6 +1529,8 @@ export function buildResponsesInput(options: BuildResponsesInp suppressHiddenEmptyFallback ? false : includeThinkingSignatures, customCallIds, options.preserveAssistantMessageIds, + supportsCustomToolCalls, + options.context.tools, ); const outputItems = suppressHiddenEmptyFallback ? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems) @@ -1481,9 +1543,10 @@ export function buildResponsesInput(options: BuildResponsesInp msg, options.model, options.strictResponsesPairing, - options.supportsImageDetailOriginal, + supportsImageDetailOriginal, knownCallIds, customCallIds, + supportsCustomToolCalls, ); } msgIndex++; @@ -1516,6 +1579,8 @@ export function convertResponsesAssistantMessage( includeThinkingSignatures = true, customCallIds?: Set, preserveMessageIds = false, + supportsCustomToolCalls = true, + tools?: readonly Tool[], ): ResponseInput { const outputItems: ResponseInput = []; let unsignedTextBlocks = 0; @@ -1587,7 +1652,7 @@ export function convertResponsesAssistantMessage( itemId = undefined; } knownCallIds.add(normalized.callId); - if (block.customWireName) { + if (block.customWireName && supportsCustomToolCalls) { const rawInput = typeof block.arguments?.input === "string" ? block.arguments.input : ""; customCallIds?.add(normalized.callId); outputItems.push({ @@ -1599,11 +1664,15 @@ export function convertResponsesAssistantMessage( } as ResponseInput[number]); continue; } + const functionName = + block.customWireName && !supportsCustomToolCalls + ? resolveReplayCustomToolName(block.customWireName, tools) + : block.name; outputItems.push({ type: "function_call", ...(itemId ? { id: itemId } : {}), call_id: normalized.callId, - name: block.name, + name: functionName, arguments: JSON.stringify(block.arguments), }); } @@ -1619,6 +1688,7 @@ export function appendResponsesToolResultMessages( supportsImageDetailOriginal: boolean, knownCallIds: ReadonlySet, customCallIds?: ReadonlySet, + supportsCustomToolCalls = true, ): void { const supportsImages = model.input.includes("image"); const textResult = toolResult.content @@ -1648,7 +1718,7 @@ export function appendResponsesToolResultMessages( } as ResponseInput[number]); return; } - if (customCallIds?.has(normalized.callId)) { + if (supportsCustomToolCalls && customCallIds?.has(normalized.callId)) { messages.push({ type: "custom_tool_call_output", call_id: normalized.callId, diff --git a/packages/ai/src/utils.ts b/packages/ai/src/utils.ts index 0abc70cb1..988c16aa5 100644 --- a/packages/ai/src/utils.ts +++ b/packages/ai/src/utils.ts @@ -65,10 +65,50 @@ export function truncateResponseItemId(id: string, prefix: string): string { return `${prefix}_${Bun.hash(id).toString(36)}`; } -export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array>): ResponseInput { +interface OpenAIResponsesReplaySanitizeOptions { + supportsImageDetailOriginal?: boolean; +} + +function isReplayRecord(value: unknown): value is Record { + if (!value || typeof value !== "object") return false; + return !Array.isArray(value); +} + +function sanitizeReplayValueForCompatibility(value: unknown, options: OpenAIResponsesReplaySanitizeOptions): unknown { + if (options.supportsImageDetailOriginal !== false) return value; + if (Array.isArray(value)) { + let changed = false; + const sanitized = value.map(item => { + const next = sanitizeReplayValueForCompatibility(item, options); + if (next !== item) changed = true; + return next; + }); + return changed ? sanitized : value; + } + if (!isReplayRecord(value)) return value; + + let changed = false; + const sanitized: Record = {}; + for (const key in value) { + const child = value[key]; + const next = sanitizeReplayValueForCompatibility(child, options); + if (next !== child) changed = true; + sanitized[key] = next; + } + if (value.type === "input_image" && value.detail === "original") { + sanitized.detail = "auto"; + changed = true; + } + return changed ? sanitized : value; +} + +export function sanitizeOpenAIResponsesHistoryItemsForReplay( + items: Array>, + options: OpenAIResponsesReplaySanitizeOptions = {}, +): ResponseInput { const normalizedCallIds = new Map(); return items.flatMap(item => { - const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds); + const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds, options); return sanitized ? [sanitized] : []; }); } @@ -82,8 +122,9 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array>, + options: OpenAIResponsesReplaySanitizeOptions = {}, ): ResponseInput | undefined { - const sanitized = sanitizeOpenAIResponsesHistoryItemsForReplay(items); + const sanitized = sanitizeOpenAIResponsesHistoryItemsForReplay(items, options); let hasReplayableAssistantOutput = false; for (const item of sanitized) { @@ -153,6 +194,7 @@ export function sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(items: Re function sanitizeOpenAIResponsesHistoryItemForReplay( item: Record, normalizedCallIds: Map, + options: OpenAIResponsesReplaySanitizeOptions, ): OpenAIResponsesReplayItem | undefined { if (item.type === "item_reference") return undefined; if (item.type === "image_generation_call") return sanitizeOpenAIResponsesImageGenerationCallForReplay(item); @@ -164,7 +206,8 @@ function sanitizeOpenAIResponsesHistoryItemForReplay( sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds); } - return sanitizedItem as unknown as OpenAIResponsesReplayItem; + const compatibleItem = sanitizeReplayValueForCompatibility(sanitizedItem, options); + return compatibleItem as unknown as OpenAIResponsesReplayItem; } function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record): OpenAIResponsesReplayItem { diff --git a/packages/ai/test/openai-responses-history-payload.test.ts b/packages/ai/test/openai-responses-history-payload.test.ts index 9520fabf0..a1a4dc84e 100644 --- a/packages/ai/test/openai-responses-history-payload.test.ts +++ b/packages/ai/test/openai-responses-history-payload.test.ts @@ -5,11 +5,12 @@ import { } from "@oh-my-pi/pi-ai/providers/openai-codex-responses"; import { type OpenAIResponsesOptions, streamOpenAIResponses } from "@oh-my-pi/pi-ai/providers/openai-responses"; import { buildResponsesInput } from "@oh-my-pi/pi-ai/providers/openai-shared"; -import type { Context, Model, ModelSpec, ProviderSessionState } from "@oh-my-pi/pi-ai/types"; +import type { Context, Model, ModelSpec, ProviderSessionState, Tool } from "@oh-my-pi/pi-ai/types"; import { createOpenAIResponsesHistoryPayload, truncateResponseItemId } from "@oh-my-pi/pi-ai/utils"; import { buildModel } from "@oh-my-pi/pi-catalog/build"; -import { getBundledModel } from "@oh-my-pi/pi-catalog/models"; +import { type GeneratedProvider, getBundledModel } from "@oh-my-pi/pi-catalog/models"; import * as piUtils from "@oh-my-pi/pi-utils"; +import { type } from "arktype"; const TEST_INSTALLATION_ID = "00000000-0000-4000-8000-000000000001"; @@ -35,13 +36,42 @@ function createCodexToken(accountId: string): string { return `${header}.${payload}.signature`; } -function getOpenAIReasoningModel( - provider: Parameters[0], - id: string, -): Model<"openai-responses"> { - return getBundledModel(provider, id) as Model<"openai-responses">; +function getOpenAIReasoningModel(provider: GeneratedProvider, id: string): Model<"openai-responses"> { + const model = getBundledModel<"openai-responses">(provider, id); + return model; } +const ISSUE_5002_PATCH = "*** Begin Patch\n*** End Patch\n"; +const ISSUE_5002_TOOL_OUTPUT = "patch applied"; +const issue5002XaiOAuthModel = buildModel({ + id: "grok-build", + name: "Grok Build", + api: "openai-responses", + provider: "xai-oauth", + baseUrl: "https://api.x.ai/v1", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 256000, + maxTokens: 64000, +} satisfies ModelSpec<"openai-responses">); + +const issue5002ZeroUsage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; +const issue5002EditTool: Tool = { + name: "edit", + customWireName: "apply_patch", + description: "Apply a hashline patch", + parameters: type({ input: "string" }), + customFormat: { syntax: "lark", definition: 'start: "*** Begin Patch" LF\nLF: /\\n/' }, +}; + const preservedHistoryItems = [ { type: "message", role: "user", content: [{ type: "input_text", text: "Preserved user" }] }, { type: "compaction", encrypted_content: "enc_123" }, @@ -298,6 +328,38 @@ function findResponsesInputItem(input: unknown[] | undefined, type: string): Rec }) as Record | undefined; } +function isIssue5002Record(value: unknown): value is Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + return true; +} + +function findResponsesInputItemByCallId( + input: unknown[], + type: string, + callId: string, +): Record | undefined { + for (const item of input) { + if (!isIssue5002Record(item)) continue; + if (item.type === type && item.call_id === callId) return item; + } + return undefined; +} + +function collectResponsesInputImageDetails(input: unknown): string[] { + const details: string[] = []; + const visit = (node: unknown): void => { + if (Array.isArray(node)) { + for (const child of node) visit(child); + return; + } + if (!isIssue5002Record(node)) return; + if (node.type === "input_image" && typeof node.detail === "string") details.push(node.detail); + for (const key in node) visit(node[key]); + }; + visit(input); + return details; +} + function containsUserInputText(input: unknown[] | undefined, text: string): boolean { return (input ?? []).some(item => { if (!item || typeof item !== "object") return false; @@ -366,11 +428,188 @@ describe("OpenAI responses history payload", () => { }); assertWireOrder(openaiItems); - const codexModel = getBundledModel("openai-codex", "gpt-5.2-codex") as Model<"openai-codex-responses">; + const codexModel = getBundledModel<"openai-codex-responses">("openai-codex", "gpt-5.2-codex"); const codexItems = convertCodexResponsesMessages(codexModel, makeContext("openai-codex")); assertWireOrder(codexItems); }); + it("adapts reconstructed apply_patch replay for xai-oauth while preserving OpenAI custom replay", () => { + const context: Context = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "previous frame" }, + { type: "image", mimeType: "image/png", data: "ZmFrZQ==", detail: "original" }, + ], + timestamp: Date.now(), + }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "call_apply", + name: "apply_patch", + arguments: { input: ISSUE_5002_PATCH }, + customWireName: "apply_patch", + }, + ], + api: "openai-responses", + provider: "openai", + model: "gpt-5-mini", + usage: issue5002ZeroUsage, + stopReason: "toolUse", + timestamp: Date.now(), + }, + { + role: "toolResult", + toolCallId: "call_apply", + toolName: "edit", + content: [{ type: "text", text: ISSUE_5002_TOOL_OUTPUT }], + isError: false, + timestamp: Date.now(), + }, + ], + tools: [issue5002EditTool], + }; + + const xaiInput = buildResponsesInput({ + model: issue5002XaiOAuthModel, + context, + strictResponsesPairing: false, + supportsImageDetailOriginal: issue5002XaiOAuthModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: issue5002XaiOAuthModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call", "call_apply")).toEqual({ + type: "function_call", + call_id: "call_apply", + name: "edit", + arguments: JSON.stringify({ input: ISSUE_5002_PATCH }), + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call_output", "call_apply")).toEqual({ + type: "function_call_output", + call_id: "call_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(JSON.stringify(xaiInput)).not.toContain("custom_tool_call"); + expect(collectResponsesInputImageDetails(xaiInput)).toEqual(["auto"]); + + const openaiModel = getOpenAIReasoningModel("openai", "gpt-5-mini"); + const openaiInput = buildResponsesInput({ + model: openaiModel, + context, + strictResponsesPairing: false, + supportsImageDetailOriginal: openaiModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: openaiModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call", "call_apply")).toEqual({ + type: "custom_tool_call", + call_id: "call_apply", + name: "apply_patch", + input: ISSUE_5002_PATCH, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call_output", "call_apply")).toEqual({ + type: "custom_tool_call_output", + call_id: "call_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(collectResponsesInputImageDetails(openaiInput)).toEqual(["original"]); + }); + + it("adapts persisted native apply_patch Responses items for xai-oauth continuations", () => { + const nativeHistoryItems = [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "previous native frame" }, + { type: "input_image", detail: "original", image_url: "data:image/png;base64,ZmFrZQ==" }, + ], + }, + { type: "custom_tool_call", call_id: "call_native_apply", name: "apply_patch", input: ISSUE_5002_PATCH }, + { + type: "custom_tool_call_output", + call_id: "call_native_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }, + ]; + const xaiContext: Context = { + messages: [ + { + role: "assistant", + content: [{ type: "text", text: "fallback should not be replayed" }], + api: "openai-responses", + provider: "xai-oauth", + model: issue5002XaiOAuthModel.id, + usage: issue5002ZeroUsage, + stopReason: "stop", + providerPayload: createOpenAIResponsesHistoryPayload("xai-oauth", nativeHistoryItems), + timestamp: Date.now(), + }, + { role: "user", content: "continue", timestamp: Date.now() }, + ], + }; + + const xaiInput = buildResponsesInput({ + model: issue5002XaiOAuthModel, + context: xaiContext, + strictResponsesPairing: false, + supportsImageDetailOriginal: issue5002XaiOAuthModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: issue5002XaiOAuthModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call", "call_native_apply")).toEqual({ + type: "function_call", + call_id: "call_native_apply", + name: "edit", + arguments: JSON.stringify({ input: ISSUE_5002_PATCH }), + }); + expect(findResponsesInputItemByCallId(xaiInput, "function_call_output", "call_native_apply")).toEqual({ + type: "function_call_output", + call_id: "call_native_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(JSON.stringify(xaiInput)).not.toContain("custom_tool_call"); + expect(collectResponsesInputImageDetails(xaiInput)).toEqual(["auto"]); + + const openaiModel = getOpenAIReasoningModel("openai", "gpt-5-mini"); + const openaiContext: Context = { + messages: [ + { + role: "assistant", + content: [{ type: "text", text: "fallback should not be replayed" }], + api: "openai-responses", + provider: "openai", + model: openaiModel.id, + usage: issue5002ZeroUsage, + stopReason: "stop", + providerPayload: createOpenAIResponsesHistoryPayload("openai", nativeHistoryItems), + timestamp: Date.now(), + }, + { role: "user", content: "continue", timestamp: Date.now() }, + ], + }; + const openaiInput = buildResponsesInput({ + model: openaiModel, + context: openaiContext, + strictResponsesPairing: false, + supportsImageDetailOriginal: openaiModel.compat.supportsImageDetailOriginal, + nativeHistory: { replay: true, filterReasoning: openaiModel.compat.filterReasoningHistory }, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call", "call_native_apply")).toEqual({ + type: "custom_tool_call", + call_id: "call_native_apply", + name: "apply_patch", + input: ISSUE_5002_PATCH, + }); + expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call_output", "call_native_apply")).toEqual({ + type: "custom_tool_call_output", + call_id: "call_native_apply", + output: ISSUE_5002_TOOL_OUTPUT, + }); + expect(collectResponsesInputImageDetails(openaiInput)).toEqual(["original"]); + }); + it("prepends multiple OpenAI developer instructions in order without changing prompt cache key routing", async () => { const model = getOpenAIReasoningModel("openai", "gpt-5-mini"); const payload = (await captureResponsesPayload( @@ -1063,7 +1302,7 @@ describe("OpenAI responses history payload", () => { { role: "user", content: "Resume", timestamp: Date.now() }, ], }; - const model = getBundledModel("openai-codex", "gpt-5.2-codex") as Model<"openai-codex-responses">; + const model = getBundledModel<"openai-codex-responses">("openai-codex", "gpt-5.2-codex"); const payload = (await captureCodexPayload(model, context)) as { input?: unknown[] }; const functionCallItem = findResponsesInputItem(payload.input, "function_call"); const functionCallOutputItem = findResponsesInputItem(payload.input, "function_call_output");