fix(ai): adapted xai responses replay shapes

Convert freeform custom_tool_call history to function_call pairs and
clamp input_image.detail original to auto when replaying into xAI OAuth
Responses, so session continuations stop 422ing.

Fixes #5002
This commit is contained in:
Victor Araújo
2026-07-10 15:24:27 -03:00
parent 395e4a5fcf
commit 9b4dcaa114
4 changed files with 375 additions and 20 deletions
+3
View File
@@ -7,6 +7,9 @@
### Changed
- Enforced `all_turns` reasoning context for all Responses Lite requests
### Fixed
- Fixed xAI OAuth Responses continuations replaying OpenAI-only `custom_tool_call`/`custom_tool_call_output` history and `input_image.detail: "original"` frames; replay now downgrades those to xAI-compatible function calls and `detail: "auto"`. ([#5002](https://github.com/can1357/oh-my-pi/issues/5002))
## [16.4.0] - 2026-07-10
+77 -7
View File
@@ -1356,6 +1356,52 @@ export function convertResponsesInputContent(
return normalizedContent.length > 0 ? normalizedContent : undefined;
}
interface ResponsesReplayCompatibilityOptions {
supportsCustomToolCalls: boolean;
tools: readonly Tool[] | undefined;
}
function resolveReplayCustomToolName(wireName: string, tools: readonly Tool[] | undefined): string {
if (tools) {
for (const tool of tools) {
if (tool.customWireName === wireName) return tool.name;
}
}
if (wireName === "apply_patch") return "edit";
return wireName;
}
function adaptResponsesReplayItemsForModel(
input: ResponseInput,
options: ResponsesReplayCompatibilityOptions,
): ResponseInput {
let changed = false;
const adapted: ResponseInput = [];
for (const item of input) {
let next = item;
if (!options.supportsCustomToolCalls && item.type === "custom_tool_call") {
changed = true;
next = {
type: "function_call",
...(item.id ? { id: item.id } : {}),
call_id: item.call_id,
name: resolveReplayCustomToolName(item.name, options.tools),
arguments: JSON.stringify({ input: item.input }),
...(item.namespace ? { namespace: item.namespace } : {}),
};
} else if (!options.supportsCustomToolCalls && item.type === "custom_tool_call_output") {
changed = true;
next = {
type: "function_call_output",
call_id: item.call_id,
output: item.output,
};
}
adapted.push(next);
}
return changed ? adapted : input;
}
export interface BuildResponsesInputOptions<TApi extends Api> {
model: Model<TApi>;
context: Context;
@@ -1380,6 +1426,13 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
messages.push({ role: options.systemRole as "system" | "developer", content: systemPrompt });
}
const supportsImageDetailOriginal =
options.model.provider === "xai-oauth" ? false : options.supportsImageDetailOriginal;
const supportsCustomToolCalls = options.model.applyPatchToolType === "freeform";
const replayCompatibility: ResponsesReplayCompatibilityOptions = {
supportsCustomToolCalls,
tools: options.context.tools,
};
let knownCallIds = new Set<string>();
const customCallIds = new Set<string>();
const transformedMessages = transformMessages(
@@ -1407,7 +1460,10 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
}) ??
false);
if (historyItems && shouldReplayPayloadItems) {
messages.push(...sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems)));
const sanitizedItems = sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems), {
supportsImageDetailOriginal,
});
messages.push(...adaptResponsesReplayItemsForModel(sanitizedItems, replayCompatibility));
knownCallIds = collectKnownCallIds(messages);
for (const id of collectCustomCallIds(messages)) customCallIds.add(id);
msgIndex++;
@@ -1416,7 +1472,7 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
const content = convertResponsesInputContent(
msg.content,
options.model.input.includes("image"),
options.supportsImageDetailOriginal,
supportsImageDetailOriginal,
);
if (!content) continue;
messages.push({
@@ -1444,9 +1500,13 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
const historyItems = providerPayload?.items;
let suppressHiddenEmptyFallback = false;
if (historyItems) {
const sanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay(
const rawSanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay(
filterReasoning(historyItems),
{ supportsImageDetailOriginal },
);
const sanitizedHistoryItems = rawSanitizedHistoryItems
? adaptResponsesReplayItemsForModel(rawSanitizedHistoryItems, replayCompatibility)
: undefined;
if (nativeReplayEnabled && sanitizedHistoryItems) {
if (providerPayload?.dt) {
messages.push(...sanitizedHistoryItems);
@@ -1469,6 +1529,8 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
suppressHiddenEmptyFallback ? false : includeThinkingSignatures,
customCallIds,
options.preserveAssistantMessageIds,
supportsCustomToolCalls,
options.context.tools,
);
const outputItems = suppressHiddenEmptyFallback
? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems)
@@ -1481,9 +1543,10 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
msg,
options.model,
options.strictResponsesPairing,
options.supportsImageDetailOriginal,
supportsImageDetailOriginal,
knownCallIds,
customCallIds,
supportsCustomToolCalls,
);
}
msgIndex++;
@@ -1516,6 +1579,8 @@ export function convertResponsesAssistantMessage<TApi extends Api>(
includeThinkingSignatures = true,
customCallIds?: Set<string>,
preserveMessageIds = false,
supportsCustomToolCalls = true,
tools?: readonly Tool[],
): ResponseInput {
const outputItems: ResponseInput = [];
let unsignedTextBlocks = 0;
@@ -1587,7 +1652,7 @@ export function convertResponsesAssistantMessage<TApi extends Api>(
itemId = undefined;
}
knownCallIds.add(normalized.callId);
if (block.customWireName) {
if (block.customWireName && supportsCustomToolCalls) {
const rawInput = typeof block.arguments?.input === "string" ? block.arguments.input : "";
customCallIds?.add(normalized.callId);
outputItems.push({
@@ -1599,11 +1664,15 @@ export function convertResponsesAssistantMessage<TApi extends Api>(
} as ResponseInput[number]);
continue;
}
const functionName =
block.customWireName && !supportsCustomToolCalls
? resolveReplayCustomToolName(block.customWireName, tools)
: block.name;
outputItems.push({
type: "function_call",
...(itemId ? { id: itemId } : {}),
call_id: normalized.callId,
name: block.name,
name: functionName,
arguments: JSON.stringify(block.arguments),
});
}
@@ -1619,6 +1688,7 @@ export function appendResponsesToolResultMessages<TApi extends Api>(
supportsImageDetailOriginal: boolean,
knownCallIds: ReadonlySet<string>,
customCallIds?: ReadonlySet<string>,
supportsCustomToolCalls = true,
): void {
const supportsImages = model.input.includes("image");
const textResult = toolResult.content
@@ -1648,7 +1718,7 @@ export function appendResponsesToolResultMessages<TApi extends Api>(
} as ResponseInput[number]);
return;
}
if (customCallIds?.has(normalized.callId)) {
if (supportsCustomToolCalls && customCallIds?.has(normalized.callId)) {
messages.push({
type: "custom_tool_call_output",
call_id: normalized.callId,
+47 -4
View File
@@ -65,10 +65,50 @@ export function truncateResponseItemId(id: string, prefix: string): string {
return `${prefix}_${Bun.hash(id).toString(36)}`;
}
export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array<Record<string, unknown>>): ResponseInput {
interface OpenAIResponsesReplaySanitizeOptions {
supportsImageDetailOriginal?: boolean;
}
function isReplayRecord(value: unknown): value is Record<string, unknown> {
if (!value || typeof value !== "object") return false;
return !Array.isArray(value);
}
function sanitizeReplayValueForCompatibility(value: unknown, options: OpenAIResponsesReplaySanitizeOptions): unknown {
if (options.supportsImageDetailOriginal !== false) return value;
if (Array.isArray(value)) {
let changed = false;
const sanitized = value.map(item => {
const next = sanitizeReplayValueForCompatibility(item, options);
if (next !== item) changed = true;
return next;
});
return changed ? sanitized : value;
}
if (!isReplayRecord(value)) return value;
let changed = false;
const sanitized: Record<string, unknown> = {};
for (const key in value) {
const child = value[key];
const next = sanitizeReplayValueForCompatibility(child, options);
if (next !== child) changed = true;
sanitized[key] = next;
}
if (value.type === "input_image" && value.detail === "original") {
sanitized.detail = "auto";
changed = true;
}
return changed ? sanitized : value;
}
export function sanitizeOpenAIResponsesHistoryItemsForReplay(
items: Array<Record<string, unknown>>,
options: OpenAIResponsesReplaySanitizeOptions = {},
): ResponseInput {
const normalizedCallIds = new Map<string, string>();
return items.flatMap(item => {
const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds);
const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds, options);
return sanitized ? [sanitized] : [];
});
}
@@ -82,8 +122,9 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array<Record
*/
export function sanitizeOpenAIResponsesAssistantHistoryItemsForReplay(
items: Array<Record<string, unknown>>,
options: OpenAIResponsesReplaySanitizeOptions = {},
): ResponseInput | undefined {
const sanitized = sanitizeOpenAIResponsesHistoryItemsForReplay(items);
const sanitized = sanitizeOpenAIResponsesHistoryItemsForReplay(items, options);
let hasReplayableAssistantOutput = false;
for (const item of sanitized) {
@@ -153,6 +194,7 @@ export function sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(items: Re
function sanitizeOpenAIResponsesHistoryItemForReplay(
item: Record<string, unknown>,
normalizedCallIds: Map<string, string>,
options: OpenAIResponsesReplaySanitizeOptions,
): OpenAIResponsesReplayItem | undefined {
if (item.type === "item_reference") return undefined;
if (item.type === "image_generation_call") return sanitizeOpenAIResponsesImageGenerationCallForReplay(item);
@@ -164,7 +206,8 @@ function sanitizeOpenAIResponsesHistoryItemForReplay(
sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds);
}
return sanitizedItem as unknown as OpenAIResponsesReplayItem;
const compatibleItem = sanitizeReplayValueForCompatibility(sanitizedItem, options);
return compatibleItem as unknown as OpenAIResponsesReplayItem;
}
function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record<string, unknown>): OpenAIResponsesReplayItem {
@@ -5,11 +5,12 @@ import {
} from "@oh-my-pi/pi-ai/providers/openai-codex-responses";
import { type OpenAIResponsesOptions, streamOpenAIResponses } from "@oh-my-pi/pi-ai/providers/openai-responses";
import { buildResponsesInput } from "@oh-my-pi/pi-ai/providers/openai-shared";
import type { Context, Model, ModelSpec, ProviderSessionState } from "@oh-my-pi/pi-ai/types";
import type { Context, Model, ModelSpec, ProviderSessionState, Tool } from "@oh-my-pi/pi-ai/types";
import { createOpenAIResponsesHistoryPayload, truncateResponseItemId } from "@oh-my-pi/pi-ai/utils";
import { buildModel } from "@oh-my-pi/pi-catalog/build";
import { getBundledModel } from "@oh-my-pi/pi-catalog/models";
import { type GeneratedProvider, getBundledModel } from "@oh-my-pi/pi-catalog/models";
import * as piUtils from "@oh-my-pi/pi-utils";
import { type } from "arktype";
const TEST_INSTALLATION_ID = "00000000-0000-4000-8000-000000000001";
@@ -35,13 +36,42 @@ function createCodexToken(accountId: string): string {
return `${header}.${payload}.signature`;
}
function getOpenAIReasoningModel(
provider: Parameters<typeof getBundledModel>[0],
id: string,
): Model<"openai-responses"> {
return getBundledModel(provider, id) as Model<"openai-responses">;
function getOpenAIReasoningModel(provider: GeneratedProvider, id: string): Model<"openai-responses"> {
const model = getBundledModel<"openai-responses">(provider, id);
return model;
}
const ISSUE_5002_PATCH = "*** Begin Patch\n*** End Patch\n";
const ISSUE_5002_TOOL_OUTPUT = "patch applied";
const issue5002XaiOAuthModel = buildModel({
id: "grok-build",
name: "Grok Build",
api: "openai-responses",
provider: "xai-oauth",
baseUrl: "https://api.x.ai/v1",
reasoning: true,
input: ["text", "image"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 256000,
maxTokens: 64000,
} satisfies ModelSpec<"openai-responses">);
const issue5002ZeroUsage = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
const issue5002EditTool: Tool = {
name: "edit",
customWireName: "apply_patch",
description: "Apply a hashline patch",
parameters: type({ input: "string" }),
customFormat: { syntax: "lark", definition: 'start: "*** Begin Patch" LF\nLF: /\\n/' },
};
const preservedHistoryItems = [
{ type: "message", role: "user", content: [{ type: "input_text", text: "Preserved user" }] },
{ type: "compaction", encrypted_content: "enc_123" },
@@ -298,6 +328,38 @@ function findResponsesInputItem(input: unknown[] | undefined, type: string): Rec
}) as Record<string, unknown> | undefined;
}
function isIssue5002Record(value: unknown): value is Record<string, unknown> {
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
return true;
}
function findResponsesInputItemByCallId(
input: unknown[],
type: string,
callId: string,
): Record<string, unknown> | undefined {
for (const item of input) {
if (!isIssue5002Record(item)) continue;
if (item.type === type && item.call_id === callId) return item;
}
return undefined;
}
function collectResponsesInputImageDetails(input: unknown): string[] {
const details: string[] = [];
const visit = (node: unknown): void => {
if (Array.isArray(node)) {
for (const child of node) visit(child);
return;
}
if (!isIssue5002Record(node)) return;
if (node.type === "input_image" && typeof node.detail === "string") details.push(node.detail);
for (const key in node) visit(node[key]);
};
visit(input);
return details;
}
function containsUserInputText(input: unknown[] | undefined, text: string): boolean {
return (input ?? []).some(item => {
if (!item || typeof item !== "object") return false;
@@ -366,11 +428,188 @@ describe("OpenAI responses history payload", () => {
});
assertWireOrder(openaiItems);
const codexModel = getBundledModel("openai-codex", "gpt-5.2-codex") as Model<"openai-codex-responses">;
const codexModel = getBundledModel<"openai-codex-responses">("openai-codex", "gpt-5.2-codex");
const codexItems = convertCodexResponsesMessages(codexModel, makeContext("openai-codex"));
assertWireOrder(codexItems);
});
it("adapts reconstructed apply_patch replay for xai-oauth while preserving OpenAI custom replay", () => {
const context: Context = {
messages: [
{
role: "user",
content: [
{ type: "text", text: "previous frame" },
{ type: "image", mimeType: "image/png", data: "ZmFrZQ==", detail: "original" },
],
timestamp: Date.now(),
},
{
role: "assistant",
content: [
{
type: "toolCall",
id: "call_apply",
name: "apply_patch",
arguments: { input: ISSUE_5002_PATCH },
customWireName: "apply_patch",
},
],
api: "openai-responses",
provider: "openai",
model: "gpt-5-mini",
usage: issue5002ZeroUsage,
stopReason: "toolUse",
timestamp: Date.now(),
},
{
role: "toolResult",
toolCallId: "call_apply",
toolName: "edit",
content: [{ type: "text", text: ISSUE_5002_TOOL_OUTPUT }],
isError: false,
timestamp: Date.now(),
},
],
tools: [issue5002EditTool],
};
const xaiInput = buildResponsesInput({
model: issue5002XaiOAuthModel,
context,
strictResponsesPairing: false,
supportsImageDetailOriginal: issue5002XaiOAuthModel.compat.supportsImageDetailOriginal,
nativeHistory: { replay: true, filterReasoning: issue5002XaiOAuthModel.compat.filterReasoningHistory },
});
expect(findResponsesInputItemByCallId(xaiInput, "function_call", "call_apply")).toEqual({
type: "function_call",
call_id: "call_apply",
name: "edit",
arguments: JSON.stringify({ input: ISSUE_5002_PATCH }),
});
expect(findResponsesInputItemByCallId(xaiInput, "function_call_output", "call_apply")).toEqual({
type: "function_call_output",
call_id: "call_apply",
output: ISSUE_5002_TOOL_OUTPUT,
});
expect(JSON.stringify(xaiInput)).not.toContain("custom_tool_call");
expect(collectResponsesInputImageDetails(xaiInput)).toEqual(["auto"]);
const openaiModel = getOpenAIReasoningModel("openai", "gpt-5-mini");
const openaiInput = buildResponsesInput({
model: openaiModel,
context,
strictResponsesPairing: false,
supportsImageDetailOriginal: openaiModel.compat.supportsImageDetailOriginal,
nativeHistory: { replay: true, filterReasoning: openaiModel.compat.filterReasoningHistory },
});
expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call", "call_apply")).toEqual({
type: "custom_tool_call",
call_id: "call_apply",
name: "apply_patch",
input: ISSUE_5002_PATCH,
});
expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call_output", "call_apply")).toEqual({
type: "custom_tool_call_output",
call_id: "call_apply",
output: ISSUE_5002_TOOL_OUTPUT,
});
expect(collectResponsesInputImageDetails(openaiInput)).toEqual(["original"]);
});
it("adapts persisted native apply_patch Responses items for xai-oauth continuations", () => {
const nativeHistoryItems = [
{
type: "message",
role: "user",
content: [
{ type: "input_text", text: "previous native frame" },
{ type: "input_image", detail: "original", image_url: "data:image/png;base64,ZmFrZQ==" },
],
},
{ type: "custom_tool_call", call_id: "call_native_apply", name: "apply_patch", input: ISSUE_5002_PATCH },
{
type: "custom_tool_call_output",
call_id: "call_native_apply",
output: ISSUE_5002_TOOL_OUTPUT,
},
];
const xaiContext: Context = {
messages: [
{
role: "assistant",
content: [{ type: "text", text: "fallback should not be replayed" }],
api: "openai-responses",
provider: "xai-oauth",
model: issue5002XaiOAuthModel.id,
usage: issue5002ZeroUsage,
stopReason: "stop",
providerPayload: createOpenAIResponsesHistoryPayload("xai-oauth", nativeHistoryItems),
timestamp: Date.now(),
},
{ role: "user", content: "continue", timestamp: Date.now() },
],
};
const xaiInput = buildResponsesInput({
model: issue5002XaiOAuthModel,
context: xaiContext,
strictResponsesPairing: false,
supportsImageDetailOriginal: issue5002XaiOAuthModel.compat.supportsImageDetailOriginal,
nativeHistory: { replay: true, filterReasoning: issue5002XaiOAuthModel.compat.filterReasoningHistory },
});
expect(findResponsesInputItemByCallId(xaiInput, "function_call", "call_native_apply")).toEqual({
type: "function_call",
call_id: "call_native_apply",
name: "edit",
arguments: JSON.stringify({ input: ISSUE_5002_PATCH }),
});
expect(findResponsesInputItemByCallId(xaiInput, "function_call_output", "call_native_apply")).toEqual({
type: "function_call_output",
call_id: "call_native_apply",
output: ISSUE_5002_TOOL_OUTPUT,
});
expect(JSON.stringify(xaiInput)).not.toContain("custom_tool_call");
expect(collectResponsesInputImageDetails(xaiInput)).toEqual(["auto"]);
const openaiModel = getOpenAIReasoningModel("openai", "gpt-5-mini");
const openaiContext: Context = {
messages: [
{
role: "assistant",
content: [{ type: "text", text: "fallback should not be replayed" }],
api: "openai-responses",
provider: "openai",
model: openaiModel.id,
usage: issue5002ZeroUsage,
stopReason: "stop",
providerPayload: createOpenAIResponsesHistoryPayload("openai", nativeHistoryItems),
timestamp: Date.now(),
},
{ role: "user", content: "continue", timestamp: Date.now() },
],
};
const openaiInput = buildResponsesInput({
model: openaiModel,
context: openaiContext,
strictResponsesPairing: false,
supportsImageDetailOriginal: openaiModel.compat.supportsImageDetailOriginal,
nativeHistory: { replay: true, filterReasoning: openaiModel.compat.filterReasoningHistory },
});
expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call", "call_native_apply")).toEqual({
type: "custom_tool_call",
call_id: "call_native_apply",
name: "apply_patch",
input: ISSUE_5002_PATCH,
});
expect(findResponsesInputItemByCallId(openaiInput, "custom_tool_call_output", "call_native_apply")).toEqual({
type: "custom_tool_call_output",
call_id: "call_native_apply",
output: ISSUE_5002_TOOL_OUTPUT,
});
expect(collectResponsesInputImageDetails(openaiInput)).toEqual(["original"]);
});
it("prepends multiple OpenAI developer instructions in order without changing prompt cache key routing", async () => {
const model = getOpenAIReasoningModel("openai", "gpt-5-mini");
const payload = (await captureResponsesPayload(
@@ -1063,7 +1302,7 @@ describe("OpenAI responses history payload", () => {
{ role: "user", content: "Resume", timestamp: Date.now() },
],
};
const model = getBundledModel("openai-codex", "gpt-5.2-codex") as Model<"openai-codex-responses">;
const model = getBundledModel<"openai-codex-responses">("openai-codex", "gpt-5.2-codex");
const payload = (await captureCodexPayload(model, context)) as { input?: unknown[] };
const functionCallItem = findResponsesInputItem(payload.input, "function_call");
const functionCallOutputItem = findResponsesInputItem(payload.input, "function_call_output");