smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai): tightened xai replay adaptation to catalog compat

Browse Source 
Drive image-detail clamping from resolved model compat instead of a
provider hardcode, clamp input_image only on known paths, and skip
custom-tool adaptation when freeform is supported.
This commit is contained in:
Victor Araújo
2026-07-10 15:27:24 -03:00
parent 6195193435
commit 0e3e5ab72a
3 changed files with 91 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/providers/openai-shared.ts
+54 -33
View File
@@ -1356,48 +1356,61 @@ export function convertResponsesInputContent(
return normalizedContent.length > 0 ? normalizedContent : undefined;
}
interface ResponsesReplayCompatibilityOptions {
supportsCustomToolCalls: boolean;
tools: readonly Tool[] | undefined;
}
function resolveReplayCustomToolName(wireName: string, tools: readonly Tool[] | undefined): string {
if (tools) {
for (const tool of tools) {
if (tool.customWireName === wireName) return tool.name;
}
/**
* Map freeform custom-tool wire names back to the internal tool name for
* providers that only accept function_call / function_call_output.
* Built once per request; `apply_patch` → `edit` is the OMP default.
*/
function buildCustomToolWireNameMap(tools: readonly Tool[] | undefined): ReadonlyMap<string, string> | undefined {
if (!tools?.length) return undefined;
const map = new Map<string, string>();
for (const tool of tools) {
if (tool.customWireName) map.set(tool.customWireName, tool.name);
}
if (wireName === "apply_patch") return "edit";
return wireName;
return map.size > 0 ? map : undefined;
}
function resolveReplayCustomToolName(wireName: string, wireNameMap: ReadonlyMap<string, string> | undefined): string {
return wireNameMap?.get(wireName) ?? (wireName === "apply_patch" ? "edit" : wireName);
}
/**
* Downgrade OpenAI-only custom tool items when the target model does not
* advertise freeform custom tools (`applyPatchToolType === "freeform"`).
* No-op (returns the same array reference) when freeform is supported.
*/
function adaptResponsesReplayItemsForModel(
input: ResponseInput,
options: ResponsesReplayCompatibilityOptions,
supportsCustomToolCalls: boolean,
wireNameMap: ReadonlyMap<string, string> | undefined,
): ResponseInput {
if (supportsCustomToolCalls) return input;
let changed = false;
const adapted: ResponseInput = [];
for (const item of input) {
let next = item;
if (!options.supportsCustomToolCalls && item.type === "custom_tool_call") {
if (item.type === "custom_tool_call") {
changed = true;
next = {
adapted.push({
type: "function_call",
...(item.id ? { id: item.id } : {}),
call_id: item.call_id,
name: resolveReplayCustomToolName(item.name, options.tools),
name: resolveReplayCustomToolName(item.name, wireNameMap),
arguments: JSON.stringify({ input: item.input }),
...(item.namespace ? { namespace: item.namespace } : {}),
};
} else if (!options.supportsCustomToolCalls && item.type === "custom_tool_call_output") {
});
continue;
}
if (item.type === "custom_tool_call_output") {
changed = true;
next = {
adapted.push({
type: "function_call_output",
call_id: item.call_id,
output: item.output,
};
});
continue;
}
adapted.push(next);
adapted.push(item);
}
return changed ? adapted : input;
}
@@ -1426,13 +1439,15 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
messages.push({ role: options.systemRole as "system" | "developer", content: systemPrompt });
}
const supportsImageDetailOriginal =
options.model.provider === "xai-oauth" ? false : options.supportsImageDetailOriginal;
// Compat is resolved by the catalog (e.g. Copilot / xai-oauth reject
// `detail: "original"`). Do not re-branch on provider id here.
const supportsImageDetailOriginal = options.supportsImageDetailOriginal;
// Freeform custom tools (`custom_tool_call`) only when the catalog says so;
// same gate as tool conversion (`applyPatchToolType === "freeform"`).
const supportsCustomToolCalls = options.model.applyPatchToolType === "freeform";
const replayCompatibility: ResponsesReplayCompatibilityOptions = {
supportsCustomToolCalls,
tools: options.context.tools,
};
const customToolWireNameMap = supportsCustomToolCalls
? undefined
: buildCustomToolWireNameMap(options.context.tools);
let knownCallIds = new Set<string>();
const customCallIds = new Set<string>();
const transformedMessages = transformMessages(
@@ -1463,7 +1478,9 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
const sanitizedItems = sanitizeOpenAIResponsesHistoryItemsForReplay(filterReasoning(historyItems), {
supportsImageDetailOriginal,
});
messages.push(...adaptResponsesReplayItemsForModel(sanitizedItems, replayCompatibility));
messages.push(
...adaptResponsesReplayItemsForModel(sanitizedItems, supportsCustomToolCalls, customToolWireNameMap),
);
knownCallIds = collectKnownCallIds(messages);
for (const id of collectCustomCallIds(messages)) customCallIds.add(id);
msgIndex++;
@@ -1505,7 +1522,11 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
{ supportsImageDetailOriginal },
);
const sanitizedHistoryItems = rawSanitizedHistoryItems
? adaptResponsesReplayItemsForModel(rawSanitizedHistoryItems, replayCompatibility)
? adaptResponsesReplayItemsForModel(
rawSanitizedHistoryItems,
supportsCustomToolCalls,
customToolWireNameMap,
)
: undefined;
if (nativeReplayEnabled && sanitizedHistoryItems) {
if (providerPayload?.dt) {
@@ -1530,7 +1551,7 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
customCallIds,
options.preserveAssistantMessageIds,
supportsCustomToolCalls,
options.context.tools,
customToolWireNameMap,
);
const outputItems = suppressHiddenEmptyFallback
? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems)
@@ -1580,7 +1601,7 @@ export function convertResponsesAssistantMessage<TApi extends Api>(
customCallIds?: Set<string>,
preserveMessageIds = false,
supportsCustomToolCalls = true,
tools?: readonly Tool[],
customToolWireNameMap?: ReadonlyMap<string, string>,
): ResponseInput {
const outputItems: ResponseInput = [];
let unsignedTextBlocks = 0;
@@ -1666,7 +1687,7 @@ export function convertResponsesAssistantMessage<TApi extends Api>(
}
const functionName =
block.customWireName && !supportsCustomToolCalls
? resolveReplayCustomToolName(block.customWireName, tools)
? resolveReplayCustomToolName(block.customWireName, customToolWireNameMap)
: block.name;
outputItems.push({
type: "function_call",
packages/ai/src/utils.ts
+29 -30
View File
@@ -69,37 +69,32 @@ interface OpenAIResponsesReplaySanitizeOptions {
supportsImageDetailOriginal?: boolean;
}
function isReplayRecord(value: unknown): value is Record<string, unknown> {
if (!value || typeof value !== "object") return false;
return !Array.isArray(value);
}
/**
* Clamp `detail: "original"` only where Responses input_image parts live —
* top-level items and `message.content[]`. Avoids a deep tree walk/clone of
* every history node on providers that reject native-resolution images.
*/
function clampReplayItemImageDetail(
item: Record<string, unknown>,
supportsImageDetailOriginal: boolean,
): Record<string, unknown> {
if (supportsImageDetailOriginal) return item;
function sanitizeReplayValueForCompatibility(value: unknown, options: OpenAIResponsesReplaySanitizeOptions): unknown {
if (options.supportsImageDetailOriginal !== false) return value;
if (Array.isArray(value)) {
let changed = false;
const sanitized = value.map(item => {
const next = sanitizeReplayValueForCompatibility(item, options);
if (next !== item) changed = true;
return next;
});
return changed ? sanitized : value;
if (item.type === "input_image" && item.detail === "original") {
return { ...item, detail: "auto" };
}
if (!isReplayRecord(value)) return value;
if (item.type !== "message" || !Array.isArray(item.content)) return item;
let changed = false;
const sanitized: Record<string, unknown> = {};
for (const key in value) {
const child = value[key];
const next = sanitizeReplayValueForCompatibility(child, options);
if (next !== child) changed = true;
sanitized[key] = next;
}
if (value.type === "input_image" && value.detail === "original") {
sanitized.detail = "auto";
const content = item.content.map(part => {
if (!part || typeof part !== "object" || Array.isArray(part)) return part;
const record = part as Record<string, unknown>;
if (record.type !== "input_image" || record.detail !== "original") return part;
changed = true;
}
return changed ? sanitized : value;
return { ...record, detail: "auto" };
});
return changed ? { ...item, content } : item;
}
export function sanitizeOpenAIResponsesHistoryItemsForReplay(
@@ -107,8 +102,13 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay(
options: OpenAIResponsesReplaySanitizeOptions = {},
): ResponseInput {
const normalizedCallIds = new Map<string, string>();
const supportsImageDetailOriginal = options.supportsImageDetailOriginal !== false;
return items.flatMap(item => {
const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(item, normalizedCallIds, options);
const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(
item,
normalizedCallIds,
supportsImageDetailOriginal,
);
return sanitized ? [sanitized] : [];
});
}
@@ -194,7 +194,7 @@ export function sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(items: Re
function sanitizeOpenAIResponsesHistoryItemForReplay(
item: Record<string, unknown>,
normalizedCallIds: Map<string, string>,
options: OpenAIResponsesReplaySanitizeOptions,
supportsImageDetailOriginal: boolean,
): OpenAIResponsesReplayItem | undefined {
if (item.type === "item_reference") return undefined;
if (item.type === "image_generation_call") return sanitizeOpenAIResponsesImageGenerationCallForReplay(item);
@@ -206,8 +206,7 @@ function sanitizeOpenAIResponsesHistoryItemForReplay(
sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds);
}
const compatibleItem = sanitizeReplayValueForCompatibility(sanitizedItem, options);
return compatibleItem as unknown as OpenAIResponsesReplayItem;
return clampReplayItemImageDetail(sanitizedItem, supportsImageDetailOriginal) as unknown as OpenAIResponsesReplayItem;
}
function sanitizeOpenAIResponsesReasoningItemForReplay(item: Record<string, unknown>): OpenAIResponsesReplayItem {
packages/catalog/src/compat/openai.ts
+8 -6
View File
@@ -595,12 +595,14 @@ export function buildOpenAIResponsesCompat(spec: OpenAIResponsesSpecLike): Resol
// Azure OpenAI and GitHub Copilot Responses paths require tool results
// to strictly match prior tool calls when building Responses inputs.
strictResponsesPairing: isAzure || spec.provider === "github-copilot",
// GitHub Copilot's Responses endpoint rejects the `detail: "original"`
// image hint with a 400; every other host preserves native-resolution
// frames (snapcompact relies on `original`). Detect Copilot by provider id
// or base-URL host (mirroring the Anthropic compat builder) so a model
// pointed at the Copilot host under a different provider id still clamps.
supportsImageDetailOriginal: !modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"),
// GitHub Copilot and xAI OAuth reject `detail: "original"` (400 / 422).
// Every other host preserves native-resolution frames (snapcompact relies
// on `original`). Detect Copilot by provider id or base-URL host so a
// model pointed at the Copilot host under a different provider id still
// clamps; xai-oauth is provider-id only (same host family as paid `xai`).
supportsImageDetailOriginal:
spec.provider !== "xai-oauth" &&
!modelMatchesHost({ provider: spec.provider, baseUrl }, "githubCopilot"),
reasoningEffortMap: {},
supportsReasoningParams: true,
thinkingFormat,