diff --git a/packages/ai/src/providers/openai-shared.ts b/packages/ai/src/providers/openai-shared.ts index ad4687219..ddb71bcaa 100644 --- a/packages/ai/src/providers/openai-shared.ts +++ b/packages/ai/src/providers/openai-shared.ts @@ -69,6 +69,7 @@ import { sanitizeOpenAIResponsesAssistantFallbackItemsForReplay, sanitizeOpenAIResponsesAssistantHistoryItemsForReplay, sanitizeOpenAIResponsesHistoryItemsForReplay, + stripUnpairedOpenAIResponsesComputerReasoningIdsForReplay, } from "../utils"; import { clearStreamingPartialJson, @@ -1759,7 +1760,8 @@ export function buildResponsesInput(options: BuildResponsesInp } const withRepairedOutputs = options.repairOrphanOutputs ? repairOrphanResponsesToolOutputs(messages) : messages; - return repairOrphanResponsesToolCalls(withRepairedOutputs); + const withRepairedCalls = repairOrphanResponsesToolCalls(withRepairedOutputs); + return stripUnpairedOpenAIResponsesComputerReasoningIdsForReplay(withRepairedCalls); } type ResponsesReplayAssistantMessage = Omit & { id?: string }; diff --git a/packages/ai/src/utils.ts b/packages/ai/src/utils.ts index 389ee009f..c1ab931da 100644 --- a/packages/ai/src/utils.ts +++ b/packages/ai/src/utils.ts @@ -125,52 +125,102 @@ function isOpenAIResponsesClientInputBoundary(item: Record): bo } } +function collectOpenAIResponsesComputerLinkedReasoningItems( + items: Array>, + requireLaterOutput: boolean, +): Set> { + let computerCallsWithLaterOutputs: Set> | undefined; + if (requireLaterOutput) { + computerCallsWithLaterOutputs = new Set(); + const laterComputerOutputCallIds = new Set(); + for (let index = items.length - 1; index >= 0; index--) { + const item = items[index]!; + if (item.type === "computer_call_output" && typeof item.call_id === "string") { + laterComputerOutputCallIds.add(item.call_id); + } else if ( + item.type === "computer_call" && + typeof item.id === "string" && + typeof item.call_id === "string" && + laterComputerOutputCallIds.has(item.call_id) + ) { + computerCallsWithLaterOutputs.add(item); + } + } + } + + const computerLinkedReasoningItems = new Set>(); + const responseReasoningItems: Array> = []; + for (const item of items) { + if (isOpenAIResponsesClientInputBoundary(item)) { + responseReasoningItems.length = 0; + } else if (item.type === "reasoning") { + responseReasoningItems.push(item); + } else if ( + item.type === "computer_call" && + typeof item.id === "string" && + (!computerCallsWithLaterOutputs || computerCallsWithLaterOutputs.has(item)) + ) { + for (const reasoningItem of responseReasoningItems) computerLinkedReasoningItems.add(reasoningItem); + } + } + return computerLinkedReasoningItems; +} + +const provisionalOpenAIResponsesComputerReasoningItems = new WeakSet(); + export function sanitizeOpenAIResponsesHistoryItemsForReplay( items: Array>, options: OpenAIResponsesReplaySanitizeOptions = {}, ): ResponseInput { const normalizedCallIds = new Map(); const supportsImageDetailOriginal = options.supportsImageDetailOriginal !== false; - const supportsComputerUse = options.supportsComputerUse !== false; - // Stateless native computer history is an atomic Responses chain: replaying - // a `computer_call` ID requires the reasoning item IDs from that response. - const computerLinkedReasoningItems = new Set>(); - const responseReasoningItems: Array> = []; - const computerCallsWithLaterOutputs = new Set>(); - const laterComputerOutputCallIds = new Set(); - for (let index = items.length - 1; index >= 0; index--) { - const item = items[index]!; - if (item.type === "computer_call_output" && typeof item.call_id === "string") { - laterComputerOutputCallIds.add(item.call_id); - } else if ( - item.type === "computer_call" && - typeof item.id === "string" && - typeof item.call_id === "string" && - laterComputerOutputCallIds.has(item.call_id) - ) { - computerCallsWithLaterOutputs.add(item); - } - } - for (const item of items) { - if (isOpenAIResponsesClientInputBoundary(item)) { - responseReasoningItems.length = 0; - } else if (item.type === "reasoning") { - responseReasoningItems.push(item); - } else if (supportsComputerUse && computerCallsWithLaterOutputs.has(item)) { - for (const reasoningItem of responseReasoningItems) computerLinkedReasoningItems.add(reasoningItem); - } - } + const computerLinkedReasoningItems = + options.supportsComputerUse === false + ? undefined + : collectOpenAIResponsesComputerLinkedReasoningItems(items, false); return items.flatMap(item => { + const preserveForComputer = computerLinkedReasoningItems?.has(item) === true; const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay( item, normalizedCallIds, supportsImageDetailOriginal, - computerLinkedReasoningItems.has(item), + preserveForComputer, ); + if (preserveForComputer && sanitized?.type === "reasoning") { + provisionalOpenAIResponsesComputerReasoningItems.add(sanitized); + } return sanitized ? [sanitized] : []; }); } +/** + * Finalize provisional native-computer reasoning IDs after the complete + * Responses input has been rebuilt, model-adapted, and orphan-repaired. + */ +export function stripUnpairedOpenAIResponsesComputerReasoningIdsForReplay(items: ResponseInput): ResponseInput { + const records = items as unknown as Array>; + const linkedReasoningItems = collectOpenAIResponsesComputerLinkedReasoningItems(records, true); + let sanitized: ResponseInput | undefined; + + for (let index = 0; index < items.length; index++) { + const item = items[index]!; + const record = records[index]!; + if ( + item.type !== "reasoning" || + !provisionalOpenAIResponsesComputerReasoningItems.has(item) || + typeof record.id !== "string" || + linkedReasoningItems.has(record) + ) { + sanitized?.push(item); + continue; + } + if (!sanitized) sanitized = items.slice(0, index); + const { id: _id, ...withoutId } = record; + sanitized.push(withoutId as unknown as ResponseInput[number]); + } + return sanitized ?? items; +} + /** * Sanitize assistant-native Responses history for replay. * diff --git a/packages/ai/test/openai-responses-history-payload.test.ts b/packages/ai/test/openai-responses-history-payload.test.ts index ed0eda70c..629a3ae79 100644 --- a/packages/ai/test/openai-responses-history-payload.test.ts +++ b/packages/ai/test/openai-responses-history-payload.test.ts @@ -1277,6 +1277,77 @@ describe("OpenAI responses history payload", () => { ]); }); + it("preserves linked reasoning when the screenshot is a later tool result", async () => { + const context: Context = { + messages: [ + { + ...makeAssistantMessage( + [ + { + type: "reasoning", + id: "rs_split_computer_turn", + summary: [], + encrypted_content: "encrypted-split-computer-reasoning", + status: "completed", + }, + { + type: "computer_call", + id: "cu_split_computer_turn", + call_id: "call_split_computer_turn", + action: { type: "screenshot" }, + pending_safety_checks: [], + status: "completed", + }, + ], + true, + "openai", + "gpt-5.4", + ), + content: [ + { + type: "toolCall" as const, + id: "call_split_computer_turn|cu_split_computer_turn", + name: "computer", + arguments: { actions: [{ type: "screenshot" }] }, + providerMetadata: { + type: "computer" as const, + providerItemId: "cu_split_computer_turn", + actions: [{ type: "screenshot" as const }], + pendingSafetyChecks: [], + }, + }, + ], + }, + { + role: "toolResult", + toolCallId: "call_split_computer_turn|cu_split_computer_turn", + toolName: "computer", + content: [{ type: "image", data: "AAEC", mimeType: "image/png" }], + isError: false, + timestamp: Date.now(), + providerMetadata: { + type: "computer", + screenshot: { type: "computer_screenshot", image_url: "data:image/png;base64,AAEC" }, + acknowledgedSafetyChecks: [], + }, + }, + { role: "user", content: "continue after split persistence", timestamp: Date.now() }, + ], + }; + + const model = getOpenAIReasoningModel("openai", "gpt-5.4"); + const payload = (await captureResponsesPayload(model, context)) as { input?: unknown[] }; + + expect(findResponsesInputItem(payload.input, "reasoning")?.id).toBe("rs_split_computer_turn"); + expect(findResponsesInputItem(payload.input, "computer_call")).toMatchObject({ + id: "cu_split_computer_turn", + call_id: "call_split_computer_turn", + }); + expect(findResponsesInputItem(payload.input, "computer_call_output")).toMatchObject({ + call_id: "call_split_computer_turn", + }); + }); + it("backward compat: old full-snapshot payloads still replace history for legacy same-provider assistant turns", async () => { const fullSnapshotItems = [ { type: "message", role: "user", content: [{ type: "input_text", text: "Canonical user" }] },