fix(ai): finalize computer reasoning after replay repair

2026-07-27 14:26:13 -06:00
parent 98114a1a46
commit dd65086b79
3 changed files with 154 additions and 31 deletions
@@ -69,6 +69,7 @@ import {
 	sanitizeOpenAIResponsesAssistantFallbackItemsForReplay,
 	sanitizeOpenAIResponsesAssistantHistoryItemsForReplay,
 	sanitizeOpenAIResponsesHistoryItemsForReplay,
+	stripUnpairedOpenAIResponsesComputerReasoningIdsForReplay,
 } from "../utils";
 import {
 	clearStreamingPartialJson,
@@ -1759,7 +1760,8 @@ export function buildResponsesInput<TApi extends Api>(options: BuildResponsesInp
 	}

 	const withRepairedOutputs = options.repairOrphanOutputs ? repairOrphanResponsesToolOutputs(messages) : messages;
-	return repairOrphanResponsesToolCalls(withRepairedOutputs);
+	const withRepairedCalls = repairOrphanResponsesToolCalls(withRepairedOutputs);
+	return stripUnpairedOpenAIResponsesComputerReasoningIdsForReplay(withRepairedCalls);
 }

 type ResponsesReplayAssistantMessage = Omit<ResponseOutputMessage, "id"> & { id?: string };
@@ -125,52 +125,102 @@ function isOpenAIResponsesClientInputBoundary(item: Record<string, unknown>): bo
 	}
 }

+function collectOpenAIResponsesComputerLinkedReasoningItems(
+	items: Array<Record<string, unknown>>,
+	requireLaterOutput: boolean,
+): Set<Record<string, unknown>> {
+	let computerCallsWithLaterOutputs: Set<Record<string, unknown>> | undefined;
+	if (requireLaterOutput) {
+		computerCallsWithLaterOutputs = new Set();
+		const laterComputerOutputCallIds = new Set<string>();
+		for (let index = items.length - 1; index >= 0; index--) {
+			const item = items[index]!;
+			if (item.type === "computer_call_output" && typeof item.call_id === "string") {
+				laterComputerOutputCallIds.add(item.call_id);
+			} else if (
+				item.type === "computer_call" &&
+				typeof item.id === "string" &&
+				typeof item.call_id === "string" &&
+				laterComputerOutputCallIds.has(item.call_id)
+			) {
+				computerCallsWithLaterOutputs.add(item);
+			}
+		}
+	}
+
+	const computerLinkedReasoningItems = new Set<Record<string, unknown>>();
+	const responseReasoningItems: Array<Record<string, unknown>> = [];
+	for (const item of items) {
+		if (isOpenAIResponsesClientInputBoundary(item)) {
+			responseReasoningItems.length = 0;
+		} else if (item.type === "reasoning") {
+			responseReasoningItems.push(item);
+		} else if (
+			item.type === "computer_call" &&
+			typeof item.id === "string" &&
+			(!computerCallsWithLaterOutputs || computerCallsWithLaterOutputs.has(item))
+		) {
+			for (const reasoningItem of responseReasoningItems) computerLinkedReasoningItems.add(reasoningItem);
+		}
+	}
+	return computerLinkedReasoningItems;
+}
+
+const provisionalOpenAIResponsesComputerReasoningItems = new WeakSet<object>();
+
 export function sanitizeOpenAIResponsesHistoryItemsForReplay(
 	items: Array<Record<string, unknown>>,
 	options: OpenAIResponsesReplaySanitizeOptions = {},
 ): ResponseInput {
 	const normalizedCallIds = new Map<string, string>();
 	const supportsImageDetailOriginal = options.supportsImageDetailOriginal !== false;
-	const supportsComputerUse = options.supportsComputerUse !== false;
-	// Stateless native computer history is an atomic Responses chain: replaying
-	// a `computer_call` ID requires the reasoning item IDs from that response.
-	const computerLinkedReasoningItems = new Set<Record<string, unknown>>();
-	const responseReasoningItems: Array<Record<string, unknown>> = [];
-	const computerCallsWithLaterOutputs = new Set<Record<string, unknown>>();
-	const laterComputerOutputCallIds = new Set<string>();
-	for (let index = items.length - 1; index >= 0; index--) {
-		const item = items[index]!;
-		if (item.type === "computer_call_output" && typeof item.call_id === "string") {
-			laterComputerOutputCallIds.add(item.call_id);
-		} else if (
-			item.type === "computer_call" &&
-			typeof item.id === "string" &&
-			typeof item.call_id === "string" &&
-			laterComputerOutputCallIds.has(item.call_id)
-		) {
-			computerCallsWithLaterOutputs.add(item);
-		}
-	}
-	for (const item of items) {
-		if (isOpenAIResponsesClientInputBoundary(item)) {
-			responseReasoningItems.length = 0;
-		} else if (item.type === "reasoning") {
-			responseReasoningItems.push(item);
-		} else if (supportsComputerUse && computerCallsWithLaterOutputs.has(item)) {
-			for (const reasoningItem of responseReasoningItems) computerLinkedReasoningItems.add(reasoningItem);
-		}
-	}
+	const computerLinkedReasoningItems =
+		options.supportsComputerUse === false
+			? undefined
+			: collectOpenAIResponsesComputerLinkedReasoningItems(items, false);
 	return items.flatMap(item => {
+		const preserveForComputer = computerLinkedReasoningItems?.has(item) === true;
 		const sanitized = sanitizeOpenAIResponsesHistoryItemForReplay(
 			item,
 			normalizedCallIds,
 			supportsImageDetailOriginal,
-			computerLinkedReasoningItems.has(item),
+			preserveForComputer,
 		);
+		if (preserveForComputer && sanitized?.type === "reasoning") {
+			provisionalOpenAIResponsesComputerReasoningItems.add(sanitized);
+		}
 		return sanitized ? [sanitized] : [];
 	});
 }

+/**
+ * Finalize provisional native-computer reasoning IDs after the complete
+ * Responses input has been rebuilt, model-adapted, and orphan-repaired.
+ */
+export function stripUnpairedOpenAIResponsesComputerReasoningIdsForReplay(items: ResponseInput): ResponseInput {
+	const records = items as unknown as Array<Record<string, unknown>>;
+	const linkedReasoningItems = collectOpenAIResponsesComputerLinkedReasoningItems(records, true);
+	let sanitized: ResponseInput | undefined;
+
+	for (let index = 0; index < items.length; index++) {
+		const item = items[index]!;
+		const record = records[index]!;
+		if (
+			item.type !== "reasoning" ||
+			!provisionalOpenAIResponsesComputerReasoningItems.has(item) ||
+			typeof record.id !== "string" ||
+			linkedReasoningItems.has(record)
+		) {
+			sanitized?.push(item);
+			continue;
+		}
+		if (!sanitized) sanitized = items.slice(0, index);
+		const { id: _id, ...withoutId } = record;
+		sanitized.push(withoutId as unknown as ResponseInput[number]);
+	}
+	return sanitized ?? items;
+}
+
 /**
 * Sanitize assistant-native Responses history for replay.
 *
@@ -1277,6 +1277,77 @@ describe("OpenAI responses history payload", () => {
 		]);
 	});

+	it("preserves linked reasoning when the screenshot is a later tool result", async () => {
+		const context: Context = {
+			messages: [
+				{
+					...makeAssistantMessage(
+						[
+							{
+								type: "reasoning",
+								id: "rs_split_computer_turn",
+								summary: [],
+								encrypted_content: "encrypted-split-computer-reasoning",
+								status: "completed",
+							},
+							{
+								type: "computer_call",
+								id: "cu_split_computer_turn",
+								call_id: "call_split_computer_turn",
+								action: { type: "screenshot" },
+								pending_safety_checks: [],
+								status: "completed",
+							},
+						],
+						true,
+						"openai",
+						"gpt-5.4",
+					),
+					content: [
+						{
+							type: "toolCall" as const,
+							id: "call_split_computer_turn|cu_split_computer_turn",
+							name: "computer",
+							arguments: { actions: [{ type: "screenshot" }] },
+							providerMetadata: {
+								type: "computer" as const,
+								providerItemId: "cu_split_computer_turn",
+								actions: [{ type: "screenshot" as const }],
+								pendingSafetyChecks: [],
+							},
+						},
+					],
+				},
+				{
+					role: "toolResult",
+					toolCallId: "call_split_computer_turn|cu_split_computer_turn",
+					toolName: "computer",
+					content: [{ type: "image", data: "AAEC", mimeType: "image/png" }],
+					isError: false,
+					timestamp: Date.now(),
+					providerMetadata: {
+						type: "computer",
+						screenshot: { type: "computer_screenshot", image_url: "data:image/png;base64,AAEC" },
+						acknowledgedSafetyChecks: [],
+					},
+				},
+				{ role: "user", content: "continue after split persistence", timestamp: Date.now() },
+			],
+		};
+
+		const model = getOpenAIReasoningModel("openai", "gpt-5.4");
+		const payload = (await captureResponsesPayload(model, context)) as { input?: unknown[] };
+
+		expect(findResponsesInputItem(payload.input, "reasoning")?.id).toBe("rs_split_computer_turn");
+		expect(findResponsesInputItem(payload.input, "computer_call")).toMatchObject({
+			id: "cu_split_computer_turn",
+			call_id: "call_split_computer_turn",
+		});
+		expect(findResponsesInputItem(payload.input, "computer_call_output")).toMatchObject({
+			call_id: "call_split_computer_turn",
+		});
+	});
+
 	it("backward compat: old full-snapshot payloads still replace history for legacy same-provider assistant turns", async () => {
 		const fullSnapshotItems = [
 			{ type: "message", role: "user", content: [{ type: "input_text", text: "Canonical user" }] },