From aad7ee8fa3d37be2b9b7c2c246e605ad78e0923b Mon Sep 17 00:00:00 2001 From: can1357 Date: Fri, 24 Jul 2026 06:10:00 +0200 Subject: [PATCH] feat(ai): implemented lite response overrides and computer call unrolling for codex - Updated the OpenAI Codex provider to unroll native computer calls and tool outputs into standard function calls. - Added support for the `PI_CODEX_RESPONSES_LITE` environment variable override via the request transformer. - Updated documentation and tests to cover lite response resolution and native computer response unrolling. --- docs/environment-variables.md | 1 + packages/ai/CHANGELOG.md | 2 + .../src/providers/openai-codex-responses.ts | 148 ++++++++++++------ .../openai-codex/request-transformer.ts | 13 +- .../test/openai-codex-responses-lite.test.ts | 31 ++-- .../ai/test/openai-computer-contract.test.ts | 106 ++++++++++++- 6 files changed, 237 insertions(+), 64 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 30314fdf6..f5f6536da 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -213,6 +213,7 @@ OAuth host chain: `KIMI_CODE_OAUTH_HOST` → `KIMI_OAUTH_HOST` → `https://auth | ------------------------------------------ | ---------------------------------------------------- | | `PI_CODEX_DEBUG` | `1`/`true` enables Codex provider debug logging | | `PI_CODEX_WEBSOCKET` | `1`/`true` enables websocket transport preference | +| `PI_CODEX_RESPONSES_LITE` | `1`/`true` forces Responses Lite; `0`/`false` forces the standard Responses body; unset uses the model catalog default | | `PI_OPENAI_STATEFUL` | Overrides the stateful-chaining default for the platform OpenAI Responses API (`previous_response_id`, forces `store: true`): on by default against api.openai.com, off elsewhere | | `PI_CODEX_WEBSOCKET_IDLE_TIMEOUT_MS` | Positive integer override (default 300000) | | `PI_CODEX_WEBSOCKET_RETRY_BUDGET` | Non-negative integer override (default 5) | diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index effa59b40..4aeb648ab 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -5,9 +5,11 @@ ### Fixed - Fixed stateful OpenAI Responses explicit cache breakpoints being restored onto edited historical messages, ensuring full replays recompute the latest stable cache boundary. +- Fixed ChatGPT Codex standard and Lite transports rejecting or hiding native computer-use payloads by unrolling the tool definition, forced choice, `computer_call`, and `computer_call_output` into ordinary function-tool forms. ### Added - Added OpenAI Responses native computer-use transport, including batched actions and exact `computer_call`/`computer_call_output` replay with pending/acknowledged safety checks and `image_url`/`file_id` output references. Models without native support receive the same action surface as a regular function tool; provider-specific tool-choice forcing is used where supported. +- Added `PI_CODEX_RESPONSES_LITE` to override the catalog-selected Codex Responses transport for diagnostics (`1`/`true` forces Lite; `0`/`false` forces the standard body). ## [17.1.0] - 2026-07-24 diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts index 1e23bbba3..daa7cbf82 100644 --- a/packages/ai/src/providers/openai-codex-responses.ts +++ b/packages/ai/src/providers/openai-codex-responses.ts @@ -41,6 +41,7 @@ import type { Tool, ToolCall, ToolChoice, + ToolResultMessage, Usage, } from "../types"; import { @@ -1095,11 +1096,6 @@ export function normalizeCodexToolChoice( ): string | Record | undefined { if (!choice) return undefined; if (typeof choice === "string") return choice; - if (choice.type === "computer") { - return model?.supportsComputerUse === true && tools.some(tool => tool.native?.type === "computer") - ? { type: "computer" } - : undefined; - } const allowFreeform = model ? model.applyPatchToolType === "freeform" : false; const mapName = (name: string): Record | undefined => { const directTool = tools.find(tool => tool.name === name); @@ -1112,6 +1108,10 @@ export function normalizeCodexToolChoice( ? { type: "custom", name: customTool.customWireName ?? customTool.name } : { type: "function", name: offeredTool.name }; }; + if (choice.type === "computer") { + const computer = tools.find(tool => tool.native?.type === "computer"); + return computer ? { type: "function", name: computer.name } : undefined; + } if (choice.type === "function") { if ("function" in choice && choice.function?.name) { return mapName(choice.function.name); @@ -1125,6 +1125,74 @@ export function normalizeCodexToolChoice( } return undefined; } +function unrollCodexComputerItems(items: ResponseInput, supportsImageDetailOriginal: boolean): ResponseInput { + const unrolled: ResponseInput = []; + for (const item of items) { + if (item.type === "computer_call") { + const actions = item.actions ?? (item.action ? [item.action] : []); + unrolled.push({ + type: "function_call", + call_id: item.call_id, + name: "computer", + arguments: JSON.stringify({ actions }), + status: item.status, + }); + continue; + } + if (item.type === "computer_call_output") { + const image = + typeof item.output.image_url === "string" && item.output.image_url.length > 0 + ? ({ + type: "input_image", + detail: supportsImageDetailOriginal ? "original" : "auto", + image_url: item.output.image_url, + } satisfies ResponseInputContent) + : typeof item.output.file_id === "string" && item.output.file_id.length > 0 + ? ({ + type: "input_image", + detail: supportsImageDetailOriginal ? "original" : "auto", + file_id: item.output.file_id, + } satisfies ResponseInputContent) + : undefined; + unrolled.push({ + type: "function_call_output", + call_id: item.call_id, + output: image ? "(see attached image)" : "", + }); + if (image) { + unrolled.push({ + role: "user", + content: [{ type: "input_text", text: "Attached image from computer tool result:" }, image], + }); + } + continue; + } + unrolled.push(item); + } + return unrolled; +} + +function unrollCodexComputerAssistantMessage(message: AssistantMessage): AssistantMessage { + let changed = false; + const content = message.content.map(block => { + if (block.type !== "toolCall" || block.providerMetadata?.type !== "computer") return block; + changed = true; + const call: ToolCall = { + ...block, + arguments: { actions: structuredCloneJSON(block.providerMetadata.actions) }, + }; + delete call.providerMetadata; + return call; + }); + return changed ? { ...message, content } : message; +} + +function unrollCodexComputerToolResult(message: ToolResultMessage): ToolResultMessage { + if (message.providerMetadata?.type !== "computer") return message; + const result: ToolResultMessage = { ...message }; + delete result.providerMetadata; + return result; +} function getCodexServiceTierCostMultiplier( model: Pick, "id">, @@ -4011,7 +4079,6 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex // messages can be replayed as `custom_tool_call_output` rather than // `function_call_output` (OpenAI rejects mismatched pairs). const customCallIds = new Set(); - const computerCallIds = new Set(); const knownCallIds = new Set(); for (const msg of transformedMessages) { @@ -4022,23 +4089,19 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex | undefined; if (historyItems) { const redactedHistoryItems = redactSensitiveInObject(historyItems).result as Array; - for (const item of redactedHistoryItems) { - const maybe = item as { type?: string; call_id?: string }; - if (maybe.type === "custom_tool_call" && typeof maybe.call_id === "string") { - customCallIds.add(maybe.call_id); + const replayItems = unrollCodexComputerItems( + redactedHistoryItems, + model.compat.supportsImageDetailOriginal, + ); + for (const item of replayItems) { + if (item.type === "custom_tool_call") { + customCallIds.add(item.call_id); } - if (maybe.type === "computer_call" && typeof maybe.call_id === "string") { - computerCallIds.add(maybe.call_id); + if ((item.type === "function_call" || item.type === "custom_tool_call") && item.call_id) { + knownCallIds.add(item.call_id); } - if ( - (maybe.type === "function_call" || - maybe.type === "custom_tool_call" || - maybe.type === "computer_call") && - typeof maybe.call_id === "string" - ) - knownCallIds.add(maybe.call_id); } - messages.push(...redactedHistoryItems); + messages.push(...replayItems); msgIndex += 1; continue; } @@ -4064,26 +4127,22 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex if (historyItems) { const sanitizedHistoryItems = sanitizeOpenAIResponsesAssistantHistoryItemsForReplay(historyItems); if (sanitizedHistoryItems) { - for (const item of sanitizedHistoryItems) { - const maybe = item as { type?: string; call_id?: string }; - if (maybe.type === "custom_tool_call" && typeof maybe.call_id === "string") { - customCallIds.add(maybe.call_id); + const replayItems = unrollCodexComputerItems( + sanitizedHistoryItems, + model.compat.supportsImageDetailOriginal, + ); + for (const item of replayItems) { + if (item.type === "custom_tool_call") { + customCallIds.add(item.call_id); } - if (maybe.type === "computer_call" && typeof maybe.call_id === "string") { - computerCallIds.add(maybe.call_id); + if ((item.type === "function_call" || item.type === "custom_tool_call") && item.call_id) { + knownCallIds.add(item.call_id); } - if ( - (maybe.type === "function_call" || - maybe.type === "custom_tool_call" || - maybe.type === "computer_call") && - typeof maybe.call_id === "string" - ) - knownCallIds.add(maybe.call_id); } if (providerPayload?.dt) { - messages.push(...sanitizedHistoryItems); + messages.push(...replayItems); } else { - messages.splice(0, messages.length, ...sanitizedHistoryItems); + messages.splice(0, messages.length, ...replayItems); // Keep customCallIds from the pre-splice state since historyItems may re-introduce them. } msgIndex += 1; @@ -4093,7 +4152,7 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex } const convertedOutputItems = convertResponsesAssistantMessage( - msg as AssistantMessage, + unrollCodexComputerAssistantMessage(msg as AssistantMessage), model, msgIndex, knownCallIds, @@ -4101,8 +4160,6 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex customCallIds, false, true, - undefined, - computerCallIds, ); const outputItems = suppressHiddenEmptyFallback ? sanitizeOpenAIResponsesAssistantFallbackItemsForReplay(convertedOutputItems) @@ -4117,14 +4174,13 @@ function convertMessages(model: Model<"openai-codex-responses">, context: Contex if (msg.role === "toolResult") { appendResponsesToolResultMessages( messages, - msg, + unrollCodexComputerToolResult(msg), model, false, model.compat.supportsImageDetailOriginal, knownCallIds, customCallIds, true, - computerCallIds, ); } @@ -4165,8 +4221,8 @@ type CodexToolPayload = name: string; description: string; format: { type: "grammar"; syntax: "lark" | "regex"; definition: string }; - } - | { type: "computer"; name?: never }; + }; + /** @internal Exported for tests. */ export function convertOpenAICodexResponsesTools( tools: Tool[], @@ -4175,12 +4231,8 @@ export function convertOpenAICodexResponsesTools( const allowFreeform = model.applyPatchToolType === "freeform"; const payloads: CodexToolPayload[] = []; for (const tool of tools) { - if (tool.native?.type === "computer" && model.supportsComputerUse === true) { - payloads.push({ type: "computer" }); - continue; - } - // Models without native computer support fall through and receive the - // tool as a plain function tool so function-calling models can drive it. + // The ChatGPT Codex endpoints reject the native `{ type: "computer" }` + // shape, so both standard and Lite transports expose it as a function. if (allowFreeform && tool.customFormat) { payloads.push({ type: "custom", diff --git a/packages/ai/src/providers/openai-codex/request-transformer.ts b/packages/ai/src/providers/openai-codex/request-transformer.ts index a6419f4d0..6bb4c1937 100644 --- a/packages/ai/src/providers/openai-codex/request-transformer.ts +++ b/packages/ai/src/providers/openai-codex/request-transformer.ts @@ -1,6 +1,7 @@ import { Effort } from "@oh-my-pi/pi-catalog/effort"; import { supportsAllTurnsReasoningContext, supportsCodexReasoningSummary } from "@oh-my-pi/pi-catalog/identity"; import { requireSupportedEffort } from "@oh-my-pi/pi-catalog/model-thinking"; +import { $env } from "@oh-my-pi/pi-utils"; import type { Model } from "../../types"; import { mapOpenAIReasoningEffort } from "../openai-shared"; @@ -92,14 +93,20 @@ export interface RequestBody { /** * Resolve whether a Codex request uses the Responses Lite transport: an - * explicit option wins, otherwise the model's catalog flag (codex-rs - * `model_info.use_responses_lite`) decides. + * explicit option wins, then the `PI_CODEX_RESPONSES_LITE` env override + * (`1`/`true` forces Lite, `0`/`false` forces the full Responses body), + * otherwise the model's catalog flag (codex-rs `model_info.use_responses_lite`) + * decides. */ export function resolveCodexResponsesLite( model: Model<"openai-codex-responses">, requested: boolean | undefined, ): boolean { - return requested ?? model.useResponsesLite === true; + if (requested !== undefined) return requested; + const env = $env.PI_CODEX_RESPONSES_LITE?.trim().toLowerCase(); + if (env === "1" || env === "true") return true; + if (env === "0" || env === "false") return false; + return model.useResponsesLite === true; } /** diff --git a/packages/ai/test/openai-codex-responses-lite.test.ts b/packages/ai/test/openai-codex-responses-lite.test.ts index ee6bd4bf7..b1787d40a 100644 --- a/packages/ai/test/openai-codex-responses-lite.test.ts +++ b/packages/ai/test/openai-codex-responses-lite.test.ts @@ -394,17 +394,30 @@ describe("openai-codex Responses Lite input shaping", () => { }); }); - it("defaults lite from the model useResponsesLite flag and honors explicit opt-out", async () => { + it("resolves Lite from explicit options, the environment, then the model default", async () => { + const previous = Bun.env.PI_CODEX_RESPONSES_LITE; const model = createCodexModel("gpt-5.6-terra", { useResponsesLite: true }); - const lite = await transformRequestBody({ model: model.id, instructions: "sys" }, model, {}); - expect(lite.instructions).toBeUndefined(); - expect(lite.input?.[0]?.type).toBe("additional_tools"); + try { + delete Bun.env.PI_CODEX_RESPONSES_LITE; + const modelDefault = await transformRequestBody({ model: model.id, instructions: "sys" }, model, {}); + expect(modelDefault.instructions).toBeUndefined(); + expect(modelDefault.input?.[0]?.type).toBe("additional_tools"); - const optOut = await transformRequestBody({ model: model.id, instructions: "sys" }, model, { - responsesLite: false, - }); - expect(optOut.instructions).toBe("sys"); - expect(optOut.input?.some(item => item.type === "additional_tools")).toBe(false); + Bun.env.PI_CODEX_RESPONSES_LITE = "false"; + const envOptOut = await transformRequestBody({ model: model.id, instructions: "sys" }, model, {}); + expect(envOptOut.instructions).toBe("sys"); + expect(envOptOut.input?.some(item => item.type === "additional_tools")).toBe(false); + + Bun.env.PI_CODEX_RESPONSES_LITE = "true"; + const explicitOptOut = await transformRequestBody({ model: model.id, instructions: "sys" }, model, { + responsesLite: false, + }); + expect(explicitOptOut.instructions).toBe("sys"); + expect(explicitOptOut.input?.some(item => item.type === "additional_tools")).toBe(false); + } finally { + if (previous === undefined) delete Bun.env.PI_CODEX_RESPONSES_LITE; + else Bun.env.PI_CODEX_RESPONSES_LITE = previous; + } }); }); diff --git a/packages/ai/test/openai-computer-contract.test.ts b/packages/ai/test/openai-computer-contract.test.ts index 81e6a5d61..51d08cc8e 100644 --- a/packages/ai/test/openai-computer-contract.test.ts +++ b/packages/ai/test/openai-computer-contract.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test"; import { + convertCodexResponsesMessages, convertOpenAICodexResponsesTools, normalizeCodexToolChoice, } from "@oh-my-pi/pi-ai/providers/openai-codex-responses"; @@ -119,7 +120,10 @@ describe("OpenAI GA computer contract", () => { expect( normalizeCodexToolChoice({ type: "function", name: "computer" }, [computerTool], codexUnsupported), ).toEqual({ type: "function", name: "computer" }); - expect(normalizeCodexToolChoice({ type: "computer" }, [computerTool], codexUnsupported)).toBeUndefined(); + expect(normalizeCodexToolChoice({ type: "computer" }, [computerTool], codexUnsupported)).toEqual({ + type: "function", + name: "computer", + }); }); test("parses batched streamed actions, stable item id, and safety checks", async () => { @@ -399,10 +403,104 @@ describe("OpenAI GA computer contract", () => { expect(replay.some(item => item.type === "function_call" && item.call_id === "call_new")).toBe(true); }); - test("uses the same native shape and forced choice for Codex", () => { + test("unrolls the native computer tool and forced choice for Codex", () => { const codex = model("openai-codex-responses"); - expect(convertOpenAICodexResponsesTools([computerTool], codex)).toEqual([{ type: "computer" }]); - expect(normalizeCodexToolChoice({ type: "computer" }, [computerTool], codex)).toEqual({ type: "computer" }); + expect(convertOpenAICodexResponsesTools([computerTool], codex)).toMatchObject([ + { type: "function", name: "computer", description: "Control the host desktop" }, + ]); + expect(normalizeCodexToolChoice({ type: "computer" }, [computerTool], codex)).toEqual({ + type: "function", + name: "computer", + }); expect(normalizeCodexToolChoice({ type: "computer" }, [], codex)).toBeUndefined(); }); + + test("unrolls native computer response history for Codex replay", () => { + const codex = model("openai-codex-responses"); + const previous = { + ...assistant([]), + api: "openai-codex-responses" as const, + provider: "openai-codex", + model: codex.id, + providerPayload: { + type: "openaiResponsesHistory" as const, + provider: "openai-codex", + dt: true, + items: [ + { + type: "computer_call", + id: "item_codex_computer", + call_id: "call_codex_computer", + actions: [{ type: "screenshot" }], + pending_safety_checks: [], + status: "completed", + }, + { + type: "computer_call_output", + call_id: "call_codex_computer", + output: { type: "computer_screenshot", file_id: "file_codex_computer" }, + acknowledged_safety_checks: [], + }, + ], + }, + }; + const replay = convertCodexResponsesMessages(codex, { messages: [previous] }); + expect(replay.some(item => item.type === "computer_call" || item.type === "computer_call_output")).toBe(false); + const call = replay.find(item => item.type === "function_call" && item.call_id === "call_codex_computer"); + expect(call).toMatchObject({ type: "function_call", name: "computer" }); + if (call?.type !== "function_call") throw new Error("Expected unrolled computer function call"); + expect(JSON.parse(call.arguments)).toEqual({ actions: [{ type: "screenshot" }] }); + expect(replay.some(item => item.type === "function_call_output" && item.call_id === "call_codex_computer")).toBe( + true, + ); + expect(JSON.stringify(replay)).toContain("file_codex_computer"); + }); + + test("unrolls internal computer calls and screenshot results for Codex replay", () => { + const codex = model("openai-codex-responses"); + const call = { + ...assistant([ + { + type: "toolCall" as const, + id: "call_internal_computer|item_internal_computer", + name: "computer", + arguments: {}, + providerMetadata: { + type: "computer" as const, + providerItemId: "item_internal_computer", + actions: [{ type: "screenshot" as const }], + pendingSafetyChecks: [], + }, + }, + ]), + api: "openai-codex-responses" as const, + provider: "openai-codex", + model: codex.id, + }; + const result: ToolResultMessage = { + role: "toolResult", + toolCallId: "call_internal_computer|item_internal_computer", + toolName: "computer", + content: [{ type: "image", data: "cG5n", mimeType: "image/png", detail: "original" }], + isError: false, + timestamp: 2, + providerMetadata: { + type: "computer", + screenshot: { type: "computer_screenshot", image_url: "data:image/png;base64,cG5n" }, + acknowledgedSafetyChecks: [], + }, + }; + const replay = convertCodexResponsesMessages(codex, { messages: [call, result] }); + expect(replay.some(item => item.type === "computer_call" || item.type === "computer_call_output")).toBe(false); + const functionCall = replay.find( + item => item.type === "function_call" && item.call_id === "call_internal_computer", + ); + expect(functionCall).toMatchObject({ type: "function_call", name: "computer" }); + if (functionCall?.type !== "function_call") throw new Error("Expected unrolled computer function call"); + expect(JSON.parse(functionCall.arguments)).toEqual({ actions: [{ type: "screenshot" }] }); + expect( + replay.some(item => item.type === "function_call_output" && item.call_id === "call_internal_computer"), + ).toBe(true); + expect(JSON.stringify(replay)).toContain("data:image/png;base64,cG5n"); + }); });