diff --git a/docs/computer-use.md b/docs/computer-use.md index b3d705f53..5c5e5c9fd 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -16,8 +16,8 @@ computer: enabled: true backend: auto display: all - maxWidth: 1280 - maxHeight: 900 + maxWidth: 1920 + maxHeight: 1200 tools: approvalMode: write @@ -46,7 +46,7 @@ omp config set computer.enabled true omp config get computer.enabled ``` -Inside a running session, the `/computer` slash command (`/computer`, `/computer on|off|status`) toggles the tool for that session only; it never writes settings files. Backend, display, and image-size settings still snapshot when the session's desktop controller is created, so change those in config and start a new session. +Inside a running session, the `/computer` slash command (`/computer`, `/computer on|off|status`) toggles the tool for that session only; it never writes settings files. Backend, display, and image-size settings snapshot when the desktop controller is created. A model switch that crosses the Claude-family sizing boundary recreates the controller and resnapshots those settings; changing config alone does not, so start a new session after a settings change. ### Settings @@ -55,8 +55,8 @@ Inside a running session, the `/computer` slash command (`/computer`, `/computer | `computer.enabled` | `false` | Register the essential `computer` tool. | | `computer.backend` | `auto` | `auto` or `native`. Both require a native backend; neither falls back to browser or software automation. | | `computer.display` | `all` | Composite every active display, or select one numeric native display ID. | -| `computer.maxWidth` | `1280` | Maximum composite screenshot width in pixels. Larger values are capped at `1280` to preserve screenshot-relative coordinates across providers. | -| `computer.maxHeight` | `900` | Maximum composite screenshot height in pixels. Larger values are capped at `900` to preserve screenshot-relative coordinates across providers. | +| `computer.maxWidth` | `1920` | Maximum composite screenshot width in pixels. Claude-family image transports cap the effective width at `1280`; other models retain the configured limit. | +| `computer.maxHeight` | `1200` | Maximum composite screenshot height in pixels. Claude-family image transports cap the effective height at `896`; other models retain the configured limit. | The first successful result lists each display ID, name, logical rectangle, screenshot-pixel rectangle, scale, and primary status. Use one of those IDs as a string when you want a single display: @@ -107,14 +107,14 @@ A batch containing only `screenshot` and `wait` is observation-only. Any click, ## Screenshot coordinates and image mapping -Always choose coordinates from the immediately preceding successful computer result. Every coordinate action in one batch maps through that same prior frame. Do not use OS logical coordinates, CSS pixels, terminal cell positions, coordinates copied from another screenshot, or an in-batch `screenshot` marker as a new frame. +Always choose coordinates from the immediately preceding successful computer result returned by the current desktop controller. Every coordinate action in one batch maps through that same prior frame. A model switch that crosses the Claude-family sizing boundary recreates the controller and invalidates the prior frame, so capture a fresh screenshot before the next coordinate action. Do not use OS logical coordinates, CSS pixels, terminal cell positions, coordinates copied from another screenshot, or an in-batch `screenshot` marker as a new frame. For each capture, OMP: 1. Enumerates the selected native displays and their global logical rectangles. 2. Captures every selected display at native pixel density. 3. Builds one logical bounding rectangle, including negative monitor origins. -4. Chooses one render scale that preserves the desktop layout and stays within the configured and provider-safe `maxWidth` and `maxHeight` limits. +4. Chooses one render scale that preserves the desktop layout and stays within the configured `maxWidth` and `maxHeight` limits. Claude-family image transports additionally cap the effective frame at `1280×896`, below their verified resize threshold; other providers retain the configured limits. 5. Places each resized display image into the composite and returns a PNG. Each result's `displays` metadata maps both spaces: diff --git a/docs/settings.md b/docs/settings.md index e31cc1599..296b96b2b 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -487,8 +487,8 @@ computer: enabled: true backend: auto display: all - maxWidth: 1280 - maxHeight: 900 + maxWidth: 1920 + maxHeight: 1200 ``` | Key | Type | Default | Notes | @@ -496,10 +496,10 @@ computer: | `computer.enabled` | boolean | `false` | Enable the native computer tool. Natively capable OpenAI GA models use the `{ "type": "computer" }` wire form; every other function-calling model gets `computer` as a regular function tool. The `/computer` slash command toggles this for the current session only. | | `computer.backend` | enum | `auto` | `auto` or `native`; both require native capture/input and never fall back to browser automation. | | `computer.display` | string | `all` | Composite all active displays, or use a numeric display ID reported by a successful computer result. | -| `computer.maxWidth` | number | `1280` | Maximum composite screenshot width in pixels; larger values are capped at `1280` to preserve screenshot-relative coordinates across providers. | -| `computer.maxHeight` | number | `900` | Maximum composite screenshot height in pixels; larger values are capped at `900` to preserve screenshot-relative coordinates across providers. | +| `computer.maxWidth` | number | `1920` | Maximum composite screenshot width in pixels. Claude-family image transports cap the effective width at `1280` to avoid an unreported provider resize. | +| `computer.maxHeight` | number | `1200` | Maximum composite screenshot height in pixels. Claude-family image transports cap the effective height at `896` to keep the frame below the verified image budget. | -Computer settings are captured when the session tool is constructed; start a new session after changing them. Before enabling input, configure `tools.approvalMode` or `tools.approval.computer` and grant platform permissions. See [Native computer use](./computer-use.md) for supported providers, actions, coordinate mapping, displays, platform setup, safety, Files behavior, troubleshooting, and verified limitations. +Computer settings are captured when the desktop controller is created. A model switch that crosses the Claude-family sizing boundary recreates the controller and resnapshots those settings; changing config alone does not, so start a new session after a settings change. The recreated controller has no prior coordinate frame, so capture a fresh screenshot before the next pointer action. Before enabling input, configure `tools.approvalMode` or `tools.approval.computer` and grant platform permissions. See [Native computer use](./computer-use.md) for supported providers, actions, coordinate mapping, displays, platform setup, safety, Files behavior, troubleshooting, and verified limitations. ### Shell, eval, and LSP diff --git a/docs/tools/computer.md b/docs/tools/computer.md index e9cb85c13..33cc8ded2 100644 --- a/docs/tools/computer.md +++ b/docs/tools/computer.md @@ -38,10 +38,10 @@ Unlike `browser`, `computer` operates the entire visible host session. It can ac | `computer.enabled` | boolean | `false` | Register tool. | | `computer.backend` | `auto \| native` | `auto` | Both prohibit non-native fallback. | | `computer.display` | string | `all` | `all` or numeric native monitor ID. | -| `computer.maxWidth` | number | `1280` | Maximum composite PNG width; larger values are capped at `1280` for provider-safe coordinate fidelity. | -| `computer.maxHeight` | number | `900` | Maximum composite PNG height; larger values are capped at `900` for provider-safe coordinate fidelity. | +| `computer.maxWidth` | number | `1920` | Maximum composite PNG width. Claude-family image transports cap the effective width at `1280`. | +| `computer.maxHeight` | number | `1200` | Maximum composite PNG height. Claude-family image transports cap the effective height at `896`. | -Constructor snapshots these settings into one `DesktopSessionOptions`. No setting is reread per call. +The controller snapshots these settings into one `DesktopSessionOptions`. Crossing the Claude-family sizing boundary during a model switch recreates the controller, resnapshots the options, and invalidates the prior coordinate frame; the next pointer action requires a fresh screenshot. ## Inputs diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index e598f6068..8fbb4aa25 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -69,7 +69,7 @@ function buildToolResult(toolCallId: string, timestamp: number): ToolResultMessa toolName: "read", content: [ { type: "text", text: "Read image file [image/png]" }, - { type: "image", data: "ZmFrZQ==", mimeType: "image/png" }, + { type: "image", data: "ZmFrZQ==", mimeType: "image/png", detail: "original" }, ], isError: false, timestamp, @@ -119,7 +119,7 @@ describe("openai-completions convertMessages", () => { ]); }); - it("batches tool-result images after consecutive tool results", () => { + it("batches tool-result images without unsupported original-detail metadata", () => { const baseModel = getBundledModel("openai", "gpt-4o-mini") as Model<"openai-completions">; const model: Model<"openai-completions"> = { ...baseModel, @@ -159,8 +159,13 @@ describe("openai-completions convertMessages", () => { expect(imageMessage.role).toBe("user"); expect(Array.isArray(imageMessage.content)).toBe(true); - const imageParts = (imageMessage.content as Array<{ type?: string }>).filter(part => part?.type === "image_url"); - expect(imageParts.length).toBe(2); + const imageParts = ( + imageMessage.content as Array<{ type?: string; image_url?: { url: string; detail?: string } }> + ).filter(part => part?.type === "image_url"); + expect(imageParts).toEqual([ + { type: "image_url", image_url: { url: "data:image/png;base64,ZmFrZQ==" } }, + { type: "image_url", image_url: { url: "data:image/png;base64,ZmFrZQ==" } }, + ]); }); it("serializes assistant tool-call turns with string content for strict OpenAI-compatible backends", () => { const baseModel = getBundledModel("openai", "gpt-4o-mini") as Model<"openai-completions">; diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 8d311fa97..1eac59436 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -14,7 +14,7 @@ - `omp usage` revalidates the broker credential snapshot before rendering: live usage reports were previously paired with a disk-cached account list up to an hour old, so a just-completed re-login (org-less row upserted to org-scoped) rendered as a phantom duplicate until the cache expired. ### Fixed -- Fixed screenshot-relative pointer actions missing their visible targets when a function-tool provider silently downscaled a large computer screenshot; computer captures now stay within provider-safe dimensions before establishing the native coordinate frame ([#6596](https://github.com/can1357/oh-my-pi/pull/6596)). +- Fixed screenshot-relative pointer actions missing their visible targets when Claude-family image transports silently downscaled a large computer screenshot; those sessions now establish the native coordinate frame below the verified image-resize threshold without changing the public capture defaults for other models ([#6596](https://github.com/can1357/oh-my-pi/pull/6596) by [@wolfiesch](https://github.com/wolfiesch)). ## [17.1.3] - 2026-07-24 diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 7e9009f2d..8d808ad55 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -3871,23 +3871,23 @@ export const SETTINGS_SCHEMA = { "computer.maxWidth": { type: "number", - default: 1280, + default: 1920, ui: { tab: "tools", group: "Computer", label: "Computer Screenshot Width", - description: "Maximum composite screenshot width in pixels (provider-safe cap: 1280)", + description: "Maximum composite screenshot width in pixels", }, }, "computer.maxHeight": { type: "number", - default: 900, + default: 1200, ui: { tab: "tools", group: "Computer", label: "Computer Screenshot Height", - description: "Maximum composite screenshot height in pixels (provider-safe cap: 900)", + description: "Maximum composite screenshot height in pixels", }, }, diff --git a/packages/coding-agent/src/tools/computer.ts b/packages/coding-agent/src/tools/computer.ts index d504379ae..427cb263b 100644 --- a/packages/coding-agent/src/tools/computer.ts +++ b/packages/coding-agent/src/tools/computer.ts @@ -5,7 +5,8 @@ import type { AgentToolUpdateCallback, ToolApprovalDecision, } from "@oh-my-pi/pi-agent-core"; -import type { ComputerAction, ComputerSafetyCheck, ComputerToolCallMetadata } from "@oh-my-pi/pi-ai"; +import type { ComputerAction, ComputerSafetyCheck, ComputerToolCallMetadata, Model } from "@oh-my-pi/pi-ai"; +import { isClaudeModelId } from "@oh-my-pi/pi-catalog/identity"; import type { DesktopAction, DesktopCapabilities, @@ -21,10 +22,32 @@ import { type ComputerController, ComputerSupervisor, registerComputerController import type { ToolSession } from "./index"; import { ToolError, throwIfAborted } from "./tool-errors"; -// Function-tool providers may downscale larger screenshots without exposing the -// transformed dimensions. Keep the native coordinate frame below that threshold. -const PROVIDER_SAFE_MAX_WIDTH = 1280; -const PROVIDER_SAFE_MAX_HEIGHT = 900; +// Claude-family image transports resize screenshots above their image budget +// without returning transformed dimensions. Keep only those native coordinate +// frames below the empirically verified threshold; other providers retain their +// configured limits instead of inheriting a universal, unproven ceiling. +const CLAUDE_MAX_CAPTURE_WIDTH = 1280; +const CLAUDE_MAX_CAPTURE_HEIGHT = 896; + +function usesClaudeImageSizing(model: Model | undefined): boolean { + if (!model) return false; + return ( + isClaudeModelId(model.id) || + (model.requestModelId !== undefined && isClaudeModelId(model.requestModelId)) || + (typeof model.name === "string" && /^claude(?:\s|$)/i.test(model.name)) + ); +} + +function captureOptions(session: ToolSession, claudeImageSizing: boolean): DesktopSessionOptions { + const maxWidth = session.settings.get("computer.maxWidth"); + const maxHeight = session.settings.get("computer.maxHeight"); + return { + backend: session.settings.get("computer.backend"), + display: session.settings.get("computer.display"), + maxWidth: claudeImageSizing ? Math.min(maxWidth, CLAUDE_MAX_CAPTURE_WIDTH) : maxWidth, + maxHeight: claudeImageSizing ? Math.min(maxHeight, CLAUDE_MAX_CAPTURE_HEIGHT) : maxHeight, + }; +} // Desktop actions cross the N-API boundary as i32; out-of-range JS numbers // must fail closed here instead of truncating in the napi conversion. @@ -361,8 +384,10 @@ export class ComputerTool implements AgentTool void; + readonly #createController: ComputerControllerFactory; + #controller: ComputerController; + #unregisterOwner: () => void; + #usesClaudeImageSizing: boolean; #closed = false; #description?: string; @@ -370,12 +395,9 @@ export class ComputerTool implements AgentTool new ComputerSupervisor(options), ) { - this.#controller = createController({ - backend: session.settings.get("computer.backend"), - display: session.settings.get("computer.display"), - maxWidth: Math.min(session.settings.get("computer.maxWidth"), PROVIDER_SAFE_MAX_WIDTH), - maxHeight: Math.min(session.settings.get("computer.maxHeight"), PROVIDER_SAFE_MAX_HEIGHT), - }); + this.#createController = createController; + this.#usesClaudeImageSizing = usesClaudeImageSizing(session.getActiveModel?.()); + this.#controller = createController(captureOptions(session, this.#usesClaudeImageSizing)); this.#unregisterOwner = registerComputerController( session.getEvalKernelOwnerId?.() ?? undefined, this.#controller, @@ -386,6 +408,19 @@ export class ComputerTool implements AgentTool { + const nextUsesClaudeImageSizing = usesClaudeImageSizing(this.session.getActiveModel?.()); + if (nextUsesClaudeImageSizing === this.#usesClaudeImageSizing) return; + + const previous = this.#controller; + const next = this.#createController(captureOptions(this.session, nextUsesClaudeImageSizing)); + this.#unregisterOwner(); + this.#controller = next; + this.#unregisterOwner = registerComputerController(this.session.getEvalKernelOwnerId?.() ?? undefined, next); + this.#usesClaudeImageSizing = nextUsesClaudeImageSizing; + await previous.close(); + } + async execute( _toolCallId: string, params: ComputerParams, @@ -401,6 +436,8 @@ export class ComputerTool implements AgentTool 0 && context?.providerSafetyApproved !== true) { throw new ToolError("Provider safety checks require interactive approval before computer input"); } + await this.#refreshControllerForModel(); + throwIfAborted(signal); const capture = await this.#controller.execute(actions.map(toDesktopAction), signal); throwIfAborted(signal); const data = Buffer.from(capture.data).toBase64(); diff --git a/packages/coding-agent/test/tools/computer.test.ts b/packages/coding-agent/test/tools/computer.test.ts index 2de6b2b3a..62f80a139 100644 --- a/packages/coding-agent/test/tools/computer.test.ts +++ b/packages/coding-agent/test/tools/computer.test.ts @@ -179,13 +179,14 @@ class NonClosingWorker implements ComputerWorkerHandle { } } -function toolSession(settings: Settings): ToolSession { +function toolSession(settings: Settings, model?: Model): ToolSession { return { cwd: ".", hasUI: false, settings, getSessionFile: () => null, getSessionSpawns: () => null, + getActiveModel: () => model, } as ToolSession; } @@ -488,14 +489,15 @@ describe("computer tool", () => { await tool.close(); }); - it("preserves configured capture limits below the provider-safe ceiling", async () => { + it("preserves smaller configured capture limits for Claude-family transports", async () => { const settings = Settings.isolated({ "computer.enabled": true, "computer.maxWidth": 960, "computer.maxHeight": 640, }); + const model = { id: "claude-sonnet-4-6", api: "openai-completions" } as unknown as Model; let receivedOptions: DesktopSessionOptions | undefined; - const tool = new ComputerTool(toolSession(settings), options => { + const tool = new ComputerTool(toolSession(settings, model), options => { receivedOptions = options; return new FakeController(); }); @@ -504,6 +506,84 @@ describe("computer tool", () => { await tool.close(); }); + it("caps Claude-family captures without changing the public defaults", async () => { + const settings = Settings.isolated({ "computer.enabled": true }); + const model = { id: "claude-opus-4-8", api: "openai-completions" } as unknown as Model; + let receivedOptions: DesktopSessionOptions | undefined; + const tool = new ComputerTool(toolSession(settings, model), options => { + receivedOptions = options; + return new FakeController(); + }); + + expect(settings.get("computer.maxWidth")).toBe(1920); + expect(settings.get("computer.maxHeight")).toBe(1200); + expect(receivedOptions).toMatchObject({ maxWidth: 1280, maxHeight: 896 }); + await tool.close(); + }); + + it("recognizes Claude aliases without classifying every Anthropic-API model as Claude", async () => { + const settings = Settings.isolated({ "computer.enabled": true }); + const claudeModels = [ + { id: "anthropic/claude-sonnet-4-6" }, + { id: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }, + { id: "local-alias", requestModelId: "claude-opus-4-8" }, + { id: "claude-opus-4-8", requestModelId: "upstream-alias" }, + { id: "opaque-proxy-alias", name: "Claude Sonnet 4.6" }, + ] as unknown as Model[]; + for (const model of claudeModels) { + let receivedOptions: DesktopSessionOptions | undefined; + const tool = new ComputerTool(toolSession(settings, model), options => { + receivedOptions = options; + return new FakeController(); + }); + expect(receivedOptions).toMatchObject({ maxWidth: 1280, maxHeight: 896 }); + await tool.close(); + } + + for (const model of [{ id: "MiniMax-M2.5", api: "anthropic-messages" }, undefined] as Array< + Model | undefined + >) { + let receivedOptions: DesktopSessionOptions | undefined; + const tool = new ComputerTool(toolSession(settings, model), options => { + receivedOptions = options; + return new FakeController(); + }); + expect(receivedOptions).toMatchObject({ maxWidth: 1920, maxHeight: 1200 }); + await tool.close(); + } + }); + + it("recreates the controller when model switches cross the Claude sizing boundary", async () => { + const settings = Settings.isolated({ "computer.enabled": true }); + const gpt = { id: "gpt-5.6", api: "openai-responses" } as unknown as Model; + const claude = { id: "claude-sonnet-4-6", api: "openai-completions" } as unknown as Model; + let activeModel = gpt; + const session = toolSession(settings); + session.getActiveModel = () => activeModel; + const receivedOptions: DesktopSessionOptions[] = []; + const controllers: FakeController[] = []; + const tool = new ComputerTool(session, options => { + receivedOptions.push(options); + const controller = new FakeController(); + controllers.push(controller); + return controller; + }); + + activeModel = claude; + await tool.execute("call", { actions: [{ type: "screenshot" }] }); + activeModel = gpt; + await tool.execute("call", { actions: [{ type: "screenshot" }] }); + + expect(receivedOptions.map(options => [options.maxWidth, options.maxHeight])).toEqual([ + [1920, 1200], + [1280, 896], + [1920, 1200], + ]); + expect(controllers.map(controller => controller.closeCount)).toEqual([1, 1, 0]); + expect(controllers.map(controller => controller.batches.length)).toEqual([0, 1, 1]); + await tool.close(); + }); + it("uses registered native options, adapts every GA field, and returns exactly one fresh PNG with metadata", async () => { const settings = Settings.isolated({ "computer.enabled": true, @@ -512,9 +592,14 @@ describe("computer tool", () => { "computer.maxWidth": 1600, "computer.maxHeight": 1000, }); + const model = { + id: "gpt-5.6", + api: "openai-responses", + supportsComputerUse: true, + } as unknown as Model; const controller = new FakeController(); let receivedOptions: DesktopSessionOptions | undefined; - const tool = new ComputerTool(toolSession(settings), options => { + const tool = new ComputerTool(toolSession(settings, model), options => { receivedOptions = options; return controller; }); @@ -542,8 +627,8 @@ describe("computer tool", () => { expect(receivedOptions).toEqual({ backend: "native", display: "display-1", - maxWidth: 1280, - maxHeight: 900, + maxWidth: 1600, + maxHeight: 1000, }); expect(controller.batches).toEqual([ [