diff --git a/packages/coding-agent/src/cli.ts b/packages/coding-agent/src/cli.ts index c1194fb9d..ba32da564 100755 --- a/packages/coding-agent/src/cli.ts +++ b/packages/coding-agent/src/cli.ts @@ -28,8 +28,8 @@ import { declareWorkerHostEntry, installWorkerInbox } from "@oh-my-pi/pi-utils/w import { installProfileAlias, resolveProfileAliasCommandFromProcess } from "./cli/profile-alias"; import { extractProfileFlags } from "./cli/profile-bootstrap"; import { DAEMON_BROKER_WORKER_ARG } from "./launch/protocol"; -import { startComputerWorker } from "./tools/computer/worker-entry"; import { smokeTestComputerWorker } from "./tools/computer/supervisor"; +import { startComputerWorker } from "./tools/computer/worker-entry"; if (Bun.semver.order(Bun.version, MIN_BUN_VERSION) < 0) { process.stderr.write( diff --git a/packages/coding-agent/src/prompts/system/system-prompt.md b/packages/coding-agent/src/prompts/system/system-prompt.md index c865074d2..24a210409 100644 --- a/packages/coding-agent/src/prompts/system/system-prompt.md +++ b/packages/coding-agent/src/prompts/system/system-prompt.md @@ -79,6 +79,15 @@ Special URLs for internal resources; with most FS/bash tools they auto-resolve t {{/if}} {{/if}} +{{#has tools "computer"}} +# Computer Use +The `{{toolRefs.computer}}` tool is explicitly enabled and available in this session. +- MUST use `{{toolRefs.computer}}` for requests to view or control host desktop applications. +- NEVER claim Computer Use is unavailable while `{{toolRefs.computer}}` appears in the tool inventory. +- While fulfilling host-desktop requests, NEVER substitute Browser, Bash, Eval, AppleScript, accessibility commands, or `screencapture` unless the user explicitly requests that mechanism or `{{toolRefs.computer}}` returns an error. +- Inspect the fresh screenshot returned by every successful `{{toolRefs.computer}}` call before choosing the next action. +{{/has}} + {{#if xdevTools.length}} # xd:// Tool Devices Additional tools are mounted as virtual devices, executed by writing a JSON args object as `content` to `xd://` via `{{toolRefs.write}}`. diff --git a/packages/coding-agent/src/session/session-tools.ts b/packages/coding-agent/src/session/session-tools.ts index fe0eab70a..7ef77e430 100644 --- a/packages/coding-agent/src/session/session-tools.ts +++ b/packages/coding-agent/src/session/session-tools.ts @@ -17,6 +17,7 @@ import type { MemoryBackendStartOptions } from "../memory-backend/types"; import xdevMountNoticePrompt from "../prompts/system/xdev-mount-notice.md" with { type: "text" }; import { usesCodexTaskPrompt } from "../task/prompt-policy"; import { isMCPToolName, normalizeToolNames } from "../tools/builtin-names"; +import { computerExposureMode } from "../tools/computer/exposure"; import { wrapToolWithMetaNotice } from "../tools/output-meta"; import { ToolAbortError, ToolError } from "../tools/tool-errors"; import { isMountableUnderXdev, type XdevRegistry } from "../tools/xdev"; @@ -294,6 +295,16 @@ export class SessionTools { return usesCodexTaskPrompt(model) ? "task-policy:gpt-5.6" : "task-policy:default"; } + #logComputerState(message: string, enabled: boolean): void { + const model = this.#host.model(); + logger.debug(message, { + enabled, + active: this.getEnabledToolNames().includes("computer"), + model: model ? formatModelString(model) : undefined, + exposure: computerExposureMode(model), + }); + } + /** Rebuilds model-dependent tool prompts after a model change. */ async syncAfterModelChange(previousEditMode: EditMode): Promise { const currentEditMode = this.resolveActiveEditMode(); @@ -303,6 +314,20 @@ export class SessionTools { if (editModeChanged || modelChanged) { await this.refreshBaseSystemPrompt(); } + const computerExpected = this.#host.settings.get("computer.enabled"); + const computerActive = this.getEnabledToolNames().includes("computer"); + if (computerExpected && !computerActive) { + const model = this.#host.model(); + const modelName = model ? formatModelString(model) : "the current model"; + logger.warn("Enabled computer tool missing after model change", { model: modelName }); + this.#host.emitNotice( + "warning", + `Computer use remains enabled, but the computer tool is unavailable to ${modelName}.`, + "computer", + ); + } else if (computerExpected) { + this.#logComputerState("Computer tool retained after model change", true); + } } /** Enabled MCP tools in their current presentation partition. */ @@ -713,16 +738,24 @@ export class SessionTools { * tool (e.g. restricted child sessions have no factory). */ async setComputerToolEnabled(enabled: boolean): Promise { + const logState = (): void => this.#logComputerState("Computer tool state changed", enabled); const active = this.getEnabledToolNames(); if (!enabled) { if (active.includes("computer")) { await this.applyActiveToolsByName(active.filter(name => name !== "computer")); } + logState(); return true; } if (!this.#toolRegistry.has("computer")) { const tool = await this.#createComputerTool?.(); - if (tool?.name !== "computer") return false; + if (tool?.name !== "computer") { + const model = this.#host.model(); + logger.warn("Computer tool could not be created", { + model: model ? formatModelString(model) : undefined, + }); + return false; + } const wrapped = this.#wrapRuntimeTool(tool); this.#toolRegistry.set(wrapped.name, wrapped); this.#builtInToolNames.add(wrapped.name); @@ -730,6 +763,7 @@ export class SessionTools { if (!active.includes("computer")) { await this.applyActiveToolsByName([...active, "computer"]); } + logState(); return true; } diff --git a/packages/coding-agent/src/slash-commands/builtin-registry.ts b/packages/coding-agent/src/slash-commands/builtin-registry.ts index 40b219a44..b1c9d5dd1 100644 --- a/packages/coding-agent/src/slash-commands/builtin-registry.ts +++ b/packages/coding-agent/src/slash-commands/builtin-registry.ts @@ -7,7 +7,12 @@ import { APP_NAME, getProjectDir, setProjectDir } from "@oh-my-pi/pi-utils"; import { reset as resetCapabilities } from "../capability"; import { COLLAB_GUEST_ALLOWED_COMMANDS, CollabGuestLink } from "../collab/guest"; import { CollabHost } from "../collab/host"; -import { expandRoleAlias, getModelMatchPreferences, resolveCliModel } from "../config/model-resolver"; +import { + expandRoleAlias, + formatModelString, + getModelMatchPreferences, + resolveCliModel, +} from "../config/model-resolver"; import { applyProviderGlobalsFromSettings } from "../config/provider-globals"; import type { SettingPath, SettingValue } from "../config/settings"; import { settings } from "../config/settings"; @@ -37,6 +42,7 @@ import type { SessionOAuthAccountList } from "../session/agent-session-types"; import { COMPACT_MODES, parseCompactArgs } from "../session/compact-modes"; import { resolveResumableSession } from "../session/session-listing"; import { formatShakeSummary, type ShakeMode } from "../session/shake-types"; +import { computerExposureMode } from "../tools/computer/exposure"; import { expandTilde, resolveToCwd } from "../tools/path-utils"; import { urlHyperlinkAlways } from "../tui"; import { @@ -90,9 +96,23 @@ function formatFastModeStatus(session: AgentSession): string { return session.isFastModeEnabled() ? "on" : "off"; } -/** `/computer status` label for the session-effective `computer.enabled` value. */ +/** Detailed, session-effective `/computer status` diagnostics. */ function formatComputerUseStatus(session: AgentSession): string { - return session.settings.get("computer.enabled") ? "on" : "off"; + const enabled = session.settings.get("computer.enabled"); + const active = session.getEnabledToolNames().includes("computer"); + const model = session.model; + const modelName = model ? formatModelString(model) : "none"; + const exposure = !enabled || !active ? "not exposed" : computerExposureMode(model); + const toolState = active ? "active" : enabled ? "unavailable" : "inactive"; + return [ + `Computer use: ${enabled ? "enabled" : "disabled"}`, + `tool: ${toolState}`, + `backend: ${session.settings.get("computer.backend")}`, + `display: ${session.settings.get("computer.display")}`, + `capture: ${session.settings.get("computer.maxWidth")}×${session.settings.get("computer.maxHeight")}`, + `model: ${modelName}`, + `exposure: ${exposure}`, + ].join(" · "); } /** @@ -107,7 +127,9 @@ async function applyComputerUseToggle(session: AgentSession, enable: boolean): P return "Computer use is unavailable in this session."; } session.settings.override("computer.enabled", enable); - return `Computer use ${enable ? "enabled" : "disabled"} for this session.`; + return enable + ? `Computer use enabled for this session. ${formatComputerUseStatus(session)}` + : "Computer use disabled for this session."; } const AUTOCOMPLETE_DETAIL_LIMIT = 48; @@ -573,11 +595,12 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray = [ { name: "status", description: "Show computer use status" }, ], allowArgs: true, - getTuiAutocompleteDescription: runtime => `Computer: ${formatComputerUseStatus(runtime.ctx.session)}`, + getTuiAutocompleteDescription: runtime => + `Computer: ${runtime.ctx.session.settings.get("computer.enabled") ? "on" : "off"}`, handle: async (command, runtime) => { const arg = command.args.trim().toLowerCase(); if (arg === "status") { - await runtime.output(`Computer use is ${formatComputerUseStatus(runtime.session)}.`); + await runtime.output(formatComputerUseStatus(runtime.session)); return commandConsumed(); } if (!arg || arg === "toggle" || arg === "on" || arg === "off") { @@ -590,7 +613,7 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray = [ handleTui: async (command, runtime) => { const arg = command.args.trim().toLowerCase(); if (arg === "status") { - runtime.ctx.showStatus(`Computer use is ${formatComputerUseStatus(runtime.ctx.session)}.`); + runtime.ctx.showStatus(formatComputerUseStatus(runtime.ctx.session)); runtime.ctx.editor.setText(""); return; } diff --git a/packages/coding-agent/src/tools/computer/exposure.ts b/packages/coding-agent/src/tools/computer/exposure.ts new file mode 100644 index 000000000..419b80b0f --- /dev/null +++ b/packages/coding-agent/src/tools/computer/exposure.ts @@ -0,0 +1,9 @@ +import type { Model } from "@oh-my-pi/pi-ai"; + +export type ComputerExposureMode = "native" | "function" | "unavailable"; + +/** Match the provider transport's effective Computer Use tool representation. */ +export function computerExposureMode(model: Model | undefined): ComputerExposureMode { + if (!model) return "unavailable"; + return model.supportsComputerUse === true ? "native" : "function"; +} diff --git a/packages/coding-agent/src/tools/computer/supervisor.ts b/packages/coding-agent/src/tools/computer/supervisor.ts index 69f23e2d0..c5963877c 100644 --- a/packages/coding-agent/src/tools/computer/supervisor.ts +++ b/packages/coding-agent/src/tools/computer/supervisor.ts @@ -1,5 +1,5 @@ import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives"; -import { withTimeout, workerHostEntry } from "@oh-my-pi/pi-utils"; +import { logger, withTimeout, workerHostEntry } from "@oh-my-pi/pi-utils"; import { ToolAbortError, ToolError } from "../tool-errors"; import { COMPUTER_WORKER_ARG, @@ -138,11 +138,23 @@ export class ComputerSupervisor implements ComputerController { if (this.#startPromise) return this.#startPromise; const ready = Promise.withResolvers(); try { + logger.debug("Starting native computer worker", { + backend: this.options.backend, + display: this.options.display, + maxWidth: this.options.maxWidth, + maxHeight: this.options.maxHeight, + }); const worker = this.createWorker(); this.#worker = worker; this.#unsubscribeMessage = worker.onMessage(message => { if (message.type === "ready") { this.#capabilities = message.capabilities; + logger.debug("Native computer worker ready", { + backend: message.capabilities.backend, + capturePermission: message.capabilities.capturePermission, + inputPermission: message.capabilities.inputPermission, + displayCount: message.capabilities.displayCount, + }); ready.resolve(); return; } @@ -151,10 +163,22 @@ export class ComputerSupervisor implements ComputerController { const pending = this.#pending.get(message.id); this.#pending.delete(message.id); pending?.resolve(message.capture); + logger.debug("Native computer capture completed", { + requestId: message.id, + backend: message.capture.backend, + capturePermission: message.capture.capturePermission, + inputPermission: message.capture.inputPermission, + width: message.capture.width, + height: message.capture.height, + }); return; } if (message.type === "error") { const error = workerError(message.error); + logger.warn("Native computer worker request failed", { + requestId: message.id, + errorName: message.error.name, + }); if (message.id) { const pending = this.#pending.get(message.id); this.#pending.delete(message.id); @@ -184,6 +208,9 @@ export class ComputerSupervisor implements ComputerController { } async #terminate(reason: unknown): Promise { + logger.debug("Terminating native computer worker", { + reason: reason instanceof Error ? reason.name : typeof reason, + }); const worker = this.#worker; this.#worker = undefined; this.#startPromise = undefined; diff --git a/packages/coding-agent/test/sdk-computer-tool-toggle.test.ts b/packages/coding-agent/test/sdk-computer-tool-toggle.test.ts index cff545486..9fcd1e24c 100644 --- a/packages/coding-agent/test/sdk-computer-tool-toggle.test.ts +++ b/packages/coding-agent/test/sdk-computer-tool-toggle.test.ts @@ -28,6 +28,7 @@ describe("AgentSession.setComputerToolEnabled", () => { registryDir = path.join(os.tmpdir(), `pi-computer-toggle-${Snowflake.next()}`); fs.mkdirSync(registryDir, { recursive: true }); authStorage = await AuthStorage.create(path.join(registryDir, "auth.db")); + authStorage.setRuntimeApiKey("google", "test-key"); modelRegistry = new ModelRegistry(authStorage); }); @@ -70,6 +71,14 @@ describe("AgentSession.setComputerToolEnabled", () => { // Re-enable reuses the retained registry entry. expect(await session.setComputerToolEnabled(true)).toBe(true); + session.settings.override("computer.enabled", true); + expect(session.getEnabledToolNames()).toContain("computer"); + + const gemini = getBundledModel("google", "gemini-2.5-flash"); + if (!gemini) throw new Error("Expected bundled Google Gemini model to exist"); + await session.setModel(gemini); + expect(session.model).toBe(gemini); + expect(session.settings.get("computer.enabled")).toBe(true); expect(session.getEnabledToolNames()).toContain("computer"); }); }); diff --git a/packages/coding-agent/test/slash-commands/computer.test.ts b/packages/coding-agent/test/slash-commands/computer.test.ts index 46780e227..0b392f5df 100644 --- a/packages/coding-agent/test/slash-commands/computer.test.ts +++ b/packages/coding-agent/test/slash-commands/computer.test.ts @@ -2,20 +2,50 @@ import { describe, expect, it, vi } from "bun:test"; import { executeAcpBuiltinSlashCommand } from "@oh-my-pi/pi-coding-agent/slash-commands/acp-builtins"; import type { SlashCommandRuntime } from "@oh-my-pi/pi-coding-agent/slash-commands/types"; -function acpRuntime(options?: { enabled?: boolean; applyResult?: boolean }) { - const store = { "computer.enabled": options?.enabled ?? false }; +function acpRuntime(options?: { + enabled?: boolean; + applyResult?: boolean; + supportsComputerUse?: boolean; + codex?: boolean; +}) { + const store = { + "computer.enabled": options?.enabled ?? false, + "computer.backend": "auto", + "computer.display": "all", + "computer.maxWidth": 1920, + "computer.maxHeight": 1200, + }; const get = vi.fn((path: string) => store[path as keyof typeof store]); const override = vi.fn((path: string, value: boolean) => { - store[path as keyof typeof store] = value; + if (path === "computer.enabled") store[path] = value; }); const set = vi.fn(); const setComputerToolEnabled = vi.fn(async () => options?.applyResult ?? true); + const getEnabledToolNames = vi.fn(() => (store["computer.enabled"] ? ["computer"] : [])); const output = vi.fn(); + const model = options?.codex + ? { + provider: "openai-codex", + id: "gpt-5.6-sol", + api: "openai-codex-responses", + supportsComputerUse: options.supportsComputerUse ?? false, + } + : { + provider: "google", + id: "gemini-2.5-flash", + api: "google-generative-ai", + supportsComputerUse: options?.supportsComputerUse ?? false, + }; const runtime = { - session: { settings: { get, override, set }, setComputerToolEnabled }, + session: { + settings: { get, override, set }, + setComputerToolEnabled, + getEnabledToolNames, + model, + }, output, } as unknown as SlashCommandRuntime; - return { get, override, set, setComputerToolEnabled, output, runtime }; + return { get, override, set, setComputerToolEnabled, getEnabledToolNames, output, runtime }; } describe("/computer slash command", () => { @@ -28,7 +58,9 @@ describe("/computer slash command", () => { expect(h.setComputerToolEnabled).toHaveBeenCalledWith(true); expect(h.override).toHaveBeenCalledWith("computer.enabled", true); expect(h.set).not.toHaveBeenCalled(); - expect(h.output).toHaveBeenCalledWith("Computer use enabled for this session."); + expect(h.output).toHaveBeenCalledWith( + "Computer use enabled for this session. Computer use: enabled · tool: active · backend: auto · display: all · capture: 1920×1200 · model: google/gemini-2.5-flash · exposure: function", + ); }); it("toggles an enabled session off", async () => { @@ -60,7 +92,29 @@ describe("/computer slash command", () => { expect(h.setComputerToolEnabled).not.toHaveBeenCalled(); expect(h.override).not.toHaveBeenCalled(); - expect(h.output).toHaveBeenCalledWith("Computer use is on."); + expect(h.output).toHaveBeenCalledWith( + "Computer use: enabled · tool: active · backend: auto · display: all · capture: 1920×1200 · model: google/gemini-2.5-flash · exposure: function", + ); + }); + + it("reports subscription Codex computer exposure as a callable function", async () => { + const h = acpRuntime({ enabled: true, codex: true }); + + await executeAcpBuiltinSlashCommand("/computer status", h.runtime); + + expect(h.output).toHaveBeenCalledWith( + "Computer use: enabled · tool: active · backend: auto · display: all · capture: 1920×1200 · model: openai-codex/gpt-5.6-sol · exposure: function", + ); + }); + + it("reports explicit Codex native opt-in without masking the override", async () => { + const h = acpRuntime({ enabled: true, codex: true, supportsComputerUse: true }); + + await executeAcpBuiltinSlashCommand("/computer status", h.runtime); + + expect(h.output).toHaveBeenCalledWith( + "Computer use: enabled · tool: active · backend: auto · display: all · capture: 1920×1200 · model: openai-codex/gpt-5.6-sol · exposure: native", + ); }); it("leaves the override untouched when the session cannot build the tool", async () => { diff --git a/packages/coding-agent/test/system-prompt-inventory.test.ts b/packages/coding-agent/test/system-prompt-inventory.test.ts index 39c94e611..52bfe7834 100644 --- a/packages/coding-agent/test/system-prompt-inventory.test.ts +++ b/packages/coding-agent/test/system-prompt-inventory.test.ts @@ -142,6 +142,32 @@ describe("system prompt tool inventory", () => { expect(text).not.toContain("Reads files from disk."); }); + it("keeps enabled computer routing explicit in compact native-tool mode", async () => { + const tools = new Map(TOOLS); + tools.set("computer", { + label: "Computer", + description: "Controls the host desktop.", + parameters: { type: "object", properties: {} }, + }); + const { systemPrompt } = await buildSystemPrompt({ + cwd: tempDir, + contextFiles: [], + skills: [], + rules: [], + toolNames: ["read", "computer"], + tools, + workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, + nativeTools: true, + inlineToolDescriptors: false, + }); + const text = systemPrompt.join("\n\n"); + expect(text).toContain("# Computer Use"); + expect(text).toContain("The `computer` tool is explicitly enabled and available"); + expect(text).toContain("MUST use `computer` for requests to view or control host desktop applications"); + expect(text).toContain("NEVER claim Computer Use is unavailable"); + expect(text).toContain("Inspect the fresh screenshot returned by every successful `computer` call"); + }); + it("renders `# Tool:` sections (not a name list) when tools are not native", async () => { const text = await render({ nativeTools: false, inlineToolDescriptors: false }); expect(text).toContain("# Tool: read");