feat(computer): unified native addon, /computer toggle, function tool

- Replaced the separate GUI-linked pi_natives.desktop.linux-x64 addon with
  a pure-Rust X11 backend (x11rb RustConnection capture via RandR/GetImage,
  XTest input with keysym mapping) compiled into the core addon on every
  published target; Linux arm64 and musl are now supported and headless
  hosts load the addon unaffected.
- Removed the native-desktop-linux cargo feature, desktop_unsupported.rs,
  lazy desktop loader, second napi build, desktop packaging/CI steps, GUI
  build dependencies, and the now-unreferenced vendored libspa crate;
  reverted setup-system-deps to main.
- Preserved the desktop input hardening semantics on the unified backend:
  XTest layouts reject negative origins and coordinates beyond 0..=32767,
  batch coordinates stay bound to the frame last returned to JS with
  intermediate screenshots deferred, coordinate input requires a
  previously returned frame, and failed chord releases still release
  every held key.
- Enforced a 60s worker-side execute deadline (DESKTOP_DEADLINE_EXCEEDED):
  no input is emitted after expiry and wait-heavy batches are rejected
  upfront.
- Added int32 fail-closed validation for coordinates, drag points, and
  scroll deltas at the JS ingress and gateway schema.
- Exposed computer to models without native OpenAI computer-use support as
  a regular function tool with a typed GA action schema across OpenAI,
  Azure, and Codex Responses providers, including named forced choice.
- Added the /computer slash command (on/off/status/toggle) for
  session-only enablement via runtime tool registration in SessionTools.
- Updated docs, changelogs, and contract tests accordingly.
This commit is contained in:
can1357
2026-07-24 02:23:56 +02:00
committed by usr-bin-roygbiv
parent 4fe03e25cb
commit 681d7daf65
72 changed files with 1894 additions and 12477 deletions
@@ -6,7 +6,13 @@ import type { ExtensionRunner } from "@oh-my-pi/pi-coding-agent/extensibility/ex
import { ExtensionToolWrapper } from "@oh-my-pi/pi-coding-agent/extensibility/extensions";
import { getThemeByName } from "@oh-my-pi/pi-coding-agent/modes/theme/theme";
import { buildSystemPrompt } from "@oh-my-pi/pi-coding-agent/system-prompt";
import { ComputerTool, computerApproval, createTools, type ToolSession } from "@oh-my-pi/pi-coding-agent/tools";
import {
type ComputerParams,
ComputerTool,
computerApproval,
createTools,
type ToolSession,
} from "@oh-my-pi/pi-coding-agent/tools";
import type {
ComputerWorkerInbound,
ComputerWorkerOutbound,
@@ -23,6 +29,7 @@ import { ComputerWorkerCore, type NativeDesktopSession } from "@oh-my-pi/pi-codi
import { computerToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/computer-renderer";
import { buildNamedToolChoice, isToolChoiceActive } from "@oh-my-pi/pi-coding-agent/utils/tool-choice";
import type { DesktopAction, DesktopCapabilities, DesktopCapture, DesktopSessionOptions } from "@oh-my-pi/pi-natives";
import { type as arkType } from "arktype";
const capabilities: DesktopCapabilities = {
capture: true,
@@ -358,12 +365,12 @@ describe("computer supervisor", () => {
});
describe("computer tool choice", () => {
it("uses native forced choice only for models declaring computer-use support", () => {
it("uses native forced choice for GA models and named function choice for the fallback", () => {
const supported = { api: "openai-responses", supportsComputerUse: true } as unknown as Model<Api>;
expect(buildNamedToolChoice("computer", supported)).toEqual({ type: "computer" });
for (const api of ["openai-responses", "openai-codex-responses", "azure-openai-responses"] as const) {
const unsupported = { api, supportsComputerUse: false } as unknown as Model<Api>;
expect(buildNamedToolChoice("computer", unsupported)).toBeUndefined();
expect(buildNamedToolChoice("computer", unsupported)).toEqual({ type: "function", name: "computer" });
}
expect(isToolChoiceActive({ type: "computer" }, [{ name: "computer" }])).toBe(true);
expect(isToolChoiceActive({ type: "computer" }, [{ name: "read" }])).toBe(false);
@@ -378,6 +385,78 @@ describe("computer tool", () => {
expect(enabled.map(tool => [tool.name, tool.loadMode])).toEqual([["computer", "essential"]]);
});
it("accepts each GA action shape through the params schema and rejects malformed coordinates", () => {
const tool = new ComputerTool(
toolSession(Settings.isolated({ "computer.enabled": true })),
() => new FakeController(),
);
const ok = tool.parameters({
actions: [
{ type: "click", x: 1, y: 2, button: "left" },
{ type: "double_click", x: 3, y: 4 },
{
type: "drag",
path: [
{ x: 0, y: 0 },
{ x: 9, y: 9 },
],
},
{ type: "keypress", keys: ["CTRL", "A"] },
{ type: "move", x: 5, y: 6 },
{ type: "screenshot" },
{ type: "scroll", x: 7, y: 8, scroll_x: -10, scroll_y: 20 },
{ type: "type", text: "hello" },
{ type: "wait" },
],
});
expect(ok instanceof arkType.errors).toBe(false);
for (const actions of [
[{ type: "click", x: -1, y: 2, button: "left" }],
[{ type: "move", x: 0.5, y: 0 }],
[{ type: "scroll", x: 0, y: 0, scroll_x: 2 ** 31, scroll_y: 0 }],
]) {
expect(tool.parameters({ actions }) instanceof arkType.errors).toBe(true);
}
});
it("executes function-call params.actions and defaults empty batches to a screenshot", async () => {
const controller = new FakeController();
const tool = new ComputerTool(toolSession(Settings.isolated({ "computer.enabled": true })), () => controller);
const result = await tool.execute("call", { actions: [{ type: "click", x: 5, y: 6, button: "left" }] });
expect(result.content).toEqual([{ type: "image", data: "AQ==", mimeType: "image/png", detail: "original" }]);
await tool.execute("call", {});
await tool.execute("call", { actions: [] });
expect(controller.batches).toEqual([
[{ type: "click", x: 5, y: 6, button: "left" }],
[{ type: "screenshot" }],
[{ type: "screenshot" }],
]);
await tool.close();
});
it("fails closed on non-integer, negative, or out-of-int32-range coordinates", async () => {
const controller = new FakeController();
const tool = new ComputerTool(toolSession(Settings.isolated({ "computer.enabled": true })), () => controller);
const invalidBatches = [
[{ type: "click", x: 1.5, y: 2, button: "left" }],
[{ type: "move", x: -1, y: 2 }],
[{ type: "move", x: 2 ** 31, y: 0 }],
[{ type: "scroll", x: 0, y: 0, scroll_x: 0, scroll_y: -(2 ** 31) - 1 }],
[{ type: "drag", path: [{ x: 0, y: -3 }] }],
] as unknown as ComputerParams["actions"][];
for (const actions of invalidBatches) {
await expect(tool.execute("call", { actions })).rejects.toThrow("Computer call contains an invalid action");
}
expect(controller.batches).toHaveLength(0);
await tool.execute("call", {
actions: [{ type: "scroll", x: 0, y: 0, scroll_x: -2_147_483_648, scroll_y: 2_147_483_647 }],
});
expect(controller.batches).toEqual([
[{ type: "scroll", x: 0, y: 0, scroll_x: -2_147_483_648, scroll_y: 2_147_483_647 }],
]);
await tool.close();
});
it("uses registered native options, adapts every GA field, and returns exactly one fresh PNG with metadata", async () => {
const settings = Settings.isolated({
"computer.enabled": true,