From 73add3f98c9eef9d34c13499ae2a388e97f21754 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 27 Jul 2026 00:39:09 +0000 Subject: [PATCH 1/4] perf(coding-agent): lazily construct computer schema --- packages/coding-agent/src/tools/computer.ts | 109 +++++++++++------- .../test/computer-lazy-schema.test.ts | 58 ++++++++++ .../computer-schema-construction-preload.ts | 19 +++ .../computer-schema-construction-probe.ts | 101 ++++++++++++++++ 4 files changed, 245 insertions(+), 42 deletions(-) create mode 100644 packages/coding-agent/test/computer-lazy-schema.test.ts create mode 100644 packages/coding-agent/test/fixtures/computer-schema-construction-preload.ts create mode 100644 packages/coding-agent/test/fixtures/computer-schema-construction-probe.ts diff --git a/packages/coding-agent/src/tools/computer.ts b/packages/coding-agent/src/tools/computer.ts index 6217766fc..691743e5d 100644 --- a/packages/coding-agent/src/tools/computer.ts +++ b/packages/coding-agent/src/tools/computer.ts @@ -14,8 +14,8 @@ import type { DesktopDisplay, DesktopSessionOptions, } from "@oh-my-pi/pi-natives"; -import { prompt, sanitizeText } from "@oh-my-pi/pi-utils"; -import { type } from "arktype"; +import { once, prompt, sanitizeText } from "@oh-my-pi/pi-utils"; +import { type Type, type } from "arktype"; import computerDescription from "../prompts/tools/computer.md" with { type: "text" }; import { truncateForPrompt } from "./approval"; import { type ComputerController, ComputerSupervisor, registerComputerController } from "./computer/supervisor"; @@ -57,47 +57,70 @@ function captureOptions(session: ToolSession, coordinateSafeImageSizing: boolean const INT32_MIN = -2_147_483_648; const INT32_MAX = 2_147_483_647; -const coordinateSchema = type("0 <= number.integer <= 2147483647"); -const scrollDeltaSchema = type("-2147483648 <= number.integer <= 2147483647"); +type ComputerSchemaPoint = { + x: number; + y: number; +}; -const pointSchema = type({ - x: coordinateSchema.describe("x pixel coordinate"), - y: coordinateSchema.describe("y pixel coordinate"), - "+": "reject", +type ComputerSchemaAction = { + type: ComputerAction["type"]; + x?: number; + y?: number; + button?: "left" | "right" | "wheel" | "back" | "forward"; + path?: ComputerSchemaPoint[]; + keys?: string[] | null; + scroll_x?: number; + scroll_y?: number; + text?: string; +}; + +export type ComputerParams = { + actions?: ComputerSchemaAction[]; +}; + +type ComputerSchema = Type; + +const getComputerSchema: () => ComputerSchema = once(() => { + const coordinateSchema = type("0 <= number.integer <= 2147483647"); + const scrollDeltaSchema = type("-2147483648 <= number.integer <= 2147483647"); + + const pointSchema = type({ + x: coordinateSchema.describe("x pixel coordinate"), + y: coordinateSchema.describe("y pixel coordinate"), + "+": "reject", + }); + + const computerActionSchema = type({ + type: type( + "'click' | 'double_click' | 'drag' | 'keypress' | 'move' | 'screenshot' | 'scroll' | 'type' | 'wait'", + ).describe("action kind"), + "x?": coordinateSchema.describe( + "x pixel coordinate in the most recent screenshot (click, double_click, move, scroll)", + ), + "y?": coordinateSchema.describe( + "y pixel coordinate in the most recent screenshot (click, double_click, move, scroll)", + ), + "button?": type("'left' | 'right' | 'wheel' | 'back' | 'forward'").describe("mouse button; required for click"), + "path?": pointSchema.array().atLeastLength(2).describe("waypoints from press to release; required for drag"), + "keys?": type("string[] | null").describe( + "key names (e.g. CTRL, SHIFT, ENTER, A); required chord for keypress, optional held modifiers for pointer actions", + ), + "scroll_x?": scrollDeltaSchema.describe("horizontal scroll delta in pixels; required for scroll"), + "scroll_y?": scrollDeltaSchema.describe( + "vertical scroll delta in pixels, positive scrolls content down; required for scroll", + ), + "text?": type("string").describe("literal text to type; required for type"), + "+": "reject", + }); + + return type({ + "actions?": computerActionSchema + .array() + .describe("ordered actions executed as one batch; omit or pass [] to just capture a screenshot"), + "+": "reject", + }); }); -const computerActionSchema = type({ - type: type( - "'click' | 'double_click' | 'drag' | 'keypress' | 'move' | 'screenshot' | 'scroll' | 'type' | 'wait'", - ).describe("action kind"), - "x?": coordinateSchema.describe( - "x pixel coordinate in the most recent screenshot (click, double_click, move, scroll)", - ), - "y?": coordinateSchema.describe( - "y pixel coordinate in the most recent screenshot (click, double_click, move, scroll)", - ), - "button?": type("'left' | 'right' | 'wheel' | 'back' | 'forward'").describe("mouse button; required for click"), - "path?": pointSchema.array().atLeastLength(2).describe("waypoints from press to release; required for drag"), - "keys?": type("string[] | null").describe( - "key names (e.g. CTRL, SHIFT, ENTER, A); required chord for keypress, optional held modifiers for pointer actions", - ), - "scroll_x?": scrollDeltaSchema.describe("horizontal scroll delta in pixels; required for scroll"), - "scroll_y?": scrollDeltaSchema.describe( - "vertical scroll delta in pixels, positive scrolls content down; required for scroll", - ), - "text?": type("string").describe("literal text to type; required for type"), - "+": "reject", -}); - -const computerSchema = type({ - "actions?": computerActionSchema - .array() - .describe("ordered actions executed as one batch; omit or pass [] to just capture a screenshot"), - "+": "reject", -}); - -export type ComputerParams = typeof computerSchema.infer; - export interface ComputerToolDetails { width: number; height: number; @@ -374,14 +397,16 @@ function approvalActionSummary(actions: unknown): string[] { return truncateForPrompt(lines.join("\n"), 2_000).split("\n"); } -export class ComputerTool implements AgentTool { +export class ComputerTool implements AgentTool { readonly name = "computer"; readonly native = { type: "computer" } as const; readonly label = "Computer"; readonly loadMode = "essential" as const; readonly concurrency = "exclusive" as const; readonly summary = "Capture and control the host desktop through native OS APIs"; - readonly parameters = computerSchema; + get parameters(): ComputerSchema { + return getComputerSchema(); + } readonly strict = false; readonly approval = computerApproval; readonly formatApprovalDetails = (args: unknown): string[] => { diff --git a/packages/coding-agent/test/computer-lazy-schema.test.ts b/packages/coding-agent/test/computer-lazy-schema.test.ts new file mode 100644 index 000000000..671751e30 --- /dev/null +++ b/packages/coding-agent/test/computer-lazy-schema.test.ts @@ -0,0 +1,58 @@ +import { expect, test } from "bun:test"; +import * as path from "node:path"; + +const preloadPath = path.join(import.meta.dir, "fixtures", "computer-schema-construction-preload.ts"); +const probePath = path.join(import.meta.dir, "fixtures", "computer-schema-construction-probe.ts"); + +test("computer schema is constructed once on first parameters access", async () => { + const proc = Bun.spawn([process.execPath, "--preload", preloadPath, probePath], { + cwd: path.join(import.meta.dir, "../../.."), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + expect(exitCode, stderr).toBe(0); + expect(JSON.parse(stdout)).toEqual({ + counts: { + afterModuleImport: 0, + afterDefaultOffFactory: 0, + afterToolConstruction: 0, + afterFirstParametersAccess: 1, + afterRepeatedParametersAccess: 1, + afterSecondToolParametersAccess: 1, + afterValidation: 1, + }, + disabledToolCount: 0, + schema: { + callable: true, + repeatedIdentity: true, + crossToolIdentity: true, + validAccepted: true, + validOutput: { + actions: [ + { type: "click", x: 1, y: 2, button: "left", keys: null }, + { type: "double_click", x: 3, y: 4 }, + { + type: "drag", + path: [ + { x: 0, y: 0 }, + { x: 9, y: 9 }, + ], + }, + { type: "keypress", keys: ["CTRL", "A"] }, + { type: "move", x: 5, y: 6 }, + { type: "screenshot" }, + { type: "scroll", x: 7, y: 8, scroll_x: -10, scroll_y: 20 }, + { type: "type", text: "hello" }, + { type: "wait" }, + ], + }, + invalidRejected: [true, true, true, true, true, true], + }, + }); +}, 20_000); diff --git a/packages/coding-agent/test/fixtures/computer-schema-construction-preload.ts b/packages/coding-agent/test/fixtures/computer-schema-construction-preload.ts new file mode 100644 index 000000000..742d378ae --- /dev/null +++ b/packages/coding-agent/test/fixtures/computer-schema-construction-preload.ts @@ -0,0 +1,19 @@ +import { spyOn } from "bun:test"; +import * as arktype from "arktype"; + +declare global { + var __computerCoordinateSchemaConstructionCount: number; +} + +const coordinateDefinition = "0 <= number.integer <= 2147483647"; +globalThis.__computerCoordinateSchemaConstructionCount = 0; +const originalType = arktype.type; +const countedType = ((...args: unknown[]) => { + if (args[0] === coordinateDefinition) { + globalThis.__computerCoordinateSchemaConstructionCount += 1; + } + return Reflect.apply(originalType, undefined, args); +}) as typeof originalType; +Object.assign(countedType, originalType); +const typeSpy = spyOn(arktype, "type").mockImplementation(countedType); +Object.assign(typeSpy, originalType); diff --git a/packages/coding-agent/test/fixtures/computer-schema-construction-probe.ts b/packages/coding-agent/test/fixtures/computer-schema-construction-probe.ts new file mode 100644 index 000000000..d991cbf6d --- /dev/null +++ b/packages/coding-agent/test/fixtures/computer-schema-construction-probe.ts @@ -0,0 +1,101 @@ +import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; +import { BUILTIN_TOOLS, ComputerTool, createTools, type ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; +import { type as arkType } from "arktype"; + +declare global { + var __computerCoordinateSchemaConstructionCount: number; +} + +const count = () => globalThis.__computerCoordinateSchemaConstructionCount; +const toolSession = (settings: Settings): ToolSession => + ({ + cwd: ".", + hasUI: false, + settings, + getSessionFile: () => null, + getSessionSpawns: () => null, + }) as ToolSession; + +const counts = { + afterModuleImport: count(), + afterDefaultOffFactory: -1, + afterToolConstruction: -1, + afterFirstParametersAccess: -1, + afterRepeatedParametersAccess: -1, + afterSecondToolParametersAccess: -1, + afterValidation: -1, +}; + +const disabledTools = await createTools(toolSession(Settings.isolated()), ["computer"]); +counts.afterDefaultOffFactory = count(); + +const firstTool = await BUILTIN_TOOLS.computer(toolSession(Settings.isolated())); +const secondTool = await BUILTIN_TOOLS.computer(toolSession(Settings.isolated())); +if (!(firstTool instanceof ComputerTool) || !(secondTool instanceof ComputerTool)) { + throw new Error("Expected the built-in computer factory to construct ComputerTool instances"); +} +counts.afterToolConstruction = count(); + +const firstSchema = firstTool.parameters; +counts.afterFirstParametersAccess = count(); +const repeatedSchema = firstTool.parameters; +counts.afterRepeatedParametersAccess = count(); +const secondToolSchema = secondTool.parameters; +counts.afterSecondToolParametersAccess = count(); + +const validInput = { + actions: [ + { type: "click", x: 1, y: 2, button: "left", keys: null }, + { type: "double_click", x: 3, y: 4 }, + { + type: "drag", + path: [ + { x: 0, y: 0 }, + { x: 9, y: 9 }, + ], + }, + { type: "keypress", keys: ["CTRL", "A"] }, + { type: "move", x: 5, y: 6 }, + { type: "screenshot" }, + { type: "scroll", x: 7, y: 8, scroll_x: -10, scroll_y: 20 }, + { type: "type", text: "hello" }, + { type: "wait" }, + ], +}; +const validOutput = firstSchema(validInput); +const invalidOutputs = [ + firstSchema({ actions: [{ type: "click", x: -1, y: 2, button: "left" }] }), + firstSchema({ actions: [{ type: "move", x: 0.5, y: 0 }] }), + firstSchema({ actions: [{ type: "scroll", x: 0, y: 0, scroll_x: 2 ** 31, scroll_y: 0 }] }), + firstSchema({ actions: [{ type: "drag", path: [{ x: 0, y: 0 }] }] }), + firstSchema({ + actions: [ + { + type: "drag", + path: [ + { x: 0, y: 0, label: "unexpected" }, + { x: 1, y: 1 }, + ], + }, + ], + }), + firstSchema({ actions: [], unexpected: true }), +]; +counts.afterValidation = count(); + +await Promise.all([firstTool.close(), secondTool.close()]); + +process.stdout.write( + JSON.stringify({ + counts, + disabledToolCount: disabledTools.length, + schema: { + callable: typeof firstSchema === "function", + repeatedIdentity: firstSchema === repeatedSchema, + crossToolIdentity: firstSchema === secondToolSchema, + validAccepted: !(validOutput instanceof arkType.errors), + validOutput, + invalidRejected: invalidOutputs.map(output => output instanceof arkType.errors), + }, + }), +); From 7f32d6fcbe352fd138665942f9050d26c2e4a659 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 27 Jul 2026 00:48:44 +0000 Subject: [PATCH 2/4] docs(coding-agent): note lazy computer schema --- packages/coding-agent/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 8286d275b..411e43251 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- Reduced default startup resident memory by constructing the default-off ComputerTool ArkType schema only on first parameter access, then reusing it across tool instances without changing validation or tool behavior ([#6742](https://github.com/can1357/oh-my-pi/pull/6742) by [@usr-bin-roygbiv](https://github.com/usr-bin-roygbiv)). + ## [17.1.4] - 2026-07-26 ### Added From 76425c45bbe29435e5ae873390df5aceafc92b53 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 27 Jul 2026 00:51:35 +0000 Subject: [PATCH 3/4] refactor(coding-agent): assert computer schema type parity --- packages/coding-agent/src/tools/computer.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/src/tools/computer.ts b/packages/coding-agent/src/tools/computer.ts index 691743e5d..9e49be5a7 100644 --- a/packages/coding-agent/src/tools/computer.ts +++ b/packages/coding-agent/src/tools/computer.ts @@ -78,7 +78,13 @@ export type ComputerParams = { actions?: ComputerSchemaAction[]; }; -type ComputerSchema = Type; +type IsSameType = [Left] extends [Right] ? ([Right] extends [Left] ? true : false) : false; +type ComputerSchema> = IsSameType< + ComputerParams, + Schema["infer"] +> extends true + ? Schema + : never; const getComputerSchema: () => ComputerSchema = once(() => { const coordinateSchema = type("0 <= number.integer <= 2147483647"); @@ -113,12 +119,13 @@ const getComputerSchema: () => ComputerSchema = once(() => { "+": "reject", }); - return type({ + const computerSchema = type({ "actions?": computerActionSchema .array() .describe("ordered actions executed as one batch; omit or pass [] to just capture a screenshot"), "+": "reject", }); + return computerSchema satisfies ComputerSchema; }); export interface ComputerToolDetails { From 2755226e59b9a9c632a241d39b59a2262b961807 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Mon, 27 Jul 2026 01:21:27 +0000 Subject: [PATCH 4/4] style(coding-agent): format schema parity type --- packages/coding-agent/src/tools/computer.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/coding-agent/src/tools/computer.ts b/packages/coding-agent/src/tools/computer.ts index 9e49be5a7..53f74aea5 100644 --- a/packages/coding-agent/src/tools/computer.ts +++ b/packages/coding-agent/src/tools/computer.ts @@ -79,12 +79,8 @@ export type ComputerParams = { }; type IsSameType = [Left] extends [Right] ? ([Right] extends [Left] ? true : false) : false; -type ComputerSchema> = IsSameType< - ComputerParams, - Schema["infer"] -> extends true - ? Schema - : never; +type ComputerSchema> = + IsSameType extends true ? Schema : never; const getComputerSchema: () => ComputerSchema = once(() => { const coordinateSchema = type("0 <= number.integer <= 2147483647");