Files
oh-my-pi/packages/coding-agent/test/write-xdev-dispatch.test.ts
T
Nik Divjak 630f9e5324 fix(xdev): bound device summaries in UTF-8 bytes and flag untrusted metadata
Catalog summaries of mounted xd:// devices are inlined verbatim into the
system prompt. External devices (MCP servers, plugins) supply that text, and
it was bounded only by character count: a summary of multi-byte script passed
roughly three times the intended budget, and control characters survived into
the prompt where they can forge structure.

Summaries now go through a single sanitize-and-bound step that strips C0/C1
control characters and bounds the result in UTF-8 bytes via the central
truncateHeadBytes helper, so a cut lands on a code point boundary and never
renders a partial code point. The built-in/external distinction is derived
once per entry, and that same boolean both selects the description cap and is
exposed as `dynamic`, so the cap and the flag cannot disagree. The prompt uses
the flag to state that dynamic summaries are untrusted metadata, and the mount
notice says the same for newly appeared devices.

(cherry picked from commit 5989da6235d820bc687779a791e655e6f1b2df0f)
2026-07-29 23:09:00 +02:00

528 lines
23 KiB
TypeScript

import { describe, expect, it } from "bun:test";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as path from "node:path";
import type { AgentTool } from "@oh-my-pi/pi-agent-core";
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
import * as themeModule from "@oh-my-pi/pi-coding-agent/modes/theme/theme";
import { ToolChoiceQueue } from "@oh-my-pi/pi-coding-agent/session/tool-choice-queue";
import { createTools, type Tool, type ToolSession } from "@oh-my-pi/pi-coding-agent/tools";
import { githubToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/gh-renderer";
import { ToolError } from "@oh-my-pi/pi-coding-agent/tools/tool-errors";
import { WriteTool, writeToolRenderer } from "@oh-my-pi/pi-coding-agent/tools/write";
import {
listXdevTools,
resolveMountedXdevTool,
XDEV_DOCS_PER_DEVICE_CAP,
XDEV_DOCS_TOTAL_BUDGET,
XDEV_EXTERNAL_DESCRIPTION_CAP,
type XdevState,
xdevDocs,
xdevDocsAll,
xdevEntries,
} from "@oh-my-pi/pi-coding-agent/tools/xdev";
import { removeWithRetries } from "@oh-my-pi/pi-utils";
import { type } from "arktype";
// xdev mounting is default-on: discoverable tools like ast_edit unmount into
// xd://, and a plain `write xd://ast_edit` dispatches them. These guard the
// resolution-device symbols write.ts pulls from ./resolve — a missing import
// threw `ReferenceError: isResolutionDeviceName is not defined` on *every*
// xd:// write, in both the executor (approval + execute) and the streaming
// renderer (surfacing as the error text inside a generic Write frame).
function xdevSession(cwd: string, overrides: Partial<ToolSession> = {}): ToolSession {
return {
cwd,
hasUI: true,
getSessionFile: () => null,
getSessionSpawns: () => "*",
settings: Settings.isolated({}),
...overrides,
};
}
function createTestXdevState(tools: Tool[], builtInNames: Iterable<string> = tools.map(tool => tool.name)): XdevState {
return {
tools: new Map(tools.map(tool => [tool.name, tool])),
mountedNames: new Set(tools.map(tool => tool.name)),
builtInNames: new Set(builtInNames),
isActive: () => false,
};
}
describe("read and write route xd:// device URLs", () => {
it("lists, documents, and dispatches an ast_edit device", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-"));
try {
const filePath = path.join(tempDir, "legacy.ts");
await Bun.write(filePath, "legacyWrap(x, value)\n");
const queue = new ToolChoiceQueue();
const tools = await createTools(
xdevSession(tempDir, {
getToolChoiceQueue: () => queue,
buildToolChoice: () => ({ type: "tool" as const, name: "resolve" }),
steer: () => {},
}),
);
// xdev on: ast_edit is unmounted into xd://; write stays in the toolset.
const write = tools.find(entry => entry.name === "write");
const read = tools.find(entry => entry.name === "read");
expect(read).toBeDefined();
expect(write).toBeDefined();
expect(tools.some(entry => entry.name === "ast_edit")).toBe(false);
const listing = await read!.execute("read-xd-list", { path: "xd://" });
expect(listing.content.find(entry => entry.type === "text")?.text).toContain("xd://ast_edit");
const docs = await read!.execute("read-xd-docs", { path: "xd://ast_edit" });
expect(docs.content.find(entry => entry.type === "text")?.text).toContain("# ast_edit");
const content = JSON.stringify({
ops: [{ pat: "legacyWrap($A, $B)", out: "modernWrap($A, $B)" }],
paths: [filePath],
});
// The write gate decodes the device payload and evaluates the mounted
// tool's own approval. ast_edit is write-tier for a filesystem path.
const approval = write!.approval;
expect(typeof approval).toBe("function");
if (typeof approval === "function") {
expect(approval({ path: "xd://ast_edit", content })).toBe("write");
}
// Execute dispatches through the xdev registry to the mounted ast_edit,
// staging a preview (not a direct apply).
const previewResult = await write!.execute("write-xdev-preview", { path: "xd://ast_edit", content });
expect(previewResult.isError).toBeUndefined();
expect(previewResult.details?.xdev?.tool).toBe("ast_edit");
expect(previewResult.details?.xdev?.mode).toBe("execute");
const previewText = previewResult.content.find(entry => entry.type === "text")?.text ?? "";
expect(previewText).toContain("modernWrap");
// The staged preview applies through the resolve queue and rewrites disk.
const invoker = queue.peekPendingInvoker();
expect(invoker).toBeDefined();
await invoker!({ action: "apply", reason: "apply xdev ast edit" });
expect(await Bun.file(filePath).text()).toContain("modernWrap(x, value)");
} finally {
await removeWithRetries(tempDir);
}
});
it("rejects near-miss xd addresses before filesystem fallback", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-near-miss-"));
try {
const tools = await createTools(xdevSession(tempDir));
const write = tools.find(entry => entry.name === "write");
expect(write).toBeDefined();
for (const target of ["xdt://web_search", "xd:/web_search", "xd/web_search"]) {
await expect(write!.execute(`write-${target}`, { path: target, content: "{}" })).rejects.toThrow(
"Did you mean 'xd://web_search'?",
);
}
expect(await Bun.file(path.join(tempDir, "xdt:/web_search")).exists()).toBe(false);
expect(await Bun.file(path.join(tempDir, "xd/web_search")).exists()).toBe(false);
const escaped = await write!.execute("write-explicit-path", {
path: "./xd/web_search",
content: "intentional file",
});
expect(escaped.isError).toBeUndefined();
expect(await Bun.file(path.join(tempDir, "xd/web_search")).text()).toBe("intentional file");
// conflict:// has no router handler but is a documented write scheme —
// the guard must let it reach the conflict resolver, not reject it.
await expect(write!.execute("write-conflict", { path: "conflict://1", content: "x" })).rejects.toThrow(
"Conflict #1 not found",
);
} finally {
await removeWithRetries(tempDir);
}
});
it("resolves function-valued device approvals per payload and fails closed on bad content", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-approval-"));
try {
const filePath = path.join(tempDir, "target.ts");
await Bun.write(filePath, "legacyWrap(x, value)\n");
const tools = await createTools(xdevSession(tempDir));
const write = tools.find(entry => entry.name === "write");
expect(write).toBeDefined();
const approval = write!.approval;
expect(typeof approval).toBe("function");
if (typeof approval !== "function") throw new Error("expected a function approval");
const tier = (path: string, content: string) => approval({ path, content });
// ast_edit on a filesystem path → write; on internal URLs only → read.
const astFsPath = JSON.stringify({
ops: [{ pat: "legacyWrap($A, $B)", out: "modernWrap($A, $B)" }],
paths: [filePath],
});
const astInternalPath = JSON.stringify({
ops: [{ pat: "a", out: "b" }],
paths: ["artifact://abc"],
});
expect(tier("xd://ast_edit", astFsPath)).toBe("write");
expect(tier("xd://ast_edit", astInternalPath)).toBe("read");
// debug: inspection action → read; a real launch → exec (control).
expect(tier("xd://debug", JSON.stringify({ action: "sessions" }))).toBe("read");
expect(tier("xd://debug", JSON.stringify({ action: "launch", program: "./app" }))).toBe("exec");
// Fail closed: malformed JSON, non-object or schema-invalid payloads,
// missing content, and unknown devices all stay exec so the gate never
// under-prompts.
expect(tier("xd://ast_edit", "{ not json")).toBe("exec");
expect(tier("xd://ast_edit", "[1,2,3]")).toBe("exec");
expect(tier("xd://ast_edit", '"a string"')).toBe("exec");
expect(tier("xd://ast_edit", JSON.stringify({ paths: [null] }))).toBe("exec");
expect(approval({ path: "xd://ast_edit" })).toBe("exec");
expect(tier("xd://no_such_device", "{}")).toBe("exec");
} finally {
await removeWithRetries(tempDir);
}
});
it("renderCall withholds a partial xd:// URL, then delegates once settled", async () => {
await themeModule.initTheme();
const uiTheme = (await themeModule.getThemeByName("dark")) ?? (await themeModule.getThemeByName("light"));
if (!uiTheme) throw new Error("expected an initialized theme");
const options = { expanded: false, isPartial: true };
const content = JSON.stringify({
ops: [{ pat: "legacyWrap($A, $B)", out: "modernWrap($A, $B)" }],
paths: ["/tmp/legacy.ts"],
});
// Path still streaming (no content field yet): render nothing so the user
// never sees a half-typed "xd://ast_" frame.
expect(writeToolRenderer.renderCall({ path: "xd://ast_e" }, options, uiTheme)).toBeUndefined();
// Path settled + content streaming: delegate to the mounted tool's renderer
// instead of throwing ReferenceError inside a generic Write frame.
const rendered = writeToolRenderer.renderCall({ path: "xd://ast_edit", content }, options, uiTheme);
expect(rendered).toBeDefined();
});
it("renders device execution errors as the mounted tool instead of write", async () => {
await themeModule.initTheme();
const uiTheme = (await themeModule.getThemeByName("dark")) ?? (await themeModule.getThemeByName("light"));
if (!uiTheme) throw new Error("expected an initialized theme");
const githubDevice = {
name: "github",
label: "GitHub",
description: "fixture",
parameters: type({ op: "string" }),
...githubToolRenderer,
async execute() {
throw new ToolError("gh: Not Found (HTTP 404)");
},
};
const xdev = createTestXdevState([githubDevice]);
const write = new WriteTool(xdevSession(process.cwd(), { xdev }));
const content = JSON.stringify({ op: "repo_view" });
const result = await write.execute("write-xdev-error", { path: "xd://github", content });
expect(result.isError).toBe(true);
expect(result.details?.xdev).toMatchObject({
tool: "github",
mode: "execute",
args: { op: "repo_view" },
});
const component = writeToolRenderer.renderResult(
result,
{
expanded: false,
isPartial: false,
renderContext: { resolveXdevMounted: name => resolveMountedXdevTool(xdev, name) },
},
uiTheme,
{ path: "xd://github", content },
);
const rendered = Bun.stripANSI(component.render(80).join("\n"));
expect(rendered).toContain("GitHub Repo");
expect(rendered).toContain("gh: Not Found (HTTP 404)");
expect(rendered).not.toContain("Write");
});
it("keeps the generic custom-tool card when a mounted device has no renderer", async () => {
await themeModule.initTheme();
const uiTheme = (await themeModule.getThemeByName("dark")) ?? (await themeModule.getThemeByName("light"));
if (!uiTheme) throw new Error("expected an initialized theme");
const weatherDevice: AgentTool = {
name: "weather",
label: "Weather",
description: "Gets the weather",
parameters: type({ query: "string" }),
async execute() {
return { content: [{ type: "text", text: "Tokyo: 22°C" }] };
},
};
const xdev = createTestXdevState([weatherDevice]);
const write = new WriteTool(xdevSession(process.cwd(), { xdev }));
const content = JSON.stringify({ query: "Tokyo" });
const result = await write.execute("write-xdev-default-renderer", {
path: "xd://weather",
content,
});
const component = writeToolRenderer.renderResult(
result,
{
expanded: false,
isPartial: false,
renderContext: { resolveXdevMounted: name => resolveMountedXdevTool(xdev, name) },
},
uiTheme,
{ path: "xd://weather", content },
);
const lines = component.render(80);
const rendered = Bun.stripANSI(lines.join("\n"));
const backgroundProbe = uiTheme.bg("toolSuccessBg", "|");
const backgroundPrefix = backgroundProbe.slice(0, backgroundProbe.indexOf("|"));
expect(rendered).toContain("Weather");
expect(rendered).toContain('query="Tokyo"');
expect(rendered).toContain("Tokyo: 22°C");
expect(backgroundPrefix).not.toBe("");
expect(lines.some(line => line.includes(backgroundPrefix))).toBe(true);
});
// Dynamic device summaries are third-party text inlined into the system
// prompt. A character bound is not a byte bound: a multi-byte summary passes
// several times the intended budget, and cutting a byte budget by character
// index splits code points.
it("bounds dynamic device summaries in UTF-8 bytes on a code point boundary", () => {
const multiByteTail = "あ".repeat(XDEV_EXTERNAL_DESCRIPTION_CAP);
const dynamicDevice: AgentTool = {
name: "mcp__weather__forecast",
label: "Forecast",
description: "Weather forecast for a place.",
summary: `Napoved\u0007vremena ${multiByteTail}`,
parameters: type({ query: "string" }),
async execute() {
return { content: [{ type: "text", text: "" }] };
},
};
const builtInDevice: AgentTool = {
name: "weather",
label: "Weather",
description: "Weather for a place.",
summary: `Gets the weather ${multiByteTail}`,
parameters: type({ query: "string" }),
async execute() {
return { content: [{ type: "text", text: "" }] };
},
};
const xdev = createTestXdevState([builtInDevice, dynamicDevice], ["weather"]);
const entries = new Map(xdevEntries(xdev).map(entry => [entry.name, entry]));
const dynamic = entries.get("mcp__weather__forecast");
if (!dynamic) throw new Error("expected the dynamic device entry");
expect(dynamic.dynamic).toBe(true);
// Control characters collapse to a space instead of reaching the prompt.
expect(dynamic.summary.startsWith("Napoved vremena ")).toBe(true);
expect(dynamic.summary.endsWith("…")).toBe(true);
const body = dynamic.summary.slice(0, -1);
const bodyBytes = Buffer.byteLength(body, "utf-8");
expect(bodyBytes).toBeLessThanOrEqual(XDEV_EXTERNAL_DESCRIPTION_CAP);
// The cut backs off at most one code point: the character straddling the
// budget is dropped whole rather than split.
expect(bodyBytes).toBeGreaterThan(XDEV_EXTERNAL_DESCRIPTION_CAP - 3);
expect(body.endsWith("あ")).toBe(true);
// A split code point would decode to U+FFFD and fail the round trip.
expect(Buffer.from(body, "utf-8").toString("utf-8")).toBe(body);
// The same boolean drives the cap and the flag, so a built-in device is
// never capped and never reported as untrusted.
const builtIn = entries.get("weather");
if (!builtIn) throw new Error("expected the built-in device entry");
expect(builtIn.dynamic).toBe(false);
expect(builtIn.summary).toBe(`Gets the weather ${multiByteTail}`);
});
it("docsAll inlines small device docs and falls back to a listing past the caps", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-docs-"));
try {
const session = xdevSession(tempDir);
expect(session.settings.get("tools.xdevDocs")).toBe("builtins");
await createTools(session);
const xdev = session.xdev;
if (!xdev) throw new Error("expected xdev state");
const mounted = listXdevTools(xdev);
expect(mounted.length).toBeGreaterThan(0);
// One device with a pathological description must fall back to the
// listing without starving the rest of the catalog.
const giant = Object.create(mounted[0]!) as (typeof mounted)[number];
Object.defineProperty(giant, "name", { value: "giant_mcp_tool" });
Object.defineProperty(giant, "description", { value: "x".repeat(XDEV_DOCS_PER_DEVICE_CAP + 1) });
xdev.tools.set(giant.name, giant);
xdev.mountedNames.add(giant.name);
xdev.builtInNames.add(giant.name);
const docs = xdevDocsAll(xdev);
expect(docs.length).toBeLessThan(XDEV_DOCS_TOTAL_BUDGET + XDEV_DOCS_PER_DEVICE_CAP);
expect(docs).toContain(`## ${mounted[0]!.name}`);
expect(docs).toContain("## Additional devices (docs on demand)");
expect(docs).toContain("- xd://giant_mcp_tool —");
expect(docs).not.toContain("## giant_mcp_tool");
} finally {
await removeWithRetries(tempDir);
}
});
it("docsAll supports inline, builtins, and catalog prompt modes", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-external-"));
try {
const session = xdevSession(tempDir);
expect(session.settings.get("tools.xdevDocs")).toBe("builtins");
await createTools(session);
const xdev = session.xdev;
if (!xdev) throw new Error("expected xdev state");
const mounted = listXdevTools(xdev);
const builtInMountedNames = [...xdev.mountedNames];
const longDescription = `LEDE ${"y".repeat(XDEV_EXTERNAL_DESCRIPTION_CAP * 3)} TAIL`;
const external = Object.create(mounted[0]!) as (typeof mounted)[number];
Object.defineProperty(external, "name", { value: "mcp_external_tool" });
Object.defineProperty(external, "description", { value: longDescription });
Object.defineProperty(external, "summary", {
value: `SUMMARY ${"z".repeat(XDEV_EXTERNAL_DESCRIPTION_CAP * 3)} TAIL`,
});
xdev.tools.set(external.name, external);
xdev.mountedNames.add(external.name);
const inlineDocs = xdevDocsAll(xdev, "inline");
expect(inlineDocs).toContain("## mcp_external_tool");
expect(inlineDocs).toContain("LEDE ");
expect(inlineDocs).not.toContain("TAIL");
expect(inlineDocs).toContain("… (full docs: read xd://mcp_external_tool)");
const builtinsDocs = xdevDocsAll(xdev, "builtins");
expect(builtinsDocs).toContain("## ");
expect(builtinsDocs).not.toContain("## mcp_external_tool");
expect(builtinsDocs).toContain("- xd://mcp_external_tool —");
expect(builtinsDocs).not.toContain("TAIL");
const catalogDocs = xdevDocsAll(xdev, "catalog");
expect(catalogDocs).not.toContain(`## ${mounted[0]!.name}`);
expect(catalogDocs).toContain("- xd://");
expect(catalogDocs).toContain("- xd://mcp_external_tool —");
expect(xdevDocs(xdev, "mcp_external_tool")).toContain("TAIL");
const contextMode = Object.create(mounted[0]!) as (typeof mounted)[number];
Object.defineProperty(contextMode, "name", { value: "mcp__context_mode_ctx_execute" });
const unrelatedMcp = Object.create(mounted[0]!) as (typeof mounted)[number];
Object.defineProperty(unrelatedMcp, "name", { value: "mcp__other_server_execute" });
xdev.tools.set(contextMode.name, contextMode);
xdev.tools.set(unrelatedMcp.name, unrelatedMcp);
xdev.mountedNames.clear();
for (const name of [...builtInMountedNames, contextMode.name, unrelatedMcp.name]) xdev.mountedNames.add(name);
const allowlistedDocs = xdevDocsAll(xdev, "builtins", ["mcp__context_mode_*"]);
expect(allowlistedDocs).toContain("## mcp__context_mode_ctx_execute");
expect(allowlistedDocs).not.toContain("## mcp__other_server_execute");
expect(allowlistedDocs).toContain("- xd://mcp__other_server_execute —");
const catalogWithAllowlistDocs = xdevDocsAll(xdev, "catalog", ["mcp__context_mode_*"]);
expect(catalogWithAllowlistDocs).not.toContain("## mcp__context_mode_ctx_execute");
// Malformed user config (scalar or non-string entries reach the
// registry unvalidated) degrades to the catalog listing instead of
// throwing while the system prompt is built.
const scalarAllowlistDocs = xdevDocsAll(xdev, "builtins", "mcp__context_mode_*" as never);
expect(scalarAllowlistDocs).toContain("- xd://mcp__context_mode_ctx_execute —");
const nonStringAllowlistDocs = xdevDocsAll(xdev, "builtins", [123] as never);
expect(nonStringAllowlistDocs).toContain("- xd://mcp__context_mode_ctx_execute —");
} finally {
await removeWithRetries(tempDir);
}
});
});
describe("web_search stays top-level under xdev", () => {
it("keeps web_search direct and out of the mounted-name set with default config", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-websearch-"));
try {
const session = xdevSession(tempDir);
// Default config: tools.xdev is on.
expect(session.settings.get("tools.xdev")).toBe(true);
const tools = await createTools(session);
// Regression for #5973: models call web_search directly, so it must
// remain a top-level function and never mount behind the xd:// device.
expect(tools.some(entry => entry.name === "web_search")).toBe(true);
const mounted = session.xdev ? [...session.xdev.mountedNames] : [];
expect(mounted).not.toContain("web_search");
const write = tools.find(tool => tool.name === "write");
const read = tools.find(tool => tool.name === "read");
expect(write).toBeDefined();
expect(read).toBeDefined();
const docs = await read!.execute("read-xdev-web-search", { path: "xd://web_search" });
expect(docs.content.find(entry => entry.type === "text")?.text).toContain("# web_search");
// Missing required args fails schema validation after routing to web_search,
// rather than failing lookup because the tool is top-level.
const dispatched = await write!.execute("write-xdev-web-search", {
path: "xd://web_search",
content: "{}",
});
expect(dispatched.isError).toBe(true);
expect(dispatched.details?.xdev?.tool).toBe("web_search");
expect(dispatched.content.find(entry => entry.type === "text")?.text).not.toContain("No such tool");
} finally {
await removeWithRetries(tempDir);
}
});
});
describe("xd:// and top-level calls share the canonical tool map", () => {
it("dispatches and documents an unmounted top-level tool, and still rejects unknown names", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "write-xdev-fallback-"));
try {
await Bun.write(path.join(tempDir, "haystack.txt"), "alpha\nfallback-needle\nomega\n");
const session = xdevSession(tempDir);
const tools = await createTools(session);
const write = tools.find(entry => entry.name === "write");
const read = tools.find(entry => entry.name === "read");
expect(write).toBeDefined();
expect(read).toBeDefined();
// grep is kept top-level (XDEV_KEEP_TOP_LEVEL) and thus not a mounted
// device — the unified namespace must still dispatch it via xd://.
const mounted = [...session.xdev!.mountedNames];
expect(mounted).not.toContain("grep");
const dispatched = await write!.execute("write-xdev-fallback-grep", {
path: "xd://grep",
content: JSON.stringify({ pattern: "fallback-needle", path: tempDir }),
});
expect(dispatched.isError).toBeUndefined();
expect(dispatched.details?.xdev?.tool).toBe("grep");
expect(dispatched.content.find(entry => entry.type === "text")?.text).toContain("fallback-needle");
// Docs resolve through the same fallback.
const docs = await read!.execute("read-xdev-fallback-grep", { path: "xd://grep" });
expect(docs.content.find(entry => entry.type === "text")?.text).toContain("# grep");
// Genuinely unknown names still fail with the catalog error.
const unknown = await write!.execute("write-xdev-fallback-unknown", {
path: "xd://no_such_tool",
content: "{}",
});
expect(unknown.isError).toBe(true);
expect(unknown.content.find(entry => entry.type === "text")?.text).toContain(
"No such tool: xd://no_such_tool",
);
} finally {
await removeWithRetries(tempDir);
}
});
});