import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import { buildSystemPrompt as buildSdkSystemPrompt } from "@oh-my-pi/pi-coding-agent/sdk"; import { buildSystemPrompt, buildSystemPromptToolMetadata, DEFAULT_SYSTEM_PROMPT_TOOL_NAMES, projectSystemPromptToolMetadata, type SystemPromptToolMetadata, } from "@oh-my-pi/pi-coding-agent/system-prompt"; import { createTools, type Tool, type ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; import { cleanupTempHome } from "./helpers/temp-home-cleanup"; const EMPTY_TREE = { rootPath: "", rendered: "", truncated: false, totalLines: 0, agentsMdFiles: [], }; const TOOLS = new Map([ [ "read", { label: "Read", description: "Reads files from disk.", parameters: { type: "object", properties: { path: { type: "string" } } }, }, ], [ "bash", { label: "Bash", description: "Executes a shell command.", parameters: { type: "object", properties: { command: { type: "string" } } }, }, ], ]); const DIRECT_WEB_SEARCH: SystemPromptToolMetadata = { label: "Direct Web", description: "Provider-callable direct search.", parameters: { type: "object", properties: {} }, }; const SDK_TOOL: Tool = { name: "sdk_custom", label: "SDK Custom", description: "SDK-provided custom tool.", parameters: { type: "object", properties: {} }, approval: "read", async execute() { return { content: [{ type: "text", text: "ok" }] }; }, }; interface MetadataGetterCounts { label: number; wireName: number; description: number; parameters: number; examples: number; } function emptyMetadataGetterCounts(): MetadataGetterCounts { return { label: 0, wireName: 0, description: 0, parameters: 0, examples: 0 }; } describe("system prompt tool inventory", () => { let tempDir = ""; let tempHomeDir = ""; let originalHome: string | undefined; beforeEach(() => { tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "pi-prompt-inv-")); tempHomeDir = fs.mkdtempSync(path.join(os.tmpdir(), "pi-prompt-inv-home-")); originalHome = process.env.HOME; process.env.HOME = tempHomeDir; }); afterEach(cleanupTempHome(() => ({ tempDir, tempHomeDir, originalHome }))); async function render(opts: { nativeTools: boolean; inlineToolDescriptors: boolean }): Promise { const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["read", "bash"], tools: TOOLS, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: opts.nativeTools, inlineToolDescriptors: opts.inlineToolDescriptors, }); return systemPrompt.join("\n\n"); } function inventoryFrom(text: string): string { // Isolate the tool list across prompt layouts by stopping at the next // top-level or regular section heading. const inventoryStart = ["# Tool Inventory", "# Inventory"].map(header => text.indexOf(header)).find(index => index >= 0) ?? -1; expect(inventoryStart).toBeGreaterThan(-1); const sectionEnds = ["\nENV\n", "\nTOOL POLICY", "\n§ ", "\n# "] .map(marker => text.indexOf(marker, inventoryStart + 1)) .filter(index => index > inventoryStart); const inventoryEnd = sectionEnds.length > 0 ? Math.min(...sectionEnds) : text.length; return text.slice(inventoryStart, inventoryEnd); } async function renderMountedWebSearch(opts: { nativeTools: boolean; directDefinition: boolean; dynamic?: boolean; }): Promise<{ text: string; inventory: string }> { const tools = new Map(TOOLS); if (opts.directDefinition) tools.set("web_search", DIRECT_WEB_SEARCH); const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["read", "web_search"], tools, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: opts.nativeTools, inlineToolDescriptors: false, xdevTools: [{ name: "web_search", summary: "Searches the web.", dynamic: opts.dynamic }], xdevDocs: "Mounted web search documentation.", }); const text = systemPrompt.join("\n\n"); return { text, inventory: opts.nativeTools ? inventoryFrom(text) : text }; } function makeToolSession(settings: Settings): ToolSession { return { cwd: tempDir, hasUI: false, getSessionFile: () => null, getSessionSpawns: () => "*", settings, } as ToolSession; } it("preserves the one-argument full metadata builder", () => { const metadata = buildSystemPromptToolMetadata(new Map([[SDK_TOOL.name, SDK_TOOL]])); expect(Array.from(metadata.keys())).toEqual(["sdk_custom"]); expect(metadata.get("sdk_custom")).toMatchObject({ label: "SDK Custom", description: "SDK-provided custom tool.", parameters: { type: "object", properties: {} }, }); }); it("preserves the legacy metadata overrides map", () => { const metadata = buildSystemPromptToolMetadata(new Map([[SDK_TOOL.name, SDK_TOOL]]), { sdk_custom: { label: "Overridden label", description: "Overridden description.", wireName: "sdk_custom_wire", }, }); expect(metadata.get("sdk_custom")).toMatchObject({ label: "Overridden label", description: "Overridden description.", parameters: { type: "object", properties: {} }, wireName: "sdk_custom_wire", }); }); it("snapshots every full metadata getter once per rebuild and keeps fresh values", async () => { let revision = 1; const reads = new Map(); const makeTool = (name: string): Tool => { const counts = emptyMetadataGetterCounts(); reads.set(name, counts); return { name, approval: "read", get label() { counts.label += 1; return `${name} label r${revision}`; }, get customWireName() { counts.wireName += 1; return `${name}_wire_r${revision}`; }, get description() { counts.description += 1; return `${name} description r${revision}`; }, get parameters() { counts.parameters += 1; return { type: "object", properties: { [`arg_r${revision}`]: { type: "string" } }, required: [`arg_r${revision}`], }; }, get examples() { counts.examples += 1; return [{ caption: `${name} example r${revision}`, note: `note r${revision}` }]; }, async execute() { return { content: [{ type: "text", text: "ok" }] }; }, }; }; const tools = new Map([ ["read", makeTool("read")], ["edit", makeTool("edit")], ]); const first = projectSystemPromptToolMetadata(tools, { mode: "full" }); expect(Array.from(first.keys())).toEqual(["read", "edit"]); expect(first.get("edit")).toEqual({ label: "edit label r1", description: "edit description r1", parameters: { type: "object", properties: { arg_r1: { type: "string" } }, required: ["arg_r1"], }, examples: [{ caption: "edit example r1", note: "note r1" }], wireName: "edit_wire_r1", }); expect(Array.from(reads.values())).toEqual([ { label: 1, wireName: 1, description: 1, parameters: 1, examples: 1 }, { label: 1, wireName: 1, description: 1, parameters: 1, examples: 1 }, ]); const firstPrompt = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["edit", "read"], tools: first, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: false, inlineToolDescriptors: false, }); const firstText = firstPrompt.systemPrompt.join("\n\n"); expect(firstText.indexOf("type edit_wire_r1 = (")).toBeLessThan(firstText.indexOf("type read_wire_r1 = (")); expect(firstText).toContain("edit description r1"); expect(firstText).toContain("arg_r1: string,"); revision = 2; const second = projectSystemPromptToolMetadata(tools, { mode: "full" }); expect(second.get("edit")?.description).toBe("edit description r2"); expect(second.get("edit")?.wireName).toBe("edit_wire_r2"); expect(first.get("edit")?.description).toBe("edit description r1"); expect(Array.from(reads.values())).toEqual([ { label: 2, wireName: 2, description: 2, parameters: 2, examples: 2 }, { label: 2, wireName: 2, description: 2, parameters: 2, examples: 2 }, ]); const secondPrompt = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["edit", "read"], tools: second, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: false, inlineToolDescriptors: false, }); const secondText = secondPrompt.systemPrompt.join("\n\n"); expect(secondText.indexOf("type edit_wire_r2 = (")).toBeLessThan(secondText.indexOf("type read_wire_r2 = (")); expect(secondText).toContain("edit description r2"); expect(secondText).toContain("arg_r2: string,"); expect(secondText).not.toContain("edit description r1"); }); it("projects compact metadata in active order without reading descriptors or inactive tools", async () => { const reads = new Map(); const makeTool = (name: string, label: string, wireName?: string): Tool => { const counts = emptyMetadataGetterCounts(); reads.set(name, counts); return { name, approval: "read", get label() { counts.label += 1; return label; }, get customWireName() { counts.wireName += 1; return wireName; }, get description(): string { counts.description += 1; throw new Error(`${name} description getter was read`); }, get parameters(): Tool["parameters"] { counts.parameters += 1; throw new Error(`${name} parameters getter was read`); }, get examples(): Tool["examples"] { counts.examples += 1; throw new Error(`${name} examples getter was read`); }, async execute() { return { content: [{ type: "text", text: "ok" }] }; }, }; }; const tools = new Map([ ["inactive", makeTool("inactive", "Inactive")], ["read", makeTool("read", "Read")], ["edit", makeTool("edit", "Edit", "apply_patch")], ]); const metadata = projectSystemPromptToolMetadata(tools, { mode: "compact", toolNames: ["edit", "read"], }); expect(Array.from(metadata.keys())).toEqual(["edit", "read"]); expect(metadata.get("edit")).toMatchObject({ label: "Edit", wireName: "apply_patch" }); expect(metadata.get("read")).toMatchObject({ label: "Read" }); expect(reads.get("inactive")).toEqual(emptyMetadataGetterCounts()); expect(reads.get("edit")).toEqual({ label: 1, wireName: 1, description: 0, parameters: 0, examples: 0, }); expect(reads.get("read")).toEqual({ label: 1, wireName: 1, description: 0, parameters: 0, examples: 0, }); const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["edit", "read"], tools: metadata, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: true, inlineToolDescriptors: false, }); expect(inventoryFrom(systemPrompt.join("\n\n")).trim()).toBe( "# Tool Inventory\n- Edit: `apply_patch`\n- Read: `read`", ); }); it("does not construct descriptor records for a compact native inventory", async () => { const reads = new Map(); const makeMetadata = (name: string, label: string, wireName?: string): SystemPromptToolMetadata => { const counts = emptyMetadataGetterCounts(); reads.set(name, counts); return { get label() { counts.label += 1; return label; }, get wireName() { counts.wireName += 1; return wireName; }, get description(): string { counts.description += 1; throw new Error(`${name} description getter was read`); }, get parameters(): SystemPromptToolMetadata["parameters"] { counts.parameters += 1; throw new Error(`${name} parameters getter was read`); }, get examples(): SystemPromptToolMetadata["examples"] { counts.examples += 1; throw new Error(`${name} examples getter was read`); }, }; }; const metadata = new Map([ ["read", makeMetadata("read", "Read")], ["edit", makeMetadata("edit", "Edit", "apply_patch")], ]); const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["edit", "read"], tools: metadata, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: true, inlineToolDescriptors: false, }); expect(inventoryFrom(systemPrompt.join("\n\n")).trim()).toBe( "# Tool Inventory\n- Edit: `apply_patch`\n- Read: `read`", ); expect(Array.from(reads.values())).toEqual([ { label: 1, wireName: 1, description: 0, parameters: 0, examples: 0 }, { label: 1, wireName: 1, description: 0, parameters: 0, examples: 0 }, ]); }); it("renders a compact name list only when native tools are active and descriptors stay in schemas", async () => { const text = await render({ nativeTools: true, inlineToolDescriptors: false }); expect(text).toContain("- Read: `read`"); expect(text).toContain("- Bash: `bash`"); // No full per-tool sections in list mode. expect(text).not.toContain("namespace functions"); expect(text).not.toContain("Reads files from disk."); }); it("keeps enabled computer routing explicit in compact native-tool mode", async () => { const tools = new Map(TOOLS); tools.set("computer", { label: "Computer", description: "Controls the host desktop.", parameters: { type: "object", properties: {} }, }); const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames: ["read", "computer"], tools, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: true, inlineToolDescriptors: false, }); const text = systemPrompt.join("\n\n"); expect(text).toContain("# Computer Use"); }); it("renders the functions namespace (not a name list) when tools are not native", async () => { const text = await render({ nativeTools: false, inlineToolDescriptors: false }); expect(text).toContain("namespace functions {"); expect(text).toContain("type read = (_: {"); expect(text).toContain("type bash = (_: {"); expect(text).toContain("Reads files from disk."); expect(text).not.toContain("- Read: `read`"); // The legacy `` wrapper is gone. expect(text).not.toContain(" { const text = await render({ nativeTools: true, inlineToolDescriptors: true }); expect(text).toContain("type read = (_: {"); expect(text).toContain("Executes a shell command."); expect(text).not.toContain("- Read: `read`"); }); it.each([ ["compact", true], ["inline", false], ] as const)("omits xd-only tools from the %s inventory", async (_mode, nativeTools) => { const { text, inventory } = await renderMountedWebSearch({ nativeTools, directDefinition: false }); expect(inventory).toContain(nativeTools ? "`read`" : "type read = (_: {"); expect(inventory).not.toContain(nativeTools ? "`web_search`" : "type web_search = ("); expect(text).toContain("# xd:// Tool Devices"); expect(text).toContain("Mounted web search documentation."); }); it.each([ ["compact", true], ["inline", false], ] as const)("keeps direct tools that share an xd device name in the %s inventory", async (_mode, nativeTools) => { const { inventory } = await renderMountedWebSearch({ nativeTools, directDefinition: true }); expect(inventory).toContain(nativeTools ? "- Direct Web: `web_search`" : "type web_search = ("); if (!nativeTools) expect(inventory).toContain(DIRECT_WEB_SEARCH.description); }); it("uses a conservative fallback inventory when no tools map is provided", async () => { const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, }); const inventory = inventoryFrom(systemPrompt.join("\n\n")); for (const toolName of DEFAULT_SYSTEM_PROMPT_TOOL_NAMES) { expect(inventory).toContain(`- \`${toolName}\``); } expect(inventory).not.toContain("- `browser`"); expect(inventory).not.toContain("- `task`"); expect(inventory).not.toContain("- `eval`"); }); it("omits eval prompt guidance when every eval backend is disabled", async () => { const settings = Settings.isolated({ "eval.py": false, "eval.js": false, "eval.rb": false, "eval.jl": false, }); const session = makeToolSession(settings); const tools = await createTools(session, ["bash", "eval"]); const toolNames = tools.map(tool => tool.name); const bash = tools.find(tool => tool.name === "bash"); expect(toolNames).toContain("bash"); expect(toolNames).not.toContain("eval"); expect(bash?.description).toContain("purpose-built tool"); expect(bash?.description).not.toContain("eval` cell"); expect(bash?.description).not.toContain("use `eval` cells"); expect(bash?.description).not.toContain("Prefer `eval`"); expect(bash?.description).not.toContain("`grep` tool"); expect(bash?.description).not.toContain("`ls` → `read`"); expect(bash?.description).not.toContain("`find` → the `glob` tool"); const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], rules: [], toolNames, tools: buildSystemPromptToolMetadata(new Map(tools.map(tool => [tool.name, tool]))), workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, nativeTools: true, inlineToolDescriptors: true, }); const text = systemPrompt.join("\n\n"); expect(text).not.toContain("Default for any compute"); expect(text).not.toContain("use `eval` cells"); }); it("SDK wrapper renders provided tools instead of the fallback inventory", async () => { const { systemPrompt } = await buildSdkSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], tools: [SDK_TOOL], }); const inventory = inventoryFrom(systemPrompt.join("\n\n")); expect(inventory).toContain("- SDK Custom: `sdk_custom`"); expect(inventory).not.toContain("- `read`"); }); it("SDK wrapper preserves an explicit empty tool list", async () => { const { systemPrompt } = await buildSdkSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [], tools: [], }); const text = systemPrompt.join("\n\n"); expect(text).not.toContain("# Inventory"); expect(text).not.toContain("- `read`"); }); it("keeps visible skills when no tools map is provided", async () => { const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [ { name: "prompt-authoring", description: "Prompt authoring workflow", filePath: path.join(tempDir, "SKILL.md"), baseDir: tempDir, source: "test", }, ], rules: [], workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, }); const text = systemPrompt.join("\n\n"); expect(text).toContain("- prompt-authoring: Prompt authoring workflow"); }); it("omits skills when active tool names exclude read", async () => { const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [ { name: "search-only-skill", description: "Should not render without read", filePath: path.join(tempDir, "SKILL.md"), baseDir: tempDir, source: "test", }, ], rules: [], toolNames: ["bash"], tools: TOOLS, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, }); const text = systemPrompt.join("\n\n"); expect(text).not.toContain("search-only-skill"); }); it("omits hidden skills even when read is active", async () => { const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [ { name: "hidden-workflow", description: "Hidden prompt workflow", filePath: path.join(tempDir, "SKILL.md"), baseDir: tempDir, source: "test", hide: true, }, ], rules: [], toolNames: ["read"], tools: TOOLS, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, }); const text = systemPrompt.join("\n\n"); expect(text).not.toContain("hidden-workflow"); }); it("tells the agent to read matching skills before work", async () => { const { systemPrompt } = await buildSystemPrompt({ cwd: tempDir, contextFiles: [], skills: [ { name: "frontend-design", description: "Frontend UI workflow", filePath: path.join(tempDir, "SKILL.md"), baseDir: tempDir, source: "test", }, ], rules: [], toolNames: ["read"], tools: TOOLS, workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, }); const text = systemPrompt.join("\n\n"); expect(text).toContain(""); expect(text).toContain("- frontend-design: Frontend UI workflow"); }); it("omits the read-only scout delegation gate when scout is unavailable", async () => { const opts = { toolNames: ["read", "bash", "task"], tools: TOOLS }; const withScout = ( await buildSystemPrompt({ ...opts, cwd: tempDir, contextFiles: [], skills: [], rules: [], workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, scoutAvailable: true, }) ).systemPrompt.join("\n\n"); const withoutScout = ( await buildSystemPrompt({ ...opts, cwd: tempDir, contextFiles: [], skills: [], rules: [], workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, scoutAvailable: false, }) ).systemPrompt.join("\n\n"); expect(withScout).toContain("one read-only scout while working is allowed"); expect(withoutScout).not.toContain("read-only scout"); }); it("does not require browser verification when the browser tool is absent (issue #8139)", async () => { const opts = { cwd: tempDir, contextFiles: [], skills: [], rules: [], workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, }; const tools = new Map(TOOLS); const withoutBrowser = ( await buildSystemPrompt({ ...opts, toolNames: ["read", "bash"], tools, nativeTools: true, inlineToolDescriptors: false, }) ).systemPrompt.join("\n\n"); expect(withoutBrowser).not.toContain("browser-drive with `browser`"); expect(withoutBrowser).not.toContain("browser-drive with browser"); expect(withoutBrowser).toContain("TUI/CLI"); expect(withoutBrowser).toContain("behavioral test or smoke test"); tools.set("browser", { label: "Browser", description: "Drives a real Chromium tab.", parameters: { type: "object", properties: {} }, }); const withBrowser = ( await buildSystemPrompt({ ...opts, toolNames: ["read", "bash", "browser"], tools, nativeTools: true, inlineToolDescriptors: false, }) ).systemPrompt.join("\n\n"); expect(withBrowser).toContain("browser-drive with `browser`"); // A browser-only session still needs the smoke-test fallback for // native-desktop surfaces (no computer tool). expect(withBrowser).toContain("behavioral test or smoke test"); }); it("omits todo workflow guidance when the todo tool is absent", async () => { const opts = { cwd: tempDir, contextFiles: [], skills: [], rules: [], workspaceTree: { ...EMPTY_TREE, rootPath: tempDir }, tools: TOOLS, nativeTools: true, inlineToolDescriptors: false, }; const withoutTodo = (await buildSystemPrompt({ ...opts, toolNames: ["read", "bash"] })).systemPrompt.join("\n\n"); expect(withoutTodo).not.toContain("Todo calls NEVER alone"); expect(withoutTodo).not.toContain("batch each with turn's real calls"); const withTodo = (await buildSystemPrompt({ ...opts, toolNames: ["read", "bash", "todo"] })).systemPrompt.join( "\n\n", ); expect(withTodo).toContain("Todo calls NEVER alone"); }); });