diff --git a/docs/settings.md b/docs/settings.md index 6837dd29e..e02ff0d78 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -478,7 +478,7 @@ tools: | `tools.artifactTailBytes` | number | `20` | KB of tail kept inline on spill. | | `tools.artifactTailLines` | number | `500` | Max tail lines kept inline on spill. | -Individual built-in tools are toggled by their own keys, e.g. `bash.enabled`, `launch.enabled`, `eval.py`, `eval.js`, `glob.enabled`, `grep.enabled`, `fetch.enabled`, `browser.enabled`, `computer.enabled`, `astEdit.enabled`, `astGrep.enabled`, `web_search.enabled`, and `inspect_image.enabled`. +Individual built-in tools are toggled by their own keys, e.g. `bash.enabled`, `launch.enabled`, `eval.py`, `eval.js`, `glob.enabled`, `grep.enabled`, `fetch.enabled`, `browser.enabled`, `computer.enabled`, `astEdit.enabled`, `astGrep.enabled`, and `web_search.enabled`. The `inspect_image` tool is controlled by the tri-state `inspect_image.mode` (`auto`|`on`|`off`, default `auto`): `auto` exposes it only when the active model lacks native image input, and the `/vision` slash command overrides the mode per session. ### Native computer use @@ -744,6 +744,7 @@ Applied whenever raw settings are loaded (global, project, overlays, and runtime | Old | New | |---|---| +| `inspect_image.enabled` boolean | `inspect_image.mode` (`true` → `on`, `false` → `off`) | | `queueMode` | `steeringMode` | | `ask.timeout` in milliseconds (value `> 1000`) | seconds (divided by 1000) | | flat `theme: ""` string | `theme.dark` / `theme.light` (slot chosen by luminance; built-in `light`/`dark` are dropped to use defaults) | diff --git a/docs/tools/inspect_image.md b/docs/tools/inspect_image.md index 36454eb70..3b8404ce8 100644 --- a/docs/tools/inspect_image.md +++ b/docs/tools/inspect_image.md @@ -75,7 +75,7 @@ TUI rendering adds presentation-only truncation from `packages/coding-agent/src/ ## Limits & Caps - Supported detected input formats: `image/png`, `image/jpeg`, `image/gif`, `image/webp` (`SUPPORTED_IMAGE_MIME_TYPES` in `packages/utils/src/mime.ts`). - Metadata sniff cap: `DEFAULT_IMAGE_METADATA_HEADER_BYTES = 256 * 1024` bytes. Format detection only reads up to 256 KiB from the file header. -- Availability is gated by `inspect_image.enabled`, default `false`, in `packages/coding-agent/src/config/settings-schema.ts` / `packages/coding-agent/src/tools/index.ts`. +- Availability is gated by `inspect_image.mode` (`auto`|`on`|`off`, default `auto`) in `packages/coding-agent/src/config/settings-schema.ts`, resolved with the session-scoped `/vision` override and the active model's image capability in `packages/coding-agent/src/utils/inspect-image-mode.ts` / `packages/coding-agent/src/tools/index.ts`. `auto` registers the tool only when the active model lacks native image input; the legacy `inspect_image.enabled` boolean migrates to `mode` (`true`→`on`, `false`→`off`). - Upload input cap: `MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024` bytes (20 MiB) in `packages/coding-agent/src/utils/image-loading.ts`. - Auto-resize defaults in `packages/coding-agent/src/utils/image-resize.ts`: - `maxWidth: 1568` diff --git a/docs/tools/read.md b/docs/tools/read.md index 390517a6c..b2cd29b13 100644 --- a/docs/tools/read.md +++ b/docs/tools/read.md @@ -188,7 +188,7 @@ URL selectors are parsed separately in `packages/coding-agent/src/tools/fetch.ts ### Images - Image detection is metadata-based (`readImageMetadata()`). - Max accepted image size is `20 MiB` (`MAX_IMAGE_INPUT_BYTES`, re-exported as `MAX_IMAGE_SIZE`). Larger files throw. -- If `inspect_image.enabled` is true, `read` returns metadata only (MIME, bytes, dimensions, channels, alpha) plus a suggestion to call `inspect_image`. +- If the effective `inspect_image` state is active (mode `on`, or `auto` with an active model that lacks native image input), `read` returns metadata only (MIME, bytes, dimensions, channels, alpha) plus a suggestion to call `inspect_image`. - Otherwise it calls `loadImageInput()` and returns: - a text note from the image loader - an inline image block diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 13827c45a..22d49c2ce 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -13,6 +13,11 @@ - Session listing now caches parsed headers keyed on file stat identity (mtime + size), so repeated resume-picker opens and startup scans re-read only changed session files - Reduced per-keystroke editor dispatch overhead: keybinding resolution happens once per input chunk and the per-action interception chain is gated behind a single canonical-key set probe - `xd://` device docs now render the parameter schema as a comment-annotated TypeScript type (via `jsonSchemaToTypeScript`, the same renderer the in-band tool inventory uses) instead of a raw JSON Schema dump, shrinking system-prompt device sections while keeping descriptions inline. +- Added a `/vision [on|off|auto|status]` slash command for session-scoped control of the `inspect_image` vision-delegation tool, modeled on `/computer`: `on`/`off` force the tool for the current session only, `auto` returns to the persisted setting, and `status` reports the effective mode, session override, tool state, and active-model image capability. + +### Changed + +- Replaced the `inspect_image.enabled` boolean with the tri-state `inspect_image.mode` (`auto`|`on`|`off`, default `auto`). In `auto` the tool is registered only when the active model lacks native image input, so vision-capable models (e.g. `kimi-code/k3`) read images inline with their own capabilities instead of delegating to a separate vision model; the tool set is re-evaluated on every model switch with a status notice when it flips. The `read` tool now follows the effective state dynamically rather than the raw setting, so it returns decoded image blocks again whenever `inspect_image` is hidden. Existing `inspect_image.enabled: true/false` configs migrate to `inspect_image.mode: on/off`. ## [17.1.6] - 2026-07-27 diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 2755a1ff2..ba43939d3 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -3835,14 +3835,28 @@ export const SETTINGS_SCHEMA = { }, }, + // Legacy boolean kept only for back-compat migration to `inspect_image.mode` + // (see config/settings.ts). Hidden from UI. "inspect_image.enabled": { type: "boolean", default: false, + }, + + "inspect_image.mode": { + type: "enum", + values: ["auto", "on", "off"] as const, + default: "auto", ui: { tab: "tools", group: "Available Tools", label: "Inspect Image", - description: "Enable the inspect_image tool, delegating image understanding to a vision-capable model", + description: + "Controls the inspect_image tool, which delegates image understanding to a vision-capable model. 'auto' exposes it only when the active model lacks native image input; 'on' always exposes it; 'off' never does.", + options: [ + { value: "auto", label: "Auto (only for models without vision)" }, + { value: "on", label: "On" }, + { value: "off", label: "Off" }, + ], }, }, diff --git a/packages/coding-agent/src/config/settings.ts b/packages/coding-agent/src/config/settings.ts index e20843ce3..d8e391096 100644 --- a/packages/coding-agent/src/config/settings.ts +++ b/packages/coding-agent/src/config/settings.ts @@ -39,6 +39,7 @@ import { isLightTheme, setAutoThemeMapping, setColorBlindMode, setSymbolPreset } import { AgentStorage } from "../session/agent-storage"; import { AUTO_IMAGE_PROVIDER_ORDER, isImageProviderId } from "../tools/image-providers"; import { type EditMode, normalizeEditMode } from "../utils/edit-mode"; +import { INSPECT_IMAGE_MODES } from "../utils/inspect-image-mode"; import { isSearchProviderId, SEARCH_PROVIDER_ORDER } from "../web/search/types"; import { withFileLock } from "./file-lock"; import { @@ -1366,6 +1367,40 @@ export class Settings { } } + // inspect_image.enabled (boolean) -> inspect_image.mode (enum). Explicit + // user choices are preserved: true -> "on", false -> "off". Configs with + // no legacy key get the new "auto" default, which hides the tool for + // models with native image input. Handles nested and quoted-dotted + // ("inspect_image.enabled") sources; the target is always the nested + // form, which is the only shape the resolver reads. + const inspectImageObj = isRecord(raw.inspect_image) ? (raw.inspect_image as Record) : undefined; + const legacyEnabled = + typeof inspectImageObj?.enabled === "boolean" + ? inspectImageObj.enabled + : typeof raw["inspect_image.enabled"] === "boolean" + ? (raw["inspect_image.enabled"] as boolean) + : undefined; + if (legacyEnabled !== undefined) { + if (!inspectImageObj) { + raw.inspect_image = {}; + } + const target = raw.inspect_image as Record; + const flatMode = raw["inspect_image.mode"]; + if (target.mode === undefined) { + // A quoted-dotted explicit mode wins over the legacy boolean but + // must be normalized into the nested form the resolver reads. + target.mode = + typeof flatMode === "string" && (INSPECT_IMAGE_MODES as readonly string[]).includes(flatMode) + ? flatMode + : legacyEnabled + ? "on" + : "off"; + } + delete target.enabled; + delete raw["inspect_image.enabled"]; + delete raw["inspect_image.mode"]; + } + // task.isolation.enabled (boolean) -> task.isolation.mode (enum) const taskObj = raw.task as Record | undefined; const isolationObj = taskObj?.isolation as Record | undefined; diff --git a/packages/coding-agent/src/modes/controllers/selector-controller.ts b/packages/coding-agent/src/modes/controllers/selector-controller.ts index d6602a4bb..b3b7ab8e9 100644 --- a/packages/coding-agent/src/modes/controllers/selector-controller.ts +++ b/packages/coding-agent/src/modes/controllers/selector-controller.ts @@ -455,6 +455,11 @@ export class SelectorController { this.ctx.showError(`Failed to apply memory backend: ${err}`); }); break; + case "inspect_image.mode": + void this.ctx.session.applyInspectImageModeChange().catch(err => { + this.ctx.showError(`Failed to apply vision mode: ${err}`); + }); + break; case "autocompleteMaxVisible": this.ctx.editor.setAutocompleteMaxVisible(typeof value === "number" ? value : Number(value)); diff --git a/packages/coding-agent/src/sdk.ts b/packages/coding-agent/src/sdk.ts index 28084db47..9568607aa 100644 --- a/packages/coding-agent/src/sdk.ts +++ b/packages/coding-agent/src/sdk.ts @@ -1682,6 +1682,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} getModelString: () => (hasExplicitModel && model ? formatModelString(model) : undefined), getActiveModelString, getActiveModel: () => agent?.state.model ?? model, + getInspectImageModeOverride: () => session?.getInspectImageModeOverride(), getServiceTierByFamily: () => session?.serviceTierByFamily, getImageAttachments: () => session?.getImageAttachments() ?? [], getPlanModeState: () => session?.getPlanModeState(), @@ -3178,6 +3179,9 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} createComputerTool: restrictToolNames ? undefined : async () => (await BUILTIN_TOOLS.computer(toolSession)) ?? null, + createInspectImageTool: restrictToolNames + ? undefined + : async () => (await BUILTIN_TOOLS.inspect_image(toolSession)) ?? null, createVibeTools: (options.taskDepth ?? 0) === 0 && !options.parentTaskPrefix ? () => createVibeTools(toolSession) diff --git a/packages/coding-agent/src/session/agent-session-types.ts b/packages/coding-agent/src/session/agent-session-types.ts index e6e4a2bf9..b5513df3a 100644 --- a/packages/coding-agent/src/session/agent-session-types.ts +++ b/packages/coding-agent/src/session/agent-session-types.ts @@ -139,6 +139,8 @@ export interface AgentSessionConfig { createMemoryTools?: () => Promise; /** Creates the built-in `computer` tool for session-scoped runtime enablement (see {@link AgentSession.setComputerToolEnabled}). */ createComputerTool?: () => Promise; + /** Creates the built-in `inspect_image` tool for session-scoped runtime enablement (see {@link AgentSession.setInspectImageMode}). */ + createInspectImageTool?: () => Promise; /** Model registry for API key resolution and model discovery. */ modelRegistry: ModelRegistry; /** Tool registry for LSP and settings. */ diff --git a/packages/coding-agent/src/session/agent-session.ts b/packages/coding-agent/src/session/agent-session.ts index b1b1bb1a8..a250de9b3 100644 --- a/packages/coding-agent/src/session/agent-session.ts +++ b/packages/coding-agent/src/session/agent-session.ts @@ -200,6 +200,7 @@ import type { EditMode } from "../utils/edit-mode"; import { resolveFileDisplayMode } from "../utils/file-display-mode"; import { extractFileMentions, generateFileMentionMessages } from "../utils/file-mentions"; import { normalizeModelContextImages } from "../utils/image-loading"; +import type { InspectImageMode } from "../utils/inspect-image-mode"; import { generateSessionTitle } from "../utils/title-generator"; import { buildNamedToolChoice, isToolChoiceActive } from "../utils/tool-choice"; import type { VibeModeState } from "../vibe/state"; @@ -433,6 +434,8 @@ export class AgentSession { #scheduledHiddenNextTurnGeneration: number | undefined = undefined; #queuedMessageDrainScheduled = false; #planModeState: PlanModeState | undefined; + /** Session-scoped `/vision` override; undefined = follow persisted `inspect_image.mode`. */ + #inspectImageModeOverride: InspectImageMode | undefined; #vibeModeState: VibeModeState | undefined; #goalModeState: GoalModeState | undefined; #goalRuntime: GoalRuntime; @@ -1085,12 +1088,17 @@ export class AgentSession { emitNotice: (level, message, source) => this.emitNotice(level, message, source), notifyCommandMetadataChanged: () => this.#notifyCommandMetadataChanged(), localProtocolOptions: () => this.#localProtocolOptions(), + getInspectImageModeOverride: () => this.#inspectImageModeOverride, + setInspectImageModeOverride: mode => { + this.#inspectImageModeOverride = mode; + }, }; this.#tools = new SessionTools(sessionToolsHost, { autoApprove: config.autoApprove, toolRegistry: config.toolRegistry, createVibeTools: config.createVibeTools, createComputerTool: config.createComputerTool, + createInspectImageTool: config.createInspectImageTool, builtInToolNames: config.builtInToolNames, presentationPinnedToolNames: config.presentationPinnedToolNames, ensureWriteRegistered: config.ensureWriteRegistered, @@ -4002,6 +4010,34 @@ export class AgentSession { return this.#tools.setComputerToolEnabled(enabled); } + /** + * Session-scoped inspect_image mode (`/vision`). `auto` clears the override + * and returns to the persisted `inspect_image.mode` setting; `on`/`off` + * force the tool for this session only. See {@link SessionTools.setInspectImageMode}. + */ + setInspectImageMode(mode: InspectImageMode): Promise { + return this.#tools.setInspectImageMode(mode); + } + + /** Effective inspect_image state for `/vision status`. */ + inspectImageState(): { mode: InspectImageMode; active: boolean; model: string | undefined } { + return this.#tools.inspectImageState(); + } + + /** Session-scoped `/vision` override; undefined means "follow the persisted setting". */ + getInspectImageModeOverride(): InspectImageMode | undefined { + return this.#inspectImageModeOverride; + } + + /** + * Reconciles the inspect_image tool set after the persisted + * `inspect_image.mode` setting changed (e.g. via the settings selector), so + * the new value takes effect immediately instead of on the next model switch. + */ + applyInspectImageModeChange(): Promise { + return this.#tools.reconcileInspectImageTool(); + } + /** Cancels the local rollout-memory startup owned by this session. */ cancelLocalMemoryStartup(): void { this.#memory.cancelLocalMemoryStartup(); @@ -6469,6 +6505,14 @@ export class AgentSession { // Re-evaluate append-only context mode — provider or setting may have changed this.#syncAppendOnlyContext(model); + + // inspect_image auto mode keys off model image capability. Reconcile + // centrally here so retry-fallback model changes (turn-recovery.ts), + // which bypass syncAfterModelChange, cannot leave the tool set stale. + // Idempotent; syncAfterModelChange's own reconcile is then a no-op. + void this.#tools.reconcileInspectImageAfterModelChange().catch(error => { + logger.warn("inspect_image reconcile after model change failed", { error: String(error) }); + }); } #closeCodexProviderSessionsForHistoryRewrite(): void { diff --git a/packages/coding-agent/src/session/session-tools.ts b/packages/coding-agent/src/session/session-tools.ts index 2384523f6..7129a629c 100644 --- a/packages/coding-agent/src/session/session-tools.ts +++ b/packages/coding-agent/src/session/session-tools.ts @@ -23,6 +23,7 @@ import { wrapToolWithMetaNotice } from "../tools/output-meta"; import { ToolAbortError, ToolError } from "../tools/tool-errors"; import { isMountableUnderXdev, type XdevRegistry } from "../tools/xdev"; import { type EditMode, resolveEditMode } from "../utils/edit-mode"; +import { type InspectImageMode, isInspectImageToolActive } from "../utils/inspect-image-mode"; import { formatLocalCalendarDate } from "../utils/local-date"; import { extractPermissionLocations, @@ -56,6 +57,9 @@ export interface SessionToolsHost { emitNotice(level: "info" | "warning" | "error", message: string, source?: string): void; notifyCommandMetadataChanged(): void; localProtocolOptions(): LocalProtocolOptions; + /** Session-scoped `/vision` override; undefined means "follow the persisted setting". */ + getInspectImageModeOverride(): InspectImageMode | undefined; + setInspectImageModeOverride(mode: InspectImageMode | undefined): void; } interface SessionToolsOptions { @@ -63,6 +67,8 @@ interface SessionToolsOptions { toolRegistry?: Map; createVibeTools?: () => AgentTool[]; createComputerTool?: () => Promise; + /** Creates the built-in `inspect_image` tool for session-scoped runtime enablement (see {@link SessionTools.setInspectImageMode}). */ + createInspectImageTool?: () => Promise; builtInToolNames?: Iterable; presentationPinnedToolNames?: ReadonlySet; ensureWriteRegistered?: () => Promise; @@ -161,6 +167,7 @@ export class SessionTools { #toolRegistry: Map; #createVibeTools: (() => AgentTool[]) | undefined; #createComputerTool: SessionToolsOptions["createComputerTool"]; + #createInspectImageTool: SessionToolsOptions["createInspectImageTool"]; #installedVibeToolNames = new Set(); #builtInToolNames: Set; #rpcHostToolNames = new Set(); @@ -190,6 +197,7 @@ export class SessionTools { this.#toolRegistry = options.toolRegistry ?? new Map(); this.#createVibeTools = options.createVibeTools; this.#createComputerTool = options.createComputerTool; + this.#createInspectImageTool = options.createInspectImageTool; this.#builtInToolNames = new Set(options.builtInToolNames ?? []); this.#presentationPinnedToolNames = options.presentationPinnedToolNames; this.#ensureWriteRegistered = options.ensureWriteRegistered; @@ -403,6 +411,10 @@ export class SessionTools { } else if (computerExpected) { this.#logComputerState("Computer tool retained after model change", true); } + + // inspect_image auto mode keys off model image capability, so a model + // switch can flip the tool either way. + await this.reconcileInspectImageAfterModelChange(); } /** Enabled MCP tools in their current presentation partition. */ @@ -849,6 +861,108 @@ export class SessionTools { return true; } + /** Current effective inspect_image state for `/vision status`. */ + inspectImageState(): { mode: InspectImageMode; active: boolean; model: string | undefined } { + const model = this.#host.model(); + return { + mode: this.#host.getInspectImageModeOverride() ?? this.#host.settings.get("inspect_image.mode"), + active: this.getEnabledToolNames().includes("inspect_image"), + model: model ? formatModelString(model) : undefined, + }; + } + + /** + * Brings the active tool set in line with the effective inspect_image state + * (mode setting, `/vision` override, active-model image capability). + * Mirrors {@link setComputerToolEnabled}: enabling builds the tool through + * the config factory on first use and reuses the registry entry afterwards. + * Idempotent — safe to call from every model/settings change path. + * + * @returns false when the tool should be active but this session cannot + * build it (e.g. restricted child sessions have no factory). + */ + async reconcileInspectImageTool(): Promise { + const expected = isInspectImageToolActive({ + settings: this.#host.settings, + getActiveModel: () => this.#host.model(), + getInspectImageModeOverride: () => this.#host.getInspectImageModeOverride(), + }); + // Keep the read tool's advertised description in sync BEFORE any prompt + // rebuild below, passing the post-change availability so the prompt never + // lags a flip in either direction. Per-read lazy sync is the backstop. + const syncReadDescription = (available: boolean): void => { + const readTool = this.#toolRegistry.get("read") as + | { syncInspectImageState?: (available?: boolean) => boolean } + | undefined; + readTool?.syncInspectImageState?.(available); + }; + const active = this.getEnabledToolNames(); + const isActive = active.includes("inspect_image"); + if (expected === isActive) { + syncReadDescription(isActive); + return true; + } + if (!expected) { + syncReadDescription(false); + await this.applyActiveToolsByName(active.filter(name => name !== "inspect_image")); + return true; + } + if (!this.#toolRegistry.has("inspect_image")) { + const tool = await this.#createInspectImageTool?.(); + if (tool?.name !== "inspect_image") { + logger.warn("inspect_image tool could not be created", { + model: this.#host.model()?.id, + }); + syncReadDescription(false); + return false; + } + const wrapped = this.#wrapRuntimeTool(tool); + this.#toolRegistry.set(wrapped.name, wrapped); + this.#builtInToolNames.add(wrapped.name); + } + syncReadDescription(true); + await this.applyActiveToolsByName([...active, "inspect_image"]); + return true; + } + + /** + * Reconciles inspect_image after a model change and surfaces a notice when + * the visible tool set actually flipped. Called from every model-change + * path — including retry-fallback switches that bypass + * {@link syncAfterModelChange}. + */ + async reconcileInspectImageAfterModelChange(): Promise { + const before = this.getEnabledToolNames().includes("inspect_image"); + const reconciled = await this.reconcileInspectImageTool(); + const after = this.getEnabledToolNames().includes("inspect_image"); + if (!reconciled || before === after) return; + const model = this.#host.model(); + const modelName = model ? formatModelString(model) : "the current model"; + this.#host.emitNotice( + "info", + after + ? `inspect_image is now available: ${modelName} has no native image input.` + : `inspect_image is now hidden: ${modelName} supports image input natively. Override with /vision on.`, + "vision", + ); + } + + /** + * Session-scoped `/vision` override. `auto` clears the override so the + * persisted `inspect_image.mode` setting (itself possibly `auto`) decides; + * `on`/`off` force the tool for this session only. Takes effect before the + * next model call. + * + * @returns false when `on` was requested but the tool cannot be built here. + */ + async setInspectImageMode(mode: InspectImageMode): Promise { + this.#host.setInspectImageModeOverride(mode === "auto" ? undefined : mode); + const applied = await this.reconcileInspectImageTool(); + const { active, model } = this.inspectImageState(); + logger.debug("inspect_image mode changed", { mode, active, model }); + return applied; + } + /** Rebuilds the stable base prompt for the current tools and model. */ async refreshBaseSystemPrompt(): Promise { if (this.#host.isDisposed() || !this.#rebuildSystemPrompt) return; diff --git a/packages/coding-agent/src/slash-commands/builtin-registry.ts b/packages/coding-agent/src/slash-commands/builtin-registry.ts index 4f1df5674..faf6930e9 100644 --- a/packages/coding-agent/src/slash-commands/builtin-registry.ts +++ b/packages/coding-agent/src/slash-commands/builtin-registry.ts @@ -53,6 +53,7 @@ import { renderChangelogEntries, } from "../utils/changelog"; import { copyToClipboard } from "../utils/clipboard"; +import type { InspectImageMode } from "../utils/inspect-image-mode"; import { CollabQrCodeComponent } from "./helpers/collab-qrcode"; import { buildContextReportText } from "./helpers/context-report"; import { formatDuration } from "./helpers/format"; @@ -151,6 +152,33 @@ async function applyComputerUseToggle(session: AgentSession, enable: boolean): P : "Computer use disabled for this session."; } +/** Session-effective `/vision status` line. */ +function formatVisionStatus(session: AgentSession): string { + const { mode, active, model } = session.inspectImageState(); + const override = session.getInspectImageModeOverride(); + const modelObj = session.model; + const capability = modelObj + ? modelObj.input.includes("image") + ? "native image input" + : "no native image input" + : "no active model"; + return [ + `inspect_image: ${active ? "active" : "inactive"}`, + `mode: ${mode}${override ? " (session override)" : ""}`, + ...(override ? [`configured: ${session.settings.get("inspect_image.mode")}`] : []), + `model: ${model ?? "none"} (${capability})`, + ].join(" · "); +} + +/** Applies a `/vision` mode for this session and returns the operator feedback line. */ +async function applyVisionMode(session: AgentSession, mode: InspectImageMode): Promise { + const applied = await session.setInspectImageMode(mode); + if (!applied) { + return "inspect_image is unavailable in this session."; + } + return `Vision mode: ${mode}. ${formatVisionStatus(session)}`; +} + const AUTOCOMPLETE_DETAIL_LIMIT = 48; function shortDetail(value: string, limit = AUTOCOMPLETE_DETAIL_LIMIT): string { @@ -647,6 +675,47 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray = [ runtime.ctx.editor.setText(""); }, }, + { + name: "vision", + description: "Control the inspect_image vision-delegation tool for this session", + acpDescription: "Toggle vision delegation", + acpInputHint: "[on|off|auto|status]", + subcommands: [ + { name: "on", description: "Always expose inspect_image this session" }, + { name: "off", description: "Never expose inspect_image this session" }, + { name: "auto", description: "Follow inspect_image.mode (auto hides it for vision-capable models)" }, + { name: "status", description: "Show inspect_image status" }, + ], + allowArgs: true, + getTuiAutocompleteDescription: runtime => `Vision: ${runtime.ctx.session.inspectImageState().mode}`, + handle: async (command, runtime) => { + const arg = command.args.trim().toLowerCase(); + if (arg === "status") { + await runtime.output(formatVisionStatus(runtime.session)); + return commandConsumed(); + } + if (arg === "on" || arg === "off" || arg === "auto") { + await runtime.output(await applyVisionMode(runtime.session, arg)); + return commandConsumed(); + } + return usage("Usage: /vision [on|off|auto|status]", runtime); + }, + handleTui: async (command, runtime) => { + const arg = command.args.trim().toLowerCase(); + if (arg === "status") { + runtime.ctx.showStatus(formatVisionStatus(runtime.ctx.session)); + runtime.ctx.editor.setText(""); + return; + } + if (arg === "on" || arg === "off" || arg === "auto") { + runtime.ctx.showStatus(await applyVisionMode(runtime.ctx.session, arg)); + runtime.ctx.editor.setText(""); + return; + } + runtime.ctx.showStatus("Usage: /vision [on|off|auto|status]"); + runtime.ctx.editor.setText(""); + }, + }, { name: "prewalk", description: "Switch to a fast/cheap model at the next action (works even without --prewalk)", diff --git a/packages/coding-agent/src/tools/index.ts b/packages/coding-agent/src/tools/index.ts index 2a1d0966d..1ca113245 100644 --- a/packages/coding-agent/src/tools/index.ts +++ b/packages/coding-agent/src/tools/index.ts @@ -32,6 +32,7 @@ import { TaskTool } from "../task"; import type { AgentOutputManager } from "../task/output-manager"; import { canSpawnAtDepth, type StructuredSubagentSchemaMode } from "../task/types"; import type { EventBus } from "../utils/event-bus"; +import { type InspectImageMode, isInspectImageToolActive } from "../utils/inspect-image-mode"; import { WebSearchTool } from "../web/search"; import type { WorkspaceTree } from "../workspace-tree"; import { AskTool } from "./ask"; @@ -263,6 +264,8 @@ export interface ToolSession { getActiveModelString?: () => string | undefined; /** Get the current session model object (provider/api capabilities), regardless of how it was chosen. */ getActiveModel?: () => Model | undefined; + /** Session-scoped inspect_image mode override set by `/vision`; wins over the persisted setting. */ + getInspectImageModeOverride?: () => InspectImageMode | undefined; /** Get the session's live per-family service tiers (undefined = none). Source of truth for subagent `tier.subagent: inherit`. */ getServiceTierByFamily?: () => ServiceTierByFamily | undefined; /** Auth storage for passing to subagents (avoids re-discovery) */ @@ -555,7 +558,7 @@ export async function createTools(session: ToolSession, toolNames?: string[]): P if (name === "github") return session.settings.get("github.enabled"); if (name === "ast_grep") return session.settings.get("astGrep.enabled"); if (name === "ast_edit") return session.settings.get("astEdit.enabled"); - if (name === "inspect_image") return session.settings.get("inspect_image.enabled"); + if (name === "inspect_image") return isInspectImageToolActive(session); if (name === "web_search") return session.settings.get("web_search.enabled"); if (name === "ask") return session.settings.get("ask.enabled"); if (name === "browser") return session.settings.get("browser.enabled"); diff --git a/packages/coding-agent/src/tools/read.ts b/packages/coding-agent/src/tools/read.ts index e03814702..4cf544f79 100644 --- a/packages/coding-agent/src/tools/read.ts +++ b/packages/coding-agent/src/tools/read.ts @@ -65,6 +65,7 @@ import { MAX_IMAGE_INPUT_BYTES, webpExclusionForModel, } from "../utils/image-loading"; +import { isInspectImageToolActive } from "../utils/inspect-image-mode"; import { CONVERTIBLE_EXTENSIONS, convertFileWithMarkit } from "../utils/markit"; import { isSampleProfilePath, renderSampleProfile } from "../utils/sample-profile"; import { type ArchiveReader, formatArchiveEntryLines, openArchive, parseArchivePathCandidates } from "../utils/zip"; @@ -859,31 +860,62 @@ export class ReadTool implements AgentTool { pathTargetsSsh(String((args as { path?: unknown }).path ?? "")) ? "exec" : "read"; readonly label = "Read"; readonly loadMode = "essential"; - readonly description: string; + description: string; readonly parameters = readSchema; readonly strict = true; readonly #autoResizeImages: boolean; readonly #defaultLimit: number; - readonly #inspectImageEnabled: boolean; + #inspectImageActive: boolean; constructor(private readonly session: ToolSession) { - const displayMode = resolveFileDisplayMode(session); this.#autoResizeImages = session.settings.get("images.autoResize"); this.#defaultLimit = Math.max( 1, Math.min(session.settings.get("read.defaultLimit") ?? DEFAULT_MAX_LINES, DEFAULT_MAX_LINES), ); - this.#inspectImageEnabled = session.settings.get("inspect_image.enabled"); - this.description = prompt.render(readDescription, { + this.#inspectImageActive = session.isToolActive?.("inspect_image") ?? isInspectImageToolActive(session); + this.description = this.#renderDescription(); + } + + /** + * Re-render the tool description for the current display mode and the + * effective inspect_image state (mode setting, `/vision` override, and + * active-model image capability all feed it, so it can change at runtime). + */ + #renderDescription(): string { + const displayMode = resolveFileDisplayMode(this.session); + return prompt.render(readDescription, { DEFAULT_LIMIT: String(this.#defaultLimit), DEFAULT_MAX_LINES: String(DEFAULT_MAX_LINES), IS_HL_MODE: displayMode.hashLines, IS_LINE_NUMBER_MODE: !displayMode.hashLines && displayMode.lineNumbers, - INSPECT_IMAGE_ENABLED: this.#inspectImageEnabled, + INSPECT_IMAGE_ENABLED: this.#inspectImageActive, }); } + /** + * Re-evaluate the effective inspect_image state; it can flip when the model + * or the `/vision` override changes after this tool was constructed. Keeps + * the behavior branch and the advertised description in lockstep. Called + * per image read and by tool reconciliation before prompt rebuilds. + * + * Actual tool availability wins over the mode computation: restricted + * sessions (explicit tool slates without `inspect_image`, e.g. subagents) + * must never see metadata-only reads pointing at an absent tool. Sessions + * without an `isToolActive` predicate (tests, embedded use) fall back to + * the mode check. + */ + syncInspectImageState(availableOverride?: boolean): boolean { + const active = + availableOverride ?? this.session.isToolActive?.("inspect_image") ?? isInspectImageToolActive(this.session); + if (active !== this.#inspectImageActive) { + this.#inspectImageActive = active; + this.description = this.#renderDescription(); + } + return active; + } + /** * Recover the active approved plan when a model rewrites its `local://` URL * as a same-basename path in the working-directory root. @@ -1260,10 +1292,10 @@ export class ReadTool implements AgentTool { /** * Build content blocks for an on-disk image file: an `inspect_image` - * metadata note when inspection is enabled, otherwise the decoded image + * metadata note when inspection is active, otherwise the decoded image * block. Shared by the plain-file read path and the `local://` image fast - * path so both honor `inspect_image.enabled`, the size cap, and auto-resize - * identically. Too-large / unsupported images surface as {@link ToolError}. + * path so both honor the effective inspect_image state, the size cap, and + * auto-resize identically. Too-large / unsupported images surface as {@link ToolError}. */ async #loadImageContent(options: { readPath: string; @@ -1273,7 +1305,7 @@ export class ReadTool implements AgentTool { fileSize: number; }): Promise<{ content: Array; details: ReadToolDetails; sourcePath: string }> { const { readPath, absolutePath, mimeType, imageMetadata, fileSize } = options; - if (this.#inspectImageEnabled) { + if (this.syncInspectImageState()) { const outputMime = imageMetadata?.mimeType ?? mimeType; const metadataLines = [ "Image metadata:", diff --git a/packages/coding-agent/src/utils/inspect-image-mode.ts b/packages/coding-agent/src/utils/inspect-image-mode.ts new file mode 100644 index 000000000..684e9859f --- /dev/null +++ b/packages/coding-agent/src/utils/inspect-image-mode.ts @@ -0,0 +1,39 @@ +/** + * Effective-state resolution for the `inspect_image` tool. + * + * The tool delegates image understanding to a (possibly different) + * vision-capable model. That indirection is only useful when the active model + * cannot consume images itself; when it can (`model.input` includes + * `"image"`), the tool is redundant and its presence actively degrades the + * `read` tool, which reduces image reads to metadata-only plus an + * inspect_image suggestion. `auto` mode therefore exposes the tool only for + * models without native image input. `on`/`off` force registration regardless + * of model capability. A session-scoped override (the `/vision` command) + * takes precedence over the persisted setting for the current session only. + */ +import type { Model } from "@oh-my-pi/pi-ai"; +import type { Settings } from "../config/settings"; + +export type InspectImageMode = "auto" | "on" | "off"; + +export const INSPECT_IMAGE_MODES = ["auto", "on", "off"] as const; + +/** Minimal session surface needed to resolve the effective inspect_image state. */ +export interface InspectImageModeContext { + settings: Pick; + getActiveModel?: () => Model | undefined; + getInspectImageModeOverride?: () => InspectImageMode | undefined; +} + +/** + * Whether the `inspect_image` tool should be registered/active right now. + * `auto` registers it only when the active model lacks native image input; + * an unresolved model is treated as text-only so the tool stays available. + */ +export function isInspectImageToolActive(session: InspectImageModeContext): boolean { + const mode = session.getInspectImageModeOverride?.() ?? session.settings.get("inspect_image.mode"); + if (mode === "on") return true; + if (mode === "off") return false; + const model = session.getActiveModel?.(); + return !(model?.input?.includes("image") ?? false); +} diff --git a/packages/coding-agent/test/inspect-image-mode.test.ts b/packages/coding-agent/test/inspect-image-mode.test.ts new file mode 100644 index 000000000..dbc2ae672 --- /dev/null +++ b/packages/coding-agent/test/inspect-image-mode.test.ts @@ -0,0 +1,132 @@ +/** + * Regression tests for the inspect_image tri-state mode (`inspect_image.mode`), + * the `/vision` session override precedence, and the legacy + * `inspect_image.enabled` → `inspect_image.mode` migration. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { Model } from "@oh-my-pi/pi-ai"; +import { Settings } from "../src/config/settings"; +import type { ToolSession } from "../src/tools/index"; +import { ReadTool } from "../src/tools/read"; +import { isInspectImageToolActive } from "../src/utils/inspect-image-mode"; + +const visionModel = { provider: "kimi-code", id: "k3", input: ["text", "image"] } as unknown as Model; +const textModel = { provider: "openai", id: "gpt-x", input: ["text"] } as unknown as Model; + +function activeFor( + overrides: Record, + model: Model | undefined, + sessionOverride?: "on" | "off", +): boolean { + const settings = Settings.isolated(overrides); + return isInspectImageToolActive({ + settings, + getActiveModel: () => model, + getInspectImageModeOverride: () => sessionOverride, + }); +} + +describe("isInspectImageToolActive", () => { + test("auto hides the tool for image-capable models", () => { + expect(activeFor({}, visionModel)).toBe(false); + }); + + test("auto exposes the tool for text-only models", () => { + expect(activeFor({}, textModel)).toBe(true); + }); + + test("auto treats an unresolved model as text-only", () => { + expect(activeFor({}, undefined)).toBe(true); + }); + + test("on forces the tool even for image-capable models", () => { + expect(activeFor({ "inspect_image.mode": "on" }, visionModel)).toBe(true); + }); + + test("off suppresses the tool even for text-only models", () => { + expect(activeFor({ "inspect_image.mode": "off" }, textModel)).toBe(false); + }); + + test("session override wins over the persisted setting", () => { + expect(activeFor({}, visionModel, "on")).toBe(true); + expect(activeFor({ "inspect_image.mode": "on" }, visionModel, "off")).toBe(false); + }); +}); + +describe("inspect_image.enabled migration", () => { + test("nested enabled=true migrates to mode on", () => { + const settings = Settings.isolated({ "inspect_image.enabled": true }); + expect(settings.get("inspect_image.mode")).toBe("on"); + }); + + test("nested enabled=false migrates to mode off", () => { + const settings = Settings.isolated({ "inspect_image.enabled": false }); + expect(settings.get("inspect_image.mode")).toBe("off"); + }); + + test("explicit mode wins over a stale legacy key", () => { + const settings = Settings.isolated({ "inspect_image.enabled": true, "inspect_image.mode": "off" }); + expect(settings.get("inspect_image.mode")).toBe("off"); + }); + + describe("flat dotted key in config.yml", () => { + let agentDir: string; + afterEach(() => { + if (agentDir) fs.rmSync(agentDir, { recursive: true, force: true }); + }); + + test("flat inspect_image.enabled migrates to mode", async () => { + agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-vision-migration-")); + fs.writeFileSync(path.join(agentDir, "config.yml"), '"inspect_image.enabled": true\n'); + const settings = await Settings.loadReadOnly({ agentDir, cwd: agentDir }); + expect(settings.get("inspect_image.mode")).toBe("on"); + }); + + test("nested inspect_image.enabled in config.yml migrates to mode", async () => { + agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-vision-migration-")); + fs.writeFileSync(path.join(agentDir, "config.yml"), "inspect_image:\n enabled: false\n"); + const settings = await Settings.loadReadOnly({ agentDir, cwd: agentDir }); + expect(settings.get("inspect_image.mode")).toBe("off"); + }); + + test("flat explicit mode survives alongside a flat legacy key", async () => { + agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-vision-migration-")); + fs.writeFileSync( + path.join(agentDir, "config.yml"), + '"inspect_image.enabled": true\n"inspect_image.mode": "off"\n', + ); + const settings = await Settings.loadReadOnly({ agentDir, cwd: agentDir }); + expect(settings.get("inspect_image.mode")).toBe("off"); + }); + }); +}); + +describe("read tool follows actual tool availability", () => { + function readSession(inspectImageActive: boolean): ToolSession { + return { + cwd: os.tmpdir(), + hasUI: false, + settings: Settings.isolated(), + getSessionFile: () => null, + getSessionSpawns: () => null, + getActiveModel: () => textModel, + isToolActive: (name: string) => name === "inspect_image" && inspectImageActive, + } as unknown as ToolSession; + } + + test("restricted session (tool absent) never advertises inspect_image", () => { + // auto mode + text-only model would compute active=true, but the tool is + // not in this session's slate, so read must serve inline image blocks. + const tool = new ReadTool(readSession(false)); + expect(tool.description).not.toContain("call `inspect_image`"); + expect(tool.syncInspectImageState()).toBe(false); + }); + + test("session with the tool registered advertises it", () => { + const tool = new ReadTool(readSession(true)); + expect(tool.syncInspectImageState()).toBe(true); + }); +});