Merge PR #6830: feat(coding-agent): capability-aware inspect_image with tri-state mode and /vision toggle (@epsilver)
This commit is contained in:
+2
-1
@@ -478,7 +478,7 @@ tools:
|
||||
| `tools.artifactTailBytes` | number | `20` | KB of tail kept inline on spill. |
|
||||
| `tools.artifactTailLines` | number | `500` | Max tail lines kept inline on spill. |
|
||||
|
||||
Individual built-in tools are toggled by their own keys, e.g. `bash.enabled`, `launch.enabled`, `eval.py`, `eval.js`, `glob.enabled`, `grep.enabled`, `fetch.enabled`, `browser.enabled`, `computer.enabled`, `astEdit.enabled`, `astGrep.enabled`, `web_search.enabled`, and `inspect_image.enabled`.
|
||||
Individual built-in tools are toggled by their own keys, e.g. `bash.enabled`, `launch.enabled`, `eval.py`, `eval.js`, `glob.enabled`, `grep.enabled`, `fetch.enabled`, `browser.enabled`, `computer.enabled`, `astEdit.enabled`, `astGrep.enabled`, and `web_search.enabled`. The `inspect_image` tool is controlled by the tri-state `inspect_image.mode` (`auto`|`on`|`off`, default `auto`): `auto` exposes it only when the active model lacks native image input, and the `/vision` slash command overrides the mode per session.
|
||||
|
||||
### Native computer use
|
||||
|
||||
@@ -744,6 +744,7 @@ Applied whenever raw settings are loaded (global, project, overlays, and runtime
|
||||
|
||||
| Old | New |
|
||||
|---|---|
|
||||
| `inspect_image.enabled` boolean | `inspect_image.mode` (`true` → `on`, `false` → `off`) |
|
||||
| `queueMode` | `steeringMode` |
|
||||
| `ask.timeout` in milliseconds (value `> 1000`) | seconds (divided by 1000) |
|
||||
| flat `theme: "<name>"` string | `theme.dark` / `theme.light` (slot chosen by luminance; built-in `light`/`dark` are dropped to use defaults) |
|
||||
|
||||
@@ -75,7 +75,7 @@ TUI rendering adds presentation-only truncation from `packages/coding-agent/src/
|
||||
## Limits & Caps
|
||||
- Supported detected input formats: `image/png`, `image/jpeg`, `image/gif`, `image/webp` (`SUPPORTED_IMAGE_MIME_TYPES` in `packages/utils/src/mime.ts`).
|
||||
- Metadata sniff cap: `DEFAULT_IMAGE_METADATA_HEADER_BYTES = 256 * 1024` bytes. Format detection only reads up to 256 KiB from the file header.
|
||||
- Availability is gated by `inspect_image.enabled`, default `false`, in `packages/coding-agent/src/config/settings-schema.ts` / `packages/coding-agent/src/tools/index.ts`.
|
||||
- Availability is gated by `inspect_image.mode` (`auto`|`on`|`off`, default `auto`) in `packages/coding-agent/src/config/settings-schema.ts`, resolved with the session-scoped `/vision` override and the active model's image capability in `packages/coding-agent/src/utils/inspect-image-mode.ts` / `packages/coding-agent/src/tools/index.ts`. `auto` registers the tool only when the active model lacks native image input; the legacy `inspect_image.enabled` boolean migrates to `mode` (`true`→`on`, `false`→`off`).
|
||||
- Upload input cap: `MAX_IMAGE_INPUT_BYTES = 20 * 1024 * 1024` bytes (20 MiB) in `packages/coding-agent/src/utils/image-loading.ts`.
|
||||
- Auto-resize defaults in `packages/coding-agent/src/utils/image-resize.ts`:
|
||||
- `maxWidth: 1568`
|
||||
|
||||
+1
-1
@@ -188,7 +188,7 @@ URL selectors are parsed separately in `packages/coding-agent/src/tools/fetch.ts
|
||||
### Images
|
||||
- Image detection is metadata-based (`readImageMetadata()`).
|
||||
- Max accepted image size is `20 MiB` (`MAX_IMAGE_INPUT_BYTES`, re-exported as `MAX_IMAGE_SIZE`). Larger files throw.
|
||||
- If `inspect_image.enabled` is true, `read` returns metadata only (MIME, bytes, dimensions, channels, alpha) plus a suggestion to call `inspect_image`.
|
||||
- If the effective `inspect_image` state is active (mode `on`, or `auto` with an active model that lacks native image input), `read` returns metadata only (MIME, bytes, dimensions, channels, alpha) plus a suggestion to call `inspect_image`.
|
||||
- Otherwise it calls `loadImageInput()` and returns:
|
||||
- a text note from the image loader
|
||||
- an inline image block
|
||||
|
||||
@@ -13,6 +13,11 @@
|
||||
- Session listing now caches parsed headers keyed on file stat identity (mtime + size), so repeated resume-picker opens and startup scans re-read only changed session files
|
||||
- Reduced per-keystroke editor dispatch overhead: keybinding resolution happens once per input chunk and the per-action interception chain is gated behind a single canonical-key set probe
|
||||
- `xd://` device docs now render the parameter schema as a comment-annotated TypeScript type (via `jsonSchemaToTypeScript`, the same renderer the in-band tool inventory uses) instead of a raw JSON Schema dump, shrinking system-prompt device sections while keeping descriptions inline.
|
||||
- Added a `/vision [on|off|auto|status]` slash command for session-scoped control of the `inspect_image` vision-delegation tool, modeled on `/computer`: `on`/`off` force the tool for the current session only, `auto` returns to the persisted setting, and `status` reports the effective mode, session override, tool state, and active-model image capability.
|
||||
|
||||
### Changed
|
||||
|
||||
- Replaced the `inspect_image.enabled` boolean with the tri-state `inspect_image.mode` (`auto`|`on`|`off`, default `auto`). In `auto` the tool is registered only when the active model lacks native image input, so vision-capable models (e.g. `kimi-code/k3`) read images inline with their own capabilities instead of delegating to a separate vision model; the tool set is re-evaluated on every model switch with a status notice when it flips. The `read` tool now follows the effective state dynamically rather than the raw setting, so it returns decoded image blocks again whenever `inspect_image` is hidden. Existing `inspect_image.enabled: true/false` configs migrate to `inspect_image.mode: on/off`.
|
||||
|
||||
## [17.1.6] - 2026-07-27
|
||||
|
||||
|
||||
@@ -3835,14 +3835,28 @@ export const SETTINGS_SCHEMA = {
|
||||
},
|
||||
},
|
||||
|
||||
// Legacy boolean kept only for back-compat migration to `inspect_image.mode`
|
||||
// (see config/settings.ts). Hidden from UI.
|
||||
"inspect_image.enabled": {
|
||||
type: "boolean",
|
||||
default: false,
|
||||
},
|
||||
|
||||
"inspect_image.mode": {
|
||||
type: "enum",
|
||||
values: ["auto", "on", "off"] as const,
|
||||
default: "auto",
|
||||
ui: {
|
||||
tab: "tools",
|
||||
group: "Available Tools",
|
||||
label: "Inspect Image",
|
||||
description: "Enable the inspect_image tool, delegating image understanding to a vision-capable model",
|
||||
description:
|
||||
"Controls the inspect_image tool, which delegates image understanding to a vision-capable model. 'auto' exposes it only when the active model lacks native image input; 'on' always exposes it; 'off' never does.",
|
||||
options: [
|
||||
{ value: "auto", label: "Auto (only for models without vision)" },
|
||||
{ value: "on", label: "On" },
|
||||
{ value: "off", label: "Off" },
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ import { isLightTheme, setAutoThemeMapping, setColorBlindMode, setSymbolPreset }
|
||||
import { AgentStorage } from "../session/agent-storage";
|
||||
import { AUTO_IMAGE_PROVIDER_ORDER, isImageProviderId } from "../tools/image-providers";
|
||||
import { type EditMode, normalizeEditMode } from "../utils/edit-mode";
|
||||
import { INSPECT_IMAGE_MODES } from "../utils/inspect-image-mode";
|
||||
import { isSearchProviderId, SEARCH_PROVIDER_ORDER } from "../web/search/types";
|
||||
import { withFileLock } from "./file-lock";
|
||||
import {
|
||||
@@ -1366,6 +1367,40 @@ export class Settings {
|
||||
}
|
||||
}
|
||||
|
||||
// inspect_image.enabled (boolean) -> inspect_image.mode (enum). Explicit
|
||||
// user choices are preserved: true -> "on", false -> "off". Configs with
|
||||
// no legacy key get the new "auto" default, which hides the tool for
|
||||
// models with native image input. Handles nested and quoted-dotted
|
||||
// ("inspect_image.enabled") sources; the target is always the nested
|
||||
// form, which is the only shape the resolver reads.
|
||||
const inspectImageObj = isRecord(raw.inspect_image) ? (raw.inspect_image as Record<string, unknown>) : undefined;
|
||||
const legacyEnabled =
|
||||
typeof inspectImageObj?.enabled === "boolean"
|
||||
? inspectImageObj.enabled
|
||||
: typeof raw["inspect_image.enabled"] === "boolean"
|
||||
? (raw["inspect_image.enabled"] as boolean)
|
||||
: undefined;
|
||||
if (legacyEnabled !== undefined) {
|
||||
if (!inspectImageObj) {
|
||||
raw.inspect_image = {};
|
||||
}
|
||||
const target = raw.inspect_image as Record<string, unknown>;
|
||||
const flatMode = raw["inspect_image.mode"];
|
||||
if (target.mode === undefined) {
|
||||
// A quoted-dotted explicit mode wins over the legacy boolean but
|
||||
// must be normalized into the nested form the resolver reads.
|
||||
target.mode =
|
||||
typeof flatMode === "string" && (INSPECT_IMAGE_MODES as readonly string[]).includes(flatMode)
|
||||
? flatMode
|
||||
: legacyEnabled
|
||||
? "on"
|
||||
: "off";
|
||||
}
|
||||
delete target.enabled;
|
||||
delete raw["inspect_image.enabled"];
|
||||
delete raw["inspect_image.mode"];
|
||||
}
|
||||
|
||||
// task.isolation.enabled (boolean) -> task.isolation.mode (enum)
|
||||
const taskObj = raw.task as Record<string, unknown> | undefined;
|
||||
const isolationObj = taskObj?.isolation as Record<string, unknown> | undefined;
|
||||
|
||||
@@ -455,6 +455,11 @@ export class SelectorController {
|
||||
this.ctx.showError(`Failed to apply memory backend: ${err}`);
|
||||
});
|
||||
break;
|
||||
case "inspect_image.mode":
|
||||
void this.ctx.session.applyInspectImageModeChange().catch(err => {
|
||||
this.ctx.showError(`Failed to apply vision mode: ${err}`);
|
||||
});
|
||||
break;
|
||||
|
||||
case "autocompleteMaxVisible":
|
||||
this.ctx.editor.setAutocompleteMaxVisible(typeof value === "number" ? value : Number(value));
|
||||
|
||||
@@ -1682,6 +1682,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
||||
getModelString: () => (hasExplicitModel && model ? formatModelString(model) : undefined),
|
||||
getActiveModelString,
|
||||
getActiveModel: () => agent?.state.model ?? model,
|
||||
getInspectImageModeOverride: () => session?.getInspectImageModeOverride(),
|
||||
getServiceTierByFamily: () => session?.serviceTierByFamily,
|
||||
getImageAttachments: () => session?.getImageAttachments() ?? [],
|
||||
getPlanModeState: () => session?.getPlanModeState(),
|
||||
@@ -3178,6 +3179,9 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {}
|
||||
createComputerTool: restrictToolNames
|
||||
? undefined
|
||||
: async () => (await BUILTIN_TOOLS.computer(toolSession)) ?? null,
|
||||
createInspectImageTool: restrictToolNames
|
||||
? undefined
|
||||
: async () => (await BUILTIN_TOOLS.inspect_image(toolSession)) ?? null,
|
||||
createVibeTools:
|
||||
(options.taskDepth ?? 0) === 0 && !options.parentTaskPrefix
|
||||
? () => createVibeTools(toolSession)
|
||||
|
||||
@@ -139,6 +139,8 @@ export interface AgentSessionConfig {
|
||||
createMemoryTools?: () => Promise<AgentTool[]>;
|
||||
/** Creates the built-in `computer` tool for session-scoped runtime enablement (see {@link AgentSession.setComputerToolEnabled}). */
|
||||
createComputerTool?: () => Promise<AgentTool | null>;
|
||||
/** Creates the built-in `inspect_image` tool for session-scoped runtime enablement (see {@link AgentSession.setInspectImageMode}). */
|
||||
createInspectImageTool?: () => Promise<AgentTool | null>;
|
||||
/** Model registry for API key resolution and model discovery. */
|
||||
modelRegistry: ModelRegistry;
|
||||
/** Tool registry for LSP and settings. */
|
||||
|
||||
@@ -200,6 +200,7 @@ import type { EditMode } from "../utils/edit-mode";
|
||||
import { resolveFileDisplayMode } from "../utils/file-display-mode";
|
||||
import { extractFileMentions, generateFileMentionMessages } from "../utils/file-mentions";
|
||||
import { normalizeModelContextImages } from "../utils/image-loading";
|
||||
import type { InspectImageMode } from "../utils/inspect-image-mode";
|
||||
import { generateSessionTitle } from "../utils/title-generator";
|
||||
import { buildNamedToolChoice, isToolChoiceActive } from "../utils/tool-choice";
|
||||
import type { VibeModeState } from "../vibe/state";
|
||||
@@ -433,6 +434,8 @@ export class AgentSession {
|
||||
#scheduledHiddenNextTurnGeneration: number | undefined = undefined;
|
||||
#queuedMessageDrainScheduled = false;
|
||||
#planModeState: PlanModeState | undefined;
|
||||
/** Session-scoped `/vision` override; undefined = follow persisted `inspect_image.mode`. */
|
||||
#inspectImageModeOverride: InspectImageMode | undefined;
|
||||
#vibeModeState: VibeModeState | undefined;
|
||||
#goalModeState: GoalModeState | undefined;
|
||||
#goalRuntime: GoalRuntime;
|
||||
@@ -1085,12 +1088,17 @@ export class AgentSession {
|
||||
emitNotice: (level, message, source) => this.emitNotice(level, message, source),
|
||||
notifyCommandMetadataChanged: () => this.#notifyCommandMetadataChanged(),
|
||||
localProtocolOptions: () => this.#localProtocolOptions(),
|
||||
getInspectImageModeOverride: () => this.#inspectImageModeOverride,
|
||||
setInspectImageModeOverride: mode => {
|
||||
this.#inspectImageModeOverride = mode;
|
||||
},
|
||||
};
|
||||
this.#tools = new SessionTools(sessionToolsHost, {
|
||||
autoApprove: config.autoApprove,
|
||||
toolRegistry: config.toolRegistry,
|
||||
createVibeTools: config.createVibeTools,
|
||||
createComputerTool: config.createComputerTool,
|
||||
createInspectImageTool: config.createInspectImageTool,
|
||||
builtInToolNames: config.builtInToolNames,
|
||||
presentationPinnedToolNames: config.presentationPinnedToolNames,
|
||||
ensureWriteRegistered: config.ensureWriteRegistered,
|
||||
@@ -4002,6 +4010,34 @@ export class AgentSession {
|
||||
return this.#tools.setComputerToolEnabled(enabled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Session-scoped inspect_image mode (`/vision`). `auto` clears the override
|
||||
* and returns to the persisted `inspect_image.mode` setting; `on`/`off`
|
||||
* force the tool for this session only. See {@link SessionTools.setInspectImageMode}.
|
||||
*/
|
||||
setInspectImageMode(mode: InspectImageMode): Promise<boolean> {
|
||||
return this.#tools.setInspectImageMode(mode);
|
||||
}
|
||||
|
||||
/** Effective inspect_image state for `/vision status`. */
|
||||
inspectImageState(): { mode: InspectImageMode; active: boolean; model: string | undefined } {
|
||||
return this.#tools.inspectImageState();
|
||||
}
|
||||
|
||||
/** Session-scoped `/vision` override; undefined means "follow the persisted setting". */
|
||||
getInspectImageModeOverride(): InspectImageMode | undefined {
|
||||
return this.#inspectImageModeOverride;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconciles the inspect_image tool set after the persisted
|
||||
* `inspect_image.mode` setting changed (e.g. via the settings selector), so
|
||||
* the new value takes effect immediately instead of on the next model switch.
|
||||
*/
|
||||
applyInspectImageModeChange(): Promise<boolean> {
|
||||
return this.#tools.reconcileInspectImageTool();
|
||||
}
|
||||
|
||||
/** Cancels the local rollout-memory startup owned by this session. */
|
||||
cancelLocalMemoryStartup(): void {
|
||||
this.#memory.cancelLocalMemoryStartup();
|
||||
@@ -6469,6 +6505,14 @@ export class AgentSession {
|
||||
|
||||
// Re-evaluate append-only context mode — provider or setting may have changed
|
||||
this.#syncAppendOnlyContext(model);
|
||||
|
||||
// inspect_image auto mode keys off model image capability. Reconcile
|
||||
// centrally here so retry-fallback model changes (turn-recovery.ts),
|
||||
// which bypass syncAfterModelChange, cannot leave the tool set stale.
|
||||
// Idempotent; syncAfterModelChange's own reconcile is then a no-op.
|
||||
void this.#tools.reconcileInspectImageAfterModelChange().catch(error => {
|
||||
logger.warn("inspect_image reconcile after model change failed", { error: String(error) });
|
||||
});
|
||||
}
|
||||
|
||||
#closeCodexProviderSessionsForHistoryRewrite(): void {
|
||||
|
||||
@@ -23,6 +23,7 @@ import { wrapToolWithMetaNotice } from "../tools/output-meta";
|
||||
import { ToolAbortError, ToolError } from "../tools/tool-errors";
|
||||
import { isMountableUnderXdev, type XdevRegistry } from "../tools/xdev";
|
||||
import { type EditMode, resolveEditMode } from "../utils/edit-mode";
|
||||
import { type InspectImageMode, isInspectImageToolActive } from "../utils/inspect-image-mode";
|
||||
import { formatLocalCalendarDate } from "../utils/local-date";
|
||||
import {
|
||||
extractPermissionLocations,
|
||||
@@ -56,6 +57,9 @@ export interface SessionToolsHost {
|
||||
emitNotice(level: "info" | "warning" | "error", message: string, source?: string): void;
|
||||
notifyCommandMetadataChanged(): void;
|
||||
localProtocolOptions(): LocalProtocolOptions;
|
||||
/** Session-scoped `/vision` override; undefined means "follow the persisted setting". */
|
||||
getInspectImageModeOverride(): InspectImageMode | undefined;
|
||||
setInspectImageModeOverride(mode: InspectImageMode | undefined): void;
|
||||
}
|
||||
|
||||
interface SessionToolsOptions {
|
||||
@@ -63,6 +67,8 @@ interface SessionToolsOptions {
|
||||
toolRegistry?: Map<string, AgentTool>;
|
||||
createVibeTools?: () => AgentTool[];
|
||||
createComputerTool?: () => Promise<AgentTool | null>;
|
||||
/** Creates the built-in `inspect_image` tool for session-scoped runtime enablement (see {@link SessionTools.setInspectImageMode}). */
|
||||
createInspectImageTool?: () => Promise<AgentTool | null>;
|
||||
builtInToolNames?: Iterable<string>;
|
||||
presentationPinnedToolNames?: ReadonlySet<string>;
|
||||
ensureWriteRegistered?: () => Promise<boolean>;
|
||||
@@ -161,6 +167,7 @@ export class SessionTools {
|
||||
#toolRegistry: Map<string, AgentTool>;
|
||||
#createVibeTools: (() => AgentTool[]) | undefined;
|
||||
#createComputerTool: SessionToolsOptions["createComputerTool"];
|
||||
#createInspectImageTool: SessionToolsOptions["createInspectImageTool"];
|
||||
#installedVibeToolNames = new Set<string>();
|
||||
#builtInToolNames: Set<string>;
|
||||
#rpcHostToolNames = new Set<string>();
|
||||
@@ -190,6 +197,7 @@ export class SessionTools {
|
||||
this.#toolRegistry = options.toolRegistry ?? new Map();
|
||||
this.#createVibeTools = options.createVibeTools;
|
||||
this.#createComputerTool = options.createComputerTool;
|
||||
this.#createInspectImageTool = options.createInspectImageTool;
|
||||
this.#builtInToolNames = new Set(options.builtInToolNames ?? []);
|
||||
this.#presentationPinnedToolNames = options.presentationPinnedToolNames;
|
||||
this.#ensureWriteRegistered = options.ensureWriteRegistered;
|
||||
@@ -403,6 +411,10 @@ export class SessionTools {
|
||||
} else if (computerExpected) {
|
||||
this.#logComputerState("Computer tool retained after model change", true);
|
||||
}
|
||||
|
||||
// inspect_image auto mode keys off model image capability, so a model
|
||||
// switch can flip the tool either way.
|
||||
await this.reconcileInspectImageAfterModelChange();
|
||||
}
|
||||
|
||||
/** Enabled MCP tools in their current presentation partition. */
|
||||
@@ -849,6 +861,108 @@ export class SessionTools {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Current effective inspect_image state for `/vision status`. */
|
||||
inspectImageState(): { mode: InspectImageMode; active: boolean; model: string | undefined } {
|
||||
const model = this.#host.model();
|
||||
return {
|
||||
mode: this.#host.getInspectImageModeOverride() ?? this.#host.settings.get("inspect_image.mode"),
|
||||
active: this.getEnabledToolNames().includes("inspect_image"),
|
||||
model: model ? formatModelString(model) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings the active tool set in line with the effective inspect_image state
|
||||
* (mode setting, `/vision` override, active-model image capability).
|
||||
* Mirrors {@link setComputerToolEnabled}: enabling builds the tool through
|
||||
* the config factory on first use and reuses the registry entry afterwards.
|
||||
* Idempotent — safe to call from every model/settings change path.
|
||||
*
|
||||
* @returns false when the tool should be active but this session cannot
|
||||
* build it (e.g. restricted child sessions have no factory).
|
||||
*/
|
||||
async reconcileInspectImageTool(): Promise<boolean> {
|
||||
const expected = isInspectImageToolActive({
|
||||
settings: this.#host.settings,
|
||||
getActiveModel: () => this.#host.model(),
|
||||
getInspectImageModeOverride: () => this.#host.getInspectImageModeOverride(),
|
||||
});
|
||||
// Keep the read tool's advertised description in sync BEFORE any prompt
|
||||
// rebuild below, passing the post-change availability so the prompt never
|
||||
// lags a flip in either direction. Per-read lazy sync is the backstop.
|
||||
const syncReadDescription = (available: boolean): void => {
|
||||
const readTool = this.#toolRegistry.get("read") as
|
||||
| { syncInspectImageState?: (available?: boolean) => boolean }
|
||||
| undefined;
|
||||
readTool?.syncInspectImageState?.(available);
|
||||
};
|
||||
const active = this.getEnabledToolNames();
|
||||
const isActive = active.includes("inspect_image");
|
||||
if (expected === isActive) {
|
||||
syncReadDescription(isActive);
|
||||
return true;
|
||||
}
|
||||
if (!expected) {
|
||||
syncReadDescription(false);
|
||||
await this.applyActiveToolsByName(active.filter(name => name !== "inspect_image"));
|
||||
return true;
|
||||
}
|
||||
if (!this.#toolRegistry.has("inspect_image")) {
|
||||
const tool = await this.#createInspectImageTool?.();
|
||||
if (tool?.name !== "inspect_image") {
|
||||
logger.warn("inspect_image tool could not be created", {
|
||||
model: this.#host.model()?.id,
|
||||
});
|
||||
syncReadDescription(false);
|
||||
return false;
|
||||
}
|
||||
const wrapped = this.#wrapRuntimeTool(tool);
|
||||
this.#toolRegistry.set(wrapped.name, wrapped);
|
||||
this.#builtInToolNames.add(wrapped.name);
|
||||
}
|
||||
syncReadDescription(true);
|
||||
await this.applyActiveToolsByName([...active, "inspect_image"]);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconciles inspect_image after a model change and surfaces a notice when
|
||||
* the visible tool set actually flipped. Called from every model-change
|
||||
* path — including retry-fallback switches that bypass
|
||||
* {@link syncAfterModelChange}.
|
||||
*/
|
||||
async reconcileInspectImageAfterModelChange(): Promise<void> {
|
||||
const before = this.getEnabledToolNames().includes("inspect_image");
|
||||
const reconciled = await this.reconcileInspectImageTool();
|
||||
const after = this.getEnabledToolNames().includes("inspect_image");
|
||||
if (!reconciled || before === after) return;
|
||||
const model = this.#host.model();
|
||||
const modelName = model ? formatModelString(model) : "the current model";
|
||||
this.#host.emitNotice(
|
||||
"info",
|
||||
after
|
||||
? `inspect_image is now available: ${modelName} has no native image input.`
|
||||
: `inspect_image is now hidden: ${modelName} supports image input natively. Override with /vision on.`,
|
||||
"vision",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Session-scoped `/vision` override. `auto` clears the override so the
|
||||
* persisted `inspect_image.mode` setting (itself possibly `auto`) decides;
|
||||
* `on`/`off` force the tool for this session only. Takes effect before the
|
||||
* next model call.
|
||||
*
|
||||
* @returns false when `on` was requested but the tool cannot be built here.
|
||||
*/
|
||||
async setInspectImageMode(mode: InspectImageMode): Promise<boolean> {
|
||||
this.#host.setInspectImageModeOverride(mode === "auto" ? undefined : mode);
|
||||
const applied = await this.reconcileInspectImageTool();
|
||||
const { active, model } = this.inspectImageState();
|
||||
logger.debug("inspect_image mode changed", { mode, active, model });
|
||||
return applied;
|
||||
}
|
||||
|
||||
/** Rebuilds the stable base prompt for the current tools and model. */
|
||||
async refreshBaseSystemPrompt(): Promise<void> {
|
||||
if (this.#host.isDisposed() || !this.#rebuildSystemPrompt) return;
|
||||
|
||||
@@ -53,6 +53,7 @@ import {
|
||||
renderChangelogEntries,
|
||||
} from "../utils/changelog";
|
||||
import { copyToClipboard } from "../utils/clipboard";
|
||||
import type { InspectImageMode } from "../utils/inspect-image-mode";
|
||||
import { CollabQrCodeComponent } from "./helpers/collab-qrcode";
|
||||
import { buildContextReportText } from "./helpers/context-report";
|
||||
import { formatDuration } from "./helpers/format";
|
||||
@@ -151,6 +152,33 @@ async function applyComputerUseToggle(session: AgentSession, enable: boolean): P
|
||||
: "Computer use disabled for this session.";
|
||||
}
|
||||
|
||||
/** Session-effective `/vision status` line. */
|
||||
function formatVisionStatus(session: AgentSession): string {
|
||||
const { mode, active, model } = session.inspectImageState();
|
||||
const override = session.getInspectImageModeOverride();
|
||||
const modelObj = session.model;
|
||||
const capability = modelObj
|
||||
? modelObj.input.includes("image")
|
||||
? "native image input"
|
||||
: "no native image input"
|
||||
: "no active model";
|
||||
return [
|
||||
`inspect_image: ${active ? "active" : "inactive"}`,
|
||||
`mode: ${mode}${override ? " (session override)" : ""}`,
|
||||
...(override ? [`configured: ${session.settings.get("inspect_image.mode")}`] : []),
|
||||
`model: ${model ?? "none"} (${capability})`,
|
||||
].join(" · ");
|
||||
}
|
||||
|
||||
/** Applies a `/vision` mode for this session and returns the operator feedback line. */
|
||||
async function applyVisionMode(session: AgentSession, mode: InspectImageMode): Promise<string> {
|
||||
const applied = await session.setInspectImageMode(mode);
|
||||
if (!applied) {
|
||||
return "inspect_image is unavailable in this session.";
|
||||
}
|
||||
return `Vision mode: ${mode}. ${formatVisionStatus(session)}`;
|
||||
}
|
||||
|
||||
const AUTOCOMPLETE_DETAIL_LIMIT = 48;
|
||||
|
||||
function shortDetail(value: string, limit = AUTOCOMPLETE_DETAIL_LIMIT): string {
|
||||
@@ -647,6 +675,47 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray<SlashCommandSpec> = [
|
||||
runtime.ctx.editor.setText("");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "vision",
|
||||
description: "Control the inspect_image vision-delegation tool for this session",
|
||||
acpDescription: "Toggle vision delegation",
|
||||
acpInputHint: "[on|off|auto|status]",
|
||||
subcommands: [
|
||||
{ name: "on", description: "Always expose inspect_image this session" },
|
||||
{ name: "off", description: "Never expose inspect_image this session" },
|
||||
{ name: "auto", description: "Follow inspect_image.mode (auto hides it for vision-capable models)" },
|
||||
{ name: "status", description: "Show inspect_image status" },
|
||||
],
|
||||
allowArgs: true,
|
||||
getTuiAutocompleteDescription: runtime => `Vision: ${runtime.ctx.session.inspectImageState().mode}`,
|
||||
handle: async (command, runtime) => {
|
||||
const arg = command.args.trim().toLowerCase();
|
||||
if (arg === "status") {
|
||||
await runtime.output(formatVisionStatus(runtime.session));
|
||||
return commandConsumed();
|
||||
}
|
||||
if (arg === "on" || arg === "off" || arg === "auto") {
|
||||
await runtime.output(await applyVisionMode(runtime.session, arg));
|
||||
return commandConsumed();
|
||||
}
|
||||
return usage("Usage: /vision [on|off|auto|status]", runtime);
|
||||
},
|
||||
handleTui: async (command, runtime) => {
|
||||
const arg = command.args.trim().toLowerCase();
|
||||
if (arg === "status") {
|
||||
runtime.ctx.showStatus(formatVisionStatus(runtime.ctx.session));
|
||||
runtime.ctx.editor.setText("");
|
||||
return;
|
||||
}
|
||||
if (arg === "on" || arg === "off" || arg === "auto") {
|
||||
runtime.ctx.showStatus(await applyVisionMode(runtime.ctx.session, arg));
|
||||
runtime.ctx.editor.setText("");
|
||||
return;
|
||||
}
|
||||
runtime.ctx.showStatus("Usage: /vision [on|off|auto|status]");
|
||||
runtime.ctx.editor.setText("");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "prewalk",
|
||||
description: "Switch to a fast/cheap model at the next action (works even without --prewalk)",
|
||||
|
||||
@@ -32,6 +32,7 @@ import { TaskTool } from "../task";
|
||||
import type { AgentOutputManager } from "../task/output-manager";
|
||||
import { canSpawnAtDepth, type StructuredSubagentSchemaMode } from "../task/types";
|
||||
import type { EventBus } from "../utils/event-bus";
|
||||
import { type InspectImageMode, isInspectImageToolActive } from "../utils/inspect-image-mode";
|
||||
import { WebSearchTool } from "../web/search";
|
||||
import type { WorkspaceTree } from "../workspace-tree";
|
||||
import { AskTool } from "./ask";
|
||||
@@ -263,6 +264,8 @@ export interface ToolSession {
|
||||
getActiveModelString?: () => string | undefined;
|
||||
/** Get the current session model object (provider/api capabilities), regardless of how it was chosen. */
|
||||
getActiveModel?: () => Model | undefined;
|
||||
/** Session-scoped inspect_image mode override set by `/vision`; wins over the persisted setting. */
|
||||
getInspectImageModeOverride?: () => InspectImageMode | undefined;
|
||||
/** Get the session's live per-family service tiers (undefined = none). Source of truth for subagent `tier.subagent: inherit`. */
|
||||
getServiceTierByFamily?: () => ServiceTierByFamily | undefined;
|
||||
/** Auth storage for passing to subagents (avoids re-discovery) */
|
||||
@@ -555,7 +558,7 @@ export async function createTools(session: ToolSession, toolNames?: string[]): P
|
||||
if (name === "github") return session.settings.get("github.enabled");
|
||||
if (name === "ast_grep") return session.settings.get("astGrep.enabled");
|
||||
if (name === "ast_edit") return session.settings.get("astEdit.enabled");
|
||||
if (name === "inspect_image") return session.settings.get("inspect_image.enabled");
|
||||
if (name === "inspect_image") return isInspectImageToolActive(session);
|
||||
if (name === "web_search") return session.settings.get("web_search.enabled");
|
||||
if (name === "ask") return session.settings.get("ask.enabled");
|
||||
if (name === "browser") return session.settings.get("browser.enabled");
|
||||
|
||||
@@ -65,6 +65,7 @@ import {
|
||||
MAX_IMAGE_INPUT_BYTES,
|
||||
webpExclusionForModel,
|
||||
} from "../utils/image-loading";
|
||||
import { isInspectImageToolActive } from "../utils/inspect-image-mode";
|
||||
import { CONVERTIBLE_EXTENSIONS, convertFileWithMarkit } from "../utils/markit";
|
||||
import { isSampleProfilePath, renderSampleProfile } from "../utils/sample-profile";
|
||||
import { type ArchiveReader, formatArchiveEntryLines, openArchive, parseArchivePathCandidates } from "../utils/zip";
|
||||
@@ -859,31 +860,62 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
pathTargetsSsh(String((args as { path?: unknown }).path ?? "")) ? "exec" : "read";
|
||||
readonly label = "Read";
|
||||
readonly loadMode = "essential";
|
||||
readonly description: string;
|
||||
description: string;
|
||||
readonly parameters = readSchema;
|
||||
readonly strict = true;
|
||||
|
||||
readonly #autoResizeImages: boolean;
|
||||
readonly #defaultLimit: number;
|
||||
readonly #inspectImageEnabled: boolean;
|
||||
#inspectImageActive: boolean;
|
||||
|
||||
constructor(private readonly session: ToolSession) {
|
||||
const displayMode = resolveFileDisplayMode(session);
|
||||
this.#autoResizeImages = session.settings.get("images.autoResize");
|
||||
this.#defaultLimit = Math.max(
|
||||
1,
|
||||
Math.min(session.settings.get("read.defaultLimit") ?? DEFAULT_MAX_LINES, DEFAULT_MAX_LINES),
|
||||
);
|
||||
this.#inspectImageEnabled = session.settings.get("inspect_image.enabled");
|
||||
this.description = prompt.render(readDescription, {
|
||||
this.#inspectImageActive = session.isToolActive?.("inspect_image") ?? isInspectImageToolActive(session);
|
||||
this.description = this.#renderDescription();
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-render the tool description for the current display mode and the
|
||||
* effective inspect_image state (mode setting, `/vision` override, and
|
||||
* active-model image capability all feed it, so it can change at runtime).
|
||||
*/
|
||||
#renderDescription(): string {
|
||||
const displayMode = resolveFileDisplayMode(this.session);
|
||||
return prompt.render(readDescription, {
|
||||
DEFAULT_LIMIT: String(this.#defaultLimit),
|
||||
DEFAULT_MAX_LINES: String(DEFAULT_MAX_LINES),
|
||||
IS_HL_MODE: displayMode.hashLines,
|
||||
IS_LINE_NUMBER_MODE: !displayMode.hashLines && displayMode.lineNumbers,
|
||||
INSPECT_IMAGE_ENABLED: this.#inspectImageEnabled,
|
||||
INSPECT_IMAGE_ENABLED: this.#inspectImageActive,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-evaluate the effective inspect_image state; it can flip when the model
|
||||
* or the `/vision` override changes after this tool was constructed. Keeps
|
||||
* the behavior branch and the advertised description in lockstep. Called
|
||||
* per image read and by tool reconciliation before prompt rebuilds.
|
||||
*
|
||||
* Actual tool availability wins over the mode computation: restricted
|
||||
* sessions (explicit tool slates without `inspect_image`, e.g. subagents)
|
||||
* must never see metadata-only reads pointing at an absent tool. Sessions
|
||||
* without an `isToolActive` predicate (tests, embedded use) fall back to
|
||||
* the mode check.
|
||||
*/
|
||||
syncInspectImageState(availableOverride?: boolean): boolean {
|
||||
const active =
|
||||
availableOverride ?? this.session.isToolActive?.("inspect_image") ?? isInspectImageToolActive(this.session);
|
||||
if (active !== this.#inspectImageActive) {
|
||||
this.#inspectImageActive = active;
|
||||
this.description = this.#renderDescription();
|
||||
}
|
||||
return active;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover the active approved plan when a model rewrites its `local://` URL
|
||||
* as a same-basename path in the working-directory root.
|
||||
@@ -1260,10 +1292,10 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
|
||||
/**
|
||||
* Build content blocks for an on-disk image file: an `inspect_image`
|
||||
* metadata note when inspection is enabled, otherwise the decoded image
|
||||
* metadata note when inspection is active, otherwise the decoded image
|
||||
* block. Shared by the plain-file read path and the `local://` image fast
|
||||
* path so both honor `inspect_image.enabled`, the size cap, and auto-resize
|
||||
* identically. Too-large / unsupported images surface as {@link ToolError}.
|
||||
* path so both honor the effective inspect_image state, the size cap, and
|
||||
* auto-resize identically. Too-large / unsupported images surface as {@link ToolError}.
|
||||
*/
|
||||
async #loadImageContent(options: {
|
||||
readPath: string;
|
||||
@@ -1273,7 +1305,7 @@ export class ReadTool implements AgentTool<typeof readSchema, ReadToolDetails> {
|
||||
fileSize: number;
|
||||
}): Promise<{ content: Array<TextContent | ImageContent>; details: ReadToolDetails; sourcePath: string }> {
|
||||
const { readPath, absolutePath, mimeType, imageMetadata, fileSize } = options;
|
||||
if (this.#inspectImageEnabled) {
|
||||
if (this.syncInspectImageState()) {
|
||||
const outputMime = imageMetadata?.mimeType ?? mimeType;
|
||||
const metadataLines = [
|
||||
"Image metadata:",
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Effective-state resolution for the `inspect_image` tool.
|
||||
*
|
||||
* The tool delegates image understanding to a (possibly different)
|
||||
* vision-capable model. That indirection is only useful when the active model
|
||||
* cannot consume images itself; when it can (`model.input` includes
|
||||
* `"image"`), the tool is redundant and its presence actively degrades the
|
||||
* `read` tool, which reduces image reads to metadata-only plus an
|
||||
* inspect_image suggestion. `auto` mode therefore exposes the tool only for
|
||||
* models without native image input. `on`/`off` force registration regardless
|
||||
* of model capability. A session-scoped override (the `/vision` command)
|
||||
* takes precedence over the persisted setting for the current session only.
|
||||
*/
|
||||
import type { Model } from "@oh-my-pi/pi-ai";
|
||||
import type { Settings } from "../config/settings";
|
||||
|
||||
export type InspectImageMode = "auto" | "on" | "off";
|
||||
|
||||
export const INSPECT_IMAGE_MODES = ["auto", "on", "off"] as const;
|
||||
|
||||
/** Minimal session surface needed to resolve the effective inspect_image state. */
|
||||
export interface InspectImageModeContext {
|
||||
settings: Pick<Settings, "get">;
|
||||
getActiveModel?: () => Model | undefined;
|
||||
getInspectImageModeOverride?: () => InspectImageMode | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the `inspect_image` tool should be registered/active right now.
|
||||
* `auto` registers it only when the active model lacks native image input;
|
||||
* an unresolved model is treated as text-only so the tool stays available.
|
||||
*/
|
||||
export function isInspectImageToolActive(session: InspectImageModeContext): boolean {
|
||||
const mode = session.getInspectImageModeOverride?.() ?? session.settings.get("inspect_image.mode");
|
||||
if (mode === "on") return true;
|
||||
if (mode === "off") return false;
|
||||
const model = session.getActiveModel?.();
|
||||
return !(model?.input?.includes("image") ?? false);
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* Regression tests for the inspect_image tri-state mode (`inspect_image.mode`),
|
||||
* the `/vision` session override precedence, and the legacy
|
||||
* `inspect_image.enabled` → `inspect_image.mode` migration.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import type { Model } from "@oh-my-pi/pi-ai";
|
||||
import { Settings } from "../src/config/settings";
|
||||
import type { ToolSession } from "../src/tools/index";
|
||||
import { ReadTool } from "../src/tools/read";
|
||||
import { isInspectImageToolActive } from "../src/utils/inspect-image-mode";
|
||||
|
||||
const visionModel = { provider: "kimi-code", id: "k3", input: ["text", "image"] } as unknown as Model;
|
||||
const textModel = { provider: "openai", id: "gpt-x", input: ["text"] } as unknown as Model;
|
||||
|
||||
function activeFor(
|
||||
overrides: Record<string, unknown>,
|
||||
model: Model | undefined,
|
||||
sessionOverride?: "on" | "off",
|
||||
): boolean {
|
||||
const settings = Settings.isolated(overrides);
|
||||
return isInspectImageToolActive({
|
||||
settings,
|
||||
getActiveModel: () => model,
|
||||
getInspectImageModeOverride: () => sessionOverride,
|
||||
});
|
||||
}
|
||||
|
||||
describe("isInspectImageToolActive", () => {
|
||||
test("auto hides the tool for image-capable models", () => {
|
||||
expect(activeFor({}, visionModel)).toBe(false);
|
||||
});
|
||||
|
||||
test("auto exposes the tool for text-only models", () => {
|
||||
expect(activeFor({}, textModel)).toBe(true);
|
||||
});
|
||||
|
||||
test("auto treats an unresolved model as text-only", () => {
|
||||
expect(activeFor({}, undefined)).toBe(true);
|
||||
});
|
||||
|
||||
test("on forces the tool even for image-capable models", () => {
|
||||
expect(activeFor({ "inspect_image.mode": "on" }, visionModel)).toBe(true);
|
||||
});
|
||||
|
||||
test("off suppresses the tool even for text-only models", () => {
|
||||
expect(activeFor({ "inspect_image.mode": "off" }, textModel)).toBe(false);
|
||||
});
|
||||
|
||||
test("session override wins over the persisted setting", () => {
|
||||
expect(activeFor({}, visionModel, "on")).toBe(true);
|
||||
expect(activeFor({ "inspect_image.mode": "on" }, visionModel, "off")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("inspect_image.enabled migration", () => {
|
||||
test("nested enabled=true migrates to mode on", () => {
|
||||
const settings = Settings.isolated({ "inspect_image.enabled": true });
|
||||
expect(settings.get("inspect_image.mode")).toBe("on");
|
||||
});
|
||||
|
||||
test("nested enabled=false migrates to mode off", () => {
|
||||
const settings = Settings.isolated({ "inspect_image.enabled": false });
|
||||
expect(settings.get("inspect_image.mode")).toBe("off");
|
||||
});
|
||||
|
||||
test("explicit mode wins over a stale legacy key", () => {
|
||||
const settings = Settings.isolated({ "inspect_image.enabled": true, "inspect_image.mode": "off" });
|
||||
expect(settings.get("inspect_image.mode")).toBe("off");
|
||||
});
|
||||
|
||||
describe("flat dotted key in config.yml", () => {
|
||||
let agentDir: string;
|
||||
afterEach(() => {
|
||||
if (agentDir) fs.rmSync(agentDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("flat inspect_image.enabled migrates to mode", async () => {
|
||||
agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-vision-migration-"));
|
||||
fs.writeFileSync(path.join(agentDir, "config.yml"), '"inspect_image.enabled": true\n');
|
||||
const settings = await Settings.loadReadOnly({ agentDir, cwd: agentDir });
|
||||
expect(settings.get("inspect_image.mode")).toBe("on");
|
||||
});
|
||||
|
||||
test("nested inspect_image.enabled in config.yml migrates to mode", async () => {
|
||||
agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-vision-migration-"));
|
||||
fs.writeFileSync(path.join(agentDir, "config.yml"), "inspect_image:\n enabled: false\n");
|
||||
const settings = await Settings.loadReadOnly({ agentDir, cwd: agentDir });
|
||||
expect(settings.get("inspect_image.mode")).toBe("off");
|
||||
});
|
||||
|
||||
test("flat explicit mode survives alongside a flat legacy key", async () => {
|
||||
agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "omp-vision-migration-"));
|
||||
fs.writeFileSync(
|
||||
path.join(agentDir, "config.yml"),
|
||||
'"inspect_image.enabled": true\n"inspect_image.mode": "off"\n',
|
||||
);
|
||||
const settings = await Settings.loadReadOnly({ agentDir, cwd: agentDir });
|
||||
expect(settings.get("inspect_image.mode")).toBe("off");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("read tool follows actual tool availability", () => {
|
||||
function readSession(inspectImageActive: boolean): ToolSession {
|
||||
return {
|
||||
cwd: os.tmpdir(),
|
||||
hasUI: false,
|
||||
settings: Settings.isolated(),
|
||||
getSessionFile: () => null,
|
||||
getSessionSpawns: () => null,
|
||||
getActiveModel: () => textModel,
|
||||
isToolActive: (name: string) => name === "inspect_image" && inspectImageActive,
|
||||
} as unknown as ToolSession;
|
||||
}
|
||||
|
||||
test("restricted session (tool absent) never advertises inspect_image", () => {
|
||||
// auto mode + text-only model would compute active=true, but the tool is
|
||||
// not in this session's slate, so read must serve inline image blocks.
|
||||
const tool = new ReadTool(readSession(false));
|
||||
expect(tool.description).not.toContain("call `inspect_image`");
|
||||
expect(tool.syncInspectImageState()).toBe(false);
|
||||
});
|
||||
|
||||
test("session with the tool registered advertises it", () => {
|
||||
const tool = new ReadTool(readSession(true));
|
||||
expect(tool.syncInspectImageState()).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user