diff --git a/packages/catalog/CHANGELOG.md b/packages/catalog/CHANGELOG.md index 344434576..0160cec7a 100644 --- a/packages/catalog/CHANGELOG.md +++ b/packages/catalog/CHANGELOG.md @@ -8,6 +8,10 @@ - Fixed MiniMax Anthropic-compatible M2/M3 thinking metadata to expose the adaptive transport and keep M2 mandatory reasoning floored ([#2928](https://github.com/can1357/oh-my-pi/issues/2928)). +### Fixed + +- Fixed LM Studio dynamic discovery to use native `/api/v0/models` metadata so VLM models advertise image input. ([#2945](https://github.com/can1357/oh-my-pi/issues/2945)) + ## [16.0.6] - 2026-06-18 ### Added diff --git a/packages/catalog/src/provider-models/openai-compat.ts b/packages/catalog/src/provider-models/openai-compat.ts index 2de54c971..658d22492 100644 --- a/packages/catalog/src/provider-models/openai-compat.ts +++ b/packages/catalog/src/provider-models/openai-compat.ts @@ -2184,6 +2184,86 @@ export function kimiCodeModelManagerOptions( // 12.5. LM Studio // --------------------------------------------------------------------------- +/** Native LM Studio metadata keyed by model id from `/api/v0/models`. */ +export interface LmStudioNativeModelMetadata { + input: ("text" | "image")[]; + contextWindow?: number; +} + +/** Options for LM Studio's optional native metadata probe. */ +export interface LmStudioNativeModelMetadataOptions { + headers?: Record; + signal?: AbortSignal; +} + +const LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS = 250; + +function toLmStudioNativeBaseUrl(baseUrl: string): string { + const trimmed = baseUrl.trim(); + const normalized = trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed; + return normalized.endsWith("/v1") ? normalized.slice(0, -3) : normalized; +} + +function getLmStudioCapabilityNames(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap(item => (typeof item === "string" ? [item.toLowerCase()] : [])); +} + +function getLmStudioNativeInput(entry: Record): ("text" | "image")[] { + const modelType = typeof entry.type === "string" ? entry.type.toLowerCase() : ""; + const capabilities = getLmStudioCapabilityNames(entry.capabilities); + const supportsImage = modelType === "vlm" || capabilities.includes("vision") || capabilities.includes("image"); + return supportsImage ? ["text", "image"] : ["text"]; +} + +function getLmStudioNativeContextWindow(entry: Record): number | undefined { + return ( + toPositiveNumber(entry.max_context_length, null) ?? + toPositiveNumber(entry.context_length, null) ?? + toPositiveNumber(entry.max_model_len, null) ?? + undefined + ); +} + +/** Fetches LM Studio native model metadata used to mark VLM models as image-capable. */ +export async function fetchLmStudioNativeModelMetadata( + baseUrl: string, + fetchImpl: FetchImpl = fetch, + options?: LmStudioNativeModelMetadataOptions, +): Promise | null> { + const nativeBaseUrl = toLmStudioNativeBaseUrl(baseUrl); + try { + const response = await fetchImpl(`${nativeBaseUrl}/api/v0/models`, { + method: "GET", + headers: { Accept: "application/json", ...(options?.headers ?? {}) }, + signal: options?.signal ?? AbortSignal.timeout(LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS), + }); + if (!response.ok) { + return null; + } + const payload = await response.json(); + if (!isRecord(payload) || !Array.isArray(payload.data)) { + return null; + } + const metadata = new Map(); + for (const entry of payload.data) { + if (!isRecord(entry) || typeof entry.id !== "string" || entry.id.length === 0) { + continue; + } + const contextWindow = getLmStudioNativeContextWindow(entry); + metadata.set(entry.id, { + input: getLmStudioNativeInput(entry), + ...(contextWindow === undefined ? {} : { contextWindow }), + }); + } + return metadata; + } catch { + return null; + } +} + export interface LmStudioModelManagerConfig { apiKey?: string; baseUrl?: string; @@ -2198,8 +2278,11 @@ export function lmStudioModelManagerOptions( const references = createBundledReferenceMap<"openai-completions">("lm-studio" as any); return { providerId: "lm-studio", - fetchDynamicModels: () => - fetchOpenAICompatibleModels({ + fetchDynamicModels: async () => { + const nativeMetadataPromise = fetchLmStudioNativeModelMetadata(baseUrl, config?.fetch, { + headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined, + }); + const models = await fetchOpenAICompatibleModels({ api: "openai-completions", provider: "lm-studio", baseUrl, @@ -2209,7 +2292,26 @@ export function lmStudioModelManagerOptions( return mapWithBundledReference(entry, defaults, reference); }, fetch: config?.fetch, - }), + }); + if (!models) { + return models; + } + const nativeMetadata = await nativeMetadataPromise; + if (!nativeMetadata) { + return models; + } + return models.map(model => { + const metadata = nativeMetadata.get(model.id); + if (!metadata) { + return model; + } + return { + ...model, + input: metadata.input, + contextWindow: metadata.contextWindow ?? model.contextWindow, + }; + }); + }, }; } diff --git a/packages/catalog/test/lm-studio-provider.test.ts b/packages/catalog/test/lm-studio-provider.test.ts new file mode 100644 index 000000000..6b8aee0fd --- /dev/null +++ b/packages/catalog/test/lm-studio-provider.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, test, vi } from "bun:test"; +import { lmStudioModelManagerOptions } from "@oh-my-pi/pi-catalog/provider-models/openai-compat"; +import type { FetchImpl } from "@oh-my-pi/pi-catalog/types"; + +describe("lm studio local provider discovery", () => { + test("marks native VLM models as image-capable", async () => { + const requestedUrls: string[] = []; + const fetchMock: FetchImpl = vi.fn(async input => { + const url = String(input); + requestedUrls.push(url); + if (url === "http://127.0.0.1:1234/api/v0/models") { + return new Response( + JSON.stringify({ + data: [ + { + id: "qwen/qwen3.6-27b", + type: "vlm", + capabilities: ["tool_use"], + max_context_length: 262144, + }, + { id: "plain-llm", type: "llm" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url === "http://127.0.0.1:1234/v1/models") { + return new Response( + JSON.stringify({ + data: [ + { id: "qwen/qwen3.6-27b", object: "model" }, + { id: "plain-llm", object: "model" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + throw new Error(`Unexpected URL: ${url}`); + }); + + const models = await lmStudioModelManagerOptions({ fetch: fetchMock }).fetchDynamicModels?.(); + const vision = models?.find(model => model.id === "qwen/qwen3.6-27b"); + const text = models?.find(model => model.id === "plain-llm"); + + expect(requestedUrls).toContain("http://127.0.0.1:1234/api/v0/models"); + expect(vision?.input).toEqual(["text", "image"]); + expect(vision?.contextWindow).toBe(262144); + expect(text?.input).toEqual(["text"]); + }); + + test("falls back to the OpenAI-compatible catalog when native metadata hangs", async () => { + let nativeAborted = false; + let openAiCatalogStartedBeforeAbort = false; + const fetchMock: FetchImpl = vi.fn(async (input, init) => { + const url = String(input); + if (url === "http://127.0.0.1:11434/api/v0/models") { + const pending = Promise.withResolvers(); + const abort = () => { + nativeAborted = true; + pending.reject(new DOMException("Aborted", "AbortError")); + }; + if (init?.signal?.aborted) { + abort(); + } else { + init?.signal?.addEventListener("abort", abort, { once: true }); + } + return pending.promise; + } + if (url === "http://127.0.0.1:11434/v1/models") { + openAiCatalogStartedBeforeAbort = !nativeAborted; + return new Response(JSON.stringify({ data: [{ id: "omlx-model", object: "model" }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + throw new Error(`Unexpected URL: ${url}`); + }); + + const models = await lmStudioModelManagerOptions({ + baseUrl: "http://127.0.0.1:11434/v1", + fetch: fetchMock, + }).fetchDynamicModels?.(); + + expect(openAiCatalogStartedBeforeAbort).toBe(true); + expect(nativeAborted).toBe(true); + expect(models?.find(model => model.id === "omlx-model")?.input).toEqual(["text"]); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b579e1f76..cf53d1521 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -65,6 +65,7 @@ - Fixed `/plan ` and `/goal ` to preserve the typed slash-command line in TUI input history when entering those modes from off ([#2887](https://github.com/can1357/oh-my-pi/issues/2887)). - Fixed `/model` in the TUI to open the active-session model switcher instead of the role-assignment picker ([#2846](https://github.com/can1357/oh-my-pi/issues/2846)). - Fixed Perplexity web search collapsing every upstream failure to a generic `401 No authentication method available` once all auth methods failed: the fallback loop now rethrows the last classified provider error (`402`/credits-exhausted, `429`, `5xx`), so quota and rate-limit failures are no longer mis-reported as authorization errors. The generic 401 is now only a defensive fallback for the no-method-ran case. +- Fixed LM Studio runtime discovery to use native `/api/v0/models` metadata so `inspect_image` can select VLM models. ([#2945](https://github.com/can1357/oh-my-pi/issues/2945)) ### Security diff --git a/packages/coding-agent/src/config/model-discovery.ts b/packages/coding-agent/src/config/model-discovery.ts index 9ea35cba4..fc5226dc7 100644 --- a/packages/coding-agent/src/config/model-discovery.ts +++ b/packages/coding-agent/src/config/model-discovery.ts @@ -13,6 +13,7 @@ import { resolveModelReference, stripBracketedModelIdAffixes, } from "@oh-my-pi/pi-catalog/identity"; +import { fetchLmStudioNativeModelMetadata } from "@oh-my-pi/pi-catalog/provider-models/openai-compat"; import type { ModelSpec } from "@oh-my-pi/pi-catalog/types"; import { isRecord } from "@oh-my-pi/pi-utils"; import type { ProviderDiscovery } from "./models-config-schema"; @@ -379,18 +380,25 @@ export async function discoverOpenAIModelsList( const baseHeaders: Record = { ...(providerConfig.headers ?? {}) }; let headers = baseHeaders; const attempt = async (h: Record) => { - const res = await ctx.fetch(modelsUrl, { - headers: h, - signal: AbortSignal.timeout(10_000), - }); + const nativeMetadataPromise = + providerConfig.discovery.type === "lm-studio" + ? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, { headers: h }) + : Promise.resolve(null); + const [res, nativeMetadata] = await Promise.all([ + ctx.fetch(modelsUrl, { + headers: h, + signal: AbortSignal.timeout(10_000), + }), + nativeMetadataPromise, + ]); if (!res.ok) { throw new Error(`HTTP ${res.status} from ${modelsUrl}`); } headers = h; - return res; + return [res, nativeMetadata] as const; }; const apiKey = await ctx.getBearerApiKeyResolver(providerConfig.provider); - const response = apiKey + const [response, nativeMetadata] = apiKey ? await withAuth(apiKey, key => attempt({ ...baseHeaders, Authorization: `Bearer ${key}` })) : await attempt(baseHeaders); const payload = (await response.json()) as { @@ -401,8 +409,12 @@ export async function discoverOpenAIModelsList( for (const item of models) { const id = item.id; if (!id) continue; + const nativeMetadataForModel = nativeMetadata?.get(id); const contextWindow = - toPositiveNumberOrUndefined(item.max_model_len) ?? toPositiveNumberOrUndefined(item.context_length) ?? 128000; + toPositiveNumberOrUndefined(item.max_model_len) ?? + toPositiveNumberOrUndefined(item.context_length) ?? + nativeMetadataForModel?.contextWindow ?? + 128000; discovered.push( buildModel({ id, @@ -411,7 +423,7 @@ export async function discoverOpenAIModelsList( provider: providerConfig.provider, baseUrl, reasoning: false, - input: ["text"], + input: nativeMetadataForModel?.input ?? ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow, maxTokens: Math.min(contextWindow, discoveryDefaultMaxTokens(providerConfig.api)), diff --git a/packages/coding-agent/test/lm-studio-fix.test.ts b/packages/coding-agent/test/lm-studio-fix.test.ts index 8e314a107..87a6b102e 100644 --- a/packages/coding-agent/test/lm-studio-fix.test.ts +++ b/packages/coding-agent/test/lm-studio-fix.test.ts @@ -60,6 +60,53 @@ describe("ModelRegistry LM Studio Fixes", () => { expect(available.some(m => m.provider === "lm-studio")).toBe(true); }); + test("marks LM Studio native VLM models as image-capable", async () => { + const fetchMock: FetchImpl = input => { + const url = String(input); + if (url === "http://127.0.0.1:1234/api/v0/models") { + return Promise.resolve( + new Response( + JSON.stringify({ + data: [ + { + id: "qwen/qwen3.6-27b", + type: "vlm", + capabilities: ["tool_use"], + max_context_length: 262144, + }, + { id: "plain-llm", type: "llm" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + } + if (url === "http://127.0.0.1:1234/v1/models") { + return Promise.resolve( + new Response( + JSON.stringify({ + data: [ + { id: "qwen/qwen3.6-27b", object: "model" }, + { id: "plain-llm", object: "model" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + } + return Promise.resolve(new Response(null, { status: 404 })); + }; + + const registry = new ModelRegistry(authStorage, modelsJsonPath, { fetch: fetchMock }); + await registry.refresh(); + + const vision = registry.find("lm-studio", "qwen/qwen3.6-27b"); + const text = registry.find("lm-studio", "plain-llm"); + expect(vision?.input).toEqual(["text", "image"]); + expect(vision?.contextWindow).toBe(262144); + expect(text?.input).toEqual(["text"]); + }); + test("LM_STUDIO_BASE_URL can target any local OpenAI-compatible /v1 server", async () => { const originalBaseUrl = Bun.env.LM_STUDIO_BASE_URL; Bun.env.LM_STUDIO_BASE_URL = "http://127.0.0.1:11434/v1";