From 6cac1d06a4c16e021c271455353cd83f4c9e1959 Mon Sep 17 00:00:00 2001 From: roboomp Date: Thu, 18 Jun 2026 07:38:36 +0000 Subject: [PATCH 1/2] fix(providers): detected lm studio vlm inputs Queried LM Studio native /api/v0/models metadata during catalog and runtime discovery so type=vlm models advertise image input. Added regressions for both provider-manager and runtime discovery paths.\n\nFixes #2945 --- packages/catalog/CHANGELOG.md | 4 + .../src/provider-models/openai-compat.ts | 94 ++++++++++++++++++- .../catalog/test/lm-studio-provider.test.ts | 50 ++++++++++ packages/coding-agent/CHANGELOG.md | 1 + .../src/config/model-discovery.ts | 28 ++++-- .../coding-agent/test/lm-studio-fix.test.ts | 47 ++++++++++ 6 files changed, 212 insertions(+), 12 deletions(-) create mode 100644 packages/catalog/test/lm-studio-provider.test.ts diff --git a/packages/catalog/CHANGELOG.md b/packages/catalog/CHANGELOG.md index 8644bbdee..da5b2d1eb 100644 --- a/packages/catalog/CHANGELOG.md +++ b/packages/catalog/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed LM Studio dynamic discovery to use native `/api/v0/models` metadata so VLM models advertise image input. ([#2945](https://github.com/can1357/oh-my-pi/issues/2945)) + ## [16.0.6] - 2026-06-18 ### Added diff --git a/packages/catalog/src/provider-models/openai-compat.ts b/packages/catalog/src/provider-models/openai-compat.ts index 2de54c971..b7e7a94e5 100644 --- a/packages/catalog/src/provider-models/openai-compat.ts +++ b/packages/catalog/src/provider-models/openai-compat.ts @@ -2184,6 +2184,77 @@ export function kimiCodeModelManagerOptions( // 12.5. LM Studio // --------------------------------------------------------------------------- +/** Native LM Studio metadata keyed by model id from `/api/v0/models`. */ +export interface LmStudioNativeModelMetadata { + input: ("text" | "image")[]; + contextWindow?: number; +} + +function toLmStudioNativeBaseUrl(baseUrl: string): string { + const trimmed = baseUrl.trim(); + const normalized = trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed; + return normalized.endsWith("/v1") ? normalized.slice(0, -3) : normalized; +} + +function getLmStudioCapabilityNames(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap(item => (typeof item === "string" ? [item.toLowerCase()] : [])); +} + +function getLmStudioNativeInput(entry: Record): ("text" | "image")[] { + const modelType = typeof entry.type === "string" ? entry.type.toLowerCase() : ""; + const capabilities = getLmStudioCapabilityNames(entry.capabilities); + const supportsImage = modelType === "vlm" || capabilities.includes("vision") || capabilities.includes("image"); + return supportsImage ? ["text", "image"] : ["text"]; +} + +function getLmStudioNativeContextWindow(entry: Record): number | undefined { + return ( + toPositiveNumber(entry.max_context_length, null) ?? + toPositiveNumber(entry.context_length, null) ?? + toPositiveNumber(entry.max_model_len, null) ?? + undefined + ); +} + +/** Fetches LM Studio native model metadata used to mark VLM models as image-capable. */ +export async function fetchLmStudioNativeModelMetadata( + baseUrl: string, + fetchImpl: FetchImpl = fetch, + headers?: Record, +): Promise | null> { + const nativeBaseUrl = toLmStudioNativeBaseUrl(baseUrl); + try { + const response = await fetchImpl(`${nativeBaseUrl}/api/v0/models`, { + method: "GET", + headers: { Accept: "application/json", ...(headers ?? {}) }, + }); + if (!response.ok) { + return null; + } + const payload = await response.json(); + if (!isRecord(payload) || !Array.isArray(payload.data)) { + return null; + } + const metadata = new Map(); + for (const entry of payload.data) { + if (!isRecord(entry) || typeof entry.id !== "string" || entry.id.length === 0) { + continue; + } + const contextWindow = getLmStudioNativeContextWindow(entry); + metadata.set(entry.id, { + input: getLmStudioNativeInput(entry), + ...(contextWindow === undefined ? {} : { contextWindow }), + }); + } + return metadata; + } catch { + return null; + } +} + export interface LmStudioModelManagerConfig { apiKey?: string; baseUrl?: string; @@ -2198,18 +2269,33 @@ export function lmStudioModelManagerOptions( const references = createBundledReferenceMap<"openai-completions">("lm-studio" as any); return { providerId: "lm-studio", - fetchDynamicModels: () => - fetchOpenAICompatibleModels({ + fetchDynamicModels: async () => { + const nativeMetadata = await fetchLmStudioNativeModelMetadata( + baseUrl, + config?.fetch, + apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined, + ); + return fetchOpenAICompatibleModels({ api: "openai-completions", provider: "lm-studio", baseUrl, apiKey, mapModel: (entry, defaults) => { const reference = references.get(defaults.id); - return mapWithBundledReference(entry, defaults, reference); + const mapped = mapWithBundledReference(entry, defaults, reference); + const metadata = nativeMetadata?.get(mapped.id); + if (!metadata) { + return mapped; + } + return { + ...mapped, + input: metadata.input, + contextWindow: metadata.contextWindow ?? mapped.contextWindow, + }; }, fetch: config?.fetch, - }), + }); + }, }; } diff --git a/packages/catalog/test/lm-studio-provider.test.ts b/packages/catalog/test/lm-studio-provider.test.ts new file mode 100644 index 000000000..77f34eb9a --- /dev/null +++ b/packages/catalog/test/lm-studio-provider.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test, vi } from "bun:test"; +import { lmStudioModelManagerOptions } from "@oh-my-pi/pi-catalog/provider-models/openai-compat"; +import type { FetchImpl } from "@oh-my-pi/pi-catalog/types"; + +describe("lm studio local provider discovery", () => { + test("marks native VLM models as image-capable", async () => { + const requestedUrls: string[] = []; + const fetchMock: FetchImpl = vi.fn(async input => { + const url = String(input); + requestedUrls.push(url); + if (url === "http://127.0.0.1:1234/api/v0/models") { + return new Response( + JSON.stringify({ + data: [ + { + id: "qwen/qwen3.6-27b", + type: "vlm", + capabilities: ["tool_use"], + max_context_length: 262144, + }, + { id: "plain-llm", type: "llm" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + if (url === "http://127.0.0.1:1234/v1/models") { + return new Response( + JSON.stringify({ + data: [ + { id: "qwen/qwen3.6-27b", object: "model" }, + { id: "plain-llm", object: "model" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + throw new Error(`Unexpected URL: ${url}`); + }); + + const models = await lmStudioModelManagerOptions({ fetch: fetchMock }).fetchDynamicModels?.(); + const vision = models?.find(model => model.id === "qwen/qwen3.6-27b"); + const text = models?.find(model => model.id === "plain-llm"); + + expect(requestedUrls).toContain("http://127.0.0.1:1234/api/v0/models"); + expect(vision?.input).toEqual(["text", "image"]); + expect(vision?.contextWindow).toBe(262144); + expect(text?.input).toEqual(["text"]); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 99b745097..c4118a8e5 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -50,6 +50,7 @@ - Fixed `/plan ` and `/goal ` to preserve the typed slash-command line in TUI input history when entering those modes from off ([#2887](https://github.com/can1357/oh-my-pi/issues/2887)). - Fixed `/model` in the TUI to open the active-session model switcher instead of the role-assignment picker ([#2846](https://github.com/can1357/oh-my-pi/issues/2846)). - Fixed Perplexity web search collapsing every upstream failure to a generic `401 No authentication method available` once all auth methods failed: the fallback loop now rethrows the last classified provider error (`402`/credits-exhausted, `429`, `5xx`), so quota and rate-limit failures are no longer mis-reported as authorization errors. The generic 401 is now only a defensive fallback for the no-method-ran case. +- Fixed LM Studio runtime discovery to use native `/api/v0/models` metadata so `inspect_image` can select VLM models. ([#2945](https://github.com/can1357/oh-my-pi/issues/2945)) ### Security diff --git a/packages/coding-agent/src/config/model-discovery.ts b/packages/coding-agent/src/config/model-discovery.ts index 9ea35cba4..f652b5f07 100644 --- a/packages/coding-agent/src/config/model-discovery.ts +++ b/packages/coding-agent/src/config/model-discovery.ts @@ -13,6 +13,7 @@ import { resolveModelReference, stripBracketedModelIdAffixes, } from "@oh-my-pi/pi-catalog/identity"; +import { fetchLmStudioNativeModelMetadata } from "@oh-my-pi/pi-catalog/provider-models/openai-compat"; import type { ModelSpec } from "@oh-my-pi/pi-catalog/types"; import { isRecord } from "@oh-my-pi/pi-utils"; import type { ProviderDiscovery } from "./models-config-schema"; @@ -379,18 +380,25 @@ export async function discoverOpenAIModelsList( const baseHeaders: Record = { ...(providerConfig.headers ?? {}) }; let headers = baseHeaders; const attempt = async (h: Record) => { - const res = await ctx.fetch(modelsUrl, { - headers: h, - signal: AbortSignal.timeout(10_000), - }); + const nativeMetadataPromise = + providerConfig.discovery.type === "lm-studio" + ? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, h) + : Promise.resolve(null); + const [res, nativeMetadata] = await Promise.all([ + ctx.fetch(modelsUrl, { + headers: h, + signal: AbortSignal.timeout(10_000), + }), + nativeMetadataPromise, + ]); if (!res.ok) { throw new Error(`HTTP ${res.status} from ${modelsUrl}`); } headers = h; - return res; + return [res, nativeMetadata] as const; }; const apiKey = await ctx.getBearerApiKeyResolver(providerConfig.provider); - const response = apiKey + const [response, nativeMetadata] = apiKey ? await withAuth(apiKey, key => attempt({ ...baseHeaders, Authorization: `Bearer ${key}` })) : await attempt(baseHeaders); const payload = (await response.json()) as { @@ -401,8 +409,12 @@ export async function discoverOpenAIModelsList( for (const item of models) { const id = item.id; if (!id) continue; + const nativeMetadataForModel = nativeMetadata?.get(id); const contextWindow = - toPositiveNumberOrUndefined(item.max_model_len) ?? toPositiveNumberOrUndefined(item.context_length) ?? 128000; + toPositiveNumberOrUndefined(item.max_model_len) ?? + toPositiveNumberOrUndefined(item.context_length) ?? + nativeMetadataForModel?.contextWindow ?? + 128000; discovered.push( buildModel({ id, @@ -411,7 +423,7 @@ export async function discoverOpenAIModelsList( provider: providerConfig.provider, baseUrl, reasoning: false, - input: ["text"], + input: nativeMetadataForModel?.input ?? ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow, maxTokens: Math.min(contextWindow, discoveryDefaultMaxTokens(providerConfig.api)), diff --git a/packages/coding-agent/test/lm-studio-fix.test.ts b/packages/coding-agent/test/lm-studio-fix.test.ts index 8e314a107..87a6b102e 100644 --- a/packages/coding-agent/test/lm-studio-fix.test.ts +++ b/packages/coding-agent/test/lm-studio-fix.test.ts @@ -60,6 +60,53 @@ describe("ModelRegistry LM Studio Fixes", () => { expect(available.some(m => m.provider === "lm-studio")).toBe(true); }); + test("marks LM Studio native VLM models as image-capable", async () => { + const fetchMock: FetchImpl = input => { + const url = String(input); + if (url === "http://127.0.0.1:1234/api/v0/models") { + return Promise.resolve( + new Response( + JSON.stringify({ + data: [ + { + id: "qwen/qwen3.6-27b", + type: "vlm", + capabilities: ["tool_use"], + max_context_length: 262144, + }, + { id: "plain-llm", type: "llm" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + } + if (url === "http://127.0.0.1:1234/v1/models") { + return Promise.resolve( + new Response( + JSON.stringify({ + data: [ + { id: "qwen/qwen3.6-27b", object: "model" }, + { id: "plain-llm", object: "model" }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + } + return Promise.resolve(new Response(null, { status: 404 })); + }; + + const registry = new ModelRegistry(authStorage, modelsJsonPath, { fetch: fetchMock }); + await registry.refresh(); + + const vision = registry.find("lm-studio", "qwen/qwen3.6-27b"); + const text = registry.find("lm-studio", "plain-llm"); + expect(vision?.input).toEqual(["text", "image"]); + expect(vision?.contextWindow).toBe(262144); + expect(text?.input).toEqual(["text"]); + }); + test("LM_STUDIO_BASE_URL can target any local OpenAI-compatible /v1 server", async () => { const originalBaseUrl = Bun.env.LM_STUDIO_BASE_URL; Bun.env.LM_STUDIO_BASE_URL = "http://127.0.0.1:11434/v1"; From 94140c49cd2bfd62b1b32e05f968498f225cec1c Mon Sep 17 00:00:00 2001 From: roboomp Date: Thu, 18 Jun 2026 07:58:20 +0000 Subject: [PATCH 2/2] fix(providers): bounded lm studio native probe Limited the optional LM Studio /api/v0/models metadata lookup with an abort timeout and started the catalog request independently so OpenAI-compatible servers without the native endpoint still refresh. Added a regression for hung native metadata probes.\n\nFixes #2945 --- .../src/provider-models/openai-compat.ts | 52 ++++++++++++------- .../catalog/test/lm-studio-provider.test.ts | 38 ++++++++++++++ .../src/config/model-discovery.ts | 2 +- 3 files changed, 73 insertions(+), 19 deletions(-) diff --git a/packages/catalog/src/provider-models/openai-compat.ts b/packages/catalog/src/provider-models/openai-compat.ts index b7e7a94e5..658d22492 100644 --- a/packages/catalog/src/provider-models/openai-compat.ts +++ b/packages/catalog/src/provider-models/openai-compat.ts @@ -2190,6 +2190,14 @@ export interface LmStudioNativeModelMetadata { contextWindow?: number; } +/** Options for LM Studio's optional native metadata probe. */ +export interface LmStudioNativeModelMetadataOptions { + headers?: Record; + signal?: AbortSignal; +} + +const LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS = 250; + function toLmStudioNativeBaseUrl(baseUrl: string): string { const trimmed = baseUrl.trim(); const normalized = trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed; @@ -2223,13 +2231,14 @@ function getLmStudioNativeContextWindow(entry: Record): number export async function fetchLmStudioNativeModelMetadata( baseUrl: string, fetchImpl: FetchImpl = fetch, - headers?: Record, + options?: LmStudioNativeModelMetadataOptions, ): Promise | null> { const nativeBaseUrl = toLmStudioNativeBaseUrl(baseUrl); try { const response = await fetchImpl(`${nativeBaseUrl}/api/v0/models`, { method: "GET", - headers: { Accept: "application/json", ...(headers ?? {}) }, + headers: { Accept: "application/json", ...(options?.headers ?? {}) }, + signal: options?.signal ?? AbortSignal.timeout(LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS), }); if (!response.ok) { return null; @@ -2270,31 +2279,38 @@ export function lmStudioModelManagerOptions( return { providerId: "lm-studio", fetchDynamicModels: async () => { - const nativeMetadata = await fetchLmStudioNativeModelMetadata( - baseUrl, - config?.fetch, - apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined, - ); - return fetchOpenAICompatibleModels({ + const nativeMetadataPromise = fetchLmStudioNativeModelMetadata(baseUrl, config?.fetch, { + headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined, + }); + const models = await fetchOpenAICompatibleModels({ api: "openai-completions", provider: "lm-studio", baseUrl, apiKey, mapModel: (entry, defaults) => { const reference = references.get(defaults.id); - const mapped = mapWithBundledReference(entry, defaults, reference); - const metadata = nativeMetadata?.get(mapped.id); - if (!metadata) { - return mapped; - } - return { - ...mapped, - input: metadata.input, - contextWindow: metadata.contextWindow ?? mapped.contextWindow, - }; + return mapWithBundledReference(entry, defaults, reference); }, fetch: config?.fetch, }); + if (!models) { + return models; + } + const nativeMetadata = await nativeMetadataPromise; + if (!nativeMetadata) { + return models; + } + return models.map(model => { + const metadata = nativeMetadata.get(model.id); + if (!metadata) { + return model; + } + return { + ...model, + input: metadata.input, + contextWindow: metadata.contextWindow ?? model.contextWindow, + }; + }); }, }; } diff --git a/packages/catalog/test/lm-studio-provider.test.ts b/packages/catalog/test/lm-studio-provider.test.ts index 77f34eb9a..6b8aee0fd 100644 --- a/packages/catalog/test/lm-studio-provider.test.ts +++ b/packages/catalog/test/lm-studio-provider.test.ts @@ -47,4 +47,42 @@ describe("lm studio local provider discovery", () => { expect(vision?.contextWindow).toBe(262144); expect(text?.input).toEqual(["text"]); }); + + test("falls back to the OpenAI-compatible catalog when native metadata hangs", async () => { + let nativeAborted = false; + let openAiCatalogStartedBeforeAbort = false; + const fetchMock: FetchImpl = vi.fn(async (input, init) => { + const url = String(input); + if (url === "http://127.0.0.1:11434/api/v0/models") { + const pending = Promise.withResolvers(); + const abort = () => { + nativeAborted = true; + pending.reject(new DOMException("Aborted", "AbortError")); + }; + if (init?.signal?.aborted) { + abort(); + } else { + init?.signal?.addEventListener("abort", abort, { once: true }); + } + return pending.promise; + } + if (url === "http://127.0.0.1:11434/v1/models") { + openAiCatalogStartedBeforeAbort = !nativeAborted; + return new Response(JSON.stringify({ data: [{ id: "omlx-model", object: "model" }] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + throw new Error(`Unexpected URL: ${url}`); + }); + + const models = await lmStudioModelManagerOptions({ + baseUrl: "http://127.0.0.1:11434/v1", + fetch: fetchMock, + }).fetchDynamicModels?.(); + + expect(openAiCatalogStartedBeforeAbort).toBe(true); + expect(nativeAborted).toBe(true); + expect(models?.find(model => model.id === "omlx-model")?.input).toEqual(["text"]); + }); }); diff --git a/packages/coding-agent/src/config/model-discovery.ts b/packages/coding-agent/src/config/model-discovery.ts index f652b5f07..fc5226dc7 100644 --- a/packages/coding-agent/src/config/model-discovery.ts +++ b/packages/coding-agent/src/config/model-discovery.ts @@ -382,7 +382,7 @@ export async function discoverOpenAIModelsList( const attempt = async (h: Record) => { const nativeMetadataPromise = providerConfig.discovery.type === "lm-studio" - ? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, h) + ? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, { headers: h }) : Promise.resolve(null); const [res, nativeMetadata] = await Promise.all([ ctx.fetch(modelsUrl, {