Merge remote-tracking branch 'origin/farm/d3ff0cb8/detect-lm-studio-vision'

2026-06-18 16:58:42 +02:00
parent 8fcb45b9fe 94140c49cd
commit 1a5faf4a33
6 changed files with 265 additions and 11 deletions
@@ -8,6 +8,10 @@

 - Fixed MiniMax Anthropic-compatible M2/M3 thinking metadata to expose the adaptive transport and keep M2 mandatory reasoning floored ([#2928](https://github.com/can1357/oh-my-pi/issues/2928)).

+### Fixed
+
+- Fixed LM Studio dynamic discovery to use native `/api/v0/models` metadata so VLM models advertise image input. ([#2945](https://github.com/can1357/oh-my-pi/issues/2945))
+
 ## [16.0.6] - 2026-06-18

 ### Added
@@ -2184,6 +2184,86 @@ export function kimiCodeModelManagerOptions(
 // 12.5. LM Studio
 // ---------------------------------------------------------------------------

+/** Native LM Studio metadata keyed by model id from `/api/v0/models`. */
+export interface LmStudioNativeModelMetadata {
+	input: ("text" | "image")[];
+	contextWindow?: number;
+}
+
+/** Options for LM Studio's optional native metadata probe. */
+export interface LmStudioNativeModelMetadataOptions {
+	headers?: Record<string, string>;
+	signal?: AbortSignal;
+}
+
+const LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS = 250;
+
+function toLmStudioNativeBaseUrl(baseUrl: string): string {
+	const trimmed = baseUrl.trim();
+	const normalized = trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed;
+	return normalized.endsWith("/v1") ? normalized.slice(0, -3) : normalized;
+}
+
+function getLmStudioCapabilityNames(value: unknown): string[] {
+	if (!Array.isArray(value)) {
+		return [];
+	}
+	return value.flatMap(item => (typeof item === "string" ? [item.toLowerCase()] : []));
+}
+
+function getLmStudioNativeInput(entry: Record<string, unknown>): ("text" | "image")[] {
+	const modelType = typeof entry.type === "string" ? entry.type.toLowerCase() : "";
+	const capabilities = getLmStudioCapabilityNames(entry.capabilities);
+	const supportsImage = modelType === "vlm" || capabilities.includes("vision") || capabilities.includes("image");
+	return supportsImage ? ["text", "image"] : ["text"];
+}
+
+function getLmStudioNativeContextWindow(entry: Record<string, unknown>): number | undefined {
+	return (
+		toPositiveNumber(entry.max_context_length, null) ??
+		toPositiveNumber(entry.context_length, null) ??
+		toPositiveNumber(entry.max_model_len, null) ??
+		undefined
+	);
+}
+
+/** Fetches LM Studio native model metadata used to mark VLM models as image-capable. */
+export async function fetchLmStudioNativeModelMetadata(
+	baseUrl: string,
+	fetchImpl: FetchImpl = fetch,
+	options?: LmStudioNativeModelMetadataOptions,
+): Promise<Map<string, LmStudioNativeModelMetadata> | null> {
+	const nativeBaseUrl = toLmStudioNativeBaseUrl(baseUrl);
+	try {
+		const response = await fetchImpl(`${nativeBaseUrl}/api/v0/models`, {
+			method: "GET",
+			headers: { Accept: "application/json", ...(options?.headers ?? {}) },
+			signal: options?.signal ?? AbortSignal.timeout(LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS),
+		});
+		if (!response.ok) {
+			return null;
+		}
+		const payload = await response.json();
+		if (!isRecord(payload) || !Array.isArray(payload.data)) {
+			return null;
+		}
+		const metadata = new Map<string, LmStudioNativeModelMetadata>();
+		for (const entry of payload.data) {
+			if (!isRecord(entry) || typeof entry.id !== "string" || entry.id.length === 0) {
+				continue;
+			}
+			const contextWindow = getLmStudioNativeContextWindow(entry);
+			metadata.set(entry.id, {
+				input: getLmStudioNativeInput(entry),
+				...(contextWindow === undefined ? {} : { contextWindow }),
+			});
+		}
+		return metadata;
+	} catch {
+		return null;
+	}
+}
+
 export interface LmStudioModelManagerConfig {
 	apiKey?: string;
 	baseUrl?: string;
@@ -2198,8 +2278,11 @@ export function lmStudioModelManagerOptions(
 	const references = createBundledReferenceMap<"openai-completions">("lm-studio" as any);
 	return {
 		providerId: "lm-studio",
-		fetchDynamicModels: () =>
-			fetchOpenAICompatibleModels({
+		fetchDynamicModels: async () => {
+			const nativeMetadataPromise = fetchLmStudioNativeModelMetadata(baseUrl, config?.fetch, {
+				headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
+			});
+			const models = await fetchOpenAICompatibleModels({
 				api: "openai-completions",
 				provider: "lm-studio",
 				baseUrl,
@@ -2209,7 +2292,26 @@ export function lmStudioModelManagerOptions(
 					return mapWithBundledReference(entry, defaults, reference);
 				},
 				fetch: config?.fetch,
-			}),
+			});
+			if (!models) {
+				return models;
+			}
+			const nativeMetadata = await nativeMetadataPromise;
+			if (!nativeMetadata) {
+				return models;
+			}
+			return models.map(model => {
+				const metadata = nativeMetadata.get(model.id);
+				if (!metadata) {
+					return model;
+				}
+				return {
+					...model,
+					input: metadata.input,
+					contextWindow: metadata.contextWindow ?? model.contextWindow,
+				};
+			});
+		},
 	};
 }

@@ -0,0 +1,88 @@
+import { describe, expect, test, vi } from "bun:test";
+import { lmStudioModelManagerOptions } from "@oh-my-pi/pi-catalog/provider-models/openai-compat";
+import type { FetchImpl } from "@oh-my-pi/pi-catalog/types";
+
+describe("lm studio local provider discovery", () => {
+	test("marks native VLM models as image-capable", async () => {
+		const requestedUrls: string[] = [];
+		const fetchMock: FetchImpl = vi.fn(async input => {
+			const url = String(input);
+			requestedUrls.push(url);
+			if (url === "http://127.0.0.1:1234/api/v0/models") {
+				return new Response(
+					JSON.stringify({
+						data: [
+							{
+								id: "qwen/qwen3.6-27b",
+								type: "vlm",
+								capabilities: ["tool_use"],
+								max_context_length: 262144,
+							},
+							{ id: "plain-llm", type: "llm" },
+						],
+					}),
+					{ status: 200, headers: { "Content-Type": "application/json" } },
+				);
+			}
+			if (url === "http://127.0.0.1:1234/v1/models") {
+				return new Response(
+					JSON.stringify({
+						data: [
+							{ id: "qwen/qwen3.6-27b", object: "model" },
+							{ id: "plain-llm", object: "model" },
+						],
+					}),
+					{ status: 200, headers: { "Content-Type": "application/json" } },
+				);
+			}
+			throw new Error(`Unexpected URL: ${url}`);
+		});
+
+		const models = await lmStudioModelManagerOptions({ fetch: fetchMock }).fetchDynamicModels?.();
+		const vision = models?.find(model => model.id === "qwen/qwen3.6-27b");
+		const text = models?.find(model => model.id === "plain-llm");
+
+		expect(requestedUrls).toContain("http://127.0.0.1:1234/api/v0/models");
+		expect(vision?.input).toEqual(["text", "image"]);
+		expect(vision?.contextWindow).toBe(262144);
+		expect(text?.input).toEqual(["text"]);
+	});
+
+	test("falls back to the OpenAI-compatible catalog when native metadata hangs", async () => {
+		let nativeAborted = false;
+		let openAiCatalogStartedBeforeAbort = false;
+		const fetchMock: FetchImpl = vi.fn(async (input, init) => {
+			const url = String(input);
+			if (url === "http://127.0.0.1:11434/api/v0/models") {
+				const pending = Promise.withResolvers<Response>();
+				const abort = () => {
+					nativeAborted = true;
+					pending.reject(new DOMException("Aborted", "AbortError"));
+				};
+				if (init?.signal?.aborted) {
+					abort();
+				} else {
+					init?.signal?.addEventListener("abort", abort, { once: true });
+				}
+				return pending.promise;
+			}
+			if (url === "http://127.0.0.1:11434/v1/models") {
+				openAiCatalogStartedBeforeAbort = !nativeAborted;
+				return new Response(JSON.stringify({ data: [{ id: "omlx-model", object: "model" }] }), {
+					status: 200,
+					headers: { "Content-Type": "application/json" },
+				});
+			}
+			throw new Error(`Unexpected URL: ${url}`);
+		});
+
+		const models = await lmStudioModelManagerOptions({
+			baseUrl: "http://127.0.0.1:11434/v1",
+			fetch: fetchMock,
+		}).fetchDynamicModels?.();
+
+		expect(openAiCatalogStartedBeforeAbort).toBe(true);
+		expect(nativeAborted).toBe(true);
+		expect(models?.find(model => model.id === "omlx-model")?.input).toEqual(["text"]);
+	});
+});
@@ -65,6 +65,7 @@
 - Fixed `/plan <prompt>` and `/goal <objective>` to preserve the typed slash-command line in TUI input history when entering those modes from off ([#2887](https://github.com/can1357/oh-my-pi/issues/2887)).
 - Fixed `/model` in the TUI to open the active-session model switcher instead of the role-assignment picker ([#2846](https://github.com/can1357/oh-my-pi/issues/2846)).
 - Fixed Perplexity web search collapsing every upstream failure to a generic `401 No authentication method available` once all auth methods failed: the fallback loop now rethrows the last classified provider error (`402`/credits-exhausted, `429`, `5xx`), so quota and rate-limit failures are no longer mis-reported as authorization errors. The generic 401 is now only a defensive fallback for the no-method-ran case.
+- Fixed LM Studio runtime discovery to use native `/api/v0/models` metadata so `inspect_image` can select VLM models. ([#2945](https://github.com/can1357/oh-my-pi/issues/2945))

 ### Security

@@ -13,6 +13,7 @@ import {
 	resolveModelReference,
 	stripBracketedModelIdAffixes,
 } from "@oh-my-pi/pi-catalog/identity";
+import { fetchLmStudioNativeModelMetadata } from "@oh-my-pi/pi-catalog/provider-models/openai-compat";
 import type { ModelSpec } from "@oh-my-pi/pi-catalog/types";
 import { isRecord } from "@oh-my-pi/pi-utils";
 import type { ProviderDiscovery } from "./models-config-schema";
@@ -379,18 +380,25 @@ export async function discoverOpenAIModelsList(
 	const baseHeaders: Record<string, string> = { ...(providerConfig.headers ?? {}) };
 	let headers = baseHeaders;
 	const attempt = async (h: Record<string, string>) => {
-		const res = await ctx.fetch(modelsUrl, {
-			headers: h,
-			signal: AbortSignal.timeout(10_000),
-		});
+		const nativeMetadataPromise =
+			providerConfig.discovery.type === "lm-studio"
+				? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, { headers: h })
+				: Promise.resolve(null);
+		const [res, nativeMetadata] = await Promise.all([
+			ctx.fetch(modelsUrl, {
+				headers: h,
+				signal: AbortSignal.timeout(10_000),
+			}),
+			nativeMetadataPromise,
+		]);
 		if (!res.ok) {
 			throw new Error(`HTTP ${res.status} from ${modelsUrl}`);
 		}
 		headers = h;
-		return res;
+		return [res, nativeMetadata] as const;
 	};
 	const apiKey = await ctx.getBearerApiKeyResolver(providerConfig.provider);
-	const response = apiKey
+	const [response, nativeMetadata] = apiKey
 		? await withAuth(apiKey, key => attempt({ ...baseHeaders, Authorization: `Bearer ${key}` }))
 		: await attempt(baseHeaders);
 	const payload = (await response.json()) as {
@@ -401,8 +409,12 @@ export async function discoverOpenAIModelsList(
 	for (const item of models) {
 		const id = item.id;
 		if (!id) continue;
+		const nativeMetadataForModel = nativeMetadata?.get(id);
 		const contextWindow =
-			toPositiveNumberOrUndefined(item.max_model_len) ?? toPositiveNumberOrUndefined(item.context_length) ?? 128000;
+			toPositiveNumberOrUndefined(item.max_model_len) ??
+			toPositiveNumberOrUndefined(item.context_length) ??
+			nativeMetadataForModel?.contextWindow ??
+			128000;
 		discovered.push(
 			buildModel({
 				id,
@@ -411,7 +423,7 @@ export async function discoverOpenAIModelsList(
 				provider: providerConfig.provider,
 				baseUrl,
 				reasoning: false,
-				input: ["text"],
+				input: nativeMetadataForModel?.input ?? ["text"],
 				cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
 				contextWindow,
 				maxTokens: Math.min(contextWindow, discoveryDefaultMaxTokens(providerConfig.api)),
@@ -60,6 +60,53 @@ describe("ModelRegistry LM Studio Fixes", () => {
 		expect(available.some(m => m.provider === "lm-studio")).toBe(true);
 	});

+	test("marks LM Studio native VLM models as image-capable", async () => {
+		const fetchMock: FetchImpl = input => {
+			const url = String(input);
+			if (url === "http://127.0.0.1:1234/api/v0/models") {
+				return Promise.resolve(
+					new Response(
+						JSON.stringify({
+							data: [
+								{
+									id: "qwen/qwen3.6-27b",
+									type: "vlm",
+									capabilities: ["tool_use"],
+									max_context_length: 262144,
+								},
+								{ id: "plain-llm", type: "llm" },
+							],
+						}),
+						{ status: 200, headers: { "Content-Type": "application/json" } },
+					),
+				);
+			}
+			if (url === "http://127.0.0.1:1234/v1/models") {
+				return Promise.resolve(
+					new Response(
+						JSON.stringify({
+							data: [
+								{ id: "qwen/qwen3.6-27b", object: "model" },
+								{ id: "plain-llm", object: "model" },
+							],
+						}),
+						{ status: 200, headers: { "Content-Type": "application/json" } },
+					),
+				);
+			}
+			return Promise.resolve(new Response(null, { status: 404 }));
+		};
+
+		const registry = new ModelRegistry(authStorage, modelsJsonPath, { fetch: fetchMock });
+		await registry.refresh();
+
+		const vision = registry.find("lm-studio", "qwen/qwen3.6-27b");
+		const text = registry.find("lm-studio", "plain-llm");
+		expect(vision?.input).toEqual(["text", "image"]);
+		expect(vision?.contextWindow).toBe(262144);
+		expect(text?.input).toEqual(["text"]);
+	});
+
 	test("LM_STUDIO_BASE_URL can target any local OpenAI-compatible /v1 server", async () => {
 		const originalBaseUrl = Bun.env.LM_STUDIO_BASE_URL;
 		Bun.env.LM_STUDIO_BASE_URL = "http://127.0.0.1:11434/v1";