fix(providers): bounded lm studio native probe
Limited the optional LM Studio /api/v0/models metadata lookup with an abort timeout and started the catalog request independently so OpenAI-compatible servers without the native endpoint still refresh. Added a regression for hung native metadata probes.\n\nFixes #2945
This commit is contained in:
@@ -2190,6 +2190,14 @@ export interface LmStudioNativeModelMetadata {
|
||||
contextWindow?: number;
|
||||
}
|
||||
|
||||
/** Options for LM Studio's optional native metadata probe. */
|
||||
export interface LmStudioNativeModelMetadataOptions {
|
||||
headers?: Record<string, string>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
const LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS = 250;
|
||||
|
||||
function toLmStudioNativeBaseUrl(baseUrl: string): string {
|
||||
const trimmed = baseUrl.trim();
|
||||
const normalized = trimmed.endsWith("/") ? trimmed.slice(0, -1) : trimmed;
|
||||
@@ -2223,13 +2231,14 @@ function getLmStudioNativeContextWindow(entry: Record<string, unknown>): number
|
||||
export async function fetchLmStudioNativeModelMetadata(
|
||||
baseUrl: string,
|
||||
fetchImpl: FetchImpl = fetch,
|
||||
headers?: Record<string, string>,
|
||||
options?: LmStudioNativeModelMetadataOptions,
|
||||
): Promise<Map<string, LmStudioNativeModelMetadata> | null> {
|
||||
const nativeBaseUrl = toLmStudioNativeBaseUrl(baseUrl);
|
||||
try {
|
||||
const response = await fetchImpl(`${nativeBaseUrl}/api/v0/models`, {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", ...(headers ?? {}) },
|
||||
headers: { Accept: "application/json", ...(options?.headers ?? {}) },
|
||||
signal: options?.signal ?? AbortSignal.timeout(LM_STUDIO_NATIVE_METADATA_TIMEOUT_MS),
|
||||
});
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
@@ -2270,31 +2279,38 @@ export function lmStudioModelManagerOptions(
|
||||
return {
|
||||
providerId: "lm-studio",
|
||||
fetchDynamicModels: async () => {
|
||||
const nativeMetadata = await fetchLmStudioNativeModelMetadata(
|
||||
baseUrl,
|
||||
config?.fetch,
|
||||
apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
|
||||
);
|
||||
return fetchOpenAICompatibleModels({
|
||||
const nativeMetadataPromise = fetchLmStudioNativeModelMetadata(baseUrl, config?.fetch, {
|
||||
headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined,
|
||||
});
|
||||
const models = await fetchOpenAICompatibleModels({
|
||||
api: "openai-completions",
|
||||
provider: "lm-studio",
|
||||
baseUrl,
|
||||
apiKey,
|
||||
mapModel: (entry, defaults) => {
|
||||
const reference = references.get(defaults.id);
|
||||
const mapped = mapWithBundledReference(entry, defaults, reference);
|
||||
const metadata = nativeMetadata?.get(mapped.id);
|
||||
if (!metadata) {
|
||||
return mapped;
|
||||
}
|
||||
return {
|
||||
...mapped,
|
||||
input: metadata.input,
|
||||
contextWindow: metadata.contextWindow ?? mapped.contextWindow,
|
||||
};
|
||||
return mapWithBundledReference(entry, defaults, reference);
|
||||
},
|
||||
fetch: config?.fetch,
|
||||
});
|
||||
if (!models) {
|
||||
return models;
|
||||
}
|
||||
const nativeMetadata = await nativeMetadataPromise;
|
||||
if (!nativeMetadata) {
|
||||
return models;
|
||||
}
|
||||
return models.map(model => {
|
||||
const metadata = nativeMetadata.get(model.id);
|
||||
if (!metadata) {
|
||||
return model;
|
||||
}
|
||||
return {
|
||||
...model,
|
||||
input: metadata.input,
|
||||
contextWindow: metadata.contextWindow ?? model.contextWindow,
|
||||
};
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -47,4 +47,42 @@ describe("lm studio local provider discovery", () => {
|
||||
expect(vision?.contextWindow).toBe(262144);
|
||||
expect(text?.input).toEqual(["text"]);
|
||||
});
|
||||
|
||||
test("falls back to the OpenAI-compatible catalog when native metadata hangs", async () => {
|
||||
let nativeAborted = false;
|
||||
let openAiCatalogStartedBeforeAbort = false;
|
||||
const fetchMock: FetchImpl = vi.fn(async (input, init) => {
|
||||
const url = String(input);
|
||||
if (url === "http://127.0.0.1:11434/api/v0/models") {
|
||||
const pending = Promise.withResolvers<Response>();
|
||||
const abort = () => {
|
||||
nativeAborted = true;
|
||||
pending.reject(new DOMException("Aborted", "AbortError"));
|
||||
};
|
||||
if (init?.signal?.aborted) {
|
||||
abort();
|
||||
} else {
|
||||
init?.signal?.addEventListener("abort", abort, { once: true });
|
||||
}
|
||||
return pending.promise;
|
||||
}
|
||||
if (url === "http://127.0.0.1:11434/v1/models") {
|
||||
openAiCatalogStartedBeforeAbort = !nativeAborted;
|
||||
return new Response(JSON.stringify({ data: [{ id: "omlx-model", object: "model" }] }), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
}
|
||||
throw new Error(`Unexpected URL: ${url}`);
|
||||
});
|
||||
|
||||
const models = await lmStudioModelManagerOptions({
|
||||
baseUrl: "http://127.0.0.1:11434/v1",
|
||||
fetch: fetchMock,
|
||||
}).fetchDynamicModels?.();
|
||||
|
||||
expect(openAiCatalogStartedBeforeAbort).toBe(true);
|
||||
expect(nativeAborted).toBe(true);
|
||||
expect(models?.find(model => model.id === "omlx-model")?.input).toEqual(["text"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -382,7 +382,7 @@ export async function discoverOpenAIModelsList(
|
||||
const attempt = async (h: Record<string, string>) => {
|
||||
const nativeMetadataPromise =
|
||||
providerConfig.discovery.type === "lm-studio"
|
||||
? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, h)
|
||||
? fetchLmStudioNativeModelMetadata(baseUrl, ctx.fetch, { headers: h })
|
||||
: Promise.resolve(null);
|
||||
const [res, nativeMetadata] = await Promise.all([
|
||||
ctx.fetch(modelsUrl, {
|
||||
|
||||
Reference in New Issue
Block a user