diff --git a/packages/catalog/src/provider-models/openai-compat.ts b/packages/catalog/src/provider-models/openai-compat.ts index c1e2a046b..83cd58136 100644 --- a/packages/catalog/src/provider-models/openai-compat.ts +++ b/packages/catalog/src/provider-models/openai-compat.ts @@ -3036,6 +3036,10 @@ type LiteLLMRichEndpointModel = { model: ModelSpec; supportsVision: unknown; supportsReasoning: unknown; + hasContextWindow: boolean; + hasMaxTokens: boolean; + hasToolMetadata: boolean; + hasSupportedOpenAIParams: boolean; }; const LITELLM_RICH_ENDPOINTS = ["/model_group/info", "/v2/model/info", "/model/info", "/v1/model/info"] as const; @@ -3307,13 +3311,23 @@ async function fetchLiteLLMRichEndpoint( const model = mapLiteLLMRichEntry(entry, options, runtimeBaseUrl); if (model) { const supportsVision = getLiteLLMMetadataValue(entry, "supports_vision"); + const supportsReasoning = getLiteLLMMetadataValue(entry, "supports_reasoning"); + const supportsFunctionCalling = getLiteLLMMetadataValue(entry, "supports_function_calling"); + const supportedOpenAIParams = getSupportedOpenAIParams(entry); if (supportsVision !== true && supportsVision !== false) { incompleteVisionMetadata = true; } deduped.set(model.id, { model, supportsVision, - supportsReasoning: getLiteLLMMetadataValue(entry, "supports_reasoning"), + supportsReasoning, + hasContextWindow: toPositiveNumber(getLiteLLMMetadataValue(entry, "max_input_tokens"), null) !== null, + hasMaxTokens: toPositiveNumber(getLiteLLMMetadataValue(entry, "max_output_tokens"), null) !== null, + hasToolMetadata: + supportsFunctionCalling === true || + supportsFunctionCalling === false || + supportedOpenAIParams !== undefined, + hasSupportedOpenAIParams: supportedOpenAIParams !== undefined, }); } } @@ -3341,25 +3355,40 @@ export async function fetchLiteLLMRichModels( if (!result) { continue; } + const hadPriorModels = deduped.size > 0; for (const next of result.models) { const existing = deduped.get(next.model.id); if (!existing) { - deduped.set(next.model.id, next); + if (!hadPriorModels) { + deduped.set(next.model.id, next); + } continue; } - const model = { + const model: ModelSpec = { ...existing.model, - ...next.model, name: next.model.name === next.model.id ? existing.model.name : next.model.name, + contextWindow: next.hasContextWindow ? next.model.contextWindow : existing.model.contextWindow, + maxTokens: next.hasMaxTokens ? next.model.maxTokens : existing.model.maxTokens, input: next.supportsVision === true || next.supportsVision === false ? next.model.input : existing.model.input, reasoning: typeof next.supportsReasoning === "boolean" ? next.model.reasoning : existing.model.reasoning, + compat: next.hasSupportedOpenAIParams ? next.model.compat : existing.model.compat, }; + if (next.hasToolMetadata) { + model.supportsTools = next.model.supportsTools; + } deduped.set(next.model.id, { ...next, model }); } - if (!result.incompleteVisionMetadata) { + let hasIncompleteVisionMetadata = false; + for (const entry of deduped.values()) { + if (entry.supportsVision !== true && entry.supportsVision !== false) { + hasIncompleteVisionMetadata = true; + break; + } + } + if (!hasIncompleteVisionMetadata) { break; } } diff --git a/packages/catalog/test/litellm-provider.test.ts b/packages/catalog/test/litellm-provider.test.ts index 2524d014d..c353ac16f 100644 --- a/packages/catalog/test/litellm-provider.test.ts +++ b/packages/catalog/test/litellm-provider.test.ts @@ -490,14 +490,16 @@ describe("LiteLLM provider discovery", () => { { model_group: "vision-proxy-model", model_name: "Vision Proxy Model", - max_input_tokens: 128_000, - max_output_tokens: 16_000, + max_input_tokens: 64_000, + max_output_tokens: 8_000, }, ], }); } if (url === "http://primary:4000/v2/model/info") { - return new Response("{}", { status: 404 }); + return Response.json({ + data: [{ model_name: "unrelated-v2-model", model_info: { supports_vision: false } }], + }); } if (url === "http://primary:4000/model/info") { return Response.json({ @@ -506,8 +508,6 @@ describe("LiteLLM provider discovery", () => { { model_name: "vision-proxy-model", model_info: { - max_input_tokens: 128_000, - max_output_tokens: 16_000, supports_vision: true, }, }, @@ -531,12 +531,13 @@ describe("LiteLLM provider discovery", () => { expect(calls).toContain("http://primary:4000/v2/model/info"); expect(calls).toContain("http://primary:4000/model/info"); expect(calls).not.toContain("http://primary:4000/v1/models"); + expect(models).toHaveLength(1); expect(models?.find(model => model.id === "vision-proxy-model")).toMatchObject({ id: "vision-proxy-model", name: "Vision Proxy Model", input: ["text", "image"], - contextWindow: 128_000, - maxTokens: 16_000, + contextWindow: 64_000, + maxTokens: 8_000, }); });