diff --git a/docs/models.md b/docs/models.md index 61335e182..e77cfb840 100644 --- a/docs/models.md +++ b/docs/models.md @@ -300,7 +300,9 @@ When `litellm` is active (for example through `LITELLM_API_KEY` or stored auth), - base URL: explicit provider `baseUrl` / `models.yml` config, otherwise `LITELLM_BASE_URL`, otherwise `http://localhost:4000/v1` - auth mode: `LITELLM_API_KEY` or stored LiteLLM auth when the proxy requires a key -Runtime discovery probes LiteLLM management metadata first: `GET /model_group/info`, then `GET /v2/model/info`, then falls back to the OpenAI-compatible `GET /models` list. Rich metadata maps `max_input_tokens`, `max_output_tokens`, `supports_vision`, and `supports_reasoning`; bare fallback ids are enriched against bundled reference metadata when available. +Runtime discovery probes LiteLLM management metadata in order: `GET /model_group/info`, `GET /v2/model/info`, `GET /model/info`, and `GET /v1/model/info`. The configured key must be authorized to read at least one of these routes; on deployments that restrict management endpoints, grant the route through LiteLLM's `allowed_routes` access controls or use a master/admin key for discovery. + +If every metadata route is unavailable, discovery falls back to the OpenAI-compatible `GET /models` list. A forbidden or failed metadata request is logged once with its endpoint and status; `404` is treated as an absent route. Rich metadata maps per-model context and capability fields, while bare fallback ids are enriched against bundled reference metadata when available. Models absent from the bundled catalog can therefore have unknown context and pricing after fallback. ### Explicit provider discovery diff --git a/packages/catalog/CHANGELOG.md b/packages/catalog/CHANGELOG.md index c268dc798..5b0256de4 100644 --- a/packages/catalog/CHANGELOG.md +++ b/packages/catalog/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Logged LiteLLM rich-metadata endpoint failures once with their endpoint and status before falling back to incomplete `/v1/models` data ([#5801](https://github.com/can1357/oh-my-pi/issues/5801)). + ## [17.0.2] - 2026-07-17 ### Changed diff --git a/packages/catalog/src/provider-models/openai-compat.ts b/packages/catalog/src/provider-models/openai-compat.ts index 27621ff34..74db37217 100644 --- a/packages/catalog/src/provider-models/openai-compat.ts +++ b/packages/catalog/src/provider-models/openai-compat.ts @@ -1,3 +1,4 @@ +import * as logger from "@oh-my-pi/pi-utils/logger"; import { fetchOpenAICompatibleModels, type OpenAICompatibleModelMapperContext, @@ -3267,16 +3268,41 @@ type LiteLLMRichEndpointModel = { hasToolMetadata: boolean; hasSupportedOpenAIParams: boolean; }; +type LiteLLMRichEndpointFailure = { + endpoint: string; + reason: "http-status" | "invalid-json" | "network-error"; + status?: number; + error?: unknown; +}; +type LiteLLMRichEndpointResult = + | { models: LiteLLMRichEndpointModel[]; incompleteVisionMetadata: boolean } + | { failure: LiteLLMRichEndpointFailure }; const LITELLM_RICH_ENDPOINTS = ["/model_group/info", "/v2/model/info", "/model/info", "/v1/model/info"] as const; export const OPENAI_COMPAT_DISCOVERY_DEFAULT_CONTEXT_WINDOW = 128_000; export const OPENAI_COMPAT_DISCOVERY_DEFAULT_MAX_TOKENS = 32_768; const UNKNOWN_PROXY_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } as const; +const warnedLiteLLMMetadataBases = new Set(); const LITELLM_UNUSABLE_SENTINEL_IDS: Record = { "all-team-models": true, "all-proxy-models": true, "no-default-models": true, }; +function warnLiteLLMMetadataFallback(managementBaseUrl: string, failure: LiteLLMRichEndpointFailure): void { + if (warnedLiteLLMMetadataBases.has(managementBaseUrl)) { + return; + } + warnedLiteLLMMetadataBases.add(managementBaseUrl); + logger.warn("LiteLLM rich model metadata unavailable; falling back to /v1/models", { + endpoint: `${managementBaseUrl}${failure.endpoint}`, + status: failure.status ?? "unavailable", + reason: failure.reason, + ...(failure.status === 403 + ? { requiredPermission: "Grant this LiteLLM key access to the model metadata endpoints" } + : {}), + ...(failure.error !== undefined ? { error: failure.error } : {}), + }); +} export function normalizeLiteLLMManagementBaseUrl(baseUrl: string): string { const trimmed = baseUrl.trim().replace(/\/+$/g, ""); @@ -3499,7 +3525,7 @@ async function fetchLiteLLMRichEndpoint( managementBaseUrl: string, runtimeBaseUrl: string, signal?: AbortSignal, -): Promise<{ models: LiteLLMRichEndpointModel[]; incompleteVisionMetadata: boolean } | null> { +): Promise | null> { const fetchImpl = discoveryFetch(options.fetch); const requestHeaders: Record = { Accept: "application/json", @@ -3515,17 +3541,17 @@ async function fetchLiteLLMRichEndpoint( headers: requestHeaders, signal, }); - } catch { - return null; + } catch (error) { + return { failure: { endpoint, reason: "network-error", error } }; } if (!response.ok) { - return null; + return response.status === 404 ? null : { failure: { endpoint, reason: "http-status", status: response.status } }; } let payload: unknown; try { payload = await response.json(); - } catch { - return null; + } catch (error) { + return { failure: { endpoint, reason: "invalid-json", status: response.status, error } }; } const entries = extractLiteLLMRichEntries(payload); if (!entries || entries.length === 0) { @@ -3576,11 +3602,18 @@ export async function fetchLiteLLMRichModels( } const fetchModels = async (signal?: AbortSignal): Promise[] | null> => { const deduped = new Map>(); + let metadataFailure: LiteLLMRichEndpointFailure | undefined; for (const endpoint of LITELLM_RICH_ENDPOINTS) { const result = await fetchLiteLLMRichEndpoint(endpoint, options, managementBaseUrl, runtimeBaseUrl, signal); if (!result) { continue; } + if ("failure" in result) { + if (!metadataFailure || (metadataFailure.status !== 403 && result.failure.status === 403)) { + metadataFailure = result.failure; + } + continue; + } const hadPriorModels = deduped.size > 0; for (const next of result.models) { const existing = deduped.get(next.model.id); @@ -3619,6 +3652,9 @@ export async function fetchLiteLLMRichModels( } } if (deduped.size === 0) { + if (metadataFailure) { + warnLiteLLMMetadataFallback(managementBaseUrl, metadataFailure); + } return null; } return Array.from(deduped.values()) diff --git a/packages/catalog/test/litellm-provider.test.ts b/packages/catalog/test/litellm-provider.test.ts index 48ded780f..08b791c25 100644 --- a/packages/catalog/test/litellm-provider.test.ts +++ b/packages/catalog/test/litellm-provider.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, test, vi } from "bun:test"; import { fetchLiteLLMRichModels, litellmModelManagerOptions } from "@oh-my-pi/pi-catalog/provider-models/openai-compat"; import type { FetchImpl } from "@oh-my-pi/pi-catalog/types"; +import * as logger from "@oh-my-pi/pi-utils/logger"; const ORIGINAL_LITELLM_BASE_URL = Bun.env.LITELLM_BASE_URL; const MODELS_DEV_URL = "https://models.dev/api.json"; @@ -238,6 +239,58 @@ describe("LiteLLM provider discovery", () => { }); }); + test("warns once when forbidden rich metadata forces /v1/models fallback", async () => { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = inputUrl(input); + if (url === MODELS_DEV_URL) { + return Response.json({}); + } + if (url === "http://forbidden:4000/v1/models") { + return Response.json({ data: [{ id: "hosted_vllm/private-model" }] }); + } + return new Response("Forbidden", { status: 403 }); + }) as FetchImpl; + const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const options = litellmModelManagerOptions({ + apiKey: "sk-restricted", + baseUrl: "http://forbidden:4000/v1", + fetch: fetchMock, + }); + + const models = await options.fetchDynamicModels?.(); + await options.fetchDynamicModels?.(); + + expect(models?.[0]).toMatchObject({ + id: "hosted_vllm/private-model", + contextWindow: null, + maxTokens: null, + }); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(warnSpy).toHaveBeenCalledWith( + "LiteLLM rich model metadata unavailable; falling back to /v1/models", + expect.objectContaining({ + endpoint: "http://forbidden:4000/model_group/info", + status: 403, + reason: "http-status", + }), + ); + }); + + test("treats missing rich metadata endpoints as absent without warning", async () => { + const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => {}); + + const models = await fetchLiteLLMRichModels({ + api: "openai-completions", + provider: "litellm", + apiKey: "sk-restricted", + baseUrl: "http://missing:4000/v1", + fetch: async () => new Response("Not Found", { status: 404 }), + }); + + expect(models).toBeNull(); + expect(warnSpy).not.toHaveBeenCalled(); + }); + test("enriches LiteLLM rich models missing from models.dev with bundled reasoning metadata", async () => { const fetchMock = vi.fn(async (input: string | URL | Request) => { const url = inputUrl(input);