From bcdcccf100ab6596e748037cd0d9fd3ca27ddfc5 Mon Sep 17 00:00:00 2001 From: can1357 Date: Wed, 28 Jan 2026 14:14:44 +0100 Subject: [PATCH] feat(coding-agent/config): added model preference matching system for intelligent model selection - Added model preference matching system with ModelMatchPreferences and ModelPreferenceContext interfaces to enable intelligent model selection based on usage history and provider preferences. - Implemented buildPreferenceContext and pickPreferredModel functions to rank and select models based on usage order, provider history, and deprioritization settings. - Enhanced parseModelPattern function to accept optional ModelMatchPreferences parameter, enabling model resolution to consider user preferences and usage history. - Updated model matching logic to prioritize models based on historical usage patterns and provider preferences when multiple candidates match a pattern. --- .../src/commit/model-selection.ts | 30 ++-- .../coding-agent/src/config/model-resolver.ts | 133 +++++++++++++++--- packages/coding-agent/src/main.ts | 10 +- .../src/prompts/agents/designer.md | 2 +- packages/coding-agent/src/task/executor.ts | 7 +- .../coding-agent/test/model-resolver.test.ts | 51 ++++++- 6 files changed, 204 insertions(+), 29 deletions(-) diff --git a/packages/coding-agent/src/commit/model-selection.ts b/packages/coding-agent/src/commit/model-selection.ts index a127282d9..ae1e0f7e0 100644 --- a/packages/coding-agent/src/commit/model-selection.ts +++ b/packages/coding-agent/src/commit/model-selection.ts @@ -11,9 +11,10 @@ export async function resolvePrimaryModel( }, ): Promise<{ model: Model; apiKey: string }> { const available = modelRegistry.getAvailable(); + const matchPreferences = { usageOrder: settingsManager.getStorage()?.getModelUsageOrder() }; const model = override - ? resolveModelFromString(expandRoleAlias(override, settingsManager), available) - : resolveModelFromSettings(settingsManager, available); + ? resolveModelFromString(expandRoleAlias(override, settingsManager), available, matchPreferences) + : resolveModelFromSettings(settingsManager, available, matchPreferences); if (!model) { throw new Error("No model available for commit generation"); } @@ -34,15 +35,16 @@ export async function resolveSmolModel( fallbackApiKey: string, ): Promise<{ model: Model; apiKey: string }> { const available = modelRegistry.getAvailable(); + const matchPreferences = { usageOrder: settingsManager.getStorage()?.getModelUsageOrder() }; const role = settingsManager.getModelRole("smol"); - const roleModel = role ? resolveModelFromString(role, available) : undefined; + const roleModel = role ? resolveModelFromString(role, available, matchPreferences) : undefined; if (roleModel) { const apiKey = await modelRegistry.getApiKey(roleModel); if (apiKey) return { model: roleModel, apiKey }; } for (const pattern of SMOL_MODEL_PRIORITY) { - const candidate = parseModelPattern(pattern, available).model; + const candidate = parseModelPattern(pattern, available, matchPreferences).model; if (!candidate) continue; const apiKey = await modelRegistry.getApiKey(candidate); if (apiKey) return { model: candidate, apiKey }; @@ -51,23 +53,35 @@ export async function resolveSmolModel( return { model: fallbackModel, apiKey: fallbackApiKey }; } -function resolveModelFromSettings(settingsManager: SettingsManager, available: Model[]): Model | undefined { +function resolveModelFromSettings( + settingsManager: SettingsManager, + available: Model[], + matchPreferences: { usageOrder?: string[] }, +): Model | undefined { const roles = ["commit", "smol", "default"]; for (const role of roles) { const configured = settingsManager.getModelRole(role); if (!configured) continue; - const resolved = resolveModelFromString(expandRoleAlias(configured, settingsManager), available); + const resolved = resolveModelFromString( + expandRoleAlias(configured, settingsManager), + available, + matchPreferences, + ); if (resolved) return resolved; } return available[0]; } -function resolveModelFromString(value: string, available: Model[]): Model | undefined { +function resolveModelFromString( + value: string, + available: Model[], + matchPreferences: { usageOrder?: string[] }, +): Model | undefined { const parsed = parseModelString(value); if (parsed) { return available.find(model => model.provider === parsed.provider && model.id === parsed.id); } - return parseModelPattern(value, available).model; + return parseModelPattern(value, available, matchPreferences).model; } function expandRoleAlias(value: string, settingsManager: SettingsManager): string { diff --git a/packages/coding-agent/src/config/model-resolver.ts b/packages/coding-agent/src/config/model-resolver.ts index 54540a932..4e2a04fad 100644 --- a/packages/coding-agent/src/config/model-resolver.ts +++ b/packages/coding-agent/src/config/model-resolver.ts @@ -60,6 +60,76 @@ export function formatModelString(model: Model): string { return `${model.provider}/${model.id}`; } +export interface ModelMatchPreferences { + /** Most-recently-used model keys (provider/modelId) to prefer when ambiguous. */ + usageOrder?: string[]; + /** Providers to deprioritize when no recent usage is available. */ + deprioritizeProviders?: string[]; +} + +interface ModelPreferenceContext { + modelUsageRank: Map; + providerUsageRank: Map; + deprioritizedProviders: Set; + modelOrder: Map; +} + +function buildPreferenceContext( + availableModels: Model[], + preferences: ModelMatchPreferences | undefined, +): ModelPreferenceContext { + const modelUsageRank = new Map(); + const providerUsageRank = new Map(); + const usageOrder = preferences?.usageOrder ?? []; + for (let i = 0; i < usageOrder.length; i += 1) { + const key = usageOrder[i]; + if (!modelUsageRank.has(key)) { + modelUsageRank.set(key, i); + } + const parsed = parseModelString(key); + if (parsed && !providerUsageRank.has(parsed.provider)) { + providerUsageRank.set(parsed.provider, i); + } + } + + const deprioritizedProviders = new Set(preferences?.deprioritizeProviders ?? ["openrouter"]); + const modelOrder = new Map(); + for (let i = 0; i < availableModels.length; i += 1) { + modelOrder.set(formatModelString(availableModels[i]), i); + } + + return { modelUsageRank, providerUsageRank, deprioritizedProviders, modelOrder }; +} + +function pickPreferredModel(candidates: Model[], context: ModelPreferenceContext): Model { + if (candidates.length <= 1) return candidates[0]; + return [...candidates].sort((a, b) => { + const aKey = formatModelString(a); + const bKey = formatModelString(b); + const aUsage = context.modelUsageRank.get(aKey); + const bUsage = context.modelUsageRank.get(bKey); + if (aUsage !== undefined || bUsage !== undefined) { + return (aUsage ?? Number.POSITIVE_INFINITY) - (bUsage ?? Number.POSITIVE_INFINITY); + } + + const aProviderUsage = context.providerUsageRank.get(a.provider); + const bProviderUsage = context.providerUsageRank.get(b.provider); + if (aProviderUsage !== undefined || bProviderUsage !== undefined) { + return (aProviderUsage ?? Number.POSITIVE_INFINITY) - (bProviderUsage ?? Number.POSITIVE_INFINITY); + } + + const aDeprioritized = context.deprioritizedProviders.has(a.provider); + const bDeprioritized = context.deprioritizedProviders.has(b.provider); + if (aDeprioritized !== bDeprioritized) { + return aDeprioritized ? 1 : -1; + } + + const aOrder = context.modelOrder.get(aKey) ?? 0; + const bOrder = context.modelOrder.get(bKey) ?? 0; + return aOrder - bOrder; + })[0]; +} + /** * Helper to check if a model ID looks like an alias (no date suffix) * Dates are typically in format: -20241022 or -20250929 @@ -77,7 +147,11 @@ function isAlias(id: string): boolean { * Try to match a pattern to a model from the available models list. * Returns the matched model or undefined if no match found. */ -function tryMatchModel(modelPattern: string, availableModels: Model[]): Model | undefined { +function tryMatchModel( + modelPattern: string, + availableModels: Model[], + context: ModelPreferenceContext, +): Model | undefined { // Check for provider/modelId format (provider is everything before the first /) const slashIndex = modelPattern.indexOf("/"); if (slashIndex !== -1) { @@ -93,9 +167,9 @@ function tryMatchModel(modelPattern: string, availableModels: Model[]): Mod } // Check for exact ID match (case-insensitive) - const exactMatch = availableModels.find(m => m.id.toLowerCase() === modelPattern.toLowerCase()); - if (exactMatch) { - return exactMatch; + const exactMatches = availableModels.filter(m => m.id.toLowerCase() === modelPattern.toLowerCase()); + if (exactMatches.length > 0) { + return pickPreferredModel(exactMatches, context); } // No exact match - fall back to partial matching @@ -114,14 +188,19 @@ function tryMatchModel(modelPattern: string, availableModels: Model[]): Mod const datedVersions = matches.filter(m => !isAlias(m.id)); if (aliases.length > 0) { - // Prefer alias - if multiple aliases, pick the one that sorts highest - aliases.sort((a, b) => b.id.localeCompare(a.id)); - return aliases[0]; - } else { - // No alias found, pick latest dated version - datedVersions.sort((a, b) => b.id.localeCompare(a.id)); + return pickPreferredModel(aliases, context); + } + if (datedVersions.length === 0) return undefined; + + if (datedVersions.length === 1) { return datedVersions[0]; } + + const sortedById = [...datedVersions].sort((a, b) => b.id.localeCompare(a.id)); + const topId = sortedById[0]?.id; + if (!topId) return undefined; + const topCandidates = sortedById.filter(model => model.id === topId); + return pickPreferredModel(topCandidates, context); } export interface ParsedModelResult { @@ -145,9 +224,13 @@ export interface ParsedModelResult { * * @internal Exported for testing */ -export function parseModelPattern(pattern: string, availableModels: Model[]): ParsedModelResult { +function parseModelPatternWithContext( + pattern: string, + availableModels: Model[], + context: ModelPreferenceContext, +): ParsedModelResult { // Try exact match first - const exactMatch = tryMatchModel(pattern, availableModels); + const exactMatch = tryMatchModel(pattern, availableModels, context); if (exactMatch) { return { model: exactMatch, thinkingLevel: undefined, warning: undefined, explicitThinkingLevel: false }; } @@ -164,7 +247,7 @@ export function parseModelPattern(pattern: string, availableModels: Model[] if (isValidThinkingLevel(suffix)) { // Valid thinking level - recurse on prefix and use this level - const result = parseModelPattern(prefix, availableModels); + const result = parseModelPatternWithContext(prefix, availableModels, context); if (result.model) { // Only use this thinking level if no warning from inner recursion const explicitThinkingLevel = !result.warning; @@ -179,7 +262,7 @@ export function parseModelPattern(pattern: string, availableModels: Model[] } // Invalid suffix - recurse on prefix and warn - const result = parseModelPattern(prefix, availableModels); + const result = parseModelPatternWithContext(prefix, availableModels, context); if (result.model) { return { model: result.model, @@ -191,6 +274,15 @@ export function parseModelPattern(pattern: string, availableModels: Model[] return result; } +export function parseModelPattern( + pattern: string, + availableModels: Model[], + preferences?: ModelMatchPreferences, +): ParsedModelResult { + const context = buildPreferenceContext(availableModels, preferences); + return parseModelPatternWithContext(pattern, availableModels, context); +} + /** * Resolve model patterns to actual Model objects with optional thinking levels * Format: "pattern:level" where :level is optional @@ -202,8 +294,13 @@ export function parseModelPattern(pattern: string, availableModels: Model[] * The algorithm tries to match the full pattern first, then progressively * strips colon-suffixes to find a match. */ -export async function resolveModelScope(patterns: string[], modelRegistry: ModelRegistry): Promise { +export async function resolveModelScope( + patterns: string[], + modelRegistry: ModelRegistry, + preferences?: ModelMatchPreferences, +): Promise { const availableModels = modelRegistry.getAvailable(); + const context = buildPreferenceContext(availableModels, preferences); const scopedModels: ScopedModel[] = []; for (const pattern of patterns) { @@ -245,7 +342,11 @@ export async function resolveModelScope(patterns: string[], modelRegistry: Model continue; } - const { model, thinkingLevel, warning, explicitThinkingLevel } = parseModelPattern(pattern, availableModels); + const { model, thinkingLevel, warning, explicitThinkingLevel } = parseModelPatternWithContext( + pattern, + availableModels, + context, + ); if (warning) { console.warn(chalk.yellow(`Warning: ${warning}`)); diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index 830cc44d4..adce94ebb 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -373,7 +373,10 @@ async function buildSessionOptions( // Model from CLI (--model) - uses same fuzzy matching as --models if (parsed.model) { const available = modelRegistry.getAvailable(); - const { model, warning } = parseModelPattern(parsed.model, available); + const modelMatchPreferences = { + usageOrder: settingsManager.getStorage()?.getModelUsageOrder(), + }; + const { model, warning } = parseModelPattern(parsed.model, available, modelMatchPreferences); if (warning) { writeStderr(chalk.yellow(`Warning: ${warning}`)); } @@ -652,8 +655,11 @@ export async function main(args: string[]) { let scopedModels: ScopedModel[] = []; const modelPatterns = parsed.models ?? settingsManager.getEnabledModels(); + const modelMatchPreferences = { + usageOrder: settingsManager.getStorage()?.getModelUsageOrder(), + }; if (modelPatterns && modelPatterns.length > 0) { - scopedModels = await resolveModelScope(modelPatterns, modelRegistry); + scopedModels = await resolveModelScope(modelPatterns, modelRegistry, modelMatchPreferences); time("resolveModelScope"); } diff --git a/packages/coding-agent/src/prompts/agents/designer.md b/packages/coding-agent/src/prompts/agents/designer.md index 5aa0bcf51..6b13acb64 100644 --- a/packages/coding-agent/src/prompts/agents/designer.md +++ b/packages/coding-agent/src/prompts/agents/designer.md @@ -72,4 +72,4 @@ These are fingerprints of generic AI-generated interfaces. Avoid them: Every interface should make someone ask "how was this made?" not "which AI made this?" Commit to a clear aesthetic direction and execute with precision. Keep going until the implementation is complete. This matters. - + \ No newline at end of file diff --git a/packages/coding-agent/src/task/executor.ts b/packages/coding-agent/src/task/executor.ts index 02cc31f62..57d8891f7 100644 --- a/packages/coding-agent/src/task/executor.ts +++ b/packages/coding-agent/src/task/executor.ts @@ -134,6 +134,7 @@ function resolveModelOverride( settingsManager?: SettingsManager, ): { model?: Model; thinkingLevel?: ThinkingLevel } { if (modelPatterns.length === 0) return {}; + const matchPreferences = { usageOrder: settingsManager?.getStorage()?.getModelUsageOrder() }; const roles = settingsManager?.serialize().modelRoles as Record | undefined; for (const pattern of modelPatterns) { const normalized = pattern.trim().toLowerCase(); @@ -148,7 +149,11 @@ function resolveModelOverride( effectivePattern = configured; } } - const { model, thinkingLevel } = parseModelPattern(effectivePattern, modelRegistry.getAvailable()); + const { model, thinkingLevel } = parseModelPattern( + effectivePattern, + modelRegistry.getAvailable(), + matchPreferences, + ); if (model) { return { model, thinkingLevel: thinkingLevel !== "off" ? thinkingLevel : undefined }; } diff --git a/packages/coding-agent/test/model-resolver.test.ts b/packages/coding-agent/test/model-resolver.test.ts index d3297f6fd..c406833b5 100644 --- a/packages/coding-agent/test/model-resolver.test.ts +++ b/packages/coding-agent/test/model-resolver.test.ts @@ -58,7 +58,34 @@ const mockOpenRouterModels: Model<"anthropic-messages">[] = [ }, ]; -const allModels = [...mockModels, ...mockOpenRouterModels]; +const mockProviderOverlapModels: Model<"anthropic-messages">[] = [ + { + id: "kimi-k2.5", + name: "Kimi K2.5", + api: "anthropic-messages", + provider: "kimi-code", + baseUrl: "https://api.kimi.ai", + reasoning: false, + input: ["text"], + cost: { input: 2, output: 6, cacheRead: 0.2, cacheWrite: 2 }, + contextWindow: 128000, + maxTokens: 8192, + }, + { + id: "moonshotai/kimi-k2.5", + name: "Kimi K2.5 (OpenRouter)", + api: "anthropic-messages", + provider: "openrouter", + baseUrl: "https://openrouter.ai/api/v1", + reasoning: false, + input: ["text"], + cost: { input: 2.2, output: 6.2, cacheRead: 0.22, cacheWrite: 2.2 }, + contextWindow: 128000, + maxTokens: 8192, + }, +]; + +const allModels = [...mockModels, ...mockOpenRouterModels, ...mockProviderOverlapModels]; describe("parseModelPattern", () => { describe("simple patterns without colons", () => { @@ -213,4 +240,26 @@ describe("parseModelPattern", () => { expect(result.warning).toContain("Invalid thinking level"); }); }); + + describe("preference logic", () => { + test("prefers most recently used model when multiple providers match", () => { + const result = parseModelPattern("k2.5", allModels, { + usageOrder: ["kimi-code/kimi-k2.5"], + }); + expect(result.model?.provider).toBe("kimi-code"); + }); + + test("falls back to deprioritizing openrouter when no usage data", () => { + const result = parseModelPattern("k2.5", allModels, { usageOrder: [] }); + expect(result.model?.provider).toBe("kimi-code"); + }); + + test("respects most recently used provider even if openrouter", () => { + const result = parseModelPattern("k2.5", allModels, { + usageOrder: ["openrouter/moonshotai/kimi-k2.5"], + }); + expect(result.model?.provider).toBe("openrouter"); + expect(result.model?.id).toBe("moonshotai/kimi-k2.5"); + }); + }); });