feat(coding-agent/config): added model preference matching system for intelligent model selection

- Added model preference matching system with ModelMatchPreferences and ModelPreferenceContext interfaces to enable intelligent model selection based on usage history and provider preferences.
- Implemented buildPreferenceContext and pickPreferredModel functions to rank and select models based on usage order, provider history, and deprioritization settings.
- Enhanced parseModelPattern function to accept optional ModelMatchPreferences parameter, enabling model resolution to consider user preferences and usage history.
- Updated model matching logic to prioritize models based on historical usage patterns and provider preferences when multiple candidates match a pattern.
This commit is contained in:
can1357
2026-01-28 14:14:44 +01:00
parent b260dbb635
commit bcdcccf100
6 changed files with 204 additions and 29 deletions
@@ -11,9 +11,10 @@ export async function resolvePrimaryModel(
},
): Promise<{ model: Model<Api>; apiKey: string }> {
const available = modelRegistry.getAvailable();
const matchPreferences = { usageOrder: settingsManager.getStorage()?.getModelUsageOrder() };
const model = override
? resolveModelFromString(expandRoleAlias(override, settingsManager), available)
: resolveModelFromSettings(settingsManager, available);
? resolveModelFromString(expandRoleAlias(override, settingsManager), available, matchPreferences)
: resolveModelFromSettings(settingsManager, available, matchPreferences);
if (!model) {
throw new Error("No model available for commit generation");
}
@@ -34,15 +35,16 @@ export async function resolveSmolModel(
fallbackApiKey: string,
): Promise<{ model: Model<Api>; apiKey: string }> {
const available = modelRegistry.getAvailable();
const matchPreferences = { usageOrder: settingsManager.getStorage()?.getModelUsageOrder() };
const role = settingsManager.getModelRole("smol");
const roleModel = role ? resolveModelFromString(role, available) : undefined;
const roleModel = role ? resolveModelFromString(role, available, matchPreferences) : undefined;
if (roleModel) {
const apiKey = await modelRegistry.getApiKey(roleModel);
if (apiKey) return { model: roleModel, apiKey };
}
for (const pattern of SMOL_MODEL_PRIORITY) {
const candidate = parseModelPattern(pattern, available).model;
const candidate = parseModelPattern(pattern, available, matchPreferences).model;
if (!candidate) continue;
const apiKey = await modelRegistry.getApiKey(candidate);
if (apiKey) return { model: candidate, apiKey };
@@ -51,23 +53,35 @@ export async function resolveSmolModel(
return { model: fallbackModel, apiKey: fallbackApiKey };
}
function resolveModelFromSettings(settingsManager: SettingsManager, available: Model<Api>[]): Model<Api> | undefined {
function resolveModelFromSettings(
settingsManager: SettingsManager,
available: Model<Api>[],
matchPreferences: { usageOrder?: string[] },
): Model<Api> | undefined {
const roles = ["commit", "smol", "default"];
for (const role of roles) {
const configured = settingsManager.getModelRole(role);
if (!configured) continue;
const resolved = resolveModelFromString(expandRoleAlias(configured, settingsManager), available);
const resolved = resolveModelFromString(
expandRoleAlias(configured, settingsManager),
available,
matchPreferences,
);
if (resolved) return resolved;
}
return available[0];
}
function resolveModelFromString(value: string, available: Model<Api>[]): Model<Api> | undefined {
function resolveModelFromString(
value: string,
available: Model<Api>[],
matchPreferences: { usageOrder?: string[] },
): Model<Api> | undefined {
const parsed = parseModelString(value);
if (parsed) {
return available.find(model => model.provider === parsed.provider && model.id === parsed.id);
}
return parseModelPattern(value, available).model;
return parseModelPattern(value, available, matchPreferences).model;
}
function expandRoleAlias(value: string, settingsManager: SettingsManager): string {
@@ -60,6 +60,76 @@ export function formatModelString(model: Model<Api>): string {
return `${model.provider}/${model.id}`;
}
export interface ModelMatchPreferences {
/** Most-recently-used model keys (provider/modelId) to prefer when ambiguous. */
usageOrder?: string[];
/** Providers to deprioritize when no recent usage is available. */
deprioritizeProviders?: string[];
}
interface ModelPreferenceContext {
modelUsageRank: Map<string, number>;
providerUsageRank: Map<string, number>;
deprioritizedProviders: Set<string>;
modelOrder: Map<string, number>;
}
function buildPreferenceContext(
availableModels: Model<Api>[],
preferences: ModelMatchPreferences | undefined,
): ModelPreferenceContext {
const modelUsageRank = new Map<string, number>();
const providerUsageRank = new Map<string, number>();
const usageOrder = preferences?.usageOrder ?? [];
for (let i = 0; i < usageOrder.length; i += 1) {
const key = usageOrder[i];
if (!modelUsageRank.has(key)) {
modelUsageRank.set(key, i);
}
const parsed = parseModelString(key);
if (parsed && !providerUsageRank.has(parsed.provider)) {
providerUsageRank.set(parsed.provider, i);
}
}
const deprioritizedProviders = new Set(preferences?.deprioritizeProviders ?? ["openrouter"]);
const modelOrder = new Map<string, number>();
for (let i = 0; i < availableModels.length; i += 1) {
modelOrder.set(formatModelString(availableModels[i]), i);
}
return { modelUsageRank, providerUsageRank, deprioritizedProviders, modelOrder };
}
function pickPreferredModel(candidates: Model<Api>[], context: ModelPreferenceContext): Model<Api> {
if (candidates.length <= 1) return candidates[0];
return [...candidates].sort((a, b) => {
const aKey = formatModelString(a);
const bKey = formatModelString(b);
const aUsage = context.modelUsageRank.get(aKey);
const bUsage = context.modelUsageRank.get(bKey);
if (aUsage !== undefined || bUsage !== undefined) {
return (aUsage ?? Number.POSITIVE_INFINITY) - (bUsage ?? Number.POSITIVE_INFINITY);
}
const aProviderUsage = context.providerUsageRank.get(a.provider);
const bProviderUsage = context.providerUsageRank.get(b.provider);
if (aProviderUsage !== undefined || bProviderUsage !== undefined) {
return (aProviderUsage ?? Number.POSITIVE_INFINITY) - (bProviderUsage ?? Number.POSITIVE_INFINITY);
}
const aDeprioritized = context.deprioritizedProviders.has(a.provider);
const bDeprioritized = context.deprioritizedProviders.has(b.provider);
if (aDeprioritized !== bDeprioritized) {
return aDeprioritized ? 1 : -1;
}
const aOrder = context.modelOrder.get(aKey) ?? 0;
const bOrder = context.modelOrder.get(bKey) ?? 0;
return aOrder - bOrder;
})[0];
}
/**
* Helper to check if a model ID looks like an alias (no date suffix)
* Dates are typically in format: -20241022 or -20250929
@@ -77,7 +147,11 @@ function isAlias(id: string): boolean {
* Try to match a pattern to a model from the available models list.
* Returns the matched model or undefined if no match found.
*/
function tryMatchModel(modelPattern: string, availableModels: Model<Api>[]): Model<Api> | undefined {
function tryMatchModel(
modelPattern: string,
availableModels: Model<Api>[],
context: ModelPreferenceContext,
): Model<Api> | undefined {
// Check for provider/modelId format (provider is everything before the first /)
const slashIndex = modelPattern.indexOf("/");
if (slashIndex !== -1) {
@@ -93,9 +167,9 @@ function tryMatchModel(modelPattern: string, availableModels: Model<Api>[]): Mod
}
// Check for exact ID match (case-insensitive)
const exactMatch = availableModels.find(m => m.id.toLowerCase() === modelPattern.toLowerCase());
if (exactMatch) {
return exactMatch;
const exactMatches = availableModels.filter(m => m.id.toLowerCase() === modelPattern.toLowerCase());
if (exactMatches.length > 0) {
return pickPreferredModel(exactMatches, context);
}
// No exact match - fall back to partial matching
@@ -114,14 +188,19 @@ function tryMatchModel(modelPattern: string, availableModels: Model<Api>[]): Mod
const datedVersions = matches.filter(m => !isAlias(m.id));
if (aliases.length > 0) {
// Prefer alias - if multiple aliases, pick the one that sorts highest
aliases.sort((a, b) => b.id.localeCompare(a.id));
return aliases[0];
} else {
// No alias found, pick latest dated version
datedVersions.sort((a, b) => b.id.localeCompare(a.id));
return pickPreferredModel(aliases, context);
}
if (datedVersions.length === 0) return undefined;
if (datedVersions.length === 1) {
return datedVersions[0];
}
const sortedById = [...datedVersions].sort((a, b) => b.id.localeCompare(a.id));
const topId = sortedById[0]?.id;
if (!topId) return undefined;
const topCandidates = sortedById.filter(model => model.id === topId);
return pickPreferredModel(topCandidates, context);
}
export interface ParsedModelResult {
@@ -145,9 +224,13 @@ export interface ParsedModelResult {
*
* @internal Exported for testing
*/
export function parseModelPattern(pattern: string, availableModels: Model<Api>[]): ParsedModelResult {
function parseModelPatternWithContext(
pattern: string,
availableModels: Model<Api>[],
context: ModelPreferenceContext,
): ParsedModelResult {
// Try exact match first
const exactMatch = tryMatchModel(pattern, availableModels);
const exactMatch = tryMatchModel(pattern, availableModels, context);
if (exactMatch) {
return { model: exactMatch, thinkingLevel: undefined, warning: undefined, explicitThinkingLevel: false };
}
@@ -164,7 +247,7 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
if (isValidThinkingLevel(suffix)) {
// Valid thinking level - recurse on prefix and use this level
const result = parseModelPattern(prefix, availableModels);
const result = parseModelPatternWithContext(prefix, availableModels, context);
if (result.model) {
// Only use this thinking level if no warning from inner recursion
const explicitThinkingLevel = !result.warning;
@@ -179,7 +262,7 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
}
// Invalid suffix - recurse on prefix and warn
const result = parseModelPattern(prefix, availableModels);
const result = parseModelPatternWithContext(prefix, availableModels, context);
if (result.model) {
return {
model: result.model,
@@ -191,6 +274,15 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
return result;
}
export function parseModelPattern(
pattern: string,
availableModels: Model<Api>[],
preferences?: ModelMatchPreferences,
): ParsedModelResult {
const context = buildPreferenceContext(availableModels, preferences);
return parseModelPatternWithContext(pattern, availableModels, context);
}
/**
* Resolve model patterns to actual Model objects with optional thinking levels
* Format: "pattern:level" where :level is optional
@@ -202,8 +294,13 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
* The algorithm tries to match the full pattern first, then progressively
* strips colon-suffixes to find a match.
*/
export async function resolveModelScope(patterns: string[], modelRegistry: ModelRegistry): Promise<ScopedModel[]> {
export async function resolveModelScope(
patterns: string[],
modelRegistry: ModelRegistry,
preferences?: ModelMatchPreferences,
): Promise<ScopedModel[]> {
const availableModels = modelRegistry.getAvailable();
const context = buildPreferenceContext(availableModels, preferences);
const scopedModels: ScopedModel[] = [];
for (const pattern of patterns) {
@@ -245,7 +342,11 @@ export async function resolveModelScope(patterns: string[], modelRegistry: Model
continue;
}
const { model, thinkingLevel, warning, explicitThinkingLevel } = parseModelPattern(pattern, availableModels);
const { model, thinkingLevel, warning, explicitThinkingLevel } = parseModelPatternWithContext(
pattern,
availableModels,
context,
);
if (warning) {
console.warn(chalk.yellow(`Warning: ${warning}`));
+8 -2
View File
@@ -373,7 +373,10 @@ async function buildSessionOptions(
// Model from CLI (--model) - uses same fuzzy matching as --models
if (parsed.model) {
const available = modelRegistry.getAvailable();
const { model, warning } = parseModelPattern(parsed.model, available);
const modelMatchPreferences = {
usageOrder: settingsManager.getStorage()?.getModelUsageOrder(),
};
const { model, warning } = parseModelPattern(parsed.model, available, modelMatchPreferences);
if (warning) {
writeStderr(chalk.yellow(`Warning: ${warning}`));
}
@@ -652,8 +655,11 @@ export async function main(args: string[]) {
let scopedModels: ScopedModel[] = [];
const modelPatterns = parsed.models ?? settingsManager.getEnabledModels();
const modelMatchPreferences = {
usageOrder: settingsManager.getStorage()?.getModelUsageOrder(),
};
if (modelPatterns && modelPatterns.length > 0) {
scopedModels = await resolveModelScope(modelPatterns, modelRegistry);
scopedModels = await resolveModelScope(modelPatterns, modelRegistry, modelMatchPreferences);
time("resolveModelScope");
}
@@ -72,4 +72,4 @@ These are fingerprints of generic AI-generated interfaces. Avoid them:
Every interface should make someone ask "how was this made?" not "which AI made this?"
Commit to a clear aesthetic direction and execute with precision.
Keep going until the implementation is complete. This matters.
</critical>
</critical>
+6 -1
View File
@@ -134,6 +134,7 @@ function resolveModelOverride(
settingsManager?: SettingsManager,
): { model?: Model<Api>; thinkingLevel?: ThinkingLevel } {
if (modelPatterns.length === 0) return {};
const matchPreferences = { usageOrder: settingsManager?.getStorage()?.getModelUsageOrder() };
const roles = settingsManager?.serialize().modelRoles as Record<string, string> | undefined;
for (const pattern of modelPatterns) {
const normalized = pattern.trim().toLowerCase();
@@ -148,7 +149,11 @@ function resolveModelOverride(
effectivePattern = configured;
}
}
const { model, thinkingLevel } = parseModelPattern(effectivePattern, modelRegistry.getAvailable());
const { model, thinkingLevel } = parseModelPattern(
effectivePattern,
modelRegistry.getAvailable(),
matchPreferences,
);
if (model) {
return { model, thinkingLevel: thinkingLevel !== "off" ? thinkingLevel : undefined };
}
@@ -58,7 +58,34 @@ const mockOpenRouterModels: Model<"anthropic-messages">[] = [
},
];
const allModels = [...mockModels, ...mockOpenRouterModels];
const mockProviderOverlapModels: Model<"anthropic-messages">[] = [
{
id: "kimi-k2.5",
name: "Kimi K2.5",
api: "anthropic-messages",
provider: "kimi-code",
baseUrl: "https://api.kimi.ai",
reasoning: false,
input: ["text"],
cost: { input: 2, output: 6, cacheRead: 0.2, cacheWrite: 2 },
contextWindow: 128000,
maxTokens: 8192,
},
{
id: "moonshotai/kimi-k2.5",
name: "Kimi K2.5 (OpenRouter)",
api: "anthropic-messages",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
reasoning: false,
input: ["text"],
cost: { input: 2.2, output: 6.2, cacheRead: 0.22, cacheWrite: 2.2 },
contextWindow: 128000,
maxTokens: 8192,
},
];
const allModels = [...mockModels, ...mockOpenRouterModels, ...mockProviderOverlapModels];
describe("parseModelPattern", () => {
describe("simple patterns without colons", () => {
@@ -213,4 +240,26 @@ describe("parseModelPattern", () => {
expect(result.warning).toContain("Invalid thinking level");
});
});
describe("preference logic", () => {
test("prefers most recently used model when multiple providers match", () => {
const result = parseModelPattern("k2.5", allModels, {
usageOrder: ["kimi-code/kimi-k2.5"],
});
expect(result.model?.provider).toBe("kimi-code");
});
test("falls back to deprioritizing openrouter when no usage data", () => {
const result = parseModelPattern("k2.5", allModels, { usageOrder: [] });
expect(result.model?.provider).toBe("kimi-code");
});
test("respects most recently used provider even if openrouter", () => {
const result = parseModelPattern("k2.5", allModels, {
usageOrder: ["openrouter/moonshotai/kimi-k2.5"],
});
expect(result.model?.provider).toBe("openrouter");
expect(result.model?.id).toBe("moonshotai/kimi-k2.5");
});
});
});