feat(coding-agent/config): added model preference matching system for intelligent model selection
- Added model preference matching system with ModelMatchPreferences and ModelPreferenceContext interfaces to enable intelligent model selection based on usage history and provider preferences. - Implemented buildPreferenceContext and pickPreferredModel functions to rank and select models based on usage order, provider history, and deprioritization settings. - Enhanced parseModelPattern function to accept optional ModelMatchPreferences parameter, enabling model resolution to consider user preferences and usage history. - Updated model matching logic to prioritize models based on historical usage patterns and provider preferences when multiple candidates match a pattern.
This commit is contained in:
@@ -11,9 +11,10 @@ export async function resolvePrimaryModel(
|
||||
},
|
||||
): Promise<{ model: Model<Api>; apiKey: string }> {
|
||||
const available = modelRegistry.getAvailable();
|
||||
const matchPreferences = { usageOrder: settingsManager.getStorage()?.getModelUsageOrder() };
|
||||
const model = override
|
||||
? resolveModelFromString(expandRoleAlias(override, settingsManager), available)
|
||||
: resolveModelFromSettings(settingsManager, available);
|
||||
? resolveModelFromString(expandRoleAlias(override, settingsManager), available, matchPreferences)
|
||||
: resolveModelFromSettings(settingsManager, available, matchPreferences);
|
||||
if (!model) {
|
||||
throw new Error("No model available for commit generation");
|
||||
}
|
||||
@@ -34,15 +35,16 @@ export async function resolveSmolModel(
|
||||
fallbackApiKey: string,
|
||||
): Promise<{ model: Model<Api>; apiKey: string }> {
|
||||
const available = modelRegistry.getAvailable();
|
||||
const matchPreferences = { usageOrder: settingsManager.getStorage()?.getModelUsageOrder() };
|
||||
const role = settingsManager.getModelRole("smol");
|
||||
const roleModel = role ? resolveModelFromString(role, available) : undefined;
|
||||
const roleModel = role ? resolveModelFromString(role, available, matchPreferences) : undefined;
|
||||
if (roleModel) {
|
||||
const apiKey = await modelRegistry.getApiKey(roleModel);
|
||||
if (apiKey) return { model: roleModel, apiKey };
|
||||
}
|
||||
|
||||
for (const pattern of SMOL_MODEL_PRIORITY) {
|
||||
const candidate = parseModelPattern(pattern, available).model;
|
||||
const candidate = parseModelPattern(pattern, available, matchPreferences).model;
|
||||
if (!candidate) continue;
|
||||
const apiKey = await modelRegistry.getApiKey(candidate);
|
||||
if (apiKey) return { model: candidate, apiKey };
|
||||
@@ -51,23 +53,35 @@ export async function resolveSmolModel(
|
||||
return { model: fallbackModel, apiKey: fallbackApiKey };
|
||||
}
|
||||
|
||||
function resolveModelFromSettings(settingsManager: SettingsManager, available: Model<Api>[]): Model<Api> | undefined {
|
||||
function resolveModelFromSettings(
|
||||
settingsManager: SettingsManager,
|
||||
available: Model<Api>[],
|
||||
matchPreferences: { usageOrder?: string[] },
|
||||
): Model<Api> | undefined {
|
||||
const roles = ["commit", "smol", "default"];
|
||||
for (const role of roles) {
|
||||
const configured = settingsManager.getModelRole(role);
|
||||
if (!configured) continue;
|
||||
const resolved = resolveModelFromString(expandRoleAlias(configured, settingsManager), available);
|
||||
const resolved = resolveModelFromString(
|
||||
expandRoleAlias(configured, settingsManager),
|
||||
available,
|
||||
matchPreferences,
|
||||
);
|
||||
if (resolved) return resolved;
|
||||
}
|
||||
return available[0];
|
||||
}
|
||||
|
||||
function resolveModelFromString(value: string, available: Model<Api>[]): Model<Api> | undefined {
|
||||
function resolveModelFromString(
|
||||
value: string,
|
||||
available: Model<Api>[],
|
||||
matchPreferences: { usageOrder?: string[] },
|
||||
): Model<Api> | undefined {
|
||||
const parsed = parseModelString(value);
|
||||
if (parsed) {
|
||||
return available.find(model => model.provider === parsed.provider && model.id === parsed.id);
|
||||
}
|
||||
return parseModelPattern(value, available).model;
|
||||
return parseModelPattern(value, available, matchPreferences).model;
|
||||
}
|
||||
|
||||
function expandRoleAlias(value: string, settingsManager: SettingsManager): string {
|
||||
|
||||
@@ -60,6 +60,76 @@ export function formatModelString(model: Model<Api>): string {
|
||||
return `${model.provider}/${model.id}`;
|
||||
}
|
||||
|
||||
export interface ModelMatchPreferences {
|
||||
/** Most-recently-used model keys (provider/modelId) to prefer when ambiguous. */
|
||||
usageOrder?: string[];
|
||||
/** Providers to deprioritize when no recent usage is available. */
|
||||
deprioritizeProviders?: string[];
|
||||
}
|
||||
|
||||
interface ModelPreferenceContext {
|
||||
modelUsageRank: Map<string, number>;
|
||||
providerUsageRank: Map<string, number>;
|
||||
deprioritizedProviders: Set<string>;
|
||||
modelOrder: Map<string, number>;
|
||||
}
|
||||
|
||||
function buildPreferenceContext(
|
||||
availableModels: Model<Api>[],
|
||||
preferences: ModelMatchPreferences | undefined,
|
||||
): ModelPreferenceContext {
|
||||
const modelUsageRank = new Map<string, number>();
|
||||
const providerUsageRank = new Map<string, number>();
|
||||
const usageOrder = preferences?.usageOrder ?? [];
|
||||
for (let i = 0; i < usageOrder.length; i += 1) {
|
||||
const key = usageOrder[i];
|
||||
if (!modelUsageRank.has(key)) {
|
||||
modelUsageRank.set(key, i);
|
||||
}
|
||||
const parsed = parseModelString(key);
|
||||
if (parsed && !providerUsageRank.has(parsed.provider)) {
|
||||
providerUsageRank.set(parsed.provider, i);
|
||||
}
|
||||
}
|
||||
|
||||
const deprioritizedProviders = new Set(preferences?.deprioritizeProviders ?? ["openrouter"]);
|
||||
const modelOrder = new Map<string, number>();
|
||||
for (let i = 0; i < availableModels.length; i += 1) {
|
||||
modelOrder.set(formatModelString(availableModels[i]), i);
|
||||
}
|
||||
|
||||
return { modelUsageRank, providerUsageRank, deprioritizedProviders, modelOrder };
|
||||
}
|
||||
|
||||
function pickPreferredModel(candidates: Model<Api>[], context: ModelPreferenceContext): Model<Api> {
|
||||
if (candidates.length <= 1) return candidates[0];
|
||||
return [...candidates].sort((a, b) => {
|
||||
const aKey = formatModelString(a);
|
||||
const bKey = formatModelString(b);
|
||||
const aUsage = context.modelUsageRank.get(aKey);
|
||||
const bUsage = context.modelUsageRank.get(bKey);
|
||||
if (aUsage !== undefined || bUsage !== undefined) {
|
||||
return (aUsage ?? Number.POSITIVE_INFINITY) - (bUsage ?? Number.POSITIVE_INFINITY);
|
||||
}
|
||||
|
||||
const aProviderUsage = context.providerUsageRank.get(a.provider);
|
||||
const bProviderUsage = context.providerUsageRank.get(b.provider);
|
||||
if (aProviderUsage !== undefined || bProviderUsage !== undefined) {
|
||||
return (aProviderUsage ?? Number.POSITIVE_INFINITY) - (bProviderUsage ?? Number.POSITIVE_INFINITY);
|
||||
}
|
||||
|
||||
const aDeprioritized = context.deprioritizedProviders.has(a.provider);
|
||||
const bDeprioritized = context.deprioritizedProviders.has(b.provider);
|
||||
if (aDeprioritized !== bDeprioritized) {
|
||||
return aDeprioritized ? 1 : -1;
|
||||
}
|
||||
|
||||
const aOrder = context.modelOrder.get(aKey) ?? 0;
|
||||
const bOrder = context.modelOrder.get(bKey) ?? 0;
|
||||
return aOrder - bOrder;
|
||||
})[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to check if a model ID looks like an alias (no date suffix)
|
||||
* Dates are typically in format: -20241022 or -20250929
|
||||
@@ -77,7 +147,11 @@ function isAlias(id: string): boolean {
|
||||
* Try to match a pattern to a model from the available models list.
|
||||
* Returns the matched model or undefined if no match found.
|
||||
*/
|
||||
function tryMatchModel(modelPattern: string, availableModels: Model<Api>[]): Model<Api> | undefined {
|
||||
function tryMatchModel(
|
||||
modelPattern: string,
|
||||
availableModels: Model<Api>[],
|
||||
context: ModelPreferenceContext,
|
||||
): Model<Api> | undefined {
|
||||
// Check for provider/modelId format (provider is everything before the first /)
|
||||
const slashIndex = modelPattern.indexOf("/");
|
||||
if (slashIndex !== -1) {
|
||||
@@ -93,9 +167,9 @@ function tryMatchModel(modelPattern: string, availableModels: Model<Api>[]): Mod
|
||||
}
|
||||
|
||||
// Check for exact ID match (case-insensitive)
|
||||
const exactMatch = availableModels.find(m => m.id.toLowerCase() === modelPattern.toLowerCase());
|
||||
if (exactMatch) {
|
||||
return exactMatch;
|
||||
const exactMatches = availableModels.filter(m => m.id.toLowerCase() === modelPattern.toLowerCase());
|
||||
if (exactMatches.length > 0) {
|
||||
return pickPreferredModel(exactMatches, context);
|
||||
}
|
||||
|
||||
// No exact match - fall back to partial matching
|
||||
@@ -114,14 +188,19 @@ function tryMatchModel(modelPattern: string, availableModels: Model<Api>[]): Mod
|
||||
const datedVersions = matches.filter(m => !isAlias(m.id));
|
||||
|
||||
if (aliases.length > 0) {
|
||||
// Prefer alias - if multiple aliases, pick the one that sorts highest
|
||||
aliases.sort((a, b) => b.id.localeCompare(a.id));
|
||||
return aliases[0];
|
||||
} else {
|
||||
// No alias found, pick latest dated version
|
||||
datedVersions.sort((a, b) => b.id.localeCompare(a.id));
|
||||
return pickPreferredModel(aliases, context);
|
||||
}
|
||||
if (datedVersions.length === 0) return undefined;
|
||||
|
||||
if (datedVersions.length === 1) {
|
||||
return datedVersions[0];
|
||||
}
|
||||
|
||||
const sortedById = [...datedVersions].sort((a, b) => b.id.localeCompare(a.id));
|
||||
const topId = sortedById[0]?.id;
|
||||
if (!topId) return undefined;
|
||||
const topCandidates = sortedById.filter(model => model.id === topId);
|
||||
return pickPreferredModel(topCandidates, context);
|
||||
}
|
||||
|
||||
export interface ParsedModelResult {
|
||||
@@ -145,9 +224,13 @@ export interface ParsedModelResult {
|
||||
*
|
||||
* @internal Exported for testing
|
||||
*/
|
||||
export function parseModelPattern(pattern: string, availableModels: Model<Api>[]): ParsedModelResult {
|
||||
function parseModelPatternWithContext(
|
||||
pattern: string,
|
||||
availableModels: Model<Api>[],
|
||||
context: ModelPreferenceContext,
|
||||
): ParsedModelResult {
|
||||
// Try exact match first
|
||||
const exactMatch = tryMatchModel(pattern, availableModels);
|
||||
const exactMatch = tryMatchModel(pattern, availableModels, context);
|
||||
if (exactMatch) {
|
||||
return { model: exactMatch, thinkingLevel: undefined, warning: undefined, explicitThinkingLevel: false };
|
||||
}
|
||||
@@ -164,7 +247,7 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
|
||||
|
||||
if (isValidThinkingLevel(suffix)) {
|
||||
// Valid thinking level - recurse on prefix and use this level
|
||||
const result = parseModelPattern(prefix, availableModels);
|
||||
const result = parseModelPatternWithContext(prefix, availableModels, context);
|
||||
if (result.model) {
|
||||
// Only use this thinking level if no warning from inner recursion
|
||||
const explicitThinkingLevel = !result.warning;
|
||||
@@ -179,7 +262,7 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
|
||||
}
|
||||
|
||||
// Invalid suffix - recurse on prefix and warn
|
||||
const result = parseModelPattern(prefix, availableModels);
|
||||
const result = parseModelPatternWithContext(prefix, availableModels, context);
|
||||
if (result.model) {
|
||||
return {
|
||||
model: result.model,
|
||||
@@ -191,6 +274,15 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
|
||||
return result;
|
||||
}
|
||||
|
||||
export function parseModelPattern(
|
||||
pattern: string,
|
||||
availableModels: Model<Api>[],
|
||||
preferences?: ModelMatchPreferences,
|
||||
): ParsedModelResult {
|
||||
const context = buildPreferenceContext(availableModels, preferences);
|
||||
return parseModelPatternWithContext(pattern, availableModels, context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve model patterns to actual Model objects with optional thinking levels
|
||||
* Format: "pattern:level" where :level is optional
|
||||
@@ -202,8 +294,13 @@ export function parseModelPattern(pattern: string, availableModels: Model<Api>[]
|
||||
* The algorithm tries to match the full pattern first, then progressively
|
||||
* strips colon-suffixes to find a match.
|
||||
*/
|
||||
export async function resolveModelScope(patterns: string[], modelRegistry: ModelRegistry): Promise<ScopedModel[]> {
|
||||
export async function resolveModelScope(
|
||||
patterns: string[],
|
||||
modelRegistry: ModelRegistry,
|
||||
preferences?: ModelMatchPreferences,
|
||||
): Promise<ScopedModel[]> {
|
||||
const availableModels = modelRegistry.getAvailable();
|
||||
const context = buildPreferenceContext(availableModels, preferences);
|
||||
const scopedModels: ScopedModel[] = [];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
@@ -245,7 +342,11 @@ export async function resolveModelScope(patterns: string[], modelRegistry: Model
|
||||
continue;
|
||||
}
|
||||
|
||||
const { model, thinkingLevel, warning, explicitThinkingLevel } = parseModelPattern(pattern, availableModels);
|
||||
const { model, thinkingLevel, warning, explicitThinkingLevel } = parseModelPatternWithContext(
|
||||
pattern,
|
||||
availableModels,
|
||||
context,
|
||||
);
|
||||
|
||||
if (warning) {
|
||||
console.warn(chalk.yellow(`Warning: ${warning}`));
|
||||
|
||||
@@ -373,7 +373,10 @@ async function buildSessionOptions(
|
||||
// Model from CLI (--model) - uses same fuzzy matching as --models
|
||||
if (parsed.model) {
|
||||
const available = modelRegistry.getAvailable();
|
||||
const { model, warning } = parseModelPattern(parsed.model, available);
|
||||
const modelMatchPreferences = {
|
||||
usageOrder: settingsManager.getStorage()?.getModelUsageOrder(),
|
||||
};
|
||||
const { model, warning } = parseModelPattern(parsed.model, available, modelMatchPreferences);
|
||||
if (warning) {
|
||||
writeStderr(chalk.yellow(`Warning: ${warning}`));
|
||||
}
|
||||
@@ -652,8 +655,11 @@ export async function main(args: string[]) {
|
||||
|
||||
let scopedModels: ScopedModel[] = [];
|
||||
const modelPatterns = parsed.models ?? settingsManager.getEnabledModels();
|
||||
const modelMatchPreferences = {
|
||||
usageOrder: settingsManager.getStorage()?.getModelUsageOrder(),
|
||||
};
|
||||
if (modelPatterns && modelPatterns.length > 0) {
|
||||
scopedModels = await resolveModelScope(modelPatterns, modelRegistry);
|
||||
scopedModels = await resolveModelScope(modelPatterns, modelRegistry, modelMatchPreferences);
|
||||
time("resolveModelScope");
|
||||
}
|
||||
|
||||
|
||||
@@ -72,4 +72,4 @@ These are fingerprints of generic AI-generated interfaces. Avoid them:
|
||||
Every interface should make someone ask "how was this made?" not "which AI made this?"
|
||||
Commit to a clear aesthetic direction and execute with precision.
|
||||
Keep going until the implementation is complete. This matters.
|
||||
</critical>
|
||||
</critical>
|
||||
@@ -134,6 +134,7 @@ function resolveModelOverride(
|
||||
settingsManager?: SettingsManager,
|
||||
): { model?: Model<Api>; thinkingLevel?: ThinkingLevel } {
|
||||
if (modelPatterns.length === 0) return {};
|
||||
const matchPreferences = { usageOrder: settingsManager?.getStorage()?.getModelUsageOrder() };
|
||||
const roles = settingsManager?.serialize().modelRoles as Record<string, string> | undefined;
|
||||
for (const pattern of modelPatterns) {
|
||||
const normalized = pattern.trim().toLowerCase();
|
||||
@@ -148,7 +149,11 @@ function resolveModelOverride(
|
||||
effectivePattern = configured;
|
||||
}
|
||||
}
|
||||
const { model, thinkingLevel } = parseModelPattern(effectivePattern, modelRegistry.getAvailable());
|
||||
const { model, thinkingLevel } = parseModelPattern(
|
||||
effectivePattern,
|
||||
modelRegistry.getAvailable(),
|
||||
matchPreferences,
|
||||
);
|
||||
if (model) {
|
||||
return { model, thinkingLevel: thinkingLevel !== "off" ? thinkingLevel : undefined };
|
||||
}
|
||||
|
||||
@@ -58,7 +58,34 @@ const mockOpenRouterModels: Model<"anthropic-messages">[] = [
|
||||
},
|
||||
];
|
||||
|
||||
const allModels = [...mockModels, ...mockOpenRouterModels];
|
||||
const mockProviderOverlapModels: Model<"anthropic-messages">[] = [
|
||||
{
|
||||
id: "kimi-k2.5",
|
||||
name: "Kimi K2.5",
|
||||
api: "anthropic-messages",
|
||||
provider: "kimi-code",
|
||||
baseUrl: "https://api.kimi.ai",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 2, output: 6, cacheRead: 0.2, cacheWrite: 2 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
{
|
||||
id: "moonshotai/kimi-k2.5",
|
||||
name: "Kimi K2.5 (OpenRouter)",
|
||||
api: "anthropic-messages",
|
||||
provider: "openrouter",
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 2.2, output: 6.2, cacheRead: 0.22, cacheWrite: 2.2 },
|
||||
contextWindow: 128000,
|
||||
maxTokens: 8192,
|
||||
},
|
||||
];
|
||||
|
||||
const allModels = [...mockModels, ...mockOpenRouterModels, ...mockProviderOverlapModels];
|
||||
|
||||
describe("parseModelPattern", () => {
|
||||
describe("simple patterns without colons", () => {
|
||||
@@ -213,4 +240,26 @@ describe("parseModelPattern", () => {
|
||||
expect(result.warning).toContain("Invalid thinking level");
|
||||
});
|
||||
});
|
||||
|
||||
describe("preference logic", () => {
|
||||
test("prefers most recently used model when multiple providers match", () => {
|
||||
const result = parseModelPattern("k2.5", allModels, {
|
||||
usageOrder: ["kimi-code/kimi-k2.5"],
|
||||
});
|
||||
expect(result.model?.provider).toBe("kimi-code");
|
||||
});
|
||||
|
||||
test("falls back to deprioritizing openrouter when no usage data", () => {
|
||||
const result = parseModelPattern("k2.5", allModels, { usageOrder: [] });
|
||||
expect(result.model?.provider).toBe("kimi-code");
|
||||
});
|
||||
|
||||
test("respects most recently used provider even if openrouter", () => {
|
||||
const result = parseModelPattern("k2.5", allModels, {
|
||||
usageOrder: ["openrouter/moonshotai/kimi-k2.5"],
|
||||
});
|
||||
expect(result.model?.provider).toBe("openrouter");
|
||||
expect(result.model?.id).toBe("moonshotai/kimi-k2.5");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user