feat(coding-agent): migrated settings and models to models.yml

- Settings now stored in ~/.omp/agent/config.yaml instead of agent.db
- Models config now supports models.yaml (preferred) alongside models.json
- Auto-migration from agent.db settings and models.json on first run
- AgentStorage.saveSettings() deprecated (getSettings() kept for migration)
- storage-migration.ts now only handles auth.json migration
This commit is contained in:
can1357
2026-01-23 13:46:01 +01:00
parent 6e5fa5b24d
commit fb68e72d78
6 changed files with 186 additions and 133 deletions
+12 -2
View File
@@ -69,16 +69,26 @@ export function getModelsPath(): string {
return join(getAgentDir(), "models.json");
}
/** Get path to models.yml (preferred over models.json) */
export function getModelsYamlPath(): string {
return join(getAgentDir(), "models.yml");
}
/** Get path to auth.json */
export function getAuthPath(): string {
return join(getAgentDir(), "auth.json");
}
/** Get path to settings.json */
export function getSettingsPath(): string {
/** Get path to settings.json (legacy, use getConfigPath for config.yaml) */
export function getLegacySettingsPath(): string {
return join(getAgentDir(), "settings.json");
}
/** Get path to config.yml (user settings) */
export function getConfigPath(): string {
return join(getAgentDir(), "config.yml");
}
/**
* Gets the path to agent.db (SQLite database for settings and auth storage).
* @param agentDir - Base agent directory, defaults to ~/.omp/agent
@@ -3,6 +3,7 @@
*/
import { existsSync, readFileSync } from "node:fs";
import { extname } from "node:path";
import {
type Api,
getGitHubCopilotBaseUrl,
@@ -14,6 +15,7 @@ import {
import { logger } from "@oh-my-pi/pi-utils";
import { type Static, Type } from "@sinclair/typebox";
import AjvModule from "ajv";
import { YAML } from "bun";
import type { AuthStorage } from "$c/session/auth-storage";
const Ajv = (AjvModule as any).default || AjvModule;
@@ -262,14 +264,21 @@ export class ModelRegistry {
});
}
private loadCustomModels(modelsJsonPath: string): CustomModelsResult {
if (!existsSync(modelsJsonPath)) {
private loadCustomModels(modelsPath: string): CustomModelsResult {
if (!existsSync(modelsPath)) {
return emptyCustomModelsResult();
}
try {
const content = readFileSync(modelsJsonPath, "utf-8");
const config: ModelsConfig = JSON.parse(content);
const content = readFileSync(modelsPath, "utf-8");
const ext = extname(modelsPath).toLowerCase();
let config: ModelsConfig;
if (ext === ".yaml" || ext === ".yml") {
config = YAML.parse(content) as ModelsConfig;
} else {
config = JSON.parse(content) as ModelsConfig;
}
// Validate schema
const ajv = new Ajv();
@@ -278,7 +287,7 @@ export class ModelRegistry {
const errors =
validate.errors?.map((e: any) => ` - ${e.instancePath || "root"}: ${e.message}`).join("\n") ||
"Unknown schema error";
return emptyCustomModelsResult(`Invalid models.json schema:\n${errors}\n\nFile: ${modelsJsonPath}`);
return emptyCustomModelsResult(`Invalid models config schema:\n${errors}\n\nFile: ${modelsPath}`);
}
// Additional validation
@@ -309,10 +318,10 @@ export class ModelRegistry {
return { models: this.parseModels(config), replacedProviders, overrides, error: undefined };
} catch (error) {
if (error instanceof SyntaxError) {
return emptyCustomModelsResult(`Failed to parse models.json: ${error.message}\n\nFile: ${modelsJsonPath}`);
return emptyCustomModelsResult(`Failed to parse models config: ${error.message}\n\nFile: ${modelsPath}`);
}
return emptyCustomModelsResult(
`Failed to load models.json: ${error instanceof Error ? error.message : error}\n\nFile: ${modelsJsonPath}`,
`Failed to load models config: ${error instanceof Error ? error.message : error}\n\nFile: ${modelsPath}`,
);
}
}
@@ -1,8 +1,10 @@
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import { rename } from "node:fs/promises";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { logger } from "@oh-my-pi/pi-utils";
import { YAML } from "bun";
import { type Settings as SettingsItem, settingsCapability } from "$c/capability/settings";
import { getAgentDbPath, getAgentDir } from "$c/config";
import { getAgentDbPath, getAgentDir, getConfigPath } from "$c/config";
import { loadCapability } from "$c/discovery";
import type { SymbolPreset } from "$c/modes/theme/theme";
import { AgentStorage } from "$c/session/agent-storage";
@@ -424,8 +426,10 @@ function deepMergeSettings(base: Settings, overrides: Settings): Settings {
}
export class SettingsManager {
/** SQLite storage for persisted settings (null for in-memory mode) */
/** SQLite storage for auth/cache (null for in-memory mode) */
private storage: AgentStorage | null;
/** Path to config.yml (null for in-memory mode) */
private configPath: string | null;
private cwd: string | null;
private globalSettings: Settings;
private overrides: Settings;
@@ -434,7 +438,8 @@ export class SettingsManager {
/**
* Private constructor - use static factory methods instead.
* @param storage - SQLite storage instance for persistence, or null for in-memory mode
* @param storage - SQLite storage instance for auth/cache, or null for in-memory mode
* @param configPath - Path to config.yml for persistence, or null for in-memory mode
* @param cwd - Current working directory for project settings discovery
* @param initialSettings - Initial global settings to use
* @param persist - Whether to persist settings changes to storage
@@ -442,12 +447,14 @@ export class SettingsManager {
*/
private constructor(
storage: AgentStorage | null,
configPath: string | null,
cwd: string | null,
initialSettings: Settings,
persist: boolean,
projectSettings: Settings,
) {
this.storage = storage;
this.configPath = configPath;
this.cwd = cwd;
this.persist = persist;
this.globalSettings = initialSettings;
@@ -479,14 +486,17 @@ export class SettingsManager {
}
/**
* Create a SettingsManager that loads from persistent SQLite storage.
* Create a SettingsManager that loads from persistent config.yml.
* @param cwd - Current working directory for project settings discovery
* @param agentDir - Agent directory containing agent.db
* @param agentDir - Agent directory containing config.yml
* @returns Configured SettingsManager with merged global and user settings
*/
static async create(cwd: string = process.cwd(), agentDir: string = getAgentDir()): Promise<SettingsManager> {
const configPath = getConfigPath();
const storage = AgentStorage.open(getAgentDbPath(agentDir));
await SettingsManager.migrateLegacySettingsFile(storage, agentDir);
// Migrate from legacy storage if config.yml doesn't exist
await SettingsManager.migrateToYaml(storage, agentDir, configPath);
// Use capability API to load user-level settings from all providers
const result = await loadCapability(settingsCapability.id, { cwd });
@@ -499,14 +509,14 @@ export class SettingsManager {
}
}
// Load persisted settings from agent.db (legacy settings.json is migrated separately)
const storedSettings = SettingsManager.loadFromStorage(storage);
// Load persisted settings from config.yml
const storedSettings = SettingsManager.loadFromYaml(configPath);
globalSettings = deepMergeSettings(globalSettings, storedSettings);
// Load project settings before construction (constructor is sync)
const projectSettings = await SettingsManager.loadProjectSettingsStatic(cwd);
return new SettingsManager(storage, cwd, globalSettings, true, projectSettings);
return new SettingsManager(storage, configPath, cwd, globalSettings, true, projectSettings);
}
/**
@@ -515,7 +525,7 @@ export class SettingsManager {
* @returns SettingsManager that won't persist changes to disk
*/
static inMemory(settings: Partial<Settings> = {}): SettingsManager {
return new SettingsManager(null, null, settings, false, {});
return new SettingsManager(null, null, null, settings, false, {});
}
/**
@@ -534,41 +544,83 @@ export class SettingsManager {
}
/**
* Load settings from SQLite storage, applying any schema migrations.
* @param storage - AgentStorage instance, or null for in-memory mode
* @returns Parsed and migrated settings, or empty object if storage is null/empty
* Load settings from config.yml, applying any schema migrations.
* @param configPath - Path to config.yml, or null for in-memory mode
* @returns Parsed and migrated settings, or empty object if file doesn't exist
*/
private static loadFromStorage(storage: AgentStorage | null): Settings {
if (!storage) {
private static loadFromYaml(configPath: string | null): Settings {
if (!configPath || !existsSync(configPath)) {
return {};
}
const settings = storage.getSettings();
if (!settings) {
try {
const content = readFileSync(configPath, "utf-8");
const parsed = YAML.parse(content);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return {};
}
return SettingsManager.migrateSettings(parsed as Record<string, unknown>);
} catch (error) {
logger.warn("SettingsManager failed to load config.yml", { path: configPath, error: String(error) });
return {};
}
return SettingsManager.migrateSettings(settings as Record<string, unknown>);
}
private static async migrateLegacySettingsFile(storage: AgentStorage, agentDir: string): Promise<void> {
const settingsPath = join(agentDir, "settings.json");
const settingsFile = Bun.file(settingsPath);
if (!(await settingsFile.exists())) return;
if (storage.getSettings() !== null) return;
/**
* Migrate settings from legacy sources to config.yml.
* Migration order: settings.json -> agent.db -> config.yml
* Only migrates if config.yml doesn't exist.
*/
private static async migrateToYaml(storage: AgentStorage, agentDir: string, configPath: string): Promise<void> {
// Skip if config.yml already exists
if (existsSync(configPath)) return;
let settings: Settings = {};
let migrated = false;
// 1. Try to migrate from settings.json (oldest legacy format)
const settingsJsonPath = join(agentDir, "settings.json");
try {
const parsed = JSON.parse(await settingsFile.text());
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return;
}
const migrated = SettingsManager.migrateSettings(parsed as Record<string, unknown>);
storage.saveSettings(migrated);
try {
await rename(settingsPath, `${settingsPath}.bak`);
} catch (error) {
logger.warn("SettingsManager failed to backup settings.json", { error: String(error) });
const settingsFile = Bun.file(settingsJsonPath);
if (await settingsFile.exists()) {
const parsed = JSON.parse(await settingsFile.text());
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
settings = deepMergeSettings(settings, SettingsManager.migrateSettings(parsed));
migrated = true;
// Backup settings.json
try {
await rename(settingsJsonPath, `${settingsJsonPath}.bak`);
} catch (error) {
logger.warn("SettingsManager failed to backup settings.json", { error: String(error) });
}
}
}
} catch (error) {
logger.warn("SettingsManager failed to migrate settings.json", { error: String(error) });
logger.warn("SettingsManager failed to read settings.json", { error: String(error) });
}
// 2. Migrate from agent.db settings table
try {
const dbSettings = storage.getSettings();
if (dbSettings) {
settings = deepMergeSettings(
settings,
SettingsManager.migrateSettings(dbSettings as Record<string, unknown>),
);
migrated = true;
}
} catch (error) {
logger.warn("SettingsManager failed to read agent.db settings", { error: String(error) });
}
// 3. Write merged settings to config.yml if we found any
if (migrated && Object.keys(settings).length > 0) {
try {
mkdirSync(dirname(configPath), { recursive: true });
await Bun.write(configPath, YAML.stringify(settings));
logger.debug("SettingsManager migrated settings to config.yml", { path: configPath });
} catch (error) {
logger.warn("SettingsManager failed to write config.yml", { path: configPath, error: String(error) });
}
}
}
@@ -620,16 +672,19 @@ export class SettingsManager {
}
/**
* Persist current global settings to SQLite storage and rebuild merged settings.
* Merges with any concurrent changes in storage before saving.
* Persist current global settings to config.yml and rebuild merged settings.
* Merges with any concurrent changes in config file before saving.
*/
private async save(): Promise<void> {
if (this.persist && this.storage) {
if (this.persist && this.configPath) {
try {
const currentSettings = this.storage.getSettings() ?? {};
// Read current settings from file to merge with any concurrent changes
const currentSettings = SettingsManager.loadFromYaml(this.configPath);
const mergedSettings = deepMergeSettings(currentSettings, this.globalSettings);
this.globalSettings = mergedSettings;
this.storage.saveSettings(this.globalSettings);
// Write YAML
mkdirSync(dirname(this.configPath), { recursive: true });
await Bun.write(this.configPath, YAML.stringify(this.globalSettings));
} catch (error) {
logger.warn("SettingsManager save failed", { error: String(error) });
}
+44 -8
View File
@@ -26,17 +26,19 @@
* ```
*/
import { existsSync } from "node:fs";
import { rename } from "node:fs/promises";
import { join } from "node:path";
import { Agent, type AgentEvent, type AgentMessage, type AgentTool, type ThinkingLevel } from "@oh-my-pi/pi-agent-core";
import { type Message, type Model, supportsXhigh } from "@oh-my-pi/pi-ai";
import type { Component } from "@oh-my-pi/pi-tui";
// Import discovery to register all providers on startup
import { logger, postmortem } from "@oh-my-pi/pi-utils";
import { YAML } from "bun";
import chalk from "chalk";
// Import discovery to register all providers on startup
import { loadCapability } from "$c/capability/index";
import { type Rule, ruleCapability } from "$c/capability/rule";
import { getAgentDir, getConfigDirPaths } from "$c/config";
import "./discovery";
import { CursorExecHandlers } from "$c/cursor";
import { initializeWithSettings } from "$c/discovery";
import { TtsrManager } from "$c/export/ttsr";
@@ -47,6 +49,7 @@ import { ModelRegistry } from "./config/model-registry";
import { formatModelString, parseModelString } from "./config/model-resolver";
import { loadPromptTemplates as loadPromptTemplatesInternal, type PromptTemplate } from "./config/prompt-templates";
import { type Settings, SettingsManager, type SkillsSettings } from "./config/settings-manager";
import "./discovery";
import {
type CustomCommandsLoadResult,
loadCustomCommands as loadCustomCommandsInternal,
@@ -269,24 +272,57 @@ export async function discoverAuthStorage(agentDir: string = getDefaultAgentDir(
/**
* Create a ModelRegistry with fallback support.
* Reads from primary path first, then falls back to legacy paths (.pi, .claude).
* Prefers models.yml over models.json. Reads from primary path first,
* then falls back to legacy paths (.pi, .claude).
*/
export async function discoverModels(
authStorage: AuthStorage,
agentDir: string = getDefaultAgentDir(),
): Promise<ModelRegistry> {
const primaryPath = join(agentDir, "models.json");
// Get all models.json paths (user-level only), excluding the primary
const allPaths = getConfigDirPaths("models.json", { project: false });
const fallbackPaths = allPaths.filter((p) => p !== primaryPath);
const yamlPath = join(agentDir, "models.yml");
const jsonPath = join(agentDir, "models.json");
// Migrate models.json to models.yml if yaml doesn't exist but json does
if (!existsSync(yamlPath) && existsSync(jsonPath)) {
await migrateModelsJsonToYaml(jsonPath, yamlPath);
}
// Prefer models.yml, fall back to models.json
const primaryPath = existsSync(yamlPath) ? yamlPath : jsonPath;
// Get all models config paths (user-level only), excluding the primary
const yamlPaths = getConfigDirPaths("models.yml", { project: false });
const jsonPaths = getConfigDirPaths("models.json", { project: false });
const allPaths = [...yamlPaths, ...jsonPaths];
const fallbackPaths = allPaths.filter((p) => p !== primaryPath && existsSync(p));
logger.debug("discoverModels", { primaryPath, fallbackPaths });
const registry = new ModelRegistry(authStorage, primaryPath, fallbackPaths);
await registry.refresh();
registry.refresh();
return registry;
}
/**
* Migrate models.json to models.yml.
* Creates models.yml from models.json and renames the json file to .bak.
*/
async function migrateModelsJsonToYaml(jsonPath: string, yamlPath: string): Promise<void> {
try {
const content = await Bun.file(jsonPath).text();
const parsed = JSON.parse(content);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
logger.warn("migrateModelsJsonToYaml: invalid models.json structure", { path: jsonPath });
return;
}
await Bun.write(yamlPath, YAML.stringify(parsed));
await rename(jsonPath, `${jsonPath}.bak`);
logger.debug("migrateModelsJsonToYaml: migrated models.json to models.yml", { from: jsonPath, to: yamlPath });
} catch (error) {
logger.warn("migrateModelsJsonToYaml: migration failed", { error: String(error) });
}
}
/**
* Discover extensions from cwd.
*/
@@ -116,8 +116,6 @@ export class AgentStorage {
private static instances = new Map<string, AgentStorage>();
private listSettingsStmt: Statement;
private insertSettingStmt: Statement;
private deleteSettingsStmt: Statement;
private getCacheStmt: Statement;
private upsertCacheStmt: Statement;
private deleteExpiredCacheStmt: Statement;
@@ -137,10 +135,6 @@ export class AgentStorage {
this.hardenPermissions(dbPath);
this.listSettingsStmt = this.db.prepare("SELECT key, value FROM settings");
this.insertSettingStmt = this.db.prepare(
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, unixepoch())",
);
this.deleteSettingsStmt = this.db.prepare("DELETE FROM settings");
this.getCacheStmt = this.db.prepare("SELECT value FROM cache WHERE key = ? AND expires_at > unixepoch()");
this.upsertCacheStmt = this.db.prepare(
@@ -276,8 +270,11 @@ CREATE TABLE settings (
}
/**
* Retrieves all settings from storage.
* Retrieves all settings from storage (legacy, for migration only).
* Settings are now stored in config.yml. This method is only used
* during migration from agent.db to config.yml.
* @returns Settings object, or null if no settings are stored
* @deprecated Use config.yml instead. This is only for migration.
*/
getSettings(): Settings | null {
const rows = (this.listSettingsStmt.all() as SettingsRow[]) ?? [];
@@ -297,26 +294,13 @@ CREATE TABLE settings (
}
/**
* Atomically replaces all settings in storage.
* Uses delete-then-insert within a transaction for consistency.
* @param settings - Settings object to persist
* @deprecated Settings are now stored in config.yml, not agent.db.
* This method is kept for backward compatibility but does nothing.
*/
saveSettings(settings: Settings): void {
const entries = Object.entries(settings).filter(([, value]) => value !== undefined);
const replace = this.db.transaction((rows: Array<[string, unknown]>) => {
this.deleteSettingsStmt.run();
for (const [key, value] of rows) {
const serialized = JSON.stringify(value);
if (serialized === undefined) continue;
this.insertSettingStmt.run(key, serialized);
}
logger.warn("AgentStorage.saveSettings is deprecated - settings are now stored in config.yml", {
keys: Object.keys(settings),
});
try {
replace(entries);
} catch (error) {
logger.error("AgentStorage failed to save settings", { error: String(error) });
}
}
/**
@@ -1,12 +1,14 @@
/**
* Migrates legacy JSON storage (settings.json, auth.json) to SQLite-based agent.db.
* Settings migrate only when the DB has no settings; auth merges per-provider when missing.
* Migrates legacy auth.json to SQLite-based agent.db.
* Auth credentials merge per-provider when missing.
* Original JSON files are backed up to .bak and removed after successful migration.
*
* NOTE: Settings migration is now handled by SettingsManager.migrateToYaml(),
* which migrates from both settings.json and agent.db to config.yaml.
*/
import { logger } from "@oh-my-pi/pi-utils";
import { getAgentDbPath } from "$c/config";
import type { Settings } from "$c/config/settings-manager";
import { AgentStorage } from "./agent-storage";
import type { AuthCredential, AuthCredentialEntry, AuthStorageData } from "./auth-storage";
@@ -14,7 +16,7 @@ import type { AuthCredential, AuthCredentialEntry, AuthStorageData } from "./aut
type MigrationPaths = {
/** Directory containing agent.db */
agentDir: string;
/** Path to legacy settings.json file */
/** Path to legacy settings.json file (kept for API compatibility, no longer used) */
settingsPath: string;
/** Candidate paths to search for auth.json (checked in order) */
authPaths: string[];
@@ -22,7 +24,7 @@ type MigrationPaths = {
/** Result of the JSON-to-SQLite storage migration. */
export interface StorageMigrationResult {
/** Whether settings.json was migrated to agent.db */
/** Whether settings.json was migrated (always false - settings migration is handled elsewhere) */
migratedSettings: boolean;
/** Whether auth.json was migrated to agent.db */
migratedAuth: boolean;
@@ -39,20 +41,6 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === "object" && !Array.isArray(value);
}
/**
* Transforms legacy settings to current schema (e.g., queueMode -> steeringMode).
* @param settings - Settings object potentially containing deprecated keys
* @returns Settings with deprecated keys renamed to current equivalents
*/
function migrateLegacySettings(settings: Settings): Settings {
const migrated = { ...settings } as Record<string, unknown>;
if ("queueMode" in migrated && !("steeringMode" in migrated)) {
migrated.steeringMode = migrated.queueMode;
delete migrated.queueMode;
}
return migrated as Settings;
}
/**
* Normalizes credential entries to array format (legacy stored single credentials).
* @param entry - Single credential or array of credentials
@@ -99,32 +87,6 @@ async function backupJson(path: string): Promise<void> {
}
}
/**
* Migrates settings.json to SQLite storage if DB is empty.
* @param storage - AgentStorage instance to migrate into
* @param settingsPath - Path to legacy settings.json
* @param warnings - Array to collect non-fatal warnings
* @returns True if migration was performed
*/
async function migrateSettings(storage: AgentStorage, settingsPath: string, warnings: string[]): Promise<boolean> {
const settingsFile = Bun.file(settingsPath);
const settingsExists = await settingsFile.exists();
const hasDbSettings = storage.getSettings() !== null;
if (!settingsExists) return false;
if (hasDbSettings) {
warnings.push(`settings.json exists but agent.db is authoritative: ${settingsPath}`);
return false;
}
const settingsJson = await readJsonFile<Settings>(settingsPath);
if (!settingsJson) return false;
storage.saveSettings(migrateLegacySettings(settingsJson));
await backupJson(settingsPath);
return true;
}
/**
* Finds the first valid auth.json from candidate paths (checked in priority order).
* @param authPaths - Candidate paths to search (e.g., project-local before global)
@@ -191,8 +153,8 @@ async function migrateAuth(storage: AgentStorage, authPaths: string[], warnings:
}
/**
* Migrates legacy JSON files (settings.json, auth.json) to SQLite-based agent.db.
* Settings migrate only when the DB has no settings; auth merges per-provider when missing.
* Migrates legacy auth.json to SQLite-based agent.db.
* Settings migration is handled separately by SettingsManager.migrateToYaml().
* @param paths - Configuration specifying locations of legacy files and target DB
* @returns Result indicating what was migrated and any warnings encountered
*/
@@ -200,10 +162,7 @@ export async function migrateJsonStorage(paths: MigrationPaths): Promise<Storage
const storage = AgentStorage.open(getAgentDbPath(paths.agentDir));
const warnings: string[] = [];
const [migratedSettings, migratedAuth] = await Promise.all([
migrateSettings(storage, paths.settingsPath, warnings),
migrateAuth(storage, paths.authPaths, warnings),
]);
const migratedAuth = await migrateAuth(storage, paths.authPaths, warnings);
if (warnings.length > 0) {
for (const warning of warnings) {
@@ -211,5 +170,5 @@ export async function migrateJsonStorage(paths: MigrationPaths): Promise<Storage
}
}
return { migratedSettings, migratedAuth, warnings };
return { migratedSettings: false, migratedAuth, warnings };
}