21d152c70c
F6: defer the JSON -> YAML migration out of ConfigFile's constructor and
add an async path so the boot sequence stops blocking the event loop on
sync I/O. New ConfigFile.tryLoadAsync/loadAsync/loadOrDefaultAsync/
getMtimeMsAsync and static ConfigFile.warmup. The migration is now
idempotent (per-process cache) so relocate() does not re-run it.
ModelRegistry.create(authStorage, modelsPath?) is a new async factory
that runs the warmup before the sync constructor's bundled-model load.
Production call sites (main.ts, sdk.ts, task/executor.ts, commit
pipelines, SDK example) all switched. Sync new ModelRegistry(...)
constructor is still supported for tests.
F2: rewrite MemorySessionStorage's mirror as { chunks: string[]; byteLen;
mtimeMs } so writeLineSync appends a single chunk in O(1) instead of
read-modify-writing the entire file (which was O(N) per append, O(N^2)
per session). statSync now reports true UTF-8 byte length instead of
character count. readTextPrefix walks chunks until the byte budget is
exhausted instead of materialising the full mirror.
Also rolls in per-package CHANGELOG entries for F1-F8.
373 lines
10 KiB
TypeScript
373 lines
10 KiB
TypeScript
import * as fs from "node:fs";
|
|
import * as path from "node:path";
|
|
import { getAgentDir, isEnoent, logger } from "@oh-my-pi/pi-utils";
|
|
import { JSONC, YAML } from "bun";
|
|
import type { ZodType } from "zod/v4";
|
|
|
|
/** Minimal subset of the AJV ConfigSchemaError shape this module actually relies on. */
|
|
interface ConfigSchemaError {
|
|
instancePath: string;
|
|
message: string | undefined;
|
|
}
|
|
|
|
/**
|
|
* Module-private cache of (jsonPath, ymlPath) pairs we already migrated this
|
|
* process. Prevents `ConfigFile.relocate()` / repeated `tryLoad()` calls from
|
|
* re-running the migration over and over on the boot path.
|
|
*/
|
|
const migratedPaths = new Set<string>();
|
|
|
|
function migrationKey(jsonPath: string, ymlPath: string): string {
|
|
return `${jsonPath}\u0000${ymlPath}`;
|
|
}
|
|
|
|
/**
|
|
* Synchronous JSON → YAML migration kept for callers that still want the
|
|
* eager path (settings init, tests that observe migration completion).
|
|
* Idempotent — re-running is a no-op.
|
|
*/
|
|
function migrateJsonToYml(jsonPath: string, ymlPath: string) {
|
|
const key = migrationKey(jsonPath, ymlPath);
|
|
if (migratedPaths.has(key)) return;
|
|
try {
|
|
if (fs.existsSync(ymlPath)) {
|
|
migratedPaths.add(key);
|
|
return;
|
|
}
|
|
if (!fs.existsSync(jsonPath)) {
|
|
migratedPaths.add(key);
|
|
return;
|
|
}
|
|
|
|
const content = fs.readFileSync(jsonPath, "utf-8");
|
|
const parsed = JSON.parse(content);
|
|
if (!parsed) {
|
|
logger.warn("migrateJsonToYml: invalid json structure", { path: jsonPath });
|
|
migratedPaths.add(key);
|
|
return;
|
|
}
|
|
fs.writeFileSync(ymlPath, YAML.stringify(parsed, null, 2));
|
|
migratedPaths.add(key);
|
|
} catch (error) {
|
|
logger.warn("migrateJsonToYml: migration failed", { error: String(error) });
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Async sibling of `migrateJsonToYml`. Uses Bun.file so the boot path no
|
|
* longer blocks on a sync FS read before the first await. Idempotent and
|
|
* shares the process-wide `migratedPaths` cache with the sync path.
|
|
*/
|
|
async function migrateJsonToYmlAsync(jsonPath: string, ymlPath: string) {
|
|
const key = migrationKey(jsonPath, ymlPath);
|
|
if (migratedPaths.has(key)) return;
|
|
try {
|
|
if (await Bun.file(ymlPath).exists()) {
|
|
migratedPaths.add(key);
|
|
return;
|
|
}
|
|
let content: string;
|
|
try {
|
|
content = await Bun.file(jsonPath).text();
|
|
} catch (err) {
|
|
if (isEnoent(err)) {
|
|
migratedPaths.add(key);
|
|
return;
|
|
}
|
|
throw err;
|
|
}
|
|
const parsed = JSON.parse(content);
|
|
if (!parsed) {
|
|
logger.warn("migrateJsonToYmlAsync: invalid json structure", { path: jsonPath });
|
|
migratedPaths.add(key);
|
|
return;
|
|
}
|
|
await Bun.write(ymlPath, YAML.stringify(parsed, null, 2));
|
|
migratedPaths.add(key);
|
|
} catch (error) {
|
|
logger.warn("migrateJsonToYmlAsync: migration failed", { error: String(error) });
|
|
}
|
|
}
|
|
|
|
export interface IConfigFile<T> {
|
|
readonly id: string;
|
|
readonly schema: ZodType<T>;
|
|
path?(): string;
|
|
load(): T | null;
|
|
invalidate?(): void;
|
|
}
|
|
|
|
export class ConfigError extends Error {
|
|
readonly #message: string;
|
|
constructor(
|
|
public readonly id: string,
|
|
public readonly schemaErrors: ConfigSchemaError[] | null | undefined,
|
|
public readonly other?: { err: unknown; stage: string },
|
|
) {
|
|
let messages: string[] | undefined;
|
|
let cause: Error | undefined;
|
|
let klass: string;
|
|
|
|
if (schemaErrors) {
|
|
klass = "Schema";
|
|
messages = schemaErrors.map(e => `${e.instancePath || "root"}: ${e.message}`);
|
|
} else if (other) {
|
|
klass = other.stage;
|
|
if (other.err instanceof Error) {
|
|
messages = [other.err.message];
|
|
cause = other.err;
|
|
} else {
|
|
messages = [String(other.err)];
|
|
}
|
|
} else {
|
|
klass = "Unknown";
|
|
}
|
|
|
|
const title = `Failed to load config file ${id}, ${klass} error:`;
|
|
let message: string;
|
|
switch (messages?.length ?? 0) {
|
|
case 0:
|
|
message = title.slice(0, -1);
|
|
break;
|
|
case 1:
|
|
message = `${title} ${messages![0]}`;
|
|
break;
|
|
default:
|
|
message = `${title}\n${messages!.map(m => ` - ${m}`).join("\n")}`;
|
|
}
|
|
|
|
super(message, { cause });
|
|
this.name = "LoadError";
|
|
this.#message = message;
|
|
}
|
|
|
|
get message(): string {
|
|
return this.#message;
|
|
}
|
|
|
|
toString(): string {
|
|
return this.message;
|
|
}
|
|
}
|
|
|
|
export type LoadStatus = "ok" | "error" | "not-found";
|
|
|
|
export type LoadResult<T> =
|
|
| { value?: null; error: ConfigError; status: "error" }
|
|
| { value: T; error?: undefined; status: "ok" }
|
|
| { value?: null; error?: unknown; status: "not-found" };
|
|
|
|
export class ConfigFile<T> implements IConfigFile<T> {
|
|
readonly #basePath: string;
|
|
readonly #jsonMigrationPath: string | null;
|
|
#cache?: LoadResult<T>;
|
|
#auxValidate?: (value: T) => void;
|
|
|
|
constructor(
|
|
readonly id: string,
|
|
readonly schema: ZodType<T>,
|
|
configPath: string = path.join(getAgentDir(), `${id}.yml`),
|
|
) {
|
|
this.#basePath = configPath;
|
|
if (configPath.endsWith(".yml")) {
|
|
this.#jsonMigrationPath = `${configPath.slice(0, -4)}.json`;
|
|
} else if (configPath.endsWith(".yaml")) {
|
|
this.#jsonMigrationPath = `${configPath.slice(0, -5)}.json`;
|
|
} else if (configPath.endsWith(".json") || configPath.endsWith(".jsonc")) {
|
|
// JSON configs are still supported without migration.
|
|
this.#jsonMigrationPath = null;
|
|
} else {
|
|
throw new Error(`Invalid config file path: ${configPath}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run the JSON → YAML migration synchronously, if applicable. Idempotent.
|
|
* Sync callers (tests, settings init) hit this implicitly via {@link tryLoad}.
|
|
*/
|
|
#ensureMigratedSync(): void {
|
|
if (this.#jsonMigrationPath) {
|
|
migrateJsonToYml(this.#jsonMigrationPath, this.#basePath);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Async sibling of {@link #ensureMigratedSync}. Boot-path callers should
|
|
* `await ConfigFile.warmup(file)` before doing any sync `tryLoad`/`load`
|
|
* so the migration's I/O happens off the event-loop's hot path.
|
|
*/
|
|
async #ensureMigratedAsync(): Promise<void> {
|
|
if (this.#jsonMigrationPath) {
|
|
await migrateJsonToYmlAsync(this.#jsonMigrationPath, this.#basePath);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run any pending JSON → YAML migration asynchronously, ahead of a sync
|
|
* `tryLoad()` on the boot path. Safe to call multiple times; subsequent
|
|
* calls are O(1) thanks to the module-level migration cache.
|
|
*/
|
|
static warmup<U>(file: ConfigFile<U>): Promise<void> {
|
|
return file.#ensureMigratedAsync();
|
|
}
|
|
|
|
relocate(configPath?: string): ConfigFile<T> {
|
|
if (!configPath || configPath === this.#basePath) return this;
|
|
const result = new ConfigFile<T>(this.id, this.schema, configPath);
|
|
result.#auxValidate = this.#auxValidate;
|
|
return result;
|
|
}
|
|
|
|
getMtimeMs(): number | null {
|
|
try {
|
|
return fs.statSync(this.path()).mtimeMs;
|
|
} catch (err) {
|
|
if (isEnoent(err)) return null;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
async getMtimeMsAsync(): Promise<number | null> {
|
|
const file = Bun.file(this.path());
|
|
if (!(await file.exists())) return null;
|
|
const lm = file.lastModified;
|
|
return typeof lm === "number" && Number.isFinite(lm) ? lm : null;
|
|
}
|
|
|
|
withValidation(name: string, validate: (value: T) => void): this {
|
|
const prev = this.#auxValidate;
|
|
this.#auxValidate = (value: T) => {
|
|
prev?.(value);
|
|
try {
|
|
validate(value);
|
|
} catch (error) {
|
|
throw new ConfigError(this.id, undefined, { err: error, stage: `Validate(${name})` });
|
|
}
|
|
};
|
|
return this;
|
|
}
|
|
|
|
createDefault(): T {
|
|
const parsed = this.schema.safeParse({});
|
|
if (parsed.success) return parsed.data;
|
|
const fallback = this.schema.safeParse(undefined);
|
|
if (fallback.success) return fallback.data;
|
|
throw new ConfigError(this.id, undefined, {
|
|
err: new Error("Schema produced no default value"),
|
|
stage: "createDefault",
|
|
});
|
|
}
|
|
|
|
#storeCache(result: LoadResult<T>): LoadResult<T> {
|
|
this.#cache = result;
|
|
return result;
|
|
}
|
|
|
|
#parseContent(content: string): LoadResult<T> {
|
|
try {
|
|
let parsed: unknown;
|
|
if (this.#basePath.endsWith(".json") || this.#basePath.endsWith(".jsonc")) {
|
|
parsed = JSONC.parse(content);
|
|
} else if (this.#basePath.endsWith(".yml") || this.#basePath.endsWith(".yaml")) {
|
|
parsed = YAML.parse(content);
|
|
} else {
|
|
throw new Error(`Invalid config file path: ${this.#basePath}`);
|
|
}
|
|
|
|
const checked = this.schema.safeParse(parsed);
|
|
if (!checked.success) {
|
|
const schemaErrors: ConfigSchemaError[] = [];
|
|
for (const issue of checked.error.issues) {
|
|
const instancePath = issue.path.length === 0 ? "" : `/${issue.path.map(String).join("/")}`;
|
|
schemaErrors.push({ instancePath, message: issue.message });
|
|
if (schemaErrors.length >= 50) break;
|
|
}
|
|
const error = new ConfigError(this.id, schemaErrors);
|
|
logger.warn("Failed to parse config file", { path: this.path(), error });
|
|
return this.#storeCache({ error, status: "error" });
|
|
}
|
|
const value = checked.data;
|
|
try {
|
|
this.#auxValidate?.(value);
|
|
} catch (error) {
|
|
const wrapped =
|
|
error instanceof ConfigError
|
|
? error
|
|
: new ConfigError(this.id, undefined, { err: error, stage: "AuxValidate" });
|
|
return this.#storeCache({ error: wrapped, status: "error" });
|
|
}
|
|
return this.#storeCache({ value, status: "ok" });
|
|
} catch (error) {
|
|
logger.warn("Failed to parse config file", { path: this.path(), error });
|
|
return this.#storeCache({
|
|
error: new ConfigError(this.id, undefined, { err: error, stage: "Unexpected" }),
|
|
status: "error",
|
|
});
|
|
}
|
|
}
|
|
|
|
tryLoad(): LoadResult<T> {
|
|
if (this.#cache) return this.#cache;
|
|
this.#ensureMigratedSync();
|
|
|
|
let content: string;
|
|
try {
|
|
content = fs.readFileSync(this.path(), "utf-8").trim();
|
|
} catch (error) {
|
|
if (isEnoent(error)) {
|
|
return this.#storeCache({ status: "not-found" });
|
|
}
|
|
logger.warn("Failed to read config file", { path: this.path(), error });
|
|
return this.#storeCache({
|
|
error: new ConfigError(this.id, undefined, { err: error, stage: "Read" }),
|
|
status: "error",
|
|
});
|
|
}
|
|
return this.#parseContent(content);
|
|
}
|
|
|
|
async tryLoadAsync(): Promise<LoadResult<T>> {
|
|
if (this.#cache) return this.#cache;
|
|
await this.#ensureMigratedAsync();
|
|
|
|
let content: string;
|
|
try {
|
|
content = (await Bun.file(this.path()).text()).trim();
|
|
} catch (error) {
|
|
if (isEnoent(error)) {
|
|
return this.#storeCache({ status: "not-found" });
|
|
}
|
|
logger.warn("Failed to read config file", { path: this.path(), error });
|
|
return this.#storeCache({
|
|
error: new ConfigError(this.id, undefined, { err: error, stage: "Read" }),
|
|
status: "error",
|
|
});
|
|
}
|
|
return this.#parseContent(content);
|
|
}
|
|
|
|
load(): T | null {
|
|
return this.tryLoad().value ?? null;
|
|
}
|
|
|
|
async loadAsync(): Promise<T | null> {
|
|
return (await this.tryLoadAsync()).value ?? null;
|
|
}
|
|
|
|
loadOrDefault(): T {
|
|
return this.tryLoad().value ?? this.createDefault();
|
|
}
|
|
|
|
async loadOrDefaultAsync(): Promise<T> {
|
|
return (await this.tryLoadAsync()).value ?? this.createDefault();
|
|
}
|
|
|
|
path(): string {
|
|
return this.#basePath;
|
|
}
|
|
|
|
invalidate() {
|
|
this.#cache = undefined;
|
|
}
|
|
}
|