From 77f641268d9802a686453435d6d67e381a269380 Mon Sep 17 00:00:00 2001 From: can1357 Date: Tue, 14 Jul 2026 01:30:40 +0200 Subject: [PATCH] feat(metaharness): migrated harbor-manager to metaharness and updated harness logic - Renamed and migrated package-level artifacts from `harbor-manager` to `metaharness`. - Updated metaharness core services (`server`, `runner`, `store`, `experiments`) with substantive logic edits. - Synced CLI, local agent, and config wiring to align with the metaharness package structure. - Refreshed benchmark-related documentation/prompts and expanded tests for the revised metaharness flow. --- bun.lock | 32 ++-- package.json | 2 +- .../{harbor-manager => metaharness}/README.md | 24 ++- .../adapters/edit/bun-imports.d.ts | 0 .../adapters/edit/cli.ts | 2 +- .../adapters/edit/prompts/benchmark-retry.md | 0 .../adapters/edit/prompts/benchmark-system.md | 0 .../adapters/edit/prompts/benchmark-task.md | 0 .../adapters/edit/report.ts | 0 .../adapters/edit/runner.test.ts | 0 .../adapters/edit/runner.ts | 0 .../adapters/edit/tsconfig.json | 0 .../agent/omp_local.py | 4 +- .../package.json | 6 +- .../src/adapters/snapcompact.py | 0 .../src/benchmarks.test.ts | 0 .../src/benchmarks.ts | 0 .../src/experiments.test.ts | 0 .../src/experiments.ts | 30 +++- .../src/launch-args.ts | 0 .../src/manager.test.ts | 101 ++++++++++- .../src/runner.test.ts | 0 .../src/runner.ts | 12 +- .../src/server.ts | 168 +++++++++++++++--- .../src/store.ts | 45 ++++- .../src/web/app.tsx | 6 +- .../src/web/index.html | 2 +- .../tsconfig.json | 0 28 files changed, 360 insertions(+), 74 deletions(-) rename packages/{harbor-manager => metaharness}/README.md (84%) rename packages/{harbor-manager => metaharness}/adapters/edit/bun-imports.d.ts (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/cli.ts (98%) rename packages/{harbor-manager => metaharness}/adapters/edit/prompts/benchmark-retry.md (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/prompts/benchmark-system.md (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/prompts/benchmark-task.md (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/report.ts (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/runner.test.ts (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/runner.ts (100%) rename packages/{harbor-manager => metaharness}/adapters/edit/tsconfig.json (100%) rename packages/{harbor-manager => metaharness}/agent/omp_local.py (99%) rename packages/{harbor-manager => metaharness}/package.json (92%) rename packages/{harbor-manager => metaharness}/src/adapters/snapcompact.py (100%) rename packages/{harbor-manager => metaharness}/src/benchmarks.test.ts (100%) rename packages/{harbor-manager => metaharness}/src/benchmarks.ts (100%) rename packages/{harbor-manager => metaharness}/src/experiments.test.ts (100%) rename packages/{harbor-manager => metaharness}/src/experiments.ts (94%) rename packages/{harbor-manager => metaharness}/src/launch-args.ts (100%) rename packages/{harbor-manager => metaharness}/src/manager.test.ts (80%) rename packages/{harbor-manager => metaharness}/src/runner.test.ts (100%) rename packages/{harbor-manager => metaharness}/src/runner.ts (99%) rename packages/{harbor-manager => metaharness}/src/server.ts (79%) rename packages/{harbor-manager => metaharness}/src/store.ts (92%) rename packages/{harbor-manager => metaharness}/src/web/app.tsx (99%) rename packages/{harbor-manager => metaharness}/src/web/index.html (93%) rename packages/{harbor-manager => metaharness}/tsconfig.json (100%) diff --git a/bun.lock b/bun.lock index 6e26f48e8..36dffdeec 100644 --- a/bun.lock +++ b/bun.lock @@ -136,11 +136,22 @@ "@types/react-dom": "catalog:", }, }, - "packages/harbor-manager": { - "name": "@oh-my-pi/harbor-manager", + "packages/hashline": { + "name": "@oh-my-pi/hashline", + "version": "16.4.8", + "dependencies": { + "diff": "catalog:", + "lru-cache": "catalog:", + }, + "devDependencies": { + "@types/bun": "catalog:", + }, + }, + "packages/metaharness": { + "name": "@oh-my-pi/pi-metaharness", "version": "0.0.1", "bin": { - "harbor-manager": "src/server.ts", + "metaharness": "src/server.ts", }, "dependencies": { "@oh-my-pi/hashline": "catalog:", @@ -167,17 +178,6 @@ "react-refresh": "^0.18.0", }, }, - "packages/hashline": { - "name": "@oh-my-pi/hashline", - "version": "16.4.8", - "dependencies": { - "diff": "catalog:", - "lru-cache": "catalog:", - }, - "devDependencies": { - "@types/bun": "catalog:", - }, - }, "packages/mnemopi": { "name": "@oh-my-pi/pi-mnemopi", "version": "16.4.8", @@ -759,8 +759,6 @@ "@oh-my-pi/collab-web": ["@oh-my-pi/collab-web@workspace:packages/collab-web"], - "@oh-my-pi/harbor-manager": ["@oh-my-pi/harbor-manager@workspace:packages/harbor-manager"], - "@oh-my-pi/hashline": ["@oh-my-pi/hashline@workspace:packages/hashline"], "@oh-my-pi/omp-stats": ["@oh-my-pi/omp-stats@workspace:packages/stats"], @@ -773,6 +771,8 @@ "@oh-my-pi/pi-coding-agent": ["@oh-my-pi/pi-coding-agent@workspace:packages/coding-agent"], + "@oh-my-pi/pi-metaharness": ["@oh-my-pi/pi-metaharness@workspace:packages/metaharness"], + "@oh-my-pi/pi-mnemopi": ["@oh-my-pi/pi-mnemopi@workspace:packages/mnemopi"], "@oh-my-pi/pi-natives": ["@oh-my-pi/pi-natives@workspace:packages/natives"], diff --git a/package.json b/package.json index da161644f..3005d3142 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "collab:relay": "bun --cwd=packages/collab-web run relay", "collab:mock-host": "bun --cwd=packages/collab-web run mock-host", "collab:web:build": "bun --cwd=packages/collab-web run build", - "hmgr": "bun --cwd=packages/harbor-manager run dev", + "meta": "bun --cwd=packages/metaharness run dev", "claude:trace": "bun scripts/claude-trace.ts", "build": "bun run --workspaces --if-present build", "build:native": "bun --cwd=packages/natives run build", diff --git a/packages/harbor-manager/README.md b/packages/metaharness/README.md similarity index 84% rename from packages/harbor-manager/README.md rename to packages/metaharness/README.md index 85d614ca2..57330fe4c 100644 --- a/packages/harbor-manager/README.md +++ b/packages/metaharness/README.md @@ -1,4 +1,4 @@ -# @oh-my-pi/harbor-manager +# @oh-my-pi/pi-metaharness One manager for repository benchmarks. Harbor, TypeScript edit, and SnapCompact runs use the same experiment → run → trace model, SQLite store, REST/SSE API, @@ -30,8 +30,19 @@ bun run serve --port 4700 ## Server - `GET /` — experiments, runs, normalized traces, and a launch form for every benchmark. -- `GET /api/experiments` — experiment summaries across all benchmark types. -- `GET /api/runs` — uniform run rows with benchmark, score, progress, spend, and tokens. +- `GET /api/experiments[?q=]` — experiment summaries across all benchmark types + (`q` filters by id/goal substring). +- `POST /api/experiments` — register an experiment before its first arm. Body + `{ "id": "sb2", "goal": "..." }`; the id is the dash-free token job names + group under (`sb2-n8` → experiment `sb2`). +- `GET /api/experiments/:id` — arms, per-task matrix, and calibrated projections. +- `PUT /api/experiments/:id` — update the goal and per-run role/note/label. +- `POST /api/experiments/:id/arms` — launch a comparable arm; sample + config + inherited from a sibling. +- `DELETE /api/experiments/:id` — delete every arm (DB rows **and** job dirs) + plus the goal row; rejected while any arm is running. +- `GET /api/runs[?experiment=&status=&benchmark=]` — uniform run rows with + benchmark, score, progress, spend, and tokens. - `POST /api/runs` — launch through a benchmark adapter. Body: ```json @@ -51,7 +62,10 @@ bun run serve --port 4700 `include`, `timeoutMultiplier`, and `prewalk`; edit uses `include` as task IDs; SnapCompact uses `conditions` and treats `tasks` as the passage limit. - `GET /api/runs/:name` — `{ run, traces }` (syncs native artifacts on read). -- `DELETE /api/runs/:name` — cancel a manager-launched run. +- `POST /api/runs/:name/cancel` — cancel a manager-launched run. +- `DELETE /api/runs/:name` — permanently delete a finished run (DB row **and** + job dir; a surviving dir would be re-discovered on restart); rejected while + the run is live. - `POST /api/runs/:name/resume` — resume an incomplete harbor run in place: completed trials (and their spend) are reused, interrupted/pending trials re-run, and errored trials retried (body `{ "filterErrorTypes": [...] }` @@ -62,7 +76,7 @@ bun run serve --port 4700 - `GET /api/runs/:name/traces/:trace[?raw=1]` — normalized or native trace. - `GET /api/events` — SSE stream of run-list snapshots (sent on change). -State lives in `/_manager/harbor-manager.sqlite`; the filesystem +State lives in `/_manager/metaharness.sqlite`; the filesystem stays the source of truth and historical CLI runs are auto-discovered. ## Harbor runner options (excerpt) diff --git a/packages/harbor-manager/adapters/edit/bun-imports.d.ts b/packages/metaharness/adapters/edit/bun-imports.d.ts similarity index 100% rename from packages/harbor-manager/adapters/edit/bun-imports.d.ts rename to packages/metaharness/adapters/edit/bun-imports.d.ts diff --git a/packages/harbor-manager/adapters/edit/cli.ts b/packages/metaharness/adapters/edit/cli.ts similarity index 98% rename from packages/harbor-manager/adapters/edit/cli.ts rename to packages/metaharness/adapters/edit/cli.ts index dbbe8ce04..eff9e91de 100644 --- a/packages/harbor-manager/adapters/edit/cli.ts +++ b/packages/metaharness/adapters/edit/cli.ts @@ -11,7 +11,7 @@ import { type BenchmarkConfig, runBenchmark } from "./runner"; const EDIT_PACKAGE = path.resolve(import.meta.dir, "..", "..", "..", "typescript-edit-benchmark"); async function extractFixtures(): Promise<{ dir: string; temp: TempDir }> { - const temp = await TempDir.create("@harbor-edit-fixtures-"); + const temp = await TempDir.create("@metaharness-edit-fixtures-"); const archive = new Bun.Archive(await Bun.file(path.join(EDIT_PACKAGE, "fixtures.tar.gz")).arrayBuffer()); for (const [filePath, file] of await archive.files()) { await Bun.write(path.join(temp.path(), filePath), file); diff --git a/packages/harbor-manager/adapters/edit/prompts/benchmark-retry.md b/packages/metaharness/adapters/edit/prompts/benchmark-retry.md similarity index 100% rename from packages/harbor-manager/adapters/edit/prompts/benchmark-retry.md rename to packages/metaharness/adapters/edit/prompts/benchmark-retry.md diff --git a/packages/harbor-manager/adapters/edit/prompts/benchmark-system.md b/packages/metaharness/adapters/edit/prompts/benchmark-system.md similarity index 100% rename from packages/harbor-manager/adapters/edit/prompts/benchmark-system.md rename to packages/metaharness/adapters/edit/prompts/benchmark-system.md diff --git a/packages/harbor-manager/adapters/edit/prompts/benchmark-task.md b/packages/metaharness/adapters/edit/prompts/benchmark-task.md similarity index 100% rename from packages/harbor-manager/adapters/edit/prompts/benchmark-task.md rename to packages/metaharness/adapters/edit/prompts/benchmark-task.md diff --git a/packages/harbor-manager/adapters/edit/report.ts b/packages/metaharness/adapters/edit/report.ts similarity index 100% rename from packages/harbor-manager/adapters/edit/report.ts rename to packages/metaharness/adapters/edit/report.ts diff --git a/packages/harbor-manager/adapters/edit/runner.test.ts b/packages/metaharness/adapters/edit/runner.test.ts similarity index 100% rename from packages/harbor-manager/adapters/edit/runner.test.ts rename to packages/metaharness/adapters/edit/runner.test.ts diff --git a/packages/harbor-manager/adapters/edit/runner.ts b/packages/metaharness/adapters/edit/runner.ts similarity index 100% rename from packages/harbor-manager/adapters/edit/runner.ts rename to packages/metaharness/adapters/edit/runner.ts diff --git a/packages/harbor-manager/adapters/edit/tsconfig.json b/packages/metaharness/adapters/edit/tsconfig.json similarity index 100% rename from packages/harbor-manager/adapters/edit/tsconfig.json rename to packages/metaharness/adapters/edit/tsconfig.json diff --git a/packages/harbor-manager/agent/omp_local.py b/packages/metaharness/agent/omp_local.py similarity index 99% rename from packages/harbor-manager/agent/omp_local.py rename to packages/metaharness/agent/omp_local.py index c133fbfc5..46c4fb9d0 100644 --- a/packages/harbor-manager/agent/omp_local.py +++ b/packages/metaharness/agent/omp_local.py @@ -433,7 +433,7 @@ class OmpLocal(BaseInstalledAgent): ) def _generate_models_yaml(self) -> str: - lines = ["# Generated by harbor-manager runner — routes auth via host gateway.", "providers:"] + lines = ["# Generated by metaharness runner — routes auth via host gateway.", "providers:"] for provider in self._gateway_providers: lines += [ f" {provider}:", @@ -450,7 +450,7 @@ class OmpLocal(BaseInstalledAgent): web_search can't authenticate through the gateway, so it's off by default. """ lines = [ - "# Generated by harbor-manager runner.", + "# Generated by metaharness runner.", "web_search:", f" enabled: {'true' if self._web_search else 'false'}", ] diff --git a/packages/harbor-manager/package.json b/packages/metaharness/package.json similarity index 92% rename from packages/harbor-manager/package.json rename to packages/metaharness/package.json index 3b7e14fee..b50e9c348 100644 --- a/packages/harbor-manager/package.json +++ b/packages/metaharness/package.json @@ -1,7 +1,7 @@ { "type": "module", "private": true, - "name": "@oh-my-pi/harbor-manager", + "name": "@oh-my-pi/pi-metaharness", "version": "0.0.1", "description": "Unified benchmark runners plus Harbor run storage, REST/SSE APIs, and a live web dashboard", "homepage": "https://omp.sh", @@ -10,10 +10,10 @@ "repository": { "type": "git", "url": "git+https://github.com/can1357/oh-my-pi.git", - "directory": "packages/harbor-manager" + "directory": "packages/metaharness" }, "bin": { - "harbor-manager": "src/server.ts" + "metaharness": "src/server.ts" }, "scripts": { "check": "biome check . && bun run check:types", diff --git a/packages/harbor-manager/src/adapters/snapcompact.py b/packages/metaharness/src/adapters/snapcompact.py similarity index 100% rename from packages/harbor-manager/src/adapters/snapcompact.py rename to packages/metaharness/src/adapters/snapcompact.py diff --git a/packages/harbor-manager/src/benchmarks.test.ts b/packages/metaharness/src/benchmarks.test.ts similarity index 100% rename from packages/harbor-manager/src/benchmarks.test.ts rename to packages/metaharness/src/benchmarks.test.ts diff --git a/packages/harbor-manager/src/benchmarks.ts b/packages/metaharness/src/benchmarks.ts similarity index 100% rename from packages/harbor-manager/src/benchmarks.ts rename to packages/metaharness/src/benchmarks.ts diff --git a/packages/harbor-manager/src/experiments.test.ts b/packages/metaharness/src/experiments.test.ts similarity index 100% rename from packages/harbor-manager/src/experiments.test.ts rename to packages/metaharness/src/experiments.test.ts diff --git a/packages/harbor-manager/src/experiments.ts b/packages/metaharness/src/experiments.ts similarity index 94% rename from packages/harbor-manager/src/experiments.ts rename to packages/metaharness/src/experiments.ts index 423084c1f..5d5a59b1a 100644 --- a/packages/harbor-manager/src/experiments.ts +++ b/packages/metaharness/src/experiments.ts @@ -204,7 +204,7 @@ export function buildExperiments(store: RunStore): ExperimentSummary[] { for (const [id, runs] of groups) { out.push({ id, - goal: store.getExperimentGoal(id), + goal: store.getExperimentMeta(id)?.goal ?? "", arms: runs.length, runningArms: runs.filter(r => r.status === "running").length, datasets: [...new Set(runs.map(r => r.dataset).filter(Boolean))], @@ -218,6 +218,26 @@ export function buildExperiments(store: RunStore): ExperimentSummary[] { updatedAt: Math.max(...runs.map(r => r.finishedAt ?? Date.now())), }); } + // Registered-but-empty experiments (created via POST /api/experiments, no + // arms yet) are still browsable: zeroed rollups, goal from the meta row. + for (const meta of store.listExperimentMeta()) { + if (groups.has(meta.id)) continue; + out.push({ + id: meta.id, + goal: meta.goal, + arms: 0, + runningArms: 0, + datasets: [], + nTotal: 0, + done: 0, + pass: 0, + fail: 0, + error: 0, + costUsd: 0, + createdAt: meta.updatedAt, + updatedAt: meta.updatedAt, + }); + } out.sort((a, b) => b.updatedAt - a.updatedAt); return out; } @@ -259,7 +279,11 @@ export function pickMergedTrials(traces: TraceRow[]): TraceRow[] { export function experimentDetail(store: RunStore, id: string): ExperimentDetail | null { const runs = store.listRuns().filter(r => experimentOf(r.jobName) === id); - if (runs.length === 0) return null; + if (runs.length === 0) { + // Registered but armless (POST /api/experiments): still readable. + const meta = store.getExperimentMeta(id); + return meta ? { id, goal: meta.goal, arms: [], tasks: [], matrix: {} } : null; + } // One row per CANONICAL arm: `-fix`/`-backfill` re-runs merge into their // base arm — per-task best trial, summed spend. const groups = new Map(); @@ -347,5 +371,5 @@ export function experimentDetail(store: RunStore, id: string): ExperimentDetail // "reference rows, then treatments". const roleRank = (role: string) => (role === "baseline" ? 0 : role === "variant" ? 1 : 2); arms.sort((a, b) => roleRank(a.run.role) - roleRank(b.run.role) || a.arm.localeCompare(b.arm)); - return { id, goal: store.getExperimentGoal(id), arms, tasks: [...tasks].sort(), matrix }; + return { id, goal: store.getExperimentMeta(id)?.goal ?? "", arms, tasks: [...tasks].sort(), matrix }; } diff --git a/packages/harbor-manager/src/launch-args.ts b/packages/metaharness/src/launch-args.ts similarity index 100% rename from packages/harbor-manager/src/launch-args.ts rename to packages/metaharness/src/launch-args.ts diff --git a/packages/harbor-manager/src/manager.test.ts b/packages/metaharness/src/manager.test.ts similarity index 80% rename from packages/harbor-manager/src/manager.test.ts rename to packages/metaharness/src/manager.test.ts index 4f709d82d..70044461e 100644 --- a/packages/harbor-manager/src/manager.test.ts +++ b/packages/metaharness/src/manager.test.ts @@ -19,7 +19,7 @@ afterEach(() => { }); function makeJobsDir(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "harbor-manager-test-")); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "metaharness-test-")); cleanups.push(() => fs.rmSync(dir, { recursive: true, force: true })); return dir; } @@ -238,10 +238,13 @@ describe("ManagerServer API", () => { }); expect(badLaunch.status).toBe(400); - const cancelUnknown = (await (await fetch(`${base}/api/runs/nope`, { method: "DELETE" })).json()) as { + const cancelUnknown = (await (await fetch(`${base}/api/runs/nope/cancel`, { method: "POST" })).json()) as { cancelled: boolean; }; expect(cancelUnknown.cancelled).toBe(false); + + const deleteUnknown = await fetch(`${base}/api/runs/nope`, { method: "DELETE" }); + expect(deleteUnknown.status).toBe(404); }); it("serves edit and SnapCompact metrics and native traces through one API", async () => { @@ -366,6 +369,100 @@ describe("ManagerServer API", () => { expect(await resumeError("job-live")).toMatch(/already running/); expect(await resumeError("job-bare")).toMatch(/no harbor config.json/); }); + + it("experiment CRUD: create is browsable, delete removes rows + job dirs, live arms are protected", async () => { + const jobsDir = makeJobsDir(); + const manager = new ManagerServer(jobsDir); + // Two finished arms of experiment `crud` and one live run in a different experiment. + for (const jobName of ["crud-base", "crud-treat"]) { + manager.store.registerLaunch({ + benchmark: "harbor", + jobName, + dataset: "terminal-bench@2.0", + agent: "omp", + models: ["m/x"], + pid: process.pid, + }); + manager.store.markExit(jobName, 0); + } + manager.store.registerLaunch({ + benchmark: "harbor", + jobName: "live-run", + dataset: "terminal-bench@2.0", + agent: "omp", + models: ["m/x"], + pid: process.pid, + }); + const server = manager.start(0); + cleanups.push(() => { + void manager.stop(); + }); + const base = `http://localhost:${server.port}`; + + // Create: registered id is browsable before any run exists. + const created = await fetch(`${base}/api/experiments`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ id: "fresh", goal: "does X beat Y?" }), + }); + expect(created.status).toBe(201); + const list = (await (await fetch(`${base}/api/experiments`)).json()) as Array<{ + id: string; + goal: string; + arms: number; + }>; + const fresh = list.find(e => e.id === "fresh"); + expect(fresh).toMatchObject({ goal: "does X beat Y?", arms: 0 }); + const freshDetail = (await (await fetch(`${base}/api/experiments/fresh`)).json()) as { + goal: string; + arms: unknown[]; + }; + expect(freshDetail).toMatchObject({ goal: "does X beat Y?", arms: [] }); + + // Create: dashed / empty ids can never own a run — rejected. + for (const id of ["bad-id", ""]) { + const res = await fetch(`${base}/api/experiments`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ id }), + }); + expect(res.status).toBe(400); + } + + // Browse: list filters. + const filtered = (await (await fetch(`${base}/api/runs?experiment=crud`)).json()) as Array<{ + jobName: string; + }>; + expect(filtered.map(r => r.jobName).sort()).toEqual(["crud-base", "crud-treat"]); + const running = (await (await fetch(`${base}/api/runs?status=running`)).json()) as Array<{ + jobName: string; + }>; + expect(running.map(r => r.jobName)).toEqual(["live-run"]); + const q = (await (await fetch(`${base}/api/experiments?q=fresh`)).json()) as Array<{ id: string }>; + expect(q.map(e => e.id)).toEqual(["fresh"]); + + // Delete run: live runs are protected, finished runs vanish from DB and disk. + const liveDelete = await fetch(`${base}/api/runs/live-run`, { method: "DELETE" }); + expect(liveDelete.status).toBe(400); + const runDelete = await fetch(`${base}/api/runs/crud-treat`, { method: "DELETE" }); + expect(runDelete.status).toBe(200); + expect(fs.existsSync(path.join(jobsDir, "crud-treat"))).toBe(false); + expect(manager.store.getRun("crud-treat")).toBeNull(); + + // Delete experiment: remaining arm rows + dirs + goal row all go; 404 after. + const expDelete = (await (await fetch(`${base}/api/experiments/crud`, { method: "DELETE" })).json()) as { + deletedRuns: string[]; + }; + expect(expDelete.deletedRuns).toEqual(["crud-base"]); + expect(fs.existsSync(path.join(jobsDir, "crud-base"))).toBe(false); + expect((await fetch(`${base}/api/experiments/crud`)).status).toBe(404); + expect((await fetch(`${base}/api/experiments/unknown`, { method: "DELETE" })).status).toBe(404); + + // Delete experiment with a live arm: refused, nothing removed. + const liveExpDelete = await fetch(`${base}/api/experiments/live`, { method: "DELETE" }); + expect(liveExpDelete.status).toBe(400); + expect(manager.store.getRun("live-run")).not.toBeNull(); + }); }); describe("resolveArmLaunch", () => { diff --git a/packages/harbor-manager/src/runner.test.ts b/packages/metaharness/src/runner.test.ts similarity index 100% rename from packages/harbor-manager/src/runner.test.ts rename to packages/metaharness/src/runner.test.ts diff --git a/packages/harbor-manager/src/runner.ts b/packages/metaharness/src/runner.ts similarity index 99% rename from packages/harbor-manager/src/runner.ts rename to packages/metaharness/src/runner.ts index b832a3d63..1bb8ac385 100755 --- a/packages/harbor-manager/src/runner.ts +++ b/packages/metaharness/src/runner.ts @@ -14,9 +14,9 @@ import * as path from "node:path"; * process renders a live dashboard (progress / success% / spend / tokens / ETA) * by polling each trial's `result.json`. On completion it writes a markdown report. * - * harbor-manager harbor --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 - * harbor-manager harbor --agent oracle --tasks 2 # cheap pipeline smoke - * harbor-manager harbor --help + * metaharness harbor --model anthropic/claude-sonnet-4-6 --tasks 20 --concurrency 4 + * metaharness harbor --agent oracle --tasks 2 # cheap pipeline smoke + * metaharness harbor --help */ import type { Server } from "bun"; import { harborRunnerArgs, type LaunchRequest } from "./launch-args"; @@ -128,9 +128,9 @@ function defaultConfig(): Config { }; } -const HELP = `harbor-manager runner (local omp) +const HELP = `metaharness runner (local omp) -Usage: harbor-manager harbor [options] [-- ] +Usage: metaharness harbor [options] [-- ] Commands: cleanup Force-remove ALL leftover Harbor containers + networks, then exit @@ -1240,7 +1240,7 @@ function deriveProviders(cfg: Config): string[] { function writeModelsYaml(benchDir: string, cfg: Config): string { const providers = deriveProviders(cfg); - const lines = ["# Generated by harbor-manager — auth via host pm2 gateway.", "providers:"]; + const lines = ["# Generated by metaharness — auth via host pm2 gateway.", "providers:"]; for (const p of providers) { lines.push(` ${p}:`); lines.push(` baseUrl: ${cfg.gatewayUrl}`); diff --git a/packages/harbor-manager/src/server.ts b/packages/metaharness/src/server.ts similarity index 79% rename from packages/harbor-manager/src/server.ts rename to packages/metaharness/src/server.ts index 9c8045d51..17d31b513 100755 --- a/packages/harbor-manager/src/server.ts +++ b/packages/metaharness/src/server.ts @@ -1,16 +1,23 @@ #!/usr/bin/env bun /** - * harbor-manager server: REST + SSE API over the run store, static web + * metaharness server: REST + SSE API over the run store, static web * dashboard, and a launcher that spawns the CLI runner as a managed child. * * bun src/server.ts [--port 4700] [--jobs-dir ] * * API: - * GET /api/experiments → experiment summaries across all benchmarks - * GET /api/runs → RunRow[] + * GET /api/experiments[?q=] → experiment summaries across all benchmarks + * POST /api/experiments → register an experiment (id + goal) before its first arm + * GET /api/experiments/:id → experiment detail (arms, task matrix) + * PUT /api/experiments/:id → update goal + per-run role/note/label + * DELETE /api/experiments/:id → delete all arms (rows + job dirs) and the goal row + * POST /api/experiments/:id/arms → launch a comparable arm + * GET /api/runs[?experiment=&status=&benchmark=] → RunRow[] * POST /api/runs → launch any benchmark * GET /api/runs/:name → { run, traces } - * DELETE /api/runs/:name → cancel a managed run + * POST /api/runs/:name/cancel → cancel a managed run + * POST /api/runs/:name/resume → resume an incomplete harbor run + * DELETE /api/runs/:name → delete a finished run (row + job dir) * GET /api/runs/:name/traces/:trace → normalized trace * GET /api/events → SSE: run-list snapshots on change */ @@ -28,6 +35,13 @@ export interface ExperimentMetaUpdate { runs?: Record; } +/** POST /api/experiments body — pre-registers an experiment id with a goal. */ +export interface CreateExperimentRequest { + /** Dash-free token; runs group into it as `-` job names. */ + id: string; + goal?: string; +} + import indexHtml from "./web/index.html"; const REPO_ROOT = path.resolve(import.meta.dir, "..", "..", ".."); @@ -76,6 +90,24 @@ function parseServerArgs(argv: string[]): { port: number; jobsDir: string } { return { port, jobsDir }; } +/** Job names are single path segments; anything else could escape the jobs dir. */ +function assertSafeJobName(jobName: string): void { + if (!jobName || jobName === "." || jobName === ".." || /[/\\]/.test(jobName)) { + throw new Error(`invalid job name: ${jobName}`); + } +} + +/** True when `pid` names a live process (signal-0 probe). */ +function pidAlive(pid: number | null): boolean { + if (pid == null) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + /** * Resolve the launch request for a new arm added to an existing experiment. * Inherits the experiment's benchmark, dataset, and — crucially — the exact @@ -236,7 +268,17 @@ export class ManagerServer { return Response.json(BENCHMARK_DEFINITIONS); } if (p === "/api/experiments" && request.method === "GET") { - return Response.json(buildExperiments(this.#store)); + const q = url.searchParams.get("q")?.toLowerCase() ?? ""; + const experiments = buildExperiments(this.#store); + return Response.json( + q + ? experiments.filter(e => e.id.toLowerCase().includes(q) || e.goal.toLowerCase().includes(q)) + : experiments, + ); + } + if (p === "/api/experiments" && request.method === "POST") { + const body = (await request.json()) as CreateExperimentRequest; + return Response.json(this.createExperiment(body), { status: 201 }); } const expMatch = p.match(/^\/api\/experiments\/([^/]+)$/); if (expMatch) { @@ -245,6 +287,11 @@ export class ManagerServer { const body = (await request.json()) as ExperimentMetaUpdate; return Response.json(this.updateExperimentMeta(id, body)); } + if (request.method === "DELETE") { + const result = this.deleteExperiment(id); + if (!result) return Response.json({ error: "experiment not found" }, { status: 404 }); + return Response.json(result); + } const detail = experimentDetail(this.#store, id); if (!detail) return Response.json({ error: "experiment not found" }, { status: 404 }); return Response.json(detail); @@ -256,7 +303,14 @@ export class ManagerServer { return Response.json(this.addArm(id, body), { status: 201 }); } if (p === "/api/runs" && request.method === "GET") { - return Response.json(this.#store.listRuns()); + const experiment = url.searchParams.get("experiment"); + const status = url.searchParams.get("status"); + const benchmark = url.searchParams.get("benchmark"); + let runs = this.#store.listRuns(); + if (experiment) runs = runs.filter(r => experimentOf(r.jobName) === experiment); + if (status) runs = runs.filter(r => r.status === status); + if (benchmark) runs = runs.filter(r => r.benchmark === benchmark); + return Response.json(runs); } if (p === "/api/runs" && request.method === "POST") { const body = (await request.json()) as LaunchRequest; @@ -268,10 +322,17 @@ export class ManagerServer { const body = (await request.json().catch(() => ({}))) as { filterErrorTypes?: string[] }; return Response.json(this.resume(jobName, body), { status: 201 }); } + const cancelMatch = p.match(/^\/api\/runs\/([^/]+)\/cancel$/); + if (cancelMatch && request.method === "POST") { + return Response.json(this.cancel(decodeURIComponent(cancelMatch[1]))); + } const runMatch = p.match(/^\/api\/runs\/([^/]+)$/); if (runMatch) { const jobName = decodeURIComponent(runMatch[1]); - if (request.method === "DELETE") return Response.json(this.cancel(jobName)); + if (request.method === "DELETE") { + if (!this.deleteRun(jobName)) return Response.json({ error: "run not found" }, { status: 404 }); + return Response.json({ jobName, deleted: true }); + } const run = this.#store.syncRun(jobName); if (!run) return Response.json({ error: "run not found" }, { status: 404 }); return Response.json({ run, traces: this.#store.listTraces(jobName) }); @@ -386,16 +447,7 @@ export class ManagerServer { // Trust liveness, not the recorded status: a runner killed while a // previous server instance owned it leaves a stale `running` row with a // dead (or null) pid and nobody to fire markExit. - const pidAlive = (pid: number | null): boolean => { - if (pid == null) return false; - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } - }; - if (this.#children.has(jobName) || (run.status === "running" && pidAlive(run.pid))) { + if (this.#runLive(run)) { throw new Error(`run ${jobName} is already running`); } if (run.status === "running") this.#store.markExit(jobName, null, true); @@ -461,18 +513,80 @@ export class ManagerServer { return proc.pid; } + /** Liveness check that survives manager restarts: managed child, or a running row with a live pid. */ + #runLive(run: RunRow): boolean { + return this.#children.has(run.jobName) || (run.status === "running" && pidAlive(run.pid)); + } + + /** Register an experiment id (with an optional goal) so it is browsable before its first arm. */ + createExperiment(req: CreateExperimentRequest): { id: string; goal: string } { + const id = req.id?.trim() ?? ""; + // Dashes are structurally impossible: `experimentOf` groups job names by + // the token before the first dash, so a dashed id could never own a run. + if (!/^[A-Za-z0-9_.]+$/.test(id)) { + throw new Error("experiment id must be a non-empty token of [A-Za-z0-9_.] (runs group as `-`)"); + } + const goal = req.goal ?? this.#store.getExperimentMeta(id)?.goal ?? ""; + this.#store.setExperimentGoal(id, goal); + return { id, goal }; + } + /** Apply goal + per-run role/note metadata; used by the UI and for backfill. */ updateExperimentMeta(id: string, update: ExperimentMetaUpdate): { id: string; updatedRuns: string[] } { if (update.goal !== undefined) this.#store.setExperimentGoal(id, update.goal); const updatedRuns: string[] = []; - for (const [jobName, meta] of Object.entries(update.runs ?? {})) { + for (const jobName in update.runs) { if (experimentOf(jobName) !== id) continue; - if (this.#store.setRunMeta(jobName, meta)) updatedRuns.push(jobName); + if (this.#store.setRunMeta(jobName, update.runs[jobName])) updatedRuns.push(jobName); } this.#tick(); return { id, updatedRuns }; } + /** + * Delete an experiment: every arm's DB row, job dir, and manager log, plus + * the goal row. Refuses while any arm is live (cancel first — deleting a + * job dir under a writing runner would corrupt it). Returns null when the + * id names neither runs nor a registered experiment. + */ + deleteExperiment(id: string): { id: string; deletedRuns: string[] } | null { + const runs = this.#store.listRuns().filter(r => experimentOf(r.jobName) === id); + if (runs.length === 0 && !this.#store.getExperimentMeta(id)) return null; + const live = runs.filter(r => this.#runLive(r)); + if (live.length > 0) { + throw new Error( + `experiment ${id} has running arms (${live.map(r => r.jobName).join(", ")}); cancel them first`, + ); + } + for (const run of runs) this.#destroyRun(run.jobName); + this.#store.deleteExperimentMeta(id); + this.#tick(); + return { id, deletedRuns: runs.map(r => r.jobName) }; + } + + /** + * Permanently delete a run: DB row + trials, job dir, and manager log. + * Disk removal is not optional — discover() would resurrect a surviving + * job dir as a fresh row on the next restart. Refuses while the run is + * live; returns false when the run is unknown. + */ + deleteRun(jobName: string): boolean { + const run = this.#store.getRun(jobName); + if (!run) return false; + if (this.#runLive(run)) throw new Error(`run ${jobName} is running; cancel it first`); + this.#destroyRun(jobName); + this.#tick(); + return true; + } + + /** Remove a run's DB rows and on-disk artifacts (job dir + manager log). */ + #destroyRun(jobName: string): void { + assertSafeJobName(jobName); + this.#store.deleteRun(jobName); + fs.rmSync(path.join(this.jobsDir, jobName), { recursive: true, force: true }); + fs.rmSync(path.join(this.jobsDir, "_manager", "logs", `${jobName}.log`), { force: true }); + } + /** Add a comparable arm to an existing experiment, inheriting its sample + config. */ addArm(experimentId: string, req: AddArmRequest): { jobName: string; pid: number } { return this.launch(resolveArmLaunch(this.#store, experimentId, req)); @@ -642,20 +756,20 @@ if (import.meta.main) { // `bun --hot` re-evaluates this module in-place: retire the previous // instance first, or its sync ticker and sqlite connection leak per reload. const host = globalThis as typeof globalThis & { - __harborManagerServer?: ManagerServer; - __harborManagerHooks?: boolean; + __metaharnessServer?: ManagerServer; + __metaharnessHooks?: boolean; }; - await host.__harborManagerServer?.stop(); + await host.__metaharnessServer?.stop(); const { port, jobsDir } = parseServerArgs(process.argv.slice(2)); const manager = new ManagerServer(jobsDir); - host.__harborManagerServer = manager; + host.__metaharnessServer = manager; const server = manager.start(port); - process.stdout.write(`harbor-manager listening on http://localhost:${server.port} (jobs: ${jobsDir})\n`); + process.stdout.write(`metaharness listening on http://localhost:${server.port} (jobs: ${jobsDir})\n`); // Process-wide hooks register once; `--hot` re-evals reuse them via `host`. - if (!host.__harborManagerHooks) { - host.__harborManagerHooks = true; + if (!host.__metaharnessHooks) { + host.__metaharnessHooks = true; const shutdown = async () => { - await host.__harborManagerServer?.stop(); + await host.__metaharnessServer?.stop(); process.exit(0); }; process.on("SIGINT", shutdown); diff --git a/packages/harbor-manager/src/store.ts b/packages/metaharness/src/store.ts similarity index 92% rename from packages/harbor-manager/src/store.ts rename to packages/metaharness/src/store.ts index c57d96670..c06f24fc7 100644 --- a/packages/harbor-manager/src/store.ts +++ b/packages/metaharness/src/store.ts @@ -72,6 +72,13 @@ export interface TraceRow { tracePath: string | null; } +/** Row in the `experiments` table: goal metadata keyed by experiment id. */ +export interface ExperimentMeta { + id: string; + goal: string; + updatedAt: number; +} + export interface LaunchRecord { benchmark: BenchmarkKind; jobName: string; @@ -180,7 +187,7 @@ export class RunStore { constructor(jobsDir: string, dbPath?: string) { this.jobsDir = jobsDir; fs.mkdirSync(path.join(jobsDir, "_manager"), { recursive: true }); - this.#db = new Database(dbPath ?? path.join(jobsDir, "_manager", "harbor-manager.sqlite")); + this.#db = new Database(dbPath ?? path.join(jobsDir, "_manager", "metaharness.sqlite")); this.#db.run("PRAGMA busy_timeout = 5000"); enableWal(this.#db); this.#db.run(SCHEMA); @@ -258,9 +265,39 @@ export class RunStore { .run(id, goal, Date.now()); } - getExperimentGoal(id: string): string { - const row = this.#db.query("SELECT goal FROM experiments WHERE id = ?").get(id) as { goal: string } | null; - return row?.goal ?? ""; + /** Stored experiment metadata, or null when the id was never registered. */ + getExperimentMeta(id: string): ExperimentMeta | null { + const row = this.#db.query("SELECT id, goal, updated_at FROM experiments WHERE id = ?").get(id) as { + id: string; + goal: string; + updated_at: number; + } | null; + return row ? { id: row.id, goal: row.goal, updatedAt: row.updated_at } : null; + } + + /** Every registered experiment row, newest first. */ + listExperimentMeta(): ExperimentMeta[] { + const rows = this.#db + .query("SELECT id, goal, updated_at FROM experiments ORDER BY updated_at DESC") + .all() as Array<{ + id: string; + goal: string; + updated_at: number; + }>; + return rows.map(r => ({ id: r.id, goal: r.goal, updatedAt: r.updated_at })); + } + + /** Drop the experiment metadata row (run rows are deleted separately via deleteRun). */ + deleteExperimentMeta(id: string): void { + this.#db.query("DELETE FROM experiments WHERE id = ?").run(id); + } + + /** Delete a run row and its trials; returns false when the run is unknown. */ + deleteRun(jobName: string): boolean { + if (!this.getRun(jobName)) return false; + this.#db.query("DELETE FROM trials WHERE job_name = ?").run(jobName); + this.#db.query("DELETE FROM runs WHERE job_name = ?").run(jobName); + return true; } /** Set role/note/label metadata on an existing run row. */ diff --git a/packages/harbor-manager/src/web/app.tsx b/packages/metaharness/src/web/app.tsx similarity index 99% rename from packages/harbor-manager/src/web/app.tsx rename to packages/metaharness/src/web/app.tsx index 2b3318a60..3afe3a0be 100644 --- a/packages/harbor-manager/src/web/app.tsx +++ b/packages/metaharness/src/web/app.tsx @@ -1,5 +1,5 @@ /** - * harbor-manager dashboard. + * metaharness dashboard. * * Views (hash-routed): * #/ experiments index — runs grouped by job-name prefix @@ -1696,7 +1696,7 @@ function RunsPage({ selected }: { selected: string | null }) { if (el) el.scrollTop = el.scrollHeight; }, [traceData]); const cancel = useCallback(async (name: string) => { - if (confirm(`stop ${name}?`)) await fetch(`/api/runs/${encodeURIComponent(name)}`, { method: "DELETE" }); + if (confirm(`stop ${name}?`)) await fetch(`/api/runs/${encodeURIComponent(name)}/cancel`, { method: "POST" }); }, []); const resume = useCallback(async (name: string) => { if (!confirm(`resume ${name}? completed trials are kept; interrupted, pending, and errored ones re-run`)) return; @@ -1966,7 +1966,7 @@ function App() { return ( <>
-

harbor-manager

+

metaharness