From 96ba1a99fc50889c4c3c63ad3bfcc40ed73bd3aa Mon Sep 17 00:00:00 2001 From: can1357 Date: Mon, 13 Jul 2026 03:58:06 +0200 Subject: [PATCH] feat(harbor-manager): implemented arm orchestration and vite dev harness - Introduced `resolveArmLaunch` to inherit dataset, task samples, and configurations from existing experiment arms for consistent comparisons. - Added a POST `/api/experiments/:id/arms` endpoint to programmatically launch new arms within an experiment. - Created `scripts/dev.ts` and `vite.config.ts` to provide a unified development environment with React Fast Refresh and API proxying. - Integrated an "Add Arm" UI component in the dashboard to allow user-driven experimental variants. - Configured Vite to proxy requests to the Bun API server, improving iteration speed during dashboard development. --- bun.lock | 14 +++- package.json | 1 + packages/harbor-manager/package.json | 5 +- packages/harbor-manager/scripts/dev.ts | 39 +++++++++ packages/harbor-manager/src/manager.test.ts | 58 +++++++++++++- packages/harbor-manager/src/server.ts | 88 ++++++++++++++++++++- packages/harbor-manager/src/web/app.tsx | 88 ++++++++++++++++++++- packages/harbor-manager/src/web/index.html | 2 +- packages/harbor-manager/vite.config.ts | 24 ++++++ 9 files changed, 312 insertions(+), 7 deletions(-) create mode 100755 packages/harbor-manager/scripts/dev.ts create mode 100644 packages/harbor-manager/vite.config.ts diff --git a/bun.lock b/bun.lock index b9ec794d5..ed47a4f51 100644 --- a/bun.lock +++ b/bun.lock @@ -164,6 +164,8 @@ "@types/d3-shape": "^3.1.7", "@types/react": "^19.1.0", "@types/react-dom": "^19.1.0", + "@vitejs/plugin-react": "^5.0.4", + "vite": "catalog:", }, }, "packages/hashline": { @@ -471,6 +473,10 @@ "@babel/plugin-syntax-jsx": ["@babel/plugin-syntax-jsx@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-TSu8+mHCoEaaCDEZ0I3+6mvTBYR4PCxQwf2z9/r5Tbztv6NaLR3B9thGTTxX2WGuGHJqRiAbKPeGTJ5XWXVg6A=="], + "@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw=="], + + "@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.29.7", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q=="], + "@babel/template": ["@babel/template@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7" } }, "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg=="], "@babel/traverse": ["@babel/traverse@7.29.7", "", { "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", "@babel/helper-globals": "^7.29.7", "@babel/parser": "^7.29.7", "@babel/template": "^7.29.7", "@babel/types": "^7.29.7", "debug": "^4.3.1" } }, "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw=="], @@ -867,7 +873,7 @@ "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.1.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA=="], - "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-rc.3", "", {}, "sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q=="], "@sindresorhus/is": ["@sindresorhus/is@4.6.0", "", {}, "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw=="], @@ -953,6 +959,8 @@ "@typescript/vfs": ["@typescript/vfs@1.6.4", "", { "dependencies": { "debug": "^4.4.3" }, "peerDependencies": { "typescript": "*" } }, "sha512-PJFXFS4ZJKiJ9Qiuix6Dz/OwEIqHD7Dme1UwZhTK11vR+5dqW2ACbdndWQexBzCx+CPuMe5WBYQWCsFyGlQLlQ=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@5.2.0", "", { "dependencies": { "@babel/core": "^7.29.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-rc.3", "@types/babel__core": "^7.20.5", "react-refresh": "^0.18.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw=="], + "@xmldom/xmldom": ["@xmldom/xmldom@0.8.13", "", {}, "sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw=="], "@xterm/headless": ["@xterm/headless@6.0.0", "", {}, "sha512-5Yj1QINYCyzrZtf8OFIHi47iQtI+0qYFPHmouEfG8dHNxbZ9Tb9YGSuLcsEwj9Z+OL75GJqPyJbyoFer80a2Hw=="], @@ -1377,6 +1385,8 @@ "react-dom": ["react-dom@19.2.7", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.7" } }, "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="], + "react-refresh": ["react-refresh@0.18.0", "", {}, "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw=="], + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], "regexp-tree": ["regexp-tree@0.1.27", "", { "bin": { "regexp-tree": "bin/regexp-tree" } }, "sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA=="], @@ -1629,6 +1639,8 @@ "roarr/sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], + "rolldown/@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], + "string_decoder/safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], "wrap-ansi/string-width": ["string-width@8.2.2", "", { "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" } }, "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg=="], diff --git a/package.json b/package.json index 91b40ea2a..7e88c3232 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,7 @@ "collab:relay": "bun --cwd=packages/collab-web run relay", "collab:mock-host": "bun --cwd=packages/collab-web run mock-host", "collab:web:build": "bun --cwd=packages/collab-web run build", + "hmgr": "bun --cwd=packages/harbor-manager run dev", "claude:trace": "bun scripts/claude-trace.ts", "build": "bun run --workspaces --if-present build", "build:native": "bun --cwd=packages/natives run build", diff --git a/packages/harbor-manager/package.json b/packages/harbor-manager/package.json index c54a1be65..d6e7af6e7 100644 --- a/packages/harbor-manager/package.json +++ b/packages/harbor-manager/package.json @@ -21,6 +21,7 @@ "lint": "biome lint .", "start": "bun run src/server.ts", "serve": "bun run src/server.ts", + "dev": "bun scripts/dev.ts", "test": "bun test" }, "dependencies": { @@ -44,7 +45,9 @@ "@types/d3-scale": "^4.0.9", "@types/d3-shape": "^3.1.7", "@types/react": "^19.1.0", - "@types/react-dom": "^19.1.0" + "@types/react-dom": "^19.1.0", + "@vitejs/plugin-react": "^5.0.4", + "vite": "catalog:" }, "engines": { "bun": ">=1.3.14" diff --git a/packages/harbor-manager/scripts/dev.ts b/packages/harbor-manager/scripts/dev.ts new file mode 100755 index 000000000..5cd7c102b --- /dev/null +++ b/packages/harbor-manager/scripts/dev.ts @@ -0,0 +1,39 @@ +#!/usr/bin/env bun +/** + * Dev harness: runs the Bun API server (auto-restarting on server edits via + * `--watch`) and a Vite dev server (React Fast Refresh for the dashboard) + * together, tearing both down on one Ctrl-C. Vite proxies `/api` to the API + * server; the shared port travels through `HARBOR_API_PORT`. + * + * Extra args pass through to the API server: + * bun run dev -- --port 4700 --jobs-dir ../../runs/harbor + * + * Vite runs under Node (its bin shebang), the API under Bun; only the frontend + * hot-reloads in place, while server-side changes trigger a fast `--watch` restart. + */ +const args = Bun.argv.slice(2); +const portIndex = args.indexOf("--port"); +const apiPort = portIndex >= 0 ? (args[portIndex + 1] ?? "4700") : "4700"; +process.env.HARBOR_API_PORT = apiPort; + +const io = { stdout: "inherit", stderr: "inherit", stdin: "inherit", env: { ...process.env } } as const; +const api = Bun.spawn(["bun", "--watch", "src/server.ts", ...args], io); +const web = Bun.spawn(["vite"], io); + +let stopping = false; +const stop = (): void => { + if (stopping) return; + stopping = true; + try { + api.kill(); + } catch {} + try { + web.kill(); + } catch {} +}; +process.on("SIGINT", stop); +process.on("SIGTERM", stop); + +await Promise.race([api.exited, web.exited]); +stop(); +process.exit(0); diff --git a/packages/harbor-manager/src/manager.test.ts b/packages/harbor-manager/src/manager.test.ts index 3d26fb274..4d29296a8 100644 --- a/packages/harbor-manager/src/manager.test.ts +++ b/packages/harbor-manager/src/manager.test.ts @@ -3,7 +3,7 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { experimentDetail } from "./experiments"; -import { ManagerServer } from "./server"; +import { ManagerServer, resolveArmLaunch } from "./server"; import { RunStore } from "./store"; /** @@ -293,3 +293,59 @@ describe("ManagerServer API", () => { expect(snapTrace.entries.map(entry => entry.kind)).toEqual(["question", "answer", "reference"]); }); }); + +describe("resolveArmLaunch", () => { + it("inherits dataset + exact task sample + scale from a sibling arm", () => { + const store = new RunStore(makeJobsDir()); + cleanups.push(() => store.close()); + store.registerLaunch({ + benchmark: "harbor", + jobName: "exp-base", + dataset: "swe-bench/swe-bench-verified", + agent: "omp", + models: ["anthropic/claude-opus-4-8"], + pid: 4321, + role: "baseline", + config: { + include: ["astropy__astropy-1", "django__django-2", "sympy__sympy-3"], + tasks: 3, + concurrency: 4, + timeoutMultiplier: 2, + }, + }); + + const launch = resolveArmLaunch(store, "exp", { + arm: "n8", + model: "google/gemini-3.5-flash", + role: "variant", + note: "slide@8", + slide: { model: "google/gemini-3.5-flash", turns: 8 }, + }); + + expect(launch.jobName).toBe("exp-n8"); + expect(launch.dataset).toBe("swe-bench/swe-bench-verified"); + expect(launch.include).toEqual(["astropy__astropy-1", "django__django-2", "sympy__sympy-3"]); + expect(launch.tasks).toBe(3); + expect(launch.concurrency).toBe(4); + expect(launch.timeoutMultiplier).toBe(2); + expect(launch.model).toBe("google/gemini-3.5-flash"); + expect(launch.role).toBe("variant"); + expect(launch.slide?.turns).toBe(8); + }); + + it("rejects a duplicate arm and an unknown experiment", () => { + const store = new RunStore(makeJobsDir()); + cleanups.push(() => store.close()); + store.registerLaunch({ + benchmark: "harbor", + jobName: "exp-base", + dataset: "d", + agent: "omp", + models: ["m/x"], + pid: 1, + config: { include: ["t1"] }, + }); + expect(() => resolveArmLaunch(store, "exp", { arm: "base", model: "m/y" })).toThrow(/already exists/); + expect(() => resolveArmLaunch(store, "ghost", { arm: "x", model: "m/y" })).toThrow(/no runs to inherit/); + }); +}); diff --git a/packages/harbor-manager/src/server.ts b/packages/harbor-manager/src/server.ts index a96353d9a..e6e116ce8 100755 --- a/packages/harbor-manager/src/server.ts +++ b/packages/harbor-manager/src/server.ts @@ -64,6 +64,17 @@ export interface LaunchRequest { extraArgs?: string[]; } +/** POST /api/experiments/:id/arms body — a new comparable arm; sample+config inherited. */ +export interface AddArmRequest { + /** Arm label; becomes the `-` job name. */ + arm: string; + model: string; + slide?: LaunchRequest["slide"]; + role?: RunRole; + note?: string; + extraArgs?: string[]; +} + interface ManagedChild { proc: Subprocess; jobName: string; @@ -91,6 +102,68 @@ function parseServerArgs(argv: string[]): { port: number; jobsDir: string } { return { port, jobsDir }; } +/** + * Resolve the launch request for a new arm added to an existing experiment. + * Inherits the experiment's benchmark, dataset, and — crucially — the exact + * task sample from a sibling arm (its recorded `include`, else its observed + * trial tasks) so the arm is directly comparable. Only per-arm knobs (model, + * slide, role, note, extra args) come from `req`. Throws if the experiment has + * no runs to inherit from or the arm name is taken. + */ +export function resolveArmLaunch(store: RunStore, experimentId: string, req: AddArmRequest): LaunchRequest { + if (!req.arm || /[^\w.-]/.test(req.arm)) throw new Error("arm must be a non-empty [A-Za-z0-9_.-] token"); + if (!req.model) throw new Error("model is required"); + const siblings = store.listRuns().filter(r => experimentOf(r.jobName) === experimentId); + if (siblings.length === 0) throw new Error(`experiment '${experimentId}' has no runs to inherit from`); + // Template = the sibling with the most observed trials (most representative + // of the real sample); listRuns is newest-first so ties keep the newest. + let template = siblings[0]; + let templateTrials = store.listTraces(template.jobName).length; + for (const r of siblings.slice(1)) { + const n = store.listTraces(r.jobName).length; + if (n > templateTrials) [template, templateTrials] = [r, n]; + } + const cfg = template.config as Partial; + const str = (v: unknown): string | undefined => (typeof v === "string" && v ? v : undefined); + const numberOr = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) ? v : undefined); + const strings = (v: unknown): string[] => + Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; + // Exact task sample: prefer the intended include list, else observed trial tasks. + let include = strings(cfg.include); + if (include.length === 0) { + include = [ + ...new Set( + store + .listTraces(template.jobName) + .map(t => t.task) + .filter(Boolean), + ), + ]; + } + const jobName = `${experimentId}-${req.arm}`; + if (store.getRun(jobName)) throw new Error(`arm '${req.arm}' already exists in '${experimentId}'`); + const conditions = strings(cfg.conditions); + return { + benchmark: template.benchmark, + model: req.model, + dataset: template.dataset, + include: include.length > 0 ? include : undefined, + tasks: include.length > 0 ? include.length : numberOr(cfg.tasks), + concurrency: numberOr(cfg.concurrency), + timeoutMultiplier: numberOr(cfg.timeoutMultiplier), + attempts: numberOr(cfg.attempts), + agent: str(cfg.agent), + webSearch: cfg.webSearch === true || undefined, + prebuiltBinaries: cfg.prebuiltBinaries === true || undefined, + conditions: conditions.length > 0 ? conditions : undefined, + jobName, + slide: req.slide, + role: req.role, + note: req.note, + extraArgs: req.extraArgs, + }; +} + export class ManagerServer { #store: RunStore; #children = new Map(); @@ -144,7 +217,7 @@ export class ManagerServer { } } - /** Bundle the React dashboard once per process; served at /app.js. */ + /** Bundle the React dashboard once per process; served at /app.tsx (matches the Vite dev entry). */ async #appBundle(): Promise { if (this.#appBundleCode !== null) return this.#appBundleCode; const result = await Bun.build({ @@ -180,7 +253,7 @@ export class ManagerServer { if (p === "/" || p === "/index.html") { return new Response(Bun.file(INDEX_HTML_PATH)); } - if (p === "/app.js") { + if (p === "/app.tsx") { return new Response(await this.#appBundle(), { headers: { "content-type": "text/javascript; charset=utf-8" }, }); @@ -203,6 +276,12 @@ export class ManagerServer { if (!detail) return Response.json({ error: "experiment not found" }, { status: 404 }); return Response.json(detail); } + const armMatch = p.match(/^\/api\/experiments\/([^/]+)\/arms$/); + if (armMatch && request.method === "POST") { + const id = decodeURIComponent(armMatch[1]); + const body = (await request.json()) as AddArmRequest; + return Response.json(this.addArm(id, body), { status: 201 }); + } if (p === "/api/runs" && request.method === "GET") { return Response.json(this.#store.listRuns()); } @@ -381,6 +460,11 @@ export class ManagerServer { return { id, updatedRuns }; } + /** Add a comparable arm to an existing experiment, inheriting its sample + config. */ + addArm(experimentId: string, req: AddArmRequest): { jobName: string; pid: number } { + return this.launch(resolveArmLaunch(this.#store, experimentId, req)); + } + /** Cancel a manager-launched run (kills the runner; harbor children follow). */ cancel(jobName: string): { jobName: string; cancelled: boolean } { const child = this.#children.get(jobName); diff --git a/packages/harbor-manager/src/web/app.tsx b/packages/harbor-manager/src/web/app.tsx index 37515c981..a461d6289 100644 --- a/packages/harbor-manager/src/web/app.tsx +++ b/packages/harbor-manager/src/web/app.tsx @@ -144,6 +144,8 @@ function usePolled(url: string | null, intervalMs: number): T | null { return data; } +const INPUT_CLASS = "rounded border border-zinc-700 bg-zinc-950 px-2 py-1 text-sm"; + const STATUS_CLASS: Record = { running: "text-sky-400 border-sky-400", complete: "text-emerald-400 border-emerald-400", @@ -426,7 +428,83 @@ function Delta({ return ({body}); } +/** + * Launch a new arm into an existing experiment. The server inherits the + * experiment's dataset and exact task sample from a sibling arm, so only the + * arm-specific knobs (name, model, role, note, optional slide) are collected here. + */ +function AddArmForm({ experimentId, onDone }: { experimentId: string; onDone: () => void }) { + const [msg, setMsg] = useState(""); + const submit = useCallback( + async (ev: React.FormEvent) => { + ev.preventDefault(); + const f = new FormData(ev.currentTarget); + const body: Record = { arm: f.get("arm"), model: f.get("model") }; + if (f.get("role")) body.role = f.get("role"); + if (f.get("note")) body.note = f.get("note"); + const trigger = f.get("slideTrigger"); + if (f.get("slideModel") && trigger) { + const slide: Record = { + model: f.get("slideModel"), + plan: !!f.get("slidePlan"), + checklist: !!f.get("slideChecklist"), + }; + if (trigger === "on-action") slide.onAction = true; + else slide.turns = Number(f.get("slideTurns") || 8); + body.slide = slide; + } + setMsg("launching…"); + const res = await fetch(`/api/experiments/${encodeURIComponent(experimentId)}/arms`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const out = (await res.json()) as { jobName?: string; error?: string }; + setMsg(res.ok ? `launched ${out.jobName}` : `error: ${out.error}`); + if (res.ok) setTimeout(onDone, 900); + }, + [experimentId, onDone], + ); + return ( +
+ + + + + + + + +
+ + inherits dataset + task sample from existing arms · {msg} +
+
+ ); +} + function ExperimentPage({ id }: { id: string }) { + const [adding, setAdding] = useState(false); const detail = usePolled(`/api/experiments/${encodeURIComponent(id)}`, 3000); if (!detail) return
loading…
; const { arms, tasks, matrix, goal } = detail; @@ -459,8 +537,16 @@ function ExperimentPage({ id }: { id: string }) { )} + {goal &&

{goal}

} + {adding && setAdding(false)} />} @@ -823,7 +909,7 @@ function LaunchForm({ onDone }: { onDone: () => void }) { }, [onDone], ); - const input = "rounded border border-zinc-700 bg-zinc-950 px-2 py-1 text-sm"; + const input = INPUT_CLASS; return (