smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: reformat

Browse Source
This commit is contained in:
can1357
2026-07-15 00:08:50 +02:00
parent 7d02778c60
commit 55f5ebec49
97 changed files with 12237 additions and 3179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/metaharness/agent/omp_local.py
+52 -17
View File
@@ -153,6 +153,7 @@ def _env(name: str, default: str = "") -> str:
def _truthy(value: str) -> bool:
return value.strip().lower() in ("1", "true", "yes", "on")
def _loads(line: str) -> dict | None:
line = line.strip()
if not line.startswith("{"):
@@ -201,11 +202,15 @@ class OmpLocal(BaseInstalledAgent):
self._tarball = _env("OMP_BENCH_TARBALL")
self._pkg_version = _env("OMP_BENCH_VERSION", "latest")
self._models_yaml_path = _env("OMP_BENCH_MODELS_YAML")
self._gateway_url = _env("OMP_BENCH_GATEWAY_URL", "http://host.docker.internal:4000")
self._gateway_url = _env(
"OMP_BENCH_GATEWAY_URL", "http://host.docker.internal:4000"
)
self._gateway_token = _env("OMP_BENCH_GATEWAY_TOKEN", "no-auth-dummy")
self._gateway_providers = [
p.strip()
for p in _env("OMP_BENCH_GATEWAY_PROVIDERS", "anthropic,openai-codex").split(",")
for p in _env(
"OMP_BENCH_GATEWAY_PROVIDERS", "anthropic,openai-codex"
).split(",")
if p.strip()
]
self._thinking = _env("OMP_BENCH_THINKING")
@@ -248,7 +253,9 @@ class OmpLocal(BaseInstalledAgent):
def get_version_command(self) -> str | None:
if self._binary:
return f"{shlex.quote(self._cli)} --version"
return self._wrap(f"{shlex.quote(self._bun)} {shlex.quote(self._cli)} --version")
return self._wrap(
f"{shlex.quote(self._bun)} {shlex.quote(self._cli)} --version"
)
@override
def parse_version(self, stdout: str) -> str:
@@ -264,7 +271,7 @@ class OmpLocal(BaseInstalledAgent):
"""
bun_dir = os.path.dirname(self._bun)
return (
f'export BUN_INSTALL={shlex.quote(self._home + "/.bun")}; '
f"export BUN_INSTALL={shlex.quote(self._home + '/.bun')}; "
f'export PATH="{bun_dir}:$PATH"; '
f"{command}"
)
@@ -272,7 +279,9 @@ class OmpLocal(BaseInstalledAgent):
@override
async def install(self, environment: BaseEnvironment) -> None:
# Resolve the agent user's HOME first (root vs non-root tasks differ).
home = (await self.exec_as_agent(environment, command='printf %s "$HOME"')).stdout
home = (
await self.exec_as_agent(environment, command='printf %s "$HOME"')
).stdout
self._home = (home or "/root").strip() or "/root"
if self._binary:
@@ -304,7 +313,7 @@ class OmpLocal(BaseInstalledAgent):
"set -e; "
f"export BUN_INSTALL={shlex.quote(self._home + '/.bun')}; "
f'curl -fsSL https://bun.sh/install | bash -s "bun-v{self._bun_version}"; '
f'{shlex.quote(self._home + "/.bun/bin/bun")} --version'
f"{shlex.quote(self._home + '/.bun/bin/bun')} --version"
),
)
self._bun = f"{self._home}/.bun/bin/bun"
@@ -326,8 +335,15 @@ class OmpLocal(BaseInstalledAgent):
`node_modules` with a linux tree, and mounts a linux `bun` binary — so
setup needs zero outbound network and no rebuild for TS changes.
"""
arch = (await self.exec_as_agent(environment, command="uname -m")).stdout.strip()
norm = {"aarch64": "arm64", "arm64": "arm64", "x86_64": "x64", "amd64": "x64"}.get(arch)
arch = (
await self.exec_as_agent(environment, command="uname -m")
).stdout.strip()
norm = {
"aarch64": "arm64",
"arm64": "arm64",
"x86_64": "x64",
"amd64": "x64",
}.get(arch)
if self._source_arch and norm != self._source_arch:
raise RuntimeError(
f"source mode: container arch {arch!r} != mounted deps tree arch "
@@ -351,7 +367,9 @@ class OmpLocal(BaseInstalledAgent):
async def _install_local(self, environment: BaseEnvironment) -> str:
if not self._tarball:
raise RuntimeError("OMP_BENCH_INSTALL=local requires OMP_BENCH_TARBALL (host tarball path)")
raise RuntimeError(
"OMP_BENCH_INSTALL=local requires OMP_BENCH_TARBALL (host tarball path)"
)
await environment.upload_file(self._tarball, _TARBALL_DST)
app = f"{self._home}/.omp-bench/app"
await self.exec_as_agent(
@@ -364,7 +382,7 @@ class OmpLocal(BaseInstalledAgent):
# Bundle inlines workspace TS; only externalized deps are needed.
# Skip heavy optionals (transformers/sherpa) but add the native addon.
"bun install --production --omit=optional; "
'arch=$(uname -m); '
"arch=$(uname -m); "
'case "$arch" in aarch64|arm64) na=arm64 ;; x86_64|amd64) na=x64 ;; '
'*) echo "unsupported arch $arch" >&2; exit 4 ;; esac; '
# Native leaf MUST match the bundle version exactly (loader/API skew
@@ -379,7 +397,9 @@ class OmpLocal(BaseInstalledAgent):
async def _install_binary(self, environment: BaseEnvironment) -> str:
"""Probe container arch, upload only the matching self-contained omp binary."""
arch = (await self.exec_as_agent(environment, command="uname -m")).stdout.strip()
arch = (
await self.exec_as_agent(environment, command="uname -m")
).stdout.strip()
if arch in ("aarch64", "arm64"):
hostbin = self._binary_arm64
elif arch in ("x86_64", "amd64"):
@@ -387,11 +407,15 @@ class OmpLocal(BaseInstalledAgent):
else:
raise RuntimeError(f"binary mode: unsupported container arch {arch!r}")
if not hostbin:
raise RuntimeError(f"binary mode: no omp binary provided for container arch {arch}")
raise RuntimeError(
f"binary mode: no omp binary provided for container arch {arch}"
)
app_dir = f"{self._home}/.omp-bench"
dst = f"{app_dir}/omp"
staging = "/tmp/omp-bin"
await self.exec_as_agent(environment, command=f"mkdir -p {shlex.quote(app_dir)}")
await self.exec_as_agent(
environment, command=f"mkdir -p {shlex.quote(app_dir)}"
)
await environment.upload_file(hostbin, staging)
await self.exec_as_agent(
environment,
@@ -422,7 +446,9 @@ class OmpLocal(BaseInstalledAgent):
else:
content = self._generate_models_yaml()
staged = _MODELS_DST
heredoc = f"cat > {_MODELS_DST} <<'OMP_MODELS_EOF'\n{content}\nOMP_MODELS_EOF"
heredoc = (
f"cat > {_MODELS_DST} <<'OMP_MODELS_EOF'\n{content}\nOMP_MODELS_EOF"
)
await self.exec_as_agent(environment, command=heredoc)
await self.exec_as_agent(
environment,
@@ -433,7 +459,10 @@ class OmpLocal(BaseInstalledAgent):
)
def _generate_models_yaml(self) -> str:
lines = ["# Generated by metaharness runner — routes auth via host gateway.", "providers:"]
lines = [
"# Generated by metaharness runner — routes auth via host gateway.",
"providers:",
]
for provider in self._gateway_providers:
lines += [
f" {provider}:",
@@ -513,7 +542,9 @@ class OmpLocal(BaseInstalledAgent):
context: AgentContext,
) -> None:
if not self.model_name or "/" not in self.model_name:
raise ValueError("model must be 'provider/model' (e.g. anthropic/claude-sonnet-4-6)")
raise ValueError(
"model must be 'provider/model' (e.g. anthropic/claude-sonnet-4-6)"
)
provider, model = self.model_name.split("/", 1)
if self._binary:
@@ -547,7 +578,11 @@ class OmpLocal(BaseInstalledAgent):
if not self._gateway_on:
run_env.update(self._collect_provider_keys(provider))
run_env.update(self._forward_env)
await self.exec_as_agent(environment, command=run if self._binary else self._wrap(run), env=run_env or None)
await self.exec_as_agent(
environment,
command=run if self._binary else self._wrap(run),
env=run_env or None,
)
@override
def populate_context_post_run(self, context: AgentContext) -> None:
packages/metaharness/src/adapters/snapcompact.py
+149 -38
View File
@@ -38,6 +38,7 @@ from pathlib import Path
HERE = Path(__file__).resolve().parent
RESEARCH = HERE.parents[2] / "snapcompact" / "research"
def find_agent_prompts() -> Path:
for parent in HERE.parents:
for candidate in (
@@ -110,7 +111,9 @@ def agent_prompt(name: str) -> str:
return re.sub(r"\{\{#if .*?\{\{/if\}\}\n?", "", text, flags=re.DOTALL)
def cached_complete(api_key: str, model: str, messages: list[dict], fresh: bool, **kw) -> tuple[str, dict]:
def cached_complete(
api_key: str, model: str, messages: list[dict], fresh: bool, **kw
) -> tuple[str, dict]:
"""complete() with response caching keyed on the full request payload.
Truncated responses (stop_reason == max_tokens) are never cached and never
@@ -135,8 +138,15 @@ def parse_condition(name: str) -> dict:
return {"name": name, "kind": name}
m = re.fullmatch(r"img-([a-z0-9]+)-([a-z-]+)", name)
if not m or m.group(1) not in FONTS or m.group(2) not in VARIANTS:
raise SystemExit(f"bad condition {name!r}; expected text|compact|handoff|img-<font>-<variant>")
return {"name": name, "kind": "image", "font": FONTS[m.group(1)], "variant": m.group(2)}
raise SystemExit(
f"bad condition {name!r}; expected text|compact|handoff|img-<font>-<variant>"
)
return {
"name": name,
"kind": "image",
"font": FONTS[m.group(1)],
"variant": m.group(2),
}
def run_chunk(cond: dict, start: int, end: int, ctx_args: dict) -> list[dict]:
@@ -148,7 +158,9 @@ def run_chunk(cond: dict, start: int, end: int, ctx_args: dict) -> list[dict]:
ctx_args["offsets"],
ctx_args["api_key"],
)
questions = squad.sample_chunk_questions(paras, offsets, start, end, args.qpc, args.seed)
questions = squad.sample_chunk_questions(
paras, offsets, start, end, args.qpc, args.seed
)
if not questions:
return []
chunk_text = flow[start:end]
@@ -157,21 +169,41 @@ def run_chunk(cond: dict, start: int, end: int, ctx_args: dict) -> list[dict]:
png = cols = rows = None
context = chunk_text
if cond["kind"] == "image":
salt = ("dimv2",) if cond["variant"] == "dim" else () # cache-bust pre-fix sticky-fg dim renders
png = CACHE / f"img-{cond['font'].name}-{cond['variant']}-{sha8(chunk_text, str(args.size), *salt)}.png"
salt = (
("dimv2",) if cond["variant"] == "dim" else ()
) # cache-bust pre-fix sticky-fg dim renders
png = (
CACHE
/ f"img-{cond['font'].name}-{cond['variant']}-{sha8(chunk_text, str(args.size), *salt)}.png"
)
if not png.exists():
render(chunk_text, cond["font"], CACHE, args.size, cond["variant"]).save(png)
render(chunk_text, cond["font"], CACHE, args.size, cond["variant"]).save(
png
)
cols, rows, _ = capacity(cond["font"], args.size)
elif cond["kind"] in ("compact", "handoff"):
prompt_file = {"compact": "compaction-summary.md", "handoff": "handoff-document.md"}[cond["kind"]]
prompt_file = {
"compact": "compaction-summary.md",
"handoff": "handoff-document.md",
}[cond["kind"]]
gen_messages = [
{"role": "user", "content": load_prompt("session-frame.md").format(context=chunk_text)},
{"role": "assistant", "content": "Noted. I have read the passages and will keep them in mind."},
{
"role": "user",
"content": load_prompt("session-frame.md").format(context=chunk_text),
},
{
"role": "assistant",
"content": "Noted. I have read the passages and will keep them in mind.",
},
{"role": "user", "content": agent_prompt(prompt_file)},
]
context, gen_usage = cached_complete(
api_key, args.model, gen_messages, args.fresh,
system=agent_prompt("summarization-system.md"), max_tokens=4096,
api_key,
args.model,
gen_messages,
args.fresh,
system=agent_prompt("summarization-system.md"),
max_tokens=4096,
)
usage_rows.append(("summarize", gen_usage))
@@ -183,16 +215,30 @@ def run_chunk(cond: dict, start: int, end: int, ctx_args: dict) -> list[dict]:
q_block = "\n".join(f"{i + 1}. {q['q']}" for i, q in enumerate(batch))
if cond["kind"] == "image":
carrier = image_block(png)
preamble = {"type": "text", "text": load_prompt("qa-image.md").format(cols=cols, rows=rows)}
preamble = {
"type": "text",
"text": load_prompt("qa-image.md").format(cols=cols, rows=rows),
}
else:
carrier = {"type": "text", "text": load_prompt("qa-text.md").format(context=context)}
carrier = {
"type": "text",
"text": load_prompt("qa-text.md").format(context=context),
}
preamble = None
if use_cache:
carrier["cache_control"] = {"type": "ephemeral"}
content = ([preamble] if preamble else []) + [carrier, {"type": "text", "text": q_block}]
content = ([preamble] if preamble else []) + [
carrier,
{"type": "text", "text": q_block},
]
messages = [{"role": "user", "content": content}]
text, usage = cached_complete(
api_key, args.model, messages, args.fresh, max_tokens=args.max_tokens, effort=args.effort
api_key,
args.model,
messages,
args.fresh,
max_tokens=args.max_tokens,
effort=args.effort,
)
usage_rows.append(("qa", usage))
answers.extend(squad.parse_numbered(text, len(batch)))
@@ -225,7 +271,9 @@ def run_chunk(cond: dict, start: int, end: int, ctx_args: dict) -> list[dict]:
return records
def aggregate(name: str, records: list[dict], price_in: float, price_out: float) -> dict:
def aggregate(
name: str, records: list[dict], price_in: float, price_out: float
) -> dict:
n = len(records)
f1s = [r["f1"] for r in records]
mean_f1 = sum(f1s) / n
@@ -251,7 +299,8 @@ def aggregate(name: str, records: list[dict], price_in: float, price_out: float)
"cache_w": cache_w,
"cache_r": cache_r,
# Anthropic pricing: cache write 1.25x input, cache read 0.1x input (5m TTL).
"cost_usd": (tok_in + 1.25 * cache_w + 0.1 * cache_r) / 1e6 * price_in + tok_out / 1e6 * price_out,
"cost_usd": (tok_in + 1.25 * cache_w + 0.1 * cache_r) / 1e6 * price_in
+ tok_out / 1e6 * price_out,
"f1_by_quartile": quart,
}
@@ -261,23 +310,59 @@ def main() -> None:
ap.add_argument("--model", default="claude-fable-5")
ap.add_argument("--conditions", default=DEFAULT_CONDITIONS)
ap.add_argument("--qpc", type=int, default=30, help="questions sampled per chunk")
ap.add_argument("--qpb", type=int, default=0, help="questions per API call (batches the chunk); 0 = all at once")
ap.add_argument("--cache", choices=["auto", "on", "off"], default="auto",
help="prompt-cache the carrier block; auto = on when --qpb is set")
ap.add_argument("--max-tokens", type=int, default=8192, help="output budget per QA call (incl. thinking)")
ap.add_argument("--effort", choices=["low", "medium", "high", "xhigh", "max"], default=None,
help="adaptive-thinking effort for QA calls; default = provider default")
ap.add_argument(
"--qpb",
type=int,
default=0,
help="questions per API call (batches the chunk); 0 = all at once",
)
ap.add_argument(
"--cache",
choices=["auto", "on", "off"],
default="auto",
help="prompt-cache the carrier block; auto = on when --qpb is set",
)
ap.add_argument(
"--max-tokens",
type=int,
default=8192,
help="output budget per QA call (incl. thinking)",
)
ap.add_argument(
"--effort",
choices=["low", "medium", "high", "xhigh", "max"],
default=None,
help="adaptive-thinking effort for QA calls; default = provider default",
)
ap.add_argument("--seed", type=int, default=42)
ap.add_argument("--size", type=int, default=1568)
ap.add_argument("--workers", type=int, default=4)
ap.add_argument("--limit-chars", type=int, default=0, help="cap corpus size; 0 = full dev set")
ap.add_argument("--limit-paras", type=int, default=0, help="cap corpus to first N passages; 0 = all")
ap.add_argument(
"--limit-chars", type=int, default=0, help="cap corpus size; 0 = full dev set"
)
ap.add_argument(
"--limit-paras",
type=int,
default=0,
help="cap corpus to first N passages; 0 = all",
)
ap.add_argument("--fresh", action="store_true", help="ignore cached responses")
ap.add_argument("--report", action="store_true", help="aggregate cached records only; no API calls")
ap.add_argument("--price-in", type=float, default=10.0, help="$ per 1M input tokens")
ap.add_argument("--price-out", type=float, default=50.0, help="$ per 1M output tokens")
ap.add_argument(
"--report",
action="store_true",
help="aggregate cached records only; no API calls",
)
ap.add_argument(
"--price-in", type=float, default=10.0, help="$ per 1M input tokens"
)
ap.add_argument(
"--price-out", type=float, default=50.0, help="$ per 1M output tokens"
)
ap.add_argument("--env", default="~/.env")
ap.add_argument("--output-dir", help="write records.jsonl and summary.json directly to this directory")
ap.add_argument(
"--output-dir",
help="write records.jsonl and summary.json directly to this directory",
)
args = ap.parse_args()
CACHE.mkdir(exist_ok=True)
@@ -301,26 +386,43 @@ def main() -> None:
if args.limit_paras:
paras = paras[: args.limit_paras]
flow, offsets = squad.build_flow(paras, args.limit_chars or None)
conditions = [parse_condition(c.strip()) for c in args.conditions.split(",") if c.strip()]
conditions = [
parse_condition(c.strip()) for c in args.conditions.split(",") if c.strip()
]
tasks: list[tuple[dict, int, int]] = []
for cond in conditions:
budget = capacity(cond["font"], args.size)[2] if cond["kind"] == "image" else TEXT_CHUNK
budget = (
capacity(cond["font"], args.size)[2]
if cond["kind"] == "image"
else TEXT_CHUNK
)
for start in range(0, len(flow), budget):
tasks.append((cond, start, min(start + budget, len(flow))))
calls = len(tasks) + sum(1 for c, *_ in tasks if c["kind"] in ("compact", "handoff"))
calls = len(tasks) + sum(
1 for c, *_ in tasks if c["kind"] in ("compact", "handoff")
)
print(
f"corpus={len(flow):,} chars ({len(offsets):,} passages), {len(conditions)} conditions, "
f"{len(tasks)} chunks, <= {calls} API calls, qpc={args.qpc}, model={args.model}"
)
api_key = "" if args.report else load_api_key(args.env)
ctx_args = {"args": args, "flow": flow, "paras": paras, "offsets": offsets, "api_key": api_key}
ctx_args = {
"args": args,
"flow": flow,
"paras": paras,
"offsets": offsets,
"api_key": api_key,
}
records: list[dict] = []
done = 0
with (run_dir / "records.jsonl").open("w") as records_file:
with ThreadPoolExecutor(args.workers) as pool:
futures = [pool.submit(run_chunk, cond, start, end, ctx_args) for cond, start, end in tasks]
futures = [
pool.submit(run_chunk, cond, start, end, ctx_args)
for cond, start, end in tasks
]
for fut in futures:
chunk_records = fut.result()
records.extend(chunk_records)
@@ -332,12 +434,19 @@ def main() -> None:
print(f" {done}/{len(tasks)} chunks", flush=True)
rows = [
aggregate(cond["name"], [r for r in records if r["cond"] == cond["name"]], args.price_in, args.price_out)
aggregate(
cond["name"],
[r for r in records if r["cond"] == cond["name"]],
args.price_in,
args.price_out,
)
for cond in conditions
if any(r["cond"] == cond["name"] for r in records)
]
rows.sort(key=lambda r: -r["f1"])
(run_dir / "summary.json").write_text(json.dumps({"args": vars(args), "rows": rows}, indent=1))
(run_dir / "summary.json").write_text(
json.dumps({"args": vars(args), "rows": rows}, indent=1)
)
hdr = (
f"{'condition':<15}{'n':>6}{'EM':>7}{'F1':>7}{'±se':>6}{'abst':>6}"
@@ -351,7 +460,9 @@ def main() -> None:
)
print(f"\n{'condition':<15} F1 by position quartile (Q1..Q4)")
for r in rows:
cells = " ".join(" - " if q is None else f"{q:.3f}" for q in r["f1_by_quartile"])
cells = " ".join(
" - " if q is None else f"{q:.3f}" for q in r["f1_by_quartile"]
)
print(f"{r['name']:<15} {cells}")
print(f"\nresults -> {run_dir}/")