oh-my-pi/packages/react-edit-benchmark/index.ts

#!/usr/bin/env bun
/**
 * Edit benchmark CLI entry point.
 *
 * Usage:
 *   bun run bench:edit --model anthropic/claude-sonnet-4-5
 *   bun run bench:edit --tasks core-memory-recall,operations-division
 *   bun run bench:edit --runs 5 --output report.md
 *   bun run bench:edit --fixtures fixtures.tar.gz
 */
import * as fs from "node:fs";
import { join } from "node:path";
import { parseArgs } from "node:util";
import type { ThinkingLevel } from "@oh-my-pi/pi-agent-core";
import { TempDir } from "@oh-my-pi/pi-utils";
import { generateJsonReport, generateReport } from "./report";
import { type BenchmarkConfig, type ProgressEvent, runBenchmark } from "./runner";
import { type EditTask, loadTasks, loadTasksFromDir, validateFixtures } from "./tasks";

const THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh"] as const;

function generateReportFilename(config: BenchmarkConfig, format: "markdown" | "json"): string {
	const modelName = config.model
		.split("/")
		.pop()!
		.replace(/[^a-zA-Z0-9-]/g, "_");
	const variant = config.editVariant ?? "replace";
	const timestamp = new Date().toISOString().replace(/:/g, "-").replace(/\..+$/, "").replace(/Z$/, "Z");
	const ext = format === "json" ? "json" : "md";
	return `runs/${modelName}_${variant}_${timestamp}.${ext}`;
}

function printUsage(tasks?: EditTask[]): void {
	const taskList = tasks
		? tasks.map((t) => `  ${t.id.padEnd(30)} ${t.name}`).join("\n")
		: "  (use --list to see available tasks)";
	console.log(`
Edit Benchmark - Evaluate patch application success rates

Usage:
  bun run bench:edit [options]

Options:
  --model <id>              Model ID (default: claude-sonnet-4-20250514)
  --thinking <level>        Thinking level: off, minimal, low, medium, high, xhigh
  --runs <n>                Runs per task (default: 3)
  --timeout <ms>            Timeout per run in ms (default: 120000)
  --task-concurrency <n>    Max tasks to run in parallel (default: 16)
  --tasks <ids>             Comma-separated task IDs to run (default: all)
  --fixtures <path>         Fixtures directory or .tar.gz archive (default: built-in)
  --edit-variant <v>        Edit variant: replace, patch, auto (default: auto)
  --edit-fuzzy <bool>       Fuzzy matching: true, false, auto (default: auto)
  --edit-fuzzy-threshold <n> Fuzzy threshold 0-1 or auto (default: auto)
  --auto-format             Auto-format output files after verify (debug only)
  --output <file>           Output file (default: run_<model>_<variant>_<fuzzy>_<threshold>_<timestamp>.md)
  --format <fmt>            Output format: markdown, json (default: markdown)
  --check-fixtures          Validate fixtures and exit
  --require-edit-tool-call  Require edit tool usage for success (default: false)
  --no-edit-required        Remove "must edit" prompt requirement (default: false)
  --list                    List available tasks and exit
  --help                    Show this help message

Available Tasks:
${taskList}

Examples:
  # Run full benchmark with default model
  bun run bench:edit

  # Run specific tasks
  bun run bench:edit --tasks core-memory-recall,operations-division

  # Compare different models
  bun run bench:edit --model claude-sonnet-4-20250514 --output sonnet.md
  bun run bench:edit --model claude-opus-4-5-20251101 --output opus.md

  # Run with extended thinking
  bun run bench:edit --thinking high --runs 5

  # Run from a fixtures archive
  bun run bench:edit --fixtures edit-fixtures.tar.gz
`);
}

async function resolveExtractedDir(tempDir: string): Promise<string> {
	const entries = await fs.promises.readdir(tempDir, { withFileTypes: true });
	const dirs = entries.filter((entry) => entry.isDirectory());
	const files = entries.filter((entry) => entry.isFile());
	if (dirs.length === 1 && files.length === 0) {
		return join(tempDir, dirs[0]!.name);
	}
	return tempDir;
}

async function extractTarGz(archivePath: string): Promise<{ dir: string; cleanupDir: string }> {
	const tempDirObj = await TempDir.create("@reach-benchmark-fixtures-");
	const tempDir = tempDirObj.path();
	try {
		const bytes = await Bun.file(archivePath).arrayBuffer();
		const archive = new Bun.Archive(bytes);
		const files = await archive.files();

		for (const [path, file] of files) {
			const destPath = join(tempDir, path);
			await Bun.write(destPath, file);
		}
	} catch (error) {
		await tempDirObj.remove();
		const message = error instanceof Error ? error.message : String(error);
		throw new Error(`Failed to extract archive: ${message}`, { cause: error });
	}

	return { dir: await resolveExtractedDir(tempDir), cleanupDir: tempDir };
}

async function resolveFixtures(fixturesArg?: string): Promise<{ tasks: EditTask[]; cleanup?: () => Promise<void> }> {
	if (!fixturesArg) {
		return { tasks: await loadTasks() };
	}

	if (fixturesArg.endsWith(".tar.gz") || fixturesArg.endsWith(".tgz")) {
		const extracted = await extractTarGz(fixturesArg);
		return {
			tasks: await loadTasksFromDir(extracted.dir),
			cleanup: () => fs.promises.rm(extracted.cleanupDir, { recursive: true, force: true }),
		};
	}

	return { tasks: await loadTasksFromDir(fixturesArg) };
}

async function main(): Promise<void> {
	const { values } = parseArgs({
		options: {
			provider: { type: "string", default: "anthropic" },
			model: { type: "string", default: "claude-sonnet-4-20250514" },
			thinking: { type: "string" },
			runs: { type: "string", default: "3" },
			timeout: { type: "string", default: "120000" },
			"task-concurrency": { type: "string", default: "16" },
			tasks: { type: "string" },
			fixtures: { type: "string" },
			output: { type: "string" },
			format: { type: "string", default: "markdown" },
			"check-fixtures": { type: "boolean", default: false },
			"auto-format": { type: "boolean", default: false },
			"require-edit-tool-call": { type: "boolean", default: false },
			"no-edit-required": { type: "boolean", default: false },
			"edit-variant": { type: "string" },
			"edit-fuzzy": { type: "string" },
			"edit-fuzzy-threshold": { type: "string" },
			list: { type: "boolean", default: false },
			help: { type: "boolean", default: false },
		},
		allowPositionals: true,
	});

	if (values.help) {
		printUsage();
		process.exit(0);
	}

	if (values["check-fixtures"]) {
		const issues = await validateFixtures(values.fixtures);
		if (issues.length === 0) {
			console.log("Fixtures OK");
			process.exit(0);
		}
		console.error("Fixture validation failed:");
		for (const issue of issues) {
			console.error(`  - ${issue.taskId}: ${issue.message}`);
		}
		process.exit(1);
	}

	const { tasks: allTasks, cleanup } = await resolveFixtures(values.fixtures);

	if (values.list) {
		console.log("Available Tasks:\n");
		for (const task of allTasks) {
			console.log(`  ${task.id}`);
			console.log(`    Name: ${task.name}`);
			console.log(`    Files: ${task.files.join(", ")}`);
			console.log("");
		}
		process.exit(0);
	}

	let thinkingLevel: ThinkingLevel | undefined;
	if (values.thinking) {
		if (!THINKING_LEVELS.includes(values.thinking as ThinkingLevel)) {
			console.error(`Invalid thinking level: ${values.thinking}`);
			console.error(`Valid levels: ${THINKING_LEVELS.join(", ")}`);
			process.exit(1);
		}
		thinkingLevel = values.thinking as ThinkingLevel;
	}

	const runsPerTask = parseInt(values.runs!, 10);
	if (isNaN(runsPerTask) || runsPerTask < 1) {
		console.error(`Invalid runs value: ${values.runs}`);
		process.exit(1);
	}

	const timeout = parseInt(values.timeout!, 10);
	if (isNaN(timeout) || timeout < 1000) {
		console.error(`Invalid timeout value: ${values.timeout}`);
		process.exit(1);
	}

	const taskConcurrency = parseInt(values["task-concurrency"]!, 10);
	if (isNaN(taskConcurrency) || taskConcurrency < 1) {
		console.error(`Invalid task concurrency value: ${values["task-concurrency"]}`);
		process.exit(1);
	}

	let tasksToRun = allTasks;
	if (values.tasks) {
		const taskIds = values.tasks.split(",").map((s) => s.trim());
		tasksToRun = [];
		for (const id of taskIds) {
			const task = allTasks.find((t) => t.id === id);
			if (!task) {
				console.error(`Unknown task ID: ${id}`);
				console.error(`Available tasks: ${allTasks.map((t) => t.id).join(", ")}`);
				process.exit(1);
			}
			tasksToRun.push(task);
		}
	}

	const editVariant = values["edit-variant"] as "replace" | "patch" | "auto" | undefined;
	if (editVariant && !["replace", "patch", "auto"].includes(editVariant)) {
		console.error(`Invalid edit-variant: ${editVariant}. Must be replace, patch, or auto.`);
		process.exit(1);
	}

	let editFuzzy: boolean | "auto" | undefined;
	if (values["edit-fuzzy"] !== undefined) {
		if (values["edit-fuzzy"] === "auto") {
			editFuzzy = "auto";
		} else if (values["edit-fuzzy"] === "true" || values["edit-fuzzy"] === "1") {
			editFuzzy = true;
		} else if (values["edit-fuzzy"] === "false" || values["edit-fuzzy"] === "0") {
			editFuzzy = false;
		} else {
			console.error(`Invalid edit-fuzzy: ${values["edit-fuzzy"]}. Must be true, false, 1, 0, or auto.`);
			process.exit(1);
		}
	}

	let editFuzzyThreshold: number | "auto" | undefined;
	if (values["edit-fuzzy-threshold"] !== undefined) {
		if (values["edit-fuzzy-threshold"] === "auto") {
			editFuzzyThreshold = "auto";
		} else {
			const parsed = parseFloat(values["edit-fuzzy-threshold"]);
			if (isNaN(parsed) || parsed < 0 || parsed > 1) {
				console.error(`Invalid edit-fuzzy-threshold: ${values["edit-fuzzy-threshold"]}. Must be 0-1 or auto.`);
				process.exit(1);
			}
			editFuzzyThreshold = parsed;
		}
	}

	const config: BenchmarkConfig = {
		provider: values.provider!,
		model: values.model!,
		thinkingLevel,
		runsPerTask,
		timeout,
		taskConcurrency,
		autoFormat: values["auto-format"],
		requireEditToolCall: values["require-edit-tool-call"],
		noEditRequired: values["no-edit-required"],
		editVariant,
		editFuzzy,
		editFuzzyThreshold,
	};

	console.log("Edit Benchmark");
	console.log("==============");
	console.log(`Provider: ${config.provider}`);
	console.log(`Model: ${config.model}`);
	if (config.thinkingLevel) {
		console.log(`Thinking: ${config.thinkingLevel}`);
	}
	console.log(`Runs per task: ${config.runsPerTask}`);
	console.log(`Timeout: ${config.timeout}ms`);
	console.log(`Task concurrency: ${config.taskConcurrency}`);
	if (config.autoFormat) {
		console.log("Auto-format: enabled");
	}
	if (config.requireEditToolCall) {
		console.log("Require edit tool call: yes");
	}
	if (config.noEditRequired) {
		console.log("No-edit-required baseline: yes");
	}
	if (config.editVariant) {
		console.log(`Edit variant: ${config.editVariant}`);
	}
	if (config.editFuzzy !== undefined) {
		console.log(`Edit fuzzy: ${config.editFuzzy}`);
	}
	if (config.editFuzzyThreshold !== undefined) {
		console.log(`Edit fuzzy threshold: ${config.editFuzzyThreshold}`);
	}
	console.log(`Tasks: ${tasksToRun.length}`);
	console.log("");

	const progress = new LiveProgress(tasksToRun.length * config.runsPerTask, config.runsPerTask);
	const result = await runBenchmark(tasksToRun, config, (event) => {
		progress.handleEvent(event);
	});
	progress.finish();

	console.log("");
	console.log("Benchmark complete!");
	console.log(`  Success rate: ${(result.summary.overallSuccessRate * 100).toFixed(1)}%`);
	console.log(`  Total tokens: ${result.summary.totalTokens.input} in / ${result.summary.totalTokens.output} out`);
	console.log("");

	const formatType = values.format === "json" ? "json" : "markdown";
	const report = formatType === "json" ? generateJsonReport(result) : generateReport(result);
	const outputPath = values.output ?? generateReportFilename(config, formatType);

	await Bun.write(outputPath, report);
	console.log(`Report written to: ${outputPath}`);

	if (cleanup) {
		await cleanup();
	}
}

class LiveProgress {
	private readonly totalRuns: number;
	private readonly runsPerTask: number;
	private readonly isTty: boolean;
	private started = 0;
	private completed = 0;
	private success = 0;
	private totalInput = 0;
	private totalOutput = 0;
	private totalDuration = 0;
	private totalReads = 0;
	private totalEdits = 0;
	private totalWrites = 0;
	private totalEditSuccesses = 0;
	private totalToolInputChars = 0;
	private indentScores: number[] = [];
	private lastLineLength = 0;

	constructor(totalRuns: number, runsPerTask: number) {
		this.totalRuns = totalRuns;
		this.runsPerTask = runsPerTask;
		this.isTty = Boolean(process.stdout.isTTY);
	}

	handleEvent(event: ProgressEvent): void {
		if (event.status === "started") {
			this.started += 1;
			if (!this.isTty) {
				console.log(`  [${event.taskId}] Run ${event.runIndex + 1}/${this.runsPerTask} started...`);
			}
			this.renderLine();
			return;
		}

		this.completed += 1;
		if (event.result) {
			if (event.result.success) {
				this.success += 1;
			}
			this.totalInput += event.result.tokens.input;
			this.totalOutput += event.result.tokens.output;
			this.totalDuration += event.result.duration;
			this.totalReads += event.result.toolCalls.read;
			this.totalEdits += event.result.toolCalls.edit;
			this.totalWrites += event.result.toolCalls.write;
			this.totalEditSuccesses += event.result.toolCalls.editSuccesses;
			this.totalToolInputChars += event.result.toolCalls.totalInputChars;
			if (typeof event.result.indentScore === "number") {
				this.indentScores.push(event.result.indentScore);
			}
		}

		if (event.result && !event.result.success && event.result.error) {
			this.flushLine();
			console.log(`  [${event.taskId}] Run ${event.runIndex + 1}/${this.runsPerTask} failed: ${event.result.error}`);
			if (event.result.diff) {
				const diffLines = event.result.diff.split("\n").slice(0, 30);
				if (diffLines.length > 0) {
					console.log("  Diff (first 30 lines):");
					for (const line of diffLines) {
						console.log(`    ${line}`);
					}
					if (event.result.diff.split("\n").length > 30) {
						console.log("    ... (truncated)");
					}
				}
			}
		}

		if (!this.isTty) {
			const status = event.result?.success ? "completed" : "failed";
			console.log(`  [${event.taskId}] Run ${event.runIndex + 1}/${this.runsPerTask} ${status}`);
		}

		this.renderLine();
	}

	finish(): void {
		this.flushLine();
		this.printSummary();
	}

	private printSummary(): void {
		const n = this.completed;
		if (n === 0) return;

		const successRate = (this.success / n) * 100;
		const editSuccessRate = this.totalEdits > 0 ? (this.totalEditSuccesses / this.totalEdits) * 100 : 100;
		const avgIndent =
			this.indentScores.length > 0 ? this.indentScores.reduce((a, b) => a + b, 0) / this.indentScores.length : 0;

		console.log("");
		console.log("Runtime Stats:");
		console.log(`  Task success:     ${successRate.toFixed(1)}% (${this.success}/${n})`);
		console.log(`  Edit success:     ${editSuccessRate.toFixed(1)}% (${this.totalEditSuccesses}/${this.totalEdits})`);
		console.log(`  Avg indent score: ${avgIndent.toFixed(2)}`);
		console.log(`  Tool calls:       read=${this.totalReads} edit=${this.totalEdits} write=${this.totalWrites}`);
		console.log(`  Tool input chars: ${this.totalToolInputChars.toLocaleString()}`);
		console.log(
			`  Avg tokens/task:  ${Math.round(this.totalInput / n)} in / ${Math.round(this.totalOutput / n)} out`,
		);
		console.log(`  Avg time/task:    ${Math.round(this.totalDuration / n)}ms`);
	}

	private renderLine(): void {
		if (!this.isTty) {
			return;
		}
		const successRate = this.completed > 0 ? (this.success / this.completed) * 100 : 0;
		const editRate = this.totalEdits > 0 ? (this.totalEditSuccesses / this.totalEdits) * 100 : 100;
		const avgInput = this.completed > 0 ? Math.round(this.totalInput / this.completed) : 0;
		const avgOutput = this.completed > 0 ? Math.round(this.totalOutput / this.completed) : 0;
		const avgDuration = this.completed > 0 ? Math.round(this.totalDuration / this.completed) : 0;
		const inFlight = this.started - this.completed;
		const bar = this.renderBar(this.completed, this.totalRuns, 20);
		const line = `  ${bar} ${this.completed}/${this.totalRuns} task=${successRate.toFixed(0)}% edit=${editRate.toFixed(0)}% tok=${avgInput}/${avgOutput} ${avgDuration}ms r/e/w=${this.totalReads}/${this.totalEdits}/${this.totalWrites} fly=${inFlight}`;
		this.writeLine(line);
	}

	private renderBar(done: number, total: number, width: number): string {
		const ratio = total === 0 ? 0 : done / total;
		const filled = Math.round(ratio * width);
		const empty = Math.max(0, width - filled);
		return `[${"#".repeat(filled)}${"-".repeat(empty)}]`;
	}

	private writeLine(line: string): void {
		const padding = this.lastLineLength > line.length ? " ".repeat(this.lastLineLength - line.length) : "";
		process.stdout.write(`\r${line}${padding}`);
		this.lastLineLength = line.length;
	}

	private flushLine(): void {
		if (!this.isTty) {
			return;
		}
		if (this.lastLineLength > 0) {
			process.stdout.write("\r" + " ".repeat(this.lastLineLength) + "\r");
			this.lastLineLength = 0;
		}
	}
}

main().catch((err) => {
	console.error("Benchmark failed:", err);
	process.exit(1);
});