oh-my-pi/packages/react-edit-benchmark/report.ts

/**
 * Markdown report generator for edit benchmark results.
 */
import type { BenchmarkResult, TaskResult } from "./runner";

function formatDuration(ms: number): string {
	if (ms < 1000) {
		return `${ms}ms`;
	}
	return `${(ms / 1000).toFixed(1)}s`;
}

function formatNumber(n: number): string {
	return n.toLocaleString();
}

function getStatusEmoji(successRate: number, runsPerTask: number): string {
	const passing = Math.round(successRate * runsPerTask);
	if (passing === runsPerTask) return "✅";
	if (passing === 0) return "❌";
	return "⚠️";
}

function formatPassRate(successRate: number, runsPerTask: number): string {
	const passing = Math.round(successRate * runsPerTask);
	return `${passing}/${runsPerTask}`;
}

function formatRate(numerator: number, denominator: number): string {
	if (denominator === 0) return "—";
	const percent = (numerator / denominator) * 100;
	return `${percent.toFixed(1)}% (${numerator}/${denominator})`;
}

function formatPercent(ratio: number): string {
	return `${(ratio * 100).toFixed(1)}%`;
}

function escapeMarkdown(text: string): string {
	return text.replace(/\|/g, "\\|").replace(/\n/g, " ");
}

function truncateText(text: string, maxLength: number): string {
	if (text.length <= maxLength) return text;
	return `${text.slice(0, maxLength - 3)}...`;
}

function formatEditArgsBlock(args: unknown): string {
	if (!args || typeof args !== "object") return "—";
	const diff = (args as { diff?: unknown }).diff;
	if (typeof diff === "string") {
		return diff;
	}
	try {
		return JSON.stringify(args, null, 2);
	} catch {
		return "—";
	}
}

function formatToolError(error: unknown): string {
	if (typeof error === "string") return error;
	try {
		return JSON.stringify(error, null, 2);
	} catch {
		return String(error);
	}
}

function formatFiles(files: string[]): string {
	const joined = files.join(", ");
	return truncateText(joined, 60);
}

export function generateReport(result: BenchmarkResult): string {
	const { config, tasks, summary } = result;
	const runsPerTask = config.runsPerTask;
	const allRuns = tasks.flatMap(task => task.runs);
	const verifiedRuns = allRuns.filter(run => run.verificationPassed).length;
	const editToolRuns = allRuns.filter(run => run.patchApplied).length;
	const successRuns = allRuns.filter(run => run.success).length;
	const totalEditAttempts = allRuns.reduce((sum, run) => sum + run.toolCalls.edit, 0);
	const totalEditFailures = allRuns.reduce((sum, run) => sum + run.toolCalls.editFailures, 0);

	const lines: string[] = [];

	lines.push("# Edit Benchmark Report");
	lines.push("");
	lines.push("## Configuration");
	lines.push("");
	lines.push("| Setting | Value |");
	lines.push("|---------|-------|");
	lines.push(`| Date | ${result.startTime} |`);
	lines.push(`| Model | ${config.provider}/${config.model} |`);
	lines.push(`| Thinking Level | ${config.thinkingLevel ?? "default"} |`);
	lines.push(`| Runs per task | ${runsPerTask} |`);
	lines.push(`| Edit Variant | ${config.editVariant ?? "auto"} |`);
	lines.push(`| Edit Fuzzy | ${config.editFuzzy === undefined ? "auto" : config.editFuzzy} |`);
	lines.push(
		`| Edit Fuzzy Threshold | ${config.editFuzzyThreshold === undefined ? "auto" : config.editFuzzyThreshold} |`,
	);
	lines.push(`| Guided Mode | ${config.guided === false ? "no" : "yes"} |`);
	lines.push(`| Max Attempts | ${config.maxAttempts ?? 1} |`);
	lines.push(`| No-op Retry Limit | ${config.noOpRetryLimit ?? 2} |`);
	lines.push(`| Mutation Scope Window | ${config.mutationScopeWindow ?? 20} |`);
	lines.push(`| Require Edit Tool | ${config.requireEditToolCall ? "yes" : "no"} |`);
	lines.push(`| Require Read Tool | ${config.requireReadToolCall ? "yes" : "no"} |`);
	lines.push(`| No-Edit Baseline | ${config.noEditRequired ? "yes" : "no"} |`);
	lines.push("");

	lines.push("## Summary");
	lines.push("");
	lines.push("| Metric | Value |");
	lines.push("|--------|-------|");
	lines.push(`| Total Tasks | ${summary.totalTasks} |`);
	lines.push(`| Total Runs | ${summary.totalRuns} |`);
	lines.push(`| Successful Runs | ${summary.successfulRuns} |`);
	lines.push(`| **Task Success Rate** | **${formatRate(successRuns, summary.totalRuns)}** |`);
	lines.push(`| Verified Rate | ${formatRate(verifiedRuns, summary.totalRuns)} |`);
	lines.push(`| Edit Tool Usage Rate | ${formatRate(editToolRuns, summary.totalRuns)} |`);
	lines.push(`| **Edit Success Rate** | **${formatPercent(summary.editSuccessRate)}** |`);
	lines.push(`| Timeout Runs | ${summary.timeoutRuns} |`);
	if (typeof summary.mutationIntentMatchRate === "number") {
		lines.push(`| Mutation Intent Match Rate | ${formatPercent(summary.mutationIntentMatchRate)} |`);
	}
	if (config.editVariant === "patch" || config.editVariant === "hashline") {
		lines.push(`| Patch Failure Rate | ${formatRate(totalEditFailures, totalEditAttempts)} |`);
	}
	lines.push(`| Tasks All Passing | ${summary.tasksWithAllPassing} |`);
	lines.push(`| Tasks Flaky/Failing | ${summary.tasksWithAnyFailing} |`);
	lines.push("");
	lines.push("### Tool Calls");
	lines.push("");
	lines.push("| Tool | Total | Avg/Run |");
	lines.push("|------|-------|---------|");
	lines.push(`| Read | ${summary.totalToolCalls.read} | ${summary.avgToolCallsPerRun.read.toFixed(1)} |`);
	lines.push(`| Edit | ${summary.totalToolCalls.edit} | ${summary.avgToolCallsPerRun.edit.toFixed(1)} |`);
	lines.push(`| Write | ${summary.totalToolCalls.write} | ${summary.avgToolCallsPerRun.write.toFixed(1)} |`);
	lines.push(
		`| **Tool Input Chars** | ${formatNumber(summary.totalToolCalls.totalInputChars)} | ${formatNumber(Math.round(summary.avgToolCallsPerRun.totalInputChars))} |`,
	);
	lines.push("");
	lines.push("### Tokens & Time");
	lines.push("");
	lines.push("| Metric | Total | Avg/Run |");
	lines.push("|--------|-------|---------|");
	lines.push(
		`| Input Tokens | ${formatNumber(summary.totalTokens.input)} | ${formatNumber(summary.avgTokensPerRun.input)} |`,
	);
	lines.push(
		`| Output Tokens | ${formatNumber(summary.totalTokens.output)} | ${formatNumber(summary.avgTokensPerRun.output)} |`,
	);
	lines.push(
		`| Total Tokens | ${formatNumber(summary.totalTokens.total)} | ${formatNumber(summary.avgTokensPerRun.total)} |`,
	);
	lines.push(`| Duration | ${formatDuration(summary.totalDuration)} | ${formatDuration(summary.avgDurationPerRun)} |`);
	lines.push(`| **Avg Indent Score** | — | **${formatScore(summary.avgIndentScore)}** |`);
	lines.push("");

	if (summary.hashlineEditSubtypes) {
		const total = Object.values(summary.hashlineEditSubtypes).reduce((a, b) => a + b, 0);
		if (total > 0) {
			lines.push("### Hashline Edit Subtypes");
			lines.push("");
			lines.push("| Operation | Count | % |");
			lines.push("|-----------|-------|---|");
			const order = ["set_line", "replace_lines", "insert_after", "replace"];
			for (const key of order) {
				const count = summary.hashlineEditSubtypes[key] ?? 0;
				const pct = formatPercent(count / total);
				lines.push(`| ${key} | ${count} | ${pct} |`);
			}
			lines.push(`| **Total** | **${total}** | 100% |`);
			lines.push("");
		}
	}

	lines.push("## Task Results");
	lines.push("");
	lines.push("| Task | File | Success | Edit Hit | R/E/W | Tokens (In/Out) | Time | Indent |");
	lines.push("|------|------|---------|----------|-------|-----------------|------|--------|");

	for (const task of tasks) {
		const status = getStatusEmoji(task.successRate, runsPerTask);
		const passRate = formatPassRate(task.successRate, runsPerTask);
		const editHitRate = formatPercent(task.editSuccessRate);
		const toolCalls = `${task.avgToolCalls.read.toFixed(0)}/${task.avgToolCalls.edit.toFixed(0)}/${task.avgToolCalls.write.toFixed(0)}`;
		lines.push(
			`| ${escapeMarkdown(task.name)} | ${escapeMarkdown(formatFiles(task.files))} | ${passRate} ${status} | ${editHitRate} | ${toolCalls} | ${formatNumber(task.avgTokens.input)}/${formatNumber(task.avgTokens.output)} | ${formatDuration(task.avgDuration)} | ${formatScore(task.avgIndentScore)} |`,
		);
	}
	lines.push("");

	appendCategorySummary(lines, tasks);
	appendMutationSummary(lines, tasks);
	appendDifficultySummary(lines, tasks);

	if (totalEditFailures > 0) {
		lines.push("## Edit Tool Errors");
		lines.push("");
		lines.push("Failures where the edit tool returned an error or failed to apply the patch.");
		lines.push("");

		for (const task of tasks) {
			const taskFailures = task.runs.filter(run => run.editFailures.length > 0);
			if (taskFailures.length === 0) continue;
			lines.push(`### ${task.name} (${formatFiles(task.files)})`);
			lines.push("");

			for (const run of taskFailures) {
				lines.push(`#### Run ${run.runIndex + 1}`);
				lines.push("");
				for (const [index, failure] of run.editFailures.entries()) {
					const args = failure.args as { path?: unknown; operation?: unknown };
					const path = typeof args?.path === "string" ? args.path : "—";
					const operation = typeof args?.operation === "string" ? args.operation : "—";
					lines.push(`##### Attempt ${index + 1}`);
					lines.push("");
					lines.push(`- Path: ${escapeMarkdown(path)}`);
					lines.push(`- Operation: ${escapeMarkdown(operation)}`);
					lines.push("");
					lines.push("**Tool error**");
					lines.push("");
					lines.push("```");
					lines.push(formatToolError(failure.error));
					lines.push("```");
					lines.push("");
					const body = formatEditArgsBlock(failure.args);
					lines.push("**Patch args**");
					lines.push("");
					lines.push("```diff");
					lines.push(body);
					lines.push("```");
					lines.push("");
				}
			}
		}
	}

	const flakyTasks = tasks.filter(t => t.successRate > 0 && t.successRate < 1);
	if (flakyTasks.length > 0) {
		lines.push("## Flaky Tasks (partial passing)");
		lines.push("");

		for (const task of flakyTasks) {
			const passing = Math.round(task.successRate * runsPerTask);
			lines.push(`### ${task.name} (${formatFiles(task.files)}) — ${passing}/${runsPerTask}`);
			lines.push("");
			lines.push("| Run | Status | Error | Tokens (in/out) | Time |");
			lines.push("|-----|--------|-------|-----------------|------|");

			for (const run of task.runs) {
				const status = run.success ? "✅" : "❌";
				const error = run.error ? truncateText(escapeMarkdown(run.error), 50) : "—";
				lines.push(
					`| ${run.runIndex + 1} | ${status} | ${error} | ${formatNumber(run.tokens.input)} / ${formatNumber(run.tokens.output)} | ${formatDuration(run.duration)} |`,
				);
			}
			lines.push("");
		}
	}

	const failedTasks = tasks.filter(t => t.successRate === 0);
	if (failedTasks.length > 0) {
		lines.push("## Failed Tasks (0% passing)");
		lines.push("");

		for (const task of failedTasks) {
			lines.push(`### ${task.name} (${formatFiles(task.files)}) — 0/${runsPerTask}`);
			lines.push("");

			const errors = task.runs.map(r => r.error).filter(Boolean);
			const uniqueErrors = [...new Set(errors)];

			if (uniqueErrors.length === 1) {
				lines.push(`**All runs failed with same error:** ${escapeMarkdown(uniqueErrors[0]!)}`);
			} else {
				lines.push("| Run | Status | Error | Tokens (in/out) | Time |");
				lines.push("|-----|--------|-------|-----------------|------|");

				for (const run of task.runs) {
					const error = run.error ? truncateText(escapeMarkdown(run.error), 50) : "—";
					lines.push(
						`| ${run.runIndex + 1} | ❌ | ${error} | ${formatNumber(run.tokens.input)} / ${formatNumber(run.tokens.output)} | ${formatDuration(run.duration)} |`,
					);
				}
			}
			lines.push("");

			const taskDef = findTaskPrompt(task);
			if (taskDef) {
				lines.push("**Prompt:**");
				lines.push(`> ${escapeMarkdown(taskDef.prompt).split("\n").join("\n> ")}`);
				lines.push("");
			}

			const sampleResponse = task.runs.find(r => r.agentResponse)?.agentResponse;
			if (sampleResponse) {
				lines.push("**Sample agent response (run 1):**");
				lines.push("```");
				lines.push(truncateText(sampleResponse, 500));
				lines.push("```");
				lines.push("");
			}

			const sampleDiff = task.runs.find(r => r.diff)?.diff;
			if (sampleDiff) {
				lines.push("**Diff (expected vs actual):**");
				lines.push("```diff");
				lines.push(truncateText(sampleDiff, 2000));
				lines.push("```");
				lines.push("");
			}
		}
	}

	return lines.join("\n");
}

function formatScore(value: number): string {
	return value.toFixed(2);
}

function findTaskPrompt(_task: TaskResult): { prompt: string } | undefined {
	// This is a placeholder - in actual use, we'd pass task definitions alongside results
	return undefined;
}

export function generateJsonReport(result: BenchmarkResult): string {
	return JSON.stringify(result, null, 2);
}

function appendCategorySummary(lines: string[], tasks: TaskResult[]): void {
	const runs = tasks.flatMap(task => task.runs);
	const categoryStats = new Map<string, { runs: number; verified: number; editUsed: number; success: number }>();
	const difficultyByCategory = new Map<string, number[]>();

	for (const task of tasks) {
		const metadata = task.runs.find(run => run.mutationCategory || run.difficultyScore !== undefined);
		const category = metadata?.mutationCategory ?? "unknown";
		if (typeof metadata?.difficultyScore === "number") {
			const scores = difficultyByCategory.get(category) ?? [];
			scores.push(metadata.difficultyScore);
			difficultyByCategory.set(category, scores);
		}
	}

	for (const run of runs) {
		const category = run.mutationCategory ?? "unknown";
		const entry = categoryStats.get(category) ?? { runs: 0, verified: 0, editUsed: 0, success: 0 };
		entry.runs += 1;
		if (run.verificationPassed) entry.verified += 1;
		if (run.patchApplied) entry.editUsed += 1;
		if (run.success) entry.success += 1;
		categoryStats.set(category, entry);
	}

	lines.push("## Category Summary");
	lines.push("");
	lines.push("| Category | Runs | Verified | Edit Used | Success | Min/Avg/Max Difficulty |");
	lines.push("|----------|------|----------|-----------|---------|------------------------|");

	const categories = [...categoryStats.keys()].sort();
	for (const category of categories) {
		const entry = categoryStats.get(category)!;
		const scores = difficultyByCategory.get(category) ?? [];
		lines.push(
			`| ${escapeMarkdown(category)} | ${entry.runs} | ${formatRate(entry.verified, entry.runs)} | ${formatRate(entry.editUsed, entry.runs)} | ${formatRate(entry.success, entry.runs)} | ${formatDifficultyStats(scores)} |`,
		);
	}
	lines.push("");
}

function appendMutationSummary(lines: string[], tasks: TaskResult[]): void {
	const runs = tasks.flatMap(task => task.runs);
	const mutationStats = new Map<
		string,
		{ category: string; runs: number; verified: number; editUsed: number; success: number }
	>();

	for (const run of runs) {
		const mutation = run.mutationType ?? "unknown";
		const category = run.mutationCategory ?? "unknown";
		const entry = mutationStats.get(mutation) ?? {
			category,
			runs: 0,
			verified: 0,
			editUsed: 0,
			success: 0,
		};
		entry.runs += 1;
		if (run.verificationPassed) entry.verified += 1;
		if (run.patchApplied) entry.editUsed += 1;
		if (run.success) entry.success += 1;
		mutationStats.set(mutation, entry);
	}

	lines.push("## Mutation Summary");
	lines.push("");
	lines.push("| Mutation | Category | Runs | Verified | Edit Used | Success |");
	lines.push("|----------|----------|------|----------|-----------|---------|");

	const mutations = [...mutationStats.keys()].sort();
	for (const mutation of mutations) {
		const entry = mutationStats.get(mutation)!;
		lines.push(
			`| ${escapeMarkdown(mutation)} | ${escapeMarkdown(entry.category)} | ${entry.runs} | ${formatRate(entry.verified, entry.runs)} | ${formatRate(entry.editUsed, entry.runs)} | ${formatRate(entry.success, entry.runs)} |`,
		);
	}
	lines.push("");
}

function appendDifficultySummary(lines: string[], tasks: TaskResult[]): void {
	const runs = tasks.flatMap(task => task.runs);
	const buckets = [
		{ label: "0-2", min: 0, max: 2 },
		{ label: "3-5", min: 3, max: 5 },
		{ label: "6-8", min: 6, max: 8 },
		{ label: "9+", min: 9, max: Number.POSITIVE_INFINITY },
	];
	const bucketStats = new Map<string, { runs: number; verified: number; editUsed: number; success: number }>();
	const unknown = { runs: 0, verified: 0, editUsed: 0, success: 0 };

	for (const run of runs) {
		const score = run.difficultyScore;
		if (typeof score !== "number") {
			unknown.runs += 1;
			if (run.verificationPassed) unknown.verified += 1;
			if (run.patchApplied) unknown.editUsed += 1;
			if (run.success) unknown.success += 1;
			continue;
		}
		const bucket = buckets.find(entry => score >= entry.min && score <= entry.max);
		const label = bucket?.label ?? "unknown";
		const entry = bucketStats.get(label) ?? { runs: 0, verified: 0, editUsed: 0, success: 0 };
		entry.runs += 1;
		if (run.verificationPassed) entry.verified += 1;
		if (run.patchApplied) entry.editUsed += 1;
		if (run.success) entry.success += 1;
		bucketStats.set(label, entry);
	}

	lines.push("## Difficulty Summary");
	lines.push("");
	lines.push("| Difficulty Score | Runs | Verified | Edit Used | Success |");
	lines.push("|------------------|------|----------|-----------|---------|");

	for (const bucket of buckets) {
		const entry = bucketStats.get(bucket.label) ?? { runs: 0, verified: 0, editUsed: 0, success: 0 };
		lines.push(
			`| ${bucket.label} | ${entry.runs} | ${formatRate(entry.verified, entry.runs)} | ${formatRate(entry.editUsed, entry.runs)} | ${formatRate(entry.success, entry.runs)} |`,
		);
	}
	if (unknown.runs > 0) {
		lines.push(
			`| unknown | ${unknown.runs} | ${formatRate(unknown.verified, unknown.runs)} | ${formatRate(unknown.editUsed, unknown.runs)} | ${formatRate(unknown.success, unknown.runs)} |`,
		);
	}
	lines.push("");
}

function formatDifficultyStats(scores: number[]): string {
	if (scores.length === 0) return "—";
	const min = Math.min(...scores);
	const max = Math.max(...scores);
	const avg = scores.reduce((sum, score) => sum + score, 0) / scores.length;
	return `${min} / ${avg.toFixed(1)} / ${max}`;
}