/** * File verification for edit benchmark. * * Compares output files against expected fixtures with byte-for-byte equality. */ import * as fs from "node:fs/promises"; import * as path from "node:path"; import { diffLines } from "diff"; import { formatContent } from "./formatter"; export interface VerificationResult { success: boolean; error?: string; duration: number; indentScore?: number; formattedEquivalent?: boolean; diffStats?: DiffStats; diff?: string; } export interface DiffStats { linesChanged: number; charsChanged: number; } async function listFiles(rootDir: string, subPath = ""): Promise { const entries = await fs.readdir(path.join(rootDir, subPath), { withFileTypes: true }); const files: string[] = []; for (const entry of entries) { const relativePath = path.join(subPath, entry.name); if (entry.isDirectory()) { files.push(...(await listFiles(rootDir, relativePath))); } else if (entry.isFile()) { files.push(relativePath); } } return files.sort(); } function formatFileList(files: string[]): string { return files.length === 0 ? "(none)" : files.join(", "); } function createCompactDiff(expected: string, actual: string, contextLines = 3): string { const changes = diffLines(expected, actual); const output: string[] = []; let lineNum = 1; for (let i = 0; i < changes.length; i++) { const change = changes[i]!; const lines = splitLines(change.value); if (change.added || change.removed) { // Show context before (from previous unchanged chunk) if (i > 0 && !changes[i - 1]!.added && !changes[i - 1]!.removed) { const prevLines = splitLines(changes[i - 1]!.value); const contextStart = Math.max(0, prevLines.length - contextLines); if (contextStart > 0) { output.push(`@@ -${lineNum - (prevLines.length - contextStart)} @@`); } for (let j = contextStart; j < prevLines.length; j++) { output.push(` ${prevLines[j]}`); } } // Show the change const prefix = change.added ? "+" : "-"; for (const line of lines) { output.push(`${prefix}${line}`); } // Show context after (from next unchanged chunk) if (i + 1 < changes.length && !changes[i + 1]!.added && !changes[i + 1]!.removed) { const nextLines = splitLines(changes[i + 1]!.value); const contextEnd = Math.min(nextLines.length, contextLines); for (let j = 0; j < contextEnd; j++) { output.push(` ${nextLines[j]}`); } } if (!change.added) { lineNum += lines.length; } } else { lineNum += lines.length; } } return output.join("\n"); } export async function verifyExpectedFiles(expectedDir: string, actualDir: string): Promise { return verifyExpectedFileSubset(expectedDir, actualDir); } export async function verifyExpectedFileSubset( expectedDir: string, actualDir: string, files?: string[], ): Promise { const startTime = Date.now(); let totalIndentScore = 0; let fileCount = 0; try { const expectedFixtureFiles = await listFiles(expectedDir); const expectedFiles = files?.length ? files.slice().sort() : expectedFixtureFiles; const actualFiles = await listFiles(actualDir); const missingFiles = expectedFiles.filter((file) => !actualFiles.includes(file)); const extraFiles = actualFiles.filter((file) => !expectedFiles.includes(file)); const missingExpected = expectedFiles.filter((file) => !expectedFixtureFiles.includes(file)); if (missingExpected.length > 0) { return { success: false, error: `Expected files missing from fixture: ${formatFileList(missingExpected)}`, duration: Date.now() - startTime, }; } if (missingFiles.length > 0 || (files === undefined && extraFiles.length > 0)) { const parts: string[] = []; if (missingFiles.length > 0) { parts.push(`Missing files: ${formatFileList(missingFiles)}`); } if (files === undefined && extraFiles.length > 0) { parts.push(`Unexpected files: ${formatFileList(extraFiles)}`); } return { success: false, error: parts.join("; "), duration: Date.now() - startTime, }; } for (const file of expectedFiles) { const expectedPath = path.join(expectedDir, file); const actualPath = path.join(actualDir, file); const expectedRaw = await Bun.file(expectedPath).text(); const actualRaw = await Bun.file(actualPath).text(); const expectedNormalized = normalizeLineEndings(expectedRaw); const actualNormalized = normalizeLineEndings(actualRaw); const expectedFormatted = await formatContent(expectedPath, expectedNormalized); const actualFormatted = await formatContent(actualPath, actualNormalized); const formattedEquivalent = expectedFormatted.formatted === actualFormatted.formatted; // Indent score: distance between agent's raw output and formatted output // This measures how much the formatter had to fix the agent's indentation const fileIndentScore = computeIndentDistanceForDiff(actualNormalized, actualFormatted.formatted); totalIndentScore += fileIndentScore; fileCount++; // Fail if formatted versions don't match (content is wrong, not just whitespace) if (!formattedEquivalent) { const diffOutput = createCompactDiff(expectedFormatted.formatted, actualFormatted.formatted); const diffStats = computeDiffStats(expectedFormatted.formatted, actualFormatted.formatted); return { success: false, error: `File mismatch for ${file}`, duration: Date.now() - startTime, diff: diffOutput, diffStats, indentScore: fileIndentScore, formattedEquivalent, }; } } return { success: true, duration: Date.now() - startTime, indentScore: fileCount > 0 ? totalIndentScore / fileCount : 0, formattedEquivalent: true, diffStats: { linesChanged: 0, charsChanged: 0 }, }; } catch (err) { return { success: false, error: err instanceof Error ? err.message : String(err), duration: Date.now() - startTime, }; } } function computeDiffStats(expected: string, actual: string): DiffStats { const changes = diffLines(expected, actual); let linesChanged = 0; let charsChanged = 0; for (const change of changes) { if (!change.added && !change.removed) { continue; } const lines = splitLines(change.value); linesChanged += lines.length; charsChanged += change.value.length; } return { linesChanged, charsChanged }; } function computeIndentDistanceForDiff(expected: string, actual: string): number { const changes = diffLines(expected, actual); let totalDistance = 0; let samples = 0; let pendingRemoved: string[] = []; let pendingAdded: string[] = []; const flush = () => { const max = Math.max(pendingRemoved.length, pendingAdded.length); for (let i = 0; i < max; i++) { const removedLine = pendingRemoved[i] ?? ""; const addedLine = pendingAdded[i] ?? ""; totalDistance += Math.abs(countIndent(removedLine) - countIndent(addedLine)); samples += 1; } pendingRemoved = []; pendingAdded = []; }; for (const change of changes) { const lines = splitLines(change.value); if (change.removed) { pendingRemoved.push(...lines); continue; } if (change.added) { pendingAdded.push(...lines); continue; } if (pendingRemoved.length > 0 || pendingAdded.length > 0) { flush(); } } if (pendingRemoved.length > 0 || pendingAdded.length > 0) { flush(); } return samples > 0 ? totalDistance / samples : 0; } function normalizeLineEndings(value: string): string { return value.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); } function splitLines(value: string): string[] { return value.split("\n").filter((line, idx, arr) => idx < arr.length - 1 || line); } function countIndent(line: string): number { let count = 0; for (const char of line) { if (char === " ") { count += 1; } else if (char === "\t") { count += 2; } else { break; } } return count; }