#!/usr/bin/env bun /** * Generate edit benchmark cases from React source. * * The goal is testing edit precision, not bug-finding ability. The mutation can * be trivial - what matters is whether the model can surgically apply the patch * in difficult contexts: * * - Repeated lines: The exact target line appears multiple times * - Long files: 300+ lines with edit in the middle * - Similar blocks: Multiple structurally similar functions * - Dense code: Minimal whitespace makes context harder to read * - Deep nesting: Whitespace-sensitive edits at high indent levels * * Difficulty modes control both FILE SELECTION and PROMPT DETAIL: * - easy: Short files, unique lines, line number given * - medium: Medium files, function context given * - hard: Long files with similar blocks, no location hint * - nightmare: Long files where target line repeats, minimal info */ import * as fs from "node:fs"; import { basename, dirname, join, relative } from "node:path"; import { parseArgs } from "node:util"; import { TempDir } from "@oh-my-pi/pi-utils"; import { $ } from "bun"; import { ALL_MUTATIONS, CATEGORY_MAP, type Mutation, type MutationInfo } from "./mutations"; const SCRIPT_DIR = import.meta.dir; const SUPPORTED_EXTENSIONS = new Set([".js", ".jsx", ".ts", ".tsx"]); using DEFAULT_REACT_DIR = TempDir.createSync("@react-source"); const DEFAULT_OUTPUT = join(SCRIPT_DIR, "fixtures.tar.gz"); const REACT_REPO_URL = "https://github.com/facebook/react.git"; const EXCLUDE_DIRS = new Set([ "__tests__", "__mocks__", "fixtures", "fixture", "dist", "build", "node_modules", "scripts", ]); type Difficulty = "easy" | "medium" | "hard" | "nightmare"; interface FileEntry { path: string; content: string; lineCount: number; repeatedLines: Map; similarBlockCount: number; density: number; maxIndent: number; functionRanges: Array<[string, number, number]>; } interface CaseResult { caseId: string; mutation: Mutation; info: MutationInfo; filePath: string; mutatedContent: string; originalContent: string; difficulty: Difficulty; difficultyScore: number; } interface Args { reactDir: string; output: string; countPerType: number; seed: number; categories: string | null; difficulty: string; minScore: number | null; dryRun: boolean; } function parseArguments(): Args { const { values } = parseArgs({ options: { "react-dir": { type: "string", default: DEFAULT_REACT_DIR.path() }, output: { type: "string", default: DEFAULT_OUTPUT }, "count-per-type": { type: "string", default: "20" }, seed: { type: "string", default: "42" }, categories: { type: "string" }, difficulty: { type: "string", default: "easy,medium,hard,nightmare" }, "min-score": { type: "string" }, "dry-run": { type: "boolean", default: false }, }, }); return { reactDir: values["react-dir"] ?? DEFAULT_REACT_DIR.path(), output: values.output ?? DEFAULT_OUTPUT, countPerType: parseInt(values["count-per-type"] ?? "20", 10), seed: parseInt(values.seed ?? "42", 10), categories: values.categories ?? null, difficulty: values.difficulty ?? "easy,medium,hard,nightmare", minScore: values["min-score"] ? parseInt(values["min-score"], 10) : null, dryRun: values["dry-run"] ?? false, }; } async function ensureReactSource(reactDir: string): Promise { if (fs.existsSync(reactDir)) { const packagesDir = join(reactDir, "packages"); if (fs.existsSync(packagesDir)) return; throw new Error(`Directory exists but missing packages/: ${reactDir}`); } console.log(`Cloning React repository to ${reactDir}...`); fs.mkdirSync(dirname(reactDir), { recursive: true }); const result = await $`git clone --depth 1 ${REACT_REPO_URL} ${reactDir}`.quiet().nothrow(); if (result.exitCode !== 0) { const decoder = new TextDecoder(); const stderr = result.stderr ? decoder.decode(result.stderr) : ""; throw new Error(`Failed to clone React: ${stderr.trim()}`); } console.log("Clone complete."); } function isExcluded(filePath: string): boolean { const parts = filePath.split("/"); for (const part of parts) { if (EXCLUDE_DIRS.has(part)) return true; } const filename = basename(filePath); if (filename.includes(".test.") || filename.includes(".spec.") || filename.includes(".fixture.")) { return true; } return false; } function hasStructure(content: string): boolean { return ["function ", "class ", "export ", "=>"].some((token) => content.includes(token)); } async function collectFiles(reactDir: string): Promise { const packagesDir = join(reactDir, "packages"); const candidates: string[] = []; async function walk(dir: string): Promise { if (isExcluded(dir)) return; const entries = await fs.promises.readdir(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { await walk(fullPath); } else if (entry.isFile()) { const ext = "." + entry.name.split(".").pop(); if (SUPPORTED_EXTENSIONS.has(ext) && !isExcluded(fullPath)) { candidates.push(fullPath); } } } } await walk(packagesDir); return candidates.sort(); } async function readFileAsync(filePath: string): Promise { return Bun.file(filePath).text(); } function findRepeatedLines(content: string, minRepeats = 2): Map { const lines = content.split("\n"); const positions = new Map(); const trivial = new Set(["{", "}", "};", ");", "],", "});", "return;", "break;", "continue;", ""]); for (let i = 0; i < lines.length; i++) { const stripped = lines[i].trim(); if (stripped.length < 10 || trivial.has(stripped)) continue; const pos = positions.get(stripped) ?? []; pos.push(i + 1); positions.set(stripped, pos); } const result = new Map(); for (const [line, pos] of positions) { if (pos.length >= minRepeats) { result.set(line, pos); } } return result; } function countSimilarBlocks(content: string): number { const funcPattern = /(?:function\s+(\w+)|const\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>)/g; const matches = [...content.matchAll(funcPattern)]; return matches.length; } function computeDensity(content: string): number { if (!content) return 0; const nonWs = content.replace(/[\s\n\t]/g, "").length; return nonWs / content.length; } function findMaxIndent(content: string): number { let maxIndent = 0; for (const line of content.split("\n")) { if (line.trim()) { const indent = line.length - line.trimStart().length; maxIndent = Math.max(maxIndent, indent); } } return maxIndent; } function findFunctionRanges(content: string): Array<[string, number, number]> { const lines = content.split("\n"); const ranges: Array<[string, number, number]> = []; const funcPattern = /(?:function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*=>)/; let braceDepth = 0; let currentFunc: string | null = null; let funcStart = 0; for (let i = 0; i < lines.length; i++) { const line = lines[i]; const match = funcPattern.exec(line); if (match && braceDepth === 0) { if (currentFunc) { ranges.push([currentFunc, funcStart, i]); } currentFunc = match[1] ?? match[2]; funcStart = i + 1; } braceDepth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length; if (currentFunc && braceDepth === 0 && line.includes("}")) { ranges.push([currentFunc, funcStart, i + 1]); currentFunc = null; } } if (currentFunc) { ranges.push([currentFunc, funcStart, lines.length]); } return ranges; } async function analyzeFile(filePath: string): Promise { const content = await readFileAsync(filePath); const lines = content.split("\n"); return { path: filePath, content, lineCount: lines.length, repeatedLines: findRepeatedLines(content), similarBlockCount: countSimilarBlocks(content), density: computeDensity(content), maxIndent: findMaxIndent(content), functionRanges: findFunctionRanges(content), }; } async function filterFiles(paths: string[], minLines = 30, maxLines = 800): Promise { const filtered: FileEntry[] = []; for (const filePath of paths) { const content = await readFileAsync(filePath); const lineCount = content.split("\n").length; if (lineCount < minLines || lineCount > maxLines) continue; if (!hasStructure(content)) continue; const entry = await analyzeFile(filePath); filtered.push(entry); } return filtered; } function scoreDifficulty(entry: FileEntry, lineNumber: number): number { let score = 0; const lines = entry.content.split("\n"); if (lineNumber < 1 || lineNumber > lines.length) return 0; const lineContent = lines[lineNumber - 1].trim(); // File length bonus if (entry.lineCount > 300) score += 3; else if (entry.lineCount > 150) score += 1; // Middle of file bonus const middleStart = entry.lineCount * 0.33; const middleEnd = entry.lineCount * 0.66; if (lineNumber >= middleStart && lineNumber <= middleEnd) score += 2; // Repeated line bonus if (entry.repeatedLines.has(lineContent)) { const repeatCount = entry.repeatedLines.get(lineContent)!.length; score += Math.min(repeatCount, 5); } // Similar function blocks bonus if (entry.similarBlockCount >= 5) score += 3; else if (entry.similarBlockCount >= 3) score += 1; // Dense code bonus if (entry.density > 0.75) score += 2; else if (entry.density > 0.65) score += 1; // Deep nesting bonus const lineIndent = lines[lineNumber - 1].length - lines[lineNumber - 1].trimStart().length; if (lineIndent >= 16) score += 2; else if (lineIndent >= 8) score += 1; // Generic context penalty const trivialLines = new Set(["{", "}", "};", ");", "});", "return;", "break;", "continue;", ""]); const contextStart = Math.max(0, lineNumber - 3); const contextEnd = Math.min(lines.length, lineNumber + 2); let trivialContext = 0; for (let i = contextStart; i < contextEnd; i++) { if (trivialLines.has(lines[i].trim())) trivialContext++; } if (trivialContext >= 3) score += 2; // Similar variable names nearby const funcName = findContainingFunction(entry, lineNumber); if (funcName) { for (const [name, start, end] of entry.functionRanges) { if (name === funcName) { const funcLines = lines.slice(start - 1, end); const identifiers = new Set(funcLines.join("\n").match(/\b[a-z][a-zA-Z0-9]*\b/g) ?? []); let similarPairs = 0; const ids = Array.from(identifiers); for (const a of ids) { for (const b of ids) { if (a !== b && (a.startsWith(b) || b.startsWith(a) || a.includes(b) || b.includes(a))) { similarPairs++; } } } if (similarPairs >= 3) score += 2; break; } } } return score; } function findContainingFunction(entry: FileEntry, lineNumber: number): string | null { for (const [name, start, end] of entry.functionRanges) { if (lineNumber >= start && lineNumber <= end) return name; } return null; } function getCandidatesForDifficulty(files: FileEntry[], difficulty: Difficulty): FileEntry[] { switch (difficulty) { case "easy": return files.filter((f) => f.lineCount < 150 && f.repeatedLines.size < 3); case "medium": return files.filter((f) => f.lineCount >= 100 && f.lineCount <= 300); case "hard": return files.filter((f) => f.lineCount > 200 && f.similarBlockCount >= 3); case "nightmare": return files.filter((f) => f.repeatedLines.size > 0 && f.lineCount > 200); default: return files; } } async function bunCheck(content: string, suffix: string): Promise { await using tempPath = await TempDir.create("@rb-bench-check-"); try { const absPath = join(tempPath.path(), suffix); await Bun.write(absPath, content); const result = await $`timeout 5s bun build ${absPath} --no-bundle`; return result.exitCode === 0; } catch { return false; } } async function isParsable(content: string, suffix: string): Promise { if (content.includes("@flow")) return true; try { return await bunCheck(content, suffix); } catch { return true; } } function regionAvailable(usedLines: Map, filePath: string, lineNumber: number): boolean { const used = usedLines.get(filePath) ?? []; return !used.some((ln) => Math.abs(ln - lineNumber) <= 3); } function recordRegion(usedLines: Map, filePath: string, lineNumber: number): void { const used = usedLines.get(filePath) ?? []; used.push(lineNumber); usedLines.set(filePath, used); } function buildPrompt( filePath: string, mutation: Mutation, info: MutationInfo, difficulty: Difficulty, entry: FileEntry, ): string { const header = `# Fix the bug in \`${basename(filePath)}\``; const isStructural = mutation.category === "structural"; const isMultiEdit = mutation.name === "identifier-multi-edit"; if (difficulty === "easy") { const detail = mutation.describe(info); const location = isStructural ? `The issue starts around line ${info.lineNumber}.` : `The issue is on line ${info.lineNumber}.`; return [header, detail, location, mutation.fixHint].join("\n\n"); } if (difficulty === "medium") { const detail = mutation.describe(info); const funcName = findContainingFunction(entry, info.lineNumber); let location: string; if (funcName) { location = `The issue is in the \`${funcName}\` function.`; } else { const ratio = entry.lineCount ? info.lineNumber / entry.lineCount : 0; if (ratio < 0.33) location = "The issue is near the top of the file."; else if (ratio < 0.66) location = "The issue is around the middle of the file."; else location = "The issue is near the end of the file."; } if (isMultiEdit) { location += " The same error appears in multiple places."; } return [header, detail, location, mutation.fixHint].join("\n\n"); } if (difficulty === "hard") { const detail = mutation.describe(info); if (isMultiEdit) { return [header, detail, "Find and fix all occurrences of this issue."].join("\n\n"); } if (isStructural) { return [header, detail, "The fix may involve multiple lines."].join("\n\n"); } return [header, detail, "Find and fix this issue."].join("\n\n"); } // nightmare if (isStructural) { return [header, "There is a structural bug in this file.", "Track it down and fix it with a minimal edit."].join( "\n\n", ); } if (isMultiEdit) { return [ header, "An identifier is consistently misspelled throughout this file.", "Find all occurrences and fix them.", ].join("\n\n"); } return [ header, "There is a subtle bug in this file. The fix is a one-character change.", "Track it down and fix it with a minimal edit.", ].join("\n\n"); } function createSeededRng(seed: number): () => number { let state = seed; return () => { state = (state * 1103515245 + 12345) & 0x7fffffff; return state / 0x7fffffff; }; } async function generateCase( rng: () => number, mutation: Mutation, files: FileEntry[], usedLines: Map, difficulty: Difficulty, minScore: number | null, attemptLimit = 100, ): Promise { let candidates = getCandidatesForDifficulty(files, difficulty); if (candidates.length === 0) candidates = files; let applicable = candidates.filter((entry) => mutation.canApply(entry.content)); if (applicable.length === 0 && candidates !== files) { applicable = files.filter((entry) => mutation.canApply(entry.content)); } if (applicable.length === 0) return null; const targetMinScore = minScore ?? { easy: 0, medium: 2, hard: 5, nightmare: 8 }[difficulty] ?? 0; for (let attempt = 0; attempt < attemptLimit; attempt++) { const entry = applicable[Math.floor(rng() * applicable.length)]; const [mutatedContent, info] = mutation.mutate(entry.content, rng); if (mutatedContent === entry.content) continue; if (!regionAvailable(usedLines, entry.path, info.lineNumber)) continue; const suffix = "." + entry.path.split(".").pop(); if (!(await isParsable(mutatedContent, suffix))) continue; const diffScore = scoreDifficulty(entry, info.lineNumber); if (difficulty === "nightmare") { const lines = entry.content.split("\n"); if (info.lineNumber <= lines.length) { const lineContent = lines[info.lineNumber - 1].trim(); if (!entry.repeatedLines.has(lineContent)) continue; } } if (diffScore < targetMinScore) continue; recordRegion(usedLines, entry.path, info.lineNumber); return { caseId: "", mutation, info, filePath: entry.path, mutatedContent, originalContent: entry.content, difficulty, difficultyScore: diffScore, }; } return null; } function ensureTrailingNewline(content: string): string { return content.replace(/\n*$/, "\n"); } interface TarEntry { name: string; content: string; } async function writeTarball(entries: TarEntry[], outputPath: string): Promise { const data: Record = {}; for (const entry of entries) { data[entry.name] = entry.content; } await Bun.Archive.write(outputPath, data, { compress: "gzip" }); } function buildCaseEntries(result: CaseResult, reactDir: string): TarEntry[] { const filename = basename(result.filePath); const relativePath = relative(reactDir, result.filePath); const caseDir = `fixtures/${result.caseId}`; const lines = result.originalContent.split("\n"); const lineContent = result.info.lineNumber <= lines.length ? lines[result.info.lineNumber - 1].trim() : ""; const entry: FileEntry = { path: result.filePath, content: result.originalContent, lineCount: lines.length, repeatedLines: findRepeatedLines(result.originalContent), similarBlockCount: countSimilarBlocks(result.originalContent), density: computeDensity(result.originalContent), maxIndent: findMaxIndent(result.originalContent), functionRanges: findFunctionRanges(result.originalContent), }; const isRepeated = entry.repeatedLines.has(lineContent); const prompt = buildPrompt(result.filePath, result.mutation, result.info, result.difficulty, entry); const metadata = { mutation_type: result.mutation.name, mutation_category: result.mutation.category, difficulty: result.difficulty, difficulty_score: result.difficultyScore, line_number: result.info.lineNumber, original_snippet: result.info.originalSnippet, mutated_snippet: result.info.mutatedSnippet, file_path: relativePath, context: { file_lines: entry.lineCount, is_repeated_line: isRepeated, repeat_count: entry.repeatedLines.get(lineContent)?.length ?? 0, similar_block_count: entry.similarBlockCount, density: Math.round(entry.density * 1000) / 1000, line_indent: result.info.lineNumber <= lines.length ? lines[result.info.lineNumber - 1].length - lines[result.info.lineNumber - 1].trimStart().length : 0, containing_function: findContainingFunction(entry, result.info.lineNumber), }, }; return [ { name: `${caseDir}/input/${filename}`, content: ensureTrailingNewline(result.mutatedContent) }, { name: `${caseDir}/expected/${filename}`, content: ensureTrailingNewline(result.originalContent) }, { name: `${caseDir}/prompt.md`, content: prompt }, { name: `${caseDir}/metadata.json`, content: JSON.stringify(metadata, null, 2) }, ]; } function chooseDifficulties(difficulties: Difficulty[], count: number): Difficulty[] { return Array.from({ length: count }, (_, i) => difficulties[i % difficulties.length]); } async function main(): Promise { const args = parseArguments(); const rng = createSeededRng(args.seed); const reactDir = args.reactDir; await ensureReactSource(reactDir); const rawFiles = await collectFiles(reactDir); const files = await filterFiles(rawFiles); if (files.length === 0) { console.error("No eligible files found."); return 1; } console.log(`Analyzed ${files.length} files`); const withRepeats = files.filter((f) => f.repeatedLines.size > 0).length; const longFiles = files.filter((f) => f.lineCount > 200).length; const similarBlocks = files.filter((f) => f.similarBlockCount >= 3).length; console.log(` ${withRepeats} with repeated lines, ${longFiles} long files, ${similarBlocks} with similar blocks`); const difficulties = args.difficulty .split(",") .map((s) => s.trim()) .filter(Boolean) as Difficulty[]; if (difficulties.length === 0) { console.error("No difficulties specified."); return 1; } let mutations = ALL_MUTATIONS; if (args.categories) { const categories = new Set( args.categories .split(",") .map((s) => s.trim()) .filter(Boolean), ); const unknown = Array.from(categories).filter((c) => !(c in CATEGORY_MAP)); if (unknown.length > 0) { console.error(`Unknown categories: ${unknown.join(", ")}`); return 1; } mutations = ALL_MUTATIONS.filter((m) => categories.has(m.category)); } const usedLines = new Map(); const results: CaseResult[] = []; const fallbackOrder: Difficulty[] = ["hard", "medium", "easy"]; for (const mutation of mutations) { const difficultiesForType = chooseDifficulties(difficulties, args.countPerType); let generated = 0; for (let index = 0; index < args.countPerType; index++) { const difficulty = difficultiesForType[index]; let result = await generateCase(rng, mutation, files, usedLines, difficulty, args.minScore); if (!result) { for (const fallback of fallbackOrder) { if (fallback === difficulty) continue; result = await generateCase(rng, mutation, files, usedLines, fallback, 0); if (result) { console.log(`Note: ${mutation.name} case ${index + 1} fell back from ${difficulty} to ${fallback}`); break; } } } if (!result) { console.log(`Warning: Skipping ${mutation.name} case ${index + 1} (no applicable files left)`); continue; } generated++; const caseId = `${mutation.category}-${mutation.name}-${String(index + 1).padStart(3, "0")}`; results.push({ ...result, caseId, }); } if (generated === 0) { console.log(`Warning: No cases generated for ${mutation.name} (mutation may be too rare)`); } else if (generated < args.countPerType) { console.log(`Note: Only ${generated}/${args.countPerType} cases generated for ${mutation.name}`); } } if (args.dryRun) { const byDifficulty = new Map(); for (const result of results) { const list = byDifficulty.get(result.difficulty) ?? []; list.push(result); byDifficulty.set(result.difficulty, list); } for (const diff of difficulties) { const cases = byDifficulty.get(diff) ?? []; console.log(`\n${diff.toUpperCase()} (${cases.length} cases):`); for (const result of cases.slice(0, 5)) { const rel = relative(reactDir, result.filePath); console.log(` ${result.caseId}: ${rel}`); console.log(` score=${result.difficultyScore}, lines=${result.originalContent.split("\n").length}`); } if (cases.length > 5) { console.log(` ... and ${cases.length - 5} more`); } } const scores = results.map((r) => r.difficultyScore); const min = Math.min(...scores); const max = Math.max(...scores); const avg = scores.reduce((a, b) => a + b, 0) / scores.length; console.log(`\nScore distribution: min=${min}, max=${max}, avg=${avg.toFixed(1)}`); return 0; } const tarEntries: TarEntry[] = []; for (const result of results) { tarEntries.push(...buildCaseEntries(result, reactDir)); } await writeTarball(tarEntries, args.output); console.log(`Generated ${results.length} cases in ${args.output}`); const scores = results.map((r) => r.difficultyScore); const min = Math.min(...scores); const max = Math.max(...scores); const avg = scores.reduce((a, b) => a + b, 0) / scores.length; console.log(`Score distribution: min=${min}, max=${max}, avg=${avg.toFixed(1)}`); return 0; } process.exit(await main());