feat: enhanced patch applicator with fallback variants and fuzzy matching
- Enhanced patch applicator with fallback variant generation and improved fuzzy matching. - Added support for ellipsis placeholders, top-of-file anchors, and comment-prefix normalization. - Renamed operation and moveTo parameters to op and rename across patch tool interfaces. - Added batch processing system to benchmark runner for parallel task execution. - Added comprehensive benchmark reports for Claude Haiku and GPT-5.1-codex-mini models.
This commit is contained in:
can1357
@@ -1,8 +1,17 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added fallback variant generation for patch matching with trimmed context, collapsed duplicates, and single-line reduction
|
||||
- Added comment-prefix normalization for improved fuzzy matching across different comment styles
|
||||
- Added support for ellipsis (...) and ellipsis character (…) as context placeholders in patch hunks
|
||||
- Added line number prefix stripping from diff content when sequential numbering is detected
|
||||
- Added support for 'top of file' and 'start of file' anchors in patch headers
|
||||
- Added function signature fallback matching for context anchors with empty parentheses
|
||||
- Added relaxed fuzzy threshold fallback when initial matching fails
|
||||
- Added context-relative matching for duplicate lines with preference for second forward match
|
||||
- Added `--no-title` flag to disable automatic session title generation
|
||||
- Added environment variable support for edit tool configuration (OMP_EDIT_VARIANT, OMP_EDIT_FUZZY, OMP_EDIT_FUZZY_THRESHOLD)
|
||||
- Added `allowFuzzy` option to control fuzzy matching behavior in patch operations
|
||||
@@ -23,6 +32,12 @@
|
||||
|
||||
### Changed
|
||||
|
||||
- Improved indentation adjustment to preserve original file context while fixing new lines
|
||||
- Enhanced patch parser to handle implicit context lines without space prefixes
|
||||
- Updated patch operation schema to use 'op' instead of 'operation' and 'rename' instead of 'moveTo'
|
||||
- Improved ambiguous context resolution by falling back to unique old lines when multiple matches occur
|
||||
- Enhanced fuzzy matching to try comment-prefix normalized matches before unicode normalization
|
||||
- Updated patch prompts with clearer anchor selection rules and context requirements
|
||||
- Changed default behavior of read tool to omit line numbers by default
|
||||
- Updated patch tool prompts to clarify verbatim context requirements and hunk formatting rules
|
||||
- Enhanced fuzzy matching algorithms to respect the `allowFuzzy` setting for more precise control
|
||||
@@ -73,6 +88,12 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed patch application to handle repeated context blocks by collapsing duplicates during matching
|
||||
- Fixed context line preservation to maintain original file indentation when fuzzy matched
|
||||
- Fixed ambiguous context matching to resolve duplicates using adjacent @@ anchor positioning
|
||||
- Fixed patch parser to correctly handle bare *** terminators and model hallucination markers
|
||||
- Fixed line hint parsing to support ranges (lines 3-5) and various format variations
|
||||
- Fixed function context matching to handle signatures with and without empty parentheses
|
||||
- Fixed session title generation to respect OMP_NO_TITLE environment variable
|
||||
- Fixed Python module discovery to use import.meta.dir for ES module compatibility
|
||||
- Fixed LSP writethrough batching to flush when delete operations complete a batch
|
||||
|
||||
@@ -59,6 +59,11 @@ interface Replacement {
|
||||
newLines: string[];
|
||||
}
|
||||
|
||||
interface HunkVariant {
|
||||
oldLines: string[];
|
||||
newLines: string[];
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Replacement Computation
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -69,27 +74,7 @@ function adjustLinesIndentation(patternLines: string[], actualLines: string[], n
|
||||
return newLines;
|
||||
}
|
||||
|
||||
let patternMin = Infinity;
|
||||
for (const line of patternLines) {
|
||||
if (line.trim().length > 0) {
|
||||
patternMin = Math.min(patternMin, countLeadingWhitespace(line));
|
||||
}
|
||||
}
|
||||
if (patternMin === Infinity) patternMin = 0;
|
||||
|
||||
let actualMin = Infinity;
|
||||
for (const line of actualLines) {
|
||||
if (line.trim().length > 0) {
|
||||
actualMin = Math.min(actualMin, countLeadingWhitespace(line));
|
||||
}
|
||||
}
|
||||
if (actualMin === Infinity) actualMin = 0;
|
||||
|
||||
const delta = actualMin - patternMin;
|
||||
if (delta === 0) {
|
||||
return newLines;
|
||||
}
|
||||
|
||||
// Detect indent character from actual content
|
||||
let indentChar = " ";
|
||||
for (const line of actualLines) {
|
||||
const ws = getLeadingWhitespace(line);
|
||||
@@ -99,18 +84,232 @@ function adjustLinesIndentation(patternLines: string[], actualLines: string[], n
|
||||
}
|
||||
}
|
||||
|
||||
return newLines.map((line) => {
|
||||
if (line.trim().length === 0) {
|
||||
return line;
|
||||
// Build a map from trimmed content to available (pattern index, actual index) pairs
|
||||
// This lets us find context lines and their corresponding actual content
|
||||
const contentToIndices = new Map<string, Array<{ patternIdx: number; actualIdx: number }>>();
|
||||
for (let i = 0; i < Math.min(patternLines.length, actualLines.length); i++) {
|
||||
const trimmed = patternLines[i].trim();
|
||||
if (trimmed.length === 0) continue;
|
||||
const arr = contentToIndices.get(trimmed);
|
||||
if (arr) {
|
||||
arr.push({ patternIdx: i, actualIdx: i });
|
||||
} else {
|
||||
contentToIndices.set(trimmed, [{ patternIdx: i, actualIdx: i }]);
|
||||
}
|
||||
if (delta > 0) {
|
||||
return indentChar.repeat(delta) + line;
|
||||
}
|
||||
|
||||
// Compute fallback delta from all non-empty lines (for truly new lines)
|
||||
let totalDelta = 0;
|
||||
let deltaCount = 0;
|
||||
for (let i = 0; i < Math.min(patternLines.length, actualLines.length); i++) {
|
||||
if (patternLines[i].trim().length > 0 && actualLines[i].trim().length > 0) {
|
||||
const pIndent = countLeadingWhitespace(patternLines[i]);
|
||||
const aIndent = countLeadingWhitespace(actualLines[i]);
|
||||
totalDelta += aIndent - pIndent;
|
||||
deltaCount++;
|
||||
}
|
||||
const toRemove = Math.min(-delta, countLeadingWhitespace(line));
|
||||
return line.slice(toRemove);
|
||||
}
|
||||
const avgDelta = deltaCount > 0 ? Math.round(totalDelta / deltaCount) : 0;
|
||||
|
||||
// Track which indices we've used to handle duplicate content correctly
|
||||
const usedIndices = new Set<number>();
|
||||
|
||||
return newLines.map((newLine) => {
|
||||
if (newLine.trim().length === 0) {
|
||||
return newLine;
|
||||
}
|
||||
|
||||
const trimmed = newLine.trim();
|
||||
const indices = contentToIndices.get(trimmed);
|
||||
|
||||
// Check if this is a context line (same trimmed content exists in pattern)
|
||||
if (indices) {
|
||||
for (const { patternIdx, actualIdx } of indices) {
|
||||
if (!usedIndices.has(patternIdx)) {
|
||||
usedIndices.add(patternIdx);
|
||||
// Use actual file content directly for context lines
|
||||
return actualLines[actualIdx];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This is a new/added line - apply average delta
|
||||
if (avgDelta > 0) {
|
||||
return indentChar.repeat(avgDelta) + newLine;
|
||||
}
|
||||
if (avgDelta < 0) {
|
||||
const toRemove = Math.min(-avgDelta, countLeadingWhitespace(newLine));
|
||||
return newLine.slice(toRemove);
|
||||
}
|
||||
return newLine;
|
||||
});
|
||||
}
|
||||
|
||||
function trimCommonContext(oldLines: string[], newLines: string[]): HunkVariant | undefined {
|
||||
let start = 0;
|
||||
let endOld = oldLines.length;
|
||||
let endNew = newLines.length;
|
||||
|
||||
while (start < endOld && start < endNew && oldLines[start] === newLines[start]) {
|
||||
start++;
|
||||
}
|
||||
|
||||
while (endOld > start && endNew > start && oldLines[endOld - 1] === newLines[endNew - 1]) {
|
||||
endOld--;
|
||||
endNew--;
|
||||
}
|
||||
|
||||
if (start === 0 && endOld === oldLines.length && endNew === newLines.length) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const trimmedOld = oldLines.slice(start, endOld);
|
||||
const trimmedNew = newLines.slice(start, endNew);
|
||||
if (trimmedOld.length === 0 && trimmedNew.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return { oldLines: trimmedOld, newLines: trimmedNew };
|
||||
}
|
||||
|
||||
function collapseConsecutiveSharedLines(oldLines: string[], newLines: string[]): HunkVariant | undefined {
|
||||
const shared = new Set(oldLines.filter((line) => newLines.includes(line)));
|
||||
const collapse = (lines: string[]): string[] => {
|
||||
const out: string[] = [];
|
||||
let i = 0;
|
||||
while (i < lines.length) {
|
||||
const line = lines[i];
|
||||
out.push(line);
|
||||
let j = i + 1;
|
||||
while (j < lines.length && lines[j] === line && shared.has(line)) {
|
||||
j++;
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const collapsedOld = collapse(oldLines);
|
||||
const collapsedNew = collapse(newLines);
|
||||
if (collapsedOld.length === oldLines.length && collapsedNew.length === newLines.length) {
|
||||
return undefined;
|
||||
}
|
||||
return { oldLines: collapsedOld, newLines: collapsedNew };
|
||||
}
|
||||
|
||||
function collapseRepeatedBlocks(oldLines: string[], newLines: string[]): HunkVariant | undefined {
|
||||
const shared = new Set(oldLines.filter((line) => newLines.includes(line)));
|
||||
const collapse = (lines: string[]): string[] => {
|
||||
const output = [...lines];
|
||||
let changed = false;
|
||||
let i = 0;
|
||||
while (i < output.length) {
|
||||
let collapsed = false;
|
||||
for (let size = Math.floor((output.length - i) / 2); size >= 2; size--) {
|
||||
const first = output.slice(i, i + size);
|
||||
const second = output.slice(i + size, i + size * 2);
|
||||
if (first.length !== second.length || first.length === 0) continue;
|
||||
if (!first.every((line) => shared.has(line))) continue;
|
||||
let same = true;
|
||||
for (let idx = 0; idx < size; idx++) {
|
||||
if (first[idx] !== second[idx]) {
|
||||
same = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (same) {
|
||||
output.splice(i + size, size);
|
||||
changed = true;
|
||||
collapsed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!collapsed) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return changed ? output : lines;
|
||||
};
|
||||
|
||||
const collapsedOld = collapse(oldLines);
|
||||
const collapsedNew = collapse(newLines);
|
||||
if (collapsedOld.length === oldLines.length && collapsedNew.length === newLines.length) {
|
||||
return undefined;
|
||||
}
|
||||
return { oldLines: collapsedOld, newLines: collapsedNew };
|
||||
}
|
||||
|
||||
function reduceToSingleLineChange(oldLines: string[], newLines: string[]): HunkVariant | undefined {
|
||||
if (oldLines.length !== newLines.length || oldLines.length === 0) return undefined;
|
||||
let changedIndex: number | undefined;
|
||||
for (let i = 0; i < oldLines.length; i++) {
|
||||
if (oldLines[i] !== newLines[i]) {
|
||||
if (changedIndex !== undefined) return undefined;
|
||||
changedIndex = i;
|
||||
}
|
||||
}
|
||||
if (changedIndex === undefined) return undefined;
|
||||
return { oldLines: [oldLines[changedIndex]], newLines: [newLines[changedIndex]] };
|
||||
}
|
||||
|
||||
function buildFallbackVariants(hunk: DiffHunk): HunkVariant[] {
|
||||
const variants: HunkVariant[] = [];
|
||||
const base: HunkVariant = { oldLines: hunk.oldLines, newLines: hunk.newLines };
|
||||
|
||||
const trimmed = trimCommonContext(base.oldLines, base.newLines);
|
||||
if (trimmed) variants.push(trimmed);
|
||||
|
||||
const deduped = collapseConsecutiveSharedLines(
|
||||
trimmed?.oldLines ?? base.oldLines,
|
||||
trimmed?.newLines ?? base.newLines,
|
||||
);
|
||||
if (deduped) variants.push(deduped);
|
||||
|
||||
const collapsed = collapseRepeatedBlocks(
|
||||
deduped?.oldLines ?? trimmed?.oldLines ?? base.oldLines,
|
||||
deduped?.newLines ?? trimmed?.newLines ?? base.newLines,
|
||||
);
|
||||
if (collapsed) variants.push(collapsed);
|
||||
|
||||
const singleLine = reduceToSingleLineChange(trimmed?.oldLines ?? base.oldLines, trimmed?.newLines ?? base.newLines);
|
||||
if (singleLine) variants.push(singleLine);
|
||||
|
||||
const seen = new Set<string>();
|
||||
return variants.filter((variant) => {
|
||||
if (variant.oldLines.length === 0 && variant.newLines.length === 0) return false;
|
||||
const key = `${variant.oldLines.join("\n")}||${variant.newLines.join("\n")}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function findContextRelativeMatch(
|
||||
lines: string[],
|
||||
patternLine: string,
|
||||
contextIndex: number,
|
||||
preferSecondForwardMatch: boolean,
|
||||
): number | undefined {
|
||||
const trimmed = patternLine.trim();
|
||||
const forwardMatches: number[] = [];
|
||||
for (let i = contextIndex + 1; i < lines.length; i++) {
|
||||
if (lines[i].trim() === trimmed) {
|
||||
forwardMatches.push(i);
|
||||
}
|
||||
}
|
||||
if (forwardMatches.length > 0) {
|
||||
if (preferSecondForwardMatch && forwardMatches.length > 1) {
|
||||
return forwardMatches[1];
|
||||
}
|
||||
return forwardMatches[0];
|
||||
}
|
||||
for (let i = contextIndex - 1; i >= 0; i--) {
|
||||
if (lines[i].trim() === trimmed) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Get hint index from hunk's line number */
|
||||
function getHunkHintIndex(hunk: DiffHunk, currentIndex: number): number | undefined {
|
||||
if (hunk.oldStartLine === undefined) return undefined;
|
||||
@@ -311,6 +510,51 @@ function findSequenceWithHint(
|
||||
return primaryResult;
|
||||
}
|
||||
|
||||
function attemptSequenceFallback(
|
||||
lines: string[],
|
||||
hunk: DiffHunk,
|
||||
currentIndex: number,
|
||||
lineHint: number | undefined,
|
||||
allowFuzzy: boolean,
|
||||
): number | undefined {
|
||||
if (hunk.oldLines.length === 0) return undefined;
|
||||
const matchHint = getHunkHintIndex(hunk, currentIndex);
|
||||
const fallbackResult = findSequenceWithHint(
|
||||
lines,
|
||||
hunk.oldLines,
|
||||
currentIndex,
|
||||
matchHint ?? lineHint,
|
||||
false,
|
||||
allowFuzzy,
|
||||
);
|
||||
if (fallbackResult.index !== undefined && (fallbackResult.matchCount ?? 1) <= 1) {
|
||||
const nextIndex = fallbackResult.index + 1;
|
||||
if (nextIndex <= lines.length - hunk.oldLines.length) {
|
||||
const secondMatch = seekSequence(lines, hunk.oldLines, nextIndex, false, { allowFuzzy });
|
||||
if (secondMatch.index !== undefined) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return fallbackResult.index;
|
||||
}
|
||||
|
||||
for (const variant of buildFallbackVariants(hunk)) {
|
||||
if (variant.oldLines.length === 0) continue;
|
||||
const variantResult = findSequenceWithHint(
|
||||
lines,
|
||||
variant.oldLines,
|
||||
currentIndex,
|
||||
matchHint ?? lineHint,
|
||||
false,
|
||||
allowFuzzy,
|
||||
);
|
||||
if (variantResult.index !== undefined && (variantResult.matchCount ?? 1) <= 1) {
|
||||
return variantResult.index;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a hunk using character-based fuzzy matching.
|
||||
* Used when the hunk contains only -/+ lines without context.
|
||||
@@ -328,10 +572,22 @@ function applyCharacterMatch(
|
||||
const normalizedContent = normalizeToLF(originalContent);
|
||||
const normalizedOldText = normalizeToLF(oldText);
|
||||
|
||||
const matchOutcome = findMatch(normalizedContent, normalizedOldText, {
|
||||
let matchOutcome = findMatch(normalizedContent, normalizedOldText, {
|
||||
allowFuzzy,
|
||||
threshold: fuzzyThreshold,
|
||||
});
|
||||
if (!matchOutcome.match && allowFuzzy) {
|
||||
const relaxedThreshold = Math.min(fuzzyThreshold, 0.92);
|
||||
if (relaxedThreshold < fuzzyThreshold) {
|
||||
const relaxedOutcome = findMatch(normalizedContent, normalizedOldText, {
|
||||
allowFuzzy,
|
||||
threshold: relaxedThreshold,
|
||||
});
|
||||
if (relaxedOutcome.match) {
|
||||
matchOutcome = relaxedOutcome;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for multiple exact occurrences
|
||||
if (matchOutcome.occurrences && matchOutcome.occurrences > 1) {
|
||||
@@ -389,6 +645,7 @@ function computeReplacements(
|
||||
let lineIndex = 0;
|
||||
|
||||
for (const hunk of hunks) {
|
||||
let contextIndex: number | undefined;
|
||||
if (hunk.oldStartLine !== undefined && hunk.oldStartLine < 1) {
|
||||
throw new ApplyPatchError(
|
||||
`Line hint ${hunk.oldStartLine} is out of range for ${path} (line numbers start at 1)`,
|
||||
@@ -408,37 +665,41 @@ function computeReplacements(
|
||||
if (hunk.changeContext !== undefined) {
|
||||
// Use hierarchical context matching for nested @@ anchors and space-separated contexts
|
||||
const result = findHierarchicalContext(originalLines, hunk.changeContext, lineIndex, lineHint, allowFuzzy);
|
||||
|
||||
if (result.matchCount !== undefined && result.matchCount > 1) {
|
||||
const displayContext = hunk.changeContext.includes("\n")
|
||||
? hunk.changeContext.split("\n").pop()
|
||||
: hunk.changeContext;
|
||||
throw new ApplyPatchError(
|
||||
`Found ${result.matchCount} matches for context '${displayContext}' in ${path}. ` +
|
||||
`Add more surrounding context or additional @@ anchors to make it unique.`,
|
||||
);
|
||||
}
|
||||
|
||||
const idx = result.index;
|
||||
if (idx === undefined) {
|
||||
const displayContext = hunk.changeContext.includes("\n")
|
||||
? hunk.changeContext.split("\n").join(" > ")
|
||||
: hunk.changeContext;
|
||||
throw new ApplyPatchError(`Failed to find context '${displayContext}' in ${path}`);
|
||||
}
|
||||
contextIndex = idx;
|
||||
|
||||
// If oldLines[0] matches the final context, start search at idx (not idx+1)
|
||||
// This handles the common case where @@ scope and first context line are identical
|
||||
const firstOldLine = hunk.oldLines[0];
|
||||
const finalContext = hunk.changeContext.includes("\n")
|
||||
? hunk.changeContext.split("\n").pop()?.trim()
|
||||
: hunk.changeContext.trim();
|
||||
const isHierarchicalContext =
|
||||
hunk.changeContext.includes("\n") || hunk.changeContext.trim().split(/\s+/).length > 2;
|
||||
if (firstOldLine !== undefined && (firstOldLine.trim() === finalContext || isHierarchicalContext)) {
|
||||
lineIndex = idx;
|
||||
if (idx === undefined || (result.matchCount !== undefined && result.matchCount > 1)) {
|
||||
const fallback = attemptSequenceFallback(originalLines, hunk, lineIndex, lineHint, allowFuzzy);
|
||||
if (fallback !== undefined) {
|
||||
lineIndex = fallback;
|
||||
} else if (result.matchCount !== undefined && result.matchCount > 1) {
|
||||
const displayContext = hunk.changeContext.includes("\n")
|
||||
? hunk.changeContext.split("\n").pop()
|
||||
: hunk.changeContext;
|
||||
throw new ApplyPatchError(
|
||||
`Found ${result.matchCount} matches for context '${displayContext}' in ${path}. ` +
|
||||
`Add more surrounding context or additional @@ anchors to make it unique.`,
|
||||
);
|
||||
} else {
|
||||
const displayContext = hunk.changeContext.includes("\n")
|
||||
? hunk.changeContext.split("\n").join(" > ")
|
||||
: hunk.changeContext;
|
||||
throw new ApplyPatchError(`Failed to find context '${displayContext}' in ${path}`);
|
||||
}
|
||||
} else {
|
||||
lineIndex = idx + 1;
|
||||
// If oldLines[0] matches the final context, start search at idx (not idx+1)
|
||||
// This handles the common case where @@ scope and first context line are identical
|
||||
const firstOldLine = hunk.oldLines[0];
|
||||
const finalContext = hunk.changeContext.includes("\n")
|
||||
? hunk.changeContext.split("\n").pop()?.trim()
|
||||
: hunk.changeContext.trim();
|
||||
const isHierarchicalContext =
|
||||
hunk.changeContext.includes("\n") || hunk.changeContext.trim().split(/\s+/).length > 2;
|
||||
if (firstOldLine !== undefined && (firstOldLine.trim() === finalContext || isHierarchicalContext)) {
|
||||
lineIndex = idx;
|
||||
} else {
|
||||
lineIndex = idx + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -507,6 +768,61 @@ function computeReplacements(
|
||||
);
|
||||
}
|
||||
|
||||
if (searchResult.index === undefined || (searchResult.matchCount ?? 0) > 1) {
|
||||
for (const variant of buildFallbackVariants(hunk)) {
|
||||
if (variant.oldLines.length === 0) continue;
|
||||
const variantResult = findSequenceWithHint(
|
||||
originalLines,
|
||||
variant.oldLines,
|
||||
lineIndex,
|
||||
matchHint,
|
||||
hunk.isEndOfFile,
|
||||
allowFuzzy,
|
||||
);
|
||||
if (variantResult.index !== undefined && (variantResult.matchCount ?? 1) <= 1) {
|
||||
pattern = variant.oldLines;
|
||||
newSlice = variant.newLines;
|
||||
searchResult = variantResult;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (searchResult.index === undefined && contextIndex !== undefined) {
|
||||
for (const variant of buildFallbackVariants(hunk)) {
|
||||
if (variant.oldLines.length !== 1 || variant.newLines.length !== 1) continue;
|
||||
const removedLine = variant.oldLines[0];
|
||||
const hasSharedDuplicate = hunk.newLines.some((line) => line.trim() === removedLine.trim());
|
||||
const adjacentIndex = findContextRelativeMatch(
|
||||
originalLines,
|
||||
removedLine,
|
||||
contextIndex,
|
||||
hasSharedDuplicate,
|
||||
);
|
||||
if (adjacentIndex !== undefined) {
|
||||
pattern = variant.oldLines;
|
||||
newSlice = variant.newLines;
|
||||
searchResult = { index: adjacentIndex, confidence: 0.95 };
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (searchResult.index !== undefined && contextIndex !== undefined && pattern.length === 1) {
|
||||
const trimmed = pattern[0].trim();
|
||||
let occurrenceCount = 0;
|
||||
for (const line of originalLines) {
|
||||
if (line.trim() === trimmed) occurrenceCount++;
|
||||
}
|
||||
if (occurrenceCount > 1) {
|
||||
const hasSharedDuplicate = hunk.newLines.some((line) => line.trim() === trimmed);
|
||||
const contextMatch = findContextRelativeMatch(originalLines, pattern[0], contextIndex, hasSharedDuplicate);
|
||||
if (contextMatch !== undefined) {
|
||||
searchResult = { index: contextMatch, confidence: searchResult.confidence ?? 0.95 };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (searchResult.index === undefined) {
|
||||
if (searchResult.matchCount !== undefined && searchResult.matchCount > 1) {
|
||||
throw new ApplyPatchError(
|
||||
@@ -648,15 +964,15 @@ export async function applyPatch(input: PatchInput, options: ApplyPatchOptions):
|
||||
const resolvePath = (p: string): string => resolveToCwd(p, cwd);
|
||||
const absolutePath = resolvePath(input.path);
|
||||
|
||||
if (input.moveTo) {
|
||||
const destPath = resolvePath(input.moveTo);
|
||||
if (input.rename) {
|
||||
const destPath = resolvePath(input.rename);
|
||||
if (destPath === absolutePath) {
|
||||
throw new ApplyPatchError("moveTo path is the same as source path");
|
||||
throw new ApplyPatchError("rename path is the same as source path");
|
||||
}
|
||||
}
|
||||
|
||||
// Handle CREATE operation
|
||||
if (input.operation === "create") {
|
||||
if (input.op === "create") {
|
||||
if (!input.diff) {
|
||||
throw new ApplyPatchError("Create operation requires diff (file content)");
|
||||
}
|
||||
@@ -682,7 +998,7 @@ export async function applyPatch(input: PatchInput, options: ApplyPatchOptions):
|
||||
}
|
||||
|
||||
// Handle DELETE operation
|
||||
if (input.operation === "delete") {
|
||||
if (input.op === "delete") {
|
||||
if (!(await fs.exists(absolutePath))) {
|
||||
throw new ApplyPatchError(`File not found: ${input.path}`);
|
||||
}
|
||||
@@ -729,8 +1045,8 @@ export async function applyPatch(input: PatchInput, options: ApplyPatchOptions):
|
||||
|
||||
const newContent = applyHunksToContent(normalizedContent, input.path, hunks, fuzzyThreshold, allowFuzzy);
|
||||
const finalContent = bom + restoreLineEndings(newContent, lineEnding);
|
||||
const destPath = input.moveTo ? resolvePath(input.moveTo) : absolutePath;
|
||||
const isMove = Boolean(input.moveTo) && destPath !== absolutePath;
|
||||
const destPath = input.rename ? resolvePath(input.rename) : absolutePath;
|
||||
const isMove = Boolean(input.rename) && destPath !== absolutePath;
|
||||
|
||||
if (!dryRun) {
|
||||
if (isMove) {
|
||||
|
||||
@@ -286,6 +286,26 @@ function lineIncludesPattern(line: string, pattern: string): boolean {
|
||||
return patternNorm.length / Math.max(1, lineNorm.length) >= PARTIAL_MATCH_MIN_RATIO;
|
||||
}
|
||||
|
||||
function stripCommentPrefix(line: string): string {
|
||||
let trimmed = line.trimStart();
|
||||
if (trimmed.startsWith("/*")) {
|
||||
trimmed = trimmed.slice(2);
|
||||
} else if (trimmed.startsWith("*/")) {
|
||||
trimmed = trimmed.slice(2);
|
||||
} else if (trimmed.startsWith("//")) {
|
||||
trimmed = trimmed.slice(2);
|
||||
} else if (trimmed.startsWith("*")) {
|
||||
trimmed = trimmed.slice(1);
|
||||
} else if (trimmed.startsWith("#")) {
|
||||
trimmed = trimmed.slice(1);
|
||||
} else if (trimmed.startsWith(";")) {
|
||||
trimmed = trimmed.slice(1);
|
||||
} else if (trimmed.startsWith("/") && trimmed[1] === " ") {
|
||||
trimmed = trimmed.slice(1);
|
||||
}
|
||||
return trimmed.trimStart();
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a sequence of pattern lines within content lines.
|
||||
*
|
||||
@@ -347,6 +367,13 @@ export function seekSequence(
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 3b: Comment-prefix normalized match
|
||||
for (let i = from; i <= to; i++) {
|
||||
if (matchesAt(lines, pattern, i, (a, b) => stripCommentPrefix(a) === stripCommentPrefix(b))) {
|
||||
return { index: i, confidence: 0.975 };
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 4: Normalize unicode punctuation
|
||||
for (let i = from; i <= to; i++) {
|
||||
if (matchesAt(lines, pattern, i, (a, b) => normalizeUnicode(a) === normalizeUnicode(b))) {
|
||||
@@ -474,7 +501,7 @@ export function findContextLine(
|
||||
lines: string[],
|
||||
context: string,
|
||||
startFrom: number,
|
||||
options?: { allowFuzzy?: boolean },
|
||||
options?: { allowFuzzy?: boolean; skipFunctionFallback?: boolean },
|
||||
): ContextLineResult {
|
||||
const allowFuzzy = options?.allowFuzzy ?? true;
|
||||
const trimmedContext = context.trim();
|
||||
@@ -606,5 +633,15 @@ export function findContextLine(
|
||||
return { index: bestIndex, confidence: bestScore, matchCount };
|
||||
}
|
||||
|
||||
if (!options?.skipFunctionFallback && trimmedContext.endsWith("()")) {
|
||||
const withParen = trimmedContext.replace(/\(\)\s*$/u, "(");
|
||||
const withoutParen = trimmedContext.replace(/\(\)\s*$/u, "");
|
||||
const parenResult = findContextLine(lines, withParen, startFrom, { allowFuzzy, skipFunctionFallback: true });
|
||||
if (parenResult.index !== undefined || (parenResult.matchCount ?? 0) > 0) {
|
||||
return parenResult;
|
||||
}
|
||||
return findContextLine(lines, withoutParen, startFrom, { allowFuzzy, skipFunctionFallback: true });
|
||||
}
|
||||
|
||||
return { index: undefined, confidence: bestScore };
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
|
||||
import { StringEnum } from "@oh-my-pi/pi-ai";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import patchDescription from "../../../prompts/tools/patch.md" with { type: "text" };
|
||||
import replaceDescription from "../../../prompts/tools/replace.md" with { type: "text" };
|
||||
@@ -45,8 +46,8 @@ export { computeEditDiff, computePatchDiff, generateDiffString, generateUnifiedD
|
||||
export {
|
||||
DEFAULT_FUZZY_THRESHOLD,
|
||||
findContextLine,
|
||||
findMatch,
|
||||
findMatch as findEditMatch,
|
||||
findMatch,
|
||||
seekSequence,
|
||||
} from "./fuzzy";
|
||||
|
||||
@@ -61,11 +62,9 @@ export {
|
||||
|
||||
// Parsing
|
||||
export { normalizeCreateContent, normalizeDiff, parseHunks as parseDiffHunks } from "./parser";
|
||||
// Rendering
|
||||
export type { EditRenderContext, EditToolDetails } from "./shared";
|
||||
// Rendering
|
||||
export { editToolRenderer, getLspBatchRequest } from "./shared";
|
||||
// Types
|
||||
// Legacy aliases for backwards compatibility
|
||||
export type {
|
||||
ApplyPatchOptions,
|
||||
ApplyPatchResult,
|
||||
@@ -79,14 +78,16 @@ export type {
|
||||
DiffResult as EditDiffResult,
|
||||
FileChange,
|
||||
FileSystem,
|
||||
FuzzyMatch,
|
||||
FuzzyMatch as EditMatch,
|
||||
MatchOutcome,
|
||||
FuzzyMatch,
|
||||
MatchOutcome as EditMatchOutcome,
|
||||
MatchOutcome,
|
||||
Operation,
|
||||
PatchInput,
|
||||
SequenceSearchResult,
|
||||
} from "./types";
|
||||
// Types
|
||||
// Legacy aliases for backwards compatibility
|
||||
export { ApplyPatchError, EditMatchError, ParseError } from "./types";
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -103,21 +104,22 @@ const replaceEditSchema = Type.Object({
|
||||
});
|
||||
|
||||
const patchEditSchema = Type.Object({
|
||||
path: Type.String({ description: "Path to the file (relative or absolute)" }),
|
||||
operation: Type.Union([Type.Literal("create"), Type.Literal("delete"), Type.Literal("update")], {
|
||||
description: "Operation type: create new file, delete existing file, or update file content",
|
||||
}),
|
||||
moveTo: Type.Optional(Type.String({ description: "New path for rename (update only)" })),
|
||||
path: Type.String({ description: "Path to the file" }),
|
||||
op: Type.Optional(
|
||||
StringEnum(["create", "delete", "update"], {
|
||||
description: "The operation to perform (Defaults to 'update')",
|
||||
}),
|
||||
),
|
||||
rename: Type.Optional(Type.String({ description: "New path, if moving" })),
|
||||
diff: Type.Optional(
|
||||
Type.String({
|
||||
description:
|
||||
"For create: full file content. For update: diff hunks with @@ markers, context lines, +/- changes",
|
||||
description: "Diff hunk(s) for update. Full content for create.",
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
type ReplaceParams = { path: string; oldText: string; newText: string; all?: boolean };
|
||||
type PatchParams = { path: string; operation: Operation; moveTo?: string; diff?: string };
|
||||
type PatchParams = { path: string; op?: string; rename?: string; diff?: string };
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// LSP FileSystem for patch mode
|
||||
@@ -276,17 +278,20 @@ export class EditTool implements AgentTool<typeof replaceEditSchema | typeof pat
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
// Patch mode execution
|
||||
// ─────────────────────────────────────────────────────────────────
|
||||
if ("operation" in params) {
|
||||
const { path, operation, moveTo, diff } = params as PatchParams;
|
||||
if (this.patchMode) {
|
||||
const { path, op: rawOp, rename, diff } = params as PatchParams;
|
||||
|
||||
// Normalize unrecognized operations to "update"
|
||||
const op: Operation = rawOp === "create" || rawOp === "delete" ? rawOp : "update";
|
||||
|
||||
if (path.endsWith(".ipynb")) {
|
||||
throw new Error("Cannot edit Jupyter notebooks with the Edit tool. Use the NotebookEdit tool instead.");
|
||||
}
|
||||
if (moveTo?.endsWith(".ipynb")) {
|
||||
if (rename?.endsWith(".ipynb")) {
|
||||
throw new Error("Cannot edit Jupyter notebooks with the Edit tool. Use the NotebookEdit tool instead.");
|
||||
}
|
||||
|
||||
const input: PatchInput = { path, operation, moveTo, diff };
|
||||
const input: PatchInput = { path, op, rename, diff };
|
||||
const fs = new LspFileSystem(this.writethrough, signal, batchRequest);
|
||||
const result = await applyPatch(input, {
|
||||
cwd: this.session.cwd,
|
||||
@@ -294,7 +299,7 @@ export class EditTool implements AgentTool<typeof replaceEditSchema | typeof pat
|
||||
fuzzyThreshold: this.fuzzyThreshold,
|
||||
allowFuzzy: this.allowFuzzy,
|
||||
});
|
||||
const effectiveMoveTo = result.change.newPath ? moveTo : undefined;
|
||||
const effRename = result.change.newPath ? rename : undefined;
|
||||
|
||||
// Generate diff for display
|
||||
let diffResult = { diff: "", firstChangedLine: undefined as number | undefined };
|
||||
@@ -313,12 +318,12 @@ export class EditTool implements AgentTool<typeof replaceEditSchema | typeof pat
|
||||
resultText = `Deleted ${path}`;
|
||||
break;
|
||||
case "update":
|
||||
resultText = effectiveMoveTo ? `Updated and moved ${path} to ${effectiveMoveTo}` : `Updated ${path}`;
|
||||
resultText = effRename ? `Updated and moved ${path} to ${effRename}` : `Updated ${path}`;
|
||||
break;
|
||||
}
|
||||
|
||||
let diagnostics = fs.getDiagnostics();
|
||||
if (operation === "delete" && batchRequest?.flush) {
|
||||
if (op === "delete" && batchRequest?.flush) {
|
||||
const flushedDiagnostics = await flushLspWritethroughBatch(batchRequest.id, this.session.cwd, signal);
|
||||
diagnostics ??= flushedDiagnostics;
|
||||
}
|
||||
@@ -333,8 +338,8 @@ export class EditTool implements AgentTool<typeof replaceEditSchema | typeof pat
|
||||
diff: diffResult.diff,
|
||||
firstChangedLine: diffResult.firstChangedLine,
|
||||
diagnostics,
|
||||
operation,
|
||||
moveTo: effectiveMoveTo,
|
||||
op,
|
||||
rename: effRename,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -21,8 +21,9 @@ const EMPTY_CHANGE_CONTEXT_MARKER = "@@";
|
||||
/** Regex to match unified diff hunk headers: @@ -OLD,COUNT +NEW,COUNT @@ optional-context */
|
||||
const UNIFIED_HUNK_HEADER_REGEX = /^@@\s*-(\d+)(?:,(\d+))?\s+\+(\d+)(?:,(\d+))?\s*@@(?:\s*(.*))?$/;
|
||||
|
||||
/** Regex to match @@ line N pattern (model-generated line hints) */
|
||||
const LINE_HINT_REGEX = /^line\s+(\d+)$/i;
|
||||
/** Regex to match @@ line/lines N or N-M pattern (model-generated line hints) */
|
||||
const LINE_HINT_REGEX = /^lines?\s+(\d+)(?:\s*-\s*(\d+))?(?:\s*@@)?$/i;
|
||||
const TOP_OF_FILE_REGEX = /^(top|start|beginning)\s+of\s+file$/i;
|
||||
|
||||
/**
|
||||
* Check if a line is a diff content line (context, addition, or removal).
|
||||
@@ -74,9 +75,17 @@ export function normalizeDiff(diff: string): string {
|
||||
if (lines[0]?.trim().startsWith("*** Begin Patch")) {
|
||||
lines = lines.slice(1);
|
||||
}
|
||||
// Also strip bare *** at the beginning (model hallucination)
|
||||
if (lines[0]?.trim() === "***") {
|
||||
lines = lines.slice(1);
|
||||
}
|
||||
if (lines.length > 0 && lines[lines.length - 1]?.trim().startsWith("*** End Patch")) {
|
||||
lines = lines.slice(0, -1);
|
||||
}
|
||||
// Also strip bare *** terminator (model hallucination)
|
||||
if (lines.length > 0 && lines[lines.length - 1]?.trim() === "***") {
|
||||
lines = lines.slice(0, -1);
|
||||
}
|
||||
|
||||
// Layer 2: Strip Codex-style file operation markers and unified diff metadata
|
||||
// NOTE: Do NOT strip "*** End of File" - that's a valid marker within hunks, not a wrapper
|
||||
@@ -218,9 +227,10 @@ function parseOneHunk(lines: string[], lineNumber: number, allowMissingContext:
|
||||
const headerTrimmed = headerLine.trimEnd();
|
||||
const isHeaderLine = headerLine.startsWith("@@");
|
||||
const unifiedHeader = isHeaderLine ? parseUnifiedHunkHeader(headerTrimmed) : undefined;
|
||||
const isEmptyContextMarker = /^@@\s*@@$/.test(headerTrimmed);
|
||||
|
||||
// Check for context marker
|
||||
if (isHeaderLine && headerTrimmed === EMPTY_CHANGE_CONTEXT_MARKER) {
|
||||
if (isHeaderLine && (headerTrimmed === EMPTY_CHANGE_CONTEXT_MARKER || isEmptyContextMarker)) {
|
||||
startIndex = 1;
|
||||
} else if (unifiedHeader) {
|
||||
if (unifiedHeader.oldStartLine < 1 || unifiedHeader.newStartLine < 1) {
|
||||
@@ -235,19 +245,28 @@ function parseOneHunk(lines: string[], lineNumber: number, allowMissingContext:
|
||||
} else if (isHeaderLine && headerTrimmed.startsWith(CHANGE_CONTEXT_MARKER)) {
|
||||
const contextValue = headerTrimmed.slice(CHANGE_CONTEXT_MARKER.length);
|
||||
const trimmedContextValue = contextValue.trim();
|
||||
const normalizedContextValue = trimmedContextValue.replace(/^@@\s*/u, "");
|
||||
|
||||
// Check for @@ line N pattern (model-generated line hints)
|
||||
const lineHintMatch = trimmedContextValue.match(LINE_HINT_REGEX);
|
||||
const lineHintMatch = normalizedContextValue.match(LINE_HINT_REGEX);
|
||||
if (lineHintMatch) {
|
||||
oldStartLine = Number(lineHintMatch[1]);
|
||||
newStartLine = oldStartLine;
|
||||
if (oldStartLine < 1) {
|
||||
throw new ParseError("Line hint must be >= 1", lineNumber);
|
||||
}
|
||||
} else if (TOP_OF_FILE_REGEX.test(normalizedContextValue)) {
|
||||
oldStartLine = 1;
|
||||
newStartLine = 1;
|
||||
} else if (trimmedContextValue.length > 0) {
|
||||
changeContexts.push(contextValue);
|
||||
}
|
||||
startIndex = 1;
|
||||
} else if (isHeaderLine) {
|
||||
const contextValue = headerTrimmed.slice(2).trim();
|
||||
if (contextValue.length > 0) {
|
||||
changeContexts.push(contextValue);
|
||||
}
|
||||
startIndex = 1;
|
||||
} else {
|
||||
if (!allowMissingContext) {
|
||||
throw new ParseError(`Expected hunk to start with @@ context marker, got: '${lines[0]}'`, lineNumber);
|
||||
@@ -309,6 +328,7 @@ function parseOneHunk(lines: string[], lineNumber: number, allowMissingContext:
|
||||
|
||||
for (let i = startIndex; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const trimmed = line.trim();
|
||||
|
||||
if (!isDiffContentLine(line) && line.trimEnd() === EOF_MARKER && line.startsWith(EOF_MARKER)) {
|
||||
if (parsedLines === 0) {
|
||||
@@ -319,6 +339,12 @@ function parseOneHunk(lines: string[], lineNumber: number, allowMissingContext:
|
||||
break;
|
||||
}
|
||||
|
||||
if (trimmed === "..." || trimmed === "…") {
|
||||
hunk.hasContextLines = true;
|
||||
parsedLines++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const firstChar = line[0];
|
||||
|
||||
if (firstChar === undefined || firstChar === "") {
|
||||
@@ -337,6 +363,11 @@ function parseOneHunk(lines: string[], lineNumber: number, allowMissingContext:
|
||||
} else if (firstChar === "-") {
|
||||
// Removed line
|
||||
hunk.oldLines.push(line.slice(1));
|
||||
} else if (!line.startsWith("@@")) {
|
||||
// Implicit context line (model omitted leading space)
|
||||
hunk.hasContextLines = true;
|
||||
hunk.oldLines.push(line);
|
||||
hunk.newLines.push(line);
|
||||
} else {
|
||||
if (parsedLines === 0) {
|
||||
throw new ParseError(
|
||||
@@ -354,9 +385,43 @@ function parseOneHunk(lines: string[], lineNumber: number, allowMissingContext:
|
||||
throw new ParseError("Hunk does not contain any lines", lineNumber + startIndex);
|
||||
}
|
||||
|
||||
stripLineNumberPrefixes(hunk);
|
||||
return { hunk, linesConsumed: parsedLines + startIndex };
|
||||
}
|
||||
|
||||
function stripLineNumberPrefixes(hunk: DiffHunk): void {
|
||||
const allLines = [...hunk.oldLines, ...hunk.newLines].filter((line) => line.trim().length > 0);
|
||||
if (allLines.length < 2) return;
|
||||
|
||||
const numberMatches = allLines
|
||||
.map((line) => line.match(/^\s*(\d{1,6})\s+(.+)$/u))
|
||||
.filter((match): match is RegExpMatchArray => match !== null);
|
||||
|
||||
if (numberMatches.length < Math.max(2, Math.ceil(allLines.length * 0.6))) {
|
||||
return;
|
||||
}
|
||||
|
||||
const numbers = numberMatches.map((match) => Number(match[1]));
|
||||
let sequential = 0;
|
||||
for (let i = 1; i < numbers.length; i++) {
|
||||
if (numbers[i] === numbers[i - 1] + 1) {
|
||||
sequential++;
|
||||
}
|
||||
}
|
||||
|
||||
if (numbers.length >= 3 && sequential < Math.max(1, numbers.length - 2)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const strip = (line: string): string => {
|
||||
const match = line.match(/^\s*\d{1,6}\s+(.+)$/u);
|
||||
return match ? match[1] : line;
|
||||
};
|
||||
|
||||
hunk.oldLines = hunk.oldLines.map(strip);
|
||||
hunk.newLines = hunk.newLines.map(strip);
|
||||
}
|
||||
|
||||
/** Multi-file patch markers that indicate this is not a single-file patch */
|
||||
const MULTI_FILE_MARKERS = ["*** Update File:", "*** Add File:", "*** Delete File:", "diff --git "];
|
||||
|
||||
@@ -450,7 +515,11 @@ export function parseHunks(diff: string): DiffHunk[] {
|
||||
continue;
|
||||
}
|
||||
|
||||
const { hunk, linesConsumed } = parseOneHunk(lines.slice(i), i + 1, hunks.length === 0);
|
||||
if (trimmed.startsWith("@@") && lines.slice(i + 1).every((l) => l.trim() === "")) {
|
||||
break;
|
||||
}
|
||||
|
||||
const { hunk, linesConsumed } = parseOneHunk(lines.slice(i), i + 1, true);
|
||||
hunks.push(hunk);
|
||||
i += linesConsumed;
|
||||
}
|
||||
|
||||
@@ -50,9 +50,9 @@ export interface EditToolDetails {
|
||||
/** Diagnostic result (if available) */
|
||||
diagnostics?: FileDiagnosticsResult;
|
||||
/** Operation type (patch mode only) */
|
||||
operation?: Operation;
|
||||
op?: Operation;
|
||||
/** New path after move/rename (patch mode only) */
|
||||
moveTo?: string;
|
||||
rename?: string;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
@@ -67,8 +67,8 @@ interface EditRenderArgs {
|
||||
patch?: string;
|
||||
all?: boolean;
|
||||
// Patch mode fields
|
||||
operation?: Operation;
|
||||
moveTo?: string;
|
||||
op?: Operation;
|
||||
rename?: string;
|
||||
diff?: string;
|
||||
}
|
||||
|
||||
@@ -145,12 +145,12 @@ export const editToolRenderer = {
|
||||
let pathDisplay = filePath ? uiTheme.fg("accent", filePath) : uiTheme.fg("toolOutput", uiTheme.format.ellipsis);
|
||||
|
||||
// Add arrow for move/rename operations
|
||||
if (args.moveTo) {
|
||||
pathDisplay += ` ${uiTheme.fg("dim", "→")} ${uiTheme.fg("accent", shortenPath(args.moveTo))}`;
|
||||
if (args.rename) {
|
||||
pathDisplay += ` ${uiTheme.fg("dim", "→")} ${uiTheme.fg("accent", shortenPath(args.rename))}`;
|
||||
}
|
||||
|
||||
// Show operation type for patch mode
|
||||
const opTitle = args.operation === "create" ? "Create" : args.operation === "delete" ? "Delete" : "Edit";
|
||||
const opTitle = args.op === "create" ? "Create" : args.op === "delete" ? "Delete" : "Edit";
|
||||
const text = `${ui.title(opTitle)} ${editIcon} ${pathDisplay}`;
|
||||
return new Text(text, 0, 0);
|
||||
},
|
||||
@@ -170,9 +170,9 @@ export const editToolRenderer = {
|
||||
const editDiffPreview = renderContext?.editDiffPreview;
|
||||
const renderDiffFn = renderContext?.renderDiff ?? ((t: string) => t);
|
||||
|
||||
// Get operation and moveTo from args or details
|
||||
const operation = args?.operation || result.details?.operation;
|
||||
const moveTo = args?.moveTo || result.details?.moveTo;
|
||||
// Get op and rename from args or details
|
||||
const op = args?.op || result.details?.op;
|
||||
const rename = args?.rename || result.details?.rename;
|
||||
|
||||
// Build path display with line number if available
|
||||
let pathDisplay = filePath ? uiTheme.fg("accent", filePath) : uiTheme.fg("toolOutput", uiTheme.format.ellipsis);
|
||||
@@ -183,17 +183,17 @@ export const editToolRenderer = {
|
||||
pathDisplay += uiTheme.fg("warning", `:${firstChangedLine}`);
|
||||
}
|
||||
|
||||
// Add arrow for move/rename operations
|
||||
if (moveTo) {
|
||||
pathDisplay += ` ${uiTheme.fg("dim", "→")} ${uiTheme.fg("accent", shortenPath(moveTo))}`;
|
||||
// Add arrow for rename operations
|
||||
if (rename) {
|
||||
pathDisplay += ` ${uiTheme.fg("dim", "→")} ${uiTheme.fg("accent", shortenPath(rename))}`;
|
||||
}
|
||||
|
||||
// Show operation type for patch mode
|
||||
const opTitle = operation === "create" ? "Create" : operation === "delete" ? "Delete" : "Edit";
|
||||
const opTitle = op === "create" ? "Create" : op === "delete" ? "Delete" : "Edit";
|
||||
let text = `${uiTheme.fg("toolTitle", uiTheme.bold(opTitle))} ${editIcon} ${pathDisplay}`;
|
||||
|
||||
// Skip metadata line for delete operations
|
||||
if (operation !== "delete") {
|
||||
if (op !== "delete") {
|
||||
const editLineCount = countLines(args?.newText ?? args?.oldText ?? args?.diff ?? args?.patch ?? "");
|
||||
text += `\n${formatMetadataLine(editLineCount, editLanguage, uiTheme)}`;
|
||||
}
|
||||
|
||||
@@ -75,9 +75,9 @@ export interface PatchInput {
|
||||
/** File path (relative or absolute) */
|
||||
path: string;
|
||||
/** Operation type */
|
||||
operation: Operation;
|
||||
op: Operation;
|
||||
/** New path for rename (update only) */
|
||||
moveTo?: string;
|
||||
rename?: string;
|
||||
/** File content (create) or diff hunks (update) */
|
||||
diff?: string;
|
||||
}
|
||||
|
||||
@@ -194,18 +194,18 @@ export class ToolExecutionComponent extends Container {
|
||||
if (this.toolName !== "edit") return;
|
||||
|
||||
const path = this.args?.path;
|
||||
const operation = this.args?.operation;
|
||||
const op = this.args?.op;
|
||||
|
||||
if (operation) {
|
||||
if (op) {
|
||||
const diff = this.args?.diff;
|
||||
const moveTo = this.args?.moveTo;
|
||||
const rename = this.args?.rename;
|
||||
if (!path) return;
|
||||
|
||||
const argsKey = JSON.stringify({ path, operation, moveTo, diff });
|
||||
const argsKey = JSON.stringify({ path, op, rename, diff });
|
||||
if (this.editDiffArgsKey === argsKey) return;
|
||||
this.editDiffArgsKey = argsKey;
|
||||
|
||||
computePatchDiff({ path, operation, moveTo, diff }, this.cwd, {
|
||||
computePatchDiff({ path, op, rename, diff }, this.cwd, {
|
||||
fuzzyThreshold: this.editFuzzyThreshold,
|
||||
allowFuzzy: this.editAllowFuzzy,
|
||||
}).then((result) => {
|
||||
|
||||
@@ -1,82 +1,76 @@
|
||||
<context>
|
||||
Performs patch operations on the file system. This is your primary tool for making changes to the codebase.
|
||||
</context>
|
||||
Performs patch operations on a file given a diff.
|
||||
This is your primary tool for making changes to existing files.
|
||||
|
||||
<critical>
|
||||
**Read before editing.** Always fetch the target file first and copy context lines verbatim. Patch failures almost always stem from guessed or paraphrased context that doesn't match actual file content.
|
||||
- Always read the target file before editing.
|
||||
- Copy anchors + context lines verbatim (including whitespace).
|
||||
- Output the clean patch format below.
|
||||
</critical>
|
||||
|
||||
<parameters>
|
||||
| Parameter | Description |
|
||||
|-----------|-------------|
|
||||
| `path` | Path to the file (relative or absolute) |
|
||||
| `operation` | `create` (new file), `delete` (remove file), or `update` (patch in place) |
|
||||
| `moveTo` | New path for rename (update only, optional) |
|
||||
| `diff` | For create: full file content. For update: diff hunks with context and changes |
|
||||
```ts
|
||||
type T =
|
||||
// Diff is one or more hunks, within the same file.
|
||||
// - Each hunk begins with "@@" (optionally with an anchor).
|
||||
// - Each hunk body contains only lines starting with: ' ' | '+' | '-'.
|
||||
// - Each hunk must include at least one real change (+ or -). No no-op hunks.
|
||||
| { path: string, op: "update", diff: string }
|
||||
// Diff is the full file content, no prefixes.
|
||||
| { path: string, op: "create", diff: string }
|
||||
// Omit diff for delete operation.
|
||||
| { path: string, op: "delete" }
|
||||
// New path for update-and-move operation.
|
||||
| { path: string, op: "update", rename: string, diff: string }
|
||||
```
|
||||
</parameters>
|
||||
|
||||
<diff_format>
|
||||
For `create`, the `diff` field contains the full file content (no prefixes needed).
|
||||
<hunk_header>
|
||||
Allowed:
|
||||
- `@@`
|
||||
- `@@ $ANCHOR`
|
||||
|
||||
For `update`, the `diff` field contains one or more "hunks", each introduced by @@ (optionally followed by a hunk header).
|
||||
If you include a hunk header, it must be **verbatim text from the file** (a full line or a unique substring of a line, e.g., a function signature). Do **not** use line numbers, ranges, or labels like "top of file"—those are treated as literal anchors and will fail to match. If unsure, omit the header entirely.
|
||||
Within a hunk each line starts with:
|
||||
- ` ` (space) for context lines
|
||||
- `-` for removed lines
|
||||
- `+` for added lines
|
||||
ANCHOR RULES:
|
||||
- `$ANCHOR` MUST be copied verbatim from the file as either:
|
||||
- a full existing line, OR
|
||||
- a unique substring of a single existing line.
|
||||
- NEVER use it as a comment:
|
||||
- line numbers / ranges: `line 207`, `lines 26-37`
|
||||
- location labels: `top of file`, `start`, `near imports`
|
||||
- placeholders: `@@ @@`, `...`
|
||||
</hunk_header>
|
||||
|
||||
Prefer replacing whole logical blocks (function/class/section) when possible instead of single-line edits.
|
||||
Line numbers are hints only; do not rely on them. Use semantic anchors via `@@` and surrounding context lines.
|
||||
<anchor_selection>
|
||||
ANCHOR SELECTION ALGORITHM (use in this order):
|
||||
1) If the surrounding context lines are already unique in the file, use bare `@@`.
|
||||
2) Else choose an anchor that is highly specific and stable, copied from the file, e.g.:
|
||||
- full function signature line
|
||||
- class declaration line
|
||||
- a unique string literal / error message
|
||||
- a config key with uncommon name
|
||||
3) If you get "Found multiple matches", escalate by:
|
||||
- adding more context lines, OR
|
||||
- using multiple hunks with separate nearby anchors, OR
|
||||
- using a more specific anchor substring (longer, includes identifiers).
|
||||
NEVER use generic anchors like `import`, `export`, `describe`, `function`, `const`.
|
||||
</anchor_selection>
|
||||
|
||||
For instructions on [context_before] and [context_after]:
|
||||
- By default, show 3 lines of code immediately above and 3 lines immediately below each change. If a change is within 3 lines of a previous change, do NOT duplicate the first change's [context_after] lines in the second change's [context_before] lines.
|
||||
- Context lines must be copied **verbatim**, including whitespace and punctuation. Avoid paraphrasing or reformatting.
|
||||
- If 3 lines of context is insufficient to uniquely identify the snippet of code within the file, use the @@ operator to indicate the class or function to which the snippet belongs. For instance, we might have:
|
||||
@@ class BaseClass
|
||||
[3 lines of pre-context]
|
||||
- [old_code]
|
||||
+ [new_code]
|
||||
[3 lines of post-context]
|
||||
<context_rules>
|
||||
- Include enough context lines (' ' prefixed) to make the match unique (usually 2–8 total).
|
||||
- Context lines must exist in the file exactly as written; preserve indentation/trailing spaces.
|
||||
</context_rules>
|
||||
|
||||
- If a code block is repeated so many times in a class or function such that even a single `@@` statement and 3 lines of context cannot uniquely identify the snippet of code, use multiple `@@` statements plus **extra surrounding context** to make the match unique. For instance:
|
||||
<example name="create">
|
||||
edit {"path":"hello.txt","op":"create","diff":"Hello\n"}
|
||||
</example>
|
||||
|
||||
@@ class BaseClass
|
||||
@@ def method():
|
||||
[3 lines of pre-context]
|
||||
- [old_code]
|
||||
+ [new_code]
|
||||
[3 lines of post-context]
|
||||
</diff_format>
|
||||
<example name="update">
|
||||
edit {"path":"src/app.py","op":"update","diff":"@@ def greet():\n def greet():\n-print('Hi')\n+print('Hello')\n"}
|
||||
</example>
|
||||
|
||||
<examples>
|
||||
Create a new file:
|
||||
```
|
||||
edit {"path": "hello.txt", "operation": "create", "diff": "Hello world"}
|
||||
```
|
||||
<example name="rename">
|
||||
edit {"path":"src/app.py","op":"update","rename":"src/main.py","diff":"@@\n ...\n"}
|
||||
</example>
|
||||
|
||||
Update a file:
|
||||
```
|
||||
edit {"path": "src/app.py", "operation": "update", "diff": "@@ def greet():\n-print(\"Hi\")\n+print(\"Hello, world!\")"}
|
||||
```
|
||||
|
||||
Rename and update:
|
||||
```
|
||||
edit {"path": "src/app.py", "operation": "update", "moveTo": "src/main.py", "diff": "@@ def greet():\n-print(\"Hi\")\n+print(\"Hello, world!\")"}
|
||||
```
|
||||
|
||||
Delete a file:
|
||||
```
|
||||
edit {"path": "obsolete.txt", "operation": "delete"}
|
||||
```
|
||||
</examples>
|
||||
|
||||
<rules>
|
||||
1. Read the file before editing—copy context lines exactly as they appear
|
||||
2. For create: provide full file content in `diff`
|
||||
3. For update: provide hunks with context lines and +/- changes
|
||||
4. For delete: omit the `diff` field
|
||||
5. Use relative paths only
|
||||
6. If a context or change appears more than once, add more surrounding context or additional `@@` anchors to make it unique
|
||||
7. Do not include no-op edits (identical `-` and `+` lines). If nothing changes, omit the hunk
|
||||
8. Ensure each hunk starts with @@ and contains only valid diff lines (space/+/ -). No extra preamble text
|
||||
</rules>
|
||||
<example name="delete">
|
||||
edit {"path":"obsolete.txt","op":"delete"}
|
||||
</example>
|
||||
|
||||
@@ -20,15 +20,12 @@ describe("applyPatch adversarial inputs", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects moveTo when it matches path", async () => {
|
||||
test("rejects rename when it matches path", async () => {
|
||||
const filePath = join(tempDir, "same.txt");
|
||||
await Bun.write(filePath, "foo\n");
|
||||
|
||||
await expect(
|
||||
applyPatch(
|
||||
{ path: "same.txt", operation: "update", moveTo: "same.txt", diff: "@@\n-foo\n+bar" },
|
||||
{ cwd: tempDir },
|
||||
),
|
||||
applyPatch({ path: "same.txt", op: "update", rename: "same.txt", diff: "@@\n-foo\n+bar" }, { cwd: tempDir }),
|
||||
).rejects.toThrow(ApplyPatchError);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("foo\n");
|
||||
@@ -41,7 +38,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "add-context.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@ function bar\n+ console.log('x');",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -60,7 +57,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "single.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "*** Begin Patch\n*** Update File: single.txt\n@@\n-foo\n+FOO\n*** Update File: other.txt\n@@\n-bar\n+BAR\n*** End Patch",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -75,7 +72,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "metadata-context.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n diff --git a b\n-alpha\n+ALPHA",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -91,7 +88,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "order.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@ second\n-keep\n+KEEP2\n@@ first\n-keep\n+KEEP1",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -108,7 +105,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "ambiguous-context.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@ return foo;\n- return foo;\n+ return bar;",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -124,7 +121,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "ambiguous-prefix.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n-const enabled = true\n+const enabled = false",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -140,7 +137,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "line-hint.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@ -999,0 +999,1 @@\n+tail",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -155,7 +152,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "blank-context.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n section\n-value\n+VALUE\n ",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -168,7 +165,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
const filePath = join(tempDir, "crlf.txt");
|
||||
await Bun.write(filePath, "foo\r\nbar\r\n");
|
||||
|
||||
await applyPatch({ path: "crlf.txt", operation: "update", diff: "@@\n-foo\n+FOO" }, { cwd: tempDir });
|
||||
await applyPatch({ path: "crlf.txt", op: "update", diff: "@@\n-foo\n+FOO" }, { cwd: tempDir });
|
||||
|
||||
const content = readFileSync(filePath, "utf-8");
|
||||
expect(content).toBe("FOO\r\nbar\r\n");
|
||||
@@ -178,7 +175,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
const filePath = join(tempDir, "bom.txt");
|
||||
await Bun.write(filePath, "\uFEFFfoo\r\nbar\r\n");
|
||||
|
||||
await applyPatch({ path: "bom.txt", operation: "update", diff: "@@\n-foo\n+FOO" }, { cwd: tempDir });
|
||||
await applyPatch({ path: "bom.txt", op: "update", diff: "@@\n-foo\n+FOO" }, { cwd: tempDir });
|
||||
|
||||
const content = readFileSync(filePath, "utf-8");
|
||||
expect(content).toBe("\uFEFFFOO\r\nbar\r\n");
|
||||
@@ -188,7 +185,7 @@ describe("applyPatch adversarial inputs", () => {
|
||||
const filePath = join(tempDir, "nonewline.txt");
|
||||
await Bun.write(filePath, "foo\nbar");
|
||||
|
||||
await applyPatch({ path: "nonewline.txt", operation: "update", diff: "@@\n-bar\n+baz" }, { cwd: tempDir });
|
||||
await applyPatch({ path: "nonewline.txt", op: "update", diff: "@@\n-bar\n+baz" }, { cwd: tempDir });
|
||||
|
||||
const content = readFileSync(filePath, "utf-8");
|
||||
expect(content).toBe("foo\nbaz");
|
||||
|
||||
@@ -42,7 +42,7 @@ describe("regression: indentation adjustment for line-based replacements (2B)",
|
||||
await applyPatch(
|
||||
{
|
||||
path: "indent.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ constructor() {
|
||||
-this.value = 1;
|
||||
-this.name = "test";
|
||||
@@ -76,7 +76,7 @@ describe("regression: indentation adjustment for line-based replacements (2B)",
|
||||
await applyPatch(
|
||||
{
|
||||
path: "multi-indent.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ function inner1() {
|
||||
-return 1;
|
||||
+return 10;
|
||||
@@ -113,7 +113,7 @@ describe("regression: ambiguity detection for context-less hunks (2C)", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "dupe.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "-foo\n+FOO",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -131,7 +131,7 @@ describe("regression: ambiguity detection for context-less hunks (2C)", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "multi-dupe.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n-aaa\n+AAA\n@@\n-ccc\n+CCC",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -147,7 +147,7 @@ describe("regression: ambiguity detection for context-less hunks (2C)", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "context-disambig.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n middle\n-foo\n+FOO",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -192,7 +192,7 @@ function process() {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "hints.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ -5,3 +5,3 @@ function process() {
|
||||
function process() {
|
||||
- return 2;
|
||||
@@ -226,7 +226,7 @@ def helper():
|
||||
await applyPatch(
|
||||
{
|
||||
path: "hint-priority.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ -6,2 +6,2 @@ def helper():
|
||||
def helper():
|
||||
- pass
|
||||
@@ -262,7 +262,7 @@ describe("regression: insertion uses newStartLine fallback (2E)", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "insert.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
line1
|
||||
+inserted`,
|
||||
@@ -281,7 +281,7 @@ describe("regression: insertion uses newStartLine fallback (2E)", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "insert-hint.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ -2,1 +2,2 @@
|
||||
bbb
|
||||
+inserted after bbb`,
|
||||
@@ -299,7 +299,7 @@ describe("regression: insertion uses newStartLine fallback (2E)", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "append.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
+appended line
|
||||
*** End of File`,
|
||||
@@ -433,7 +433,7 @@ describe("plan: partial line matching for @@ context", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "imports.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ import { mkdirSync, unlinkSync }
|
||||
|
||||
function cleanup() {
|
||||
@@ -461,7 +461,7 @@ describe("plan: partial line matching for @@ context", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "funcs.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ function processItems(items
|
||||
- return items.map(i => i.value);
|
||||
+ return items.filter(i => i.valid).map(i => i.value);`,
|
||||
@@ -495,7 +495,7 @@ describe("plan: unified diff format line numbers", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "lines.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ -10,3 +10,3 @@
|
||||
line 10
|
||||
-line 11
|
||||
@@ -532,7 +532,7 @@ line 9
|
||||
await applyPatch(
|
||||
{
|
||||
path: "repeat.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ -8,1 +8,1 @@
|
||||
-target line
|
||||
+MODIFIED TARGET`,
|
||||
@@ -567,7 +567,7 @@ describe("plan: Codex-style wrapped patches", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "wrapped.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `*** Begin Patch
|
||||
@@
|
||||
-old content
|
||||
@@ -588,7 +588,7 @@ describe("plan: Codex-style wrapped patches", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "partial.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
-original
|
||||
+modified
|
||||
@@ -600,6 +600,68 @@ describe("plan: Codex-style wrapped patches", () => {
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("modified\n");
|
||||
});
|
||||
|
||||
test("strips bare *** terminator (model hallucination)", async () => {
|
||||
const filePath = join(tempDir, "bare-asterisk.txt");
|
||||
await Bun.write(filePath, "line1\nline2\nline3\n");
|
||||
|
||||
// Model sometimes outputs just *** as end marker
|
||||
await applyPatch(
|
||||
{
|
||||
path: "bare-asterisk.txt",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
-line2
|
||||
+LINE TWO
|
||||
***`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("line1\nLINE TWO\nline3\n");
|
||||
});
|
||||
|
||||
test("strips bare *** terminator in multi-hunk diff", async () => {
|
||||
const filePath = join(tempDir, "multi-hunk-asterisk.txt");
|
||||
await Bun.write(filePath, "aaa\nbbb\nccc\nddd\n");
|
||||
|
||||
// Multiple hunks with *** terminator at end
|
||||
await applyPatch(
|
||||
{
|
||||
path: "multi-hunk-asterisk.txt",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
-aaa
|
||||
+AAA
|
||||
@@
|
||||
-ccc
|
||||
+CCC
|
||||
***`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("AAA\nbbb\nCCC\nddd\n");
|
||||
});
|
||||
|
||||
test("strips bare *** at beginning of diff", async () => {
|
||||
const filePath = join(tempDir, "leading-asterisk.txt");
|
||||
await Bun.write(filePath, "old\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "leading-asterisk.txt",
|
||||
op: "update",
|
||||
diff: `***
|
||||
@@
|
||||
-old
|
||||
+new`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("new\n");
|
||||
});
|
||||
|
||||
test("strips unified diff metadata lines", async () => {
|
||||
const filePath = join(tempDir, "unified-meta.txt");
|
||||
await Bun.write(filePath, "first\nsecond\nthird\n");
|
||||
@@ -608,7 +670,7 @@ describe("plan: Codex-style wrapped patches", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "unified-meta.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `diff --git a/unified-meta.txt b/unified-meta.txt
|
||||
index abc123..def456 100644
|
||||
--- a/unified-meta.txt
|
||||
@@ -642,7 +704,7 @@ describe("plan: strip + prefix from file creation", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "newfile.txt",
|
||||
operation: "create",
|
||||
op: "create",
|
||||
diff: `+line one
|
||||
+line two
|
||||
+line three`,
|
||||
@@ -657,7 +719,7 @@ describe("plan: strip + prefix from file creation", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "spaced.txt",
|
||||
operation: "create",
|
||||
op: "create",
|
||||
diff: `+ first line
|
||||
+ second line`,
|
||||
},
|
||||
@@ -671,7 +733,7 @@ describe("plan: strip + prefix from file creation", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "mixed.txt",
|
||||
operation: "create",
|
||||
op: "create",
|
||||
diff: `+line one
|
||||
regular line
|
||||
+line three`,
|
||||
@@ -703,7 +765,7 @@ describe("regression: *** End of File marker handling (2A/2G)", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "eof.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
-last line
|
||||
+modified last line
|
||||
@@ -723,7 +785,7 @@ describe("regression: *** End of File marker handling (2A/2G)", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "eof-target.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@
|
||||
-item
|
||||
+FINAL ITEM
|
||||
@@ -778,7 +840,7 @@ describe("regression: model edit attempt - @@ line N syntax (session 2026-01-19)
|
||||
await applyPatch(
|
||||
{
|
||||
path: "settings.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ line 125
|
||||
fuzzyMatch?: boolean; // default: true
|
||||
fuzzyThreshold?: number; // default: 0.95
|
||||
@@ -793,27 +855,6 @@ describe("regression: model edit attempt - @@ line N syntax (session 2026-01-19)
|
||||
expect(result).toContain("patchMode?: boolean; // default: true");
|
||||
expect(result).not.toContain("patchMode?: boolean; // default: false");
|
||||
});
|
||||
|
||||
test("@@ line N works with various formats", async () => {
|
||||
const filePath = join(tempDir, "code.ts");
|
||||
const lines = Array.from({ length: 20 }, (_, i) => `content line ${i + 1}`);
|
||||
lines[9] = "old value"; // line 10
|
||||
await Bun.write(filePath, `${lines.join("\n")}\n`);
|
||||
|
||||
// Variations the model might produce
|
||||
await applyPatch(
|
||||
{
|
||||
path: "code.ts",
|
||||
operation: "update",
|
||||
diff: `@@ line 10
|
||||
-old value
|
||||
+new value`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("new value");
|
||||
});
|
||||
});
|
||||
|
||||
describe("regression: model edit attempt - nested @@ anchors (session 2026-01-19)", () => {
|
||||
@@ -853,7 +894,7 @@ class PatchTool {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "patch.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ class PatchTool
|
||||
@@ constructor
|
||||
constructor(session: ToolSession) {
|
||||
@@ -892,7 +933,7 @@ class Beta {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "multi-class.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ class Beta
|
||||
@@ process
|
||||
process() {
|
||||
@@ -943,7 +984,7 @@ class PatchTool {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "tool.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ class PatchTool constructor
|
||||
constructor() {
|
||||
- this.value = 2;
|
||||
@@ -981,7 +1022,7 @@ function process() {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "funcs.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ function process helper
|
||||
function helper() {
|
||||
- return 2;
|
||||
@@ -1031,7 +1072,7 @@ export class EditTool implements AgentTool<typeof replaceEditSchema | typeof pat
|
||||
await applyPatch(
|
||||
{
|
||||
path: "tool.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: `@@ class EditTool
|
||||
constructor(session: ToolSession) {
|
||||
this.session = session;
|
||||
@@ -1046,7 +1087,7 @@ export class EditTool implements AgentTool<typeof replaceEditSchema | typeof pat
|
||||
expect(result).toContain("this.patchMode = true;");
|
||||
});
|
||||
|
||||
test("@@ class ClassName still rejects when multiple classes match", async () => {
|
||||
test("@@ class ClassName falls back to unique old lines when context is ambiguous", async () => {
|
||||
const filePath = join(tempDir, "multi.ts");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
@@ -1060,18 +1101,451 @@ export class EditTool implements AgentTool<Schema2, Details2> {
|
||||
`,
|
||||
);
|
||||
|
||||
// Should reject because "class EditTool" matches two lines
|
||||
await expect(
|
||||
applyPatch(
|
||||
{
|
||||
path: "multi.ts",
|
||||
operation: "update",
|
||||
diff: `@@ class EditTool
|
||||
await applyPatch(
|
||||
{
|
||||
path: "multi.ts",
|
||||
op: "update",
|
||||
diff: `@@ class EditTool
|
||||
- value = 1;
|
||||
+ value = 100;`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
),
|
||||
).rejects.toThrow(/2 matches/);
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
expect(result).toContain("value = 100;");
|
||||
expect(result).toContain("value = 2;");
|
||||
});
|
||||
});
|
||||
|
||||
describe("regression: bench edit failures (2026-01-19)", () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = join(tmpdir(), `bench-regression-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("@@ @@ is treated as empty context", async () => {
|
||||
const filePath = join(tempDir, "empty-context.txt");
|
||||
await Bun.write(filePath, "alpha\nbeta\ngamma\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "empty-context.txt",
|
||||
op: "update",
|
||||
diff: `@@ @@\n-beta\n+BETA`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("alpha\nBETA\ngamma\n");
|
||||
});
|
||||
|
||||
test.each([
|
||||
["@@ line 3 @@", 3],
|
||||
["@@ lines 3-5", 3],
|
||||
["@@ Line 3-5", 3],
|
||||
["@@ line 3-5 @@", 3],
|
||||
["@@ @@ line 3", 3],
|
||||
])("line hint variants (%s) target the correct line", async (header, targetLine) => {
|
||||
const filePath = join(tempDir, `line-hint-${targetLine}.txt`);
|
||||
await Bun.write(filePath, "line 1\nline 2\nline 3\nline 4\nline 5\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: `line-hint-${targetLine}.txt`,
|
||||
op: "update",
|
||||
diff: `${header}\n-line 3\n+LINE THREE`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const lines = readFileSync(filePath, "utf-8").split("\n");
|
||||
expect(lines[2]).toBe("LINE THREE");
|
||||
});
|
||||
|
||||
test("top of file header anchors to line 1", async () => {
|
||||
const filePath = join(tempDir, "top-of-file.txt");
|
||||
await Bun.write(filePath, "first\nsecond\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "top-of-file.txt",
|
||||
op: "update",
|
||||
diff: "@@ top of file\n-first\n+FIRST",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("FIRST\nsecond\n");
|
||||
});
|
||||
|
||||
test("function name with empty params matches signature", async () => {
|
||||
const filePath = join(tempDir, "functions.ts");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`function retryIfBlockedOn(reason: string) {\n return reason;\n}\n\nfunction describeNode(node: object) {\n return String(node);\n}\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "functions.ts",
|
||||
op: "update",
|
||||
diff: "@@ retryIfBlockedOn()\n- return reason;\n+ return reason.toUpperCase();",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "functions.ts",
|
||||
op: "update",
|
||||
diff: "@@ describeNode()\n- return String(node);\n+ return JSON.stringify(node);",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
expect(result).toContain("return reason.toUpperCase();");
|
||||
expect(result).toContain("return JSON.stringify(node);");
|
||||
});
|
||||
|
||||
test("label context falls back to unique old lines", async () => {
|
||||
const filePath = join(tempDir, "imports.js");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`import { startLoggingProfilingEvents, stopLoggingProfilingEvents } from "../SchedulerProfiling";\n\nexport function run() {\n return startLoggingProfilingEvents();\n}\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "imports.js",
|
||||
op: "update",
|
||||
diff: '@@ import block\n-import { startLoggingProfilingEvents, stopLoggingProfilingEvents } from "../SchedulerProfiling";\n+import { stopLoggingProfilingEvents, startLoggingProfilingEvents } from "../SchedulerProfiling";',
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain(
|
||||
'import { stopLoggingProfilingEvents, startLoggingProfilingEvents } from "../SchedulerProfiling";',
|
||||
);
|
||||
});
|
||||
|
||||
test("ambiguous @@ context resolves via unique old lines", async () => {
|
||||
const filePath = join(tempDir, "ambiguous.ts");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`function getState() {\n return 1;\n}\n\nfunction getState() {\n return 2;\n}\n\nfunction getState() {\n return 3;\n}\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "ambiguous.ts",
|
||||
op: "update",
|
||||
diff: "@@ function getState() {\n- return 2;\n+ return 200;",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
expect(result).toContain("return 1;");
|
||||
expect(result).toContain("return 200;");
|
||||
expect(result).toContain("return 3;");
|
||||
});
|
||||
|
||||
test("duplicate context lines collapse for matching", async () => {
|
||||
const filePath = join(tempDir, "duplicate-context.txt");
|
||||
await Bun.write(filePath, "alpha\nbeta\ngamma\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "duplicate-context.txt",
|
||||
op: "update",
|
||||
diff: "@@\n alpha\n beta\n beta\n-gamma\n+GAMMA",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("alpha\nbeta\nGAMMA\n");
|
||||
});
|
||||
|
||||
test("repeated context blocks collapse when duplicated", async () => {
|
||||
const filePath = join(tempDir, "repeated-block.txt");
|
||||
await Bun.write(filePath, "if (ready) {\n handle();\n}\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "repeated-block.txt",
|
||||
op: "update",
|
||||
diff: "@@\n if (ready) {\n handle();\n}\n if (ready) {\n handle();\n}\n- handle();\n+ handleNext();",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("if (ready) {\n handleNext();\n}\n");
|
||||
});
|
||||
|
||||
test("shared prefix/suffix context is trimmed when mismatched", async () => {
|
||||
const filePath = join(tempDir, "trim-context.txt");
|
||||
await Bun.write(filePath, "function doThing() {\n return 1;\n}\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "trim-context.txt",
|
||||
op: "update",
|
||||
diff: "@@\n // NOTE: helper\n function doThing() {\n- return 1;\n+ return 2;\n }",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("function doThing() {\n return 2;\n}\n");
|
||||
});
|
||||
|
||||
test("single-line change fallback uses the unique changed line", async () => {
|
||||
const filePath = join(tempDir, "single-line-change.txt");
|
||||
await Bun.write(filePath, "function getState() {\n return 1;\n}\n\nfunction getState() {\n return 2;\n}\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "single-line-change.txt",
|
||||
op: "update",
|
||||
diff: "@@ function getState() {\n return 2;\n- return 2;\n+ return 200;\n return 2;",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
expect(result).toContain("return 1;");
|
||||
expect(result).toContain("return 200;");
|
||||
});
|
||||
|
||||
test("implicit context lines without prefixes are accepted", async () => {
|
||||
const filePath = join(tempDir, "implicit-context.ts");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`function getMousePosition(\n relativeContainer: null,\n mouseEvent: SyntheticMouseEvent,\n) {\n if (relativeContainer !== null) {\n return initialTooltipState;\n }\n}\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "implicit-context.ts",
|
||||
op: "update",
|
||||
diff: `@@ function getMousePosition(\nrelativeContainer: null,\nmouseEvent: SyntheticMouseEvent,\n) {\n- if (relativeContainer !== null) {\n+ if (relativeContainer === null) {\n return initialTooltipState;\n }`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("if (relativeContainer === null)");
|
||||
});
|
||||
|
||||
test("context lines preserve original file indentation when fuzzy matched", async () => {
|
||||
const filePath = join(tempDir, "context-indent.js");
|
||||
// File has 4-space indentation throughout the table
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`export function describeWithPointerEvent(message, describeFn) {
|
||||
const pointerEvent = 'PointerEvent';
|
||||
const fallback = 'MouseEvent/TouchEvent';
|
||||
describe.each\`
|
||||
value | name
|
||||
$true | $pointerEvent
|
||||
$true | $fallback
|
||||
\`(\`\${message}: $name\`, entry => {
|
||||
const hasPointerEvents = entry.value;
|
||||
setPointerEvent(hasPointerEvents);
|
||||
describeFn(hasPointerEvents);
|
||||
});
|
||||
}
|
||||
`,
|
||||
);
|
||||
|
||||
// Model provides diff with 3-space indentation in context lines (one less than file)
|
||||
// The changed line should be fixed, but context lines should NOT be modified
|
||||
await applyPatch(
|
||||
{
|
||||
path: "context-indent.js",
|
||||
op: "update",
|
||||
diff: `@@ describe.each\`
|
||||
value | name
|
||||
$true | $pointerEvent
|
||||
- $true | $fallback
|
||||
+ $false | $fallback
|
||||
\`(\`\${message}: $name\`, entry => {`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
// The changed line should have correct value (false instead of true)
|
||||
expect(result).toContain("$false | $fallback");
|
||||
// Context lines should preserve original 4-space indentation, not become 3-space
|
||||
expect(result).toContain(" value | name");
|
||||
expect(result).toContain(" $true | $pointerEvent");
|
||||
});
|
||||
|
||||
test("duplicate context lines are resolved via adjacent match to @@ anchor", async () => {
|
||||
const filePath = join(tempDir, "ReactFlightDOMClientNode.js");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`const handleEnd = () => {
|
||||
if (--streamEndedCount === 2) {
|
||||
cleanup();
|
||||
}
|
||||
if (--streamEndedCount === 2) {
|
||||
finalize();
|
||||
}
|
||||
};
|
||||
`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "ReactFlightDOMClientNode.js",
|
||||
op: "update",
|
||||
diff: `@@ const handleEnd = () => {
|
||||
if (--streamEndedCount === 2) {
|
||||
- if (--streamEndedCount === 2) {
|
||||
+ if (++streamEndedCount === 2) {`,
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
const lines = result.split("\n");
|
||||
expect(lines[1]).toContain("--streamEndedCount");
|
||||
expect(lines[4]).toContain("++streamEndedCount");
|
||||
});
|
||||
|
||||
test("strip line-number prefixes from diff content", async () => {
|
||||
const filePath = join(tempDir, "line-numbers.txt");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`Permission is hereby granted, free of charge\nA copy of this software and associated docs\nThe above copyright notice\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "line-numbers.txt",
|
||||
op: "update",
|
||||
diff: "@@\n 1\tPermission is hereby granted, free of charge\n- 2\tA copy of this software and associated docs\n+ 2\tA copy of this software AND associated docs\n 3\tThe above copyright notice",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
expect(result).toContain("A copy of this software AND associated docs");
|
||||
});
|
||||
|
||||
test("ellipsis placeholder lines are ignored during matching", async () => {
|
||||
const filePath = join(tempDir, "ellipsis.ts");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`function progress(done: boolean, value: string) {\n if (done) {\n return;\n }\n const buffer = value;\n return buffer;\n}\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "ellipsis.ts",
|
||||
op: "update",
|
||||
diff: "@@ function progress\n if (done) {\n return;\n }\n...\n- const buffer = value;\n+ const buffer = value.toUpperCase();",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("const buffer = value.toUpperCase();");
|
||||
});
|
||||
|
||||
test("context anchor retryIfBlockedOn() matches signature without params", async () => {
|
||||
const filePath = join(tempDir, "context-anchor.ts");
|
||||
await Bun.write(filePath, `function retryIfBlockedOn(reason: string, blockedOn: mixed) {\n return reason;\n}\n`);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "context-anchor.ts",
|
||||
op: "update",
|
||||
diff: "@@ retryIfBlockedOn()\n- return reason;\n+ return reason.toUpperCase();",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("return reason.toUpperCase();");
|
||||
});
|
||||
|
||||
test("ambiguous context falls back to unique old lines", async () => {
|
||||
const filePath = join(tempDir, "ambiguous-context.ts");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`function getState() {\n return 1;\n}\n\nfunction getState() {\n return 2;\n}\n\nfunction getState() {\n return 3;\n}\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "ambiguous-context.ts",
|
||||
op: "update",
|
||||
diff: "@@ function getState() {\n- return 2;\n+ return 200;",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
const result = readFileSync(filePath, "utf-8");
|
||||
expect(result).toContain("return 1;");
|
||||
expect(result).toContain("return 200;");
|
||||
expect(result).toContain("return 3;");
|
||||
});
|
||||
|
||||
test("comment-prefix mismatches still match expected lines", async () => {
|
||||
const filePath = join(tempDir, "comment-prefix.txt");
|
||||
await Bun.write(
|
||||
filePath,
|
||||
`/*\n * LICENSE file in the root directory.\n * Copyright (c) Meta Platforms, Inc.\n */\n`,
|
||||
);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "comment-prefix.txt",
|
||||
op: "update",
|
||||
diff: "@@\n-/ LICENSE file in the root directory.\n+ / LICENSE file in the root directory.\n",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("/ LICENSE file in the root directory.");
|
||||
});
|
||||
|
||||
test("context-less fuzzy match applies even with spacing differences", async () => {
|
||||
const filePath = join(tempDir, "fuzzy-contextless.ts");
|
||||
await Bun.write(filePath, "const value = computeTotal(items);\n");
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "fuzzy-contextless.ts",
|
||||
op: "update",
|
||||
diff: "-const value=computeTotal(items);\n+const value = calculateTotal(items);",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("calculateTotal");
|
||||
});
|
||||
|
||||
test("@@ header without space is accepted", async () => {
|
||||
const filePath = join(tempDir, "header-nospace.ts");
|
||||
await Bun.write(filePath, `const value = 1;\nconst other = 2;\n`);
|
||||
|
||||
await applyPatch(
|
||||
{
|
||||
path: "header-nospace.ts",
|
||||
op: "update",
|
||||
diff: "@@const value = 1;\n-const value = 1;\n+const value = 100;",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toContain("const value = 100;");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -145,12 +145,12 @@ function parseLegacyPatch(patch: string): LegacyParseResult {
|
||||
/** Convert legacy hunk to new PatchInput format */
|
||||
function legacyHunkToInput(hunk: LegacyHunk): PatchInput {
|
||||
if (hunk.type === "add") {
|
||||
return { path: hunk.path, operation: "create", diff: hunk.contents };
|
||||
return { path: hunk.path, op: "create", diff: hunk.contents };
|
||||
}
|
||||
if (hunk.type === "delete") {
|
||||
return { path: hunk.path, operation: "delete" };
|
||||
return { path: hunk.path, op: "delete" };
|
||||
}
|
||||
return { path: hunk.path, operation: "update", moveTo: hunk.movePath, diff: hunk.diffBody };
|
||||
return { path: hunk.path, op: "update", rename: hunk.movePath, diff: hunk.diffBody };
|
||||
}
|
||||
|
||||
/** Apply a legacy format patch (for test fixtures) */
|
||||
@@ -436,7 +436,7 @@ describe("applyPatch", () => {
|
||||
});
|
||||
|
||||
test("create file", async () => {
|
||||
const result = await applyPatch({ path: "add.txt", operation: "create", diff: "ab\ncd" }, { cwd: tempDir });
|
||||
const result = await applyPatch({ path: "add.txt", op: "create", diff: "ab\ncd" }, { cwd: tempDir });
|
||||
|
||||
expect(result.change.type).toBe("create");
|
||||
expect(readFileSync(join(tempDir, "add.txt"), "utf-8")).toBe("ab\ncd\n");
|
||||
@@ -446,7 +446,7 @@ describe("applyPatch", () => {
|
||||
const filePath = join(tempDir, "del.txt");
|
||||
await Bun.write(filePath, "x");
|
||||
|
||||
const result = await applyPatch({ path: "del.txt", operation: "delete" }, { cwd: tempDir });
|
||||
const result = await applyPatch({ path: "del.txt", op: "delete" }, { cwd: tempDir });
|
||||
|
||||
expect(result.change.type).toBe("delete");
|
||||
expect(await Bun.file(filePath).exists()).toBe(false);
|
||||
@@ -457,7 +457,7 @@ describe("applyPatch", () => {
|
||||
await Bun.write(filePath, "foo\nbar\n");
|
||||
|
||||
const result = await applyPatch(
|
||||
{ path: "update.txt", operation: "update", diff: "@@\n foo\n-bar\n+baz" },
|
||||
{ path: "update.txt", op: "update", diff: "@@\n foo\n-bar\n+baz" },
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
@@ -470,7 +470,7 @@ describe("applyPatch", () => {
|
||||
await Bun.write(srcPath, "line\n");
|
||||
|
||||
const result = await applyPatch(
|
||||
{ path: "src.txt", operation: "update", moveTo: "dst.txt", diff: "@@\n-line\n+line2" },
|
||||
{ path: "src.txt", op: "update", rename: "dst.txt", diff: "@@\n-line\n+line2" },
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
@@ -484,10 +484,7 @@ describe("applyPatch", () => {
|
||||
const filePath = join(tempDir, "multi.txt");
|
||||
await Bun.write(filePath, "foo\nbar\nbaz\nqux\n");
|
||||
|
||||
await applyPatch(
|
||||
{ path: "multi.txt", operation: "update", diff: "@@\n-bar\n+BAR\n@@\n-qux\n+QUX" },
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
await applyPatch({ path: "multi.txt", op: "update", diff: "@@\n-bar\n+BAR\n@@\n-qux\n+QUX" }, { cwd: tempDir });
|
||||
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("foo\nBAR\nbaz\nQUX\n");
|
||||
});
|
||||
@@ -499,7 +496,7 @@ describe("applyPatch", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "scope.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@ ## [Unreleased]\n ## [Unreleased]\n \n+### Added\n+\n+- New feature\n+\n ### Changed",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -517,7 +514,7 @@ describe("applyPatch", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "unicode.py",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n-import asyncio # local import - avoids top-level dep\n+import asyncio # HELLO",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -531,7 +528,7 @@ describe("applyPatch", () => {
|
||||
await Bun.write(filePath, "original\n");
|
||||
|
||||
const result = await applyPatch(
|
||||
{ path: "dryrun.txt", operation: "update", diff: "@@\n-original\n+modified" },
|
||||
{ path: "dryrun.txt", op: "update", diff: "@@\n-original\n+modified" },
|
||||
{ cwd: tempDir, dryRun: true },
|
||||
);
|
||||
|
||||
@@ -541,7 +538,7 @@ describe("applyPatch", () => {
|
||||
|
||||
test("missing file for update fails", async () => {
|
||||
await expect(
|
||||
applyPatch({ path: "nonexistent.txt", operation: "update", diff: "@@\n-foo\n+bar" }, { cwd: tempDir }),
|
||||
applyPatch({ path: "nonexistent.txt", op: "update", diff: "@@\n-foo\n+bar" }, { cwd: tempDir }),
|
||||
).rejects.toThrow(ApplyPatchError);
|
||||
});
|
||||
|
||||
@@ -549,13 +546,11 @@ describe("applyPatch", () => {
|
||||
const filePath = join(tempDir, "nodiff.txt");
|
||||
await Bun.write(filePath, "content\n");
|
||||
|
||||
await expect(applyPatch({ path: "nodiff.txt", operation: "update" }, { cwd: tempDir })).rejects.toThrow(
|
||||
"requires diff",
|
||||
);
|
||||
await expect(applyPatch({ path: "nodiff.txt", op: "update" }, { cwd: tempDir })).rejects.toThrow("requires diff");
|
||||
});
|
||||
|
||||
test("creates parent directories for create", async () => {
|
||||
await applyPatch({ path: "nested/deep/file.txt", operation: "create", diff: "content" }, { cwd: tempDir });
|
||||
await applyPatch({ path: "nested/deep/file.txt", op: "create", diff: "content" }, { cwd: tempDir });
|
||||
|
||||
const filePath = join(tempDir, "nested/deep/file.txt");
|
||||
expect(readFileSync(filePath, "utf-8")).toBe("content\n");
|
||||
@@ -566,7 +561,7 @@ describe("applyPatch", () => {
|
||||
await Bun.write(srcPath, "line\n");
|
||||
|
||||
await applyPatch(
|
||||
{ path: "src.txt", operation: "update", moveTo: "nested/deep/dst.txt", diff: "@@\n-line\n+newline" },
|
||||
{ path: "src.txt", op: "update", rename: "nested/deep/dst.txt", diff: "@@\n-line\n+newline" },
|
||||
{ cwd: tempDir },
|
||||
);
|
||||
|
||||
@@ -603,7 +598,7 @@ describe("simple replace mode", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "fuzzy.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
// No @@ marker, just -/+ lines
|
||||
diff: '-console.log("Hello");\n+console.log("World");',
|
||||
},
|
||||
@@ -621,7 +616,7 @@ describe("simple replace mode", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "indent.ts",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
// Diff uses 0 indentation, should be adjusted to 4 spaces
|
||||
diff: "-const x = 1;\n+const x = 42;",
|
||||
},
|
||||
@@ -640,7 +635,7 @@ describe("simple replace mode", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "context.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n middle\n-foo\n+FOO",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -658,7 +653,7 @@ describe("simple replace mode", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "multi.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@\n-bbb\n+BBB\n@@\n-ddd\n+DDD",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -675,7 +670,7 @@ describe("simple replace mode", () => {
|
||||
await applyPatch(
|
||||
{
|
||||
path: "scoped.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "@@ class Foo {\n- return 1;\n+ return 42;",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
@@ -692,7 +687,7 @@ describe("simple replace mode", () => {
|
||||
applyPatch(
|
||||
{
|
||||
path: "dupe.txt",
|
||||
operation: "update",
|
||||
op: "update",
|
||||
diff: "-foo\n+FOO",
|
||||
},
|
||||
{ cwd: tempDir },
|
||||
|
||||
@@ -19,12 +19,12 @@
|
||||
* - nightmare: Long files where target line repeats, minimal info
|
||||
*/
|
||||
|
||||
import { mkdirSync, readdirSync, statSync, rmSync, existsSync } from "node:fs";
|
||||
import { join, relative, dirname, basename } from "node:path";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import { $ } from "bun";
|
||||
import { createWriteStream, existsSync, mkdirSync, readdirSync } from "node:fs";
|
||||
import { basename, dirname, join, relative } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { createGzip } from "node:zlib";
|
||||
import { pack } from "tar-stream";
|
||||
import { parseArgs } from "node:util";
|
||||
import { ALL_MUTATIONS, CATEGORY_MAP, type Mutation, type MutationInfo } from "./mutations";
|
||||
|
||||
const SCRIPT_DIR = import.meta.dir;
|
||||
@@ -369,26 +369,25 @@ function getCandidatesForDifficulty(files: FileEntry[], difficulty: Difficulty):
|
||||
}
|
||||
}
|
||||
|
||||
function bunCheck(content: string, suffix: string): boolean {
|
||||
const tempPath = `/tmp/bench-check-${Date.now()}${suffix}`;
|
||||
async function bunCheck(content: string, suffix: string): Promise<boolean> {
|
||||
const tempPath = `/tmp/bench-check-${crypto.randomUUID()}${suffix}`;
|
||||
try {
|
||||
Bun.spawnSync(["rm", "-f", tempPath]);
|
||||
await $`rm -f ${tempPath}`;
|
||||
const file = Bun.file(tempPath);
|
||||
Bun.write(file, content);
|
||||
|
||||
const result = Bun.spawnSync(["bun", "build", tempPath, "--no-bundle"], { timeout: 5000 });
|
||||
await Bun.write(file, content);
|
||||
const result = await $`timeout 5s bun build ${tempPath} --no-bundle`;
|
||||
return result.exitCode === 0;
|
||||
} catch {
|
||||
return true;
|
||||
return false;
|
||||
} finally {
|
||||
Bun.spawnSync(["rm", "-f", tempPath]);
|
||||
await $`rm -f ${tempPath}`;
|
||||
}
|
||||
}
|
||||
|
||||
function isParsable(content: string, suffix: string): boolean {
|
||||
async function isParsable(content: string, suffix: string): Promise<boolean> {
|
||||
if (content.includes("@flow")) return true;
|
||||
try {
|
||||
return bunCheck(content, suffix);
|
||||
return await bunCheck(content, suffix);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
@@ -484,7 +483,7 @@ function createSeededRng(seed: number): () => number {
|
||||
};
|
||||
}
|
||||
|
||||
function generateCase(
|
||||
async function generateCase(
|
||||
rng: () => number,
|
||||
mutation: Mutation,
|
||||
files: FileEntry[],
|
||||
@@ -492,7 +491,7 @@ function generateCase(
|
||||
difficulty: Difficulty,
|
||||
minScore: number | null,
|
||||
attemptLimit = 100,
|
||||
): CaseResult | null {
|
||||
): Promise<CaseResult | null> {
|
||||
let candidates = getCandidatesForDifficulty(files, difficulty);
|
||||
if (candidates.length === 0) candidates = files;
|
||||
|
||||
@@ -511,7 +510,7 @@ function generateCase(
|
||||
if (!regionAvailable(usedLines, entry.path, info.lineNumber)) continue;
|
||||
|
||||
const suffix = "." + entry.path.split(".").pop();
|
||||
if (!isParsable(mutatedContent, suffix)) continue;
|
||||
if (!await isParsable(mutatedContent, suffix)) continue;
|
||||
|
||||
const diffScore = scoreDifficulty(entry, info.lineNumber);
|
||||
|
||||
@@ -677,12 +676,12 @@ async function main(): Promise<number> {
|
||||
|
||||
for (let index = 0; index < args.countPerType; index++) {
|
||||
const difficulty = difficultiesForType[index];
|
||||
let result = generateCase(rng, mutation, files, usedLines, difficulty, args.minScore);
|
||||
let result = await generateCase(rng, mutation, files, usedLines, difficulty, args.minScore);
|
||||
|
||||
if (!result) {
|
||||
for (const fallback of fallbackOrder) {
|
||||
if (fallback === difficulty) continue;
|
||||
result = generateCase(rng, mutation, files, usedLines, fallback, 0);
|
||||
result = await generateCase(rng, mutation, files, usedLines, fallback, 0);
|
||||
if (result) {
|
||||
console.log(`Note: ${mutation.name} case ${index + 1} fell back from ${difficulty} to ${fallback}`);
|
||||
break;
|
||||
@@ -755,4 +754,4 @@ async function main(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
main().then((code) => process.exit(code));
|
||||
process.exit(await main());
|
||||
|
||||
@@ -9,36 +9,28 @@
|
||||
* bun run bench:edit --fixtures fixtures.tar.gz
|
||||
*/
|
||||
|
||||
import { parseArgs } from "node:util";
|
||||
import { writeFile, mkdtemp, rm } from "node:fs/promises";
|
||||
import type { ThinkingLevel } from "@oh-my-pi/pi-agent-core";
|
||||
import { readdirSync } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ThinkingLevel } from "@oh-my-pi/pi-agent-core";
|
||||
import { loadTasks, loadTasksFromDir, validateFixtures, type EditTask } from "./tasks";
|
||||
import { parseArgs } from "node:util";
|
||||
import { generateJsonReport, generateReport } from "./report";
|
||||
import { runBenchmark, type BenchmarkConfig, type ProgressEvent } from "./runner";
|
||||
import { generateReport, generateJsonReport } from "./report";
|
||||
import { loadTasks, loadTasksFromDir, validateFixtures, type EditTask } from "./tasks";
|
||||
|
||||
const THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh"] as const;
|
||||
|
||||
function generateReportFilename(config: BenchmarkConfig, format: "markdown" | "json"): string {
|
||||
const modelName = config.model.replace(/[^a-zA-Z0-9-]/g, "_");
|
||||
const variant = config.editVariant ?? "auto";
|
||||
const fuzzyLabel = config.editFuzzy === true
|
||||
? "fuzzy"
|
||||
: config.editFuzzy === false
|
||||
? "strict"
|
||||
: "auto";
|
||||
const thresholdLabel = typeof config.editFuzzyThreshold === "number"
|
||||
? Math.round(config.editFuzzyThreshold * 100).toString()
|
||||
: "auto";
|
||||
const timestamp = new Date()
|
||||
const modelName = config.model.replace(/[^a-zA-Z0-9-]/g, "_").split("/").pop()!;
|
||||
const variant = config.editVariant ?? "replace";
|
||||
const timestamp = new Date()
|
||||
.toISOString()
|
||||
.replace(/:/g, "-")
|
||||
.replace(/\..+$/, "")
|
||||
.replace(/Z$/, "Z");
|
||||
const ext = format === "json" ? "json" : "md";
|
||||
return `run_${modelName}_${variant}_${fuzzyLabel}_${thresholdLabel}_${timestamp}.${ext}`;
|
||||
return `runs/${modelName}_${variant}_${timestamp}.${ext}`;
|
||||
}
|
||||
|
||||
function printUsage(tasks?: EditTask[]): void {
|
||||
@@ -325,7 +317,7 @@ async function main(): Promise<void> {
|
||||
const report = formatType === "json" ? generateJsonReport(result) : generateReport(result);
|
||||
const outputPath = values.output ?? generateReportFilename(config, formatType);
|
||||
|
||||
await writeFile(outputPath, report);
|
||||
await Bun.write(outputPath, report);
|
||||
console.log(`Report written to: ${outputPath}`);
|
||||
|
||||
if (cleanup) {
|
||||
|
||||
@@ -5,14 +5,16 @@
|
||||
* and verifying results. Supports parallel runs for reliability measurement.
|
||||
*/
|
||||
|
||||
import { mkdtemp, rm, cp, readdir, appendFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ThinkingLevel } from "@oh-my-pi/pi-agent-core";
|
||||
import { RpcClient } from "@oh-my-pi/pi-coding-agent";
|
||||
import { extractTaskFiles, type EditTask } from "./tasks";
|
||||
import { appendFile, cp, mkdtemp, readdir, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { formatDirectory } from "./formatter";
|
||||
import { verifyExpectedFiles } from "./verify";
|
||||
import { extractTaskFiles, type EditTask } from "./tasks";
|
||||
import { verifyExpectedFileSubset, verifyExpectedFiles } from "./verify";
|
||||
|
||||
|
||||
const ROOT_DIR = await mkdtemp("reb-");
|
||||
|
||||
|
||||
export interface BenchmarkConfig {
|
||||
@@ -117,6 +119,14 @@ export interface BenchmarkResult {
|
||||
endTime: string;
|
||||
}
|
||||
|
||||
interface TaskRunItem {
|
||||
task: EditTask;
|
||||
runIndex: number;
|
||||
}
|
||||
|
||||
const BATCH_MIN_SIZE = 3;
|
||||
const BATCH_MAX_SIZE = 5;
|
||||
|
||||
async function copyFixtures(task: EditTask, destDir: string): Promise<void> {
|
||||
if (task.tarballPath) {
|
||||
await extractTaskFiles(task.tarballPath, task.id, destDir, "input");
|
||||
@@ -137,7 +147,7 @@ async function getExpectedDir(task: EditTask): Promise<{ dir: string; cleanup: (
|
||||
return { dir: task.expectedDir, cleanup: async () => {} };
|
||||
}
|
||||
if (task.tarballPath) {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), `edit-bench-expected-${task.id}-`));
|
||||
const tempDir = await mkdtemp(join(ROOT_DIR, `expected-${task.id}-`));
|
||||
await extractTaskFiles(task.tarballPath, task.id, tempDir, "expected");
|
||||
return {
|
||||
dir: tempDir,
|
||||
@@ -178,7 +188,7 @@ async function runSingleTask(
|
||||
totalInputChars: 0,
|
||||
};
|
||||
|
||||
const logFile = `/tmp/bench-${task.id}-run${runIndex}.jsonl`;
|
||||
const logFile = join(ROOT_DIR, `run-${task.id}-${runIndex}.jsonl`);
|
||||
const logEvent = async (event: unknown) => {
|
||||
await appendFile(logFile, JSON.stringify(event) + "\n");
|
||||
};
|
||||
@@ -367,6 +377,140 @@ ${config.noEditRequired
|
||||
};
|
||||
}
|
||||
|
||||
async function runBatchedTask(
|
||||
item: TaskRunItem,
|
||||
config: BenchmarkConfig,
|
||||
workDir: string,
|
||||
expectedDir: string,
|
||||
client: RpcClient,
|
||||
): Promise<TaskRunResult> {
|
||||
const startTime = Date.now();
|
||||
const task = item.task;
|
||||
const runIndex = item.runIndex;
|
||||
let error: string | undefined;
|
||||
let patchApplied = false;
|
||||
let verificationPassed = false;
|
||||
let indentScore: number | undefined;
|
||||
let formattedEquivalent: boolean | undefined;
|
||||
let diffStats: { linesChanged: number; charsChanged: number } | undefined;
|
||||
let tokens: TokenStats = { input: 0, output: 0, total: 0 };
|
||||
let agentResponse: string | undefined;
|
||||
let diff: string | undefined;
|
||||
let editFailures: EditFailure[] = [];
|
||||
let toolStats = {
|
||||
read: 0,
|
||||
edit: 0,
|
||||
write: 0,
|
||||
editSuccesses: 0,
|
||||
editFailures: 0,
|
||||
totalInputChars: 0,
|
||||
};
|
||||
|
||||
const logFile = join(ROOT_DIR, `run-${task.id}-${runIndex}.jsonl`);
|
||||
const logEvent = async (event: unknown) => {
|
||||
await appendFile(logFile, JSON.stringify(event) + "\n");
|
||||
};
|
||||
|
||||
try {
|
||||
await appendFile(
|
||||
logFile,
|
||||
`{"type":"meta","task":"${task.id}","run":${runIndex},"workDir":"${workDir}","batched":true}\n`,
|
||||
);
|
||||
|
||||
const promptWithContext = buildPrompt(task, config);
|
||||
await appendFile(logFile, `{"type":"prompt","message":${JSON.stringify(promptWithContext)}}\n`);
|
||||
|
||||
const statsBefore = await client.getSessionStats();
|
||||
const events = await collectPromptEvents(client, promptWithContext, config, logEvent);
|
||||
const statsAfter = await client.getSessionStats();
|
||||
tokens = diffTokenStats(statsBefore, statsAfter);
|
||||
await logEvent({ type: "stats", before: statsBefore, after: statsAfter });
|
||||
|
||||
agentResponse = (await client.getLastAssistantText()) ?? undefined;
|
||||
await logEvent({ type: "response", text: agentResponse });
|
||||
|
||||
const pendingEdits = new Map<string, unknown>();
|
||||
|
||||
for (const event of events) {
|
||||
if (event.type === "tool_execution_start") {
|
||||
const e = event as { toolName?: string; toolCallId?: string; args?: unknown };
|
||||
const toolName = e.toolName;
|
||||
if (toolName === "read") toolStats.read++;
|
||||
else if (toolName === "edit") {
|
||||
toolStats.edit++;
|
||||
if (e.toolCallId) pendingEdits.set(e.toolCallId, e.args);
|
||||
} else if (toolName === "write") toolStats.write++;
|
||||
|
||||
if (e.args) {
|
||||
toolStats.totalInputChars += JSON.stringify(e.args).length;
|
||||
}
|
||||
} else if (event.type === "tool_execution_end") {
|
||||
const e = event as { toolName?: string; toolCallId?: string; isError?: boolean; result?: unknown };
|
||||
if (e.toolName === "edit" && e.toolCallId && pendingEdits.has(e.toolCallId)) {
|
||||
const args = pendingEdits.get(e.toolCallId) ?? null;
|
||||
pendingEdits.delete(e.toolCallId);
|
||||
if (e.isError) {
|
||||
toolStats.editFailures++;
|
||||
const toolError = extractToolErrorMessage(e.result);
|
||||
editFailures.push({ toolCallId: e.toolCallId, args, error: toolError });
|
||||
} else {
|
||||
toolStats.editSuccesses++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
patchApplied = toolStats.edit > 0;
|
||||
|
||||
const filesToVerify = task.files.length > 0 ? task.files : undefined;
|
||||
const verification = await verifyExpectedFileSubset(expectedDir, workDir, filesToVerify);
|
||||
if (config.autoFormat) {
|
||||
await formatDirectory(workDir);
|
||||
}
|
||||
|
||||
verificationPassed = verification.success;
|
||||
indentScore = verification.indentScore;
|
||||
formattedEquivalent = verification.formattedEquivalent;
|
||||
diffStats = verification.diffStats;
|
||||
diff = verification.diff;
|
||||
if (!verification.success && verification.error) {
|
||||
error = verification.error;
|
||||
}
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err);
|
||||
await logEvent({ type: "error", error });
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
const mustUseEditTool = Boolean(config.requireEditToolCall) && !config.noEditRequired;
|
||||
const success = verificationPassed && (!mustUseEditTool || patchApplied);
|
||||
const metadata = task.metadata;
|
||||
|
||||
await logEvent({ type: "result", success, patchApplied, verificationPassed, error, duration });
|
||||
console.log(` Log: ${logFile}`);
|
||||
|
||||
return {
|
||||
runIndex,
|
||||
success,
|
||||
patchApplied,
|
||||
verificationPassed,
|
||||
seed: metadata?.seed,
|
||||
mutationType: metadata?.mutationType,
|
||||
mutationCategory: metadata?.mutationCategory,
|
||||
difficultyScore: metadata?.difficultyScore,
|
||||
error,
|
||||
tokens,
|
||||
duration,
|
||||
indentScore,
|
||||
formattedEquivalent,
|
||||
diffStats,
|
||||
agentResponse,
|
||||
diff,
|
||||
toolCalls: toolStats,
|
||||
editFailures,
|
||||
};
|
||||
}
|
||||
|
||||
function extractToolErrorMessage(result: unknown): string {
|
||||
if (typeof result === "string") return result;
|
||||
if (!result || typeof result !== "object") return "Unknown error";
|
||||
@@ -386,6 +530,300 @@ function extractToolErrorMessage(result: unknown): string {
|
||||
}
|
||||
}
|
||||
|
||||
function shuffle<T>(items: T[]): T[] {
|
||||
const copy = items.slice();
|
||||
for (let i = copy.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[copy[i], copy[j]] = [copy[j]!, copy[i]!];
|
||||
}
|
||||
return copy;
|
||||
}
|
||||
|
||||
function pickBatchSize(remaining: number): number {
|
||||
const maxSize = Math.min(BATCH_MAX_SIZE, remaining);
|
||||
const minSize = Math.min(BATCH_MIN_SIZE, maxSize);
|
||||
return minSize + Math.floor(Math.random() * (maxSize - minSize + 1));
|
||||
}
|
||||
|
||||
function taskFileKeys(task: EditTask): string[] {
|
||||
return task.files.slice().sort();
|
||||
}
|
||||
|
||||
function buildRunBatches(items: TaskRunItem[]): TaskRunItem[][] {
|
||||
const pending = shuffle(items);
|
||||
const batches: TaskRunItem[][] = [];
|
||||
|
||||
while (pending.length > 0) {
|
||||
const targetSize = pickBatchSize(pending.length);
|
||||
const batch: TaskRunItem[] = [];
|
||||
const usedFiles = new Set<string>();
|
||||
|
||||
for (let i = 0; i < pending.length && batch.length < targetSize; ) {
|
||||
const item = pending[i]!;
|
||||
const files = taskFileKeys(item.task);
|
||||
if (files.some((file) => usedFiles.has(file))) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
pending.splice(i, 1);
|
||||
batch.push(item);
|
||||
for (const file of files) {
|
||||
usedFiles.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.length === 0 && pending.length > 0) {
|
||||
batch.push(pending.shift()!);
|
||||
}
|
||||
|
||||
batches.push(shuffle(batch));
|
||||
}
|
||||
|
||||
return batches;
|
||||
}
|
||||
|
||||
function buildPrompt(task: EditTask, config: BenchmarkConfig): string {
|
||||
return `You are working in a repository with multiple unrelated files.
|
||||
|
||||
${task.prompt}
|
||||
|
||||
**Important constraints:**
|
||||
- Make the minimum change necessary. Do not refactor, improve, or "clean up" other code.
|
||||
- If you see multiple similar patterns, only change the ONE that is buggy.
|
||||
- Preserve exact code structure. Do not rearrange statements or change formatting.
|
||||
- Only modify the file(s) referenced by this request. Leave all other files unchanged.
|
||||
|
||||
${config.noEditRequired
|
||||
? "Read the relevant files first, then apply the fix."
|
||||
: "Read the relevant files first, then use the edit tool to apply the fix."}`;
|
||||
}
|
||||
|
||||
async function collectPromptEvents(
|
||||
client: RpcClient,
|
||||
prompt: string,
|
||||
config: BenchmarkConfig,
|
||||
logEvent: (event: unknown) => Promise<void>,
|
||||
): Promise<Array<{ type: string; [key: string]: unknown }>> {
|
||||
const events: Array<{ type: string; [key: string]: unknown }> = [];
|
||||
let unsubscribe: (() => void) | undefined;
|
||||
const eventsPromise = new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
unsubscribe?.();
|
||||
reject(new Error("Timeout waiting for agent_end"));
|
||||
}, config.timeout);
|
||||
|
||||
let pendingRetry = false;
|
||||
|
||||
unsubscribe = client.onEvent(async (event) => {
|
||||
events.push(event);
|
||||
|
||||
if (
|
||||
event.type === "tool_execution_start" ||
|
||||
event.type === "tool_execution_end" ||
|
||||
event.type === "message_end"
|
||||
) {
|
||||
await logEvent(event);
|
||||
}
|
||||
|
||||
if ((event.type as string) === "auto_retry_start") {
|
||||
pendingRetry = true;
|
||||
} else if (event.type === "turn_start" && pendingRetry) {
|
||||
pendingRetry = false;
|
||||
}
|
||||
|
||||
if (event.type === "agent_end") {
|
||||
if (pendingRetry) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(timer);
|
||||
unsubscribe?.();
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
await client.prompt(prompt);
|
||||
} catch (err) {
|
||||
unsubscribe?.();
|
||||
throw err;
|
||||
}
|
||||
await eventsPromise;
|
||||
return events;
|
||||
}
|
||||
|
||||
function diffTokenStats(
|
||||
before: { tokens: { input: number; output: number; total: number } },
|
||||
after: { tokens: { input: number; output: number; total: number } },
|
||||
): TokenStats {
|
||||
const input = Math.max(0, after.tokens.input - before.tokens.input);
|
||||
const output = Math.max(0, after.tokens.output - before.tokens.output);
|
||||
const total = Math.max(0, after.tokens.total - before.tokens.total);
|
||||
return { input, output, total };
|
||||
}
|
||||
|
||||
function summarizeTaskRuns(task: EditTask, runs: TaskRunResult[]): TaskResult {
|
||||
const orderedRuns = runs.slice().sort((a, b) => a.runIndex - b.runIndex);
|
||||
const n = orderedRuns.length;
|
||||
const successfulRuns = orderedRuns.filter((r) => r.success).length;
|
||||
const successRate = n > 0 ? successfulRuns / n : 0;
|
||||
|
||||
const avgTokens: TokenStats = n > 0
|
||||
? {
|
||||
input: Math.round(orderedRuns.reduce((sum, r) => sum + r.tokens.input, 0) / n),
|
||||
output: Math.round(orderedRuns.reduce((sum, r) => sum + r.tokens.output, 0) / n),
|
||||
total: Math.round(orderedRuns.reduce((sum, r) => sum + r.tokens.total, 0) / n),
|
||||
}
|
||||
: { input: 0, output: 0, total: 0 };
|
||||
|
||||
const avgDuration = n > 0 ? Math.round(orderedRuns.reduce((sum, r) => sum + r.duration, 0) / n) : 0;
|
||||
const indentScores = orderedRuns
|
||||
.map((run) => run.indentScore)
|
||||
.filter((score): score is number => typeof score === "number");
|
||||
const avgIndentScore = indentScores.length > 0
|
||||
? indentScores.reduce((sum, score) => sum + score, 0) / indentScores.length
|
||||
: 0;
|
||||
|
||||
const avgToolCalls: ToolCallStats = n > 0
|
||||
? {
|
||||
read: orderedRuns.reduce((sum, r) => sum + r.toolCalls.read, 0) / n,
|
||||
edit: orderedRuns.reduce((sum, r) => sum + r.toolCalls.edit, 0) / n,
|
||||
write: orderedRuns.reduce((sum, r) => sum + r.toolCalls.write, 0) / n,
|
||||
editSuccesses: orderedRuns.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0) / n,
|
||||
editFailures: orderedRuns.reduce((sum, r) => sum + r.toolCalls.editFailures, 0) / n,
|
||||
totalInputChars: orderedRuns.reduce((sum, r) => sum + r.toolCalls.totalInputChars, 0) / n,
|
||||
}
|
||||
: { read: 0, edit: 0, write: 0, editSuccesses: 0, editFailures: 0, totalInputChars: 0 };
|
||||
|
||||
const totalEditAttempts = orderedRuns.reduce((sum, r) => sum + r.toolCalls.edit, 0);
|
||||
const totalEditSuccesses = orderedRuns.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0);
|
||||
const editSuccessRate = totalEditAttempts > 0 ? totalEditSuccesses / totalEditAttempts : 1;
|
||||
|
||||
return {
|
||||
id: task.id,
|
||||
name: task.name,
|
||||
files: task.files,
|
||||
runs: orderedRuns,
|
||||
successRate,
|
||||
avgTokens,
|
||||
avgDuration,
|
||||
avgIndentScore,
|
||||
avgToolCalls,
|
||||
editSuccessRate,
|
||||
};
|
||||
}
|
||||
|
||||
function buildFailureResult(item: TaskRunItem, error: string): TaskRunResult {
|
||||
return {
|
||||
runIndex: item.runIndex,
|
||||
success: false,
|
||||
patchApplied: false,
|
||||
verificationPassed: false,
|
||||
error,
|
||||
tokens: { input: 0, output: 0, total: 0 },
|
||||
duration: 0,
|
||||
toolCalls: {
|
||||
read: 0,
|
||||
edit: 0,
|
||||
write: 0,
|
||||
editSuccesses: 0,
|
||||
editFailures: 0,
|
||||
totalInputChars: 0,
|
||||
},
|
||||
editFailures: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function runBatch(
|
||||
items: TaskRunItem[],
|
||||
config: BenchmarkConfig,
|
||||
cliPath: string,
|
||||
onProgress?: (event: ProgressEvent) => void,
|
||||
): Promise<Array<{ task: EditTask; result: TaskRunResult }>> {
|
||||
const workDir = await mkdtemp(join(ROOT_DIR, "batch-"));
|
||||
const results: Array<{ task: EditTask; result: TaskRunResult }> = [];
|
||||
let client: RpcClient | null = null;
|
||||
const expectedDirs = new Map<string, { dir: string; cleanup: () => Promise<void> }>();
|
||||
|
||||
const orderedItems = shuffle(items);
|
||||
const remaining = orderedItems.slice();
|
||||
|
||||
try {
|
||||
await Promise.all(
|
||||
orderedItems.map(async (item) => {
|
||||
const expected = await getExpectedDir(item.task);
|
||||
expectedDirs.set(item.task.id, expected);
|
||||
}),
|
||||
);
|
||||
|
||||
await Promise.all(orderedItems.map((item) => copyFixtures(item.task, workDir)));
|
||||
|
||||
const env: Record<string, string> = { OMP_NO_TITLE: "1" };
|
||||
if (config.editVariant !== undefined) {
|
||||
env.OMP_EDIT_VARIANT = config.editVariant;
|
||||
}
|
||||
if (config.editFuzzy !== undefined) {
|
||||
env.OMP_EDIT_FUZZY = config.editFuzzy === "auto" ? "auto" : config.editFuzzy ? "1" : "0";
|
||||
}
|
||||
if (config.editFuzzyThreshold !== undefined) {
|
||||
env.OMP_EDIT_FUZZY_THRESHOLD = config.editFuzzyThreshold === "auto" ? "auto" : String(config.editFuzzyThreshold);
|
||||
}
|
||||
|
||||
client = new RpcClient({
|
||||
cliPath,
|
||||
cwd: workDir,
|
||||
provider: config.provider,
|
||||
model: config.model,
|
||||
args: ["--tools", "read,edit,write,ls"],
|
||||
env,
|
||||
});
|
||||
|
||||
await client.start();
|
||||
|
||||
if (config.thinkingLevel) {
|
||||
await client.setThinkingLevel(config.thinkingLevel);
|
||||
}
|
||||
|
||||
for (const item of orderedItems) {
|
||||
const expectedDir = expectedDirs.get(item.task.id)?.dir;
|
||||
if (!expectedDir) {
|
||||
throw new Error(`Missing expected directory for task ${item.task.id}`);
|
||||
}
|
||||
|
||||
onProgress?.({ taskId: item.task.id, runIndex: item.runIndex, status: "started" });
|
||||
const result = await runBatchedTask(item, config, workDir, expectedDir, client);
|
||||
onProgress?.({ taskId: item.task.id, runIndex: item.runIndex, status: "completed", result });
|
||||
results.push({ task: item.task, result });
|
||||
remaining.shift();
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
for (const item of remaining) {
|
||||
const result = buildFailureResult(item, message);
|
||||
onProgress?.({ taskId: item.task.id, runIndex: item.runIndex, status: "completed", result });
|
||||
results.push({ task: item.task, result });
|
||||
}
|
||||
} finally {
|
||||
for (const expected of expectedDirs.values()) {
|
||||
await expected.cleanup();
|
||||
}
|
||||
if (client) {
|
||||
try {
|
||||
await client.stop();
|
||||
} catch {
|
||||
// Ignore stop errors
|
||||
}
|
||||
}
|
||||
try {
|
||||
await rm(workDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function runTask(
|
||||
task: EditTask,
|
||||
config: BenchmarkConfig,
|
||||
@@ -398,7 +836,7 @@ export async function runTask(
|
||||
|
||||
try {
|
||||
for (let i = 0; i < config.runsPerTask; i++) {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), `edit-bench-${task.id}-`));
|
||||
const tempDir = await mkdtemp(join(ROOT_DIR, `${task.id}-`));
|
||||
tempDirs.push(tempDir);
|
||||
await copyFixtures(task, tempDir);
|
||||
}
|
||||
@@ -411,50 +849,7 @@ export async function runTask(
|
||||
});
|
||||
|
||||
const runs = await Promise.all(runPromises);
|
||||
const n = runs.length;
|
||||
|
||||
const successfulRuns = runs.filter((r) => r.success).length;
|
||||
const successRate = successfulRuns / n;
|
||||
|
||||
const avgTokens: TokenStats = {
|
||||
input: Math.round(runs.reduce((sum, r) => sum + r.tokens.input, 0) / n),
|
||||
output: Math.round(runs.reduce((sum, r) => sum + r.tokens.output, 0) / n),
|
||||
total: Math.round(runs.reduce((sum, r) => sum + r.tokens.total, 0) / n),
|
||||
};
|
||||
|
||||
const avgDuration = Math.round(runs.reduce((sum, r) => sum + r.duration, 0) / n);
|
||||
const indentScores = runs
|
||||
.map((run) => run.indentScore)
|
||||
.filter((score): score is number => typeof score === "number");
|
||||
const avgIndentScore = indentScores.length > 0
|
||||
? indentScores.reduce((sum, score) => sum + score, 0) / indentScores.length
|
||||
: 0;
|
||||
|
||||
const avgToolCalls: ToolCallStats = {
|
||||
read: runs.reduce((sum, r) => sum + r.toolCalls.read, 0) / n,
|
||||
edit: runs.reduce((sum, r) => sum + r.toolCalls.edit, 0) / n,
|
||||
write: runs.reduce((sum, r) => sum + r.toolCalls.write, 0) / n,
|
||||
editSuccesses: runs.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0) / n,
|
||||
editFailures: runs.reduce((sum, r) => sum + r.toolCalls.editFailures, 0) / n,
|
||||
totalInputChars: runs.reduce((sum, r) => sum + r.toolCalls.totalInputChars, 0) / n,
|
||||
};
|
||||
|
||||
const totalEditAttempts = runs.reduce((sum, r) => sum + r.toolCalls.edit, 0);
|
||||
const totalEditSuccesses = runs.reduce((sum, r) => sum + r.toolCalls.editSuccesses, 0);
|
||||
const editSuccessRate = totalEditAttempts > 0 ? totalEditSuccesses / totalEditAttempts : 1;
|
||||
|
||||
return {
|
||||
id: task.id,
|
||||
name: task.name,
|
||||
files: task.files,
|
||||
runs,
|
||||
successRate,
|
||||
avgTokens,
|
||||
avgDuration,
|
||||
avgIndentScore,
|
||||
avgToolCalls,
|
||||
editSuccessRate,
|
||||
};
|
||||
return summarizeTaskRuns(task, runs);
|
||||
} finally {
|
||||
await cleanupExpected();
|
||||
for (const dir of tempDirs) {
|
||||
@@ -473,27 +868,38 @@ export async function runBenchmark(
|
||||
onProgress?: (event: ProgressEvent) => void,
|
||||
): Promise<BenchmarkResult> {
|
||||
const startTime = new Date().toISOString();
|
||||
const taskResults: TaskResult[] = [];
|
||||
const runItems: TaskRunItem[] = tasks.flatMap((task) =>
|
||||
Array.from({ length: config.runsPerTask }, (_, runIndex) => ({ task, runIndex })),
|
||||
);
|
||||
|
||||
const batches = buildRunBatches(runItems);
|
||||
const resultsByTask = new Map<string, TaskRunResult[]>();
|
||||
const concurrency = Math.max(1, Math.floor(config.taskConcurrency));
|
||||
const pendingTasks = [...tasks];
|
||||
const pendingBatches = [...batches];
|
||||
const running: Promise<void>[] = [];
|
||||
const cliPath = join(import.meta.dir, "../coding-agent/src/cli.ts");
|
||||
|
||||
const runNext = async (): Promise<void> => {
|
||||
const nextTask = pendingTasks.shift();
|
||||
if (!nextTask) return;
|
||||
const result = await runTask(nextTask, config, onProgress);
|
||||
taskResults.push(result);
|
||||
const nextBatch = pendingBatches.shift();
|
||||
if (!nextBatch) return;
|
||||
const batchResults = await runBatch(nextBatch, config, cliPath, onProgress);
|
||||
for (const { task, result } of batchResults) {
|
||||
const list = resultsByTask.get(task.id) ?? [];
|
||||
list.push(result);
|
||||
resultsByTask.set(task.id, list);
|
||||
}
|
||||
await runNext();
|
||||
};
|
||||
|
||||
const slots = Math.min(concurrency, pendingTasks.length || 0);
|
||||
const slots = Math.min(concurrency, pendingBatches.length || 0);
|
||||
for (let i = 0; i < slots; i++) {
|
||||
running.push(runNext());
|
||||
}
|
||||
|
||||
await Promise.all(running);
|
||||
|
||||
const taskResults = tasks.map((task) => summarizeTaskRuns(task, resultsByTask.get(task.id) ?? []));
|
||||
|
||||
const endTime = new Date().toISOString();
|
||||
|
||||
const allRuns = taskResults.flatMap((t) => t.runs);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,972 @@
|
||||
# Edit Benchmark Report
|
||||
|
||||
## Configuration
|
||||
|
||||
| Setting | Value |
|
||||
|---------|-------|
|
||||
| Date | 2026-01-19T13:45:20.687Z |
|
||||
| Model | anthropic/p-anthropic/claude-haiku-4-5 |
|
||||
| Thinking Level | default |
|
||||
| Runs per task | 3 |
|
||||
| Edit Variant | patch |
|
||||
| Edit Fuzzy | auto |
|
||||
| Edit Fuzzy Threshold | auto |
|
||||
| Require Edit Tool | no |
|
||||
| No-Edit Baseline | no |
|
||||
|
||||
## Summary
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Total Tasks | 60 |
|
||||
| Total Runs | 180 |
|
||||
| Successful Runs | 101 |
|
||||
| **Task Success Rate** | **56.1% (101/180)** |
|
||||
| Verified Rate | 56.1% (101/180) |
|
||||
| Edit Tool Usage Rate | 98.3% (177/180) |
|
||||
| **Edit Success Rate** | **98.3%** |
|
||||
| Patch Failure Rate | 1.7% (4/240) |
|
||||
| Tasks All Passing | 20 |
|
||||
| Tasks Flaky/Failing | 40 |
|
||||
|
||||
### Tool Calls
|
||||
|
||||
| Tool | Total | Avg/Run |
|
||||
|------|-------|---------|
|
||||
| Read | 387 | 2.1 |
|
||||
| Edit | 240 | 1.3 |
|
||||
| Write | 5 | 0.0 |
|
||||
| **Tool Input Chars** | 152,698 | 848 |
|
||||
|
||||
### Tokens & Time
|
||||
|
||||
| Metric | Total | Avg/Run |
|
||||
|--------|-------|---------|
|
||||
| Input Tokens | 4,112 | 23 |
|
||||
| Output Tokens | 181,227 | 1,007 |
|
||||
| Total Tokens | 14,085,828 | 78,255 |
|
||||
| Duration | 2680.8s | 14.9s |
|
||||
| **Avg Indent Score** | — | **2.18** |
|
||||
|
||||
## Task Results
|
||||
|
||||
| Task | File | Success | Edit Hit | R/E/W | Tokens (In/Out) | Time | Indent |
|
||||
|------|------|---------|----------|-------|-----------------|------|--------|
|
||||
| Access Remove Optional Chain 001 | registerDevToolsEventLogger.js | 3/3 ✅ | 100.0% | 2/1/0 | 18/841 | 12.6s | 1.00 |
|
||||
| Access Remove Optional Chain 002 | TimelineContext.js | 0/3 ❌ | 100.0% | 1/1/0 | 14/582 | 9.9s | 1.29 |
|
||||
| Access Remove Optional Chain 003 | astUtils.js | 0/3 ❌ | 100.0% | 8/3/0 | 64/4,873 | 52.1s | 4.85 |
|
||||
| Call Swap Call Args 001 | testHelpers.js | 2/3 ⚠️ | 100.0% | 3/2/0 | 27/771 | 13.5s | 1.33 |
|
||||
| Call Swap Call Args 002 | FlamegraphChartBuilder.js | 1/3 ⚠️ | 100.0% | 2/1/0 | 22/1,325 | 17.1s | 3.79 |
|
||||
| Call Swap Call Args 003 | SyntheticEvent.js | 1/3 ⚠️ | 100.0% | 2/2/0 | 26/1,389 | 16.5s | 3.76 |
|
||||
| Duplicate Duplicate Line Flip 001 | index.js | 3/3 ✅ | 100.0% | 1/1/0 | 16/443 | 9.2s | 0.00 |
|
||||
| Duplicate Duplicate Line Flip 002 | ActivityList.js | 3/3 ✅ | 100.0% | 1/1/0 | 14/485 | 8.2s | 3.61 |
|
||||
| Duplicate Duplicate Line Flip 003 | SyntheticEvent.js | 3/3 ✅ | 100.0% | 1/1/0 | 15/880 | 12.0s | 1.02 |
|
||||
| Identifier Identifier Multi Edit 001 | TabBar.js | 2/3 ⚠️ | 100.0% | 3/3/0 | 37/1,175 | 17.6s | 3.27 |
|
||||
| Identifier Identifier Multi Edit 002 | EventPluginRegistry.js | 3/3 ✅ | 100.0% | 1/2/0 | 19/901 | 12.6s | 3.94 |
|
||||
| Identifier Identifier Multi Edit 003 | ReactPerformanceTrackProperties.js | 2/3 ⚠️ | 100.0% | 9/3/0 | 69/1,978 | 29.7s | 9.95 |
|
||||
| Import Swap Named Imports 001 | CommitFlamegraphListItem.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 15/605 | 9.2s | 2.86 |
|
||||
| Import Swap Named Imports 002 | ReactDOMTextarea.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 13/391 | 7.6s | 2.36 |
|
||||
| Import Swap Named Imports 003 | StyleEditor.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 18/1,826 | 18.8s | 1.31 |
|
||||
| Literal Flip Boolean 001 | testHelpers.js | 3/3 ✅ | 100.0% | 1/1/0 | 17/422 | 7.2s | 1.33 |
|
||||
| Literal Flip Boolean 002 | ReactNoopFlightServer.js | 0/3 ❌ | 100.0% | 1/1/0 | 16/766 | 11.8s | 1.11 |
|
||||
| Literal Flip Boolean 003 | ReactFlightDOMClientEdge.js | 1/3 ⚠️ | 75.0% | 2/1/0 | 19/788 | 12.3s | 3.58 |
|
||||
| Literal Off By One 001 | githubAPI.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 14/442 | 7.2s | 0.47 |
|
||||
| Literal Off By One 002 | code-path.js | 1/3 ⚠️ | 100.0% | 3/2/0 | 35/1,013 | 16.2s | 3.50 |
|
||||
| Literal Off By One 003 | InspectedElement.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 16/581 | 10.0s | 1.20 |
|
||||
| Operator Remove Negation 001 | ReactDOMClient.js | 1/3 ⚠️ | 100.0% | 2/1/0 | 16/693 | 51.1s | 1.08 |
|
||||
| Operator Remove Negation 002 | NativeEventsView.js | 3/3 ✅ | 100.0% | 2/1/0 | 19/620 | 10.3s | 3.03 |
|
||||
| Operator Remove Negation 003 | ReactFlightUnbundledReferences.js | 0/3 ❌ | 100.0% | 4/1/0 | 33/1,025 | 17.7s | 2.00 |
|
||||
| Operator Swap Arithmetic 001 | fallbackEvalContext.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 13/5,318 | 5.8s | 0.00 |
|
||||
| Operator Swap Arithmetic 002 | CSSShorthandProperty.js | 3/3 ✅ | 100.0% | 1/1/0 | 14/361 | 8.6s | 2.88 |
|
||||
| Operator Swap Arithmetic 003 | hooks.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 15/1,080 | 15.1s | 1.50 |
|
||||
| Operator Swap Comparison 001 | index.js | 3/3 ✅ | 100.0% | 1/1/0 | 16/365 | 7.9s | 0.00 |
|
||||
| Operator Swap Comparison 002 | ReactFlightDOMServerBrowser.js | 0/3 ❌ | 100.0% | 1/1/0 | 15/571 | 9.3s | 1.57 |
|
||||
| Operator Swap Comparison 003 | ReactFlightDOMServerNode.js | 1/3 ⚠️ | 100.0% | 2/1/0 | 22/757 | 11.7s | 1.95 |
|
||||
| Operator Swap Equality 001 | readInputData.js | 3/3 ✅ | 100.0% | 1/1/0 | 16/482 | 7.3s | 0.00 |
|
||||
| Operator Swap Equality 002 | editor.js | 2/3 ⚠️ | 75.0% | 1/1/0 | 17/506 | 8.1s | 0.00 |
|
||||
| Operator Swap Equality 003 | hooks.js | 3/3 ✅ | 100.0% | 2/1/0 | 21/618 | 11.5s | 2.25 |
|
||||
| Operator Swap Increment Decrement 001 | ReactFlightDOMClientNode.js | 3/3 ✅ | 100.0% | 1/1/0 | 14/414 | 7.4s | 1.52 |
|
||||
| Operator Swap Increment Decrement 002 | ReactFlightDOMClientNode.js | 2/3 ⚠️ | 100.0% | 3/1/0 | 23/828 | 14.0s | 1.92 |
|
||||
| Operator Swap Increment Decrement 003 | loadSourceAndMetadata.js | 3/3 ✅ | 100.0% | 3/1/0 | 29/738 | 14.3s | 3.72 |
|
||||
| Operator Swap Logical 001 | profiling.js | 2/3 ⚠️ | 100.0% | 3/2/1 | 39/1,407 | 18.3s | 0.00 |
|
||||
| Operator Swap Logical 002 | SourceMapMetadataConsumer.js | 0/3 ❌ | 100.0% | 1/2/0 | 20/1,056 | 13.8s | 3.11 |
|
||||
| Operator Swap Logical 003 | DevToolsFiberComponentStack.js | 3/3 ✅ | 100.0% | 1/1/0 | 17/533 | 9.9s | 4.13 |
|
||||
| Operator Swap Nullish 001 | getBatchRange.js | 3/3 ✅ | 100.0% | 1/1/0 | 15/524 | 8.3s | 1.33 |
|
||||
| Operator Swap Nullish 002 | EnterLeaveEventPlugin.js | 1/3 ⚠️ | 100.0% | 4/2/0 | 34/1,938 | 22.5s | 1.56 |
|
||||
| Operator Swap Nullish 003 | backend.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 15/698 | 10.8s | 3.15 |
|
||||
| Regex Swap Regex Quantifier 001 | githubAPI.js | 3/3 ✅ | 100.0% | 1/1/0 | 16/443 | 7.9s | 0.67 |
|
||||
| Regex Swap Regex Quantifier 002 | ReactFlightStackConfigV8.js | 1/3 ⚠️ | 100.0% | 4/3/0 | 39/1,902 | 24.9s | 3.06 |
|
||||
| Regex Swap Regex Quantifier 003 | utils.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 15/600 | 49.0s | 2.00 |
|
||||
| Structural Delete Statement 001 | UnsupportedVersionDialog.js | 3/3 ✅ | 100.0% | 2/1/0 | 20/556 | 9.8s | 6.22 |
|
||||
| Structural Delete Statement 002 | getComponentNameFromFiber.js | 0/3 ❌ | 100.0% | 2/1/0 | 19/566 | 10.8s | 0.62 |
|
||||
| Structural Delete Statement 003 | simulateBrowserEventDispatch.js | 1/3 ⚠️ | 100.0% | 6/3/0 | 55/3,318 | 40.8s | 4.46 |
|
||||
| Structural Remove Early Return 001 | InspectedElementStateTree.js | 0/3 ❌ | 100.0% | 3/1/0 | 26/904 | 13.6s | 0.36 |
|
||||
| Structural Remove Early Return 002 | useCommitFilteringAndNavigation.js | 2/3 ⚠️ | 100.0% | 3/1/0 | 22/737 | 14.8s | 3.79 |
|
||||
| Structural Remove Early Return 003 | ReactFiberAsyncAction.js | 0/3 ❌ | 100.0% | 1/1/0 | 14/645 | 8.7s | 1.45 |
|
||||
| Structural Swap Adjacent Lines 001 | ReactServerConsoleConfigPlain.js | 3/3 ✅ | 100.0% | 3/2/0 | 35/1,408 | 20.0s | 1.00 |
|
||||
| Structural Swap Adjacent Lines 002 | ReactNoopFlightServer.js | 0/3 ❌ | 100.0% | 2/1/0 | 17/550 | 9.9s | 0.74 |
|
||||
| Structural Swap Adjacent Lines 003 | backend.js | 0/3 ❌ | 85.7% | 7/2/0 | 56/2,514 | 32.3s | 1.03 |
|
||||
| Structural Swap If Else 001 | importFile.js | 1/3 ⚠️ | 100.0% | 2/1/0 | 21/582 | 10.8s | 0.00 |
|
||||
| Structural Swap If Else 002 | ReactNativeFiberInspector.js | 0/3 ❌ | 100.0% | 1/1/0 | 16/540 | 8.2s | 3.18 |
|
||||
| Structural Swap If Else 003 | ReactDOMFizzStaticNode.js | 2/3 ⚠️ | 80.0% | 3/2/0 | 25/912 | 13.7s | 1.26 |
|
||||
| Unicode Unicode Hyphen 001 | Rectangle.js | 3/3 ✅ | 100.0% | 1/1/0 | 15/346 | 7.1s | 3.00 |
|
||||
| Unicode Unicode Hyphen 002 | UnsupportedBridgeProtocolDialog.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 15/418 | 7.4s | 3.79 |
|
||||
| Unicode Unicode Hyphen 003 | ReactTypes.js | 3/3 ✅ | 100.0% | 1/1/0 | 17/461 | 8.4s | 1.24 |
|
||||
|
||||
## Category Summary
|
||||
|
||||
| Category | Runs | Verified | Edit Used | Success | Min/Avg/Max Difficulty |
|
||||
|----------|------|----------|-----------|---------|------------------------|
|
||||
| access | 9 | 33.3% (3/9) | 100.0% (9/9) | 33.3% (3/9) | 7 / 8.7 / 10 |
|
||||
| call | 9 | 44.4% (4/9) | 100.0% (9/9) | 44.4% (4/9) | 6 / 7.7 / 10 |
|
||||
| duplicate | 9 | 100.0% (9/9) | 100.0% (9/9) | 100.0% (9/9) | 7 / 9.7 / 12 |
|
||||
| identifier | 9 | 77.8% (7/9) | 100.0% (9/9) | 77.8% (7/9) | 6 / 9.3 / 14 |
|
||||
| import | 9 | 55.6% (5/9) | 100.0% (9/9) | 55.6% (5/9) | 2 / 4.7 / 6 |
|
||||
| literal | 18 | 44.4% (8/18) | 100.0% (18/18) | 44.4% (8/18) | 4 / 6.2 / 9 |
|
||||
| operator | 63 | 61.9% (39/63) | 96.8% (61/63) | 61.9% (39/63) | 1 / 6.5 / 13 |
|
||||
| regex | 9 | 66.7% (6/9) | 88.9% (8/9) | 66.7% (6/9) | 6 / 7.3 / 8 |
|
||||
| structural | 36 | 33.3% (12/36) | 100.0% (36/36) | 33.3% (12/36) | 4 / 7.6 / 15 |
|
||||
| unicode | 9 | 88.9% (8/9) | 100.0% (9/9) | 88.9% (8/9) | 1 / 3.0 / 6 |
|
||||
|
||||
## Mutation Summary
|
||||
|
||||
| Mutation | Category | Runs | Verified | Edit Used | Success |
|
||||
|----------|----------|------|----------|-----------|---------|
|
||||
| delete-statement | structural | 9 | 44.4% (4/9) | 100.0% (9/9) | 44.4% (4/9) |
|
||||
| duplicate-line-flip | duplicate | 9 | 100.0% (9/9) | 100.0% (9/9) | 100.0% (9/9) |
|
||||
| flip-boolean | literal | 9 | 44.4% (4/9) | 100.0% (9/9) | 44.4% (4/9) |
|
||||
| identifier-multi-edit | identifier | 9 | 77.8% (7/9) | 100.0% (9/9) | 77.8% (7/9) |
|
||||
| off-by-one | literal | 9 | 44.4% (4/9) | 100.0% (9/9) | 44.4% (4/9) |
|
||||
| remove-early-return | structural | 9 | 22.2% (2/9) | 100.0% (9/9) | 22.2% (2/9) |
|
||||
| remove-negation | operator | 9 | 44.4% (4/9) | 88.9% (8/9) | 44.4% (4/9) |
|
||||
| remove-optional-chain | access | 9 | 33.3% (3/9) | 100.0% (9/9) | 33.3% (3/9) |
|
||||
| swap-adjacent-lines | structural | 9 | 33.3% (3/9) | 100.0% (9/9) | 33.3% (3/9) |
|
||||
| swap-arithmetic | operator | 9 | 55.6% (5/9) | 100.0% (9/9) | 55.6% (5/9) |
|
||||
| swap-call-args | call | 9 | 44.4% (4/9) | 100.0% (9/9) | 44.4% (4/9) |
|
||||
| swap-comparison | operator | 9 | 44.4% (4/9) | 100.0% (9/9) | 44.4% (4/9) |
|
||||
| swap-equality | operator | 9 | 88.9% (8/9) | 100.0% (9/9) | 88.9% (8/9) |
|
||||
| swap-if-else | structural | 9 | 33.3% (3/9) | 100.0% (9/9) | 33.3% (3/9) |
|
||||
| swap-increment-decrement | operator | 9 | 88.9% (8/9) | 88.9% (8/9) | 88.9% (8/9) |
|
||||
| swap-logical | operator | 9 | 55.6% (5/9) | 100.0% (9/9) | 55.6% (5/9) |
|
||||
| swap-named-imports | import | 9 | 55.6% (5/9) | 100.0% (9/9) | 55.6% (5/9) |
|
||||
| swap-nullish | operator | 9 | 55.6% (5/9) | 100.0% (9/9) | 55.6% (5/9) |
|
||||
| swap-regex-quantifier | regex | 9 | 66.7% (6/9) | 88.9% (8/9) | 66.7% (6/9) |
|
||||
| unicode-hyphen | unicode | 9 | 88.9% (8/9) | 100.0% (9/9) | 88.9% (8/9) |
|
||||
|
||||
## Difficulty Summary
|
||||
|
||||
| Difficulty Score | Runs | Verified | Edit Used | Success |
|
||||
|------------------|------|----------|-----------|---------|
|
||||
| 0-2 | 18 | 66.7% (12/18) | 94.4% (17/18) | 66.7% (12/18) |
|
||||
| 3-5 | 36 | 52.8% (19/36) | 97.2% (35/36) | 52.8% (19/36) |
|
||||
| 6-8 | 72 | 61.1% (44/72) | 98.6% (71/72) | 61.1% (44/72) |
|
||||
| 9+ | 54 | 48.1% (26/54) | 100.0% (54/54) | 48.1% (26/54) |
|
||||
|
||||
## Edit Tool Errors
|
||||
|
||||
Failures where the edit tool returned an error or failed to apply the patch.
|
||||
|
||||
### Literal Flip Boolean 003 (ReactFlightDOMClientEdge.js)
|
||||
|
||||
#### Run 3
|
||||
|
||||
##### Attempt 1
|
||||
|
||||
- Path: ./ReactFlightDOMClientEdge.js
|
||||
- Operation: —
|
||||
|
||||
**Tool error**
|
||||
|
||||
```
|
||||
Diff contains no hunks
|
||||
```
|
||||
|
||||
**Patch args**
|
||||
|
||||
```diff
|
||||
@@ __DEV__ && options ? options.replayConsoleLogs === false : false, // defaults to false
|
||||
```
|
||||
|
||||
### Operator Swap Equality 002 (editor.js)
|
||||
|
||||
#### Run 3
|
||||
|
||||
##### Attempt 1
|
||||
|
||||
- Path: editor.js
|
||||
- Operation: —
|
||||
|
||||
**Tool error**
|
||||
|
||||
```
|
||||
Failed to find expected lines in editor.js:
|
||||
return [COMMON_EDITORS[processName]];
|
||||
}
|
||||
```
|
||||
```
|
||||
|
||||
**Patch args**
|
||||
|
||||
```diff
|
||||
@@ if (output.indexOf(processName) !== -1) {
|
||||
return [COMMON_EDITORS[processName]];
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
### Structural Swap Adjacent Lines 003 (backend.js)
|
||||
|
||||
#### Run 2
|
||||
|
||||
##### Attempt 1
|
||||
|
||||
- Path: backend.js
|
||||
- Operation: —
|
||||
|
||||
**Tool error**
|
||||
|
||||
```
|
||||
Found 2 matches for context 'const agent = new Agent(bridge, isProfiling, onReloadAndProfile);' in backend.js. Add more surrounding context or additional @@ anchors to make it unique.
|
||||
```
|
||||
|
||||
**Patch args**
|
||||
|
||||
```diff
|
||||
@@ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
if (onSettingsUpdated != null) {
|
||||
agent.addListener('updateHookSettings', onSettingsUpdated);
|
||||
}
|
||||
agent.addListener('shutdown', () => {
|
||||
if (onSettingsUpdated != null) {
|
||||
agent.removeListener('updateHookSettings', onSettingsUpdated);
|
||||
}
|
||||
|
||||
// If we received 'shutdown' from `agent`, we assume the `bridge` is already shutting down,
|
||||
// and that caused the 'shutdown' event on the `agent`, so we don't need to call `bridge.shutdown()` here.
|
||||
hook.emit('shutdown');
|
||||
});
|
||||
-
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
- if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
- onReloadAndProfileFlagsReset();
|
||||
- } @@
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
+ if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
+ onReloadAndProfileFlagsReset();
|
||||
+ }
|
||||
+
|
||||
if (onSettingsUpdated != null) {
|
||||
agent.addListener('updateHookSettings', onSettingsUpdated);
|
||||
}
|
||||
agent.addListener('shutdown', () => {
|
||||
if (onSettingsUpdated != null) {
|
||||
agent.removeListener('updateHookSettings', onSettingsUpdated);
|
||||
}
|
||||
|
||||
// If we received 'shutdown' from `agent`, we assume the `bridge` is already shutting down,
|
||||
// and that caused the 'shutdown' event on the `agent`, so we don't need to call `bridge.shutdown()` here.
|
||||
hook.emit('shutdown');
|
||||
});
|
||||
|
||||
```
|
||||
|
||||
### Structural Swap If Else 003 (ReactDOMFizzStaticNode.js)
|
||||
|
||||
#### Run 3
|
||||
|
||||
##### Attempt 1
|
||||
|
||||
- Path: ./ReactDOMFizzStaticNode.js
|
||||
- Operation: —
|
||||
|
||||
**Tool error**
|
||||
|
||||
```
|
||||
Found 4 matches for context 'if (options && options.signal) {' in ./ReactDOMFizzStaticNode.js. Add more surrounding context or additional @@ anchors to make it unique.
|
||||
```
|
||||
|
||||
**Patch args**
|
||||
|
||||
```diff
|
||||
@@ if (options && options.signal) {
|
||||
- if (options && options.signal) {
|
||||
+ if (options && options.signal) {
|
||||
+ const signal = options.signal;
|
||||
+ if (signal.aborted) {
|
||||
+ abort(request, (signal: any).reason);
|
||||
+ } else {
|
||||
const listener = () => {
|
||||
abort(request, (signal: any).reason);
|
||||
signal.removeEventListener('abort', listener);
|
||||
};
|
||||
signal.addEventListener('abort', listener);
|
||||
}
|
||||
- } else {
|
||||
- const signal = options.signal;
|
||||
- if (signal.aborted) {
|
||||
- abort(request, (signal: any).reason);
|
||||
- }
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Flaky Tasks (partial passing)
|
||||
|
||||
### Call Swap Call Args 001 (testHelpers.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 29 / 719 | 14.0s |
|
||||
| 2 | ✅ | — | 19 / 625 | 10.2s |
|
||||
| 3 | ❌ | File mismatch for testHelpers.js | 33 / 968 | 16.4s |
|
||||
|
||||
### Call Swap Call Args 002 (FlamegraphChartBuilder.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for FlamegraphChartBuilder.js | 31 / 1,850 | 23.0s |
|
||||
| 2 | ❌ | File mismatch for FlamegraphChartBuilder.js | 15 / 1,082 | 12.4s |
|
||||
| 3 | ✅ | — | 19 / 1,044 | 16.0s |
|
||||
|
||||
### Call Swap Call Args 003 (SyntheticEvent.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for SyntheticEvent.js | 45 / 2,399 | 27.2s |
|
||||
| 2 | ✅ | — | 19 / 835 | 11.8s |
|
||||
| 3 | ❌ | File mismatch for SyntheticEvent.js | 15 / 933 | 10.5s |
|
||||
|
||||
### Identifier Identifier Multi Edit 001 (TabBar.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 33 / 848 | 13.9s |
|
||||
| 2 | ✅ | — | 65 / 1,700 | 27.2s |
|
||||
| 3 | ❌ | File mismatch for TabBar.js | 13 / 978 | 11.7s |
|
||||
|
||||
### Identifier Identifier Multi Edit 003 (ReactPerformanceTrackProperties.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactPerformanceTrackProperti... | 20 / 562 | 10.5s |
|
||||
| 2 | ✅ | — | 122 / 3,600 | 51.2s |
|
||||
| 3 | ✅ | — | 64 / 1,771 | 27.5s |
|
||||
|
||||
### Import Swap Named Imports 001 (CommitFlamegraphListItem.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 449 | 7.9s |
|
||||
| 2 | ✅ | — | 15 / 725 | 10.6s |
|
||||
| 3 | ❌ | File mismatch for CommitFlamegraphListItem.js | 14 / 640 | 9.2s |
|
||||
|
||||
### Import Swap Named Imports 002 (ReactDOMTextarea.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 12 / 408 | 9.6s |
|
||||
| 2 | ❌ | File mismatch for ReactDOMTextarea.js | 13 / 380 | 6.3s |
|
||||
| 3 | ❌ | File mismatch for ReactDOMTextarea.js | 13 / 385 | 6.9s |
|
||||
|
||||
### Import Swap Named Imports 003 (StyleEditor.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 660 | 9.0s |
|
||||
| 2 | ✅ | — | 15 / 3,705 | 30.1s |
|
||||
| 3 | ❌ | File mismatch for StyleEditor.js | 24 / 1,114 | 17.4s |
|
||||
|
||||
### Literal Flip Boolean 003 (ReactFlightDOMClientEdge.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactFlightDOMClientEdge.js | 13 / 399 | 7.0s |
|
||||
| 2 | ❌ | File mismatch for ReactFlightDOMClientEdge.js | 15 / 577 | 9.9s |
|
||||
| 3 | ✅ | — | 30 / 1,388 | 20.1s |
|
||||
|
||||
### Literal Off By One 001 (githubAPI.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 468 | 7.4s |
|
||||
| 2 | ❌ | File mismatch for githubAPI.js | 15 / 358 | 6.6s |
|
||||
| 3 | ✅ | — | 13 / 501 | 7.6s |
|
||||
|
||||
### Literal Off By One 002 (code-path.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 491 | 7.6s |
|
||||
| 2 | ❌ | File mismatch for code-path.js | 75 / 2,070 | 33.2s |
|
||||
| 3 | ❌ | File mismatch for code-path.js | 15 / 478 | 7.9s |
|
||||
|
||||
### Literal Off By One 003 (InspectedElement.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for InspectedElement.js | 13 / 462 | 8.5s |
|
||||
| 2 | ✅ | — | 15 / 667 | 10.6s |
|
||||
| 3 | ❌ | File mismatch for InspectedElement.js | 19 / 613 | 10.9s |
|
||||
|
||||
### Operator Remove Negation 001 (ReactDOMClient.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 13 / 1,078 | 12.8s |
|
||||
| 2 | ❌ | File mismatch for ReactDOMClient.js | 36 / 1,000 | 19.6s |
|
||||
| 3 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.8s |
|
||||
|
||||
### Operator Swap Arithmetic 001 (fallbackEvalContext.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 311 | 6.1s |
|
||||
| 2 | ❌ | File mismatch for fallbackEvalContext.js | 12 / 15,299 | 4.8s |
|
||||
| 3 | ❌ | File mismatch for fallbackEvalContext.js | 12 / 344 | 6.4s |
|
||||
|
||||
### Operator Swap Arithmetic 003 (hooks.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for hooks.js | 15 / 1,855 | 24.5s |
|
||||
| 2 | ❌ | File mismatch for hooks.js | 15 / 1,036 | 13.0s |
|
||||
| 3 | ✅ | — | 15 / 350 | 7.9s |
|
||||
|
||||
### Operator Swap Comparison 003 (ReactFlightDOMServerNode.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 37 / 1,082 | 16.8s |
|
||||
| 2 | ❌ | File mismatch for ReactFlightDOMServerNode.js | 13 / 510 | 8.7s |
|
||||
| 3 | ❌ | File mismatch for ReactFlightDOMServerNode.js | 15 / 680 | 9.5s |
|
||||
|
||||
### Operator Swap Equality 002 (editor.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 421 | 6.5s |
|
||||
| 2 | ❌ | File mismatch for editor.js | 13 / 400 | 7.1s |
|
||||
| 3 | ✅ | — | 22 / 696 | 10.8s |
|
||||
|
||||
### Operator Swap Increment Decrement 002 (ReactFlightDOMClientNode.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 19 / 579 | 15.1s |
|
||||
| 2 | ❌ | File mismatch for ReactFlightDOMClientNode.js | 17 / 900 | 11.4s |
|
||||
| 3 | ✅ | — | 34 / 1,004 | 15.6s |
|
||||
|
||||
### Operator Swap Logical 001 (profiling.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 63 / 2,530 | 34.0s |
|
||||
| 2 | ✅ | — | 27 / 974 | 11.0s |
|
||||
| 3 | ❌ | File mismatch for profiling.js | 27 / 716 | 9.8s |
|
||||
|
||||
### Operator Swap Nullish 002 (EnterLeaveEventPlugin.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 51 / 1,515 | 23.5s |
|
||||
| 2 | ❌ | File mismatch for EnterLeaveEventPlugin.js | 13 / 3,244 | 28.3s |
|
||||
| 3 | ❌ | File mismatch for EnterLeaveEventPlugin.js | 38 / 1,054 | 15.6s |
|
||||
|
||||
### Operator Swap Nullish 003 (backend.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for backend.js | 15 / 526 | 7.5s |
|
||||
| 2 | ✅ | — | 15 / 733 | 10.2s |
|
||||
| 3 | ❌ | File mismatch for backend.js | 15 / 836 | 14.8s |
|
||||
|
||||
### Regex Swap Regex Quantifier 002 (ReactFlightStackConfigV8.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactFlightStackConfigV8.js | 82 / 3,238 | 43.9s |
|
||||
| 2 | ✅ | — | 15 / 878 | 11.1s |
|
||||
| 3 | ❌ | File mismatch for ReactFlightStackConfigV8.js | 20 / 1,589 | 19.7s |
|
||||
|
||||
### Regex Swap Regex Quantifier 003 (utils.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.8s |
|
||||
| 2 | ✅ | — | 15 / 458 | 7.9s |
|
||||
| 3 | ✅ | — | 30 / 1,343 | 18.4s |
|
||||
|
||||
### Structural Delete Statement 003 (simulateBrowserEventDispatch.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for simulateBrowserEventDispatch.js | 136 / 8,950 | 103.1s |
|
||||
| 2 | ✅ | — | 15 / 474 | 8.7s |
|
||||
| 3 | ❌ | File mismatch for simulateBrowserEventDispatch.js | 15 / 531 | 10.5s |
|
||||
|
||||
### Structural Remove Early Return 002 (useCommitFilteringAndNavigation.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for useCommitFilteringAndNavigati... | 24 / 699 | 15.6s |
|
||||
| 2 | ✅ | — | 18 / 689 | 12.6s |
|
||||
| 3 | ✅ | — | 24 / 822 | 16.2s |
|
||||
|
||||
### Structural Swap If Else 001 (importFile.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for importFile.js | 21 / 575 | 11.3s |
|
||||
| 2 | ✅ | — | 21 / 522 | 10.6s |
|
||||
| 3 | ❌ | File mismatch for importFile.js | 21 / 650 | 10.5s |
|
||||
|
||||
### Structural Swap If Else 003 (ReactDOMFizzStaticNode.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 533 | 7.6s |
|
||||
| 2 | ✅ | — | 24 / 838 | 14.4s |
|
||||
| 3 | ❌ | File mismatch for ReactDOMFizzStaticNode.js | 37 / 1,364 | 19.2s |
|
||||
|
||||
### Unicode Unicode Hyphen 002 (UnsupportedBridgeProtocolDialog.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 15 / 421 | 8.0s |
|
||||
| 2 | ✅ | — | 15 / 466 | 7.7s |
|
||||
| 3 | ❌ | File mismatch for UnsupportedBridgeProtocolDial... | 15 / 367 | 6.6s |
|
||||
|
||||
## Failed Tasks (0% passing)
|
||||
|
||||
### Access Remove Optional Chain 002 (TimelineContext.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for TimelineContext.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Done. I've restored the optional chaining operators (`?.`) at line 87 in the `getState` function within `inMemoryTimelineData`. The property access now safely chains through `store.profilerStore?.profilingData?.timelineData` instead of the unsafe direct access that was there.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -80 @@
|
||||
};
|
||||
},
|
||||
function getState() {
|
||||
- return store.profilerStore.profilingData?.timelineData || null;
|
||||
+ return store.profilerStore?.profilingData?.timelineData || null;
|
||||
},
|
||||
);
|
||||
|
||||
```
|
||||
|
||||
### Access Remove Optional Chain 003 (astUtils.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for astUtils.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Perfect! The fix is in place. The optional chaining has been restored on line 420:
|
||||
|
||||
**Before (buggy):**
|
||||
```javascript
|
||||
? path.scope.bindings[hookDeclaredVariableName]?.referencePaths
|
||||
```
|
||||
|
||||
**After (fixed):**
|
||||
```javascript
|
||||
? path.scope.bindings?.[hookDeclaredVariableName]?.referencePaths
|
||||
```
|
||||
|
||||
The issue was that `path.scope.bindings[hookDeclaredVariableName]` could be null/undefined, but optional chaining was only applied to the `.referencePaths` property access. Now it's applied to the computed ...
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -255 @@
|
||||
const nodeType = hook.node.id.type;
|
||||
switch (nodeType) {
|
||||
case AST_NODE_TYPES.ARRAY_PATTERN:
|
||||
- return !isCustomHook ? (hook.node.id.elements[0]?.name ?? null) : null;
|
||||
+ return !isCustomHook ? (hook.node.id.elements[0].name ?? null) : null;
|
||||
|
||||
case AST_NODE_TYPES.IDENTIFIER:
|
||||
return hook.node.id.name;
|
||||
@@ -383 @@
|
||||
// declared variable name as the Hook name.
|
||||
const referencePaths =
|
||||
hookDeclaredVariableName != null
|
||||
- ? path.scope.bindings[hookDeclaredVariableName]?.referencePaths
|
||||
+ ? path.scope.bindings?.[hookDeclaredVariableName]?.referencePaths
|
||||
: null;
|
||||
if (referencePaths == null) {
|
||||
pushFrame(hookDeclaredVariableName, callExpressionNode);
|
||||
```
|
||||
|
||||
### Literal Flip Boolean 002 (ReactNoopFlightServer.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactNoopFlightServer.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Fixed. The boolean literal at line 97 was inverted. The condition `options.debugChannel !== undefined` should be `options.debugChannel === undefined` to properly indicate when the debug channel is NOT available (the intended inverted value).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -37 @@
|
||||
},
|
||||
writeChunkAndReturn(destination: Destination, chunk: string): boolean {
|
||||
destination.push(chunk);
|
||||
- return true;
|
||||
+ return false;
|
||||
},
|
||||
completeWriting(destination: Destination): void {},
|
||||
close(destination: Destination): void {},
|
||||
@@ -83 @@
|
||||
undefined,
|
||||
__DEV__ && options ? options.environmentName : undefined,
|
||||
__DEV__ && options ? options.filterStackFrame : undefined,
|
||||
- __DEV__ && options && options.debugChannel !== undefined,
|
||||
+ __DEV__ && options && options.debugChannel === undefined,
|
||||
);
|
||||
const signal = options ? options.signal : undefined;
|
||||
if (signal) {
|
||||
```
|
||||
|
||||
### Operator Remove Negation 003 (ReactFlightUnbundledReferences.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactFlightUnbundledReferences.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Perfect. The negation operator has been fixed. Changed from `if (!descriptor)` to `if (descriptor)`, so now the logic correctly:
|
||||
- Returns the existing descriptor immediately if it exists
|
||||
- Only creates and defines a new descriptor if one doesn't exist
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -247 @@
|
||||
// Use a cached value
|
||||
return target.then;
|
||||
}
|
||||
- if (!target.$$async) {
|
||||
+ if (target.$$async) {
|
||||
// If this module is expected to return a Promise (such as an AsyncModule) then
|
||||
// we should resolve that with a client reference that unwraps the Promise on
|
||||
// the client.
|
||||
@@ -315 @@
|
||||
},
|
||||
getOwnPropertyDescriptor: function (target: Function, name: string | symbol): $FlowFixMe {
|
||||
let descriptor = Object.getOwnPropertyDescriptor(target, name);
|
||||
- if (!descriptor) {
|
||||
+ if (descriptor) {
|
||||
descriptor = {
|
||||
value: getReference(target, name),
|
||||
writable: false,
|
||||
```
|
||||
|
||||
### Operator Swap Comparison 002 (ReactFlightDOMServerBrowser.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactFlightDOMServerBrowser.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Perfect. The comparison operator has been fixed. Changed from `<=` (less than or equal to) with `messages.length - 1` to `<` (less than) with `messages.length`, which is the correct and idiomatic way to iterate through array elements.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -69 @@
|
||||
? readFinalStringChunk(stringDecoder, new Uint8Array(0))
|
||||
: readPartialStringChunk(stringDecoder, buffer);
|
||||
const messages = stringBuffer.split('\n');
|
||||
- for (let i = 0; i < messages.length - 1; i++) {
|
||||
+ for (let i = 0; i < messages.length; i++) {
|
||||
resolveDebugMessage(request, messages[i]);
|
||||
}
|
||||
stringBuffer = messages[messages.length - 1];
|
||||
```
|
||||
|
||||
### Operator Swap Logical 002 (SourceMapMetadataConsumer.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for SourceMapMetadataConsumer.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Fixed. Changed the first `&&` to `||` on line 50. The condition now correctly checks: "if sourceRoot exists OR (sourceRoot is absolute AND source is absolute)" before making the source relative.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -45 @@
|
||||
// be particularly problematic when the source root is a prefix of the
|
||||
// source (valid, but why??). See github issue #199 and bugzil.la/1188982.
|
||||
source =
|
||||
- sourceRoot != null && util.isAbsolute(sourceRoot) && util.isAbsolute(source)
|
||||
+ sourceRoot != null || (util.isAbsolute(sourceRoot) && util.isAbsolute(source))
|
||||
? util.relative(sourceRoot, source)
|
||||
: source;
|
||||
return util.computeSourceURL(sourceRoot, source);
|
||||
```
|
||||
|
||||
### Structural Delete Statement 002 (getComponentNameFromFiber.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for getComponentNameFromFiber.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Done. Restored the missing statement that defines `functionName` by calling `getComponentNameFromType(innerType)` in the `getWrappedName` function.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -54 @@
|
||||
|
||||
// Keep in sync with shared/getComponentNameFromType
|
||||
function getWrappedName(outerType: mixed, innerType: any, wrapperName: string): string {
|
||||
- const functionName = innerType.displayName || innerType.name || '';
|
||||
+ const functionName = getComponentNameFromType(innerType);
|
||||
return (
|
||||
(outerType: any).displayName ||
|
||||
(functionName !== '' ? `${wrapperName}(${functionName})` : wrapperName)
|
||||
```
|
||||
|
||||
### Structural Remove Early Return 001 (InspectedElementStateTree.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for InspectedElementStateTree.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Perfect. The guard clause has been restored. The fix adds the missing 3-line pattern:
|
||||
- Line 41: `if (!state) {`
|
||||
- Line 42: `return null;`
|
||||
- Line 43: `}`
|
||||
|
||||
This ensures that if `state` is null or undefined, the component returns early before attempting to use it on subsequent lines. The fix is minimal and preserves all existing code structure and formatting.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -36 @@
|
||||
store,
|
||||
}: Props): React.Node {
|
||||
const { state, type } = inspectedElement;
|
||||
- if (state == null) {
|
||||
+
|
||||
+ if (!state) {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
-
|
||||
// HostSingleton and HostHoistable may have state that we don't want to expose to users
|
||||
const isHostComponent = type === ElementTypeHostComponent;
|
||||
const entries = Object.entries(state);
|
||||
```
|
||||
|
||||
### Structural Remove Early Return 003 (ReactFiberAsyncAction.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactFiberAsyncAction.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Done. Added the missing guard clause to return early if `isomorphicDefaultTransitionIndicator === null`. This prevents attempting to call `null` as a function on line 193.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -176 @@
|
||||
export function registerDefaultIndicator(
|
||||
onDefaultTransitionIndicator: () => void | (() => void),
|
||||
): void {
|
||||
- if (!enableDefaultTransitionIndicator) {
|
||||
- return;
|
||||
- }
|
||||
if (isomorphicDefaultTransitionIndicator === undefined) {
|
||||
isomorphicDefaultTransitionIndicator = onDefaultTransitionIndicator;
|
||||
} else if (isomorphicDefaultTransitionIndicator !== onDefaultTransitionIndicator) {
|
||||
@@ -195 @@
|
||||
if (!needsIsomorphicIndicator) {
|
||||
return;
|
||||
}
|
||||
+ if (isomorphicDefaultTransitionIndicator === null) {
|
||||
+ return;
|
||||
+ }
|
||||
if (isomorphicDefaultTransitionIndicator != null && pendingIsomorphicIndicator === null) {
|
||||
try {
|
||||
pendingIsomorphicIndicator = isomorphicDefaultTransitionIndicator() || noop;
|
||||
```
|
||||
|
||||
### Structural Swap Adjacent Lines 002 (ReactNoopFlightServer.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactNoopFlightServer.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Perfect. The two adjacent lines have been swapped. The `return textEncoder.encode(content);` statement now comes before the `flushBuffered` method definition, which restores the correct order. The `stringToChunk` method is now properly closed, and `flushBuffered` follows as a separate method definition.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -14 @@
|
||||
* environment.
|
||||
*/
|
||||
|
||||
-import type { ReactClientValue } from 'react-server/src/ReactFlightServer';
|
||||
+import type {ReactClientValue} from 'react-server/src/ReactFlightServer';
|
||||
|
||||
|
||||
-import { saveModule } from 'react-noop-renderer/flight-modules';
|
||||
+import {saveModule} from 'react-noop-renderer/flight-modules';
|
||||
|
||||
import ReactFlightServer from 'react-server/flight';
|
||||
|
||||
@@ -42 @@
|
||||
completeWriting(destination: Destination): void {},
|
||||
close(destination: Destination): void {},
|
||||
closeWithError(destination: Destination, error: mixed): void {},
|
||||
- flushBuffered(destination: Destination): void {},
|
||||
stringToChunk(content: string): Uint8Array {
|
||||
return textEncoder.encode(content);
|
||||
stringToChunk(content: string): Uint8Array {
|
||||
return textEncoder.encode(content);
|
||||
+ flushBuffered(destination: Destination): void {},
|
||||
},
|
||||
stringToPrecomputedChunk(content: string): Uint8Array {
|
||||
return textEncoder.encode(content);
|
||||
@@ -58 @@
|
||||
getClientReferenceKey(reference: Object): Object {
|
||||
return reference;
|
||||
},
|
||||
- resolveClientReferenceMetadata(config: void, reference: { $$typeof: symbol, value: any }) {
|
||||
+ resolveClientReferenceMetadata(
|
||||
+ config: void,
|
||||
+ reference: {$$typeof: symbol, value: any},
|
||||
+ ) {
|
||||
return saveModule(reference.value);
|
||||
},
|
||||
});
|
||||
@@ -68 @@
|
||||
filterStackFrame?: (url: string, functionName: string) => boolean,
|
||||
identifierPrefix?: string,
|
||||
signal?: AbortSignal,
|
||||
- debugChannel?: { onMessage?: (message: string) => void },
|
||||
+ debugChannel?: {onMessage?: (message: string) => void},
|
||||
onError?: (error: mixed) => void,
|
||||
};
|
||||
|
||||
@@ -98 @@
|
||||
}
|
||||
}
|
||||
if (__DEV__ && options && options.debugChannel !== undefined) {
|
||||
- options.debugChannel.onMessage = (message) => {
|
||||
+ options.debugChannel.onMessage = message => {
|
||||
ReactNoopFlightServer.resolveDebugMessage(request, message);
|
||||
};
|
||||
}
|
||||
@@ -107 @@
|
||||
return destination;
|
||||
}
|
||||
|
||||
-export { render };
|
||||
+export {render};
|
||||
```
|
||||
|
||||
### Structural Swap Adjacent Lines 003 (backend.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for backend.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
I see the issue - my previous edit created a duplicate. Let me undo this and do it correctly. I need to reverse the order of the two statements. Let me read the exact current state:
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -9 @@
|
||||
|
||||
import Agent from 'react-devtools-shared/src/backend/agent';
|
||||
import Bridge from 'react-devtools-shared/src/bridge';
|
||||
-import { installHook } from 'react-devtools-shared/src/hook';
|
||||
-import { initBackend } from 'react-devtools-shared/src/backend';
|
||||
-import { __DEBUG__ } from 'react-devtools-shared/src/constants';
|
||||
+import {installHook} from 'react-devtools-shared/src/hook';
|
||||
+import {initBackend} from 'react-devtools-shared/src/backend';
|
||||
+import {__DEBUG__} from 'react-devtools-shared/src/constants';
|
||||
import setupNativeStyleEditor from 'react-devtools-shared/src/backend/NativeStyleEditor/setupNativeStyleEditor';
|
||||
import {
|
||||
getDefaultComponentFilters,
|
||||
@@ -18 @@
|
||||
getIsReloadAndProfileSupported,
|
||||
} from 'react-devtools-shared/src/utils';
|
||||
|
||||
-import type { BackendBridge } from 'react-devtools-shared/src/bridge';
|
||||
-import type { ComponentFilter, Wall } from 'react-devtools-shared/src/frontend/types';
|
||||
+import type {BackendBridge} from 'react-devtools-shared/src/bridge';
|
||||
import type {
|
||||
import type {
|
||||
+ ComponentFilter,
|
||||
+ Wall,
|
||||
+} from 'react-devtools-shared/src/frontend/types';
|
||||
+import type {
|
||||
DevToolsHook,
|
||||
DevToolsHookSettings,
|
||||
ProfilingSettings,
|
||||
@@ -25 @@
|
||||
DevToolsHookSettings,
|
||||
ProfilingSettings,
|
||||
} from 'react-devtools-shared/src/backend/types';
|
||||
-import type { ResolveNativeStyle } from 'react-devtools-shared/src/backend/NativeStyleEditor/setupNativeStyleEditor';
|
||||
+import type {ResolveNativeStyle} from 'react-devtools-shared/src/backend/NativeStyleEditor/setupNativeStyleEditor';
|
||||
|
||||
type ConnectOptions = {
|
||||
host?: string,
|
||||
@@ -43 @@
|
||||
onReloadAndProfileFlagsReset?: () => void,
|
||||
};
|
||||
|
||||
-let savedComponentFilters: Array<ComponentFilter> = getDefaultComponentFilters();
|
||||
+let savedComponentFilters: Array<ComponentFilter> =
|
||||
+ getDefaultComponentFilters();
|
||||
|
||||
function debug(methodName: string, ...args: Array<mixed>) {
|
||||
if (__DEBUG__) {
|
||||
@@ -57 @@
|
||||
}
|
||||
|
||||
export function initialize(
|
||||
- maybeSettingsOrSettingsPromise?: DevToolsHookSettings | Promise<DevTo...
|
||||
```
|
||||
|
||||
### Structural Swap If Else 002 (ReactNativeFiberInspector.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactNativeFiberInspector.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Fixed. Negated the condition from `if (__DEV__)` to `if (!__DEV__)` so that the error is thrown in production (where it should be unavailable) and the inspector data is returned in development mode.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -145 @@
|
||||
}
|
||||
|
||||
function getInspectorDataForViewTag(viewTag: number): InspectorData {
|
||||
- if (__DEV__) {
|
||||
+ if (!__DEV__) {
|
||||
+ throw new Error('getInspectorDataForViewTag() is not available in production');
|
||||
+ } else {
|
||||
const closestInstance = getClosestInstanceFromNode(viewTag);
|
||||
return getInspectorDataForInstance(closestInstance);
|
||||
const closestInstance = getClosestInstanceFromNode(viewTag);
|
||||
return getInspectorDataForInstance(closestInstance);
|
||||
- } else {
|
||||
- throw new Error('getInspectorDataForViewTag() is not available in production');
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,919 @@
|
||||
# Edit Benchmark Report
|
||||
|
||||
## Configuration
|
||||
|
||||
| Setting | Value |
|
||||
|---------|-------|
|
||||
| Date | 2026-01-19T13:44:45.661Z |
|
||||
| Model | anthropic/openai-codex/gpt-5.1-codex-mini |
|
||||
| Thinking Level | default |
|
||||
| Runs per task | 3 |
|
||||
| Edit Variant | patch |
|
||||
| Edit Fuzzy | auto |
|
||||
| Edit Fuzzy Threshold | auto |
|
||||
| Require Edit Tool | no |
|
||||
| No-Edit Baseline | no |
|
||||
|
||||
## Summary
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Total Tasks | 60 |
|
||||
| Total Runs | 180 |
|
||||
| Successful Runs | 96 |
|
||||
| **Task Success Rate** | **53.3% (96/180)** |
|
||||
| Verified Rate | 53.3% (96/180) |
|
||||
| Edit Tool Usage Rate | 67.2% (121/180) |
|
||||
| **Edit Success Rate** | **98.6%** |
|
||||
| Patch Failure Rate | 1.4% (2/142) |
|
||||
| Tasks All Passing | 14 |
|
||||
| Tasks Flaky/Failing | 46 |
|
||||
|
||||
### Tool Calls
|
||||
|
||||
| Tool | Total | Avg/Run |
|
||||
|------|-------|---------|
|
||||
| Read | 272 | 1.5 |
|
||||
| Edit | 142 | 0.8 |
|
||||
| Write | 0 | 0.0 |
|
||||
| **Tool Input Chars** | 60,288 | 335 |
|
||||
|
||||
### Tokens & Time
|
||||
|
||||
| Metric | Total | Avg/Run |
|
||||
|--------|-------|---------|
|
||||
| Input Tokens | 2,109,510 | 11,720 |
|
||||
| Output Tokens | 385,104 | 2,139 |
|
||||
| Total Tokens | 8,913,686 | 49,520 |
|
||||
| Duration | 5912.0s | 32.8s |
|
||||
| **Avg Indent Score** | — | **2.28** |
|
||||
|
||||
## Task Results
|
||||
|
||||
| Task | File | Success | Edit Hit | R/E/W | Tokens (In/Out) | Time | Indent |
|
||||
|------|------|---------|----------|-------|-----------------|------|--------|
|
||||
| Access Remove Optional Chain 001 | registerDevToolsEventLogger.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 7,881/1,102 | 50.7s | 1.00 |
|
||||
| Access Remove Optional Chain 002 | TimelineContext.js | 3/3 ✅ | 100.0% | 4/2/0 | 32,303/5,595 | 49.6s | 1.29 |
|
||||
| Access Remove Optional Chain 003 | astUtils.js | 0/3 ❌ | 100.0% | 3/0/0 | 34,617/5,112 | 38.7s | 4.87 |
|
||||
| Call Swap Call Args 001 | testHelpers.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 12,362/1,878 | 22.7s | 1.33 |
|
||||
| Call Swap Call Args 002 | FlamegraphChartBuilder.js | 1/3 ⚠️ | 100.0% | 4/1/0 | 25,888/3,771 | 35.0s | 3.79 |
|
||||
| Call Swap Call Args 003 | SyntheticEvent.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 10,545/862 | 48.4s | 3.76 |
|
||||
| Duplicate Duplicate Line Flip 001 | index.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 6,779/583 | 7.4s | 0.00 |
|
||||
| Duplicate Duplicate Line Flip 002 | ActivityList.js | 3/3 ✅ | 100.0% | 2/2/0 | 11,594/987 | 11.5s | 3.61 |
|
||||
| Duplicate Duplicate Line Flip 003 | SyntheticEvent.js | 0/3 ❌ | 100.0% | 0/0/0 | 926/1,601 | 92.0s | 1.02 |
|
||||
| Identifier Identifier Multi Edit 001 | TabBar.js | 1/3 ⚠️ | 100.0% | 4/2/0 | 10,937/2,373 | 25.9s | 3.33 |
|
||||
| Identifier Identifier Multi Edit 002 | EventPluginRegistry.js | 1/3 ⚠️ | 100.0% | 3/1/0 | 13,257/1,718 | 61.7s | 3.94 |
|
||||
| Identifier Identifier Multi Edit 003 | ReactPerformanceTrackProperties.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 16,552/1,505 | 15.6s | 9.95 |
|
||||
| Import Swap Named Imports 001 | CommitFlamegraphListItem.js | 3/3 ✅ | 100.0% | 2/1/0 | 9,074/1,246 | 15.8s | 2.86 |
|
||||
| Import Swap Named Imports 002 | ReactDOMTextarea.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 14,842/1,604 | 19.2s | 2.41 |
|
||||
| Import Swap Named Imports 003 | StyleEditor.js | 0/3 ❌ | 100.0% | 1/1/0 | 9,254/4,430 | 68.7s | 1.31 |
|
||||
| Literal Flip Boolean 001 | testHelpers.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 7,384/432 | 8.6s | 0.89 |
|
||||
| Literal Flip Boolean 002 | ReactNoopFlightServer.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 6,287/1,809 | 18.1s | 1.11 |
|
||||
| Literal Flip Boolean 003 | ReactFlightDOMClientEdge.js | 0/3 ❌ | 100.0% | 1/0/0 | 3,972/4,440 | 9.0s | 3.58 |
|
||||
| Literal Off By One 001 | githubAPI.js | 3/3 ✅ | 100.0% | 1/1/0 | 12,865/206 | 8.5s | 0.67 |
|
||||
| Literal Off By One 002 | code-path.js | 3/3 ✅ | 100.0% | 2/1/0 | 28,443/6,444 | 49.8s | 3.50 |
|
||||
| Literal Off By One 003 | InspectedElement.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 13,076/2,061 | 21.3s | 3.60 |
|
||||
| Operator Remove Negation 001 | ReactDOMClient.js | 2/3 ⚠️ | 100.0% | 3/1/0 | 21,759/5,975 | 90.0s | 1.08 |
|
||||
| Operator Remove Negation 002 | NativeEventsView.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 20,692/7,036 | 87.8s | 3.03 |
|
||||
| Operator Remove Negation 003 | ReactFlightUnbundledReferences.js | 0/3 ❌ | 100.0% | 0/0/0 | 0/0 | 120.0s | 0.00 |
|
||||
| Operator Swap Arithmetic 001 | fallbackEvalContext.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 2,171/293 | 46.6s | 0.00 |
|
||||
| Operator Swap Arithmetic 002 | CSSShorthandProperty.js | 1/3 ⚠️ | 100.0% | 3/1/0 | 15,150/3,112 | 27.5s | 2.88 |
|
||||
| Operator Swap Arithmetic 003 | hooks.js | 0/3 ❌ | 100.0% | 2/0/0 | 11,289/4,322 | 71.7s | 2.25 |
|
||||
| Operator Swap Comparison 001 | index.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 4,890/340 | 7.0s | 0.00 |
|
||||
| Operator Swap Comparison 002 | ReactFlightDOMServerBrowser.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 4,042/2,543 | 11.6s | 1.57 |
|
||||
| Operator Swap Comparison 003 | ReactFlightDOMServerNode.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 18,337/1,471 | 16.4s | 1.95 |
|
||||
| Operator Swap Equality 001 | readInputData.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 13,469/501 | 9.9s | 0.00 |
|
||||
| Operator Swap Equality 002 | editor.js | 3/3 ✅ | 100.0% | 2/1/0 | 20,983/1,152 | 14.3s | 0.00 |
|
||||
| Operator Swap Equality 003 | hooks.js | 3/3 ✅ | 100.0% | 1/1/0 | 12,057/556 | 9.4s | 2.25 |
|
||||
| Operator Swap Increment Decrement 001 | ReactFlightDOMClientNode.js | 3/3 ✅ | 100.0% | 1/1/0 | 8,681/290 | 6.1s | 1.52 |
|
||||
| Operator Swap Increment Decrement 002 | ReactFlightDOMClientNode.js | 3/3 ✅ | 100.0% | 1/1/0 | 6,212/284 | 7.8s | 1.92 |
|
||||
| Operator Swap Increment Decrement 003 | loadSourceAndMetadata.js | 3/3 ✅ | 100.0% | 2/1/0 | 30,476/697 | 12.5s | 3.72 |
|
||||
| Operator Swap Logical 001 | profiling.js | 3/3 ✅ | 100.0% | 1/1/0 | 5,962/436 | 13.3s | 0.00 |
|
||||
| Operator Swap Logical 002 | SourceMapMetadataConsumer.js | 1/3 ⚠️ | 100.0% | 1/0/0 | 3,294/1,651 | 13.1s | 3.07 |
|
||||
| Operator Swap Logical 003 | DevToolsFiberComponentStack.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 6,757/5,239 | 28.4s | 4.13 |
|
||||
| Operator Swap Nullish 001 | getBatchRange.js | 3/3 ✅ | 100.0% | 1/1/0 | 7,028/402 | 8.0s | 1.33 |
|
||||
| Operator Swap Nullish 002 | EnterLeaveEventPlugin.js | 1/3 ⚠️ | 100.0% | 1/0/0 | 3,257/822 | 10.3s | 1.56 |
|
||||
| Operator Swap Nullish 003 | backend.js | 0/3 ❌ | 100.0% | 5/1/0 | 20,811/5,938 | 53.4s | 3.15 |
|
||||
| Regex Swap Regex Quantifier 001 | githubAPI.js | 3/3 ✅ | 100.0% | 1/1/0 | 12,987/1,378 | 14.5s | 0.67 |
|
||||
| Regex Swap Regex Quantifier 002 | ReactFlightStackConfigV8.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 17,922/1,857 | 20.6s | 3.06 |
|
||||
| Regex Swap Regex Quantifier 003 | utils.js | 1/3 ⚠️ | 100.0% | 0/0/0 | 2,297/4,655 | 110.0s | 2.00 |
|
||||
| Structural Delete Statement 001 | UnsupportedVersionDialog.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 21,975/445 | 8.0s | 6.22 |
|
||||
| Structural Delete Statement 002 | getComponentNameFromFiber.js | 0/3 ❌ | 100.0% | 1/1/0 | 18,709/953 | 9.2s | 0.74 |
|
||||
| Structural Delete Statement 003 | simulateBrowserEventDispatch.js | 1/3 ⚠️ | 100.0% | 1/0/0 | 3,582/1,414 | 93.2s | 4.46 |
|
||||
| Structural Remove Early Return 001 | InspectedElementStateTree.js | 0/3 ❌ | 100.0% | 2/1/0 | 10,838/2,575 | 24.0s | 0.41 |
|
||||
| Structural Remove Early Return 002 | useCommitFilteringAndNavigation.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 15,643/3,704 | 29.6s | 3.81 |
|
||||
| Structural Remove Early Return 003 | ReactFiberAsyncAction.js | 0/3 ❌ | 100.0% | 0/0/0 | 5,414/2,433 | 55.9s | 1.46 |
|
||||
| Structural Swap Adjacent Lines 001 | ReactServerConsoleConfigPlain.js | 1/3 ⚠️ | 100.0% | 1/1/0 | 4,366/1,978 | 17.6s | 0.33 |
|
||||
| Structural Swap Adjacent Lines 002 | ReactNoopFlightServer.js | 3/3 ✅ | 100.0% | 1/1/0 | 6,983/2,050 | 18.0s | 1.11 |
|
||||
| Structural Swap Adjacent Lines 003 | backend.js | 0/3 ❌ | 50.0% | 2/1/0 | 15,928/5,067 | 116.0s | 3.15 |
|
||||
| Structural Swap If Else 001 | importFile.js | 0/3 ❌ | 100.0% | 1/1/0 | 2,414/2,282 | 58.0s | 0.00 |
|
||||
| Structural Swap If Else 002 | ReactNativeFiberInspector.js | 0/3 ❌ | 100.0% | 1/1/0 | 7,518/1,443 | 12.5s | 3.18 |
|
||||
| Structural Swap If Else 003 | ReactDOMFizzStaticNode.js | 2/3 ⚠️ | 100.0% | 2/1/0 | 9,895/2,558 | 23.0s | 1.89 |
|
||||
| Unicode Unicode Hyphen 001 | Rectangle.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 4,579/193 | 5.3s | 3.00 |
|
||||
| Unicode Unicode Hyphen 002 | UnsupportedBridgeProtocolDialog.js | 1/3 ⚠️ | 100.0% | 1/0/0 | 6,211/348 | 5.5s | 3.83 |
|
||||
| Unicode Unicode Hyphen 003 | ReactTypes.js | 2/3 ⚠️ | 100.0% | 1/1/0 | 9,753/216 | 6.3s | 1.24 |
|
||||
|
||||
## Category Summary
|
||||
|
||||
| Category | Runs | Verified | Edit Used | Success | Min/Avg/Max Difficulty |
|
||||
|----------|------|----------|-----------|---------|------------------------|
|
||||
| access | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) | 7 / 8.7 / 10 |
|
||||
| call | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) | 6 / 7.7 / 10 |
|
||||
| duplicate | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) | 7 / 9.7 / 12 |
|
||||
| identifier | 9 | 44.4% (4/9) | 55.6% (5/9) | 44.4% (4/9) | 6 / 9.3 / 14 |
|
||||
| import | 9 | 55.6% (5/9) | 88.9% (8/9) | 55.6% (5/9) | 2 / 4.7 / 6 |
|
||||
| literal | 18 | 66.7% (12/18) | 72.2% (13/18) | 66.7% (12/18) | 4 / 6.2 / 9 |
|
||||
| operator | 63 | 60.3% (38/63) | 69.8% (44/63) | 60.3% (38/63) | 1 / 6.5 / 13 |
|
||||
| regex | 9 | 66.7% (6/9) | 66.7% (6/9) | 66.7% (6/9) | 6 / 7.3 / 8 |
|
||||
| structural | 36 | 30.6% (11/36) | 61.1% (22/36) | 30.6% (11/36) | 4 / 7.6 / 15 |
|
||||
| unicode | 9 | 55.6% (5/9) | 55.6% (5/9) | 55.6% (5/9) | 1 / 3.0 / 6 |
|
||||
|
||||
## Mutation Summary
|
||||
|
||||
| Mutation | Category | Runs | Verified | Edit Used | Success |
|
||||
|----------|----------|------|----------|-----------|---------|
|
||||
| delete-statement | structural | 9 | 33.3% (3/9) | 55.6% (5/9) | 33.3% (3/9) |
|
||||
| duplicate-line-flip | duplicate | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) |
|
||||
| flip-boolean | literal | 9 | 44.4% (4/9) | 55.6% (5/9) | 44.4% (4/9) |
|
||||
| identifier-multi-edit | identifier | 9 | 44.4% (4/9) | 55.6% (5/9) | 44.4% (4/9) |
|
||||
| off-by-one | literal | 9 | 88.9% (8/9) | 88.9% (8/9) | 88.9% (8/9) |
|
||||
| remove-early-return | structural | 9 | 22.2% (2/9) | 66.7% (6/9) | 22.2% (2/9) |
|
||||
| remove-negation | operator | 9 | 44.4% (4/9) | 44.4% (4/9) | 44.4% (4/9) |
|
||||
| remove-optional-chain | access | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) |
|
||||
| swap-adjacent-lines | structural | 9 | 44.4% (4/9) | 55.6% (5/9) | 44.4% (4/9) |
|
||||
| swap-arithmetic | operator | 9 | 22.2% (2/9) | 44.4% (4/9) | 22.2% (2/9) |
|
||||
| swap-call-args | call | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) |
|
||||
| swap-comparison | operator | 9 | 55.6% (5/9) | 66.7% (6/9) | 55.6% (5/9) |
|
||||
| swap-equality | operator | 9 | 88.9% (8/9) | 88.9% (8/9) | 88.9% (8/9) |
|
||||
| swap-if-else | structural | 9 | 22.2% (2/9) | 66.7% (6/9) | 22.2% (2/9) |
|
||||
| swap-increment-decrement | operator | 9 | 100.0% (9/9) | 100.0% (9/9) | 100.0% (9/9) |
|
||||
| swap-logical | operator | 9 | 66.7% (6/9) | 66.7% (6/9) | 66.7% (6/9) |
|
||||
| swap-named-imports | import | 9 | 55.6% (5/9) | 88.9% (8/9) | 55.6% (5/9) |
|
||||
| swap-nullish | operator | 9 | 44.4% (4/9) | 77.8% (7/9) | 44.4% (4/9) |
|
||||
| swap-regex-quantifier | regex | 9 | 66.7% (6/9) | 66.7% (6/9) | 66.7% (6/9) |
|
||||
| unicode-hyphen | unicode | 9 | 55.6% (5/9) | 55.6% (5/9) | 55.6% (5/9) |
|
||||
|
||||
## Difficulty Summary
|
||||
|
||||
| Difficulty Score | Runs | Verified | Edit Used | Success |
|
||||
|------------------|------|----------|-----------|---------|
|
||||
| 0-2 | 18 | 61.1% (11/18) | 66.7% (12/18) | 61.1% (11/18) |
|
||||
| 3-5 | 36 | 61.1% (22/36) | 72.2% (26/36) | 61.1% (22/36) |
|
||||
| 6-8 | 72 | 51.4% (37/72) | 68.1% (49/72) | 51.4% (37/72) |
|
||||
| 9+ | 54 | 48.1% (26/54) | 63.0% (34/54) | 48.1% (26/54) |
|
||||
|
||||
## Edit Tool Errors
|
||||
|
||||
Failures where the edit tool returned an error or failed to apply the patch.
|
||||
|
||||
### Structural Swap Adjacent Lines 003 (backend.js)
|
||||
|
||||
#### Run 3
|
||||
|
||||
##### Attempt 1
|
||||
|
||||
- Path: backend.js
|
||||
- Operation: —
|
||||
|
||||
**Tool error**
|
||||
|
||||
```
|
||||
Found 2 occurrences of the text in backend.js. The text must be unique. Please provide more context to make it unique.
|
||||
```
|
||||
|
||||
**Patch args**
|
||||
|
||||
```diff
|
||||
@@
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
- if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
- onReloadAndProfileFlagsReset();
|
||||
- }
|
||||
+ if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
+ onReloadAndProfileFlagsReset();
|
||||
+ }
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
@@
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
- if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
- onReloadAndProfileFlagsReset();
|
||||
- }
|
||||
+ if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
+ onReloadAndProfileFlagsReset();
|
||||
+ }
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
|
||||
```
|
||||
|
||||
##### Attempt 2
|
||||
|
||||
- Path: backend.js
|
||||
- Operation: —
|
||||
|
||||
**Tool error**
|
||||
|
||||
```
|
||||
Found 2 occurrences of the text in backend.js. The text must be unique. Please provide more context to make it unique.
|
||||
```
|
||||
|
||||
**Patch args**
|
||||
|
||||
```diff
|
||||
@@
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
- if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
- onReloadAndProfileFlagsReset();
|
||||
- }
|
||||
+ if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
+ onReloadAndProfileFlagsReset();
|
||||
+ }
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
@@
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
- if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
- onReloadAndProfileFlagsReset();
|
||||
- }
|
||||
+ if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
+ onReloadAndProfileFlagsReset();
|
||||
+ }
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
|
||||
```
|
||||
|
||||
## Flaky Tasks (partial passing)
|
||||
|
||||
### Access Remove Optional Chain 001 (registerDevToolsEventLogger.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 5,605 / 1,567 | 14.6s |
|
||||
| 2 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 3 | ✅ | — | 18,038 / 1,739 | 17.5s |
|
||||
|
||||
### Call Swap Call Args 001 (testHelpers.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 21,086 / 2,913 | 35.2s |
|
||||
| 2 | ❌ | File mismatch for testHelpers.js | 4,654 / 458 | 12.1s |
|
||||
| 3 | ✅ | — | 11,347 / 2,262 | 20.9s |
|
||||
|
||||
### Call Swap Call Args 002 (FlamegraphChartBuilder.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for FlamegraphChartBuilder.js | 1,294 / 409 | 4.4s |
|
||||
| 2 | ❌ | File mismatch for FlamegraphChartBuilder.js | 17,766 / 7,359 | 61.4s |
|
||||
| 3 | ✅ | — | 58,604 / 3,545 | 39.2s |
|
||||
|
||||
### Call Swap Call Args 003 (SyntheticEvent.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 5,659 / 1,296 | 12.9s |
|
||||
| 2 | ✅ | — | 25,977 / 1,289 | 12.2s |
|
||||
| 3 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
|
||||
### Duplicate Duplicate Line Flip 001 (index.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 7,846 / 405 | 6.9s |
|
||||
| 2 | ✅ | — | 6,979 / 1,019 | 10.1s |
|
||||
| 3 | ❌ | File mismatch for index.js | 5,513 / 326 | 5.2s |
|
||||
|
||||
### Identifier Identifier Multi Edit 001 (TabBar.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 4,115 / 4,139 | 36.2s |
|
||||
| 2 | ❌ | File mismatch for TabBar.js | 1,412 / 98 | 3.4s |
|
||||
| 3 | ❌ | File mismatch for TabBar.js | 27,283 / 2,882 | 38.1s |
|
||||
|
||||
### Identifier Identifier Multi Edit 002 (EventPluginRegistry.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 2 | ❌ | File mismatch for EventPluginRegistry.js | 2,515 / 782 | 12.1s |
|
||||
| 3 | ✅ | — | 37,256 / 4,372 | 53.0s |
|
||||
|
||||
### Identifier Identifier Multi Edit 003 (ReactPerformanceTrackProperties.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactPerformanceTrackProperti... | 1,285 / 319 | 6.8s |
|
||||
| 2 | ✅ | — | 22,197 / 2,947 | 24.2s |
|
||||
| 3 | ✅ | — | 26,173 / 1,248 | 15.8s |
|
||||
|
||||
### Import Swap Named Imports 002 (ReactDOMTextarea.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 12,727 / 1,208 | 12.3s |
|
||||
| 2 | ❌ | File mismatch for ReactDOMTextarea.js | 11,629 / 878 | 15.3s |
|
||||
| 3 | ✅ | — | 20,170 / 2,726 | 29.8s |
|
||||
|
||||
### Literal Flip Boolean 001 (testHelpers.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 2,784 / 297 | 4.5s |
|
||||
| 2 | ❌ | File mismatch for testHelpers.js | 16,209 / 614 | 9.4s |
|
||||
| 3 | ✅ | — | 3,160 / 384 | 11.9s |
|
||||
|
||||
### Literal Flip Boolean 002 (ReactNoopFlightServer.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 4,427 / 1,740 | 16.9s |
|
||||
| 2 | ❌ | File mismatch for ReactNoopFlightServer.js | 8,473 / 1,382 | 13.3s |
|
||||
| 3 | ✅ | — | 5,960 / 2,305 | 24.2s |
|
||||
|
||||
### Literal Off By One 003 (InspectedElement.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for InspectedElement.js | 2,640 / 218 | 5.5s |
|
||||
| 2 | ✅ | — | 15,894 / 4,535 | 38.0s |
|
||||
| 3 | ✅ | — | 20,695 / 1,430 | 20.3s |
|
||||
|
||||
### Operator Remove Negation 001 (ReactDOMClient.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 10,255 / 10,311 | 84.1s |
|
||||
| 2 | ✅ | — | 55,023 / 7,614 | 66.0s |
|
||||
| 3 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
|
||||
### Operator Remove Negation 002 (NativeEventsView.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 2 | ✅ | — | 17,148 / 6,876 | 46.6s |
|
||||
| 3 | ✅ | — | 44,929 / 14,232 | 96.7s |
|
||||
|
||||
### Operator Swap Arithmetic 001 (fallbackEvalContext.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 3,503 / 736 | 14.5s |
|
||||
| 2 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 3 | ❌ | File mismatch for fallbackEvalContext.js | 3,011 / 143 | 5.2s |
|
||||
|
||||
### Operator Swap Arithmetic 002 (CSSShorthandProperty.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 9,912 / 3,717 | 30.7s |
|
||||
| 2 | ❌ | File mismatch for CSSShorthandProperty.js | 34,613 / 5,364 | 49.1s |
|
||||
| 3 | ❌ | File mismatch for CSSShorthandProperty.js | 924 / 255 | 2.6s |
|
||||
|
||||
### Operator Swap Comparison 001 (index.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 5,806 / 246 | 5.6s |
|
||||
| 2 | ❌ | File mismatch for index.js | 1,286 / 514 | 9.3s |
|
||||
| 3 | ✅ | — | 7,578 / 259 | 6.0s |
|
||||
|
||||
### Operator Swap Comparison 002 (ReactFlightDOMServerBrowser.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 8,176 / 436 | 8.0s |
|
||||
| 2 | ❌ | File mismatch for ReactFlightDOMServerBrowser.js | 2,484 / 6,789 | 19.9s |
|
||||
| 3 | ❌ | File mismatch for ReactFlightDOMServerBrowser.js | 1,466 / 404 | 7.0s |
|
||||
|
||||
### Operator Swap Comparison 003 (ReactFlightDOMServerNode.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 23,318 / 2,787 | 24.0s |
|
||||
| 2 | ✅ | — | 24,782 / 547 | 10.5s |
|
||||
| 3 | ❌ | File mismatch for ReactFlightDOMServerNode.js | 6,912 / 1,080 | 14.8s |
|
||||
|
||||
### Operator Swap Equality 001 (readInputData.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for readInputData.js | 4,748 / 535 | 6.8s |
|
||||
| 2 | ✅ | — | 5,514 / 401 | 10.0s |
|
||||
| 3 | ✅ | — | 30,144 / 567 | 13.0s |
|
||||
|
||||
### Operator Swap Logical 002 (SourceMapMetadataConsumer.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for SourceMapMetadataConsumer.js | 6,418 / 2,364 | 20.4s |
|
||||
| 2 | ❌ | File mismatch for SourceMapMetadataConsumer.js | 339 / 253 | 2.9s |
|
||||
| 3 | ✅ | — | 3,126 / 2,335 | 15.9s |
|
||||
|
||||
### Operator Swap Logical 003 (DevToolsFiberComponentStack.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 14,087 / 1,836 | 19.8s |
|
||||
| 2 | ❌ | File mismatch for DevToolsFiberComponentStack.js | 141 / 5,909 | 17.7s |
|
||||
| 3 | ✅ | — | 6,042 / 7,973 | 47.7s |
|
||||
|
||||
### Operator Swap Nullish 002 (EnterLeaveEventPlugin.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for EnterLeaveEventPlugin.js | 1,295 / 136 | 2.5s |
|
||||
| 2 | ✅ | — | 2,891 / 1,421 | 15.3s |
|
||||
| 3 | ❌ | File mismatch for EnterLeaveEventPlugin.js | 5,585 / 908 | 13.1s |
|
||||
|
||||
### Regex Swap Regex Quantifier 002 (ReactFlightStackConfigV8.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactFlightStackConfigV8.js | 2,096 / 478 | 9.7s |
|
||||
| 2 | ✅ | — | 23,700 / 3,108 | 30.6s |
|
||||
| 3 | ✅ | — | 27,971 / 1,986 | 21.5s |
|
||||
|
||||
### Regex Swap Regex Quantifier 003 (utils.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 2 | ✅ | — | 6,892 / 13,966 | 90.0s |
|
||||
| 3 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
|
||||
### Structural Delete Statement 001 (UnsupportedVersionDialog.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 47,833 / 397 | 6.9s |
|
||||
| 2 | ❌ | File mismatch for UnsupportedVersionDialog.js | 13,879 / 521 | 6.1s |
|
||||
| 3 | ✅ | — | 4,212 / 417 | 10.8s |
|
||||
|
||||
### Structural Delete Statement 003 (simulateBrowserEventDispatch.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 2 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 3 | ✅ | — | 10,745 / 4,242 | 39.7s |
|
||||
|
||||
### Structural Remove Early Return 002 (useCommitFilteringAndNavigation.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 28,047 / 4,658 | 37.8s |
|
||||
| 2 | ✅ | — | 6,714 / 2,218 | 19.6s |
|
||||
| 3 | ❌ | File mismatch for useCommitFilteringAndNavigati... | 12,167 / 4,237 | 31.5s |
|
||||
|
||||
### Structural Swap Adjacent Lines 001 (ReactServerConsoleConfigPlain.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactServerConsoleConfigPlain.js | 1,865 / 3,382 | 25.7s |
|
||||
| 2 | ❌ | File mismatch for ReactServerConsoleConfigPlain.js | 1,568 / 1,056 | 6.8s |
|
||||
| 3 | ✅ | — | 9,664 / 1,495 | 20.3s |
|
||||
|
||||
### Structural Swap If Else 003 (ReactDOMFizzStaticNode.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ✅ | — | 12,350 / 3,219 | 24.4s |
|
||||
| 2 | ✅ | — | 12,036 / 1,448 | 18.8s |
|
||||
| 3 | ❌ | File mismatch for ReactDOMFizzStaticNode.js | 5,300 / 3,006 | 25.9s |
|
||||
|
||||
### Unicode Unicode Hyphen 001 (Rectangle.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for Rectangle.js | 2,178 / 282 | 4.9s |
|
||||
| 2 | ✅ | — | 10,031 / 115 | 3.4s |
|
||||
| 3 | ✅ | — | 1,527 / 182 | 7.7s |
|
||||
|
||||
### Unicode Unicode Hyphen 002 (UnsupportedBridgeProtocolDialog.js) — 1/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for UnsupportedBridgeProtocolDial... | 3,641 / 608 | 6.1s |
|
||||
| 2 | ✅ | — | 13,601 / 280 | 6.3s |
|
||||
| 3 | ❌ | File mismatch for UnsupportedBridgeProtocolDial... | 1,390 / 155 | 4.0s |
|
||||
|
||||
### Unicode Unicode Hyphen 003 (ReactTypes.js) — 2/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactTypes.js | 1,279 / 199 | 3.4s |
|
||||
| 2 | ✅ | — | 13,933 / 248 | 7.5s |
|
||||
| 3 | ✅ | — | 14,046 / 201 | 7.8s |
|
||||
|
||||
## Failed Tasks (0% passing)
|
||||
|
||||
### Access Remove Optional Chain 003 (astUtils.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for astUtils.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Assumptions:
|
||||
- Some hook variable declarations use sparse array patterns (e.g., `const [, setFoo] = useState(0)`), so `hook.node.id.elements[0]` can be `null` or `undefined`.
|
||||
- No other parts of `astUtils.js` require touching; we only need to guard this single property access with optional chaining.
|
||||
|
||||
Plan:
|
||||
1. Focus on `getHookVariableName`’s array-pattern case, since that’s where we dereference `elements[0].name`.
|
||||
2. Reintroduce the optional chaining that was likely dropped so we don’t crash ...
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -255 @@
|
||||
const nodeType = hook.node.id.type;
|
||||
switch (nodeType) {
|
||||
case AST_NODE_TYPES.ARRAY_PATTERN:
|
||||
- return !isCustomHook ? (hook.node.id.elements[0]?.name ?? null) : null;
|
||||
+ return !isCustomHook ? (hook.node.id.elements[0].name ?? null) : null;
|
||||
|
||||
case AST_NODE_TYPES.IDENTIFIER:
|
||||
return hook.node.id.name;
|
||||
```
|
||||
|
||||
### Duplicate Duplicate Line Flip 003 (SyntheticEvent.js) — 0/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for SyntheticEvent.js | 2,777 / 4,804 | 36.1s |
|
||||
| 2 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 3 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
- `ReactFlightUnbundledReferences.js`: tightened the cache check in `getReference` by replacing the loose negated truthiness test with an explicit nullish check, so only genuinely missing entries trigger the proxy creation path.
|
||||
|
||||
Tests not run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -448 @@
|
||||
|
||||
// KeyPress is deprecated, but its replacement is not yet final and not
|
||||
// implemented in any major browser. Only KeyPress has charCode.
|
||||
- if (event.type === 'keypress') {
|
||||
+ if (event.type !== 'keypress') {
|
||||
return getEventCharCode(
|
||||
// $FlowFixMe[incompatible-call] unable to narrow to `KeyboardEvent`
|
||||
event,
|
||||
```
|
||||
|
||||
### Import Swap Named Imports 003 (StyleEditor.js) — 0/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 2 | ❌ | File mismatch for StyleEditor.js | 22,819 / 6,827 | 45.5s |
|
||||
| 3 | ❌ | File mismatch for StyleEditor.js | 4,943 / 6,464 | 40.5s |
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
- Swapped the order of `StoreContext` and `BridgeContext` in the `react-devtools-shared/src/devtools/views/context` import so the destructured names now match their intended usage.
|
||||
|
||||
Tests: Not run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -8 @@
|
||||
*/
|
||||
|
||||
import * as React from 'react';
|
||||
-import { useContext, useMemo, useRef, useState } from 'react';
|
||||
+import { useMemo, useContext, useRef, useState } from 'react';
|
||||
import { copy } from 'clipboard-js';
|
||||
import { copy } from 'clipboard-js';
|
||||
-import { BridgeContext, StoreContext } from 'react-devtools-shared/src/devtools/views/context';
|
||||
+import { StoreContext, BridgeContext } from 'react-devtools-shared/src/devtools/views/context';
|
||||
import Button from '../../Button';
|
||||
import ButtonIcon from '../../ButtonIcon';
|
||||
import { serializeDataForCopy } from '../../utils';
|
||||
```
|
||||
|
||||
### Literal Flip Boolean 003 (ReactFlightDOMClientEdge.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactFlightDOMClientEdge.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Plan:
|
||||
1. Confirm that `simulateEventDispatch` currently awaits `_dispatch` but never returns its boolean result, breaking callers that expect `dispatchEvent` semantics.
|
||||
2. Reintroduce the missing return so the async helper resolves to `_dispatch`’s value while still awaiting it.
|
||||
3. Keep the rest of the file untouched.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -100 @@
|
||||
typeof options.nonce === 'string' ? options.nonce : undefined,
|
||||
options && options.temporaryReferences ? options.temporaryReferences : undefined,
|
||||
__DEV__ && options && options.findSourceMapURL ? options.findSourceMapURL : undefined,
|
||||
- __DEV__ && options ? options.replayConsoleLogs === true : false, // defaults to false
|
||||
+ __DEV__ && options ? options.replayConsoleLogs === false : false, // defaults to false
|
||||
__DEV__ && options && options.environmentName ? options.environmentName : undefined,
|
||||
__DEV__ && options && options.startTime != null ? options.startTime : undefined,
|
||||
__DEV__ && options && options.endTime != null ? options.endTime : undefined,
|
||||
```
|
||||
|
||||
### Operator Remove Negation 003 (ReactFlightUnbundledReferences.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** Timeout waiting for agent_end
|
||||
|
||||
### Operator Swap Arithmetic 003 (hooks.js) — 0/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for hooks.js | 20,582 / 8,931 | 65.1s |
|
||||
| 2 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 3 | ❌ | File mismatch for hooks.js | 13,286 / 4,035 | 29.9s |
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
- Tightened the overflow check in `hooks.js` so `handleResize` now reports overflow only when `container.clientWidth < totalChildWidth`, matching the intended arithmetic comparison.
|
||||
|
||||
- Not run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -2 @@
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
- * LICENSE file in the root directory of this source tree.
|
||||
+ / LICENSE file in the root directory of this source tree.
|
||||
*
|
||||
* @flow
|
||||
*/
|
||||
@@ -118 @@
|
||||
|
||||
const container = ((containerRef.current: any): HTMLDivElement);
|
||||
|
||||
- const handleResize = () => setIsOverflowing(container.clientWidth <= totalChildWidth);
|
||||
+ const handleResize = () => setIsOverflowing(container.clientWidth < totalChildWidth);
|
||||
|
||||
handleResize();
|
||||
|
||||
```
|
||||
|
||||
### Operator Swap Nullish 003 (backend.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for backend.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Plan:
|
||||
- Confirmed `connectToDevTools` is the only place in `backend.js` using `??` with the two RN-style resolvers.
|
||||
- Swapped the operands so the hook-provided resolver is chosen before the optional `resolveRNStyle` argument.
|
||||
- Did not run tests (not requested).
|
||||
|
||||
Assumptions:
|
||||
- The intended behavior is to prefer `hook.resolveRNStyle` when it exists and fallback to the passed-in `resolveRNStyle`.
|
||||
- No other code needs touching to fix this single swapped operator.
|
||||
|
||||
Changes:
|
||||
- `backend.js`: swap...
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -195 @@
|
||||
// $FlowFixMe[incompatible-call] found when upgrading Flow
|
||||
bridge,
|
||||
agent,
|
||||
- ((resolveRNStyle || hook.resolveRNStyle: any): ResolveNativeStyle),
|
||||
+ ((hook.resolveRNStyle ?? resolveRNStyle: any): ResolveNativeStyle),
|
||||
nativeStyleEditorValidAttributes || hook.nativeStyleEditorValidAttributes || null,
|
||||
);
|
||||
} else {
|
||||
```
|
||||
|
||||
### Structural Delete Statement 002 (getComponentNameFromFiber.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for getComponentNameFromFiber.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Restored `functionName` capture in `getWrappedName` so the helper now mirrors the shared implementation: it pulls `innerType.displayName`/`.name` before composing the wrapper name. No tests run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -54 @@
|
||||
|
||||
// Keep in sync with shared/getComponentNameFromType
|
||||
function getWrappedName(outerType: mixed, innerType: any, wrapperName: string): string {
|
||||
- const functionName = innerType.displayName || innerType.name || '';
|
||||
+ const functionName = (innerType: any).displayName || (innerType: any).name || '';
|
||||
return (
|
||||
(outerType: any).displayName ||
|
||||
(functionName !== '' ? `${wrapperName}(${functionName})` : wrapperName)
|
||||
```
|
||||
|
||||
### Structural Remove Early Return 001 (InspectedElementStateTree.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for InspectedElementStateTree.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Changes:
|
||||
- `InspectedElementStateTree.js`: restored the null-check guard (`state == null`) immediately after destructuring so we return early before calling `Object.entries`.
|
||||
- `ReactFlightDOMServerBrowser.js`: changed the debug-channel loop condition to `i < messages.length - 1` so only complete messages are resolved before carrying the remainder forward.
|
||||
|
||||
Tests not run; not requested.
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -36 @@
|
||||
store,
|
||||
}: Props): React.Node {
|
||||
const { state, type } = inspectedElement;
|
||||
+
|
||||
if (state == null) {
|
||||
return null;
|
||||
}
|
||||
```
|
||||
|
||||
### Structural Remove Early Return 003 (ReactFiberAsyncAction.js) — 0/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | File mismatch for ReactFiberAsyncAction.js | 10,564 / 5,531 | 35.5s |
|
||||
| 2 | ❌ | File mismatch for ReactFiberAsyncAction.js | 5,678 / 1,769 | 11.5s |
|
||||
| 3 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.8s |
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Summary:
|
||||
- Restored the guard clause in `releaseIsomorphicIndicator` so it bails out when `pendingEntangledRoots` is already 0 before decrementing, preventing negative counts if the release path runs unexpectedly or twice.
|
||||
|
||||
Tests: Not run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -176 @@
|
||||
export function registerDefaultIndicator(
|
||||
onDefaultTransitionIndicator: () => void | (() => void),
|
||||
): void {
|
||||
- if (!enableDefaultTransitionIndicator) {
|
||||
- return;
|
||||
- }
|
||||
if (isomorphicDefaultTransitionIndicator === undefined) {
|
||||
isomorphicDefaultTransitionIndicator = onDefaultTransitionIndicator;
|
||||
} else if (isomorphicDefaultTransitionIndicator !== onDefaultTransitionIndicator) {
|
||||
@@ -217 @@
|
||||
}
|
||||
|
||||
function releaseIsomorphicIndicator() {
|
||||
+ if (pendingEntangledRoots === 0) {
|
||||
+ return;
|
||||
+ }
|
||||
if (--pendingEntangledRoots === 0) {
|
||||
stopIsomorphicDefaultIndicator();
|
||||
}
|
||||
```
|
||||
|
||||
### Structural Swap Adjacent Lines 003 (backend.js) — 0/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.8s |
|
||||
| 2 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.8s |
|
||||
| 3 | ❌ | File mismatch for backend.js | 47,783 / 15,200 | 106.4s |
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Assumptions: The reload-and-profile flag reset callback needs to run before instantiating the backend agent so the agent sees the reset state immediately. Swapped the two adjacent statements in both backend connection paths so `onReloadAndProfileFlagsReset()` executes before `new Agent(...)`, nothing else changed.
|
||||
|
||||
Tests not run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -77 @@
|
||||
useHttps = false,
|
||||
port = 8097,
|
||||
websocket,
|
||||
- resolveRNStyle = (null: $FlowFixMe),
|
||||
retryConnectionDelay = 2000,
|
||||
retryConnectionDelay = 2000,
|
||||
+ resolveRNStyle = (null: $FlowFixMe),
|
||||
isAppActive = () => true,
|
||||
onSettingsUpdated,
|
||||
isReloadAndProfileSupported = getIsReloadAndProfileSupported(),
|
||||
@@ -169 @@
|
||||
|
||||
// TODO (npm-packages) Warn if "isBackendStorageAPISupported"
|
||||
// $FlowFixMe[incompatible-call] found when upgrading Flow
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
onReloadAndProfileFlagsReset();
|
||||
}
|
||||
if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
onReloadAndProfileFlagsReset();
|
||||
}
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
|
||||
if (onSettingsUpdated != null) {
|
||||
agent.addListener('updateHookSettings', onSettingsUpdated);
|
||||
@@ -358 @@
|
||||
bridge.send('overrideComponentFilters', savedComponentFilters);
|
||||
}
|
||||
|
||||
- const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
onReloadAndProfileFlagsReset();
|
||||
}
|
||||
if (typeof onReloadAndProfileFlagsReset === 'function') {
|
||||
onReloadAndProfileFlagsReset();
|
||||
}
|
||||
+ const agent = new Agent(bridge, isProfiling, onReloadAndProfile);
|
||||
|
||||
if (onSettingsUpdated != null) {
|
||||
agent.addListener('updateHookSettings', onSettingsUpdated);
|
||||
```
|
||||
|
||||
### Structural Swap If Else 001 (importFile.js) — 0/3
|
||||
|
||||
| Run | Status | Error | Tokens (in/out) | Time |
|
||||
|-----|--------|-------|-----------------|------|
|
||||
| 1 | ❌ | Timeout waiting for agent_end | 0 / 0 | 120.0s |
|
||||
| 2 | ❌ | File mismatch for importFile.js | 3,867 / 509 | 9.4s |
|
||||
| 3 | ❌ | File mismatch for importFile.js | 3,375 / 6,337 | 44.7s |
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
- **EventPluginRegistry.js**: Restored the correct `eventName` identifier throughout `publishEventForPlugin`, ensuring the dispatch config map, optional property access, and registration name handling hit the right variable.
|
||||
- **TimelineContext.js**: Reintroduced optional chaining for `profilingData` so missing data no longer throws.
|
||||
- **editor.js**: Fixed the `guessEditor` process detection by inverting the comparison in the loop.
|
||||
- **importFile.js**: Corrected the error handling so `Invalid...
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -31 @@
|
||||
processedData,
|
||||
};
|
||||
} catch (error) {
|
||||
- if (error instanceof InvalidProfileError) {
|
||||
+ if (!(error instanceof InvalidProfileError)) {
|
||||
return {
|
||||
return {
|
||||
- status: 'INVALID_PROFILE_ERROR',
|
||||
+ status: 'UNEXPECTED_ERROR',
|
||||
error,
|
||||
};
|
||||
} else {
|
||||
@@ -38 @@
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
- status: 'UNEXPECTED_ERROR',
|
||||
+ status: 'INVALID_PROFILE_ERROR',
|
||||
error,
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Structural Swap If Else 002 (ReactNativeFiberInspector.js) — 0/3
|
||||
|
||||
**All runs failed with same error:** File mismatch for ReactNativeFiberInspector.js
|
||||
|
||||
**Sample agent response (run 1):**
|
||||
```
|
||||
Assumption: `getInspectorDataForViewTag` should stay available in dev builds and throw only in production, so the branch that throws must run when `__DEV__` is false.
|
||||
|
||||
Summary:
|
||||
- Flipped the condition guarding the production-only error so the `throw` now fires when `__DEV__` is false, leaving the development path (returning inspector data) inside the `else` block unchanged.
|
||||
|
||||
Testing: Not run (not requested).
|
||||
```
|
||||
|
||||
**Diff (expected vs actual):**
|
||||
```diff
|
||||
@@ -145 @@
|
||||
}
|
||||
|
||||
function getInspectorDataForViewTag(viewTag: number): InspectorData {
|
||||
- if (__DEV__) {
|
||||
+ if (!__DEV__) {
|
||||
+ throw new Error('getInspectorDataForViewTag() is not available in production');
|
||||
+ } else {
|
||||
const closestInstance = getClosestInstanceFromNode(viewTag);
|
||||
return getInspectorDataForInstance(closestInstance);
|
||||
const closestInstance = getClosestInstanceFromNode(viewTag);
|
||||
return getInspectorDataForInstance(closestInstance);
|
||||
- } else {
|
||||
- throw new Error('getInspectorDataForViewTag() is not available in production');
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
@@ -95,24 +95,42 @@ function createCompactDiff(expected: string, actual: string, contextLines = 3):
|
||||
export async function verifyExpectedFiles(
|
||||
expectedDir: string,
|
||||
actualDir: string,
|
||||
): Promise<VerificationResult> {
|
||||
return verifyExpectedFileSubset(expectedDir, actualDir);
|
||||
}
|
||||
|
||||
export async function verifyExpectedFileSubset(
|
||||
expectedDir: string,
|
||||
actualDir: string,
|
||||
files?: string[],
|
||||
): Promise<VerificationResult> {
|
||||
const startTime = Date.now();
|
||||
let totalIndentScore = 0;
|
||||
let fileCount = 0;
|
||||
|
||||
try {
|
||||
const expectedFiles = listFiles(expectedDir);
|
||||
const expectedFixtureFiles = listFiles(expectedDir);
|
||||
const expectedFiles = files?.length ? files.slice().sort() : expectedFixtureFiles;
|
||||
const actualFiles = listFiles(actualDir);
|
||||
|
||||
const missingFiles = expectedFiles.filter((file) => !actualFiles.includes(file));
|
||||
const extraFiles = actualFiles.filter((file) => !expectedFiles.includes(file));
|
||||
const missingExpected = expectedFiles.filter((file) => !expectedFixtureFiles.includes(file));
|
||||
|
||||
if (missingFiles.length > 0 || extraFiles.length > 0) {
|
||||
if (missingExpected.length > 0) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Expected files missing from fixture: ${formatFileList(missingExpected)}`,
|
||||
duration: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
if (missingFiles.length > 0 || (files === undefined && extraFiles.length > 0)) {
|
||||
const parts: string[] = [];
|
||||
if (missingFiles.length > 0) {
|
||||
parts.push(`Missing files: ${formatFileList(missingFiles)}`);
|
||||
}
|
||||
if (extraFiles.length > 0) {
|
||||
if (files === undefined && extraFiles.length > 0) {
|
||||
parts.push(`Unexpected files: ${formatFileList(extraFiles)}`);
|
||||
}
|
||||
|
||||
@@ -145,7 +163,6 @@ export async function verifyExpectedFiles(
|
||||
if (!formattedEquivalent) {
|
||||
const diffOutput = createCompactDiff(expectedFormatted.formatted, actualFormatted.formatted);
|
||||
const diffStats = computeDiffStats(expectedFormatted.formatted, actualFormatted.formatted);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: `File mismatch for ${file}`,
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
* --compact Compact output (no diff content)
|
||||
*/
|
||||
|
||||
import { readFileSync, existsSync } from "node:fs";
|
||||
import { basename } from "node:path";
|
||||
import { Glob } from "bun";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { basename } from "node:path";
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Types
|
||||
@@ -43,7 +43,7 @@ interface Message {
|
||||
interface EditAttempt {
|
||||
id: string;
|
||||
path: string;
|
||||
operation: string;
|
||||
op: string;
|
||||
diff: string;
|
||||
isError: boolean;
|
||||
resultText: string;
|
||||
@@ -90,7 +90,7 @@ function extractEditAttempts(sessionPath: string): EditAttempt[] {
|
||||
for (const item of msgContent) {
|
||||
if (item.type === "toolCall" && item.name === "edit") {
|
||||
const toolId = item.id!;
|
||||
const args = item.arguments as { path?: string; operation?: string; diff?: string };
|
||||
const args = item.arguments as { path?: string; op?: string; diff?: string };
|
||||
|
||||
// Find result
|
||||
let result: Message["message"] | null = null;
|
||||
@@ -115,7 +115,7 @@ function extractEditAttempts(sessionPath: string): EditAttempt[] {
|
||||
editAttempts.push({
|
||||
id: toolId,
|
||||
path: args.path ?? "",
|
||||
operation: args.operation ?? "update",
|
||||
op: args.op ?? "update",
|
||||
diff: args.diff ?? "",
|
||||
isError,
|
||||
resultText,
|
||||
@@ -170,7 +170,7 @@ function formatAttempt(attempt: EditAttempt, index: number, options: Options): s
|
||||
``,
|
||||
`${colorize(`### Attempt ${index}`, "bold")}: ${status}`,
|
||||
`${colorize("Path:", "dim")} ${attempt.path}`,
|
||||
`${colorize("Operation:", "dim")} ${attempt.operation}`,
|
||||
`${colorize("Operation:", "dim")} ${attempt.op}`,
|
||||
];
|
||||
|
||||
if (options.context && attempt.thinkingContext) {
|
||||
@@ -258,7 +258,7 @@ function formatJson(results: SessionResult[]): string {
|
||||
file: r.file,
|
||||
attempts: r.attempts.map((a) => ({
|
||||
path: a.path,
|
||||
operation: a.operation,
|
||||
op: a.op,
|
||||
diff: a.diff,
|
||||
isError: a.isError,
|
||||
errorType: a.errorType,
|
||||
|
||||
Reference in New Issue
Block a user