diff --git a/packages/typescript-edit-benchmark/src/report.ts b/packages/typescript-edit-benchmark/src/report.ts index 44a2ddc5c..580417c86 100644 --- a/packages/typescript-edit-benchmark/src/report.ts +++ b/packages/typescript-edit-benchmark/src/report.ts @@ -125,6 +125,9 @@ export function generateReport(result: BenchmarkResult): string { if (summary.ghostRuns > 0) { lines.push(`| Ghost Runs (0/0/0) | ${summary.ghostRuns} |`); } + if (summary.transportFailureRuns > 0) { + lines.push(`| Transport Failures (excluded) | ${summary.transportFailureRuns} |`); + } if (summary.totalTimeoutRetries > 0 || summary.totalZeroToolRetries > 0 || summary.totalProviderFailureRetries > 0) { lines.push(`| Timeout Retries | ${summary.totalTimeoutRetries} |`); lines.push(`| Zero-Tool Retries | ${summary.totalZeroToolRetries} |`); diff --git a/packages/typescript-edit-benchmark/src/runner.ts b/packages/typescript-edit-benchmark/src/runner.ts index 41aabb4fc..3f791665f 100644 --- a/packages/typescript-edit-benchmark/src/runner.ts +++ b/packages/typescript-edit-benchmark/src/runner.ts @@ -833,6 +833,8 @@ export interface BenchmarkSummary { totalProviderFailureRetries: number; /** Runs where the 0/0/0 ghost signature was detected (0 tokens, 0 tool calls) */ ghostRuns: number; + /** Runs excluded because provider/transport stalls exhausted retries (subset of ghostRuns when error matches). */ + transportFailureRuns: number; mutationIntentMatchRate?: number; /** Hashline edit subtype totals — only when editVariant is hashline */ hashlineEditSubtypes?: Record; @@ -1891,10 +1893,19 @@ function diffTokenStats( return { input, output, total }; } +function isTransportFailure(r: TaskRunResult): boolean { + if (r.success) return false; + const err = r.error ?? ""; + // Provider/transport stalls retried until the cap was hit. These don't reflect + // edit-tool quality, so we exclude them from the score denominator. + return err.includes("Timeout exhausted"); +} + function isGhostRun(r: TaskRunResult): boolean { - return ( - !r.success && r.tokens.total === 0 && r.toolCalls.read === 0 && r.toolCalls.edit === 0 && r.toolCalls.write === 0 - ); + if (r.success) return false; + const noProgress = + r.tokens.total === 0 && r.toolCalls.read === 0 && r.toolCalls.edit === 0 && r.toolCalls.write === 0; + return noProgress || isTransportFailure(r); } function summarizeTaskRuns(task: EditTask, runs: TaskRunResult[]): TaskResult { @@ -2089,6 +2100,7 @@ export async function runBenchmark( const allRuns = taskResults.flatMap(t => t.runs); const totalRuns = allRuns.length; const ghostRuns = allRuns.filter(r => isGhostRun(r)).length; + const transportFailureRuns = allRuns.filter(r => isTransportFailure(r)).length; const effectiveRuns = totalRuns - ghostRuns; const nonGhostRuns = allRuns.filter(r => !isGhostRun(r)); const successfulRuns = allRuns.filter(r => r.success).length; @@ -2197,6 +2209,7 @@ export async function runBenchmark( totalZeroToolRetries, totalProviderFailureRetries, ghostRuns, + transportFailureRuns, mutationIntentMatchRate, hashlineEditSubtypes, chunkEditSubtypes,