fix(typescript-edit-benchmark): included timeout transport failures as excluded benchmark runs

- Detected transport failures by checking failed run errors for "Timeout exhausted".
- Counted those failures in benchmark summaries and included them in ghost-like run classification, reducing effective run totals accordingly.
- Updated benchmark reporting to display excluded transport-failure runs when present.
This commit is contained in:
can1357
2026-04-26 05:12:09 +02:00
parent 652f49442f
commit ab1a5e2f3e
2 changed files with 19 additions and 3 deletions
@@ -125,6 +125,9 @@ export function generateReport(result: BenchmarkResult): string {
if (summary.ghostRuns > 0) {
lines.push(`| Ghost Runs (0/0/0) | ${summary.ghostRuns} |`);
}
if (summary.transportFailureRuns > 0) {
lines.push(`| Transport Failures (excluded) | ${summary.transportFailureRuns} |`);
}
if (summary.totalTimeoutRetries > 0 || summary.totalZeroToolRetries > 0 || summary.totalProviderFailureRetries > 0) {
lines.push(`| Timeout Retries | ${summary.totalTimeoutRetries} |`);
lines.push(`| Zero-Tool Retries | ${summary.totalZeroToolRetries} |`);
@@ -833,6 +833,8 @@ export interface BenchmarkSummary {
totalProviderFailureRetries: number;
/** Runs where the 0/0/0 ghost signature was detected (0 tokens, 0 tool calls) */
ghostRuns: number;
/** Runs excluded because provider/transport stalls exhausted retries (subset of ghostRuns when error matches). */
transportFailureRuns: number;
mutationIntentMatchRate?: number;
/** Hashline edit subtype totals — only when editVariant is hashline */
hashlineEditSubtypes?: Record<string, number>;
@@ -1891,10 +1893,19 @@ function diffTokenStats(
return { input, output, total };
}
function isTransportFailure(r: TaskRunResult): boolean {
if (r.success) return false;
const err = r.error ?? "";
// Provider/transport stalls retried until the cap was hit. These don't reflect
// edit-tool quality, so we exclude them from the score denominator.
return err.includes("Timeout exhausted");
}
function isGhostRun(r: TaskRunResult): boolean {
return (
!r.success && r.tokens.total === 0 && r.toolCalls.read === 0 && r.toolCalls.edit === 0 && r.toolCalls.write === 0
);
if (r.success) return false;
const noProgress =
r.tokens.total === 0 && r.toolCalls.read === 0 && r.toolCalls.edit === 0 && r.toolCalls.write === 0;
return noProgress || isTransportFailure(r);
}
function summarizeTaskRuns(task: EditTask, runs: TaskRunResult[]): TaskResult {
@@ -2089,6 +2100,7 @@ export async function runBenchmark(
const allRuns = taskResults.flatMap(t => t.runs);
const totalRuns = allRuns.length;
const ghostRuns = allRuns.filter(r => isGhostRun(r)).length;
const transportFailureRuns = allRuns.filter(r => isTransportFailure(r)).length;
const effectiveRuns = totalRuns - ghostRuns;
const nonGhostRuns = allRuns.filter(r => !isGhostRun(r));
const successfulRuns = allRuns.filter(r => r.success).length;
@@ -2197,6 +2209,7 @@ export async function runBenchmark(
totalZeroToolRetries,
totalProviderFailureRetries,
ghostRuns,
transportFailureRuns,
mutationIntentMatchRate,
hashlineEditSubtypes,
chunkEditSubtypes,