fix(typescript-edit-benchmark): included timeout transport failures as excluded benchmark runs
- Detected transport failures by checking failed run errors for "Timeout exhausted". - Counted those failures in benchmark summaries and included them in ghost-like run classification, reducing effective run totals accordingly. - Updated benchmark reporting to display excluded transport-failure runs when present.
This commit is contained in:
@@ -125,6 +125,9 @@ export function generateReport(result: BenchmarkResult): string {
|
||||
if (summary.ghostRuns > 0) {
|
||||
lines.push(`| Ghost Runs (0/0/0) | ${summary.ghostRuns} |`);
|
||||
}
|
||||
if (summary.transportFailureRuns > 0) {
|
||||
lines.push(`| Transport Failures (excluded) | ${summary.transportFailureRuns} |`);
|
||||
}
|
||||
if (summary.totalTimeoutRetries > 0 || summary.totalZeroToolRetries > 0 || summary.totalProviderFailureRetries > 0) {
|
||||
lines.push(`| Timeout Retries | ${summary.totalTimeoutRetries} |`);
|
||||
lines.push(`| Zero-Tool Retries | ${summary.totalZeroToolRetries} |`);
|
||||
|
||||
@@ -833,6 +833,8 @@ export interface BenchmarkSummary {
|
||||
totalProviderFailureRetries: number;
|
||||
/** Runs where the 0/0/0 ghost signature was detected (0 tokens, 0 tool calls) */
|
||||
ghostRuns: number;
|
||||
/** Runs excluded because provider/transport stalls exhausted retries (subset of ghostRuns when error matches). */
|
||||
transportFailureRuns: number;
|
||||
mutationIntentMatchRate?: number;
|
||||
/** Hashline edit subtype totals — only when editVariant is hashline */
|
||||
hashlineEditSubtypes?: Record<string, number>;
|
||||
@@ -1891,10 +1893,19 @@ function diffTokenStats(
|
||||
return { input, output, total };
|
||||
}
|
||||
|
||||
function isTransportFailure(r: TaskRunResult): boolean {
|
||||
if (r.success) return false;
|
||||
const err = r.error ?? "";
|
||||
// Provider/transport stalls retried until the cap was hit. These don't reflect
|
||||
// edit-tool quality, so we exclude them from the score denominator.
|
||||
return err.includes("Timeout exhausted");
|
||||
}
|
||||
|
||||
function isGhostRun(r: TaskRunResult): boolean {
|
||||
return (
|
||||
!r.success && r.tokens.total === 0 && r.toolCalls.read === 0 && r.toolCalls.edit === 0 && r.toolCalls.write === 0
|
||||
);
|
||||
if (r.success) return false;
|
||||
const noProgress =
|
||||
r.tokens.total === 0 && r.toolCalls.read === 0 && r.toolCalls.edit === 0 && r.toolCalls.write === 0;
|
||||
return noProgress || isTransportFailure(r);
|
||||
}
|
||||
|
||||
function summarizeTaskRuns(task: EditTask, runs: TaskRunResult[]): TaskResult {
|
||||
@@ -2089,6 +2100,7 @@ export async function runBenchmark(
|
||||
const allRuns = taskResults.flatMap(t => t.runs);
|
||||
const totalRuns = allRuns.length;
|
||||
const ghostRuns = allRuns.filter(r => isGhostRun(r)).length;
|
||||
const transportFailureRuns = allRuns.filter(r => isTransportFailure(r)).length;
|
||||
const effectiveRuns = totalRuns - ghostRuns;
|
||||
const nonGhostRuns = allRuns.filter(r => !isGhostRun(r));
|
||||
const successfulRuns = allRuns.filter(r => r.success).length;
|
||||
@@ -2197,6 +2209,7 @@ export async function runBenchmark(
|
||||
totalZeroToolRetries,
|
||||
totalProviderFailureRetries,
|
||||
ghostRuns,
|
||||
transportFailureRuns,
|
||||
mutationIntentMatchRate,
|
||||
hashlineEditSubtypes,
|
||||
chunkEditSubtypes,
|
||||
|
||||
Reference in New Issue
Block a user