diff --git a/packages/typescript-edit-benchmark/src/index.ts b/packages/typescript-edit-benchmark/src/index.ts index 0eb202dce..ca59db2a4 100644 --- a/packages/typescript-edit-benchmark/src/index.ts +++ b/packages/typescript-edit-benchmark/src/index.ts @@ -90,6 +90,7 @@ Options: --require-edit-tool-call Require edit tool usage for success (default: false) --require-read-tool-call Require read tool usage for success (default: false) --no-edit-required Remove "must edit" prompt requirement (default: false) + --no-early-stop-on-match Don't short-circuit the run when output matches expected (default: false) --list List available tasks and exit --help Show this help message @@ -191,6 +192,7 @@ async function main(): Promise { "edit-fuzzy": { type: "string" }, "edit-fuzzy-threshold": { type: "string" }, "no-in-process": { type: "boolean", default: false }, + "no-early-stop-on-match": { type: "boolean", default: false }, "max-tasks": { type: "string", default: "80" }, list: { type: "boolean", default: false }, help: { type: "boolean", default: false }, @@ -364,6 +366,7 @@ async function main(): Promise { mutationScopeWindow, connectionTimeout, inProcess: !values["no-in-process"], + earlyStopOnMatch: !values["no-early-stop-on-match"], }; const outputPath = values.output ?? generateReportFilename(config, formatType); config.conversationDumpDir = await resolveConversationDumpDir(outputPath); diff --git a/packages/typescript-edit-benchmark/src/runner.ts b/packages/typescript-edit-benchmark/src/runner.ts index fe88dbe56..86cb184d2 100644 --- a/packages/typescript-edit-benchmark/src/runner.ts +++ b/packages/typescript-edit-benchmark/src/runner.ts @@ -71,6 +71,8 @@ export interface BenchmarkConfig { requireReadToolCall?: boolean; noEditRequired?: boolean; autoFormat?: boolean; + /** If true, abort the agent loop as soon as the formatted file content matches the expected fixture. Default: true. */ + earlyStopOnMatch?: boolean; editVariant?: string; editFuzzy?: boolean | "auto"; editFuzzyThreshold?: number | "auto"; @@ -774,6 +776,8 @@ export interface TaskRunResult { mutationIntentMatched?: boolean; mutationIntentReason?: string; timeoutTelemetry?: PromptAttemptTelemetry; + /** True when the run terminated early because the formatted file content matched the expected fixture. */ + earlyStopped?: boolean; /** Retry telemetry: how many retries of each type were used */ retryStats?: { timeoutRetries: number; @@ -861,6 +865,34 @@ async function copyFixtures(task: EditTask, destDir: string): Promise { ); } +interface EarlyStopOptions { + check: () => Promise; + onMatch: () => void | Promise; +} + +function buildEarlyStop(params: { + config: BenchmarkConfig; + cwd: string; + expectedDir: string; + files: string[]; + logEvent: (event: unknown) => Promise; + attempt: number; + onMatched: () => void; +}): EarlyStopOptions | undefined { + if (params.config.earlyStopOnMatch === false) return undefined; + if (params.files.length === 0) return undefined; + return { + check: async () => { + const verification = await verifyExpectedFileSubset(params.expectedDir, params.cwd, params.files); + return verification.success; + }, + onMatch: async () => { + params.onMatched(); + await params.logEvent({ type: "early_stop", attempt: params.attempt, reason: "formatted_match" }); + }, + }; +} + async function runSingleTask( task: EditTask, runIndex: number, @@ -884,6 +916,7 @@ async function runSingleTask( let editAutocorrectCount = 0; let timeoutTelemetry: PromptAttemptTelemetry | undefined; let mutationIntentValidation: MutationIntentValidation | null = null; + let earlyStoppedByMatch = false; let conversationSnapshot: ConversationDumpSnapshot | undefined; const toolStats = { read: 0, @@ -991,7 +1024,17 @@ async function runSingleTask( const statsBefore = await client.getSessionStats(); let events: Array<{ type: string; [key: string]: unknown }>; try { - events = await collectPromptEvents(client, delivery, config, logEvent); + events = await collectPromptEvents(client, delivery, config, logEvent, buildEarlyStop({ + config, + cwd, + expectedDir, + files: task.files, + logEvent, + attempt: attempt + 1, + onMatched: () => { + earlyStoppedByMatch = true; + }, + })); } catch (err) { if (err instanceof PromptTurnLimitError) { error = err.message; @@ -1239,6 +1282,7 @@ async function runSingleTask( mutationIntentMatched: mutationIntentValidation?.matched, mutationIntentReason: mutationIntentValidation?.reason, timeoutTelemetry, + earlyStopped: earlyStoppedByMatch || undefined, retryStats: { timeoutRetries: timeoutRetriesUsed, zeroToolRetries, @@ -1624,6 +1668,10 @@ async function collectPromptEvents( delivery: BenchmarkPromptDelivery, config: BenchmarkConfig, logEvent: (event: unknown) => Promise, + earlyStop?: { + check: () => Promise; + onMatch: () => void | Promise; + }, ): Promise> { const events: Array<{ type: string; [key: string]: unknown }> = []; let unsubscribe: (() => void) | undefined; @@ -1638,6 +1686,8 @@ async function collectPromptEvents( let timer: NodeJS.Timeout | undefined; let settled = false; let receivedFirstEvent = false; + let earlyStopTriggered = false; + let earlyStopChain: Promise = Promise.resolve(); const connectionTimeout = config.connectionTimeout ?? 30_000; @@ -1682,6 +1732,30 @@ async function collectPromptEvents( ); }; + const triggerEarlyStop = () => { + if (!earlyStop || earlyStopTriggered || settled) return; + earlyStopChain = earlyStopChain + .then(async () => { + if (earlyStopTriggered || settled) return; + let matched = false; + try { + matched = await earlyStop.check(); + } catch { + return; + } + if (!matched || earlyStopTriggered || settled) return; + earlyStopTriggered = true; + try { + await earlyStop.onMatch(); + } catch { + // Swallow callback errors; we still want to short-circuit. + } + client.abort?.(); + resolveWait(); + }) + .catch(() => {}); + }; + // Start with the shorter connection timeout; upgrade to full timeout on first event timer = setTimeout(fireTimeout, connectionTimeout); @@ -1712,6 +1786,13 @@ async function collectPromptEvents( if (typedEvent.type === "tool_execution_end") { toolExecutionEnds += 1; } + if ( + typedEvent.type === "tool_execution_end" && + !(typedEvent as { isError?: boolean }).isError && + isMutationTool((typedEvent as { toolName?: unknown }).toolName) + ) { + triggerEarlyStop(); + } if (typedEvent.type === "message_end") { messageEnds += 1; } @@ -1765,6 +1846,14 @@ async function collectPromptEvents( await client.prompt(delivery.message); } } catch (err) { + if (earlyStopTriggered) { + // Abort raised inside prompt(); the run already short-circuited successfully. + if (timer) { + clearTimeout(timer); + } + unsubscribe?.(); + return events; + } if (timer) { clearTimeout(timer); }