feat(typescript-edit-benchmark): added configurable early-stop-on-match behavior
- Added a --no-early-stop-on-match CLI option, passed into benchmark configuration as earlyStopOnMatch. - Added early-stop support that verified expected files after mutation-tool completion and aborted the prompt loop on a match. - Captured an earlyStopped flag in task results and emitted an early_stop event when match-based termination occurred.
This commit is contained in:
can1357
@@ -90,6 +90,7 @@ Options:
|
||||
--require-edit-tool-call Require edit tool usage for success (default: false)
|
||||
--require-read-tool-call Require read tool usage for success (default: false)
|
||||
--no-edit-required Remove "must edit" prompt requirement (default: false)
|
||||
--no-early-stop-on-match Don't short-circuit the run when output matches expected (default: false)
|
||||
--list List available tasks and exit
|
||||
--help Show this help message
|
||||
|
||||
@@ -191,6 +192,7 @@ async function main(): Promise<void> {
|
||||
"edit-fuzzy": { type: "string" },
|
||||
"edit-fuzzy-threshold": { type: "string" },
|
||||
"no-in-process": { type: "boolean", default: false },
|
||||
"no-early-stop-on-match": { type: "boolean", default: false },
|
||||
"max-tasks": { type: "string", default: "80" },
|
||||
list: { type: "boolean", default: false },
|
||||
help: { type: "boolean", default: false },
|
||||
@@ -364,6 +366,7 @@ async function main(): Promise<void> {
|
||||
mutationScopeWindow,
|
||||
connectionTimeout,
|
||||
inProcess: !values["no-in-process"],
|
||||
earlyStopOnMatch: !values["no-early-stop-on-match"],
|
||||
};
|
||||
const outputPath = values.output ?? generateReportFilename(config, formatType);
|
||||
config.conversationDumpDir = await resolveConversationDumpDir(outputPath);
|
||||
|
||||
@@ -71,6 +71,8 @@ export interface BenchmarkConfig {
|
||||
requireReadToolCall?: boolean;
|
||||
noEditRequired?: boolean;
|
||||
autoFormat?: boolean;
|
||||
/** If true, abort the agent loop as soon as the formatted file content matches the expected fixture. Default: true. */
|
||||
earlyStopOnMatch?: boolean;
|
||||
editVariant?: string;
|
||||
editFuzzy?: boolean | "auto";
|
||||
editFuzzyThreshold?: number | "auto";
|
||||
@@ -774,6 +776,8 @@ export interface TaskRunResult {
|
||||
mutationIntentMatched?: boolean;
|
||||
mutationIntentReason?: string;
|
||||
timeoutTelemetry?: PromptAttemptTelemetry;
|
||||
/** True when the run terminated early because the formatted file content matched the expected fixture. */
|
||||
earlyStopped?: boolean;
|
||||
/** Retry telemetry: how many retries of each type were used */
|
||||
retryStats?: {
|
||||
timeoutRetries: number;
|
||||
@@ -861,6 +865,34 @@ async function copyFixtures(task: EditTask, destDir: string): Promise<void> {
|
||||
);
|
||||
}
|
||||
|
||||
interface EarlyStopOptions {
|
||||
check: () => Promise<boolean>;
|
||||
onMatch: () => void | Promise<void>;
|
||||
}
|
||||
|
||||
function buildEarlyStop(params: {
|
||||
config: BenchmarkConfig;
|
||||
cwd: string;
|
||||
expectedDir: string;
|
||||
files: string[];
|
||||
logEvent: (event: unknown) => Promise<void>;
|
||||
attempt: number;
|
||||
onMatched: () => void;
|
||||
}): EarlyStopOptions | undefined {
|
||||
if (params.config.earlyStopOnMatch === false) return undefined;
|
||||
if (params.files.length === 0) return undefined;
|
||||
return {
|
||||
check: async () => {
|
||||
const verification = await verifyExpectedFileSubset(params.expectedDir, params.cwd, params.files);
|
||||
return verification.success;
|
||||
},
|
||||
onMatch: async () => {
|
||||
params.onMatched();
|
||||
await params.logEvent({ type: "early_stop", attempt: params.attempt, reason: "formatted_match" });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runSingleTask(
|
||||
task: EditTask,
|
||||
runIndex: number,
|
||||
@@ -884,6 +916,7 @@ async function runSingleTask(
|
||||
let editAutocorrectCount = 0;
|
||||
let timeoutTelemetry: PromptAttemptTelemetry | undefined;
|
||||
let mutationIntentValidation: MutationIntentValidation | null = null;
|
||||
let earlyStoppedByMatch = false;
|
||||
let conversationSnapshot: ConversationDumpSnapshot | undefined;
|
||||
const toolStats = {
|
||||
read: 0,
|
||||
@@ -991,7 +1024,17 @@ async function runSingleTask(
|
||||
const statsBefore = await client.getSessionStats();
|
||||
let events: Array<{ type: string; [key: string]: unknown }>;
|
||||
try {
|
||||
events = await collectPromptEvents(client, delivery, config, logEvent);
|
||||
events = await collectPromptEvents(client, delivery, config, logEvent, buildEarlyStop({
|
||||
config,
|
||||
cwd,
|
||||
expectedDir,
|
||||
files: task.files,
|
||||
logEvent,
|
||||
attempt: attempt + 1,
|
||||
onMatched: () => {
|
||||
earlyStoppedByMatch = true;
|
||||
},
|
||||
}));
|
||||
} catch (err) {
|
||||
if (err instanceof PromptTurnLimitError) {
|
||||
error = err.message;
|
||||
@@ -1239,6 +1282,7 @@ async function runSingleTask(
|
||||
mutationIntentMatched: mutationIntentValidation?.matched,
|
||||
mutationIntentReason: mutationIntentValidation?.reason,
|
||||
timeoutTelemetry,
|
||||
earlyStopped: earlyStoppedByMatch || undefined,
|
||||
retryStats: {
|
||||
timeoutRetries: timeoutRetriesUsed,
|
||||
zeroToolRetries,
|
||||
@@ -1624,6 +1668,10 @@ async function collectPromptEvents(
|
||||
delivery: BenchmarkPromptDelivery,
|
||||
config: BenchmarkConfig,
|
||||
logEvent: (event: unknown) => Promise<void>,
|
||||
earlyStop?: {
|
||||
check: () => Promise<boolean>;
|
||||
onMatch: () => void | Promise<void>;
|
||||
},
|
||||
): Promise<Array<{ type: string; [key: string]: unknown }>> {
|
||||
const events: Array<{ type: string; [key: string]: unknown }> = [];
|
||||
let unsubscribe: (() => void) | undefined;
|
||||
@@ -1638,6 +1686,8 @@ async function collectPromptEvents(
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
let settled = false;
|
||||
let receivedFirstEvent = false;
|
||||
let earlyStopTriggered = false;
|
||||
let earlyStopChain: Promise<void> = Promise.resolve();
|
||||
|
||||
const connectionTimeout = config.connectionTimeout ?? 30_000;
|
||||
|
||||
@@ -1682,6 +1732,30 @@ async function collectPromptEvents(
|
||||
);
|
||||
};
|
||||
|
||||
const triggerEarlyStop = () => {
|
||||
if (!earlyStop || earlyStopTriggered || settled) return;
|
||||
earlyStopChain = earlyStopChain
|
||||
.then(async () => {
|
||||
if (earlyStopTriggered || settled) return;
|
||||
let matched = false;
|
||||
try {
|
||||
matched = await earlyStop.check();
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!matched || earlyStopTriggered || settled) return;
|
||||
earlyStopTriggered = true;
|
||||
try {
|
||||
await earlyStop.onMatch();
|
||||
} catch {
|
||||
// Swallow callback errors; we still want to short-circuit.
|
||||
}
|
||||
client.abort?.();
|
||||
resolveWait();
|
||||
})
|
||||
.catch(() => {});
|
||||
};
|
||||
|
||||
// Start with the shorter connection timeout; upgrade to full timeout on first event
|
||||
timer = setTimeout(fireTimeout, connectionTimeout);
|
||||
|
||||
@@ -1712,6 +1786,13 @@ async function collectPromptEvents(
|
||||
if (typedEvent.type === "tool_execution_end") {
|
||||
toolExecutionEnds += 1;
|
||||
}
|
||||
if (
|
||||
typedEvent.type === "tool_execution_end" &&
|
||||
!(typedEvent as { isError?: boolean }).isError &&
|
||||
isMutationTool((typedEvent as { toolName?: unknown }).toolName)
|
||||
) {
|
||||
triggerEarlyStop();
|
||||
}
|
||||
if (typedEvent.type === "message_end") {
|
||||
messageEnds += 1;
|
||||
}
|
||||
@@ -1765,6 +1846,14 @@ async function collectPromptEvents(
|
||||
await client.prompt(delivery.message);
|
||||
}
|
||||
} catch (err) {
|
||||
if (earlyStopTriggered) {
|
||||
// Abort raised inside prompt(); the run already short-circuited successfully.
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
unsubscribe?.();
|
||||
return events;
|
||||
}
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user