fix(agent): recover tools after stream parse errors

This commit is contained in:
usr_bin_roygbiv
2026-07-20 17:13:28 -05:00
parent 89d6a8f6d1
commit 7c6d691c10
5 changed files with 304 additions and 7 deletions
+4
View File
@@ -2,6 +2,10 @@
## [Unreleased]
### Fixed
- Recovered completed tool calls after transient stream JSON parse failures while keeping incomplete, unknown, refused, and sensitive calls non-executable.
## [17.0.5] - 2026-07-18
### Added
+14 -1
View File
@@ -1629,13 +1629,26 @@ function recoverTransientErrorToolTurn(
if (message.stopReason !== "error") return message;
const toolCalls = message.content.filter(block => block.type === "toolCall");
if (toolCalls.length === 0) return message;
const stopDetailType = message.stopDetails?.type;
const stopDetailCategory = message.stopDetails?.category;
if (
stopDetailType === "refusal" ||
stopDetailType === "sensitive" ||
stopDetailCategory === "refusal" ||
stopDetailCategory === "sensitive"
)
return message;
const availableToolNames = new Set<string>();
for (const tool of availableTools) {
availableToolNames.add(tool.name);
if (tool.customWireName !== undefined) availableToolNames.add(tool.customWireName);
}
if (!toolCalls.every(toolCall => availableToolNames.has(toolCall.name))) return message;
if (!AIError.isStreamReadErrorText(`${message.errorMessage ?? ""}\n${message.stopDetails?.explanation ?? ""}`))
if (
!AIError.isStreamReadErrorText(`${message.errorMessage ?? ""}\n${message.stopDetails?.explanation ?? ""}`) &&
!AIError.isTransientStreamParseError(message.errorMessage) &&
!AIError.isTransientStreamParseError(message.stopDetails?.explanation)
)
return message;
return {
...message,
+269 -6
View File
@@ -630,7 +630,7 @@ describe("agentLoop with AgentMessage", () => {
expect(finalTurn.content).toContainEqual({ type: "text", text: "done after recovery" });
});
it("does not recover completed tool calls after non-stream transient errors", async () => {
it("runs completed tool calls after a transient stream JSON parse error", async () => {
const executedParams: Array<{ value: string }> = [];
const toolSchema = type({ value: "string" });
const tool: AgentTool<typeof toolSchema, { value: string }> = {
@@ -652,9 +652,9 @@ describe("agentLoop with AgentMessage", () => {
{
content: [{ type: "toolCall", id: "tool-1", name: "echo", arguments: { value: "hello" } }],
stopReason: "error",
errorMessage: "rate_limit_error",
errorMessage: "JSON Parse error: Unterminated string",
},
{ content: ["should not continue"] },
{ content: ["done after parse recovery"] },
],
});
const config: AgentLoopConfig = { model: mock.model, convertToLlm: identityConverter };
@@ -667,12 +667,275 @@ describe("agentLoop with AgentMessage", () => {
mock.stream,
).result();
expect(executedParams).toEqual([]);
expect(executedParams).toEqual([{ value: "hello" }]);
expect(mock.calls).toHaveLength(2);
expect(messages.map(message => message.role)).toEqual(["user", "assistant", "toolResult", "assistant"]);
const recoveredTurn = messages[1] as AssistantMessage;
expect(recoveredTurn.stopReason).toBe("toolUse");
expect(recoveredTurn.stopDetails?.type).toBe("stream_interrupted_after_content");
const finalTurn = messages[3] as AssistantMessage;
expect(finalTurn.content).toContainEqual({ type: "text", text: "done after parse recovery" });
});
it("recovers only completed calls when a stream parse error interrupts the next call", async () => {
const executedParams: Array<{ value: string }> = [];
const toolSchema = type({ value: "string" });
const tool: AgentTool<typeof toolSchema, { value: string }> = {
name: "echo",
label: "Echo",
description: "Echo tool",
parameters: toolSchema,
async execute(_toolCallId, params) {
executedParams.push(params);
return {
content: [{ type: "text", text: `echoed: ${params.value}` }],
details: { value: params.value },
};
},
};
const context: AgentContext = { systemPrompt: [""], messages: [], tools: [tool] };
const completedCall = {
type: "toolCall" as const,
id: "tool-complete",
name: "echo",
arguments: { value: "complete" },
};
const incompleteCall = {
type: "toolCall" as const,
id: "tool-incomplete",
name: "echo",
arguments: { value: "incomplete" },
};
const mock = createMockModel({ responses: [{ content: ["done after partial parse recovery"] }] });
let streamCalls = 0;
const streamFn: typeof mock.stream = (model, callContext, options) => {
streamCalls++;
if (streamCalls > 1) return mock.stream(model, callContext, options);
const stream = new AssistantMessageEventStream();
queueMicrotask(() => {
const partial: AssistantMessage = {
...createAssistantMessage([completedCall, incompleteCall], "error"),
errorMessage: "provider stream parse failed",
stopDetails: {
type: "parse_error",
explanation: "JSON Parse error: Unterminated string",
},
};
stream.push({ type: "start", partial });
stream.push({ type: "toolcall_end", contentIndex: 0, toolCall: completedCall, partial });
stream.push({ type: "toolcall_start", contentIndex: 1, partial });
stream.push({ type: "toolcall_delta", contentIndex: 1, delta: '{"value":"incomplete', partial });
stream.push({ type: "error", reason: "error", error: partial });
});
return stream;
};
const config: AgentLoopConfig = { model: mock.model, convertToLlm: identityConverter };
const messages = await agentLoop([createUserMessage("run echo")], context, config, undefined, streamFn).result();
expect(executedParams).toEqual([{ value: "complete" }]);
expect(streamCalls).toBe(2);
expect(mock.calls).toHaveLength(1);
expect(messages.map(message => message.role)).toEqual(["user", "assistant", "toolResult"]);
expect(messages.map(message => message.role)).toEqual(["user", "assistant", "toolResult", "assistant"]);
const recoveredTurn = messages[1] as AssistantMessage;
expect(recoveredTurn.stopReason).toBe("toolUse");
expect(recoveredTurn.content.filter(block => block.type === "toolCall").map(block => block.id)).toEqual([
"tool-complete",
]);
expect(messages.some(message => message.role === "toolResult" && message.toolCallId === "tool-incomplete")).toBe(
false,
);
});
it("does not recover a wrapped refusal after dropping an incomplete sibling", async () => {
const executedParams: Array<{ value: string }> = [];
const toolSchema = type({ value: "string" });
const tool: AgentTool<typeof toolSchema, { value: string }> = {
name: "echo",
label: "Echo",
description: "Echo tool",
parameters: toolSchema,
async execute(_toolCallId, params) {
executedParams.push(params);
return { content: [], details: { value: params.value } };
},
};
const context: AgentContext = { systemPrompt: [""], messages: [], tools: [tool] };
const completedCall = {
type: "toolCall" as const,
id: "tool-complete",
name: "echo",
arguments: { value: "complete" },
};
const incompleteCall = {
type: "toolCall" as const,
id: "tool-incomplete",
name: "echo",
arguments: { value: "incomplete" },
};
const mock = createMockModel({ responses: [{ content: ["should not continue"] }] });
let streamCalls = 0;
const streamFn: typeof mock.stream = (model, callContext, options) => {
streamCalls++;
if (streamCalls > 1) return mock.stream(model, callContext, options);
const stream = new AssistantMessageEventStream();
queueMicrotask(() => {
const partial: AssistantMessage = {
...createAssistantMessage([completedCall, incompleteCall], "error"),
errorMessage: "provider refused output",
stopDetails: { type: "refusal", explanation: "Unexpected end of JSON input" },
};
stream.push({ type: "start", partial });
stream.push({ type: "toolcall_end", contentIndex: 0, toolCall: completedCall, partial });
stream.push({ type: "toolcall_start", contentIndex: 1, partial });
stream.push({ type: "toolcall_delta", contentIndex: 1, delta: '{"value":"incomplete', partial });
stream.push({ type: "error", reason: "error", error: partial });
});
return stream;
};
const config: AgentLoopConfig = { model: mock.model, convertToLlm: identityConverter };
const messages = await agentLoop([createUserMessage("run echo")], context, config, undefined, streamFn).result();
expect(executedParams).toEqual([]);
expect(streamCalls).toBe(1);
expect(mock.calls).toHaveLength(0);
const errorTurn = messages[1] as AssistantMessage;
expect(errorTurn.stopReason).toBe("error");
expect(errorTurn.errorMessage).toBe("rate_limit_error");
expect(errorTurn.content.filter(block => block.type === "toolCall").map(block => block.id)).toEqual([
"tool-complete",
]);
expect(errorTurn.stopDetails).toEqual({
type: "stream_interrupted_after_content",
category: "refusal",
explanation: "Unexpected end of JSON input",
});
});
it("does not recover a mixed known and unknown completed tool turn", async () => {
const executedParams: Array<{ value: string }> = [];
const toolSchema = type({ value: "string" });
const tool: AgentTool<typeof toolSchema, { value: string }> = {
name: "echo",
label: "Echo",
description: "Echo tool",
parameters: toolSchema,
async execute(_toolCallId, params) {
executedParams.push(params);
return { content: [], details: { value: params.value } };
},
};
const context: AgentContext = { systemPrompt: [""], messages: [], tools: [tool] };
const mock = createMockModel({
responses: [
{
content: [
{ type: "toolCall", id: "tool-known", name: "echo", arguments: { value: "known" } },
{ type: "toolCall", id: "tool-unknown", name: "missing", arguments: { value: "unknown" } },
],
stopReason: "error",
errorMessage: "JSON Parse error: Unterminated string",
},
{ content: ["should not continue"] },
],
});
const config: AgentLoopConfig = { model: mock.model, convertToLlm: identityConverter };
const messages = await agentLoop(
[createUserMessage("run mixed tools")],
context,
config,
undefined,
mock.stream,
).result();
expect(executedParams).toEqual([]);
expect(mock.calls).toHaveLength(1);
const errorTurn = messages[1] as AssistantMessage;
expect(errorTurn.stopReason).toBe("error");
expect(errorTurn.errorMessage).toBe("JSON Parse error: Unterminated string");
});
it("does not recover content-only transient stream parse errors", async () => {
const context: AgentContext = { systemPrompt: [""], messages: [], tools: [] };
const mock = createMockModel({
responses: [
{
content: ["partial response"],
stopReason: "error",
errorMessage: "JSON Parse error: Unterminated string",
},
{ content: ["should not continue"] },
],
});
const config: AgentLoopConfig = { model: mock.model, convertToLlm: identityConverter };
const messages = await agentLoop(
[createUserMessage("answer once")],
context,
config,
undefined,
mock.stream,
).result();
expect(mock.calls).toHaveLength(1);
expect(messages.map(message => message.role)).toEqual(["user", "assistant"]);
const errorTurn = messages[1] as AssistantMessage;
expect(errorTurn.stopReason).toBe("error");
expect(errorTurn.errorMessage).toBe("JSON Parse error: Unterminated string");
});
it("does not recover ordinary transient errors or terminal stops quoting parse diagnostics", async () => {
for (const stopDetails of [
undefined,
{ type: "refusal", explanation: "Unexpected end of JSON input" },
{ type: "sensitive", explanation: "Unexpected end of JSON input" },
]) {
const executedParams: Array<{ value: string }> = [];
const toolSchema = type({ value: "string" });
const tool: AgentTool<typeof toolSchema, { value: string }> = {
name: "echo",
label: "Echo",
description: "Echo tool",
parameters: toolSchema,
async execute(_toolCallId, params) {
executedParams.push(params);
return {
content: [{ type: "text", text: `echoed: ${params.value}` }],
details: { value: params.value },
};
},
};
const context: AgentContext = { systemPrompt: [""], messages: [], tools: [tool] };
const mock = createMockModel({
responses: [
{
content: [{ type: "toolCall", id: "tool-1", name: "echo", arguments: { value: "hello" } }],
stopReason: "error",
stopDetails,
errorMessage: "rate_limit_error",
},
{ content: ["should not continue"] },
],
});
const config: AgentLoopConfig = { model: mock.model, convertToLlm: identityConverter };
const messages = await agentLoop(
[createUserMessage("run echo")],
context,
config,
undefined,
mock.stream,
).result();
expect(executedParams).toEqual([]);
expect(mock.calls).toHaveLength(1);
expect(messages.map(message => message.role)).toEqual(["user", "assistant", "toolResult"]);
const errorTurn = messages[1] as AssistantMessage;
expect(errorTurn.stopReason).toBe("error");
expect(errorTurn.errorMessage).toBe("rate_limit_error");
expect(errorTurn.stopDetails).toEqual(stopDetails);
}
});
it("labels the synthetic tool result for a provider-error turn as not-executed and preserves the upstream error", async () => {
+3
View File
@@ -505,10 +505,13 @@ export function stringify(id: number | undefined): string {
const STREAM_PARSE_TRUNCATION_PATTERN =
/unterminated string|unexpected end of json input|unexpected end of data|unexpected eof|end of file|eof while parsing|truncated/i;
const STREAM_PARSE_DIAGNOSTIC_PATTERN =
/(?:json parse error:\s*(?:unterminated string|unexpected end of json input|unexpected end of data|unexpected eof|end of file|eof while parsing|truncated)|json\.parse:\s*(?:unterminated string|unexpected end of data)|unexpected end of json input|unexpected eof|eof while parsing)/i;
const STREAM_EVENT_ORDER_PATTERN = /stream event order|before message_start/i;
/** Transient stream corruption where the response was truncated mid-JSON. */
export function isTransientStreamParseError(error: unknown): boolean {
if (typeof error === "string") return STREAM_PARSE_DIAGNOSTIC_PATTERN.test(error);
return error instanceof Error && STREAM_PARSE_TRUNCATION_PATTERN.test(error.message);
}
+14
View File
@@ -135,4 +135,18 @@ describe("aierr flag helpers", () => {
expect(AIError.retriable(AIError.create(AIError.Flag.Transient))).toBe(true);
expect(AIError.retriable(AIError.create(AIError.Flag.UsageLimit))).toBe(true);
});
it("recognizes explicit transient stream parse diagnostics without broad text false positives", () => {
const message = "JSON Parse error: Unterminated string";
expect(AIError.isTransientStreamParseError(new Error(message))).toBe(true);
expect(AIError.isTransientStreamParseError(new Error("response truncated"))).toBe(true);
expect(AIError.isTransientStreamParseError(message)).toBe(true);
expect(AIError.isTransientStreamParseError("Unexpected end of JSON input")).toBe(true);
expect(AIError.isTransientStreamParseError("JSON.parse: unexpected end of data at line 1 column 8")).toBe(true);
expect(AIError.isTransientStreamParseError("Unexpected EOF")).toBe(true);
expect(AIError.isTransientStreamParseError("EOF while parsing")).toBe(true);
expect(AIError.isTransientStreamParseError("truncated")).toBe(false);
expect(AIError.isTransientStreamParseError("end of file")).toBe(false);
expect(AIError.isTransientStreamParseError("Unexpected token in JSON")).toBe(false);
});
});