diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index d4ccd9e9d..e3c094b2c 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,14 +1,15 @@ # Changelog ## [Unreleased] - ### Added +- Added `tryEnforceStrictSchema` function that gracefully downgrades to non-strict mode when schema enforcement fails, enabling better compatibility with malformed or circular schemas - Added `sanitizeSchemaForStrictMode` function to normalize JSON schemas by stripping non-structural keywords, converting `const` to `enum`, and expanding type arrays into `anyOf` variants - Added Kilo Gateway provider support with OpenAI-compatible model discovery, OAuth `/login kilo`, and `KILO_API_KEY` environment variable support ([#193](https://github.com/can1357/oh-my-pi/issues/193)) ### Changed +- Changed strict mode handling in OpenAI providers to use `tryEnforceStrictSchema` for safer schema enforcement with automatic fallback to non-strict mode - Enhanced `enforceStrictSchema` to properly handle schemas with type arrays containing `object` (e.g., `type: ["object", "null"]`) ### Fixed diff --git a/packages/ai/src/providers/openai-codex-responses.ts b/packages/ai/src/providers/openai-codex-responses.ts index 6b724c3cb..8627595e7 100644 --- a/packages/ai/src/providers/openai-codex-responses.ts +++ b/packages/ai/src/providers/openai-codex-responses.ts @@ -32,7 +32,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream"; import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector"; import { parseStreamingJson } from "../utils/json-parse"; import { sanitizeSurrogates } from "../utils/sanitize-unicode"; -import { enforceStrictSchema, NO_STRICT } from "../utils/typebox-helpers"; +import { NO_STRICT, tryEnforceStrictSchema } from "../utils/typebox-helpers"; import { CODEX_BASE_URL, JWT_CLAIM_PATH, @@ -1706,15 +1706,17 @@ function convertTools(tools: Tool[]): Array<{ }> { return tools.map(tool => { const strict = !NO_STRICT && tool.strict; + const baseParameters = tool.parameters as unknown as Record; + const strictResult = strict ? tryEnforceStrictSchema(baseParameters) : { schema: baseParameters, strict: false }; + const parameters = strictResult.schema; + const effectiveStrict = strict && strictResult.strict; return { type: "function", name: tool.name, description: tool.description || "", - parameters: strict - ? enforceStrictSchema(tool.parameters as unknown as Record) - : (tool.parameters as unknown as Record), + parameters, // Only include strict if provider supports it. Some reject unknown fields. - ...(strict && { strict: true }), + ...(effectiveStrict && { strict: true }), }; }); } diff --git a/packages/ai/src/providers/openai-completions.ts b/packages/ai/src/providers/openai-completions.ts index 4e79db2b3..4f8eeb7a9 100644 --- a/packages/ai/src/providers/openai-completions.ts +++ b/packages/ai/src/providers/openai-completions.ts @@ -33,7 +33,7 @@ import { parseStreamingJson } from "../utils/json-parse"; import { getKimiCommonHeaders } from "../utils/oauth/kimi"; import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { mapToOpenAICompletionsToolChoice } from "../utils/tool-choice"; -import { enforceStrictSchema, NO_STRICT } from "../utils/typebox-helpers"; +import { NO_STRICT, tryEnforceStrictSchema } from "../utils/typebox-helpers"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers"; import { transformMessages } from "./transform-messages"; @@ -977,16 +977,18 @@ export function convertMessages( function convertTools(tools: Tool[], compat: ResolvedOpenAICompat): OpenAI.Chat.Completions.ChatCompletionTool[] { return tools.map(tool => { const strict = !NO_STRICT && compat.supportsStrictMode !== false && tool.strict !== false; + const baseParameters = tool.parameters as unknown as Record; + const strictResult = strict ? tryEnforceStrictSchema(baseParameters) : { schema: baseParameters, strict: false }; + const parameters = strictResult.schema; + const effectiveStrict = strict && strictResult.strict; return { type: "function", function: { name: tool.name, description: tool.description || "", - parameters: strict - ? enforceStrictSchema(tool.parameters as unknown as Record) - : (tool.parameters as unknown as Record), + parameters, // Only include strict if provider supports it. Some reject unknown fields. - ...(strict && { strict: true }), + ...(effectiveStrict && { strict: true }), }, }; }); diff --git a/packages/ai/src/providers/openai-responses.ts b/packages/ai/src/providers/openai-responses.ts index c5e8d9e24..0a74c627e 100644 --- a/packages/ai/src/providers/openai-responses.ts +++ b/packages/ai/src/providers/openai-responses.ts @@ -34,7 +34,7 @@ import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-ins import { parseStreamingJson } from "../utils/json-parse"; import { sanitizeSurrogates } from "../utils/sanitize-unicode"; import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice"; -import { enforceStrictSchema, NO_STRICT } from "../utils/typebox-helpers"; +import { NO_STRICT, tryEnforceStrictSchema } from "../utils/typebox-helpers"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers"; import { transformMessages } from "./transform-messages"; @@ -694,15 +694,16 @@ function convertMessages( function convertTools(tools: Tool[], strictMode: boolean): OpenAITool[] { return tools.map(tool => { const strict = !NO_STRICT && strictMode && tool.strict !== false; - const parameters = strict - ? enforceStrictSchema(tool.parameters as unknown as Record) - : (tool.parameters as unknown as Record); + const baseParameters = tool.parameters as unknown as Record; + const strictResult = strict ? tryEnforceStrictSchema(baseParameters) : { schema: baseParameters, strict: false }; + const parameters = strictResult.schema; + const effectiveStrict = strict && strictResult.strict; return { type: "function", name: tool.name, description: tool.description || "", parameters, - ...(strict && { strict: true }), + ...(effectiveStrict && { strict: true }), } as OpenAITool; }); } diff --git a/packages/ai/src/utils/typebox-helpers.ts b/packages/ai/src/utils/typebox-helpers.ts index f3f1c8405..82782ced5 100644 --- a/packages/ai/src/utils/typebox-helpers.ts +++ b/packages/ai/src/utils/typebox-helpers.ts @@ -218,3 +218,14 @@ export function enforceStrictSchema(schema: Record): Record): { + schema: Record; + strict: boolean; +} { + try { + return { schema: enforceStrictSchema(schema), strict: true }; + } catch { + return { schema, strict: false }; + } +} diff --git a/packages/ai/test/typebox-helpers.test.ts b/packages/ai/test/typebox-helpers.test.ts index a5818d92b..3998c0f71 100644 --- a/packages/ai/test/typebox-helpers.test.ts +++ b/packages/ai/test/typebox-helpers.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "bun:test"; -import { enforceStrictSchema, sanitizeSchemaForStrictMode } from "@oh-my-pi/pi-ai/utils/typebox-helpers"; +import { + enforceStrictSchema, + sanitizeSchemaForStrictMode, + tryEnforceStrictSchema, +} from "@oh-my-pi/pi-ai/utils/typebox-helpers"; import { Type } from "@sinclair/typebox"; describe("sanitizeSchemaForStrictMode", () => { @@ -169,3 +173,32 @@ describe("enforceStrictSchema", () => { expect(properties.value.type).toBe("string"); }); }); + +describe("tryEnforceStrictSchema", () => { + it("downgrades to non-strict mode when strict enforcement throws", () => { + const circularSchema: Record = { + type: "object", + properties: {}, + }; + (circularSchema.properties as Record).self = circularSchema; + + const result = tryEnforceStrictSchema(circularSchema); + + expect(result.strict).toBe(false); + expect(result.schema).toBe(circularSchema); + }); + + it("keeps strict mode enabled for valid schemas", () => { + const schema = { + type: "object", + properties: { value: { type: "string" } }, + required: ["value"], + } as Record; + + const result = tryEnforceStrictSchema(schema); + + expect(result.strict).toBe(true); + expect(result.schema.additionalProperties).toBe(false); + expect(result.schema.required).toEqual(["value"]); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 9fb7f1785..8912c48ef 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## [Unreleased] + ### Breaking Changes - Removed `preloadedSkills` option from `CreateAgentSessionOptions`; skills are no longer inlined into system prompts @@ -10,6 +11,8 @@ ### Changed +- Modified schema validation to allow non-conforming output on second validation failure, enabling recovery from strict schema constraints after initial rejection +- Updated JTD-to-TypeScript conversion to gracefully fall back to 'unknown' type when conversion fails, preventing template rendering errors - Changed JTD-to-JSON Schema conversion to normalize nested JTD fragments within JSON Schema nodes, enabling mixed schema definitions - Changed output schema validation to gracefully fall back to unconstrained object when schema is invalid, instead of rejecting submissions - Changed schema sanitization to remove strict-mode incompatible constraints (minLength, pattern, etc.) from tool parameters while preserving them for runtime validation @@ -22,6 +25,8 @@ ### Fixed +- Fixed handling of circular and deeply nested output schemas to prevent stack overflow and enable successful result submission with fallback unconstrained schema +- Fixed processing of non-object output schemas (arrays, primitives, booleans) to accept valid result submissions without blocking - Fixed handling of mixed JTD and JSON Schema output definitions to properly convert all nested JTD elements (e.g., `elements` → `items`, `int32` → `integer`) - Fixed strict schema generation for output schemas with only required fields, enabling proper Claude API compatibility - Fixed handling of union type schemas (e.g., object|null) to normalize them into strict-mode compatible variants diff --git a/packages/coding-agent/src/config/prompt-templates.ts b/packages/coding-agent/src/config/prompt-templates.ts index 261baa1ac..1ae485c8b 100644 --- a/packages/coding-agent/src/config/prompt-templates.ts +++ b/packages/coding-agent/src/config/prompt-templates.ts @@ -225,7 +225,13 @@ handlebars.registerHelper("includes", (collection: unknown, item: unknown): bool */ handlebars.registerHelper("not", (value: unknown): boolean => !value); -handlebars.registerHelper("jtdToTypeScript", (schema: unknown): string => jtdToTypeScript(schema)); +handlebars.registerHelper("jtdToTypeScript", (schema: unknown): string => { + try { + return jtdToTypeScript(schema); + } catch { + return "unknown"; + } +}); handlebars.registerHelper("jsonStringify", (value: unknown): string => JSON.stringify(value)); diff --git a/packages/coding-agent/src/tools/submit-result.ts b/packages/coding-agent/src/tools/submit-result.ts index f24b1b26c..d2abce41b 100644 --- a/packages/coding-agent/src/tools/submit-result.ts +++ b/packages/coding-agent/src/tools/submit-result.ts @@ -52,6 +52,29 @@ function formatAjvErrors(errors: ErrorObject[] | null | undefined): string { .join("; "); } +function splitRootDefinitions(schema: Record): { + schemaWithoutDefinitions: Record; + rootDefinitions?: Record; +} { + const schemaWithoutDefinitions = { ...schema }; + const rootDefinitions: Record = {}; + if (typeof schemaWithoutDefinitions.$defs === "object" && schemaWithoutDefinitions.$defs !== null) { + if (!Array.isArray(schemaWithoutDefinitions.$defs)) { + rootDefinitions.$defs = schemaWithoutDefinitions.$defs; + } + delete schemaWithoutDefinitions.$defs; + } + if (typeof schemaWithoutDefinitions.definitions === "object" && schemaWithoutDefinitions.definitions !== null) { + if (!Array.isArray(schemaWithoutDefinitions.definitions)) { + rootDefinitions.definitions = schemaWithoutDefinitions.definitions; + } + delete schemaWithoutDefinitions.definitions; + } + return Object.keys(rootDefinitions).length > 0 + ? { schemaWithoutDefinitions, rootDefinitions } + : { schemaWithoutDefinitions }; +} + export class SubmitResultTool implements AgentTool { readonly name = "submit_result"; readonly label = "Submit Result"; @@ -62,65 +85,96 @@ export class SubmitResultTool implements AgentTool strict = true; readonly #validate?: ValidateFunction; + #schemaValidationFailures = 0; constructor(session: ToolSession) { - const schemaResult = normalizeSchema(session.outputSchema); - // Convert JTD to JSON Schema if needed (auto-detected) - const normalizedSchema = - schemaResult.normalized !== undefined ? jtdToJsonSchema(schemaResult.normalized) : undefined; - let schemaError = schemaResult.error; + const createParameters = (dataSchema: TSchema, rootDefinitions?: Record): TSchema => { + const baseParameters = Type.Object( + { + result: Type.Union([ + Type.Object({ data: dataSchema }, { description: "Successfully completed the task" }), + Type.Object({ + error: Type.String({ description: "Error message when the task cannot be completed" }), + }), + ]), + }, + { + additionalProperties: false, + description: "Submit either `data` for success or `error` for failure", + }, + ) as unknown as Record; + if (!rootDefinitions) return baseParameters as TSchema; + return Type.Unsafe({ ...baseParameters, ...rootDefinitions }); + }; - if (normalizedSchema !== undefined && !schemaError) { - try { - this.#validate = ajv.compile(normalizedSchema as any); - } catch (err) { - schemaError = err instanceof Error ? err.message : String(err); - } - } - - const schemaHint = formatSchema(normalizedSchema ?? session.outputSchema); - - const schemaDescription = schemaError - ? `Structured JSON output (output schema invalid; accepting unconstrained object): ${schemaError}` - : `Structured output matching the schema:\n${schemaHint}`; - const sanitizedSchema = - !schemaError && - normalizedSchema != null && - typeof normalizedSchema === "object" && - !Array.isArray(normalizedSchema) - ? sanitizeSchemaForStrictMode(normalizedSchema as Record) - : !schemaError && normalizedSchema === true - ? {} - : undefined; - - const dataSchema = - sanitizedSchema !== undefined - ? Type.Unsafe({ - ...sanitizedSchema, - description: schemaDescription, - }) - : Type.Record(Type.String(), Type.Any(), { - description: schemaError ? schemaDescription : "Structured JSON output (no schema specified)", - }); - this.parameters = Type.Object( - { - result: Type.Union([ - Type.Object({ data: dataSchema }, { description: "Successfully completed the task" }), - Type.Object({ error: Type.String({ description: "Error message when the task cannot be completed" }) }), - ]), - }, - { - additionalProperties: false, - description: "Submit either `data` for success or `error` for failure", - }, - ); + let rootDefinitions: Record | undefined; + let validate: ValidateFunction | undefined; + let dataSchema: TSchema; + let parameters: TSchema; + let strict = true; try { - const strictParameters = enforceStrictSchema(this.parameters as unknown as Record); + const schemaResult = normalizeSchema(session.outputSchema); + // Convert JTD to JSON Schema if needed (auto-detected) + const normalizedSchema = + schemaResult.normalized !== undefined ? jtdToJsonSchema(schemaResult.normalized) : undefined; + let schemaError = schemaResult.error; + + if (!schemaError && normalizedSchema === false) { + schemaError = "boolean false schema rejects all outputs"; + } + + if (normalizedSchema !== undefined && normalizedSchema !== false && !schemaError) { + try { + validate = ajv.compile(normalizedSchema as Record | boolean); + } catch (err) { + schemaError = err instanceof Error ? err.message : String(err); + } + } + + const schemaHint = formatSchema(normalizedSchema ?? session.outputSchema); + const schemaDescription = schemaError + ? `Structured JSON output (output schema invalid; accepting unconstrained object): ${schemaError}` + : `Structured output matching the schema:\n${schemaHint}`; + const sanitizedSchema = + !schemaError && + normalizedSchema != null && + typeof normalizedSchema === "object" && + !Array.isArray(normalizedSchema) + ? sanitizeSchemaForStrictMode(normalizedSchema as Record) + : !schemaError && normalizedSchema === true + ? {} + : undefined; + + if (sanitizedSchema !== undefined) { + const schemaWithDescription = { + ...sanitizedSchema, + description: schemaDescription, + }; + const splitSchema = splitRootDefinitions(schemaWithDescription); + rootDefinitions = splitSchema.rootDefinitions; + dataSchema = Type.Unsafe(splitSchema.schemaWithoutDefinitions); + } else { + dataSchema = Type.Record(Type.String(), Type.Any(), { + description: schemaError ? schemaDescription : "Structured JSON output (no schema specified)", + }); + } + parameters = createParameters(dataSchema, rootDefinitions); + const strictParameters = enforceStrictSchema(parameters as unknown as Record); JSON.stringify(strictParameters); - } catch { - this.strict = false; + } catch (err) { + const errorMsg = err instanceof Error ? err.message : String(err); + dataSchema = Type.Record(Type.String(), Type.Any(), { + description: `Structured JSON output (schema processing failed: ${errorMsg})`, + }); + parameters = createParameters(dataSchema); + validate = undefined; + strict = false; } + + this.#validate = validate; + this.parameters = parameters; + this.strict = strict; } async execute( @@ -148,17 +202,26 @@ export class SubmitResultTool implements AgentTool } const status = errorMessage !== undefined ? "aborted" : "success"; + let schemaValidationOverridden = false; if (status === "success") { if (data === undefined || data === null) { throw new Error("data is required when submit_result indicates success"); } if (this.#validate && !this.#validate(data)) { - throw new Error(`Output does not match schema: ${formatAjvErrors(this.#validate.errors)}`); + this.#schemaValidationFailures++; + if (this.#schemaValidationFailures <= 1) { + throw new Error(`Output does not match schema: ${formatAjvErrors(this.#validate.errors)}`); + } + schemaValidationOverridden = true; } } - const responseText = status === "aborted" ? `Task aborted: ${errorMessage}` : "Result submitted."; - + const responseText = + status === "aborted" + ? `Task aborted: ${errorMessage}` + : schemaValidationOverridden + ? `Result submitted (schema validation overridden after ${this.#schemaValidationFailures} failed attempt(s)).` + : "Result submitted."; return { content: [{ type: "text", text: responseText }], details: { data, status, error: errorMessage }, diff --git a/packages/coding-agent/test/agent-session-user-shortcut-hooks.test.ts b/packages/coding-agent/test/agent-session-user-shortcut-hooks.test.ts index da14620be..f20544555 100644 --- a/packages/coding-agent/test/agent-session-user-shortcut-hooks.test.ts +++ b/packages/coding-agent/test/agent-session-user-shortcut-hooks.test.ts @@ -60,6 +60,10 @@ describe("AgentSession user shortcut hooks", () => { exitCode: 0, cancelled: false, truncated: false, + totalLines: 1, + totalBytes: 18, + outputLines: 1, + outputBytes: 18, }; const emitUserBash = vi.fn().mockResolvedValue({ result: replacement }); const extensionRunner = { @@ -93,7 +97,12 @@ describe("AgentSession user shortcut hooks", () => { exitCode: 0, cancelled: false, truncated: false, + totalLines: 1, + totalBytes: 20, + outputLines: 1, + outputBytes: 20, displayOutputs: [], + stdinRequested: false, }; const emitUserPython = vi.fn().mockResolvedValue({ result: replacement }); const extensionRunner = { @@ -132,13 +141,22 @@ describe("AgentSession user shortcut hooks", () => { exitCode: 0, cancelled: false, truncated: false, + totalLines: 1, + totalBytes: 13, + outputLines: 1, + outputBytes: 13, }); vi.spyOn(pythonExecutor, "executePython").mockResolvedValue({ output: "python fallback", exitCode: 0, cancelled: false, truncated: false, + totalLines: 1, + totalBytes: 15, + outputLines: 1, + outputBytes: 15, displayOutputs: [], + stdinRequested: false, }); createSession(extensionRunner); @@ -149,7 +167,11 @@ describe("AgentSession user shortcut hooks", () => { expect(pythonResult.output).toBe("python fallback"); expect(bashExecutor.executeBash).toHaveBeenCalledTimes(1); expect(pythonExecutor.executePython).toHaveBeenCalledTimes(1); - expect(session.messages.some(message => message.role === "bashExecution" && message.excludeFromContext === true)).toBe(true); - expect(session.messages.some(message => message.role === "pythonExecution" && message.excludeFromContext === false)).toBe(true); + expect( + session.messages.some(message => message.role === "bashExecution" && message.excludeFromContext === true), + ).toBe(true); + expect( + session.messages.some(message => message.role === "pythonExecution" && message.excludeFromContext === false), + ).toBe(true); }); }); diff --git a/packages/coding-agent/test/tools/submit-result.test.ts b/packages/coding-agent/test/tools/submit-result.test.ts index 9d132fcac..23a1e5ab6 100644 --- a/packages/coding-agent/test/tools/submit-result.test.ts +++ b/packages/coding-agent/test/tools/submit-result.test.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "bun:test"; +import type { Tool, ToolCall } from "@oh-my-pi/pi-ai/types"; import { enforceStrictSchema } from "@oh-my-pi/pi-ai/utils/typebox-helpers"; +import { validateToolArguments } from "@oh-my-pi/pi-ai/utils/validation"; import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings"; import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools"; import { SubmitResultTool } from "@oh-my-pi/pi-coding-agent/tools/submit-result"; @@ -161,6 +163,66 @@ describe("SubmitResultTool", () => { tool.execute("call-mixed-invalid", { result: { data: { results: [{ issue: "185" }] } } } as never), ).rejects.toThrow("Output does not match schema"); }); + it("supports $defs/$ref output schemas in tool-argument validation and degrades after first runtime failure", async () => { + const outputSchema = { + $defs: { + A: { + type: "object", + properties: { + kind: { const: "A" }, + token: { type: "string", minLength: 10 }, + }, + required: ["kind", "token"], + additionalProperties: false, + }, + }, + anyOf: [ + { $ref: "#/$defs/A" }, + { + type: "object", + properties: { + kind: { const: "B" }, + n: { type: "integer", minimum: 10 }, + }, + required: ["kind", "n"], + additionalProperties: false, + }, + ], + }; + const tool = new SubmitResultTool(createSession({ outputSchema })); + const parametersRecord = tool.parameters as unknown as Record; + expect(toRecord(parametersRecord.$defs).A).toBeDefined(); + expect(getSuccessDataSchema(parametersRecord).$defs).toBeUndefined(); + + const toolDefinition: Tool = { + name: tool.name, + description: tool.description, + parameters: tool.parameters, + }; + const firstCall: ToolCall = { + type: "toolCall", + id: "call-ref-1", + name: tool.name, + arguments: { result: { data: { kind: "A", token: "x" } } }, + }; + const firstArgs = validateToolArguments(toolDefinition, firstCall); + await expect(tool.execute("call-ref-1", firstArgs as never)).rejects.toThrow("Output does not match schema"); + + const secondCall: ToolCall = { + type: "toolCall", + id: "call-ref-2", + name: tool.name, + arguments: { result: { data: { kind: "A", token: "x" } } }, + }; + const secondArgs = validateToolArguments(toolDefinition, secondCall); + const secondResult = await tool.execute("call-ref-2", secondArgs as never); + expect(secondResult.content).toEqual([ + { + type: "text", + text: "Result submitted (schema validation overridden after 2 failed attempt(s)).", + }, + ]); + }); it("falls back to unconstrained object data when output schema is invalid", async () => { const tool = new SubmitResultTool( createSession({ @@ -189,6 +251,69 @@ describe("SubmitResultTool", () => { error: undefined, }); }); + it("falls back to unconstrained data schema when output schema is circular", async () => { + const circularSchema: Record = { type: "object" }; + circularSchema.self = circularSchema; + + const tool = new SubmitResultTool(createSession({ outputSchema: circularSchema })); + const dataSchema = getSuccessDataSchema(tool.parameters as unknown as Record); + + expect(tool.strict).toBe(false); + expect(dataSchema.type).toBe("object"); + + const result = await tool.execute("call-circular-schema", { result: { data: { ok: true } } } as never); + expect(result.details).toEqual({ data: { ok: true }, status: "success", error: undefined }); + }); + + it("falls back to unconstrained data schema when output schema is deeply nested", async () => { + const buildDeepSchema = (depth: number): Record => { + const root: Record = { + type: "object", + properties: {}, + required: ["next"], + }; + let current = root; + + for (let i = 0; i < depth; i++) { + const next: Record = { + type: "object", + properties: {}, + required: ["next"], + }; + const currentProperties = toRecord(current.properties); + currentProperties.next = next; + current.properties = currentProperties; + current = next; + } + + current.properties = { value: { type: "string" } }; + current.required = ["value"]; + return root; + }; + + const tool = new SubmitResultTool(createSession({ outputSchema: buildDeepSchema(20_000) })); + const dataSchema = getSuccessDataSchema(tool.parameters as unknown as Record); + + expect(tool.strict).toBe(false); + expect(dataSchema.type).toBe("object"); + + const result = await tool.execute("call-deep-schema", { result: { data: { nested: true } } } as never); + expect(result.details).toEqual({ data: { nested: true }, status: "success", error: undefined }); + }); + + it("handles non-object output schemas without blocking successful result submission", async () => { + for (const outputSchema of [[], 123, false]) { + const tool = new SubmitResultTool(createSession({ outputSchema })); + const result = await tool.execute("call-non-object-schema", { + result: { data: { value: outputSchema } }, + } as never); + expect(result.details).toEqual({ + data: { value: outputSchema }, + status: "success", + error: undefined, + }); + } + }); it("keeps runtime validation against the original output schema", async () => { const outputSchema = { type: "object", @@ -213,6 +338,81 @@ describe("SubmitResultTool", () => { expect(result.details).toEqual({ data: { token: "abcd" }, status: "success", error: undefined }); }); + it("throws on first schema validation failure and accepts non-conforming data on second failure", async () => { + const outputSchema = { + type: "object", + properties: { + token: { + type: "string", + minLength: 3, + }, + }, + required: ["token"], + }; + const tool = new SubmitResultTool(createSession({ outputSchema })); + + await expect(tool.execute("call-short-1", { result: { data: { token: "ab" } } } as never)).rejects.toThrow( + "Output does not match schema", + ); + + const secondResult = await tool.execute("call-short-2", { result: { data: { token: "ab" } } } as never); + expect(secondResult.details).toEqual({ data: { token: "ab" }, status: "success", error: undefined }); + expect(secondResult.content).toEqual([ + { + type: "text", + text: "Result submitted (schema validation overridden after 2 failed attempt(s)).", + }, + ]); + }); + + it("keeps schema degradation counter at zero when submissions are valid", async () => { + const outputSchema = { + type: "object", + properties: { + token: { + type: "string", + minLength: 3, + }, + }, + required: ["token"], + }; + const tool = new SubmitResultTool(createSession({ outputSchema })); + + const firstResult = await tool.execute("call-valid-1", { result: { data: { token: "abcd" } } } as never); + expect(firstResult.content).toEqual([{ type: "text", text: "Result submitted." }]); + + const secondResult = await tool.execute("call-valid-2", { result: { data: { token: "abcde" } } } as never); + expect(secondResult.content).toEqual([{ type: "text", text: "Result submitted." }]); + + await expect( + tool.execute("call-invalid-after-valid", { result: { data: { token: "ab" } } } as never), + ).rejects.toThrow("Output does not match schema"); + }); + + it("still throws structural errors after schema validation has been degraded", async () => { + const outputSchema = { + type: "object", + properties: { + token: { + type: "string", + minLength: 3, + }, + }, + required: ["token"], + }; + const tool = new SubmitResultTool(createSession({ outputSchema })); + + await expect(tool.execute("call-struct-1", { result: { data: { token: "ab" } } } as never)).rejects.toThrow( + "Output does not match schema", + ); + await expect( + tool.execute("call-struct-2", { result: { data: { token: "ab" } } } as never), + ).resolves.toBeDefined(); + + await expect(tool.execute("call-struct-missing", {} as never)).rejects.toThrow( + "result must be an object containing either data or error", + ); + }); it("rejects submissions without a result object", async () => { const tool = new SubmitResultTool(createSession()); await expect(tool.execute("call-3", {} as never)).rejects.toThrow(