feat(coding-agent): added graceful schema validation fallback for AI providers
- Added graceful schema validation fallback mechanism that degrades to unconstrained schemas on repeated validation failures. - Enhanced schema enforcement across AI providers to use try-catch pattern with automatic fallback to non-strict mode on validation errors. - Improved handling of circular, deeply nested, and non-object output schemas with stack overflow prevention and type conversion fallbacks. - Added `tryEnforceStrictSchema()` utility function providing error-resilient schema validation with strict mode flag tracking.
This commit is contained in:
can1357
@@ -1,14 +1,15 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added `tryEnforceStrictSchema` function that gracefully downgrades to non-strict mode when schema enforcement fails, enabling better compatibility with malformed or circular schemas
|
||||
- Added `sanitizeSchemaForStrictMode` function to normalize JSON schemas by stripping non-structural keywords, converting `const` to `enum`, and expanding type arrays into `anyOf` variants
|
||||
- Added Kilo Gateway provider support with OpenAI-compatible model discovery, OAuth `/login kilo`, and `KILO_API_KEY` environment variable support ([#193](https://github.com/can1357/oh-my-pi/issues/193))
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed strict mode handling in OpenAI providers to use `tryEnforceStrictSchema` for safer schema enforcement with automatic fallback to non-strict mode
|
||||
- Enhanced `enforceStrictSchema` to properly handle schemas with type arrays containing `object` (e.g., `type: ["object", "null"]`)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -32,7 +32,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream";
|
||||
import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-inspector";
|
||||
import { parseStreamingJson } from "../utils/json-parse";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
|
||||
import { enforceStrictSchema, NO_STRICT } from "../utils/typebox-helpers";
|
||||
import { NO_STRICT, tryEnforceStrictSchema } from "../utils/typebox-helpers";
|
||||
import {
|
||||
CODEX_BASE_URL,
|
||||
JWT_CLAIM_PATH,
|
||||
@@ -1706,15 +1706,17 @@ function convertTools(tools: Tool[]): Array<{
|
||||
}> {
|
||||
return tools.map(tool => {
|
||||
const strict = !NO_STRICT && tool.strict;
|
||||
const baseParameters = tool.parameters as unknown as Record<string, unknown>;
|
||||
const strictResult = strict ? tryEnforceStrictSchema(baseParameters) : { schema: baseParameters, strict: false };
|
||||
const parameters = strictResult.schema;
|
||||
const effectiveStrict = strict && strictResult.strict;
|
||||
return {
|
||||
type: "function",
|
||||
name: tool.name,
|
||||
description: tool.description || "",
|
||||
parameters: strict
|
||||
? enforceStrictSchema(tool.parameters as unknown as Record<string, unknown>)
|
||||
: (tool.parameters as unknown as Record<string, unknown>),
|
||||
parameters,
|
||||
// Only include strict if provider supports it. Some reject unknown fields.
|
||||
...(strict && { strict: true }),
|
||||
...(effectiveStrict && { strict: true }),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ import { parseStreamingJson } from "../utils/json-parse";
|
||||
import { getKimiCommonHeaders } from "../utils/oauth/kimi";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
|
||||
import { mapToOpenAICompletionsToolChoice } from "../utils/tool-choice";
|
||||
import { enforceStrictSchema, NO_STRICT } from "../utils/typebox-helpers";
|
||||
import { NO_STRICT, tryEnforceStrictSchema } from "../utils/typebox-helpers";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers";
|
||||
import { transformMessages } from "./transform-messages";
|
||||
|
||||
@@ -977,16 +977,18 @@ export function convertMessages(
|
||||
function convertTools(tools: Tool[], compat: ResolvedOpenAICompat): OpenAI.Chat.Completions.ChatCompletionTool[] {
|
||||
return tools.map(tool => {
|
||||
const strict = !NO_STRICT && compat.supportsStrictMode !== false && tool.strict !== false;
|
||||
const baseParameters = tool.parameters as unknown as Record<string, unknown>;
|
||||
const strictResult = strict ? tryEnforceStrictSchema(baseParameters) : { schema: baseParameters, strict: false };
|
||||
const parameters = strictResult.schema;
|
||||
const effectiveStrict = strict && strictResult.strict;
|
||||
return {
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description || "",
|
||||
parameters: strict
|
||||
? enforceStrictSchema(tool.parameters as unknown as Record<string, unknown>)
|
||||
: (tool.parameters as unknown as Record<string, unknown>),
|
||||
parameters,
|
||||
// Only include strict if provider supports it. Some reject unknown fields.
|
||||
...(strict && { strict: true }),
|
||||
...(effectiveStrict && { strict: true }),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
@@ -34,7 +34,7 @@ import { finalizeErrorMessage, type RawHttpRequestDump } from "../utils/http-ins
|
||||
import { parseStreamingJson } from "../utils/json-parse";
|
||||
import { sanitizeSurrogates } from "../utils/sanitize-unicode";
|
||||
import { mapToOpenAIResponsesToolChoice } from "../utils/tool-choice";
|
||||
import { enforceStrictSchema, NO_STRICT } from "../utils/typebox-helpers";
|
||||
import { NO_STRICT, tryEnforceStrictSchema } from "../utils/typebox-helpers";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers";
|
||||
import { transformMessages } from "./transform-messages";
|
||||
|
||||
@@ -694,15 +694,16 @@ function convertMessages(
|
||||
function convertTools(tools: Tool[], strictMode: boolean): OpenAITool[] {
|
||||
return tools.map(tool => {
|
||||
const strict = !NO_STRICT && strictMode && tool.strict !== false;
|
||||
const parameters = strict
|
||||
? enforceStrictSchema(tool.parameters as unknown as Record<string, unknown>)
|
||||
: (tool.parameters as unknown as Record<string, unknown>);
|
||||
const baseParameters = tool.parameters as unknown as Record<string, unknown>;
|
||||
const strictResult = strict ? tryEnforceStrictSchema(baseParameters) : { schema: baseParameters, strict: false };
|
||||
const parameters = strictResult.schema;
|
||||
const effectiveStrict = strict && strictResult.strict;
|
||||
return {
|
||||
type: "function",
|
||||
name: tool.name,
|
||||
description: tool.description || "",
|
||||
parameters,
|
||||
...(strict && { strict: true }),
|
||||
...(effectiveStrict && { strict: true }),
|
||||
} as OpenAITool;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -218,3 +218,14 @@ export function enforceStrictSchema(schema: Record<string, unknown>): Record<str
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function tryEnforceStrictSchema(schema: Record<string, unknown>): {
|
||||
schema: Record<string, unknown>;
|
||||
strict: boolean;
|
||||
} {
|
||||
try {
|
||||
return { schema: enforceStrictSchema(schema), strict: true };
|
||||
} catch {
|
||||
return { schema, strict: false };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { enforceStrictSchema, sanitizeSchemaForStrictMode } from "@oh-my-pi/pi-ai/utils/typebox-helpers";
|
||||
import {
|
||||
enforceStrictSchema,
|
||||
sanitizeSchemaForStrictMode,
|
||||
tryEnforceStrictSchema,
|
||||
} from "@oh-my-pi/pi-ai/utils/typebox-helpers";
|
||||
import { Type } from "@sinclair/typebox";
|
||||
|
||||
describe("sanitizeSchemaForStrictMode", () => {
|
||||
@@ -169,3 +173,32 @@ describe("enforceStrictSchema", () => {
|
||||
expect(properties.value.type).toBe("string");
|
||||
});
|
||||
});
|
||||
|
||||
describe("tryEnforceStrictSchema", () => {
|
||||
it("downgrades to non-strict mode when strict enforcement throws", () => {
|
||||
const circularSchema: Record<string, unknown> = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
};
|
||||
(circularSchema.properties as Record<string, unknown>).self = circularSchema;
|
||||
|
||||
const result = tryEnforceStrictSchema(circularSchema);
|
||||
|
||||
expect(result.strict).toBe(false);
|
||||
expect(result.schema).toBe(circularSchema);
|
||||
});
|
||||
|
||||
it("keeps strict mode enabled for valid schemas", () => {
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: { value: { type: "string" } },
|
||||
required: ["value"],
|
||||
} as Record<string, unknown>;
|
||||
|
||||
const result = tryEnforceStrictSchema(schema);
|
||||
|
||||
expect(result.strict).toBe(true);
|
||||
expect(result.schema.additionalProperties).toBe(false);
|
||||
expect(result.schema.required).toEqual(["value"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- Removed `preloadedSkills` option from `CreateAgentSessionOptions`; skills are no longer inlined into system prompts
|
||||
@@ -10,6 +11,8 @@
|
||||
|
||||
### Changed
|
||||
|
||||
- Modified schema validation to allow non-conforming output on second validation failure, enabling recovery from strict schema constraints after initial rejection
|
||||
- Updated JTD-to-TypeScript conversion to gracefully fall back to 'unknown' type when conversion fails, preventing template rendering errors
|
||||
- Changed JTD-to-JSON Schema conversion to normalize nested JTD fragments within JSON Schema nodes, enabling mixed schema definitions
|
||||
- Changed output schema validation to gracefully fall back to unconstrained object when schema is invalid, instead of rejecting submissions
|
||||
- Changed schema sanitization to remove strict-mode incompatible constraints (minLength, pattern, etc.) from tool parameters while preserving them for runtime validation
|
||||
@@ -22,6 +25,8 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed handling of circular and deeply nested output schemas to prevent stack overflow and enable successful result submission with fallback unconstrained schema
|
||||
- Fixed processing of non-object output schemas (arrays, primitives, booleans) to accept valid result submissions without blocking
|
||||
- Fixed handling of mixed JTD and JSON Schema output definitions to properly convert all nested JTD elements (e.g., `elements` → `items`, `int32` → `integer`)
|
||||
- Fixed strict schema generation for output schemas with only required fields, enabling proper Claude API compatibility
|
||||
- Fixed handling of union type schemas (e.g., object|null) to normalize them into strict-mode compatible variants
|
||||
|
||||
@@ -225,7 +225,13 @@ handlebars.registerHelper("includes", (collection: unknown, item: unknown): bool
|
||||
*/
|
||||
handlebars.registerHelper("not", (value: unknown): boolean => !value);
|
||||
|
||||
handlebars.registerHelper("jtdToTypeScript", (schema: unknown): string => jtdToTypeScript(schema));
|
||||
handlebars.registerHelper("jtdToTypeScript", (schema: unknown): string => {
|
||||
try {
|
||||
return jtdToTypeScript(schema);
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
});
|
||||
|
||||
handlebars.registerHelper("jsonStringify", (value: unknown): string => JSON.stringify(value));
|
||||
|
||||
|
||||
@@ -52,6 +52,29 @@ function formatAjvErrors(errors: ErrorObject[] | null | undefined): string {
|
||||
.join("; ");
|
||||
}
|
||||
|
||||
function splitRootDefinitions(schema: Record<string, unknown>): {
|
||||
schemaWithoutDefinitions: Record<string, unknown>;
|
||||
rootDefinitions?: Record<string, unknown>;
|
||||
} {
|
||||
const schemaWithoutDefinitions = { ...schema };
|
||||
const rootDefinitions: Record<string, unknown> = {};
|
||||
if (typeof schemaWithoutDefinitions.$defs === "object" && schemaWithoutDefinitions.$defs !== null) {
|
||||
if (!Array.isArray(schemaWithoutDefinitions.$defs)) {
|
||||
rootDefinitions.$defs = schemaWithoutDefinitions.$defs;
|
||||
}
|
||||
delete schemaWithoutDefinitions.$defs;
|
||||
}
|
||||
if (typeof schemaWithoutDefinitions.definitions === "object" && schemaWithoutDefinitions.definitions !== null) {
|
||||
if (!Array.isArray(schemaWithoutDefinitions.definitions)) {
|
||||
rootDefinitions.definitions = schemaWithoutDefinitions.definitions;
|
||||
}
|
||||
delete schemaWithoutDefinitions.definitions;
|
||||
}
|
||||
return Object.keys(rootDefinitions).length > 0
|
||||
? { schemaWithoutDefinitions, rootDefinitions }
|
||||
: { schemaWithoutDefinitions };
|
||||
}
|
||||
|
||||
export class SubmitResultTool implements AgentTool<TSchema, SubmitResultDetails> {
|
||||
readonly name = "submit_result";
|
||||
readonly label = "Submit Result";
|
||||
@@ -62,65 +85,96 @@ export class SubmitResultTool implements AgentTool<TSchema, SubmitResultDetails>
|
||||
strict = true;
|
||||
|
||||
readonly #validate?: ValidateFunction;
|
||||
#schemaValidationFailures = 0;
|
||||
|
||||
constructor(session: ToolSession) {
|
||||
const schemaResult = normalizeSchema(session.outputSchema);
|
||||
// Convert JTD to JSON Schema if needed (auto-detected)
|
||||
const normalizedSchema =
|
||||
schemaResult.normalized !== undefined ? jtdToJsonSchema(schemaResult.normalized) : undefined;
|
||||
let schemaError = schemaResult.error;
|
||||
const createParameters = (dataSchema: TSchema, rootDefinitions?: Record<string, unknown>): TSchema => {
|
||||
const baseParameters = Type.Object(
|
||||
{
|
||||
result: Type.Union([
|
||||
Type.Object({ data: dataSchema }, { description: "Successfully completed the task" }),
|
||||
Type.Object({
|
||||
error: Type.String({ description: "Error message when the task cannot be completed" }),
|
||||
}),
|
||||
]),
|
||||
},
|
||||
{
|
||||
additionalProperties: false,
|
||||
description: "Submit either `data` for success or `error` for failure",
|
||||
},
|
||||
) as unknown as Record<string, unknown>;
|
||||
if (!rootDefinitions) return baseParameters as TSchema;
|
||||
return Type.Unsafe({ ...baseParameters, ...rootDefinitions });
|
||||
};
|
||||
|
||||
if (normalizedSchema !== undefined && !schemaError) {
|
||||
try {
|
||||
this.#validate = ajv.compile(normalizedSchema as any);
|
||||
} catch (err) {
|
||||
schemaError = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
const schemaHint = formatSchema(normalizedSchema ?? session.outputSchema);
|
||||
|
||||
const schemaDescription = schemaError
|
||||
? `Structured JSON output (output schema invalid; accepting unconstrained object): ${schemaError}`
|
||||
: `Structured output matching the schema:\n${schemaHint}`;
|
||||
const sanitizedSchema =
|
||||
!schemaError &&
|
||||
normalizedSchema != null &&
|
||||
typeof normalizedSchema === "object" &&
|
||||
!Array.isArray(normalizedSchema)
|
||||
? sanitizeSchemaForStrictMode(normalizedSchema as Record<string, unknown>)
|
||||
: !schemaError && normalizedSchema === true
|
||||
? {}
|
||||
: undefined;
|
||||
|
||||
const dataSchema =
|
||||
sanitizedSchema !== undefined
|
||||
? Type.Unsafe({
|
||||
...sanitizedSchema,
|
||||
description: schemaDescription,
|
||||
})
|
||||
: Type.Record(Type.String(), Type.Any(), {
|
||||
description: schemaError ? schemaDescription : "Structured JSON output (no schema specified)",
|
||||
});
|
||||
this.parameters = Type.Object(
|
||||
{
|
||||
result: Type.Union([
|
||||
Type.Object({ data: dataSchema }, { description: "Successfully completed the task" }),
|
||||
Type.Object({ error: Type.String({ description: "Error message when the task cannot be completed" }) }),
|
||||
]),
|
||||
},
|
||||
{
|
||||
additionalProperties: false,
|
||||
description: "Submit either `data` for success or `error` for failure",
|
||||
},
|
||||
);
|
||||
let rootDefinitions: Record<string, unknown> | undefined;
|
||||
let validate: ValidateFunction | undefined;
|
||||
let dataSchema: TSchema;
|
||||
let parameters: TSchema;
|
||||
let strict = true;
|
||||
|
||||
try {
|
||||
const strictParameters = enforceStrictSchema(this.parameters as unknown as Record<string, unknown>);
|
||||
const schemaResult = normalizeSchema(session.outputSchema);
|
||||
// Convert JTD to JSON Schema if needed (auto-detected)
|
||||
const normalizedSchema =
|
||||
schemaResult.normalized !== undefined ? jtdToJsonSchema(schemaResult.normalized) : undefined;
|
||||
let schemaError = schemaResult.error;
|
||||
|
||||
if (!schemaError && normalizedSchema === false) {
|
||||
schemaError = "boolean false schema rejects all outputs";
|
||||
}
|
||||
|
||||
if (normalizedSchema !== undefined && normalizedSchema !== false && !schemaError) {
|
||||
try {
|
||||
validate = ajv.compile(normalizedSchema as Record<string, unknown> | boolean);
|
||||
} catch (err) {
|
||||
schemaError = err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
const schemaHint = formatSchema(normalizedSchema ?? session.outputSchema);
|
||||
const schemaDescription = schemaError
|
||||
? `Structured JSON output (output schema invalid; accepting unconstrained object): ${schemaError}`
|
||||
: `Structured output matching the schema:\n${schemaHint}`;
|
||||
const sanitizedSchema =
|
||||
!schemaError &&
|
||||
normalizedSchema != null &&
|
||||
typeof normalizedSchema === "object" &&
|
||||
!Array.isArray(normalizedSchema)
|
||||
? sanitizeSchemaForStrictMode(normalizedSchema as Record<string, unknown>)
|
||||
: !schemaError && normalizedSchema === true
|
||||
? {}
|
||||
: undefined;
|
||||
|
||||
if (sanitizedSchema !== undefined) {
|
||||
const schemaWithDescription = {
|
||||
...sanitizedSchema,
|
||||
description: schemaDescription,
|
||||
};
|
||||
const splitSchema = splitRootDefinitions(schemaWithDescription);
|
||||
rootDefinitions = splitSchema.rootDefinitions;
|
||||
dataSchema = Type.Unsafe(splitSchema.schemaWithoutDefinitions);
|
||||
} else {
|
||||
dataSchema = Type.Record(Type.String(), Type.Any(), {
|
||||
description: schemaError ? schemaDescription : "Structured JSON output (no schema specified)",
|
||||
});
|
||||
}
|
||||
parameters = createParameters(dataSchema, rootDefinitions);
|
||||
const strictParameters = enforceStrictSchema(parameters as unknown as Record<string, unknown>);
|
||||
JSON.stringify(strictParameters);
|
||||
} catch {
|
||||
this.strict = false;
|
||||
} catch (err) {
|
||||
const errorMsg = err instanceof Error ? err.message : String(err);
|
||||
dataSchema = Type.Record(Type.String(), Type.Any(), {
|
||||
description: `Structured JSON output (schema processing failed: ${errorMsg})`,
|
||||
});
|
||||
parameters = createParameters(dataSchema);
|
||||
validate = undefined;
|
||||
strict = false;
|
||||
}
|
||||
|
||||
this.#validate = validate;
|
||||
this.parameters = parameters;
|
||||
this.strict = strict;
|
||||
}
|
||||
|
||||
async execute(
|
||||
@@ -148,17 +202,26 @@ export class SubmitResultTool implements AgentTool<TSchema, SubmitResultDetails>
|
||||
}
|
||||
|
||||
const status = errorMessage !== undefined ? "aborted" : "success";
|
||||
let schemaValidationOverridden = false;
|
||||
if (status === "success") {
|
||||
if (data === undefined || data === null) {
|
||||
throw new Error("data is required when submit_result indicates success");
|
||||
}
|
||||
if (this.#validate && !this.#validate(data)) {
|
||||
throw new Error(`Output does not match schema: ${formatAjvErrors(this.#validate.errors)}`);
|
||||
this.#schemaValidationFailures++;
|
||||
if (this.#schemaValidationFailures <= 1) {
|
||||
throw new Error(`Output does not match schema: ${formatAjvErrors(this.#validate.errors)}`);
|
||||
}
|
||||
schemaValidationOverridden = true;
|
||||
}
|
||||
}
|
||||
|
||||
const responseText = status === "aborted" ? `Task aborted: ${errorMessage}` : "Result submitted.";
|
||||
|
||||
const responseText =
|
||||
status === "aborted"
|
||||
? `Task aborted: ${errorMessage}`
|
||||
: schemaValidationOverridden
|
||||
? `Result submitted (schema validation overridden after ${this.#schemaValidationFailures} failed attempt(s)).`
|
||||
: "Result submitted.";
|
||||
return {
|
||||
content: [{ type: "text", text: responseText }],
|
||||
details: { data, status, error: errorMessage },
|
||||
|
||||
@@ -60,6 +60,10 @@ describe("AgentSession user shortcut hooks", () => {
|
||||
exitCode: 0,
|
||||
cancelled: false,
|
||||
truncated: false,
|
||||
totalLines: 1,
|
||||
totalBytes: 18,
|
||||
outputLines: 1,
|
||||
outputBytes: 18,
|
||||
};
|
||||
const emitUserBash = vi.fn().mockResolvedValue({ result: replacement });
|
||||
const extensionRunner = {
|
||||
@@ -93,7 +97,12 @@ describe("AgentSession user shortcut hooks", () => {
|
||||
exitCode: 0,
|
||||
cancelled: false,
|
||||
truncated: false,
|
||||
totalLines: 1,
|
||||
totalBytes: 20,
|
||||
outputLines: 1,
|
||||
outputBytes: 20,
|
||||
displayOutputs: [],
|
||||
stdinRequested: false,
|
||||
};
|
||||
const emitUserPython = vi.fn().mockResolvedValue({ result: replacement });
|
||||
const extensionRunner = {
|
||||
@@ -132,13 +141,22 @@ describe("AgentSession user shortcut hooks", () => {
|
||||
exitCode: 0,
|
||||
cancelled: false,
|
||||
truncated: false,
|
||||
totalLines: 1,
|
||||
totalBytes: 13,
|
||||
outputLines: 1,
|
||||
outputBytes: 13,
|
||||
});
|
||||
vi.spyOn(pythonExecutor, "executePython").mockResolvedValue({
|
||||
output: "python fallback",
|
||||
exitCode: 0,
|
||||
cancelled: false,
|
||||
truncated: false,
|
||||
totalLines: 1,
|
||||
totalBytes: 15,
|
||||
outputLines: 1,
|
||||
outputBytes: 15,
|
||||
displayOutputs: [],
|
||||
stdinRequested: false,
|
||||
});
|
||||
|
||||
createSession(extensionRunner);
|
||||
@@ -149,7 +167,11 @@ describe("AgentSession user shortcut hooks", () => {
|
||||
expect(pythonResult.output).toBe("python fallback");
|
||||
expect(bashExecutor.executeBash).toHaveBeenCalledTimes(1);
|
||||
expect(pythonExecutor.executePython).toHaveBeenCalledTimes(1);
|
||||
expect(session.messages.some(message => message.role === "bashExecution" && message.excludeFromContext === true)).toBe(true);
|
||||
expect(session.messages.some(message => message.role === "pythonExecution" && message.excludeFromContext === false)).toBe(true);
|
||||
expect(
|
||||
session.messages.some(message => message.role === "bashExecution" && message.excludeFromContext === true),
|
||||
).toBe(true);
|
||||
expect(
|
||||
session.messages.some(message => message.role === "pythonExecution" && message.excludeFromContext === false),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import type { Tool, ToolCall } from "@oh-my-pi/pi-ai/types";
|
||||
import { enforceStrictSchema } from "@oh-my-pi/pi-ai/utils/typebox-helpers";
|
||||
import { validateToolArguments } from "@oh-my-pi/pi-ai/utils/validation";
|
||||
import { Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
||||
import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools";
|
||||
import { SubmitResultTool } from "@oh-my-pi/pi-coding-agent/tools/submit-result";
|
||||
@@ -161,6 +163,66 @@ describe("SubmitResultTool", () => {
|
||||
tool.execute("call-mixed-invalid", { result: { data: { results: [{ issue: "185" }] } } } as never),
|
||||
).rejects.toThrow("Output does not match schema");
|
||||
});
|
||||
it("supports $defs/$ref output schemas in tool-argument validation and degrades after first runtime failure", async () => {
|
||||
const outputSchema = {
|
||||
$defs: {
|
||||
A: {
|
||||
type: "object",
|
||||
properties: {
|
||||
kind: { const: "A" },
|
||||
token: { type: "string", minLength: 10 },
|
||||
},
|
||||
required: ["kind", "token"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
anyOf: [
|
||||
{ $ref: "#/$defs/A" },
|
||||
{
|
||||
type: "object",
|
||||
properties: {
|
||||
kind: { const: "B" },
|
||||
n: { type: "integer", minimum: 10 },
|
||||
},
|
||||
required: ["kind", "n"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema }));
|
||||
const parametersRecord = tool.parameters as unknown as Record<string, unknown>;
|
||||
expect(toRecord(parametersRecord.$defs).A).toBeDefined();
|
||||
expect(getSuccessDataSchema(parametersRecord).$defs).toBeUndefined();
|
||||
|
||||
const toolDefinition: Tool = {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.parameters,
|
||||
};
|
||||
const firstCall: ToolCall = {
|
||||
type: "toolCall",
|
||||
id: "call-ref-1",
|
||||
name: tool.name,
|
||||
arguments: { result: { data: { kind: "A", token: "x" } } },
|
||||
};
|
||||
const firstArgs = validateToolArguments(toolDefinition, firstCall);
|
||||
await expect(tool.execute("call-ref-1", firstArgs as never)).rejects.toThrow("Output does not match schema");
|
||||
|
||||
const secondCall: ToolCall = {
|
||||
type: "toolCall",
|
||||
id: "call-ref-2",
|
||||
name: tool.name,
|
||||
arguments: { result: { data: { kind: "A", token: "x" } } },
|
||||
};
|
||||
const secondArgs = validateToolArguments(toolDefinition, secondCall);
|
||||
const secondResult = await tool.execute("call-ref-2", secondArgs as never);
|
||||
expect(secondResult.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: "Result submitted (schema validation overridden after 2 failed attempt(s)).",
|
||||
},
|
||||
]);
|
||||
});
|
||||
it("falls back to unconstrained object data when output schema is invalid", async () => {
|
||||
const tool = new SubmitResultTool(
|
||||
createSession({
|
||||
@@ -189,6 +251,69 @@ describe("SubmitResultTool", () => {
|
||||
error: undefined,
|
||||
});
|
||||
});
|
||||
it("falls back to unconstrained data schema when output schema is circular", async () => {
|
||||
const circularSchema: Record<string, unknown> = { type: "object" };
|
||||
circularSchema.self = circularSchema;
|
||||
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema: circularSchema }));
|
||||
const dataSchema = getSuccessDataSchema(tool.parameters as unknown as Record<string, unknown>);
|
||||
|
||||
expect(tool.strict).toBe(false);
|
||||
expect(dataSchema.type).toBe("object");
|
||||
|
||||
const result = await tool.execute("call-circular-schema", { result: { data: { ok: true } } } as never);
|
||||
expect(result.details).toEqual({ data: { ok: true }, status: "success", error: undefined });
|
||||
});
|
||||
|
||||
it("falls back to unconstrained data schema when output schema is deeply nested", async () => {
|
||||
const buildDeepSchema = (depth: number): Record<string, unknown> => {
|
||||
const root: Record<string, unknown> = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: ["next"],
|
||||
};
|
||||
let current = root;
|
||||
|
||||
for (let i = 0; i < depth; i++) {
|
||||
const next: Record<string, unknown> = {
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: ["next"],
|
||||
};
|
||||
const currentProperties = toRecord(current.properties);
|
||||
currentProperties.next = next;
|
||||
current.properties = currentProperties;
|
||||
current = next;
|
||||
}
|
||||
|
||||
current.properties = { value: { type: "string" } };
|
||||
current.required = ["value"];
|
||||
return root;
|
||||
};
|
||||
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema: buildDeepSchema(20_000) }));
|
||||
const dataSchema = getSuccessDataSchema(tool.parameters as unknown as Record<string, unknown>);
|
||||
|
||||
expect(tool.strict).toBe(false);
|
||||
expect(dataSchema.type).toBe("object");
|
||||
|
||||
const result = await tool.execute("call-deep-schema", { result: { data: { nested: true } } } as never);
|
||||
expect(result.details).toEqual({ data: { nested: true }, status: "success", error: undefined });
|
||||
});
|
||||
|
||||
it("handles non-object output schemas without blocking successful result submission", async () => {
|
||||
for (const outputSchema of [[], 123, false]) {
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema }));
|
||||
const result = await tool.execute("call-non-object-schema", {
|
||||
result: { data: { value: outputSchema } },
|
||||
} as never);
|
||||
expect(result.details).toEqual({
|
||||
data: { value: outputSchema },
|
||||
status: "success",
|
||||
error: undefined,
|
||||
});
|
||||
}
|
||||
});
|
||||
it("keeps runtime validation against the original output schema", async () => {
|
||||
const outputSchema = {
|
||||
type: "object",
|
||||
@@ -213,6 +338,81 @@ describe("SubmitResultTool", () => {
|
||||
expect(result.details).toEqual({ data: { token: "abcd" }, status: "success", error: undefined });
|
||||
});
|
||||
|
||||
it("throws on first schema validation failure and accepts non-conforming data on second failure", async () => {
|
||||
const outputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
token: {
|
||||
type: "string",
|
||||
minLength: 3,
|
||||
},
|
||||
},
|
||||
required: ["token"],
|
||||
};
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema }));
|
||||
|
||||
await expect(tool.execute("call-short-1", { result: { data: { token: "ab" } } } as never)).rejects.toThrow(
|
||||
"Output does not match schema",
|
||||
);
|
||||
|
||||
const secondResult = await tool.execute("call-short-2", { result: { data: { token: "ab" } } } as never);
|
||||
expect(secondResult.details).toEqual({ data: { token: "ab" }, status: "success", error: undefined });
|
||||
expect(secondResult.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: "Result submitted (schema validation overridden after 2 failed attempt(s)).",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps schema degradation counter at zero when submissions are valid", async () => {
|
||||
const outputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
token: {
|
||||
type: "string",
|
||||
minLength: 3,
|
||||
},
|
||||
},
|
||||
required: ["token"],
|
||||
};
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema }));
|
||||
|
||||
const firstResult = await tool.execute("call-valid-1", { result: { data: { token: "abcd" } } } as never);
|
||||
expect(firstResult.content).toEqual([{ type: "text", text: "Result submitted." }]);
|
||||
|
||||
const secondResult = await tool.execute("call-valid-2", { result: { data: { token: "abcde" } } } as never);
|
||||
expect(secondResult.content).toEqual([{ type: "text", text: "Result submitted." }]);
|
||||
|
||||
await expect(
|
||||
tool.execute("call-invalid-after-valid", { result: { data: { token: "ab" } } } as never),
|
||||
).rejects.toThrow("Output does not match schema");
|
||||
});
|
||||
|
||||
it("still throws structural errors after schema validation has been degraded", async () => {
|
||||
const outputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
token: {
|
||||
type: "string",
|
||||
minLength: 3,
|
||||
},
|
||||
},
|
||||
required: ["token"],
|
||||
};
|
||||
const tool = new SubmitResultTool(createSession({ outputSchema }));
|
||||
|
||||
await expect(tool.execute("call-struct-1", { result: { data: { token: "ab" } } } as never)).rejects.toThrow(
|
||||
"Output does not match schema",
|
||||
);
|
||||
await expect(
|
||||
tool.execute("call-struct-2", { result: { data: { token: "ab" } } } as never),
|
||||
).resolves.toBeDefined();
|
||||
|
||||
await expect(tool.execute("call-struct-missing", {} as never)).rejects.toThrow(
|
||||
"result must be an object containing either data or error",
|
||||
);
|
||||
});
|
||||
it("rejects submissions without a result object", async () => {
|
||||
const tool = new SubmitResultTool(createSession());
|
||||
await expect(tool.execute("call-3", {} as never)).rejects.toThrow(
|
||||
|
||||
Reference in New Issue
Block a user