smashimo/oh-my-pi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(coding-agent/eval): updated eval tests to inject runtime hooks explicitly

Browse Source 
- Refactored console-table tests to build explicit RuntimeHooks and pass them to JsRuntime.run.
- Refactored image coercion tests to pass explicit RuntimeHooks into JsRuntime.displayValue instead of constructor hooks.
This commit is contained in:
can1357
2026-05-26 15:27:54 +02:00
parent dbd7dd0dc0
commit 5d7a452f11
5 changed files with 56 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/providers/register-builtins.ts
+4 -1
View File
@@ -423,7 +423,10 @@ export const streamOpenAICompletions = createLazyStream(
loadOpenAICompletionsProviderModule,
PROVIDER_HANDLED_STREAM_TIMEOUTS,
);
export const streamOpenAIResponses = createLazyStream(loadOpenAIResponsesProviderModule, PROVIDER_HANDLED_STREAM_TIMEOUTS);
export const streamOpenAIResponses = createLazyStream(
loadOpenAIResponsesProviderModule,
PROVIDER_HANDLED_STREAM_TIMEOUTS,
);
export const streamCursor = createLazyStream(loadCursorProviderModule);
export const streamOllama = createLazyStream(loadOllamaProviderModule);
packages/ai/test/stream-timeout-defaults.test.ts
+1
View File
@@ -134,6 +134,7 @@ describe("iterateWithIdleTimeout", () => {
watchdogFired = true;
}, 10);
// biome-ignore lint/correctness/useYield: intentionally yields nothing — the test exercises the path where the source generator throws before its first yield.
async function* failingStream(): AsyncGenerator<string> {
throw new Error("stream failed");
}
packages/coding-agent/src/config/settings-schema.ts
+2 -1
View File
@@ -1912,7 +1912,8 @@ export const SETTINGS_SCHEMA = {
ui: {
tab: "tools",
label: "IRC Timeout",
description: "Drop IRC messages whose recipient does not respond within this many milliseconds (0 disables the timeout)",
description:
"Drop IRC messages whose recipient does not respond within this many milliseconds (0 disables the timeout)",
options: [
{ value: "0", label: "Disabled" },
{ value: "30000", label: "30 seconds" },
packages/coding-agent/test/eval/console-table.test.ts
+21 -16
View File
@@ -1,30 +1,35 @@
import { describe, expect, it } from "bun:test";
import { JsRuntime } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime";
import { JsRuntime, type RuntimeHooks } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime";
import type { JsDisplayOutput } from "@oh-my-pi/pi-coding-agent/eval/js/shared/types";
function makeRuntime(): { runtime: JsRuntime; texts: string[]; displays: JsDisplayOutput[] } {
function makeRuntime(): {
runtime: JsRuntime;
hooks: RuntimeHooks;
texts: string[];
displays: JsDisplayOutput[];
} {
const texts: string[] = [];
const displays: JsDisplayOutput[] = [];
const runtime = new JsRuntime({
initialCwd: process.cwd(),
sessionId: "test",
getHooks: () => ({
onText: chunk => {
texts.push(chunk);
},
onDisplay: output => {
displays.push(output);
},
callTool: async () => undefined,
}),
});
return { runtime, texts, displays };
const hooks: RuntimeHooks = {
onText: (chunk: string) => {
texts.push(chunk);
},
onDisplay: (output: JsDisplayOutput) => {
displays.push(output);
},
callTool: async () => undefined,
};
return { runtime, hooks, texts, displays };
}
describe("console.table bridge", () => {
it("renders an array of objects as an ASCII table on text output", async () => {
const { runtime, texts, displays } = makeRuntime();
await runtime.run("console.table([{ name: 'Ada', age: 36 }, { name: 'Linus', age: 54 }]);");
const { runtime, hooks, texts, displays } = makeRuntime();
await runtime.run("console.table([{ name: 'Ada', age: 36 }, { name: 'Linus', age: 54 }]);", undefined, hooks);
expect(displays).toEqual([]);
expect(texts.length).toBe(1);
const out = texts[0];
@@ -39,8 +44,8 @@ describe("console.table bridge", () => {
});
it("honors the optional columns filter", async () => {
const { runtime, texts } = makeRuntime();
await runtime.run("console.table([{ name: 'Ada', age: 36, secret: 'hidden' }], ['name']);");
const { runtime, hooks, texts } = makeRuntime();
await runtime.run("console.table([{ name: 'Ada', age: 36, secret: 'hidden' }], ['name']);", undefined, hooks);
const out = texts.join("");
expect(out).toContain("name");
expect(out).toContain("Ada");
packages/coding-agent/test/eval/display-image-coerce.test.ts
+28 -27
View File
@@ -1,9 +1,10 @@
import { describe, expect, it } from "bun:test";
import { JsRuntime } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime";
import { JsRuntime, type RuntimeHooks } from "@oh-my-pi/pi-coding-agent/eval/js/shared/runtime";
import type { JsDisplayOutput } from "@oh-my-pi/pi-coding-agent/eval/js/shared/types";
function collect(): {
runtime: JsRuntime;
hooks: RuntimeHooks;
displays: JsDisplayOutput[];
texts: string[];
} {
@@ -12,17 +13,17 @@ function collect(): {
const runtime = new JsRuntime({
initialCwd: process.cwd(),
sessionId: "test",
getHooks: () => ({
onText: chunk => {
texts.push(chunk);
},
onDisplay: output => {
displays.push(output);
},
callTool: async () => undefined,
}),
});
return { runtime, displays, texts };
const hooks: RuntimeHooks = {
onText: (chunk: string) => {
texts.push(chunk);
},
onDisplay: (output: JsDisplayOutput) => {
displays.push(output);
},
callTool: async () => undefined,
};
return { runtime, hooks, displays, texts };
}
const PNG_BYTES = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]);
@@ -30,27 +31,27 @@ const PNG_BASE64 = Buffer.from(PNG_BYTES).toString("base64");
describe("JsRuntime.displayValue image coercion", () => {
it("passes through strict base64 strings verbatim", () => {
const { runtime, displays } = collect();
runtime.displayValue({ type: "image", data: PNG_BASE64, mimeType: "image/png" });
const { runtime, hooks, displays } = collect();
runtime.displayValue({ type: "image", data: PNG_BASE64, mimeType: "image/png" }, hooks);
expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]);
});
it("base64-encodes Uint8Array data", () => {
const { runtime, displays } = collect();
runtime.displayValue({ type: "image", data: PNG_BYTES, mimeType: "image/png" });
const { runtime, hooks, displays } = collect();
runtime.displayValue({ type: "image", data: PNG_BYTES, mimeType: "image/png" }, hooks);
expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]);
});
it("base64-encodes Buffer data", () => {
const { runtime, displays } = collect();
runtime.displayValue({ type: "image", data: Buffer.from(PNG_BYTES), mimeType: "image/png" });
const { runtime, hooks, displays } = collect();
runtime.displayValue({ type: "image", data: Buffer.from(PNG_BYTES), mimeType: "image/png" }, hooks);
expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]);
});
it("base64-encodes ArrayBuffer data", () => {
const { runtime, displays } = collect();
const { runtime, hooks, displays } = collect();
const ab = PNG_BYTES.buffer.slice(PNG_BYTES.byteOffset, PNG_BYTES.byteOffset + PNG_BYTES.byteLength);
runtime.displayValue({ type: "image", data: ab, mimeType: "image/png" });
runtime.displayValue({ type: "image", data: ab, mimeType: "image/png" }, hooks);
expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]);
});
@@ -58,26 +59,26 @@ describe("JsRuntime.displayValue image coercion", () => {
// Reproduces the puppeteer footgun: page.screenshot() returns Uint8Array, and
// `uint8array.toString("base64")` silently falls through to Array.toString,
// yielding "137,80,78,71,...". Anthropic rejects that as invalid base64.
const { runtime, displays } = collect();
const { runtime, hooks, displays } = collect();
const decimalCsv = Array.from(PNG_BYTES).toString();
expect(decimalCsv).toBe("137,80,78,71,13,10,26,10");
runtime.displayValue({ type: "image", data: decimalCsv, mimeType: "image/png" });
runtime.displayValue({ type: "image", data: decimalCsv, mimeType: "image/png" }, hooks);
expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]);
});
it("recovers JSON-serialized Buffer shape ({ type: 'Buffer', data: [...] })", () => {
const { runtime, displays } = collect();
const { runtime, hooks, displays } = collect();
const jsonBuffer = JSON.parse(JSON.stringify(Buffer.from(PNG_BYTES))) as {
type: string;
data: number[];
};
runtime.displayValue({ type: "image", data: jsonBuffer, mimeType: "image/png" });
runtime.displayValue({ type: "image", data: jsonBuffer, mimeType: "image/png" }, hooks);
expect(displays).toEqual([{ type: "image", data: PNG_BASE64, mimeType: "image/png" }]);
});
it("drops images whose data is unrecognized and surfaces a diagnostic in text", () => {
const { runtime, displays, texts } = collect();
runtime.displayValue({ type: "image", data: { not: "a buffer" }, mimeType: "image/png" });
const { runtime, hooks, displays, texts } = collect();
runtime.displayValue({ type: "image", data: { not: "a buffer" }, mimeType: "image/png" }, hooks);
expect(displays).toHaveLength(0);
expect(texts.join("")).toMatch(/image dropped/);
});
@@ -85,8 +86,8 @@ describe("JsRuntime.displayValue image coercion", () => {
it("rejects strings that look base64-ish but aren't strictly valid", () => {
// Padding mid-string, whitespace, or URL-safe alphabet are all dropped — the
// Anthropic API only honors strict base64 in image sources.
const { runtime, displays, texts } = collect();
runtime.displayValue({ type: "image", data: "abcd=efg", mimeType: "image/png" });
const { runtime, hooks, displays, texts } = collect();
runtime.displayValue({ type: "image", data: "abcd=efg", mimeType: "image/png" }, hooks);
expect(displays).toHaveLength(0);
expect(texts.join("")).toMatch(/image dropped/);
});