can1357
46be698745
- Removed Kagi Universal Summarizer integration from fetch tool and YouTube scraper. - Removed `fetch.useKagiSummarizer` configuration setting from settings schema. - Simplified renderHtmlToText() and renderUrl() functions by removing Kagi summarization fallback logic. - Fixed indentation inconsistencies in test files from tabs to spaces.
477 lines
17 KiB
TypeScript
477 lines
17 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from "bun:test";
|
|
import * as fs from "node:fs";
|
|
import * as os from "node:os";
|
|
import * as path from "node:path";
|
|
import { type SettingPath, Settings } from "@oh-my-pi/pi-coding-agent/config/settings";
|
|
import type { ToolSession } from "@oh-my-pi/pi-coding-agent/tools";
|
|
import { FetchTool } from "@oh-my-pi/pi-coding-agent/tools/fetch";
|
|
import * as imageResize from "@oh-my-pi/pi-coding-agent/utils/image-resize";
|
|
import * as toolsManager from "@oh-my-pi/pi-coding-agent/utils/tools-manager";
|
|
import * as scrapers from "@oh-my-pi/pi-coding-agent/web/scrapers/types";
|
|
import * as scraperUtils from "@oh-my-pi/pi-coding-agent/web/scrapers/utils";
|
|
import * as natives from "@oh-my-pi/pi-natives";
|
|
import { hookFetch, ptree, Snowflake } from "@oh-my-pi/pi-utils";
|
|
|
|
describe("fetch tool", () => {
|
|
let testDir: string;
|
|
|
|
beforeEach(() => {
|
|
testDir = path.join(os.tmpdir(), `fetch-kagi-toggle-${Snowflake.next()}`);
|
|
fs.mkdirSync(testDir, { recursive: true });
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
fs.rmSync(testDir, { recursive: true, force: true });
|
|
});
|
|
|
|
const createSession = (overrides: Partial<Record<SettingPath, unknown>> = {}): ToolSession => {
|
|
const sessionFile = path.join(testDir, "session.jsonl");
|
|
return {
|
|
cwd: testDir,
|
|
hasUI: false,
|
|
getSessionFile: () => sessionFile,
|
|
getSessionSpawns: () => null,
|
|
settings: Settings.isolated({
|
|
"fetch.enabled": true,
|
|
...overrides,
|
|
}),
|
|
};
|
|
};
|
|
|
|
it("returns an image content block when fetching image URLs", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const imageBytes = new Uint8Array([137, 80, 78, 71]);
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/png",
|
|
finalUrl: "https://example.com/image.png",
|
|
content: "",
|
|
});
|
|
vi.spyOn(scraperUtils, "fetchBinary").mockResolvedValue({
|
|
ok: true,
|
|
buffer: imageBytes,
|
|
});
|
|
vi.spyOn(scraperUtils, "convertWithMarkitdown").mockResolvedValue({
|
|
ok: false,
|
|
content: "",
|
|
error: "markitdown unavailable",
|
|
});
|
|
vi.spyOn(imageResize, "resizeImage").mockResolvedValue({
|
|
buffer: imageBytes,
|
|
mimeType: "image/png",
|
|
originalWidth: 1,
|
|
originalHeight: 1,
|
|
width: 1,
|
|
height: 1,
|
|
wasResized: false,
|
|
get data() {
|
|
return imageBytes.toBase64();
|
|
},
|
|
});
|
|
|
|
const result = await tool.execute("fetch-image", { url: "https://example.com/image.png" });
|
|
const imageBlock = result.content.find(
|
|
(content): content is { type: "image"; data: string; mimeType: string } => content.type === "image",
|
|
);
|
|
|
|
expect(result.details?.method).toBe("image");
|
|
expect(imageBlock).toBeDefined();
|
|
expect(imageBlock?.mimeType).toBe("image/png");
|
|
expect(imageBlock?.data).toBe(imageBytes.toBase64());
|
|
});
|
|
|
|
it("resizes fetched images before emitting image content blocks", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const resizeSpy = vi.spyOn(imageResize, "resizeImage").mockResolvedValue({
|
|
buffer: new Uint8Array([1, 2, 3]),
|
|
mimeType: "image/jpeg",
|
|
originalWidth: 2000,
|
|
originalHeight: 1000,
|
|
width: 1000,
|
|
height: 500,
|
|
wasResized: true,
|
|
get data() {
|
|
return "cmVzaXplZA==";
|
|
},
|
|
});
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/png",
|
|
finalUrl: "https://example.com/image.png",
|
|
content: "",
|
|
});
|
|
vi.spyOn(scraperUtils, "fetchBinary").mockResolvedValue({
|
|
ok: true,
|
|
buffer: new Uint8Array([137, 80, 78, 71]),
|
|
});
|
|
vi.spyOn(scraperUtils, "convertWithMarkitdown").mockResolvedValue({
|
|
ok: false,
|
|
content: "",
|
|
error: "markitdown unavailable",
|
|
});
|
|
|
|
const result = await tool.execute("fetch-image-resized", { url: "https://example.com/image.png" });
|
|
const imageBlock = result.content.find(
|
|
(content): content is { type: "image"; data: string; mimeType: string } => content.type === "image",
|
|
);
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(resizeSpy).toHaveBeenCalledTimes(1);
|
|
expect(result.details?.method).toBe("image");
|
|
expect(imageBlock?.mimeType).toBe("image/jpeg");
|
|
expect(imageBlock?.data).toBe("cmVzaXplZA==");
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("displayed at 1000x500");
|
|
});
|
|
|
|
it("keeps markitdown extracted text for image responses", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const extractedText = "Converted image text content that is definitely longer than fifty characters.";
|
|
vi.spyOn(imageResize, "resizeImage").mockResolvedValue({
|
|
buffer: new Uint8Array([1, 2, 3]),
|
|
mimeType: "image/png",
|
|
originalWidth: 100,
|
|
originalHeight: 100,
|
|
width: 100,
|
|
height: 100,
|
|
wasResized: false,
|
|
get data() {
|
|
return "aW1hZ2U=";
|
|
},
|
|
});
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/png",
|
|
finalUrl: "https://example.com/image.png",
|
|
content: "",
|
|
});
|
|
vi.spyOn(scraperUtils, "fetchBinary").mockResolvedValue({
|
|
ok: true,
|
|
buffer: new Uint8Array([137, 80, 78, 71]),
|
|
});
|
|
vi.spyOn(scraperUtils, "convertWithMarkitdown").mockResolvedValue({
|
|
ok: true,
|
|
content: extractedText,
|
|
});
|
|
|
|
const result = await tool.execute("fetch-image-with-ocr", { url: "https://example.com/image.png" });
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
const imageBlock = result.content.find(
|
|
(content): content is { type: "image"; data: string; mimeType: string } => content.type === "image",
|
|
);
|
|
|
|
expect(result.details?.method).toBe("image");
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain(extractedText);
|
|
expect(imageBlock?.mimeType).toBe("image/png");
|
|
expect(imageBlock?.data).toBe("aW1hZ2U=");
|
|
});
|
|
it("falls back to text-only output for unsupported image MIME types", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const fetchBinarySpy = vi.spyOn(scraperUtils, "fetchBinary");
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/svg+xml",
|
|
finalUrl: "https://example.com/image.svg",
|
|
content: "<svg></svg>",
|
|
});
|
|
|
|
const result = await tool.execute("fetch-image-unsupported", { url: "https://example.com/image.svg" });
|
|
const imageBlock = result.content.find(content => content.type === "image");
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("raw");
|
|
expect(fetchBinarySpy).not.toHaveBeenCalled();
|
|
expect(imageBlock).toBeUndefined();
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("<svg></svg>");
|
|
});
|
|
|
|
it("uses binary conversion fallback for unsupported image MIME when extension is convertible", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const convertedText = "Converted image text from markitdown fallback with sufficient length to pass threshold.";
|
|
const fetchBinarySpy = vi.spyOn(scraperUtils, "fetchBinary").mockResolvedValue({
|
|
ok: true,
|
|
buffer: new Uint8Array([255, 216, 255, 224]),
|
|
});
|
|
const convertSpy = vi.spyOn(scraperUtils, "convertWithMarkitdown").mockResolvedValue({
|
|
ok: true,
|
|
content: convertedText,
|
|
});
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/jpg",
|
|
finalUrl: "https://example.com/image.jpg",
|
|
content: "\u0000\u0001garbage",
|
|
});
|
|
|
|
const result = await tool.execute("fetch-image-jpg-fallback", { url: "https://example.com/image.jpg" });
|
|
const imageBlock = result.content.find(content => content.type === "image");
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("markitdown");
|
|
expect(fetchBinarySpy).toHaveBeenCalledTimes(1);
|
|
expect(convertSpy).toHaveBeenCalledTimes(1);
|
|
expect(result.details?.notes).toContain("Attempting binary conversion fallback for unsupported image MIME type");
|
|
expect(imageBlock).toBeUndefined();
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain(convertedText);
|
|
});
|
|
|
|
it("does not treat text/html at .png paths as inline images", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
vi.spyOn(scraperUtils, "fetchBinary").mockResolvedValue({ ok: false, error: "not an image" });
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "text/html",
|
|
finalUrl: "https://example.com/foo.png",
|
|
content: "<html><body>not really an image</body></html>",
|
|
});
|
|
|
|
const result = await tool.execute("fetch-html-png-path", { url: "https://example.com/foo.png", raw: true });
|
|
const imageBlock = result.content.find(content => content.type === "image");
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("raw");
|
|
expect(imageBlock).toBeUndefined();
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("<html><body>not really an image</body></html>");
|
|
});
|
|
|
|
it("falls back to textual output when inline image refetch fails", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const convertSpy = vi.spyOn(scraperUtils, "convertWithMarkitdown");
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/png",
|
|
finalUrl: "https://example.com/transient.png",
|
|
content: "<html><body>temporary gateway page</body></html>",
|
|
});
|
|
const fetchBinarySpy = vi
|
|
.spyOn(scraperUtils, "fetchBinary")
|
|
.mockResolvedValue({ ok: false, error: "upstream blocked" });
|
|
|
|
const result = await tool.execute("fetch-image-refetch-failed", { url: "https://example.com/transient.png" });
|
|
const imageBlock = result.content.find(content => content.type === "image");
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("raw");
|
|
expect(imageBlock).toBeUndefined();
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("<html><body>temporary gateway page</body></html>");
|
|
expect(convertSpy).not.toHaveBeenCalled();
|
|
expect(fetchBinarySpy).toHaveBeenCalledTimes(1);
|
|
});
|
|
it("falls back to text-only output when image payload bytes are invalid", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
vi.spyOn(scrapers, "loadPage").mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "image/png",
|
|
finalUrl: "https://example.com/broken.png",
|
|
content: "<html><body>gateway error</body></html>",
|
|
});
|
|
vi.spyOn(scraperUtils, "fetchBinary").mockResolvedValue({
|
|
ok: true,
|
|
buffer: new Uint8Array([60, 104, 116, 109, 108]),
|
|
});
|
|
vi.spyOn(scraperUtils, "convertWithMarkitdown").mockResolvedValue({
|
|
ok: false,
|
|
content: "",
|
|
error: "conversion failed",
|
|
});
|
|
vi.spyOn(imageResize, "resizeImage").mockResolvedValue({
|
|
buffer: new Uint8Array([60, 104, 116, 109, 108]),
|
|
mimeType: "image/png",
|
|
originalWidth: 0,
|
|
originalHeight: 0,
|
|
width: 0,
|
|
height: 0,
|
|
wasResized: false,
|
|
get data() {
|
|
return "PGh0bWw=";
|
|
},
|
|
});
|
|
|
|
const result = await tool.execute("fetch-broken-image", { url: "https://example.com/broken.png" });
|
|
const imageBlock = result.content.find(content => content.type === "image");
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("image-invalid");
|
|
expect(imageBlock).toBeUndefined();
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("<html><body>gateway error</body></html>");
|
|
});
|
|
it("prefers rendered page content over site-wide llms.txt for deep pages", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const pageUrl = "https://bun.com/reference/bun/UnixSocketOptions";
|
|
const pageHtml = "<html><body><main><h1>UnixSocketOptions</h1><p>Page-specific docs.</p></main></body></html>";
|
|
const renderedMarkdown = `# UnixSocketOptions\n\n${"Page-specific API docs. ".repeat(8)}`;
|
|
const originalWhich = Bun.which;
|
|
|
|
Bun.which = (() => null) as typeof Bun.which;
|
|
try {
|
|
const loadPageSpy = vi.spyOn(scrapers, "loadPage").mockImplementation(async (requestedUrl: string) => {
|
|
if (requestedUrl === pageUrl) {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "text/html",
|
|
finalUrl: pageUrl,
|
|
content: pageHtml,
|
|
};
|
|
}
|
|
|
|
if (requestedUrl === `${pageUrl}.md`) {
|
|
return {
|
|
ok: false,
|
|
status: 404,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: "",
|
|
};
|
|
}
|
|
|
|
if (requestedUrl === "https://bun.com/llms.txt") {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: `# Bun\n\n${"Site-wide overview. ".repeat(12)}`,
|
|
};
|
|
}
|
|
|
|
return {
|
|
ok: false,
|
|
status: 404,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: "",
|
|
};
|
|
});
|
|
using _hook = hookFetch(() => new Response("blocked", { status: 500, statusText: "Blocked" }));
|
|
vi.spyOn(toolsManager, "ensureTool").mockResolvedValue(undefined);
|
|
vi.spyOn(natives, "htmlToMarkdown").mockResolvedValue(renderedMarkdown);
|
|
|
|
const result = await tool.execute("fetch-deep-page", { url: pageUrl });
|
|
const requestedUrls = loadPageSpy.mock.calls.map(([requestedUrl]) => requestedUrl);
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("native");
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("UnixSocketOptions");
|
|
expect(requestedUrls).not.toContain("https://bun.com/.well-known/llms.txt");
|
|
expect(requestedUrls).not.toContain("https://bun.com/llms.txt");
|
|
expect(requestedUrls).not.toContain("https://bun.com/llms.md");
|
|
} finally {
|
|
Bun.which = originalWhich;
|
|
}
|
|
});
|
|
|
|
it("uses section-scoped llms.txt fallback without requesting the site-wide file", async () => {
|
|
const session = createSession();
|
|
const tool = new FetchTool(session);
|
|
const pageUrl = "https://example.com/docs/reference/widget";
|
|
const pageHtml = "<html><body><nav>Docs</nav><main><h1>Widget</h1></main></body></html>";
|
|
const lowQualityRender = `${"Please enable JavaScript to view this page.\n".repeat(6)}${"navigation\n".repeat(4)}`;
|
|
const originalWhich = Bun.which;
|
|
const execSpy = vi.spyOn(ptree, "exec").mockResolvedValue({ ok: true, stdout: lowQualityRender } as never);
|
|
|
|
Bun.which = (() => null) as typeof Bun.which;
|
|
try {
|
|
const loadPageSpy = vi.spyOn(scrapers, "loadPage").mockImplementation(async (requestedUrl: string) => {
|
|
if (requestedUrl === pageUrl) {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "text/html",
|
|
finalUrl: pageUrl,
|
|
content: pageHtml,
|
|
};
|
|
}
|
|
|
|
if (
|
|
[
|
|
`${pageUrl}.md`,
|
|
"https://example.com/docs/reference/llms.txt",
|
|
"https://example.com/docs/reference/llms.md",
|
|
"https://example.com/docs/llms.md",
|
|
].includes(requestedUrl)
|
|
) {
|
|
return {
|
|
ok: false,
|
|
status: 404,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: "",
|
|
};
|
|
}
|
|
|
|
if (requestedUrl === "https://example.com/docs/llms.txt") {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: `# Example Docs\n\n${"Section-scoped fallback. ".repeat(10)}`,
|
|
};
|
|
}
|
|
|
|
if (requestedUrl === "https://example.com/llms.txt") {
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: `# Example\n\n${"Site-wide fallback. ".repeat(10)}`,
|
|
};
|
|
}
|
|
|
|
return {
|
|
ok: false,
|
|
status: 404,
|
|
contentType: "text/plain",
|
|
finalUrl: requestedUrl,
|
|
content: "",
|
|
};
|
|
});
|
|
using _hook = hookFetch(() => new Response("blocked", { status: 500, statusText: "Blocked" }));
|
|
vi.spyOn(toolsManager, "ensureTool").mockResolvedValue("/usr/bin/trafilatura");
|
|
|
|
const result = await tool.execute("fetch-section-llms", { url: pageUrl });
|
|
const requestedUrls = loadPageSpy.mock.calls.map(([requestedUrl]) => requestedUrl);
|
|
const textBlock = result.content.find(content => content.type === "text");
|
|
|
|
expect(result.details?.method).toBe("llms.txt");
|
|
expect(result.details?.notes).toContain("Used llms.txt fallback: https://example.com/docs/llms.txt");
|
|
expect(textBlock?.type).toBe("text");
|
|
expect(textBlock?.text).toContain("Section-scoped fallback");
|
|
expect(requestedUrls).toContain("https://example.com/docs/llms.txt");
|
|
expect(requestedUrls).not.toContain("https://example.com/.well-known/llms.txt");
|
|
expect(requestedUrls).not.toContain("https://example.com/llms.txt");
|
|
expect(requestedUrls).not.toContain("https://example.com/llms.md");
|
|
} finally {
|
|
Bun.which = originalWhich;
|
|
execSpy.mockRestore();
|
|
}
|
|
});
|
|
});
|