feat(coding-agent/tools): simplified browser screenshot persistence and return paths
- Remove the per-call `save` option from `tab.screenshot()` to simplify usage. - Update `tab.screenshot()` to return the saved file path as a promise string. - Configure screenshot persistence to use daemon path or custom `browser.screenshotDir`. - Add comprehensive tests verifying temp path return and custom directory saving.
This commit is contained in:
can1357
@@ -8,6 +8,7 @@ Drives real Chromium tab; full puppeteer access via JS.
|
||||
- `tab` helpers (drop to raw puppeteer `page` for anything uncovered):
|
||||
Element handles: `tab.ref("e5")` / `tab.id(n)` return a handle you call methods on directly — `(await tab.id(n)).click()`. Handles are NOT selectors: `tab.click`/`type`/`fill`/`waitFor*` take STRING selectors only. Snapshot refs work in any selector slot: `tab.click("e5")` ≡ `tab.click("aria-ref=e5")`.
|
||||
Simple: `tab.goto`, `tab.click`, `tab.type`, `tab.fill`, `tab.press`, `tab.scroll`, `tab.scrollIntoView`, `tab.drag`, `tab.uploadFile`, `tab.select`, `tab.screenshot`, `tab.extract`, `tab.evaluate`.
|
||||
Screenshots: `tab.screenshot({ selector?, fullPage?, silent? })` saves to `browser.screenshotDir`, or OS temp when unset, then returns the path. It NEVER accepts a path.
|
||||
Waits: `tab.waitFor`, `tab.waitForSelector`, `tab.waitForUrl`, `tab.waitForResponse`, `tab.waitForNavigation`.
|
||||
Snapshots: `tab.observe()` → accessibility tree; `tab.ariaSnapshot()` → ARIA YAML with `[ref=eN]`.
|
||||
|
||||
|
||||
@@ -163,11 +163,11 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
||||
},
|
||||
},
|
||||
{
|
||||
caption: "Screenshot to look at the page — no save path",
|
||||
caption: "Capture a screenshot and return its saved path",
|
||||
call: {
|
||||
action: "run",
|
||||
name: "docs",
|
||||
code: "await tab.screenshot();",
|
||||
code: "return await tab.screenshot();",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -37,7 +37,6 @@ import type { CmuxSocketClient } from "./socket-client";
|
||||
interface ScreenshotOptions {
|
||||
selector?: string;
|
||||
fullPage?: boolean;
|
||||
save?: string;
|
||||
silent?: boolean;
|
||||
encoding?: "base64" | "binary";
|
||||
}
|
||||
@@ -487,7 +486,7 @@ export class CmuxTab {
|
||||
return content;
|
||||
}
|
||||
|
||||
async screenshot(opts: ScreenshotOptions = {}): Promise<ScreenshotResult> {
|
||||
async screenshot(opts: ScreenshotOptions = {}): Promise<string> {
|
||||
const context = this.#requireRunContext("tab.screenshot()");
|
||||
// The cmux daemon's `browser.screenshot` captures the surface viewport
|
||||
// only — it has no element-clip or full-page mode, and Bun.Image cannot
|
||||
@@ -518,20 +517,16 @@ export class CmuxTab {
|
||||
excludeWebP: context.session.excludeWebP,
|
||||
},
|
||||
);
|
||||
const explicitPath = opts.save ? resolveToCwd(opts.save, context.session.cwd) : undefined;
|
||||
const returnedPath = typeof result.path === "string" && result.path.length > 0 ? result.path : undefined;
|
||||
const saveFullRes = !!(explicitPath || context.session.browserScreenshotDir || returnedPath);
|
||||
const saveFullRes = !!context.session.browserScreenshotDir;
|
||||
const savedBuffer = saveFullRes ? buffer : Buffer.from(resized.buffer);
|
||||
const savedMimeType = saveFullRes ? captureMime : resized.mimeType;
|
||||
const ext = savedMimeType === "image/webp" ? "webp" : savedMimeType === "image/jpeg" ? "jpg" : "png";
|
||||
const dest =
|
||||
explicitPath ??
|
||||
(context.session.browserScreenshotDir
|
||||
? path.join(
|
||||
context.session.browserScreenshotDir,
|
||||
`screenshot-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, -1)}.${ext}`,
|
||||
)
|
||||
: (returnedPath ?? path.join(os.tmpdir(), `omp-sshots-${Snowflake.next()}.${ext}`)));
|
||||
const dest = context.session.browserScreenshotDir
|
||||
? path.join(
|
||||
context.session.browserScreenshotDir,
|
||||
`screenshot-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, -1)}.${ext}`,
|
||||
)
|
||||
: path.join(os.tmpdir(), `omp-sshots-${Snowflake.next()}.${ext}`);
|
||||
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
|
||||
await Bun.write(dest, savedBuffer);
|
||||
const info: ScreenshotResult = {
|
||||
@@ -556,7 +551,7 @@ export class CmuxTab {
|
||||
context.output.push({ type: "text", text: lines.join("\n") });
|
||||
context.output.push({ type: "image", data: resized.data, mimeType: resized.mimeType });
|
||||
}
|
||||
return info;
|
||||
return dest;
|
||||
}
|
||||
|
||||
async waitForUrl(pattern: string | RegExp, opts?: { timeout?: number }): Promise<string> {
|
||||
|
||||
@@ -11,7 +11,6 @@ import type {
|
||||
ElementHandle,
|
||||
ElementScreenshotOptions,
|
||||
HTTPResponse,
|
||||
ImageFormat,
|
||||
KeyInput,
|
||||
Page,
|
||||
SerializedAXNode,
|
||||
@@ -217,7 +216,6 @@ export function resolveWaitTimeout(cellTimeoutMs: number, explicit?: number): nu
|
||||
interface ScreenshotOptions {
|
||||
selector?: string;
|
||||
fullPage?: boolean;
|
||||
save?: string;
|
||||
silent?: boolean;
|
||||
}
|
||||
|
||||
@@ -233,7 +231,7 @@ interface TabApi {
|
||||
): Promise<void>;
|
||||
observe(opts?: { includeAll?: boolean; viewportOnly?: boolean }): Promise<Observation>;
|
||||
ariaSnapshot(selector?: string, opts?: AriaSnapshotOptions): Promise<string>;
|
||||
screenshot(opts?: ScreenshotOptions): Promise<ScreenshotResult>;
|
||||
screenshot(opts?: ScreenshotOptions): Promise<string>;
|
||||
extract(format?: ReadableFormat): Promise<string>;
|
||||
click(selector: string): Promise<void>;
|
||||
type(selector: string, text: string): Promise<void>;
|
||||
@@ -714,19 +712,6 @@ export function describeScreenshot(opts?: ScreenshotOptions): string {
|
||||
return "tab.screenshot()";
|
||||
}
|
||||
|
||||
/** Map an explicit save path's extension to a puppeteer capture format (default png). */
|
||||
export function imageFormatForPath(filePath: string): ImageFormat {
|
||||
switch (path.extname(filePath).toLowerCase()) {
|
||||
case ".webp":
|
||||
return "webp";
|
||||
case ".jpg":
|
||||
case ".jpeg":
|
||||
return "jpeg";
|
||||
default:
|
||||
return "png";
|
||||
}
|
||||
}
|
||||
|
||||
/** Summarize still-running helpers (oldest first) so a cell timeout names what stalled. */
|
||||
export function describeInflight(inflight: Map<number, InflightOp>): string {
|
||||
const now = Date.now();
|
||||
@@ -1532,7 +1517,7 @@ export class WorkerCore {
|
||||
screenshots: ScreenshotResult[],
|
||||
signal: AbortSignal | undefined,
|
||||
opts: ScreenshotOptions = {},
|
||||
): Promise<ScreenshotResult> {
|
||||
): Promise<string> {
|
||||
const page = this.#requirePage();
|
||||
// Multiple tabs can share one Chromium (sibling headless tabs on a shared
|
||||
// endpoint, cdp/app attach). CDP `Page.captureScreenshot` reads the
|
||||
@@ -1542,12 +1527,8 @@ export class WorkerCore {
|
||||
// already-active or freshly-closed target never fails the capture.
|
||||
await untilAborted(signal, () => page.bringToFront()).catch(() => undefined);
|
||||
const fullPage = opts.selector ? false : (opts.fullPage ?? false);
|
||||
// An explicit save path picks the full-res capture format: puppeteer encodes
|
||||
// png/jpeg/webp natively, so `save: "shot.webp"` gets real WebP bytes instead
|
||||
// of PNG bytes hiding behind a .webp name. Unknown/missing extensions stay PNG.
|
||||
const explicitPath = opts.save ? resolveToCwd(opts.save, session.cwd) : undefined;
|
||||
const captureType = explicitPath ? imageFormatForPath(explicitPath) : "png";
|
||||
const captureMime = `image/${captureType}` as const;
|
||||
const captureType = "png";
|
||||
const captureMime = "image/png" as const;
|
||||
let buffer: Buffer;
|
||||
if (opts.selector) {
|
||||
const handle =
|
||||
@@ -1581,20 +1562,16 @@ export class WorkerCore {
|
||||
{ type: "image", data: buffer.toBase64(), mimeType: captureMime },
|
||||
{ maxWidth: 1024, maxHeight: 1024, maxBytes: 150 * 1024, jpegQuality: 70, excludeWebP: session.excludeWebP },
|
||||
);
|
||||
const saveFullRes = !!(explicitPath || session.browserScreenshotDir);
|
||||
const saveFullRes = !!session.browserScreenshotDir;
|
||||
const savedBuffer = saveFullRes ? buffer : resized.buffer;
|
||||
const savedMimeType = saveFullRes ? captureMime : resized.mimeType;
|
||||
// Names must match the bytes we actually write: full-res follows the capture
|
||||
// format, the resized buffer is whichever of PNG/JPEG/WebP encoded smallest.
|
||||
const ext = savedMimeType === "image/webp" ? "webp" : savedMimeType === "image/jpeg" ? "jpg" : "png";
|
||||
const dest =
|
||||
explicitPath ??
|
||||
(session.browserScreenshotDir
|
||||
? path.join(
|
||||
session.browserScreenshotDir,
|
||||
`screenshot-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, -1)}.${ext}`,
|
||||
)
|
||||
: path.join(os.tmpdir(), `omp-sshots-${Snowflake.next()}.${ext}`));
|
||||
const dest = session.browserScreenshotDir
|
||||
? path.join(
|
||||
session.browserScreenshotDir,
|
||||
`screenshot-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, -1)}.${ext}`,
|
||||
)
|
||||
: path.join(os.tmpdir(), `omp-sshots-${Snowflake.next()}.${ext}`);
|
||||
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
|
||||
await Bun.write(dest, savedBuffer);
|
||||
const info: ScreenshotResult = {
|
||||
@@ -1616,7 +1593,7 @@ export class WorkerCore {
|
||||
output.push({ type: "text", text: lines.join("\n") });
|
||||
output.push({ type: "image", data: resized.data, mimeType: resized.mimeType });
|
||||
}
|
||||
return info;
|
||||
return dest;
|
||||
}
|
||||
|
||||
async #drag(from: DragTarget, to: DragTarget, signal: AbortSignal): Promise<void> {
|
||||
|
||||
Reference in New Issue
Block a user