diff --git a/Cargo.lock b/Cargo.lock index 4d54c7e0b..2f2e5d5ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4227,6 +4227,7 @@ dependencies = [ "serde_json", "smallvec", "syntect", + "tempfile", "tiktoken-rs", "tokio", "tokio-util", diff --git a/crates/pi-natives/Cargo.toml b/crates/pi-natives/Cargo.toml index 334fd85ac..2d8907311 100644 --- a/crates/pi-natives/Cargo.toml +++ b/crates/pi-natives/Cargo.toml @@ -65,6 +65,7 @@ enigo = { version = "=0.6.1", default-features = false } xcap = { version = "=0.9.6", default-features = false } [target.'cfg(target_os = "macos")'.dependencies] +tempfile = "=3.27.0" core-graphics = "0.25" [target.'cfg(unix)'.dependencies] diff --git a/crates/pi-natives/src/desktop.rs b/crates/pi-natives/src/desktop.rs index 3decae4f7..5556d586f 100644 --- a/crates/pi-natives/src/desktop.rs +++ b/crates/pi-natives/src/desktop.rs @@ -5,6 +5,8 @@ //! never race each other and every coordinate action is interpreted against the //! last composite frame returned to JavaScript. +#[cfg(target_os = "macos")] +use std::process::{Command, Stdio}; use std::{ collections::HashSet, fmt, @@ -577,10 +579,90 @@ struct LayoutDisplay { #[derive(Debug)] struct MonitorSnapshot { + #[cfg(not(target_os = "macos"))] monitor: Monitor, display: LayoutDisplay, } +#[cfg(target_os = "macos")] +fn capture_quartz_screenshot(display: &LayoutDisplay) -> CoreResult { + let file = tempfile::Builder::new() + .prefix("omp-computer-") + .suffix(".png") + .tempfile() + .map_err(|error| { + DesktopError::new( + ErrorCode::CaptureFailed, + format!("failed to create temporary screenshot file: {error}"), + ) + })?; + let rect = format!("-R{},{},{},{}", display.x, display.y, display.width, display.height); + let mut child = Command::new("/usr/sbin/screencapture") + .arg("-x") + .arg(rect) + .arg(file.path()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| { + DesktopError::new( + ErrorCode::CaptureFailed, + format!("failed to start macOS screen capture: {error}"), + ) + })?; + let deadline = Instant::now() + Duration::from_secs(5); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) => {}, + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(DesktopError::new( + ErrorCode::CaptureFailed, + format!("failed while waiting for macOS screen capture: {error}"), + )); + }, + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Err(DesktopError::new( + ErrorCode::CaptureFailed, + "macOS screen capture exceeded its five-second deadline", + )); + } + thread::sleep(Duration::from_millis(10)); + }; + if !status.success() { + return Err(DesktopError::permission_or( + ErrorCode::CaptureFailed, + format!("macOS screen capture exited with {status}"), + )); + } + image::open(file.path()) + .map_err(|error| { + DesktopError::new( + ErrorCode::CaptureFailed, + format!("failed to decode macOS screenshot: {error}"), + ) + }) + .map(DynamicImage::into_rgba8) +} + +fn capture_monitor_image(snapshot: &MonitorSnapshot) -> CoreResult { + #[cfg(target_os = "macos")] + return capture_quartz_screenshot(&snapshot.display); + #[cfg(not(target_os = "macos"))] + snapshot.monitor.capture_image().map_err(|source| { + DesktopError::permission_or( + ErrorCode::CaptureFailed, + format!("capture of display `{}` failed: {source}", snapshot.display.id), + ) + }) +} + const fn same_display_rect(left: &LayoutDisplay, right: &LayoutDisplay) -> bool { left.x == right.x && left.y == right.y @@ -984,6 +1066,7 @@ impl DesktopWorker { let scale = f64::from(monitor.scale_factor().map_err(capture_metadata_error)?); let is_primary = monitor.is_primary().map_err(capture_metadata_error)?; snapshots.push(MonitorSnapshot { + #[cfg(not(target_os = "macos"))] monitor, display: LayoutDisplay { id, name, x, y, width, height, scale, is_primary }, }); @@ -1051,13 +1134,9 @@ impl DesktopWorker { let mut images = Vec::with_capacity(snapshots.len()); let mut native_scale = 1.0f64; for snapshot in &snapshots { - let image = match snapshot.monitor.capture_image() { + let image = match capture_monitor_image(snapshot) { Ok(image) => image, - Err(source) => { - let error = DesktopError::permission_or( - ErrorCode::CaptureFailed, - format!("capture of display `{}` failed: {source}", snapshot.display.id), - ); + Err(error) => { self.record_capture_failure(&error); return Err(error); }, diff --git a/docs/computer-use.md b/docs/computer-use.md index f09014b0c..6f77a3ead 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -189,7 +189,7 @@ See [Tool approval mode](./approval-mode.md) for general policy resolution. | Platform | Backend | Setup and current status | |---|---|---| -| macOS x64/arm64 | Quartz/CoreGraphics capture; Quartz/CGEvent and native input | Supported. Grant Screen Recording and Accessibility. Real remote desktop execution was verified on Apple hardware; see [Verification boundary](#verification-boundary). | +| macOS x64/arm64 | Bounded macOS `screencapture` service capture; Quartz/CGEvent and native input | Supported. Grant Screen Recording and Accessibility. Real remote desktop execution was verified on Apple hardware; see [Verification boundary](#verification-boundary). | | Linux x64/arm64, glibc/musl, X11 | Pure-Rust X11 capture and XTest input (`x11rb`), bundled in the core addon | Supported when a graphical session and `DISPLAY` are available. No GUI system libraries are required; the backend speaks the X protocol directly over the display socket. Requires the RandR and XTEST server extensions. | | Linux x64/arm64, glibc/musl, Wayland | XWayland capture; XTest input bridged by the compositor | Supported with an active XWayland `DISPLAY`. Pure Wayland capture (portal/PipeWire) is not implemented. Input delivery to native Wayland windows depends on the compositor's XWayland input bridge. | | Windows x64 | xcap capture; Win32 virtual-desktop pointer movement and native input | Implemented, including negative origins and secondary monitors. Not remotely exercised in this feature's verification. | diff --git a/docs/tools/computer.md b/docs/tools/computer.md index 0764b2104..dc59d159f 100644 --- a/docs/tools/computer.md +++ b/docs/tools/computer.md @@ -169,7 +169,7 @@ Every coordinate action in a batch maps through the same frame returned by the p | Target | Native surface | |---|---| -| `darwin-x64`, `darwin-arm64` | xcap/CoreGraphics capture, Quartz `CGEvent` pointer events, native input. Screen Recording preflight; Accessibility required operationally. | +| `darwin-x64`, `darwin-arm64` | Bounded macOS `screencapture` service capture, Quartz `CGEvent` pointer events, native input. Screen Recording preflight; Accessibility required operationally. | | `linux-x64`, `linux-arm64` (glibc and musl) | Pure-Rust X11 backend bundled in the core addon: `x11rb` RustConnection capture (RandR monitors, `GetImage`) and XTest input with keysym mapping. No GUI system libraries linked; the X protocol is spoken over the display socket. | | `win32-x64` | xcap capture, native input, `SendInput` absolute movement over the virtual desktop. | | Other targets | Native package loader rejects unsupported platform tag. | diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index e767274c5..804004ea2 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -6,6 +6,10 @@ - Added a genuine native desktop backend for computer use, bundled in the core addon on every published platform: macOS Quartz/CGEvent, Windows Win32/`SendInput`, and a pure-Rust Linux X11 backend (`x11rb` capture over the display socket, XTest input with keysym mapping) that links no GUI system libraries — so Linux x64/arm64, glibc and musl are all supported and headless hosts are unaffected. Wayland sessions work through XWayland. Execute batches enforce a 60-second native deadline (`DESKTOP_DEADLINE_EXCEEDED`) and never emit input after it expires; unsupported pure-Wayland capture and out-of-XTest-range or negative-origin coordinate layouts fail closed. +### Fixed + +- Fixed macOS computer screenshots taking roughly 30 seconds under Bun by replacing xcap's deprecated window-list capture with a bounded system capture path; direct screenshots now complete in under half a second on the verified host. + ## [17.0.8] - 2026-07-22 ### Added