fix: screenshot latency, somehow calling screencapture is faster ???
This commit is contained in:
Generated
+1
@@ -4227,6 +4227,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"syntect",
|
||||
"tempfile",
|
||||
"tiktoken-rs",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
|
||||
@@ -65,6 +65,7 @@ enigo = { version = "=0.6.1", default-features = false }
|
||||
xcap = { version = "=0.9.6", default-features = false }
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
tempfile = "=3.27.0"
|
||||
core-graphics = "0.25"
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
//! never race each other and every coordinate action is interpreted against the
|
||||
//! last composite frame returned to JavaScript.
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use std::process::{Command, Stdio};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
fmt,
|
||||
@@ -577,10 +579,90 @@ struct LayoutDisplay {
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MonitorSnapshot {
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
monitor: Monitor,
|
||||
display: LayoutDisplay,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn capture_quartz_screenshot(display: &LayoutDisplay) -> CoreResult<RgbaImage> {
|
||||
let file = tempfile::Builder::new()
|
||||
.prefix("omp-computer-")
|
||||
.suffix(".png")
|
||||
.tempfile()
|
||||
.map_err(|error| {
|
||||
DesktopError::new(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("failed to create temporary screenshot file: {error}"),
|
||||
)
|
||||
})?;
|
||||
let rect = format!("-R{},{},{},{}", display.x, display.y, display.width, display.height);
|
||||
let mut child = Command::new("/usr/sbin/screencapture")
|
||||
.arg("-x")
|
||||
.arg(rect)
|
||||
.arg(file.path())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.map_err(|error| {
|
||||
DesktopError::new(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("failed to start macOS screen capture: {error}"),
|
||||
)
|
||||
})?;
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let status = loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(status)) => break status,
|
||||
Ok(None) => {},
|
||||
Err(error) => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
return Err(DesktopError::new(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("failed while waiting for macOS screen capture: {error}"),
|
||||
));
|
||||
},
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
return Err(DesktopError::new(
|
||||
ErrorCode::CaptureFailed,
|
||||
"macOS screen capture exceeded its five-second deadline",
|
||||
));
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
};
|
||||
if !status.success() {
|
||||
return Err(DesktopError::permission_or(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("macOS screen capture exited with {status}"),
|
||||
));
|
||||
}
|
||||
image::open(file.path())
|
||||
.map_err(|error| {
|
||||
DesktopError::new(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("failed to decode macOS screenshot: {error}"),
|
||||
)
|
||||
})
|
||||
.map(DynamicImage::into_rgba8)
|
||||
}
|
||||
|
||||
fn capture_monitor_image(snapshot: &MonitorSnapshot) -> CoreResult<RgbaImage> {
|
||||
#[cfg(target_os = "macos")]
|
||||
return capture_quartz_screenshot(&snapshot.display);
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
snapshot.monitor.capture_image().map_err(|source| {
|
||||
DesktopError::permission_or(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("capture of display `{}` failed: {source}", snapshot.display.id),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
const fn same_display_rect(left: &LayoutDisplay, right: &LayoutDisplay) -> bool {
|
||||
left.x == right.x
|
||||
&& left.y == right.y
|
||||
@@ -984,6 +1066,7 @@ impl DesktopWorker {
|
||||
let scale = f64::from(monitor.scale_factor().map_err(capture_metadata_error)?);
|
||||
let is_primary = monitor.is_primary().map_err(capture_metadata_error)?;
|
||||
snapshots.push(MonitorSnapshot {
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
monitor,
|
||||
display: LayoutDisplay { id, name, x, y, width, height, scale, is_primary },
|
||||
});
|
||||
@@ -1051,13 +1134,9 @@ impl DesktopWorker {
|
||||
let mut images = Vec::with_capacity(snapshots.len());
|
||||
let mut native_scale = 1.0f64;
|
||||
for snapshot in &snapshots {
|
||||
let image = match snapshot.monitor.capture_image() {
|
||||
let image = match capture_monitor_image(snapshot) {
|
||||
Ok(image) => image,
|
||||
Err(source) => {
|
||||
let error = DesktopError::permission_or(
|
||||
ErrorCode::CaptureFailed,
|
||||
format!("capture of display `{}` failed: {source}", snapshot.display.id),
|
||||
);
|
||||
Err(error) => {
|
||||
self.record_capture_failure(&error);
|
||||
return Err(error);
|
||||
},
|
||||
|
||||
@@ -189,7 +189,7 @@ See [Tool approval mode](./approval-mode.md) for general policy resolution.
|
||||
|
||||
| Platform | Backend | Setup and current status |
|
||||
|---|---|---|
|
||||
| macOS x64/arm64 | Quartz/CoreGraphics capture; Quartz/CGEvent and native input | Supported. Grant Screen Recording and Accessibility. Real remote desktop execution was verified on Apple hardware; see [Verification boundary](#verification-boundary). |
|
||||
| macOS x64/arm64 | Bounded macOS `screencapture` service capture; Quartz/CGEvent and native input | Supported. Grant Screen Recording and Accessibility. Real remote desktop execution was verified on Apple hardware; see [Verification boundary](#verification-boundary). |
|
||||
| Linux x64/arm64, glibc/musl, X11 | Pure-Rust X11 capture and XTest input (`x11rb`), bundled in the core addon | Supported when a graphical session and `DISPLAY` are available. No GUI system libraries are required; the backend speaks the X protocol directly over the display socket. Requires the RandR and XTEST server extensions. |
|
||||
| Linux x64/arm64, glibc/musl, Wayland | XWayland capture; XTest input bridged by the compositor | Supported with an active XWayland `DISPLAY`. Pure Wayland capture (portal/PipeWire) is not implemented. Input delivery to native Wayland windows depends on the compositor's XWayland input bridge. |
|
||||
| Windows x64 | xcap capture; Win32 virtual-desktop pointer movement and native input | Implemented, including negative origins and secondary monitors. Not remotely exercised in this feature's verification. |
|
||||
|
||||
@@ -169,7 +169,7 @@ Every coordinate action in a batch maps through the same frame returned by the p
|
||||
|
||||
| Target | Native surface |
|
||||
|---|---|
|
||||
| `darwin-x64`, `darwin-arm64` | xcap/CoreGraphics capture, Quartz `CGEvent` pointer events, native input. Screen Recording preflight; Accessibility required operationally. |
|
||||
| `darwin-x64`, `darwin-arm64` | Bounded macOS `screencapture` service capture, Quartz `CGEvent` pointer events, native input. Screen Recording preflight; Accessibility required operationally. |
|
||||
| `linux-x64`, `linux-arm64` (glibc and musl) | Pure-Rust X11 backend bundled in the core addon: `x11rb` RustConnection capture (RandR monitors, `GetImage`) and XTest input with keysym mapping. No GUI system libraries linked; the X protocol is spoken over the display socket. |
|
||||
| `win32-x64` | xcap capture, native input, `SendInput` absolute movement over the virtual desktop. |
|
||||
| Other targets | Native package loader rejects unsupported platform tag. |
|
||||
|
||||
@@ -6,6 +6,10 @@
|
||||
|
||||
- Added a genuine native desktop backend for computer use, bundled in the core addon on every published platform: macOS Quartz/CGEvent, Windows Win32/`SendInput`, and a pure-Rust Linux X11 backend (`x11rb` capture over the display socket, XTest input with keysym mapping) that links no GUI system libraries — so Linux x64/arm64, glibc and musl are all supported and headless hosts are unaffected. Wayland sessions work through XWayland. Execute batches enforce a 60-second native deadline (`DESKTOP_DEADLINE_EXCEEDED`) and never emit input after it expires; unsupported pure-Wayland capture and out-of-XTest-range or negative-origin coordinate layouts fail closed.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed macOS computer screenshots taking roughly 30 seconds under Bun by replacing xcap's deprecated window-list capture with a bounded system capture path; direct screenshots now complete in under half a second on the verified host.
|
||||
|
||||
## [17.0.8] - 2026-07-22
|
||||
|
||||
### Added
|
||||
|
||||
Reference in New Issue
Block a user