fix: screenshot latency, somehow calling screencapture is faster ???

This commit is contained in:
can1357
2026-07-24 06:39:49 +02:00
parent aad7ee8fa3
commit 11eb003fc8
6 changed files with 93 additions and 8 deletions
Generated
+1
View File
@@ -4227,6 +4227,7 @@ dependencies = [
"serde_json",
"smallvec",
"syntect",
"tempfile",
"tiktoken-rs",
"tokio",
"tokio-util",
+1
View File
@@ -65,6 +65,7 @@ enigo = { version = "=0.6.1", default-features = false }
xcap = { version = "=0.9.6", default-features = false }
[target.'cfg(target_os = "macos")'.dependencies]
tempfile = "=3.27.0"
core-graphics = "0.25"
[target.'cfg(unix)'.dependencies]
+85 -6
View File
@@ -5,6 +5,8 @@
//! never race each other and every coordinate action is interpreted against the
//! last composite frame returned to JavaScript.
#[cfg(target_os = "macos")]
use std::process::{Command, Stdio};
use std::{
collections::HashSet,
fmt,
@@ -577,10 +579,90 @@ struct LayoutDisplay {
#[derive(Debug)]
struct MonitorSnapshot {
#[cfg(not(target_os = "macos"))]
monitor: Monitor,
display: LayoutDisplay,
}
#[cfg(target_os = "macos")]
fn capture_quartz_screenshot(display: &LayoutDisplay) -> CoreResult<RgbaImage> {
let file = tempfile::Builder::new()
.prefix("omp-computer-")
.suffix(".png")
.tempfile()
.map_err(|error| {
DesktopError::new(
ErrorCode::CaptureFailed,
format!("failed to create temporary screenshot file: {error}"),
)
})?;
let rect = format!("-R{},{},{},{}", display.x, display.y, display.width, display.height);
let mut child = Command::new("/usr/sbin/screencapture")
.arg("-x")
.arg(rect)
.arg(file.path())
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|error| {
DesktopError::new(
ErrorCode::CaptureFailed,
format!("failed to start macOS screen capture: {error}"),
)
})?;
let deadline = Instant::now() + Duration::from_secs(5);
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) => {},
Err(error) => {
let _ = child.kill();
let _ = child.wait();
return Err(DesktopError::new(
ErrorCode::CaptureFailed,
format!("failed while waiting for macOS screen capture: {error}"),
));
},
}
if Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
return Err(DesktopError::new(
ErrorCode::CaptureFailed,
"macOS screen capture exceeded its five-second deadline",
));
}
thread::sleep(Duration::from_millis(10));
};
if !status.success() {
return Err(DesktopError::permission_or(
ErrorCode::CaptureFailed,
format!("macOS screen capture exited with {status}"),
));
}
image::open(file.path())
.map_err(|error| {
DesktopError::new(
ErrorCode::CaptureFailed,
format!("failed to decode macOS screenshot: {error}"),
)
})
.map(DynamicImage::into_rgba8)
}
fn capture_monitor_image(snapshot: &MonitorSnapshot) -> CoreResult<RgbaImage> {
#[cfg(target_os = "macos")]
return capture_quartz_screenshot(&snapshot.display);
#[cfg(not(target_os = "macos"))]
snapshot.monitor.capture_image().map_err(|source| {
DesktopError::permission_or(
ErrorCode::CaptureFailed,
format!("capture of display `{}` failed: {source}", snapshot.display.id),
)
})
}
const fn same_display_rect(left: &LayoutDisplay, right: &LayoutDisplay) -> bool {
left.x == right.x
&& left.y == right.y
@@ -984,6 +1066,7 @@ impl DesktopWorker {
let scale = f64::from(monitor.scale_factor().map_err(capture_metadata_error)?);
let is_primary = monitor.is_primary().map_err(capture_metadata_error)?;
snapshots.push(MonitorSnapshot {
#[cfg(not(target_os = "macos"))]
monitor,
display: LayoutDisplay { id, name, x, y, width, height, scale, is_primary },
});
@@ -1051,13 +1134,9 @@ impl DesktopWorker {
let mut images = Vec::with_capacity(snapshots.len());
let mut native_scale = 1.0f64;
for snapshot in &snapshots {
let image = match snapshot.monitor.capture_image() {
let image = match capture_monitor_image(snapshot) {
Ok(image) => image,
Err(source) => {
let error = DesktopError::permission_or(
ErrorCode::CaptureFailed,
format!("capture of display `{}` failed: {source}", snapshot.display.id),
);
Err(error) => {
self.record_capture_failure(&error);
return Err(error);
},
+1 -1
View File
@@ -189,7 +189,7 @@ See [Tool approval mode](./approval-mode.md) for general policy resolution.
| Platform | Backend | Setup and current status |
|---|---|---|
| macOS x64/arm64 | Quartz/CoreGraphics capture; Quartz/CGEvent and native input | Supported. Grant Screen Recording and Accessibility. Real remote desktop execution was verified on Apple hardware; see [Verification boundary](#verification-boundary). |
| macOS x64/arm64 | Bounded macOS `screencapture` service capture; Quartz/CGEvent and native input | Supported. Grant Screen Recording and Accessibility. Real remote desktop execution was verified on Apple hardware; see [Verification boundary](#verification-boundary). |
| Linux x64/arm64, glibc/musl, X11 | Pure-Rust X11 capture and XTest input (`x11rb`), bundled in the core addon | Supported when a graphical session and `DISPLAY` are available. No GUI system libraries are required; the backend speaks the X protocol directly over the display socket. Requires the RandR and XTEST server extensions. |
| Linux x64/arm64, glibc/musl, Wayland | XWayland capture; XTest input bridged by the compositor | Supported with an active XWayland `DISPLAY`. Pure Wayland capture (portal/PipeWire) is not implemented. Input delivery to native Wayland windows depends on the compositor's XWayland input bridge. |
| Windows x64 | xcap capture; Win32 virtual-desktop pointer movement and native input | Implemented, including negative origins and secondary monitors. Not remotely exercised in this feature's verification. |
+1 -1
View File
@@ -169,7 +169,7 @@ Every coordinate action in a batch maps through the same frame returned by the p
| Target | Native surface |
|---|---|
| `darwin-x64`, `darwin-arm64` | xcap/CoreGraphics capture, Quartz `CGEvent` pointer events, native input. Screen Recording preflight; Accessibility required operationally. |
| `darwin-x64`, `darwin-arm64` | Bounded macOS `screencapture` service capture, Quartz `CGEvent` pointer events, native input. Screen Recording preflight; Accessibility required operationally. |
| `linux-x64`, `linux-arm64` (glibc and musl) | Pure-Rust X11 backend bundled in the core addon: `x11rb` RustConnection capture (RandR monitors, `GetImage`) and XTest input with keysym mapping. No GUI system libraries linked; the X protocol is spoken over the display socket. |
| `win32-x64` | xcap capture, native input, `SendInput` absolute movement over the virtual desktop. |
| Other targets | Native package loader rejects unsupported platform tag. |
+4
View File
@@ -6,6 +6,10 @@
- Added a genuine native desktop backend for computer use, bundled in the core addon on every published platform: macOS Quartz/CGEvent, Windows Win32/`SendInput`, and a pure-Rust Linux X11 backend (`x11rb` capture over the display socket, XTest input with keysym mapping) that links no GUI system libraries — so Linux x64/arm64, glibc and musl are all supported and headless hosts are unaffected. Wayland sessions work through XWayland. Execute batches enforce a 60-second native deadline (`DESKTOP_DEADLINE_EXCEEDED`) and never emit input after it expires; unsupported pure-Wayland capture and out-of-XTest-range or negative-origin coordinate layouts fail closed.
### Fixed
- Fixed macOS computer screenshots taking roughly 30 seconds under Bun by replacing xcap's deprecated window-list capture with a bounded system capture path; direct screenshots now complete in under half a second on the verified host.
## [17.0.8] - 2026-07-22
### Added