From 8d6ed17811082d9d4b2520c0cda15321fb4d9161 Mon Sep 17 00:00:00 2001 From: roboomp Date: Fri, 7 Aug 2026 06:50:16 +0000 Subject: [PATCH] fix(computer): lazily requested wayland input permission Deferred libei and RemoteDesktop setup until the first native input operation. Read-only capability, window, display, and AX calls no longer request keyboard or pointer access. Used non-persistent portal grants and retained the portal session so backend teardown closes it explicitly. Fixes #7884 --- .../src/desktop/linux/wayland/libei.rs | 102 +++++++++----- .../src/desktop/linux/wayland/mod.rs | 127 +++++++++++++----- .../src/desktop/linux/wayland/portal.rs | 2 - docs/computer-use.md | 4 +- packages/natives/CHANGELOG.md | 4 + 5 files changed, 166 insertions(+), 73 deletions(-) diff --git a/crates/pi-natives/src/desktop/linux/wayland/libei.rs b/crates/pi-natives/src/desktop/linux/wayland/libei.rs index 94d70bdf3..c091b6c45 100644 --- a/crates/pi-natives/src/desktop/linux/wayland/libei.rs +++ b/crates/pi-natives/src/desktop/linux/wayland/libei.rs @@ -4,7 +4,7 @@ use std::{ }; use ashpd::desktop::{ - PersistMode, + PersistMode, Session, remote_desktop::{DeviceType, RemoteDesktop}, }; use reis::{ @@ -12,7 +12,6 @@ use reis::{ event::{Device, DeviceCapability, EiConvertEventIterator, EiEvent}, }; -use super::portal::{REMOTE_DESKTOP_TOKEN, read_token, store_token}; use crate::desktop::{ backend::{Modifiers, MouseButton, PointerEvent}, error::{CoreResult, DesktopError}, @@ -24,24 +23,46 @@ struct EiDevice { serial: u32, } +type RemoteDesktopSession = Session<'static, RemoteDesktop<'static>>; + +struct PortalSession { + runtime: tokio::runtime::Runtime, + session: RemoteDesktopSession, +} + pub(super) struct Libei { - context: ei::Context, - pointer: Option, - keyboard: Option, - sequence: u32, + context: ei::Context, + pointer: Option, + keyboard: Option, + sequence: u32, + portal_session: Option, +} + +impl Drop for Libei { + fn drop(&mut self) { + let Some(portal) = self.portal_session.take() else { + return; + }; + let _ = portal.runtime.block_on(portal.session.close()); + } } impl Libei { pub(super) fn new() -> CoreResult { - let context = match ei::Context::connect_to_env() { - Ok(Some(context)) => context, - Ok(None) => Self::portal_context()?, + let (context, portal_session) = match ei::Context::connect_to_env() { + Ok(Some(context)) => (context, None), + Ok(None) => { + let (context, session) = Self::portal_context()?; + (context, Some(session)) + }, Err(err) => return Err(DesktopError::permission_denied(format!("LIBEI_SOCKET: {err}"))), }; - let (_connection, mut events) = context + let mut backend = + Self { context, pointer: None, keyboard: None, sequence: 1, portal_session }; + let (_connection, mut events) = backend + .context .handshake_blocking("omp-computer", ei::handshake::ContextType::Sender) .map_err(|err| DesktopError::input_failed(format!("libei handshake: {err}")))?; - let mut backend = Self { context, pointer: None, keyboard: None, sequence: 1 }; backend.discover_devices(&mut events)?; if backend.pointer.is_none() && backend.keyboard.is_none() { return Err(DesktopError::permission_denied( @@ -51,14 +72,14 @@ impl Libei { Ok(backend) } - fn portal_context() -> CoreResult { + fn portal_context() -> CoreResult<(ei::Context, PortalSession)> { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|err| { DesktopError::input_failed(format!("RemoteDesktop portal runtime: {err}")) })?; - let fd = runtime + let (fd, session) = runtime .block_on(async { let portal = RemoteDesktop::new() .await @@ -67,31 +88,40 @@ impl Libei { .create_session() .await .map_err(|err| format!("RemoteDesktop CreateSession: {err}"))?; - let restore_token = read_token(REMOTE_DESKTOP_TOKEN); - portal - .select_devices( - &session, - DeviceType::Keyboard | DeviceType::Pointer, - restore_token.as_deref(), - PersistMode::ExplicitlyRevoked, - ) - .await - .map_err(|err| format!("RemoteDesktop SelectDevices: {err}"))?; - let response = portal - .start(&session, None) - .await - .map_err(|err| format!("RemoteDesktop Start: {err}"))? - .response() - .map_err(|err| format!("RemoteDesktop permission: {err}"))?; - store_token(REMOTE_DESKTOP_TOKEN, response.restore_token()); - portal - .connect_to_eis(&session) - .await - .map_err(|err| format!("RemoteDesktop ConnectToEIS: {err}")) + let fd = async { + portal + .select_devices( + &session, + DeviceType::Keyboard | DeviceType::Pointer, + None, + PersistMode::DoNot, + ) + .await + .map_err(|err| format!("RemoteDesktop SelectDevices: {err}"))?; + portal + .start(&session, None) + .await + .map_err(|err| format!("RemoteDesktop Start: {err}"))? + .response() + .map_err(|err| format!("RemoteDesktop permission: {err}"))?; + portal + .connect_to_eis(&session) + .await + .map_err(|err| format!("RemoteDesktop ConnectToEIS: {err}")) + } + .await; + match fd { + Ok(fd) => Ok((fd, session)), + Err(err) => { + let _ = session.close().await; + Err(err) + }, + } }) .map_err(DesktopError::permission_denied)?; - ei::Context::new(UnixStream::from(fd)) - .map_err(|err| DesktopError::input_failed(format!("libei portal socket: {err}"))) + let context = ei::Context::new(UnixStream::from(fd)) + .map_err(|err| DesktopError::input_failed(format!("libei portal socket: {err}")))?; + Ok((context, PortalSession { runtime, session })) } fn discover_devices(&mut self, events: &mut EiConvertEventIterator) -> CoreResult<()> { diff --git a/crates/pi-natives/src/desktop/linux/wayland/mod.rs b/crates/pi-natives/src/desktop/linux/wayland/mod.rs index 05042729e..36c5afdb5 100644 --- a/crates/pi-natives/src/desktop/linux/wayland/mod.rs +++ b/crates/pi-natives/src/desktop/linux/wayland/mod.rs @@ -1,6 +1,7 @@ #[cfg(feature = "wayland-pipewire")] mod capture; mod libei; +#[cfg(feature = "wayland-pipewire")] mod portal; use image::RgbaImage; @@ -35,11 +36,7 @@ impl WaylandBackend { Ok(ax) => (Some(ax), None), Err(err) => (None, Some(err)), }; - let (input, input_error) = match libei::Libei::new() { - Ok(input) => (Some(input), None), - Err(err) => (None, Some(err)), - }; - Self { display, ax, ax_error, input, input_error, displays: Vec::new() } + Self { display, ax, ax_error, input: None, input_error: None, displays: Vec::new() } } fn window_input_error(target: &Target, kind: &str) -> CoreResult<()> { @@ -53,16 +50,22 @@ impl WaylandBackend { Ok(()) } - fn prepare_input(&self, target: &Target, kind: &str) -> CoreResult<()> { + fn prepare_input(&mut self, target: &Target, kind: &str) -> CoreResult<&mut libei::Libei> { Self::window_input_error(target, kind)?; - if self.input.is_none() { - return Err(self.input_error.clone().unwrap_or_else(|| { - DesktopError::permission_denied( - "RemoteDesktop portal or LIBEI_SOCKET is required for Wayland input", - ) - })); + if self.input.is_none() && self.input_error.is_none() { + match libei::Libei::new() { + Ok(input) => self.input = Some(input), + Err(err) => self.input_error = Some(err), + } } - Ok(()) + if let Some(input) = self.input.as_mut() { + return Ok(input); + } + Err(self.input_error.clone().unwrap_or_else(|| { + DesktopError::permission_denied( + "RemoteDesktop portal or LIBEI_SOCKET is required for Wayland input", + ) + })) } #[cfg(feature = "wayland-pipewire")] @@ -97,6 +100,13 @@ impl WaylandBackend { impl Backend for WaylandBackend { fn capabilities(&mut self) -> DesktopCapabilities { + let input_permission = if self.input.is_some() { + "granted" + } else if self.input_error.is_some() { + "unavailable" + } else { + "prompt-or-granted" + }; DesktopCapabilities { backend: "wayland".to_string(), display_server: Some("wayland".to_string()), @@ -104,7 +114,7 @@ impl Backend for WaylandBackend { // wayland-pipewire feature; without it capture() hard-errors, so the // capability report must not advertise a capture the binary cannot do. capture: cfg!(feature = "wayland-pipewire"), - input: self.input.is_some(), + input: self.input_error.is_none(), ax: self.ax.is_some(), background_window_input: false, delivery_modes: vec!["background".to_string()], @@ -113,11 +123,7 @@ impl Backend for WaylandBackend { } else { "unavailable".to_string() }, - input_permission: if self.input.is_some() { - "granted".to_string() - } else { - "unavailable".to_string() - }, + input_permission: input_permission.to_string(), ax_permission: if self.ax.is_some() { "granted".to_string() } else { @@ -203,20 +209,12 @@ impl Backend for WaylandBackend { _frame: &FrameGeometry, _mode: DeliveryMode, ) -> CoreResult<()> { - self.prepare_input(target, "pointer input")?; - self - .input - .as_mut() - .ok_or_else(|| DesktopError::internal("Wayland input backend disappeared"))? - .pointer(ev) + self.prepare_input(target, "pointer input")?.pointer(ev) } fn type_text(&mut self, target: &Target, text: &str, _mode: DeliveryMode) -> CoreResult<()> { - self.prepare_input(target, "keyboard input")?; self - .input - .as_mut() - .ok_or_else(|| DesktopError::internal("Wayland input backend disappeared"))? + .prepare_input(target, "keyboard input")? .type_text(text) } @@ -226,11 +224,8 @@ impl Backend for WaylandBackend { keys: &[KeyName], _mode: DeliveryMode, ) -> CoreResult<()> { - self.prepare_input(target, "keyboard input")?; self - .input - .as_mut() - .ok_or_else(|| DesktopError::internal("Wayland input backend disappeared"))? + .prepare_input(target, "keyboard input")? .key_chord(keys) } @@ -248,8 +243,17 @@ impl Backend for WaylandBackend { #[cfg(test)] mod tests { + use std::{ + io::ErrorKind, + os::unix::net::UnixListener, + sync::{Mutex, mpsc}, + thread, + }; + use super::*; + static LIBEI_ENV_LOCK: Mutex<()> = Mutex::new(()); + fn backend_without_services() -> WaylandBackend { WaylandBackend { display: DisplaySelector::All, @@ -260,6 +264,63 @@ mod tests { displays: Vec::new(), } } + fn with_fake_libei(action: impl FnOnce(&mut WaylandBackend)) -> bool { + let _guard = LIBEI_ENV_LOCK.lock().expect("lock LIBEI_SOCKET test"); + let socket = std::env::temp_dir().join(format!("omp-libei-test-{}", std::process::id())); + let _ = std::fs::remove_file(&socket); + let listener = UnixListener::bind(&socket).expect("bind fake libei socket"); + listener + .set_nonblocking(true) + .expect("make fake libei socket nonblocking"); + let (stop_tx, stop_rx) = mpsc::channel(); + let accepted = thread::spawn(move || { + loop { + match listener.accept() { + Ok(_) => return true, + Err(err) if err.kind() == ErrorKind::WouldBlock => { + if !matches!( + stop_rx.recv_timeout(std::time::Duration::from_millis(10)), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + return false; + } + }, + Err(err) => panic!("fake libei listener: {err}"), + } + } + }); + let previous = std::env::var_os("LIBEI_SOCKET"); + unsafe { std::env::set_var("LIBEI_SOCKET", &socket) }; + let mut backend = WaylandBackend::new(DisplaySelector::All); + action(&mut backend); + let _ = stop_tx.send(()); + if let Some(previous) = previous { + unsafe { std::env::set_var("LIBEI_SOCKET", previous) }; + } else { + unsafe { std::env::remove_var("LIBEI_SOCKET") }; + } + let connected = accepted.join().expect("fake libei listener"); + let _ = std::fs::remove_file(socket); + connected + } + + #[test] + fn readonly_backend_creation_does_not_connect_to_libei() { + let mut capabilities = None; + let connected = with_fake_libei(|backend| capabilities = Some(backend.capabilities())); + assert!(!connected, "read-only backend construction connected to libei"); + let capabilities = capabilities.expect("Wayland capabilities"); + assert!(capabilities.input); + assert_eq!(capabilities.input_permission, "prompt-or-granted"); + } + + #[test] + fn desktop_input_connects_to_libei_lazily() { + let connected = with_fake_libei(|backend| { + let _ = backend.type_text(&Target::Desktop, "hello", DeliveryMode::Foreground); + }); + assert!(connected, "desktop input did not connect to libei"); + } #[test] fn window_foreground_delivery_reports_compositor_constraint() { diff --git a/crates/pi-natives/src/desktop/linux/wayland/portal.rs b/crates/pi-natives/src/desktop/linux/wayland/portal.rs index a2a233ccc..16fa7d6ce 100644 --- a/crates/pi-natives/src/desktop/linux/wayland/portal.rs +++ b/crates/pi-natives/src/desktop/linux/wayland/portal.rs @@ -1,7 +1,5 @@ use std::{fs, path::PathBuf}; -pub(super) const REMOTE_DESKTOP_TOKEN: &str = "remote-desktop-token"; - fn token_path(name: &str) -> Option { let base = std::env::var_os("XDG_STATE_HOME") .map(PathBuf::from) diff --git a/docs/computer-use.md b/docs/computer-use.md index 55b3a1e27..f66829653 100644 --- a/docs/computer-use.md +++ b/docs/computer-use.md @@ -131,11 +131,11 @@ await wait( | ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | macOS x64/arm64 | ScreenCapture/Quartz plus native AX and input. Grant Screen Recording for capture and Accessibility for input/AX, then restart the launching host. | | Linux X11 x64/arm64 | X11 capture/input and AT-SPI accessibility. Requires a readable display plus RandR/XTEST. | -| Linux Wayland x64/arm64 | RemoteDesktop portal or `LIBEI_SOCKET` input and AT-SPI accessibility. ScreenCast portal/PipeWire capture ships only in builds compiled with the `wayland-pipewire` Cargo feature; released binaries omit it, so `capabilities()` reports `capture: false` there. Portal permission prompts and compositor restrictions apply; background per-window native input is unavailable. | +| Linux Wayland x64/arm64 | RemoteDesktop portal or `LIBEI_SOCKET` input and AT-SPI accessibility. ScreenCast portal/PipeWire capture ships only in builds compiled with the `wayland-pipewire` Cargo feature; released binaries omit it, so `capabilities()` reports `capture: false` there. RemoteDesktop permission is requested lazily on first native input, is not persisted, and closes with the desktop session; read-only window/AX inspection does not request it. Compositor restrictions apply; background per-window native input is unavailable. | | Windows x64 | Native display/window capture, Win32 input, and UI Automation accessibility. | | Other published targets | Unsupported unless the native addon reports capabilities. | -Inspect `desktop.capabilities()` rather than assuming capture, input, AX, or permission state. On Wayland, released builds are compiled without the `wayland-pipewire` feature, so `capabilities()` reports `capture: false`; where the feature is present, a missing portal/PipeWire feature or denied RemoteDesktop portal is reported as a capture/input/permission failure rather than falling back to X11. +Inspect `desktop.capabilities()` rather than assuming capture, input, AX, or permission state. On Wayland, input reports `prompt-or-granted` before first native input without opening a RemoteDesktop session. Released builds are compiled without the `wayland-pipewire` feature, so `capabilities()` reports `capture: false`; where the feature is present, a missing portal/PipeWire feature or denied RemoteDesktop portal is reported as a capture/input/permission failure rather than falling back to X11. ## Safety and troubleshooting diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index 04d2f4b98..ca948a3dc 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Fixed read-only Wayland `computer` calls acquiring persistent keyboard and pointer control; RemoteDesktop input permission is now requested only on first input, is not persisted, and closes with the desktop session ([#7884](https://github.com/can1357/oh-my-pi/issues/7884)). + ## [17.2.10] - 2026-08-06 ### Fixed