agent_seat_linux/
harness.rs

1use std::ffi::{OsStr, OsString};
2use std::fmt;
3use std::path::{Path, PathBuf};
4use std::process::{Child, Command, Stdio};
5use std::sync::Arc;
6use std::time::Duration;
7
8use crate::{AgentSeat, CapturedFrame, SeatApp, SeatError};
9
10/// Error returned by the high-level computer-use API.
11///
12/// Wraps the low-level [`SeatError`] when one occurs; the wrapped error is
13/// exposed through [`std::error::Error::source`].
14#[derive(Debug)]
15pub struct Error {
16    message: String,
17    source: Option<SeatError>,
18}
19
20impl Error {
21    fn new(message: impl Into<String>) -> Self {
22        Self {
23            message: message.into(),
24            source: None,
25        }
26    }
27}
28
29impl fmt::Display for Error {
30    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
31        formatter.write_str(&self.message)
32    }
33}
34
35impl std::error::Error for Error {
36    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
37        self.source
38            .as_ref()
39            .map(|error| error as &(dyn std::error::Error + 'static))
40    }
41}
42
43impl From<SeatError> for Error {
44    fn from(error: SeatError) -> Self {
45        Self {
46            message: error.to_string(),
47            source: Some(error),
48        }
49    }
50}
51
52impl From<String> for Error {
53    fn from(message: String) -> Self {
54        Self::new(message)
55    }
56}
57
58impl From<std::io::Error> for Error {
59    fn from(error: std::io::Error) -> Self {
60        Self::from(SeatError::from(error))
61    }
62}
63
64/// Result type used by the high-level API.
65pub type Result<T> = std::result::Result<T, Error>;
66
67/// Pointer buttons understood by [`ControlledApp::click`].
68#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub enum PointerButton {
70    /// Primary pointer button.
71    Left,
72    /// Secondary/context pointer button.
73    Right,
74    /// Middle pointer button, commonly used for autoscroll or paste.
75    Middle,
76}
77
78impl PointerButton {
79    fn evdev_code(self) -> u32 {
80        match self {
81            Self::Left => 0x110,
82            Self::Right => 0x111,
83            Self::Middle => 0x112,
84        }
85    }
86}
87
88/// Display transport selected for a controlled application.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub enum Transport {
91    /// Application connected directly through the private Wayland socket.
92    NativeWayland,
93    /// Application connected through the authenticated XWayland bridge.
94    Xwayland,
95}
96
97/// Cloneable description of an application to launch.
98#[derive(Debug, Clone)]
99pub struct LaunchConfig {
100    program: OsString,
101    arguments: Vec<OsString>,
102    environment: Vec<(OsString, Option<OsString>)>,
103    current_dir: Option<PathBuf>,
104    inherit_diagnostics: bool,
105}
106
107impl LaunchConfig {
108    /// Create a launch configuration for an executable name or path.
109    pub fn new(program: impl Into<OsString>) -> Self {
110        Self {
111            program: program.into(),
112            arguments: Vec::new(),
113            environment: Vec::new(),
114            current_dir: None,
115            inherit_diagnostics: false,
116        }
117    }
118
119    /// Append one command-line argument.
120    pub fn arg(mut self, argument: impl Into<OsString>) -> Self {
121        self.arguments.push(argument.into());
122        self
123    }
124
125    /// Append several command-line arguments.
126    pub fn args<I, S>(mut self, arguments: I) -> Self
127    where
128        I: IntoIterator<Item = S>,
129        S: Into<OsString>,
130    {
131        self.arguments.extend(arguments.into_iter().map(Into::into));
132        self
133    }
134
135    /// Set one environment variable for the application.
136    pub fn env(mut self, key: impl Into<OsString>, value: impl Into<OsString>) -> Self {
137        self.environment.push((key.into(), Some(value.into())));
138        self
139    }
140
141    /// Remove one inherited environment variable from the application.
142    pub fn env_remove(mut self, key: impl Into<OsString>) -> Self {
143        self.environment.push((key.into(), None));
144        self
145    }
146
147    /// Set the application's working directory.
148    pub fn current_dir(mut self, directory: impl Into<PathBuf>) -> Self {
149        self.current_dir = Some(directory.into());
150        self
151    }
152
153    /// Inherit stdout and stderr, which is useful while integrating a new
154    /// toolkit. They are null by default so controlled apps do not block on
155    /// closed harness pipes.
156    pub fn inherit_diagnostics(mut self, inherit: bool) -> Self {
157        self.inherit_diagnostics = inherit;
158        self
159    }
160
161    /// Executable configured for this launch.
162    pub fn program(&self) -> &OsStr {
163        &self.program
164    }
165
166    fn environment_value(&self, key: &str) -> Option<OsString> {
167        self.environment
168            .iter()
169            .rev()
170            .find(|(candidate, _)| candidate == OsStr::new(key))
171            .and_then(|(_, value)| value.clone())
172            .or_else(|| std::env::var_os(key))
173    }
174
175    fn command(&self) -> Command {
176        let mut command = Command::new(&self.program);
177        command.args(&self.arguments).stdin(Stdio::null());
178        if self.inherit_diagnostics {
179            command.stdout(Stdio::inherit()).stderr(Stdio::inherit());
180        } else {
181            command.stdout(Stdio::null()).stderr(Stdio::null());
182        }
183        if let Some(directory) = &self.current_dir {
184            command.current_dir(directory);
185        }
186        for (key, value) in &self.environment {
187            if let Some(value) = value {
188                command.env(key, value);
189            } else {
190                command.env_remove(key);
191            }
192        }
193        command
194    }
195}
196
197/// Builder for a standalone Linux computer-use harness.
198#[derive(Debug, Clone)]
199pub struct ComputerUseBuilder {
200    native_timeout: Duration,
201    fallback_timeout: Duration,
202    bridge_width: u16,
203    bridge_height: u16,
204}
205
206impl Default for ComputerUseBuilder {
207    fn default() -> Self {
208        Self {
209            native_timeout: Duration::from_secs(8),
210            fallback_timeout: Duration::from_secs(30),
211            bridge_width: 1280,
212            bridge_height: 800,
213        }
214    }
215}
216
217impl ComputerUseBuilder {
218    /// Change how long native Wayland gets to produce a readable frame.
219    pub fn native_timeout(mut self, timeout: Duration) -> Self {
220        self.native_timeout = timeout;
221        self
222    }
223
224    /// Change how long the XWayland fallback gets to produce a frame.
225    pub fn fallback_timeout(mut self, timeout: Duration) -> Self {
226        self.fallback_timeout = timeout;
227        self
228    }
229
230    /// Set the compatibility bridge canvas size.
231    pub fn bridge_geometry(mut self, width: u16, height: u16) -> Self {
232        self.bridge_width = width;
233        self.bridge_height = height;
234        self
235    }
236
237    /// Start the private seat and return a ready harness.
238    pub fn build(self) -> Result<ComputerUse> {
239        if self.native_timeout.is_zero() || self.fallback_timeout.is_zero() {
240            return Err(Error::new("launch timeouts must be greater than zero"));
241        }
242        let seat = AgentSeat::create().map_err(Error::from)?;
243        Ok(ComputerUse {
244            seat,
245            native_timeout: self.native_timeout,
246            fallback_timeout: self.fallback_timeout,
247            bridge_width: self.bridge_width,
248            bridge_height: self.bridge_height,
249        })
250    }
251}
252
253/// Complete app-scoped computer-use harness.
254pub struct ComputerUse {
255    seat: Arc<AgentSeat>,
256    native_timeout: Duration,
257    fallback_timeout: Duration,
258    bridge_width: u16,
259    bridge_height: u16,
260}
261
262impl ComputerUse {
263    /// Start a harness with production defaults.
264    pub fn new() -> Result<Self> {
265        ComputerUseBuilder::default().build()
266    }
267
268    /// Configure launch timeouts and bridge geometry.
269    pub fn builder() -> ComputerUseBuilder {
270        ComputerUseBuilder::default()
271    }
272
273    /// The private `WAYLAND_DISPLAY` socket, for advanced integrations.
274    pub fn socket_name(&self) -> String {
275        self.seat.socket_name()
276    }
277
278    /// Launch and bind an application. Native Wayland is tried first. If the
279    /// application does not expose a readable app-sized frame, the same launch
280    /// is retried through an authenticated XWayland bridge automatically.
281    pub fn launch(&self, config: LaunchConfig) -> Result<ControlledApp> {
282        let before = self.seat.connected_pids();
283        let mut command = config.command();
284        configure_native_wayland(&self.seat, &mut command);
285        let mut child = crate::process::spawn_owned_child(&mut command)
286            .map_err(|error| launch_error(&config, "native Wayland", error))?;
287        let pid = child.id();
288
289        let native_started = std::time::Instant::now();
290        while native_started.elapsed() < self.native_timeout {
291            if let Some(app) = self.seat.new_capturable_app(&before) {
292                self.seat.bind_app_for_pid(pid, &app);
293                return Ok(ControlledApp::new(child, app, Transport::NativeWayland));
294            }
295            if child.try_wait().map_err(Error::from)?.is_some() {
296                break;
297            }
298            std::thread::sleep(Duration::from_millis(50));
299        }
300
301        kill_and_wait(&mut child);
302        self.launch_xwayland(config)
303    }
304
305    fn launch_xwayland(&self, config: LaunchConfig) -> Result<ControlledApp> {
306        let before = self.seat.connected_pids();
307        let bridge = self
308            .seat
309            .start_xwayland_bridge(self.bridge_width, self.bridge_height)
310            .map_err(Error::from)?;
311        let mut command = config.command();
312        configure_xwayland(&config, &bridge, &mut command);
313        let mut child = crate::process::spawn_owned_child(&mut command)
314            .map_err(|error| launch_error(&config, "XWayland", error))?;
315        let pid = child.id();
316
317        let fallback_started = std::time::Instant::now();
318        let Some(app) = self
319            .seat
320            .wait_new_capturable_app(&before, self.fallback_timeout)
321        else {
322            kill_and_wait(&mut child);
323            return Err(Error::new(format!(
324                "{} did not expose a readable frame through native Wayland or XWayland",
325                Path::new(config.program()).display()
326            )));
327        };
328
329        let content_deadline = fallback_started + self.fallback_timeout;
330        while std::time::Instant::now() < content_deadline {
331            if app
332                .capture_frame()
333                .is_ok_and(|frame| frame_has_visible_content(&frame))
334            {
335                self.seat.bind_app_for_pid(pid, &app);
336                if let Err(error) = self.seat.adopt_xwayland_bridge(bridge) {
337                    kill_and_wait(&mut child);
338                    return Err(Error::from(error));
339                }
340                return Ok(ControlledApp::new(child, app, Transport::Xwayland));
341            }
342            std::thread::sleep(Duration::from_millis(50));
343        }
344
345        kill_and_wait(&mut child);
346        Err(Error::new(format!(
347            "{} connected through XWayland but did not render visible application content",
348            Path::new(config.program()).display()
349        )))
350    }
351
352    /// Explicitly stop the seat and every compatibility bridge.
353    pub fn close(&self) {
354        self.seat.close();
355    }
356}
357
358impl Drop for ComputerUse {
359    fn drop(&mut self) {
360        self.seat.close();
361    }
362}
363
364/// A launched application with capture and input methods bound to its exact
365/// seat connection.
366pub struct ControlledApp {
367    child: Option<Child>,
368    app: Arc<SeatApp>,
369    transport: Transport,
370}
371
372impl ControlledApp {
373    fn new(child: Child, app: Arc<SeatApp>, transport: Transport) -> Self {
374        Self {
375            child: Some(child),
376            app,
377            transport,
378        }
379    }
380
381    /// Spawned application process identifier.
382    pub fn pid(&self) -> u32 {
383        self.child.as_ref().map_or(self.app.pid, Child::id)
384    }
385
386    /// Transport selected after launch probing.
387    pub fn transport(&self) -> Transport {
388        self.transport
389    }
390
391    /// Capture the application's current window-scoped frame.
392    pub fn capture(&self) -> Result<CapturedFrame> {
393        self.app.capture_frame().map_err(Error::from)
394    }
395
396    /// Click at frame-local coordinates.
397    pub fn click(&self, x: f64, y: f64, button: PointerButton, count: u32) -> Result<()> {
398        self.app
399            .inject_click(x, y, button.evdev_code(), count)
400            .map_err(Error::from)
401    }
402
403    /// Click while holding a `+`-separated modifier list such as `ctrl+shift`.
404    pub fn click_with_modifiers(
405        &self,
406        x: f64,
407        y: f64,
408        button: PointerButton,
409        count: u32,
410        modifiers: &str,
411    ) -> Result<()> {
412        self.app
413            .inject_click_with_modifiers(x, y, button.evdev_code(), count, Some(modifiers))
414            .map_err(Error::from)
415    }
416
417    /// Inject a pixel scroll delta at frame-local coordinates.
418    pub fn scroll(&self, x: f64, y: f64, dx: i32, dy: i32) -> Result<()> {
419        self.app.inject_scroll(x, y, dx, dy).map_err(Error::from)
420    }
421
422    /// Press a key combination such as `ctrl+shift+a`.
423    pub fn press_key(&self, combination: &str) -> Result<()> {
424        self.app.inject_key_combo(combination).map_err(Error::from)
425    }
426
427    /// Type Unicode text through the application's private seat.
428    pub fn type_text(&self, text: &str) -> Result<()> {
429        self.app.inject_text(text).map_err(Error::from)
430    }
431
432    /// Drag between two frame-local points.
433    pub fn drag(&self, from_x: f64, from_y: f64, to_x: f64, to_y: f64) -> Result<()> {
434        self.app
435            .inject_drag(from_x, from_y, to_x, to_y)
436            .map_err(Error::from)
437    }
438
439    /// Kill and reap the controlled process. Calling this twice is safe.
440    pub fn stop(&mut self) {
441        if let Some(mut child) = self.child.take() {
442            kill_and_wait(&mut child);
443        }
444    }
445}
446
447impl Drop for ControlledApp {
448    fn drop(&mut self) {
449        self.stop();
450    }
451}
452
453fn configure_native_wayland(seat: &AgentSeat, command: &mut Command) {
454    seat.configure_command(command)
455        .env("GDK_BACKEND", "wayland")
456        .env("GSK_RENDERER", "cairo")
457        .env("QT_QPA_PLATFORM", "wayland")
458        .env("ELECTRON_OZONE_PLATFORM_HINT", "wayland")
459        .env("LIBGL_ALWAYS_SOFTWARE", "1")
460        .env_remove("DISPLAY");
461}
462
463fn configure_xwayland(
464    config: &LaunchConfig,
465    bridge: &crate::XwaylandBridge,
466    command: &mut Command,
467) {
468    let mut java_options = config
469        .environment_value("_JAVA_OPTIONS")
470        .unwrap_or_default()
471        .to_string_lossy()
472        .into_owned();
473    if !java_options.is_empty() {
474        java_options.push(' ');
475    }
476    java_options.push_str("-Dawt.toolkit.name=XToolkit");
477
478    bridge
479        .configure_command(command)
480        .env("GDK_BACKEND", "x11")
481        .env("QT_QPA_PLATFORM", "xcb")
482        .env("ELECTRON_OZONE_PLATFORM_HINT", "x11")
483        .env("SDL_VIDEODRIVER", "x11")
484        .env("LIBGL_ALWAYS_SOFTWARE", "1")
485        .env("_JAVA_OPTIONS", java_options);
486}
487
488fn launch_error(config: &LaunchConfig, transport: &str, error: std::io::Error) -> Error {
489    Error::from(SeatError::Process(format!(
490        "could not launch {} through {transport}: {error}",
491        Path::new(config.program()).display()
492    )))
493}
494
495fn kill_and_wait(child: &mut Child) {
496    let _ = child.kill();
497    let _ = child.wait();
498}
499
500fn frame_has_visible_content(frame: &CapturedFrame) -> bool {
501    let image = frame.image.to_rgb8();
502    let Some(first) = image.pixels().next().copied() else {
503        return false;
504    };
505    image.pixels().step_by(64).any(|pixel| *pixel != first)
506}
507
508#[cfg(test)]
509mod tests {
510    use super::*;
511
512    #[test]
513    fn launch_config_is_framework_neutral_and_cloneable() {
514        let config = LaunchConfig::new("demo")
515            .arg("--flag")
516            .env("DEMO", "1")
517            .env_remove("REMOVE_ME")
518            .current_dir("/tmp")
519            .inherit_diagnostics(true);
520        let cloned = config.clone();
521        assert_eq!(cloned.program(), OsStr::new("demo"));
522        assert_eq!(cloned.arguments, [OsString::from("--flag")]);
523        assert_eq!(cloned.current_dir, Some(PathBuf::from("/tmp")));
524    }
525
526    #[test]
527    fn builder_rejects_zero_timeouts_before_touching_wayland() {
528        let error = ComputerUse::builder()
529            .native_timeout(Duration::ZERO)
530            .build()
531            .err()
532            .expect("zero timeout must fail");
533        assert!(error.to_string().contains("greater than zero"));
534    }
535
536    #[test]
537    fn visible_content_rejects_uniform_bridge_roots() {
538        let blank = CapturedFrame {
539            image: image::DynamicImage::ImageRgb8(image::RgbImage::new(1280, 800)),
540            width: 1280,
541            height: 800,
542        };
543        assert!(!frame_has_visible_content(&blank));
544
545        let mut image = image::RgbImage::new(128, 128);
546        image.put_pixel(64, 64, image::Rgb([255, 255, 255]));
547        let visible = CapturedFrame {
548            image: image::DynamicImage::ImageRgb8(image),
549            width: 128,
550            height: 128,
551        };
552        assert!(frame_has_visible_content(&visible));
553    }
554
555    #[test]
556    fn error_wraps_seat_error_and_exposes_source() {
557        let error = Error::from(SeatError::Capture("no frame yet".to_string()));
558        assert!(error.to_string().contains("no frame yet"));
559        let source = std::error::Error::source(&error).expect("wrapped seat error");
560        assert!(source.to_string().contains("no frame yet"));
561    }
562
563    #[test]
564    fn io_errors_keep_their_message_through_the_harness() {
565        let io = std::io::Error::other("boom");
566        let message = io.to_string();
567        let error = Error::from(io);
568        assert_eq!(error.to_string(), message);
569        assert!(std::error::Error::source(&error).is_some());
570    }
571}
agent_seat_linux/harness.rs

agent_seat_linux/
harness.rs
