Struct ComputerUse 

pub struct ComputerUse { /* private fields */ }

Complete app-scoped computer-use harness.

Implementations

impl ComputerUse

pub fn new() -> Result<Self>

Start a harness with production defaults.

Examples found in repository

examples/agent_loop.rs (line 21)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn builder() -> ComputerUseBuilder

Configure launch timeouts and bridge geometry.

Examples found in repository

examples/basic.rs (line 10)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

pub fn socket_name(&self) -> String

The private WAYLAND_DISPLAY socket, for advanced integrations.

pub fn launch(&self, config: LaunchConfig) -> Result<ControlledApp>

Launch and bind an application. Native Wayland is tried first. If the application does not expose a readable app-sized frame, the same launch is retried through an authenticated XWayland bridge automatically.

Examples found in repository

examples/basic.rs (line 15)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

examples/agent_loop.rs (line 22)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn close(&self)

Explicitly stop the seat and every compatibility bridge.

Examples found in repository

examples/agent_loop.rs (line 55)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

Trait Implementations

impl Drop for ComputerUse

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.