Struct ControlledApp 

pub struct ControlledApp { /* private fields */ }

A launched application with capture and input methods bound to its exact seat connection.

Implementations

impl ControlledApp

pub fn pid(&self) -> u32

Spawned application process identifier.

Examples found in repository

examples/basic.rs (line 20)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

pub fn transport(&self) -> Transport

Transport selected after launch probing.

Examples found in repository

examples/basic.rs (line 21)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

examples/agent_loop.rs (line 23)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn capture(&self) -> Result<CapturedFrame>

Capture the application’s current window-scoped frame.

Examples found in repository

examples/basic.rs (line 17)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

examples/agent_loop.rs (line 26)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn click( &self, x: f64, y: f64, button: PointerButton, count: u32, ) -> Result<()>

Click at frame-local coordinates.

Examples found in repository

examples/basic.rs (lines 28-33)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

examples/agent_loop.rs (lines 35-40)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn click_with_modifiers( &self, x: f64, y: f64, button: PointerButton, count: u32, modifiers: &str, ) -> Result<()>

Click while holding a +-separated modifier list such as ctrl+shift.

pub fn scroll(&self, x: f64, y: f64, dx: i32, dy: i32) -> Result<()>

Inject a pixel scroll delta at frame-local coordinates.

pub fn press_key(&self, combination: &str) -> Result<()>

Press a key combination such as ctrl+shift+a.

Examples found in repository

examples/agent_loop.rs (line 41)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn type_text(&self, text: &str) -> Result<()>

Type Unicode text through the application’s private seat.

Examples found in repository

examples/agent_loop.rs (line 42)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

pub fn drag(&self, from_x: f64, from_y: f64, to_x: f64, to_y: f64) -> Result<()>

Drag between two frame-local points.

pub fn stop(&mut self)

Kill and reap the controlled process. Calling this twice is safe.

Examples found in repository

examples/basic.rs (line 35)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut arguments = std::env::args_os().skip(1);
    let program = arguments
        .next()
        .unwrap_or_else(|| "gnome-calculator".into());
    let harness = ComputerUse::builder()
        .native_timeout(Duration::from_secs(8))
        .fallback_timeout(Duration::from_secs(30))
        .bridge_geometry(1280, 800)
        .build()?;
    let mut app = harness.launch(LaunchConfig::new(program).args(arguments))?;

    let frame = app.capture()?;
    println!(
        "controlled pid={} transport={:?} frame={}x{}",
        app.pid(),
        app.transport(),
        frame.width,
        frame.height
    );
    frame.image.save("agent-seat-frame.png")?;

    // Harmlessly focus the center of the captured app to prove pointer input.
    app.click(
        f64::from(frame.width) / 2.0,
        f64::from(frame.height) / 2.0,
        PointerButton::Left,
        1,
    )?;

    app.stop();
    Ok(())
}

examples/agent_loop.rs (line 54)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let mut args = env::args().skip(1);
    let program = args
        .next()
        .unwrap_or_else(|| "gnome-calculator".to_string());

    let mut config = LaunchConfig::new(&program);
    for extra in args {
        config = config.arg(extra);
    }

    let harness = ComputerUse::new()?;
    let mut app = harness.launch(config)?;
    println!("launched {program} via {:?}", app.transport());

    // 1. Perceive: capture the initial frame.
    let before = app.capture()?;
    before.image.save("agent-loop-before.png")?;
    println!(
        "captured {}x{} -> agent-loop-before.png",
        before.image.width(),
        before.image.height()
    );

    // 2. Act: focus, select-all, and type, exercising keyboard + text paths.
    app.click(
        before.image.width() as f64 / 2.0,
        before.image.height() as f64 / 2.0,
        PointerButton::Left,
        1,
    )?;
    app.press_key("ctrl+a")?;
    app.type_text("agent-seat-linux")?;

    // 3. Perceive again: capture the post-action frame.
    let after = app.capture()?;
    after.image.save("agent-loop-after.png")?;
    println!(
        "captured {}x{} -> agent-loop-after.png",
        after.image.width(),
        after.image.height()
    );

    // 4. Teardown: stop the app, then close the seat.
    app.stop();
    harness.close();
    println!("done");
    Ok(())
}

Trait Implementations

impl Drop for ControlledApp

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.