Struct Computer 

pub struct Computer<B: Backend> { /* private fields */ }

A computer-use session over a Backend.

Observe an app with get_app_state, then act on it by element index or by screenshot pixel. It remembers the latest state per app so indexes stay valid until the next capture of that app.

use computer_use::{CaptureOptions, ClickOptions, ClickTarget, Computer};

let mut computer = Computer::new()?;
let state = computer.get_app_state("TextEdit", &CaptureOptions::default())?;
println!("{}", state.render());

computer.click("TextEdit", ClickTarget::Element(3), ClickOptions::default())?;
computer.type_text("TextEdit", "Hello from Rust")?;
computer.press_key("TextEdit", "super+s")?;

Implementations

impl Computer<NativeBackend>

pub fn new() -> Result<Self>

Available on Linux or macOS or Windows only.

Creates a session over this platform’s native backend.

Errors

Fails if the platform’s accessibility stack cannot be initialized.

Examples found in repository

examples/list_apps.rs (line 7)

fn main() -> computer_use::Result<()> {
    let computer = computer_use::Computer::new()?;
    for app in computer.list_apps()? {
        let status = match (app.is_running, app.is_active, app.needs_restart) {
            (true, _, true) => "restart",
            (true, true, _) => "active",
            (true, false, _) => "running",
            _ => "installed",
        };
        let pid = app.pid.map(|p| format!(" pid {p}")).unwrap_or_default();
        println!("{:<9} {} [{}]{pid}", status, app.name, app.reference());
    }
    Ok(())
}

examples/inspect.rs (line 24)

fn main() -> ExitCode {
    use computer_use::{CaptureOptions, Computer};

    let mut args = std::env::args().skip(1);
    let Some(app) = args.next() else {
        eprintln!("usage: inspect <app name | bundle id | pid:N> [screenshot.png]");
        return ExitCode::FAILURE;
    };
    let output = args.next();

    let run = || -> computer_use::Result<()> {
        let mut computer = Computer::new()?;
        let permissions = computer.permissions();
        if !permissions.all_granted() {
            eprintln!("warning: missing permissions: {permissions:?}");
        }
        let started = std::time::Instant::now();
        let state = computer.get_app_state(&app, &CaptureOptions::default())?;
        println!("{state}");
        eprintln!("\ncaptured {} elements in {:?}", state.elements.len(), started.elapsed());
        if let (Some(path), Some(shot)) = (output, &state.screenshot) {
            std::fs::write(&path, shot.png())?;
            eprintln!("saved {}x{} screenshot to {path}", shot.width(), shot.height());
        }
        Ok(())
    };
    match run() {
        Ok(()) => ExitCode::SUCCESS,
        Err(e) => {
            eprintln!("error: {e}");
            ExitCode::FAILURE
        }
    }
}

examples/textedit.rs (line 19)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

impl<B: Backend> Computer<B>

pub fn with_backend(backend: B) -> Self

Creates a session over a custom backend.

pub const fn delivery(self, delivery: Delivery) -> Self

Sets the default input delivery for synthesized events.

pub const fn settle_delay(self, delay: Duration) -> Self

Sleeps this long after every action so the UI can settle before the next capture.

Examples found in repository

examples/textedit.rs (line 19)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

pub const fn backend(&self) -> &B

The underlying backend, for platform-specific calls.

pub fn permissions(&self) -> Permissions

Permissions currently held by this process.

Examples found in repository

examples/inspect.rs (line 25)

fn main() -> ExitCode {
    use computer_use::{CaptureOptions, Computer};

    let mut args = std::env::args().skip(1);
    let Some(app) = args.next() else {
        eprintln!("usage: inspect <app name | bundle id | pid:N> [screenshot.png]");
        return ExitCode::FAILURE;
    };
    let output = args.next();

    let run = || -> computer_use::Result<()> {
        let mut computer = Computer::new()?;
        let permissions = computer.permissions();
        if !permissions.all_granted() {
            eprintln!("warning: missing permissions: {permissions:?}");
        }
        let started = std::time::Instant::now();
        let state = computer.get_app_state(&app, &CaptureOptions::default())?;
        println!("{state}");
        eprintln!("\ncaptured {} elements in {:?}", state.elements.len(), started.elapsed());
        if let (Some(path), Some(shot)) = (output, &state.screenshot) {
            std::fs::write(&path, shot.png())?;
            eprintln!("saved {}x{} screenshot to {path}", shot.width(), shot.height());
        }
        Ok(())
    };
    match run() {
        Ok(()) => ExitCode::SUCCESS,
        Err(e) => {
            eprintln!("error: {e}");
            ExitCode::FAILURE
        }
    }
}

pub fn request_permissions(&self) -> Permissions

Prompts for missing permissions where the OS supports it.

pub fn list_apps(&self) -> Result<Vec<AppInfo>>

Lists running apps first (frontmost first), then installed apps, by name.

Errors

Fails if the platform cannot enumerate apps.

Examples found in repository

examples/list_apps.rs (line 8)

fn main() -> computer_use::Result<()> {
    let computer = computer_use::Computer::new()?;
    for app in computer.list_apps()? {
        let status = match (app.is_running, app.is_active, app.needs_restart) {
            (true, _, true) => "restart",
            (true, true, _) => "active",
            (true, false, _) => "running",
            _ => "installed",
        };
        let pid = app.pid.map(|p| format!(" pid {p}")).unwrap_or_default();
        println!("{:<9} {} [{}]{pid}", status, app.name, app.reference());
    }
    Ok(())
}

pub fn resolve_app(&self, query: &str) -> Result<AppInfo>

Resolves an app query (name, bundle ID, path, or pid:<n>), preferring running apps.

Errors

Returns Error::AppNotFound when nothing matches, or Error::AmbiguousApp when several running apps match equally well.

pub fn get_app_state( &mut self, app: &str, options: &CaptureOptions, ) -> Result<&AppState>

Captures the app’s current window, replacing any earlier state (and element indexes) for that app.

Errors

Fails if the app cannot be found or launched, has no window, or permissions are missing.

Examples found in repository

examples/inspect.rs (line 30)

fn main() -> ExitCode {
    use computer_use::{CaptureOptions, Computer};

    let mut args = std::env::args().skip(1);
    let Some(app) = args.next() else {
        eprintln!("usage: inspect <app name | bundle id | pid:N> [screenshot.png]");
        return ExitCode::FAILURE;
    };
    let output = args.next();

    let run = || -> computer_use::Result<()> {
        let mut computer = Computer::new()?;
        let permissions = computer.permissions();
        if !permissions.all_granted() {
            eprintln!("warning: missing permissions: {permissions:?}");
        }
        let started = std::time::Instant::now();
        let state = computer.get_app_state(&app, &CaptureOptions::default())?;
        println!("{state}");
        eprintln!("\ncaptured {} elements in {:?}", state.elements.len(), started.elapsed());
        if let (Some(path), Some(shot)) = (output, &state.screenshot) {
            std::fs::write(&path, shot.png())?;
            eprintln!("saved {}x{} screenshot to {path}", shot.width(), shot.height());
        }
        Ok(())
    };
    match run() {
        Ok(()) => ExitCode::SUCCESS,
        Err(e) => {
            eprintln!("error: {e}");
            ExitCode::FAILURE
        }
    }
}

examples/textedit.rs (line 22)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

pub fn list_windows(&self, app: &str) -> Result<Vec<WindowInfo>>

The app’s windows, front to back where the platform knows the order. Pass one to CaptureOptions::window to capture it instead of the focused or main window.

Errors

Fails if the app is not running or the backend cannot list windows.

pub fn state(&self, app: &str) -> Option<&AppState>

The latest captured state of an app, if any.

pub fn forget(&mut self, app: &str)

Discards the remembered state of one app.

pub fn clear(&mut self)

Discards all remembered state.

pub fn query( &mut self, app: &str, query: &ElementQuery, max_nodes: usize, ) -> Result<Vec<Element>>

Finds elements in the app’s current window without taking a screenshot.

The fresh tree replaces the app’s remembered state, so the returned indexes can be used with element actions. A previous screenshot is kept when the window has not moved.

Errors

Fails like get_app_state or if the query is empty.

pub fn click( &mut self, app: &str, target: ClickTarget, options: ClickOptions, ) -> Result<ClickOutcome>

Clicks an element, a screenshot pixel, or a screen point.

Errors

Fails if there is no state for the app, the target is invalid, or delivery fails.

Examples found in repository

examples/textedit.rs (line 27)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

pub fn perform_secondary_action( &mut self, app: &str, index: usize, action: &str, ) -> Result<()>

Invokes a named accessibility action on an element (e.g. Show Menu, AXRaise).

Errors

Fails if the element does not expose the action.

pub fn scroll( &mut self, app: &str, index: usize, direction: ScrollDirection, pages: f64, ) -> Result<()>

Scrolls an element by a number of pages (fractions allowed).

Whole pages use the element’s own scroll-by-page action when it has one; otherwise wheel events are posted at the element’s center.

Errors

Fails if pages is not positive or the element has no frame.

pub fn scroll_at( &mut self, app: &str, target: ClickTarget, direction: ScrollDirection, pages: f64, ) -> Result<()>

Scrolls at an element, a screenshot pixel, or a screen point, by a number of pages (fractions allowed).

An element target behaves like scroll. For a point, wheel events are posted there; a page is 80% of the scroll area under the point when the latest state has one, else of the captured window, else 400 points.

Errors

Fails if pages is not positive, a pixel lies outside the screenshot, or delivery fails.

pub fn drag( &mut self, app: &str, from: (f64, f64), to: (f64, f64), ) -> Result<()>

Drags between two screenshot pixels.

Errors

Fails if there is no screenshot or a point lies outside it.

pub fn drag_screen(&mut self, app: &str, from: Point, to: Point) -> Result<()>

Drags between two screen points. Needs no earlier capture.

Errors

Fails if the app is not running or delivery fails.

pub fn type_text(&mut self, app: &str, text: &str) -> Result<()>

Types literal text into the app’s focused element.

With background delivery, text is inserted through accessibility when the focused element supports it, and synthesized as keystrokes otherwise.

Errors

Fails if the app is not running or delivery fails.

Examples found in repository

examples/textedit.rs (line 42)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

pub fn press_key(&mut self, app: &str, key: &str) -> Result<()>

Presses a key chord in xdotool syntax, e.g. Return, super+c, KP_0.

Errors

Fails if the chord cannot be parsed or delivered.

Examples found in repository

examples/textedit.rs (line 52)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

pub fn set_value(&mut self, app: &str, index: usize, value: &str) -> Result<()>

Sets the value of a settable element.

Errors

Fails if the element’s value is read-only.

Examples found in repository

examples/textedit.rs (line 41)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

pub fn select_text( &mut self, app: &str, index: usize, selection: &TextSelection, ) -> Result<()>

Selects text inside a text element, or places the caret before or after it.

Errors

Fails if the text is missing or ambiguous, or the element does not support selection.

Examples found in repository

examples/textedit.rs (line 48)

fn main() -> computer_use::Result<()> {
    use std::time::Duration;

    use computer_use::{
        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
    };

    const APP: &str = "TextEdit";
    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
    let tree_only = CaptureOptions::default().include_screenshot(false);

    let state = computer.get_app_state(APP, &tree_only)?;
    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
    if let Some(button) = new_doc.map(|e| e.index) {
        println!(
            "click New Document: {:?}",
            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
        );
    }

    let state = computer.get_app_state(APP, &tree_only)?;
    let text_area = state
        .find(&ElementQuery::role("AXTextArea"))
        .next()
        .map(|e| e.index)
        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
    println!(
        "click text area {text_area}: {:?}",
        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
    );
    computer.set_value(APP, text_area, "")?;
    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;

    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("text area now contains: {value:?}");

    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("selected text: {:?}", state.selected_text);

    computer.press_key(APP, "BackSpace")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("after BackSpace: {value:?}");

    computer.press_key(APP, "super+a")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    println!("after super+a, selected: {:?}", state.selected_text);

    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
    computer.type_text(APP, " The end.")?;
    let state = computer.get_app_state(APP, &tree_only)?;
    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
    println!("final text: {value:?}");

    Ok(())
}

Trait Implementations

impl<B: Backend> Debug for Computer<B>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.