Skip to main content

ElementQuery

computer_use

Struct ElementQuery 

Source 
pub struct ElementQuery {
    pub text: Option<String>,
    pub role: Option<String>,
    pub exact: bool,
    pub limit: usize,
}
Expand description

Criteria for Computer::query and AppState::find.

Fields§

§text: Option<String>

Text matched against title, description, value, placeholder, identifier, and help.

§role: Option<String>

Role matched against the display role or platform role (an AX prefix is optional).

§exact: bool

Require an exact (case-insensitive) text match instead of a substring.

§limit: usize

Maximum number of matches.

Implementations§

Source§

impl ElementQuery

Source

pub fn text(text: impl Into<String>) -> Self

Matches elements containing text.

Examples found in repository?
examples/textedit.rs (line 23)
11fn main() -> computer_use::Result<()> {
12    use std::time::Duration;
13
14    use computer_use::{
15        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
16    };
17
18    const APP: &str = "TextEdit";
19    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
20    let tree_only = CaptureOptions::default().include_screenshot(false);
21
22    let state = computer.get_app_state(APP, &tree_only)?;
23    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
24    if let Some(button) = new_doc.map(|e| e.index) {
25        println!(
26            "click New Document: {:?}",
27            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
28        );
29    }
30
31    let state = computer.get_app_state(APP, &tree_only)?;
32    let text_area = state
33        .find(&ElementQuery::role("AXTextArea"))
34        .next()
35        .map(|e| e.index)
36        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
37    println!(
38        "click text area {text_area}: {:?}",
39        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
40    );
41    computer.set_value(APP, text_area, "")?;
42    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;
43
44    let state = computer.get_app_state(APP, &tree_only)?;
45    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
46    println!("text area now contains: {value:?}");
47
48    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
49    let state = computer.get_app_state(APP, &tree_only)?;
50    println!("selected text: {:?}", state.selected_text);
51
52    computer.press_key(APP, "BackSpace")?;
53    let state = computer.get_app_state(APP, &tree_only)?;
54    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
55    println!("after BackSpace: {value:?}");
56
57    computer.press_key(APP, "super+a")?;
58    let state = computer.get_app_state(APP, &tree_only)?;
59    println!("after super+a, selected: {:?}", state.selected_text);
60
61    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
62    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
63    computer.type_text(APP, " The end.")?;
64    let state = computer.get_app_state(APP, &tree_only)?;
65    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
66    println!("final text: {value:?}");
67
68    Ok(())
69}
Source

pub fn role(role: impl Into<String>) -> Self

Matches elements with role.

Examples found in repository?
examples/textedit.rs (line 33)
11fn main() -> computer_use::Result<()> {
12    use std::time::Duration;
13
14    use computer_use::{
15        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
16    };
17
18    const APP: &str = "TextEdit";
19    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
20    let tree_only = CaptureOptions::default().include_screenshot(false);
21
22    let state = computer.get_app_state(APP, &tree_only)?;
23    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
24    if let Some(button) = new_doc.map(|e| e.index) {
25        println!(
26            "click New Document: {:?}",
27            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
28        );
29    }
30
31    let state = computer.get_app_state(APP, &tree_only)?;
32    let text_area = state
33        .find(&ElementQuery::role("AXTextArea"))
34        .next()
35        .map(|e| e.index)
36        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
37    println!(
38        "click text area {text_area}: {:?}",
39        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
40    );
41    computer.set_value(APP, text_area, "")?;
42    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;
43
44    let state = computer.get_app_state(APP, &tree_only)?;
45    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
46    println!("text area now contains: {value:?}");
47
48    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
49    let state = computer.get_app_state(APP, &tree_only)?;
50    println!("selected text: {:?}", state.selected_text);
51
52    computer.press_key(APP, "BackSpace")?;
53    let state = computer.get_app_state(APP, &tree_only)?;
54    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
55    println!("after BackSpace: {value:?}");
56
57    computer.press_key(APP, "super+a")?;
58    let state = computer.get_app_state(APP, &tree_only)?;
59    println!("after super+a, selected: {:?}", state.selected_text);
60
61    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
62    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
63    computer.type_text(APP, " The end.")?;
64    let state = computer.get_app_state(APP, &tree_only)?;
65    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
66    println!("final text: {value:?}");
67
68    Ok(())
69}
Source

pub fn with_role(self, role: impl Into<String>) -> Self

Adds a role constraint.

Examples found in repository?
examples/textedit.rs (line 23)
11fn main() -> computer_use::Result<()> {
12    use std::time::Duration;
13
14    use computer_use::{
15        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
16    };
17
18    const APP: &str = "TextEdit";
19    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
20    let tree_only = CaptureOptions::default().include_screenshot(false);
21
22    let state = computer.get_app_state(APP, &tree_only)?;
23    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
24    if let Some(button) = new_doc.map(|e| e.index) {
25        println!(
26            "click New Document: {:?}",
27            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
28        );
29    }
30
31    let state = computer.get_app_state(APP, &tree_only)?;
32    let text_area = state
33        .find(&ElementQuery::role("AXTextArea"))
34        .next()
35        .map(|e| e.index)
36        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
37    println!(
38        "click text area {text_area}: {:?}",
39        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
40    );
41    computer.set_value(APP, text_area, "")?;
42    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;
43
44    let state = computer.get_app_state(APP, &tree_only)?;
45    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
46    println!("text area now contains: {value:?}");
47
48    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
49    let state = computer.get_app_state(APP, &tree_only)?;
50    println!("selected text: {:?}", state.selected_text);
51
52    computer.press_key(APP, "BackSpace")?;
53    let state = computer.get_app_state(APP, &tree_only)?;
54    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
55    println!("after BackSpace: {value:?}");
56
57    computer.press_key(APP, "super+a")?;
58    let state = computer.get_app_state(APP, &tree_only)?;
59    println!("after super+a, selected: {:?}", state.selected_text);
60
61    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
62    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
63    computer.type_text(APP, " The end.")?;
64    let state = computer.get_app_state(APP, &tree_only)?;
65    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
66    println!("final text: {value:?}");
67
68    Ok(())
69}
Source

pub const fn exact(self, exact: bool) -> Self

Requires exact text matches.

Examples found in repository?
examples/textedit.rs (line 23)
11fn main() -> computer_use::Result<()> {
12    use std::time::Duration;
13
14    use computer_use::{
15        CaptureOptions, ClickMethod, ClickOptions, ClickTarget, Computer, ElementQuery, Error, TextSelection,
16    };
17
18    const APP: &str = "TextEdit";
19    let mut computer = Computer::new()?.settle_delay(Duration::from_millis(300));
20    let tree_only = CaptureOptions::default().include_screenshot(false);
21
22    let state = computer.get_app_state(APP, &tree_only)?;
23    let new_doc = state.find(&ElementQuery::text("New Document").with_role("button").exact(true)).next();
24    if let Some(button) = new_doc.map(|e| e.index) {
25        println!(
26            "click New Document: {:?}",
27            computer.click(APP, ClickTarget::Element(button), ClickOptions::default())?
28        );
29    }
30
31    let state = computer.get_app_state(APP, &tree_only)?;
32    let text_area = state
33        .find(&ElementQuery::role("AXTextArea"))
34        .next()
35        .map(|e| e.index)
36        .ok_or_else(|| Error::StateUnavailable("no text area".into()))?;
37    println!(
38        "click text area {text_area}: {:?}",
39        computer.click(APP, ClickTarget::Element(text_area), ClickOptions::default())?
40    );
41    computer.set_value(APP, text_area, "")?;
42    computer.type_text(APP, "Hello from computer-use-rs!\nSecond line.")?;
43
44    let state = computer.get_app_state(APP, &tree_only)?;
45    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
46    println!("text area now contains: {value:?}");
47
48    computer.select_text(APP, text_area, &TextSelection::new("computer-use-rs"))?;
49    let state = computer.get_app_state(APP, &tree_only)?;
50    println!("selected text: {:?}", state.selected_text);
51
52    computer.press_key(APP, "BackSpace")?;
53    let state = computer.get_app_state(APP, &tree_only)?;
54    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
55    println!("after BackSpace: {value:?}");
56
57    computer.press_key(APP, "super+a")?;
58    let state = computer.get_app_state(APP, &tree_only)?;
59    println!("after super+a, selected: {:?}", state.selected_text);
60
61    let focused = ClickOptions::default().method(ClickMethod::FocusedBackground);
62    println!("focused click: {:?}", computer.click(APP, ClickTarget::Element(text_area), focused)?);
63    computer.type_text(APP, " The end.")?;
64    let state = computer.get_app_state(APP, &tree_only)?;
65    let value = state.find(&ElementQuery::role("AXTextArea")).next().and_then(|e| e.value.clone());
66    println!("final text: {value:?}");
67
68    Ok(())
69}
Source

pub const fn limit(self, limit: usize) -> Self

Caps the number of results.

Source

pub fn matches(&self, element: &Element) -> bool

Whether element satisfies the query.

Trait Implementations§

Source§

impl Clone for ElementQuery

Source§

fn clone(&self) -> Self

Returns a duplicate of the value. Read more
1.0.0 (const: unstable) · Source§

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source. Read more
Source§

impl Debug for ElementQuery

Source§

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter. Read more
Source§

impl Default for ElementQuery

Source§

fn default() -> Self

Returns the “default value” for a type. Read more
Source§

impl Eq for ElementQuery

Source§

impl PartialEq for ElementQuery

Source§

fn eq(&self, other: &Self) -> bool

Equality operator ==. Read more
1.0.0 (const: unstable) · Source§

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=. Read more
Source§

impl StructuralPartialEq for ElementQuery

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> AutoreleaseSafe for T
where T: ?Sized,

Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> CloneToUninit for T
where T: Clone,

Source§

unsafe fn clone_to_uninit(&self, dest: *mut u8)

🔬This is a nightly-only experimental API. (clone_to_uninit)
Performs copy-assignment from self to dest. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> ToOwned for T
where T: Clone,

Source§

type Owned = T

The resulting type after obtaining ownership.
Source§

fn to_owned(&self) -> T

Creates owned data from borrowed data, usually by cloning. Read more
Source§

fn clone_into(&self, target: &mut T)

Uses borrowed data to replace owned data, usually by cloning. Read more
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.