Skip to main content

iphone_cli/
selector.rs

1//! Selectors: how a caller names an element on screen.
2//!
3//! Text form (CLI): `label="정확히"`, `label~="포함"`, `label/정규식/`,
4//! `role=button`, `id=…`, `value~=…`, `within=…`, `index=N`, combined with
5//! spaces. `label`, `value` and `id` all take `=`, `~=` and `/regex/`. A
6//! string with none of these is an exact label.
7//!
8//! Structured form (scripts, MCP, batch): `{label, label_contains,
9//! label_regex, value, value_contains, value_regex, role, id, id_contains,
10//! id_regex, within, index}`.
11
12use anyhow::{Result, anyhow, bail};
13use regex::Regex;
14use serde_json::Value;
15use unicode_normalization::UnicodeNormalization;
16
17use crate::ax::tree::{Element, Screen};
18
19#[derive(Debug, Clone)]
20pub enum TextMatch {
21    Exact(String),
22    Contains(String),
23    Regex(Regex),
24}
25
26fn nfc(s: &str) -> String {
27    s.nfc().collect::<String>().trim().to_string()
28}
29
30impl TextMatch {
31    fn matches(&self, text: Option<&str>, first_segment_counts: bool) -> bool {
32        let Some(text) = text else { return false };
33        let text = nfc(text);
34        match self {
35            TextMatch::Exact(want) => {
36                // Rows are often labelled "name, value" ("Wi-Fi, HomeNet"); the
37                // name alone is what a caller knows in advance.
38                text == *want
39                    || (first_segment_counts
40                        && text.split_once(", ").is_some_and(|(head, _)| head == want))
41            }
42            TextMatch::Contains(want) => text.to_lowercase().contains(&want.to_lowercase()),
43            TextMatch::Regex(re) => re.is_match(&text),
44        }
45    }
46
47    fn describe(&self, key: &str) -> String {
48        match self {
49            TextMatch::Exact(s) => format!("{key}=\"{s}\""),
50            TextMatch::Contains(s) => format!("{key}~=\"{s}\""),
51            TextMatch::Regex(r) => format!("{key}/{}/", r.as_str()),
52        }
53    }
54}
55
56#[derive(Debug, Clone, Default)]
57pub struct Selector {
58    pub label: Option<TextMatch>,
59    pub value: Option<TextMatch>,
60    pub role: Option<String>,
61    /// accessibilityIdentifier, or (exact form only) the `id` handle from
62    /// `iph read`.
63    pub id: Option<TextMatch>,
64    pub within: Option<Box<Selector>>,
65    pub index: Option<usize>,
66}
67
68const KEYS: &[&str] = &["label", "value", "role", "id", "within", "index"];
69
70/// Split a selector string into `key op value` terms. Returns None when the
71/// string is not in selector syntax at all (then it is a plain label).
72fn terms(input: &str) -> Option<Vec<(String, char, String)>> {
73    let chars: Vec<char> = input.chars().collect();
74    let mut i = 0;
75    let mut out = Vec::new();
76    while i < chars.len() {
77        while i < chars.len() && chars[i].is_whitespace() {
78            i += 1;
79        }
80        if i >= chars.len() {
81            break;
82        }
83        let start = i;
84        while i < chars.len() && chars[i].is_ascii_alphabetic() {
85            i += 1;
86        }
87        let key: String = chars[start..i].iter().collect();
88        if !KEYS.contains(&key.as_str()) || i >= chars.len() {
89            return None;
90        }
91        // Operator: `=`, `~=` or `/regex/`.
92        let op = match chars[i] {
93            '=' => {
94                i += 1;
95                '='
96            }
97            '~' if chars.get(i + 1) == Some(&'=') => {
98                i += 2;
99                '~'
100            }
101            '/' => '/',
102            _ => return None,
103        };
104        let mut val = String::new();
105        if op == '/' {
106            i += 1;
107            let mut closed = false;
108            while i < chars.len() {
109                if chars[i] == '\\' && chars.get(i + 1) == Some(&'/') {
110                    val.push('/');
111                    i += 2;
112                    continue;
113                }
114                if chars[i] == '/' {
115                    closed = true;
116                    i += 1;
117                    break;
118                }
119                val.push(chars[i]);
120                i += 1;
121            }
122            if !closed {
123                return None;
124            }
125        } else if chars.get(i) == Some(&'"') || chars.get(i) == Some(&'\'') {
126            let quote = chars[i];
127            i += 1;
128            let mut closed = false;
129            while i < chars.len() {
130                if chars[i] == '\\' && i + 1 < chars.len() {
131                    val.push(chars[i + 1]);
132                    i += 2;
133                    continue;
134                }
135                if chars[i] == quote {
136                    closed = true;
137                    i += 1;
138                    break;
139                }
140                val.push(chars[i]);
141                i += 1;
142            }
143            if !closed {
144                return None;
145            }
146        } else {
147            while i < chars.len() && !chars[i].is_whitespace() {
148                val.push(chars[i]);
149                i += 1;
150            }
151        }
152        out.push((key, op, val));
153    }
154    (!out.is_empty()).then_some(out)
155}
156
157fn text_match(op: char, val: &str) -> Result<TextMatch> {
158    Ok(match op {
159        '=' => TextMatch::Exact(nfc(val)),
160        '~' => TextMatch::Contains(nfc(val)),
161        _ => TextMatch::Regex(Regex::new(val).map_err(|e| anyhow!("bad regex /{val}/: {e}"))?),
162    })
163}
164
165impl Selector {
166    pub fn parse(input: &str) -> Result<Selector> {
167        let input = input.trim();
168        if input.is_empty() {
169            bail!("empty selector");
170        }
171        let Some(terms) = terms(input) else {
172            return Ok(Selector {
173                label: Some(TextMatch::Exact(nfc(input))),
174                ..Default::default()
175            });
176        };
177        let mut sel = Selector::default();
178        for (key, op, val) in terms {
179            match key.as_str() {
180                "label" => sel.label = Some(text_match(op, &val)?),
181                "value" => sel.value = Some(text_match(op, &val)?),
182                "role" if op == '=' => sel.role = Some(val.to_lowercase()),
183                "id" => sel.id = Some(text_match(op, &val)?),
184                "within" if op == '=' => sel.within = Some(Box::new(Selector::parse(&val)?)),
185                "index" if op == '=' => {
186                    sel.index = Some(val.parse().map_err(|_| anyhow!("index must be a number"))?)
187                }
188                _ => bail!("`{key}` does not support this operator"),
189            }
190        }
191        Ok(sel)
192    }
193
194    /// Accepts a selector string or a structured object.
195    pub fn from_value(v: &Value) -> Result<Selector> {
196        match v {
197            Value::String(s) => Selector::parse(s),
198            Value::Object(map) => {
199                let mut sel = Selector::default();
200                for (key, val) in map {
201                    let s = || -> Result<String> {
202                        match val {
203                            Value::String(s) => Ok(s.clone()),
204                            Value::Number(n) => Ok(n.to_string()),
205                            Value::Bool(b) => Ok(b.to_string()),
206                            _ => bail!("selector field `{key}` must be text"),
207                        }
208                    };
209                    match key.as_str() {
210                        "label" => sel.label = Some(TextMatch::Exact(nfc(&s()?))),
211                        "label_contains" => sel.label = Some(TextMatch::Contains(nfc(&s()?))),
212                        "label_regex" => sel.label = Some(text_match('/', &s()?)?),
213                        "value" => sel.value = Some(TextMatch::Exact(nfc(&s()?))),
214                        "value_contains" => sel.value = Some(TextMatch::Contains(nfc(&s()?))),
215                        "value_regex" => sel.value = Some(text_match('/', &s()?)?),
216                        "role" => sel.role = Some(s()?.to_lowercase()),
217                        "id" | "identifier" => sel.id = Some(TextMatch::Exact(nfc(&s()?))),
218                        "id_contains" => sel.id = Some(TextMatch::Contains(nfc(&s()?))),
219                        "id_regex" => sel.id = Some(text_match('/', &s()?)?),
220                        "within" => sel.within = Some(Box::new(Selector::from_value(val)?)),
221                        "index" => {
222                            sel.index = Some(
223                                val.as_u64()
224                                    .ok_or_else(|| anyhow!("index must be a number"))?
225                                    as usize,
226                            )
227                        }
228                        other => bail!("unknown selector field `{other}`"),
229                    }
230                }
231                if sel.is_empty() {
232                    bail!("empty selector");
233                }
234                Ok(sel)
235            }
236            _ => bail!("a selector is a string or an object"),
237        }
238    }
239
240    fn is_empty(&self) -> bool {
241        self.label.is_none()
242            && self.value.is_none()
243            && self.role.is_none()
244            && self.id.is_none()
245            && self.within.is_none()
246            && self.index.is_none()
247    }
248
249    pub fn describe(&self) -> String {
250        let mut parts = Vec::new();
251        if let Some(l) = &self.label {
252            parts.push(l.describe("label"));
253        }
254        if let Some(v) = &self.value {
255            parts.push(v.describe("value"));
256        }
257        if let Some(r) = &self.role {
258            parts.push(format!("role={r}"));
259        }
260        if let Some(i) = &self.id {
261            parts.push(i.describe("id"));
262        }
263        if let Some(w) = &self.within {
264            parts.push(format!("within=\"{}\"", w.describe().replace('"', "\\\"")));
265        }
266        if let Some(i) = self.index {
267            parts.push(format!("index={i}"));
268        }
269        parts.join(" ")
270    }
271
272    /// True when the selector reads the identifier, which only the slower
273    /// second pass of a reading carries.
274    pub fn needs_identifier(&self) -> bool {
275        self.id.is_some() || self.within.as_ref().is_some_and(|w| w.needs_identifier())
276    }
277
278    fn matches_own(&self, el: &Element) -> bool {
279        if el.role == "application" && self.role.as_deref() != Some("application") {
280            return false;
281        }
282        if let Some(l) = &self.label
283            && !l.matches(el.label.as_deref(), true) {
284                return false;
285            }
286        if let Some(v) = &self.value
287            && !v.matches(el.value.as_deref(), false) {
288                return false;
289            }
290        if let Some(r) = &self.role
291            && el.role != *r {
292                return false;
293            }
294        if let Some(id) = &self.id {
295            let by_handle = matches!(id, TextMatch::Exact(want) if el.id == *want);
296            if !by_handle && !id.matches(el.identifier.as_deref(), false) {
297                return false;
298            }
299        }
300        true
301    }
302
303    /// All matches in reading order. `index` picks one of them.
304    pub fn find<'a>(&self, screen: &'a Screen) -> Vec<&'a Element> {
305        let containers: Option<Vec<&Element>> = self.within.as_ref().map(|w| w.find(screen));
306        let mut out: Vec<&Element> = screen
307            .elements
308            .iter()
309            .filter(|el| self.matches_own(el))
310            .filter(|el| match &containers {
311                None => true,
312                Some(cs) => cs.iter().any(|c| is_inside(el, c, screen)),
313            })
314            .collect();
315        if let Some(i) = self.index {
316            out = out.get(i).map(|e| vec![*e]).unwrap_or_default();
317        }
318        out
319    }
320}
321
322/// `el` is inside `container` when it descends from it or lies within its frame.
323fn is_inside(el: &Element, container: &Element, screen: &Screen) -> bool {
324    if el.id == container.id {
325        return false;
326    }
327    let mut parent = el.parent.as_deref();
328    while let Some(pid) = parent {
329        if pid == container.id {
330            return true;
331        }
332        parent = screen.by_id(pid).and_then(|p| p.parent.as_deref());
333    }
334    match (el.frame, container.frame) {
335        (Some(a), Some(b)) => {
336            a.x >= b.x - 0.5
337                && a.y >= b.y - 0.5
338                && a.x + a.w <= b.x + b.w + 0.5
339                && a.y + a.h <= b.y + b.h + 0.5
340        }
341        _ => false,
342    }
343}
344
345/// Reduce matches that are the same tap target (same label, frames nested or
346/// equal) to one. Anything left over is a real ambiguity.
347pub fn dedupe_targets(matches: Vec<&Element>) -> Vec<&Element> {
348    let mut out: Vec<&Element> = Vec::new();
349    'next: for m in matches {
350        for kept in out.iter_mut() {
351            let (Some(a), Some(b)) = (kept.frame, m.frame) else {
352                continue;
353            };
354            let same_point = {
355                let (ax, ay) = a.center();
356                let (bx, by) = b.center();
357                (ax - bx).abs() <= 2.0 && (ay - by).abs() <= 2.0
358            };
359            let nested = a.intersection(&b).is_some_and(|i| {
360                (i.area() - a.area()).abs() < 1.0 || (i.area() - b.area()).abs() < 1.0
361            });
362            if kept.label == m.label && (same_point || nested) {
363                // Prefer the element an agent would expect to act on.
364                let rank = |e: &Element| match e.role.as_str() {
365                    "button" | "switch" | "link" | "tab" | "text_field" | "search_field" => 0,
366                    _ => 1,
367                };
368                if rank(m) < rank(kept) {
369                    *kept = m;
370                }
371                continue 'next;
372            }
373        }
374        out.push(m);
375    }
376    out
377}
378
379#[cfg(test)]
380mod tests {
381    use super::*;
382    use crate::ax::tree::Rect;
383
384    fn el(id: &str, role: &str, label: &str, frame: (f64, f64, f64, f64)) -> Element {
385        Element {
386            id: id.into(),
387            role: role.into(),
388            label: (!label.is_empty()).then(|| label.to_string()),
389            value: None,
390            identifier: None,
391            hint: None,
392            frame: Some(Rect { x: frame.0, y: frame.1, w: frame.2, h: frame.3 }),
393            traits: vec![],
394            enabled: true,
395            selected: false,
396            focused: false,
397            parent: Some("app".into()),
398            depth: 1,
399            pid: 1,
400            on_screen: true,
401            raw: None,
402            key: None,
403        }
404    }
405
406    fn screen() -> Screen {
407        let mut app = el("app", "application", "설정", (0.0, 0.0, 430.0, 932.0));
408        app.parent = None;
409        app.depth = 0;
410        let mut wifi = el("b", "button", "Wi-Fi, HomeNet", (20.0, 538.0, 390.0, 49.0));
411        wifi.identifier = Some("com.apple.settings.wifi".into());
412        let mut airplane = el("c", "switch", "에어플레인 모드", (20.0, 489.0, 390.0, 49.0));
413        airplane.value = Some("0".into());
414        Screen {
415            width: 430.0,
416            height: 932.0,
417            apps: vec![],
418            elements: vec![
419                app,
420                el("a", "header", "설정", (20.0, 119.0, 54.0, 37.0)),
421                airplane,
422                wifi,
423                el("d", "button", "일반", (20.0, 700.0, 390.0, 49.0)),
424                el("e", "button", "일반 정보", (20.0, 750.0, 390.0, 49.0)),
425            ],
426        }
427    }
428
429    fn ids(sel: &str) -> Vec<String> {
430        Selector::parse(sel)
431            .unwrap()
432            .find(&screen())
433            .iter()
434            .map(|e| e.id.clone())
435            .collect()
436    }
437
438    #[test]
439    fn plain_string_is_exact_label_and_skips_the_application() {
440        assert_eq!(ids("일반"), vec!["d"]);
441        assert_eq!(ids("설정"), vec!["a"]);
442    }
443
444    #[test]
445    fn exact_label_accepts_the_name_part_of_a_row() {
446        assert_eq!(ids("Wi-Fi"), vec!["b"]);
447        assert_eq!(ids("label=\"Wi-Fi\""), vec!["b"]);
448        assert!(ids("HomeNet").is_empty());
449    }
450
451    #[test]
452    fn contains_regex_role_id_value_index() {
453        assert_eq!(ids("label~=\"일반\""), vec!["d", "e"]);
454        assert_eq!(ids("label~=wi-fi"), vec!["b"]);
455        assert_eq!(ids("label/^일반$/"), vec!["d"]);
456        assert_eq!(ids("role=switch"), vec!["c"]);
457        assert_eq!(ids("id=com.apple.settings.wifi"), vec!["b"]);
458        assert_eq!(ids("id=e"), vec!["e"]);
459        assert_eq!(ids("id~=settings.wi"), vec!["b"]);
460        assert_eq!(ids("id/^com\\.apple\\.settings\\./"), vec!["b"]);
461        assert_eq!(ids("role=switch value=0"), vec!["c"]);
462        assert_eq!(ids("label~=일반 index=1"), vec!["e"]);
463        assert!(ids("label~=일반 index=5").is_empty());
464    }
465
466    #[test]
467    fn within_uses_frames_when_the_tree_is_flat() {
468        let mut s = screen();
469        s.elements.push(el("row", "other", "행", (20.0, 800.0, 390.0, 60.0)));
470        s.elements.push(el("inner", "button", "삭제", (350.0, 810.0, 40.0, 40.0)));
471        s.elements.push(el("outer", "button", "삭제", (350.0, 870.0, 40.0, 40.0)));
472        let sel = Selector::parse("label=\"삭제\" within=\"행\"").unwrap();
473        let found: Vec<_> = sel.find(&s).iter().map(|e| e.id.clone()).collect();
474        assert_eq!(found, vec!["inner"]);
475    }
476
477    #[test]
478    fn structured_form_matches_text_form() {
479        let v = serde_json::json!({"label_contains": "일반", "role": "button", "index": 0});
480        let sel = Selector::from_value(&v).unwrap();
481        assert_eq!(sel.find(&screen())[0].id, "d");
482        assert!(Selector::from_value(&serde_json::json!({"nope": 1})).is_err());
483        assert!(Selector::from_value(&serde_json::json!({})).is_err());
484    }
485
486    #[test]
487    fn duplicate_targets_collapse_but_real_ambiguity_stays() {
488        let a = el("t", "text", "메인", (20.0, 100.0, 390.0, 50.0));
489        let b = el("b", "button", "메인", (20.0, 100.0, 390.0, 50.0));
490        let c = el("c", "button", "메인", (20.0, 300.0, 390.0, 50.0));
491        let one = dedupe_targets(vec![&a, &b]);
492        assert_eq!(one.len(), 1);
493        assert_eq!(one[0].id, "b");
494        assert_eq!(dedupe_targets(vec![&a, &b, &c]).len(), 2);
495    }
496
497    #[test]
498    fn rejects_bad_input() {
499        assert!(Selector::parse("").is_err());
500        assert!(Selector::parse("label/[/").is_err());
501        assert!(Selector::parse("index=x").is_err());
502    }
503}