Skip to main content

computer_use_linux/
atspi_tree.rs

1use crate::diagnostics::hydrate_session_bus_env;
2use anyhow::{anyhow, Context, Result};
3use atspi::{
4    proxy::{
5        accessible::{AccessibleProxy, ObjectRefExt},
6        proxy_ext::ProxyExt,
7    },
8    CoordType, ObjectRef, ObjectRefOwned, StateSet,
9};
10// Direct dependency (p2p feature off) — see Cargo.toml for why we bypass
11// atspi's "connection" re-export.
12use atspi_connection::AccessibilityConnection;
13use futures_util::{stream, StreamExt};
14use schemars::JsonSchema;
15use serde::Serialize;
16use std::{collections::VecDeque, future::Future, time::Duration};
17use tokio::time::timeout;
18use zbus::{
19    fdo::DBusProxy,
20    names::{BusName, UniqueName},
21    zvariant::ObjectPath,
22};
23
24#[derive(Debug, Clone, Serialize, JsonSchema)]
25pub struct AccessibleAppSummary {
26    pub object_ref: String,
27    pub name: Option<String>,
28    pub pid: Option<u32>,
29    pub role: String,
30    pub child_count: i32,
31    pub bounds: Option<Bounds>,
32}
33
34#[derive(Debug, Clone, Serialize, JsonSchema)]
35pub struct AccessibilityNode {
36    pub index: u32,
37    pub parent_index: Option<u32>,
38    pub depth: u32,
39    pub object_ref: String,
40    pub role: String,
41    pub name: Option<String>,
42    pub description: Option<String>,
43    pub child_count: i32,
44    pub bounds: Option<Bounds>,
45    pub states: Vec<String>,
46    pub actions: Vec<AccessibilityAction>,
47    pub value: Option<AccessibilityValue>,
48    pub text: Option<AccessibilityText>,
49    pub supports_editable_text: bool,
50}
51
52#[derive(Debug, Clone, Serialize, JsonSchema)]
53pub struct Bounds {
54    pub x: i32,
55    pub y: i32,
56    pub width: i32,
57    pub height: i32,
58}
59
60#[derive(Debug, Clone, Serialize, JsonSchema)]
61pub struct AccessibilityAction {
62    pub index: i32,
63    pub name: String,
64    pub description: String,
65    pub keybinding: String,
66}
67
68#[derive(Debug, Clone, Serialize, JsonSchema)]
69pub struct AccessibilityValue {
70    pub current: f64,
71    pub minimum: f64,
72    pub maximum: f64,
73    pub minimum_increment: f64,
74    pub text: Option<String>,
75}
76
77#[derive(Debug, Clone, Serialize, JsonSchema)]
78pub struct AccessibilityText {
79    pub character_count: i32,
80    pub caret_offset: Option<i32>,
81    pub content: Option<String>,
82    pub truncated: bool,
83    pub selections: Vec<AccessibilityTextSelection>,
84}
85
86#[derive(Debug, Clone, Serialize, JsonSchema)]
87pub struct AccessibilityTextSelection {
88    pub start_offset: i32,
89    pub end_offset: i32,
90}
91
92#[derive(Debug, Clone)]
93pub struct ActionInvocation {
94    pub action_index: i32,
95    pub action_name: Option<String>,
96    pub ok: bool,
97}
98
99#[derive(Debug, Clone)]
100pub enum ValueSetInvocation {
101    Numeric { value: f64 },
102    EditableText,
103}
104
105const MAX_TEXT_READBACK_CHARS: i32 = 4096;
106const MAX_TEXT_SELECTIONS: i32 = 8;
107const DEFAULT_SNAPSHOT_MAX_NODES: usize = 1_000;
108const HARD_SNAPSHOT_MAX_NODES: usize = 2_000;
109const DEFAULT_SNAPSHOT_MAX_DEPTH: u32 = 32;
110const HARD_SNAPSHOT_MAX_DEPTH: u32 = 64;
111const CHILD_READ_CONCURRENCY: usize = 16;
112const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10);
113const MAX_DISCOVERY_ROOTS: usize = 256;
114const ROOT_MATCH_CHILD_LIMIT: usize = 8;
115const MAX_DISCOVERY_CHILD_READS: usize = MAX_DISCOVERY_ROOTS * ROOT_MATCH_CHILD_LIMIT;
116
117fn snapshot_child_read_budgets(max_nodes: usize) -> (usize, usize, usize) {
118    (MAX_DISCOVERY_ROOTS, MAX_DISCOVERY_CHILD_READS, max_nodes)
119}
120
121pub(crate) fn snapshot_limits(
122    requested_max_nodes: Option<usize>,
123    requested_max_depth: Option<u32>,
124) -> (usize, u32) {
125    (
126        requested_max_nodes
127            .unwrap_or(DEFAULT_SNAPSHOT_MAX_NODES)
128            .clamp(1, HARD_SNAPSHOT_MAX_NODES),
129        requested_max_depth
130            .unwrap_or(DEFAULT_SNAPSHOT_MAX_DEPTH)
131            .min(HARD_SNAPSHOT_MAX_DEPTH),
132    )
133}
134
135struct BoundedTraversal<T> {
136    queue: VecDeque<T>,
137    attempted: usize,
138    max_items: usize,
139    /// Set once an enqueue had to drop offered items for lack of capacity.
140    dropped: bool,
141}
142
143impl<T> BoundedTraversal<T> {
144    fn new(max_items: usize) -> Self {
145        Self {
146            queue: VecDeque::new(),
147            attempted: 0,
148            max_items,
149            dropped: false,
150        }
151    }
152
153    fn enqueue(&mut self, items: impl IntoIterator<Item = T>) {
154        let capacity = self.remaining_capacity();
155        let mut items = items.into_iter();
156        self.queue.extend(items.by_ref().take(capacity));
157        if items.next().is_some() {
158            self.dropped = true;
159        }
160    }
161
162    fn pop(&mut self) -> Option<T> {
163        if self.attempted >= self.max_items {
164            return None;
165        }
166        let item = self.queue.pop_front()?;
167        self.attempted += 1;
168        Some(item)
169    }
170
171    fn remaining_capacity(&self) -> usize {
172        self.max_items
173            .saturating_sub(self.attempted.saturating_add(self.queue.len()))
174    }
175
176    /// True when the budget stopped work that was actually offered. `enqueue`
177    /// is the only gate: it never exceeds `remaining_capacity`, so
178    /// `attempted + queue.len() <= max_items` always holds and the queue can
179    /// never be non-empty at the attempt cap. Dropped items are the whole story.
180    fn truncated(&self) -> bool {
181        self.dropped
182    }
183}
184
185fn bounded_child_count(reported: i32, limit: usize) -> usize {
186    usize::try_from(reported).unwrap_or_default().min(limit)
187}
188
189struct IndexedReadBatch<T> {
190    items: Vec<T>,
191    attempted: usize,
192    /// True when fewer children were attempted than the parent reported, because
193    /// the caller's limit or the shared read budget ran out. Failed reads do not
194    /// count; they were attempted.
195    incomplete: bool,
196}
197
198impl<T> IndexedReadBatch<T> {
199    fn all_failed(&self) -> bool {
200        self.attempted > 0 && self.items.is_empty()
201    }
202}
203
204async fn fetch_indexed_up_to<T, E, F, Fut>(
205    reported: i32,
206    limit: usize,
207    remaining_attempts: &mut usize,
208    fetch: F,
209) -> IndexedReadBatch<T>
210where
211    F: Fn(i32) -> Fut,
212    Fut: Future<Output = std::result::Result<T, E>>,
213{
214    let attempt_count = bounded_child_count(reported, limit).min(*remaining_attempts);
215    *remaining_attempts = (*remaining_attempts).saturating_sub(attempt_count);
216    let end_index = i32::try_from(attempt_count).unwrap_or(i32::MAX);
217
218    let items = stream::iter(0..end_index)
219        .map(fetch)
220        .buffered(CHILD_READ_CONCURRENCY)
221        .filter_map(|result| async move { result.ok() })
222        .collect()
223        .await;
224
225    IndexedReadBatch {
226        items,
227        attempted: attempt_count,
228        incomplete: attempt_count < usize::try_from(reported).unwrap_or_default(),
229    }
230}
231
232async fn children_up_to(
233    proxy: &AccessibleProxy<'_>,
234    limit: usize,
235    remaining_attempts: &mut usize,
236) -> zbus::Result<IndexedReadBatch<ObjectRefOwned>> {
237    if limit == 0 || *remaining_attempts == 0 {
238        // Budget already spent: stay I/O-free. `incomplete` is false here because
239        // nothing was asked of the parent; callers that already hold the node's
240        // child_count (snapshot_tree_inner) derive truncation from that instead.
241        return Ok(IndexedReadBatch {
242            items: Vec::new(),
243            attempted: 0,
244            incomplete: false,
245        });
246    }
247
248    let child_count = proxy.child_count().await?;
249    Ok(
250        fetch_indexed_up_to(child_count, limit, remaining_attempts, |index| {
251            proxy.get_child_at_index(index)
252        })
253        .await,
254    )
255}
256
257pub async fn list_accessible_apps(limit: usize) -> Result<Vec<AccessibleAppSummary>> {
258    let conn = connect().await?;
259    let mut remaining_child_reads = limit;
260    let roots = registry_children(&conn, limit, &mut remaining_child_reads).await?;
261    let dbus = DBusProxy::new(conn.connection()).await.ok();
262    let mut apps = Vec::new();
263
264    for object_ref in roots.into_iter().take(limit) {
265        if let Ok(proxy) = open_accessible(&conn, &object_ref).await {
266            apps.push(read_app_summary(&proxy, &object_ref, dbus.as_ref()).await);
267        }
268    }
269
270    Ok(apps)
271}
272
273#[derive(Debug, Clone)]
274pub(crate) struct AccessibilitySnapshot {
275    pub nodes: Vec<AccessibilityNode>,
276    /// True when registry roots were filtered to an app name and/or pid.
277    pub scoped: bool,
278    /// The pid every root belongs to, when roots were matched by pid. None
279    /// when they were matched by app name alone or not filtered at all: those
280    /// roots can belong to any app, so no single pid describes the tree.
281    pub root_pid: Option<u32>,
282    /// True when max_nodes, max_depth, or the child read budget stopped
283    /// traversal with unread elements left. Failed element reads do not count.
284    pub truncated: bool,
285}
286
287pub async fn snapshot_tree(
288    app_name_or_bundle_identifier: Option<&str>,
289    target_pid: Option<u32>,
290    max_nodes: usize,
291    max_depth: u32,
292) -> Result<Vec<AccessibilityNode>> {
293    Ok(snapshot_accessibility_tree(
294        app_name_or_bundle_identifier,
295        target_pid,
296        max_nodes,
297        max_depth,
298    )
299    .await?
300    .nodes)
301}
302
303pub(crate) async fn snapshot_accessibility_tree(
304    app_name_or_bundle_identifier: Option<&str>,
305    target_pid: Option<u32>,
306    max_nodes: usize,
307    max_depth: u32,
308) -> Result<AccessibilitySnapshot> {
309    let (max_nodes, max_depth) = snapshot_limits(Some(max_nodes), Some(max_depth));
310    timeout(
311        SNAPSHOT_TIMEOUT,
312        snapshot_tree_inner(
313            app_name_or_bundle_identifier,
314            target_pid,
315            max_nodes,
316            max_depth,
317        ),
318    )
319    .await
320    .context("AT-SPI snapshot exceeded its 10-second deadline")?
321}
322
323async fn snapshot_tree_inner(
324    app_name_or_bundle_identifier: Option<&str>,
325    target_pid: Option<u32>,
326    max_nodes: usize,
327    max_depth: u32,
328) -> Result<AccessibilitySnapshot> {
329    let conn = connect().await?;
330    // App discovery is bounded independently so a tiny requested tree still
331    // finds a target registered after the first accessibility root.
332    let (mut remaining_registry_reads, mut remaining_filter_reads, mut remaining_traversal_reads) =
333        snapshot_child_read_budgets(max_nodes);
334    let roots =
335        registry_children(&conn, MAX_DISCOVERY_ROOTS, &mut remaining_registry_reads).await?;
336    let selected_roots = select_roots(
337        &conn,
338        roots,
339        app_name_or_bundle_identifier,
340        target_pid,
341        &mut remaining_filter_reads,
342    )
343    .await;
344    let scoped = selected_roots.scoped;
345    let root_pid = selected_roots.pid;
346    let mut nodes = Vec::new();
347    let mut truncated = false;
348    let mut traversal = BoundedTraversal::new(max_nodes);
349
350    traversal.enqueue(
351        selected_roots
352            .roots
353            .into_iter()
354            .map(|object_ref| (object_ref, 0_u32, None)),
355    );
356
357    while let Some((object_ref, depth, parent_index)) = traversal.pop() {
358        let Ok(proxy) = open_accessible(&conn, &object_ref).await else {
359            continue;
360        };
361        let index = nodes.len() as u32;
362        let remaining = traversal.remaining_capacity();
363        let node = read_node(&proxy, &object_ref, index, parent_index, depth).await;
364        let child_refs = if depth < max_depth && remaining > 0 {
365            match children_up_to(&proxy, remaining, &mut remaining_traversal_reads).await {
366                Ok(batch) => {
367                    // A read-budget cut inside the fetch reports `incomplete`; an
368                    // already-exhausted budget returns the I/O-free empty batch, so
369                    // fall back to the child_count read_node already fetched.
370                    truncated |= batch.incomplete || (batch.attempted == 0 && node.child_count > 0);
371                    batch.items
372                }
373                Err(_) => Vec::new(),
374            }
375        } else {
376            // Depth or node cap reached: any child this node reports is unread.
377            truncated |= node.child_count > 0;
378            Vec::new()
379        };
380
381        nodes.push(node);
382
383        traversal.enqueue(
384            child_refs
385                .into_iter()
386                .map(|child| (child, depth + 1, Some(index))),
387        );
388    }
389    truncated |= traversal.truncated();
390
391    Ok(AccessibilitySnapshot {
392        nodes,
393        scoped,
394        root_pid,
395        truncated,
396    })
397}
398
399/// Compact description of the AT-SPI element that currently holds keyboard
400/// focus, used as post-input feedback for type_text/press_key.
401#[derive(Debug, Clone, Serialize, JsonSchema)]
402pub struct FocusedElementSummary {
403    pub role: String,
404    pub name: Option<String>,
405    pub editable: bool,
406    pub states: Vec<String>,
407    /// AT-SPI `Role::Terminal`, read from the role enum rather than the
408    /// localized role name. Internal routing only; not part of the output.
409    #[serde(skip)]
410    #[schemars(skip)]
411    pub is_terminal: bool,
412}
413
414const FOCUS_PROBE_MAX_NODES: usize = 400;
415const FOCUS_PROBE_MAX_DEPTH: u32 = 16;
416
417/// Find the element with the `focused` state inside the target app (by pid) or
418/// across all apps. Best-effort and bounded: returns Ok(None) when no focused
419/// element is reachable through AT-SPI (common for apps without accessibility
420/// support, e.g. Electron without --force-renderer-accessibility).
421/// Outcome of the bounded focused-element search.
422#[derive(Debug, Clone)]
423pub(crate) enum FocusProbe {
424    Found(FocusedElementSummary),
425    /// The search covered the target's whole tree and nothing holds focus.
426    NoneFocused,
427    /// A node, depth, or read limit stopped the search before it finished, so
428    /// the focused element may exist past the limit.
429    Incomplete,
430    /// A pid was given but no AT-SPI app belongs to it (xterm, or Electron
431    /// without --force-renderer-accessibility), so focus cannot be read.
432    NoAccessibleApp,
433}
434
435/// Focused element in the app owning `target_pid`, or across all apps when no
436/// pid is given. With a pid, only that app's tree is searched: falling back to
437/// every other app would report some unrelated widget as the focus.
438pub(crate) async fn probe_focused_element(target_pid: Option<u32>) -> Result<FocusProbe> {
439    focused_element_probe(target_pid, target_pid.is_some()).await
440}
441
442/// The focused element in the app that owns `target_pid`, for callers that
443/// only need an answer when one exists. Every non-`Found` probe result maps to
444/// `None`; see [`probe_focused_element`] for the distinctions.
445pub(crate) async fn focused_element_summary_in_app(
446    target_pid: u32,
447) -> Result<Option<FocusedElementSummary>> {
448    Ok(match focused_element_probe(Some(target_pid), true).await? {
449        FocusProbe::Found(summary) => Some(summary),
450        _ => None,
451    })
452}
453
454async fn focused_element_probe(
455    target_pid: Option<u32>,
456    require_scoped: bool,
457) -> Result<FocusProbe> {
458    let conn = connect().await?;
459    let mut remaining_registry_reads = MAX_DISCOVERY_ROOTS;
460    let roots =
461        registry_children(&conn, MAX_DISCOVERY_ROOTS, &mut remaining_registry_reads).await?;
462    let mut remaining_filter_reads = MAX_DISCOVERY_CHILD_READS;
463    let selected_roots =
464        select_roots(&conn, roots, None, target_pid, &mut remaining_filter_reads).await;
465    if require_scoped && !selected_roots.scoped {
466        return Ok(FocusProbe::NoAccessibleApp);
467    }
468    let mut traversal = BoundedTraversal::new(FOCUS_PROBE_MAX_NODES);
469    let mut remaining_traversal_reads = FOCUS_PROBE_MAX_NODES;
470    let mut incomplete = false;
471
472    traversal.enqueue(
473        selected_roots
474            .roots
475            .into_iter()
476            .map(|object_ref| (object_ref, 0_u32)),
477    );
478
479    while let Some((object_ref, depth)) = traversal.pop() {
480        let Ok(proxy) = open_accessible(&conn, &object_ref).await else {
481            continue;
482        };
483        let Ok(state) = proxy.get_state().await else {
484            continue;
485        };
486        if state.contains(atspi::State::Focused) {
487            let proxies = proxy.proxies().await.ok();
488            let is_terminal = matches!(proxy.get_role().await, Ok(atspi::Role::Terminal));
489            return Ok(FocusProbe::Found(FocusedElementSummary {
490                role: role_name(&proxy).await,
491                name: optional_string(proxy.name().await.ok()),
492                editable: supports_editable_text(proxies.as_ref()).await,
493                states: state_labels(state),
494                is_terminal,
495            }));
496        }
497        let remaining = traversal.remaining_capacity();
498        if depth < FOCUS_PROBE_MAX_DEPTH && remaining > 0 {
499            if let Ok(batch) =
500                children_up_to(&proxy, remaining, &mut remaining_traversal_reads).await
501            {
502                incomplete |= batch.incomplete;
503                // An exhausted shared read budget takes children_up_to's
504                // I/O-free path (attempted 0, incomplete false), so the node's
505                // own child count has to say whether children went unread.
506                // With budget left, attempted 0 just means a leaf: no read.
507                if batch.attempted == 0 && remaining_traversal_reads == 0 && !incomplete {
508                    incomplete = proxy.child_count().await.is_ok_and(|count| count > 0);
509                }
510                traversal.enqueue(batch.items.into_iter().map(|child| (child, depth + 1)));
511            }
512        } else if !incomplete {
513            // Depth or node cap: any child this node reports goes unread. One
514            // child-count read settles it, and only until the first cut is seen.
515            incomplete = proxy.child_count().await.is_ok_and(|count| count > 0);
516        }
517    }
518    if incomplete || traversal.truncated() {
519        return Ok(FocusProbe::Incomplete);
520    }
521
522    Ok(FocusProbe::NoneFocused)
523}
524
525pub async fn perform_action(
526    object_ref_id: &str,
527    requested_action: Option<&str>,
528) -> Result<ActionInvocation> {
529    perform_action_inner(object_ref_id, requested_action, false).await
530}
531
532pub(crate) async fn perform_named_action(
533    object_ref_id: &str,
534    name: &str,
535) -> Result<ActionInvocation> {
536    perform_action_inner(object_ref_id, Some(name), true).await
537}
538
539async fn perform_action_inner(
540    object_ref_id: &str,
541    requested_action: Option<&str>,
542    name_only: bool,
543) -> Result<ActionInvocation> {
544    let conn = connect().await?;
545    let object_ref = object_ref_from_id(object_ref_id)?;
546    let proxy = open_accessible(&conn, &object_ref)
547        .await
548        .with_context(|| format!("failed to open AT-SPI object {object_ref_id}"))?;
549    let action = proxy
550        .proxies()
551        .await?
552        .action()
553        .await
554        .context("element does not expose the AT-SPI Action interface")?;
555    let actions = action.get_actions().await.unwrap_or_default();
556    let action_index = if name_only {
557        select_named_action_index(&actions, requested_action.unwrap_or_default())?
558    } else {
559        select_action_index(&actions, requested_action)?
560    };
561    let action_name = actions
562        .get(action_index as usize)
563        .map(|action| action.name.clone());
564    let ok = action
565        .do_action(action_index)
566        .await
567        .with_context(|| format!("failed to invoke AT-SPI action {action_index}"))?;
568
569    Ok(ActionInvocation {
570        action_index,
571        action_name,
572        ok,
573    })
574}
575
576pub async fn set_element_value(object_ref_id: &str, value: &str) -> Result<ValueSetInvocation> {
577    let conn = connect().await?;
578    let object_ref = object_ref_from_id(object_ref_id)?;
579    let proxy = open_accessible(&conn, &object_ref)
580        .await
581        .with_context(|| format!("failed to open AT-SPI object {object_ref_id}"))?;
582    let proxies = proxy.proxies().await?;
583
584    if let Ok(numeric_value) = value.parse::<f64>() {
585        if let Ok(value_proxy) = proxies.value().await {
586            value_proxy
587                .set_current_value(numeric_value)
588                .await
589                .with_context(|| {
590                    format!("failed to set AT-SPI numeric value to {numeric_value}")
591                })?;
592            return Ok(ValueSetInvocation::Numeric {
593                value: numeric_value,
594            });
595        }
596    }
597
598    if let Ok(editable_text) = proxies.editable_text().await {
599        let ok = editable_text
600            .set_text_contents(value)
601            .await
602            .context("failed to set AT-SPI editable text contents")?;
603        if ok {
604            return Ok(ValueSetInvocation::EditableText);
605        }
606        return Err(anyhow!("AT-SPI EditableText rejected the new contents"));
607    }
608
609    if value.parse::<f64>().is_err() && proxies.value().await.is_ok() {
610        return Err(anyhow!(
611            "element exposes the AT-SPI Value interface, but the requested value is not numeric"
612        ));
613    }
614
615    Err(anyhow!(
616        "element does not expose AT-SPI Value or EditableText interfaces"
617    ))
618}
619
620async fn connect() -> Result<AccessibilityConnection> {
621    hydrate_session_bus_env();
622    AccessibilityConnection::new()
623        .await
624        .context("failed to connect to AT-SPI bus")
625}
626
627/// Open an `AccessibleProxy` for an object on the a11y bus.
628///
629/// We deliberately avoid `AccessibilityConnection::object_as_accessible` (the
630/// `P2P` trait). For apps that advertise a peer-to-peer bus address it routes
631/// reads over that socket, but for apps that don't (notably GTK4 apps such as
632/// Nautilus / Text Editor / baobab, which don't implement the legacy
633/// `GetApplicationBusAddress`) it falls back to a proxy built with only a path
634/// and *no destination*. On the shared a11y bus that proxy can't address the
635/// app and every call fails with `ServiceUnknown`, which surfaces as an empty
636/// tree (`role: "unknown"`, `child_count: 0`). `as_accessible_proxy` always
637/// pins the destination to the object's bus name, so it works for every app
638/// regardless of P2P support. See issue #31.
639async fn open_accessible<'r>(
640    conn: &AccessibilityConnection,
641    object_ref: &'r ObjectRefOwned,
642) -> Result<AccessibleProxy<'r>, atspi::AtspiError> {
643    object_ref.as_accessible_proxy(conn.connection()).await
644}
645
646async fn registry_children(
647    conn: &AccessibilityConnection,
648    limit: usize,
649    remaining_child_reads: &mut usize,
650) -> Result<Vec<ObjectRefOwned>> {
651    let root = conn
652        .root_accessible_on_registry()
653        .await
654        .context("failed to open AT-SPI registry root")?;
655    let batch = children_up_to(&root, limit, remaining_child_reads)
656        .await
657        .context("failed to read AT-SPI registry children")?;
658    if batch.all_failed() {
659        return Err(anyhow!(
660            "AT-SPI registry reported children, but every indexed child read failed"
661        ));
662    }
663    Ok(batch.items)
664}
665
666struct SelectedRoots {
667    roots: Vec<ObjectRefOwned>,
668    scoped: bool,
669    /// Set only when the roots were chosen because they belong to this pid.
670    pid: Option<u32>,
671}
672
673/// Normalized app-name filter, or `None` when the caller passed nothing usable.
674/// A `None` needle with no pid match means the snapshot covers the whole desktop.
675fn app_name_needle(app_name_or_bundle_identifier: Option<&str>) -> Option<String> {
676    app_name_or_bundle_identifier
677        .map(str::trim)
678        .filter(|value| !value.is_empty())
679        .map(|value| value.to_ascii_lowercase())
680}
681
682async fn select_roots(
683    conn: &AccessibilityConnection,
684    roots: Vec<ObjectRefOwned>,
685    app_name_or_bundle_identifier: Option<&str>,
686    target_pid: Option<u32>,
687    remaining_child_reads: &mut usize,
688) -> SelectedRoots {
689    let needle = app_name_needle(app_name_or_bundle_identifier);
690    let dbus = DBusProxy::new(conn.connection()).await.ok();
691    let mut remaining = roots;
692
693    if let Some(target_pid) = target_pid {
694        let mut pid_and_filter_matches = Vec::new();
695        let mut pid_matches = Vec::new();
696        let mut non_pid_matches = Vec::new();
697
698        for object_ref in remaining {
699            if object_ref_pid(dbus.as_ref(), &object_ref).await == Some(target_pid) {
700                if let Some(needle) = needle.as_deref() {
701                    if root_matches(conn, &object_ref, needle, remaining_child_reads).await {
702                        pid_and_filter_matches.push(object_ref);
703                    } else {
704                        pid_matches.push(object_ref);
705                    }
706                } else {
707                    pid_matches.push(object_ref);
708                }
709            } else {
710                non_pid_matches.push(object_ref);
711            }
712        }
713
714        if !pid_and_filter_matches.is_empty() {
715            return SelectedRoots {
716                roots: pid_and_filter_matches,
717                scoped: true,
718                pid: Some(target_pid),
719            };
720        }
721        if !pid_matches.is_empty() {
722            return SelectedRoots {
723                roots: pid_matches,
724                scoped: true,
725                pid: Some(target_pid),
726            };
727        }
728
729        remaining = non_pid_matches;
730    }
731
732    let Some(needle) = needle.as_deref() else {
733        return SelectedRoots {
734            roots: remaining,
735            scoped: false,
736            pid: None,
737        };
738    };
739
740    let mut selected = Vec::new();
741    for object_ref in remaining {
742        if root_matches(conn, &object_ref, needle, remaining_child_reads).await {
743            selected.push(object_ref);
744        }
745    }
746
747    SelectedRoots {
748        roots: selected,
749        scoped: true,
750        pid: None,
751    }
752}
753
754async fn root_matches(
755    conn: &AccessibilityConnection,
756    object_ref: &ObjectRefOwned,
757    needle: &str,
758    remaining_child_reads: &mut usize,
759) -> bool {
760    let Ok(proxy) = open_accessible(conn, object_ref).await else {
761        return object_ref_id(object_ref)
762            .to_ascii_lowercase()
763            .contains(needle);
764    };
765
766    if proxy_matches(&proxy, object_ref, needle).await {
767        return true;
768    }
769
770    for child_ref in children_up_to(&proxy, ROOT_MATCH_CHILD_LIMIT, remaining_child_reads)
771        .await
772        .map(|batch| batch.items)
773        .unwrap_or_default()
774    {
775        let Ok(child_proxy) = open_accessible(conn, &child_ref).await else {
776            continue;
777        };
778        if proxy_matches(&child_proxy, &child_ref, needle).await {
779            return true;
780        }
781    }
782
783    false
784}
785
786async fn proxy_matches(
787    proxy: &AccessibleProxy<'_>,
788    object_ref: &ObjectRefOwned,
789    needle: &str,
790) -> bool {
791    let name = proxy.name().await.unwrap_or_default();
792    let role = proxy.get_role_name().await.unwrap_or_default();
793    format!("{} {} {}", object_ref_id(object_ref), name, role)
794        .to_ascii_lowercase()
795        .contains(needle)
796}
797
798async fn read_app_summary(
799    proxy: &AccessibleProxy<'_>,
800    object_ref: &ObjectRefOwned,
801    dbus: Option<&DBusProxy<'_>>,
802) -> AccessibleAppSummary {
803    AccessibleAppSummary {
804        object_ref: object_ref_id(object_ref),
805        name: optional_string(proxy.name().await.ok()),
806        pid: object_ref_pid(dbus, object_ref).await,
807        role: role_name(proxy).await,
808        child_count: proxy.child_count().await.unwrap_or_default(),
809        bounds: bounds(proxy).await,
810    }
811}
812
813async fn read_node(
814    proxy: &AccessibleProxy<'_>,
815    object_ref: &ObjectRefOwned,
816    index: u32,
817    parent_index: Option<u32>,
818    depth: u32,
819) -> AccessibilityNode {
820    let proxies = proxy.proxies().await.ok();
821
822    AccessibilityNode {
823        index,
824        parent_index,
825        depth,
826        object_ref: object_ref_id(object_ref),
827        role: role_name(proxy).await,
828        name: optional_string(proxy.name().await.ok()),
829        description: optional_string(proxy.description().await.ok()),
830        child_count: proxy.child_count().await.unwrap_or_default(),
831        bounds: bounds_from_proxies(proxies.as_ref(), proxy).await,
832        states: states_from_proxy(proxy).await,
833        actions: actions_from_proxies(proxies.as_ref()).await,
834        value: value_from_proxies(proxies.as_ref()).await,
835        text: text_from_proxies(proxies.as_ref()).await,
836        supports_editable_text: supports_editable_text(proxies.as_ref()).await,
837    }
838}
839
840async fn role_name(proxy: &AccessibleProxy<'_>) -> String {
841    if let Ok(role) = proxy.get_role_name().await {
842        if !role.trim().is_empty() {
843            return role;
844        }
845    }
846    proxy
847        .get_role()
848        .await
849        .map(|role| format!("{role:?}"))
850        .unwrap_or_else(|_| "unknown".to_string())
851}
852
853async fn bounds(proxy: &AccessibleProxy<'_>) -> Option<Bounds> {
854    bounds_from_proxies(proxy.proxies().await.ok().as_ref(), proxy).await
855}
856
857/// Pid of the process that owns an AT-SPI object ref (`:bus/path`), from the
858/// accessibility bus. `None` when the owner is gone or the bus cannot say.
859pub(crate) async fn object_ref_owner_pid(object_ref_id: &str) -> Result<Option<u32>> {
860    let object_ref = object_ref_from_id(object_ref_id)?;
861    let conn = connect().await?;
862    let dbus = DBusProxy::new(conn.connection()).await.ok();
863    Ok(object_ref_pid(dbus.as_ref(), &object_ref).await)
864}
865
866async fn object_ref_pid(dbus: Option<&DBusProxy<'_>>, object_ref: &ObjectRefOwned) -> Option<u32> {
867    let dbus = dbus?;
868    let bus_name = BusName::try_from(object_ref.name_as_str()?.to_string()).ok()?;
869    dbus.get_connection_unix_process_id(bus_name).await.ok()
870}
871
872async fn bounds_from_proxies(
873    proxies: Option<&atspi::proxy::proxy_ext::Proxies<'_>>,
874    proxy: &AccessibleProxy<'_>,
875) -> Option<Bounds> {
876    let owned_proxies;
877    let proxies = if let Some(proxies) = proxies {
878        proxies
879    } else {
880        owned_proxies = proxy.proxies().await.ok()?;
881        &owned_proxies
882    };
883    let component = proxies.component().await.ok()?;
884    let (x, y, width, height) = component.get_extents(CoordType::Screen).await.ok()?;
885    normalize_bounds(Bounds {
886        x,
887        y,
888        width,
889        height,
890    })
891}
892
893fn normalize_bounds(bounds: Bounds) -> Option<Bounds> {
894    if bounds.width <= 0 || bounds.height <= 0 {
895        return None;
896    }
897    if bounds.x <= i32::MIN / 2 || bounds.y <= i32::MIN / 2 {
898        return None;
899    }
900    Some(bounds)
901}
902
903async fn actions_from_proxies(
904    proxies: Option<&atspi::proxy::proxy_ext::Proxies<'_>>,
905) -> Vec<AccessibilityAction> {
906    let Some(proxies) = proxies else {
907        return Vec::new();
908    };
909    let Ok(action_proxy) = proxies.action().await else {
910        return Vec::new();
911    };
912
913    action_proxy
914        .get_actions()
915        .await
916        .unwrap_or_default()
917        .into_iter()
918        .enumerate()
919        .map(|(index, action)| AccessibilityAction {
920            index: index as i32,
921            name: action.name,
922            description: action.description,
923            keybinding: action.keybinding,
924        })
925        .collect()
926}
927
928async fn states_from_proxy(proxy: &AccessibleProxy<'_>) -> Vec<String> {
929    proxy
930        .get_state()
931        .await
932        .map(state_labels)
933        .unwrap_or_default()
934}
935
936async fn value_from_proxies(
937    proxies: Option<&atspi::proxy::proxy_ext::Proxies<'_>>,
938) -> Option<AccessibilityValue> {
939    let value = proxies?.value().await.ok()?;
940    Some(AccessibilityValue {
941        current: value.current_value().await.ok()?,
942        minimum: value.minimum_value().await.ok()?,
943        maximum: value.maximum_value().await.ok()?,
944        minimum_increment: value.minimum_increment().await.ok()?,
945        text: optional_string(value.text().await.ok()),
946    })
947}
948
949async fn text_from_proxies(
950    proxies: Option<&atspi::proxy::proxy_ext::Proxies<'_>>,
951) -> Option<AccessibilityText> {
952    let text = proxies?.text().await.ok()?;
953    let character_count = text.character_count().await.ok()?.max(0);
954    let caret_offset = text.caret_offset().await.ok();
955    let capped_count = character_count.min(MAX_TEXT_READBACK_CHARS);
956    let content = if capped_count > 0 {
957        optional_string(text.get_text(0, capped_count).await.ok())
958    } else {
959        None
960    };
961    let selection_count = text
962        .get_nselections()
963        .await
964        .unwrap_or_default()
965        .clamp(0, MAX_TEXT_SELECTIONS);
966    let mut selections = Vec::new();
967    for index in 0..selection_count {
968        if let Ok((start_offset, end_offset)) = text.get_selection(index).await {
969            selections.push(AccessibilityTextSelection {
970                start_offset,
971                end_offset,
972            });
973        }
974    }
975
976    Some(AccessibilityText {
977        character_count,
978        caret_offset,
979        content,
980        truncated: character_count > MAX_TEXT_READBACK_CHARS,
981        selections,
982    })
983}
984
985async fn supports_editable_text(proxies: Option<&atspi::proxy::proxy_ext::Proxies<'_>>) -> bool {
986    let Some(proxies) = proxies else {
987        return false;
988    };
989    proxies.editable_text().await.is_ok()
990}
991
992fn state_labels(state_set: StateSet) -> Vec<String> {
993    state_set.iter().map(|state| state.to_string()).collect()
994}
995
996fn select_action_index(actions: &[atspi::Action], requested_action: Option<&str>) -> Result<i32> {
997    if actions.is_empty() {
998        return Err(anyhow!("element exposes no AT-SPI actions"));
999    }
1000
1001    if let Some(requested_action) = requested_action
1002        .map(str::trim)
1003        .filter(|value| !value.is_empty())
1004    {
1005        let requested_action = requested_action.to_ascii_lowercase();
1006        if let Some((index, _)) = actions.iter().enumerate().find(|(_, action)| {
1007            action.name.to_ascii_lowercase() == requested_action
1008                || action.description.to_ascii_lowercase() == requested_action
1009        }) {
1010            return Ok(index as i32);
1011        }
1012
1013        if let Ok(index) = requested_action.parse::<usize>() {
1014            if index < actions.len() {
1015                return Ok(index as i32);
1016            }
1017        }
1018
1019        return Err(anyhow!(
1020            "requested AT-SPI action was not found; available actions: {}",
1021            actions
1022                .iter()
1023                .map(|action| action.name.as_str())
1024                .collect::<Vec<_>>()
1025                .join(", ")
1026        ));
1027    }
1028
1029    Ok(if actions.len() > 1 { 1 } else { 0 })
1030}
1031
1032fn select_named_action_index(actions: &[atspi::Action], name: &str) -> Result<i32> {
1033    actions
1034        .iter()
1035        .position(|action| action.name.eq_ignore_ascii_case(name))
1036        .map(|index| index as i32)
1037        .ok_or_else(|| {
1038            anyhow!("cached AT-SPI action name is no longer available; refresh get_app_state")
1039        })
1040}
1041
1042fn optional_string(value: Option<String>) -> Option<String> {
1043    value
1044        .map(|value| value.trim().to_string())
1045        .filter(|value| !value.is_empty())
1046}
1047
1048fn object_ref_from_id(object_ref_id: &str) -> Result<ObjectRefOwned> {
1049    let (name, path) = split_object_ref_id(object_ref_id)?;
1050    let name = UniqueName::try_from(name.to_string())
1051        .with_context(|| format!("invalid AT-SPI bus name in object ref {object_ref_id}"))?;
1052    let path = ObjectPath::try_from(path.to_string())
1053        .with_context(|| format!("invalid AT-SPI object path in object ref {object_ref_id}"))?;
1054    Ok(ObjectRef::new_owned(name, path))
1055}
1056
1057fn split_object_ref_id(object_ref_id: &str) -> Result<(&str, &str)> {
1058    let Some(path_start) = object_ref_id.find('/') else {
1059        return Err(anyhow!(
1060            "invalid AT-SPI object ref '{object_ref_id}'; expected ':bus/path'"
1061        ));
1062    };
1063    let (name, path) = object_ref_id.split_at(path_start);
1064    if name.is_empty() || path.is_empty() {
1065        return Err(anyhow!(
1066            "invalid AT-SPI object ref '{object_ref_id}'; expected ':bus/path'"
1067        ));
1068    }
1069    Ok((name, path))
1070}
1071
1072fn object_ref_id(object_ref: &ObjectRefOwned) -> String {
1073    format!(
1074        "{}{}",
1075        object_ref.name_as_str().unwrap_or(""),
1076        object_ref.path_as_str()
1077    )
1078}
1079
1080#[cfg(test)]
1081mod tests {
1082    use super::*;
1083
1084    #[test]
1085    fn split_object_ref_id_separates_bus_name_and_path() {
1086        let (name, path) = split_object_ref_id(":1.42/org/a11y/atspi/accessible/7").unwrap();
1087
1088        assert_eq!(name, ":1.42");
1089        assert_eq!(path, "/org/a11y/atspi/accessible/7");
1090    }
1091
1092    #[test]
1093    fn select_action_index_uses_named_action() {
1094        let actions = vec![
1095            atspi::Action {
1096                name: "click".to_string(),
1097                description: "Clicks".to_string(),
1098                keybinding: String::new(),
1099            },
1100            atspi::Action {
1101                name: "show-menu".to_string(),
1102                description: "Shows menu".to_string(),
1103                keybinding: String::new(),
1104            },
1105        ];
1106
1107        assert_eq!(select_action_index(&actions, Some("show-menu")).unwrap(), 1);
1108    }
1109
1110    #[test]
1111    fn cached_action_name_is_revalidated_against_current_actions() {
1112        let mut actions = vec![atspi::Action {
1113            name: "show-menu".into(),
1114            description: "Click".into(),
1115            keybinding: String::new(),
1116        }];
1117        // Neither a stale index nor a matching description may invoke another action.
1118        assert!(select_named_action_index(&actions, "Click").is_err());
1119        actions.push(atspi::Action {
1120            name: "click".into(),
1121            description: String::new(),
1122            keybinding: String::new(),
1123        });
1124        assert_eq!(select_named_action_index(&actions, "Click").unwrap(), 1);
1125    }
1126
1127    #[test]
1128    fn select_action_index_defaults_to_secondary_when_available() {
1129        let actions = vec![
1130            atspi::Action {
1131                name: "click".to_string(),
1132                description: String::new(),
1133                keybinding: String::new(),
1134            },
1135            atspi::Action {
1136                name: "show-menu".to_string(),
1137                description: String::new(),
1138                keybinding: String::new(),
1139            },
1140        ];
1141
1142        assert_eq!(select_action_index(&actions, None).unwrap(), 1);
1143    }
1144
1145    #[test]
1146    fn state_labels_serialize_in_bit_order() {
1147        let labels = state_labels(StateSet::new(atspi::State::Focused | atspi::State::Checked));
1148
1149        assert_eq!(labels, vec!["checked".to_string(), "focused".to_string()]);
1150    }
1151
1152    #[test]
1153    fn default_snapshot_limits_cover_deep_gtk4_trees() {
1154        // Nautilus 50 places file-list cells below depth 20 and can expose
1155        // more than 850 raw nodes. Keep the defaults above that known shape.
1156        assert_eq!(snapshot_limits(None, None), (1_000, 32));
1157    }
1158
1159    #[test]
1160    fn app_name_needle_ignores_blank_values_and_normalizes_the_rest() {
1161        assert_eq!(app_name_needle(None), None);
1162        assert_eq!(app_name_needle(Some("")), None);
1163        assert_eq!(app_name_needle(Some("   ")), None);
1164        assert_eq!(
1165            app_name_needle(Some("Calculator")).as_deref(),
1166            Some("calculator")
1167        );
1168        assert_eq!(
1169            app_name_needle(Some(" :1.64/org/a11y/atspi/accessible/root ")).as_deref(),
1170            Some(":1.64/org/a11y/atspi/accessible/root")
1171        );
1172    }
1173
1174    #[test]
1175    fn requested_snapshot_limits_remain_bounded() {
1176        assert_eq!(snapshot_limits(Some(0), Some(0)), (1, 0));
1177        assert_eq!(snapshot_limits(Some(10_000), Some(128)), (2_000, 64));
1178    }
1179
1180    #[test]
1181    fn app_discovery_budget_is_independent_of_requested_tree_size() {
1182        assert_eq!(snapshot_child_read_budgets(1), (256, 2_048, 1));
1183    }
1184
1185    #[test]
1186    fn traversal_attempts_and_queue_share_one_work_budget() {
1187        let mut traversal = BoundedTraversal::new(4);
1188        traversal.enqueue([1]);
1189
1190        assert_eq!(traversal.pop(), Some(1));
1191        traversal.enqueue(2..=10_000);
1192        assert_eq!(traversal.queue, VecDeque::from([2, 3, 4]));
1193
1194        assert_eq!(traversal.pop(), Some(2));
1195        traversal.enqueue(5..=10_000);
1196        assert_eq!(traversal.queue, VecDeque::from([3, 4]));
1197        assert_eq!(traversal.pop(), Some(3));
1198        assert_eq!(traversal.pop(), Some(4));
1199        assert_eq!(traversal.pop(), None);
1200        assert_eq!(traversal.attempted, 4);
1201        assert!(
1202            traversal.truncated(),
1203            "dropped enqueue items must be reported"
1204        );
1205    }
1206
1207    #[test]
1208    fn traversal_that_drains_within_budget_is_not_truncated() {
1209        let mut traversal = BoundedTraversal::new(3);
1210        traversal.enqueue([1, 2, 3]);
1211        assert_eq!(traversal.pop(), Some(1));
1212        assert_eq!(traversal.pop(), Some(2));
1213        assert_eq!(traversal.pop(), Some(3));
1214        assert_eq!(traversal.pop(), None);
1215        assert!(
1216            !traversal.truncated(),
1217            "exactly max_items real nodes is complete"
1218        );
1219    }
1220
1221    #[test]
1222    fn enqueue_at_zero_capacity_marks_the_traversal_truncated() {
1223        let mut traversal = BoundedTraversal::new(2);
1224        traversal.enqueue([1, 2]);
1225        assert_eq!(traversal.pop(), Some(1));
1226        assert_eq!(traversal.pop(), Some(2));
1227        assert!(!traversal.truncated(), "nothing dropped yet");
1228        // Capacity is zero; the offered item is dropped, which is the only
1229        // path that can mark truncation.
1230        traversal.enqueue([3]);
1231        assert!(traversal.queue.is_empty());
1232        assert_eq!(traversal.pop(), None);
1233        assert!(traversal.truncated());
1234    }
1235
1236    #[test]
1237    fn child_count_is_clamped_before_indexed_reads() {
1238        assert_eq!(bounded_child_count(-1, 4), 0);
1239        assert_eq!(bounded_child_count(3, 4), 3);
1240        assert_eq!(bounded_child_count(i32::MAX, 4), 4);
1241    }
1242
1243    #[tokio::test]
1244    async fn indexed_child_reads_consume_attempts_even_when_one_fails() {
1245        let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
1246        let mut remaining_attempts = 3;
1247        let batch = fetch_indexed_up_to(100, 10, &mut remaining_attempts, {
1248            let calls = calls.clone();
1249            move |index| {
1250                let calls = calls.clone();
1251                async move {
1252                    calls.lock().unwrap().push(index);
1253                    if index == 1 {
1254                        Err(())
1255                    } else {
1256                        Ok(index)
1257                    }
1258                }
1259            }
1260        })
1261        .await;
1262
1263        assert_eq!(batch.items, vec![0, 2]);
1264        assert_eq!(batch.attempted, 3);
1265        assert!(
1266            batch.incomplete,
1267            "100 reported children, 3 attempted: budget cut must be reported"
1268        );
1269        assert!(!batch.all_failed());
1270        assert_eq!(*calls.lock().unwrap(), vec![0, 1, 2]);
1271        assert_eq!(remaining_attempts, 0);
1272
1273        let no_children = fetch_indexed_up_to(100, 10, &mut remaining_attempts, |_| async {
1274            Ok::<_, ()>(99)
1275        })
1276        .await;
1277        assert!(no_children.items.is_empty());
1278        assert_eq!(no_children.attempted, 0);
1279        assert!(!no_children.all_failed());
1280
1281        let mut failed_attempts = 2;
1282        let all_failed =
1283            fetch_indexed_up_to(2, 2, &mut failed_attempts, |_| async { Err::<i32, _>(()) }).await;
1284        assert!(all_failed.all_failed());
1285    }
1286}