gpui_rhai/
accessibility.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::sync::Arc;
3
4use thiserror::Error;
5
6use crate::{ElementGeometry, GeometryRegistry, NodeId, RetainedNode, RetainedUiTree, UiValue};
7
8#[allow(clippy::struct_excessive_bools)]
9#[derive(Clone, Debug, PartialEq)]
10pub struct AccessibilityNode {
11    pub id: NodeId,
12    pub parent: Option<NodeId>,
13    pub role: String,
14    pub name: String,
15    pub description: String,
16    pub semantic_id: Option<String>,
17    pub test_id: Option<String>,
18    pub value: Option<UiValue>,
19    pub checked: Option<UiValue>,
20    pub selected: Option<bool>,
21    pub pressed: Option<bool>,
22    pub expanded: Option<bool>,
23    pub orientation: Option<String>,
24    pub value_min: Option<f64>,
25    pub value_max: Option<f64>,
26    pub current: Option<String>,
27    pub placeholder: Option<String>,
28    pub key_shortcuts: Option<String>,
29    pub level: Option<usize>,
30    pub position_in_set: Option<usize>,
31    pub size_of_set: Option<usize>,
32    pub row_index: Option<usize>,
33    pub column_index: Option<usize>,
34    pub row_count: Option<usize>,
35    pub column_count: Option<usize>,
36    pub disabled: bool,
37    pub read_only: bool,
38    pub invalid: bool,
39    pub required: bool,
40    pub geometry: Option<ElementGeometry>,
41    pub children: Vec<NodeId>,
42}
43
44/// Immutable semantic projection committed with one successful retained tree.
45///
46/// Native accessibility and automation derive from this same projection. It
47/// deliberately excludes presentation geometry, which is joined only after a
48/// GPUI frame commits.
49#[derive(Clone, Debug, Default, PartialEq)]
50pub struct CommittedSemanticFrame {
51    tree: Arc<AccessibilityTree>,
52}
53
54impl CommittedSemanticFrame {
55    /// Compile and validate the semantic projection for a retained tree.
56    ///
57    /// # Errors
58    ///
59    /// Returns an error for duplicate semantic IDs, unsupported roles, or an
60    /// invalid retained traversal.
61    pub fn from_retained(tree: &RetainedUiTree) -> Result<Self, AccessibilityError> {
62        Ok(Self {
63            tree: Arc::new(AccessibilityTree::build(
64                tree,
65                &GeometryRegistry::new(),
66                false,
67            )?),
68        })
69    }
70
71    #[must_use]
72    pub fn root_ids(&self) -> &[NodeId] {
73        self.tree.root_ids()
74    }
75
76    #[must_use]
77    pub fn node(&self, id: NodeId) -> Option<&AccessibilityNode> {
78        self.tree.node(id)
79    }
80
81    pub fn nodes(&self) -> impl ExactSizeIterator<Item = &AccessibilityNode> {
82        self.tree.nodes()
83    }
84
85    pub(crate) fn apply_primitive_projections(
86        &mut self,
87        projections: BTreeMap<NodeId, crate::PrimitiveAccessibilityProjection>,
88    ) {
89        Arc::make_mut(&mut self.tree).apply_primitive_projections(projections);
90    }
91}
92
93#[derive(Clone, Debug, Default, PartialEq)]
94pub struct AccessibilityTree {
95    roots: Vec<NodeId>,
96    nodes: BTreeMap<NodeId, AccessibilityNode>,
97    semantic_ids: BTreeMap<String, NodeId>,
98}
99
100impl AccessibilityTree {
101    /// Build a stable semantic tree from retained identity and committed geometry.
102    ///
103    /// Nodes without semantics are flattened; their semantic descendants attach
104    /// to the nearest semantic ancestor.
105    ///
106    /// # Errors
107    ///
108    /// Returns [`AccessibilityError::DuplicateSemanticId`] for ambiguous IDs.
109    pub fn from_retained(
110        tree: &RetainedUiTree,
111        geometry: &GeometryRegistry,
112    ) -> Result<Self, AccessibilityError> {
113        let frame = CommittedSemanticFrame::from_retained(tree)?;
114        Ok(Self::from_committed(&frame, tree, geometry, false))
115    }
116
117    /// Build the semantic tree from nodes that participated in the latest
118    /// committed GPUI presentation frame.
119    ///
120    /// # Errors
121    ///
122    /// Returns [`AccessibilityError::DuplicateSemanticId`] for ambiguous IDs
123    /// among currently presented nodes.
124    pub fn from_presented(
125        tree: &RetainedUiTree,
126        geometry: &GeometryRegistry,
127    ) -> Result<Self, AccessibilityError> {
128        let frame = CommittedSemanticFrame::from_retained(tree)?;
129        Ok(Self::from_committed(&frame, tree, geometry, true))
130    }
131
132    pub(crate) fn from_committed(
133        frame: &CommittedSemanticFrame,
134        tree: &RetainedUiTree,
135        geometry: &GeometryRegistry,
136        presented_only: bool,
137    ) -> Self {
138        let mut output = Self::default();
139        for root in frame.root_ids() {
140            project_committed_node(
141                frame,
142                tree,
143                geometry,
144                *root,
145                None,
146                presented_only,
147                &mut output,
148            );
149        }
150        output
151    }
152
153    fn build(
154        tree: &RetainedUiTree,
155        geometry: &GeometryRegistry,
156        presented_only: bool,
157    ) -> Result<Self, AccessibilityError> {
158        let labels = semantic_labels(tree, geometry, presented_only)?;
159        let mut output = Self::default();
160        if let Some(root) = tree.root_id() {
161            visit_retained(
162                tree,
163                geometry,
164                root,
165                None,
166                &labels,
167                presented_only,
168                &mut output,
169            )?;
170        }
171        Ok(output)
172    }
173
174    #[must_use]
175    pub fn root_ids(&self) -> &[NodeId] {
176        &self.roots
177    }
178
179    #[must_use]
180    pub fn node(&self, id: NodeId) -> Option<&AccessibilityNode> {
181        self.nodes.get(&id)
182    }
183
184    pub fn nodes(&self) -> impl ExactSizeIterator<Item = &AccessibilityNode> {
185        self.nodes.values()
186    }
187
188    pub(crate) fn mark_runtime_error(&mut self, message: &str, mark_figures: bool) {
189        let roots = self.roots.iter().copied().collect::<BTreeSet<_>>();
190        for node in self
191            .nodes
192            .values_mut()
193            .filter(|node| roots.contains(&node.id) || (mark_figures && node.role == "figure"))
194        {
195            node.invalid = true;
196            node.description = if node.description.is_empty() {
197                message.to_owned()
198            } else {
199                format!("{} Runtime error: {message}", node.description)
200            };
201        }
202    }
203
204    #[must_use]
205    pub fn find_by_semantic_id(&self, id: &str) -> Option<&AccessibilityNode> {
206        self.semantic_ids
207            .get(id)
208            .and_then(|node| self.nodes.get(node))
209    }
210
211    pub fn find_by_role_and_name<'a>(
212        &'a self,
213        role: &'a str,
214        name: &'a str,
215    ) -> impl Iterator<Item = &'a AccessibilityNode> + 'a {
216        self.nodes
217            .values()
218            .filter(move |node| node.role == role && node.name == name)
219    }
220
221    pub fn find_by_test_id<'a>(
222        &'a self,
223        id: &'a str,
224    ) -> impl Iterator<Item = &'a AccessibilityNode> + 'a {
225        self.nodes
226            .values()
227            .filter(move |node| node.test_id.as_deref() == Some(id))
228    }
229
230    pub(crate) fn apply_primitive_projections(
231        &mut self,
232        projections: BTreeMap<NodeId, crate::PrimitiveAccessibilityProjection>,
233    ) {
234        for (id, projection) in projections {
235            let Some(node) = self.nodes.get_mut(&id) else {
236                continue;
237            };
238            if !projection.description.is_empty() {
239                if !node.description.is_empty() {
240                    node.description.push_str("; ");
241                }
242                node.description.push_str(&projection.description);
243            }
244            if projection.value.is_some() {
245                node.value = projection.value;
246            }
247        }
248    }
249}
250
251fn semantic_labels(
252    tree: &RetainedUiTree,
253    geometry: &GeometryRegistry,
254    presented_only: bool,
255) -> Result<BTreeMap<String, String>, AccessibilityError> {
256    let mut labels = BTreeMap::new();
257    for node in tree.nodes() {
258        if presented_only && !geometry.is_presented(node.id()) {
259            continue;
260        }
261        let Some(id) = string_attribute(node, "semantic_id") else {
262            continue;
263        };
264        let label = string_attribute(node, "label")
265            .or_else(|| node.text().map(ToOwned::to_owned))
266            .unwrap_or_default();
267        if labels.insert(id.clone(), label).is_some() {
268            return Err(AccessibilityError::DuplicateSemanticId(id));
269        }
270    }
271    Ok(labels)
272}
273
274fn visit_retained(
275    tree: &RetainedUiTree,
276    geometry: &GeometryRegistry,
277    id: NodeId,
278    semantic_parent: Option<NodeId>,
279    labels: &BTreeMap<String, String>,
280    presented_only: bool,
281    output: &mut AccessibilityTree,
282) -> Result<(), AccessibilityError> {
283    let retained = tree
284        .node(id)
285        .ok_or(AccessibilityError::MissingRetainedNode(id))?;
286    if presented_only && !geometry.is_presented(id) {
287        return Ok(());
288    }
289    let semantic = semantic_node(tree, retained, semantic_parent, geometry, labels)?;
290    let next_parent = if let Some(node) = semantic {
291        if semantic_parent.is_none() {
292            output.roots.push(id);
293        }
294        if let Some(parent) = semantic_parent
295            && let Some(parent) = output.nodes.get_mut(&parent)
296        {
297            parent.children.push(id);
298        }
299        if let Some(semantic_id) = &node.semantic_id
300            && output
301                .semantic_ids
302                .insert(semantic_id.clone(), id)
303                .is_some()
304        {
305            return Err(AccessibilityError::DuplicateSemanticId(semantic_id.clone()));
306        }
307        output.nodes.insert(id, node);
308        Some(id)
309    } else {
310        semantic_parent
311    };
312    for child in retained.children() {
313        visit_retained(
314            tree,
315            geometry,
316            child.node(),
317            next_parent,
318            labels,
319            presented_only,
320            output,
321        )?;
322    }
323    Ok(())
324}
325
326fn semantic_node(
327    tree: &RetainedUiTree,
328    node: &RetainedNode,
329    parent: Option<NodeId>,
330    geometry: &GeometryRegistry,
331    labels: &BTreeMap<String, String>,
332) -> Result<Option<AccessibilityNode>, AccessibilityError> {
333    let Some(role) =
334        string_attribute(node, "role").or_else(|| node.text().map(|_| "text".to_owned()))
335    else {
336        return Ok(None);
337    };
338    let _ = native_role(&role)?;
339    let explicit_name = string_attribute(node, "label");
340    let name = explicit_name
341        .or_else(|| referenced_text(node, "labelled_by", labels))
342        .or_else(|| node.text().map(ToOwned::to_owned))
343        .unwrap_or_default();
344    Ok(Some(AccessibilityNode {
345        id: node.id(),
346        parent,
347        role,
348        name,
349        description: referenced_text(node, "described_by", labels).unwrap_or_default(),
350        semantic_id: string_attribute(node, "semantic_id"),
351        test_id: string_attribute(node, "test_id"),
352        value: node.attributes().get("value").cloned(),
353        checked: node.attributes().get("checked").cloned(),
354        selected: optional_bool_attribute(node, "selected"),
355        pressed: optional_bool_attribute(node, "pressed"),
356        expanded: optional_bool_attribute(node, "expanded"),
357        orientation: string_attribute(node, "orientation"),
358        value_min: float_attribute(node, "value_min"),
359        value_max: float_attribute(node, "value_max"),
360        current: string_attribute(node, "current"),
361        placeholder: string_attribute(node, "placeholder"),
362        key_shortcuts: string_attribute(node, "key_shortcuts"),
363        level: usize_attribute(node, "level"),
364        position_in_set: usize_attribute(node, "position_in_set"),
365        size_of_set: usize_attribute(node, "size_of_set"),
366        row_index: usize_attribute(node, "row_index"),
367        column_index: usize_attribute(node, "column_index"),
368        row_count: usize_attribute(node, "row_count"),
369        column_count: usize_attribute(node, "column_count"),
370        disabled: bool_attribute(node, "disabled"),
371        read_only: bool_attribute(node, "read_only"),
372        invalid: bool_attribute(node, "invalid"),
373        required: bool_attribute(node, "required"),
374        geometry: retained_geometry(tree, node, geometry),
375        children: Vec::new(),
376    }))
377}
378
379fn project_committed_node(
380    frame: &CommittedSemanticFrame,
381    tree: &RetainedUiTree,
382    geometry: &GeometryRegistry,
383    id: NodeId,
384    parent: Option<NodeId>,
385    presented_only: bool,
386    output: &mut AccessibilityTree,
387) {
388    if presented_only && !geometry.is_presented(id) {
389        return;
390    }
391    let Some(source) = frame.node(id) else {
392        return;
393    };
394    let children = source.children.clone();
395    let mut node = source.clone();
396    node.parent = parent;
397    node.children.clear();
398    node.geometry = tree
399        .node(id)
400        .and_then(|retained| retained_geometry(tree, retained, geometry));
401    if let Some(semantic_id) = &node.semantic_id {
402        output.semantic_ids.insert(semantic_id.clone(), id);
403    }
404    if let Some(parent) = parent {
405        if let Some(parent) = output.nodes.get_mut(&parent) {
406            parent.children.push(id);
407        }
408    } else {
409        output.roots.push(id);
410    }
411    output.nodes.insert(id, node);
412    for child in children {
413        project_committed_node(
414            frame,
415            tree,
416            geometry,
417            child,
418            Some(id),
419            presented_only,
420            output,
421        );
422    }
423}
424
425pub(crate) fn native_role(role: &str) -> Result<Option<gpui::Role>, AccessibilityError> {
426    use gpui::Role;
427    let role = match role {
428        "alert" => Some(Role::Alert),
429        "button" => Some(Role::Button),
430        "checkbox" => Some(Role::CheckBox),
431        "columnheader" => Some(Role::ColumnHeader),
432        "combobox" => Some(Role::ComboBox),
433        "dialog" => Some(Role::Dialog),
434        "document" => Some(Role::Document),
435        "figure" => Some(Role::Figure),
436        "grid" => Some(Role::Grid),
437        "gridcell" => Some(Role::GridCell),
438        "group" => Some(Role::Group),
439        "heading" => Some(Role::Heading),
440        "image" => Some(Role::Image),
441        // GPUI renders actual glyph elements inside retained wrappers. Plain
442        // `text` therefore follows GPUI's own text contract: a Label with a
443        // direct value, populated by `apply_native_semantics`, rather than a
444        // value-less TextRun that native consumers cannot traverse safely.
445        "label" | "text" => Some(Role::Label),
446        "list" => Some(Role::List),
447        "listbox" => Some(Role::ListBox),
448        "listitem" => Some(Role::ListItem),
449        "menu" => Some(Role::Menu),
450        "menuitem" => Some(Role::MenuItem),
451        "navigation" => Some(Role::Navigation),
452        "option" => Some(Role::ListBoxOption),
453        "presentation" => None,
454        "progressbar" => Some(Role::ProgressIndicator),
455        "radio" => Some(Role::RadioButton),
456        "radiogroup" => Some(Role::RadioGroup),
457        "region" => Some(Role::Region),
458        "row" => Some(Role::Row),
459        "rowheader" => Some(Role::RowHeader),
460        "separator" => Some(Role::Splitter),
461        "slider" => Some(Role::Slider),
462        "status" | "statusbar" => Some(Role::Status),
463        "switch" => Some(Role::Switch),
464        "tab" => Some(Role::Tab),
465        "table" => Some(Role::Table),
466        "tablist" => Some(Role::TabList),
467        "text_field" => Some(Role::TextInput),
468        "toolbar" => Some(Role::Toolbar),
469        "tooltip" => Some(Role::Tooltip),
470        "tree" => Some(Role::Tree),
471        "treeitem" => Some(Role::TreeItem),
472        other => return Err(AccessibilityError::UnsupportedRole(other.to_owned())),
473    };
474    Ok(role)
475}
476
477fn retained_geometry(
478    tree: &RetainedUiTree,
479    node: &RetainedNode,
480    geometry: &GeometryRegistry,
481) -> Option<ElementGeometry> {
482    let mut current = Some(node.id());
483    while let Some(id) = current {
484        if let Some(bounds) = geometry.get(id) {
485            return Some(bounds);
486        }
487        current = tree.node(id).and_then(RetainedNode::parent);
488    }
489    None
490}
491
492fn referenced_text(
493    node: &RetainedNode,
494    attribute: &str,
495    labels: &BTreeMap<String, String>,
496) -> Option<String> {
497    let ids = string_attribute(node, attribute)?;
498    let mut seen = BTreeSet::new();
499    let text = ids
500        .split_whitespace()
501        .filter(|id| seen.insert((*id).to_owned()))
502        .filter_map(|id| labels.get(id))
503        .filter(|label| !label.is_empty())
504        .cloned()
505        .collect::<Vec<_>>()
506        .join(" ");
507    (!text.is_empty()).then_some(text)
508}
509
510fn string_attribute(node: &RetainedNode, name: &str) -> Option<String> {
511    match node.attributes().get(name) {
512        Some(UiValue::String(value)) => Some(value.clone()),
513        _ => None,
514    }
515}
516
517fn bool_attribute(node: &RetainedNode, name: &str) -> bool {
518    node.attributes().get(name) == Some(&UiValue::Bool(true))
519}
520
521fn optional_bool_attribute(node: &RetainedNode, name: &str) -> Option<bool> {
522    match node.attributes().get(name) {
523        Some(UiValue::Bool(value)) => Some(*value),
524        _ => None,
525    }
526}
527
528fn float_attribute(node: &RetainedNode, name: &str) -> Option<f64> {
529    match node.attributes().get(name) {
530        Some(UiValue::Float(value)) => Some(*value),
531        Some(UiValue::Integer(value)) => value.to_string().parse().ok(),
532        _ => None,
533    }
534}
535
536fn usize_attribute(node: &RetainedNode, name: &str) -> Option<usize> {
537    match node.attributes().get(name) {
538        Some(UiValue::Integer(value)) => usize::try_from(*value).ok(),
539        _ => None,
540    }
541}
542
543#[derive(Clone, Debug, Error, Eq, PartialEq)]
544pub enum AccessibilityError {
545    #[error("semantic ID `{0}` is declared more than once")]
546    DuplicateSemanticId(String),
547    #[error("retained accessibility traversal lost node {0}")]
548    MissingRetainedNode(NodeId),
549    #[error("unsupported accessibility role `{0}`")]
550    UnsupportedRole(String),
551}
552
553#[cfg(test)]
554mod tests {
555    use super::*;
556
557    #[test]
558    fn committed_frame_rejects_unknown_native_roles_before_presentation() {
559        let mut retained = RetainedUiTree::new();
560        retained
561            .reconcile(
562                crate::UiNode::text("Unknown")
563                    .with_attribute("role", UiValue::String("mystery_widget".to_owned())),
564            )
565            .unwrap();
566        assert_eq!(
567            CommittedSemanticFrame::from_retained(&retained),
568            Err(AccessibilityError::UnsupportedRole(
569                "mystery_widget".to_owned()
570            ))
571        );
572    }
573
574    #[test]
575    fn semantic_tree_flattens_layout_and_resolves_label_relationships() {
576        let label = crate::UiNode::text("Project name")
577            .with_key("label")
578            .with_attribute("semantic_id", UiValue::String("project-label".to_owned()))
579            .with_attribute("role", UiValue::String("label".to_owned()));
580        let input = crate::UiNode::text("value")
581            .with_key("input")
582            .with_attribute("role", UiValue::String("text_field".to_owned()))
583            .with_attribute("labelled_by", UiValue::String("project-label".to_owned()))
584            .with_attribute("required", UiValue::Bool(true));
585        let mut retained = RetainedUiTree::new();
586        retained
587            .reconcile(crate::UiNode::box_node(vec![label, input]))
588            .unwrap();
589        let tree = AccessibilityTree::from_retained(&retained, &GeometryRegistry::new()).unwrap();
590        let field = tree
591            .find_by_role_and_name("text_field", "Project name")
592            .next()
593            .unwrap();
594        assert!(field.required);
595        assert_eq!(tree.nodes().len(), 2);
596        assert_eq!(
597            tree.find_by_semantic_id("project-label").unwrap().name,
598            "Project name"
599        );
600    }
601
602    #[test]
603    fn semantic_descendant_inherits_nearest_realized_geometry() {
604        let root = crate::UiNode::box_node(vec![
605            crate::UiNode::text("Virtual row")
606                .with_key("row")
607                .with_attribute("role", UiValue::String("option".to_owned())),
608        ])
609        .with_key("realized-root");
610        let mut retained = RetainedUiTree::new();
611        retained.reconcile(root).unwrap();
612        let root_id = retained.root_id().unwrap();
613        let bounds = crate::GeometryBounds::new(4.0, 8.0, 100.0, 24.0).unwrap();
614        let geometry = GeometryRegistry::new();
615        geometry.update(
616            root_id,
617            ElementGeometry {
618                layout: bounds,
619                visual: bounds,
620                clip: None,
621            },
622        );
623
624        let tree = AccessibilityTree::from_retained(&retained, &geometry).unwrap();
625        assert_eq!(
626            tree.find_by_role_and_name("option", "Virtual row")
627                .next()
628                .unwrap()
629                .geometry,
630            Some(ElementGeometry {
631                layout: bounds,
632                visual: bounds,
633                clip: None,
634            })
635        );
636    }
637
638    #[test]
639    fn presented_tree_excludes_retained_but_unpainted_subtrees() {
640        let visible = crate::UiNode::text("Visible")
641            .with_key("visible")
642            .with_attribute("role", UiValue::String("button".to_owned()));
643        let hidden = crate::UiNode::text("Hidden")
644            .with_key("hidden")
645            .with_attribute("role", UiValue::String("button".to_owned()));
646        let mut retained = RetainedUiTree::new();
647        retained
648            .reconcile(crate::UiNode::box_node(vec![visible, hidden]))
649            .unwrap();
650        let root = retained.root_id().unwrap();
651        let children = retained
652            .node(root)
653            .unwrap()
654            .children()
655            .map(crate::RetainedChildLink::node)
656            .collect::<Vec<_>>();
657        let geometry = GeometryRegistry::new();
658        let bounds = crate::GeometryBounds::new(0.0, 0.0, 100.0, 24.0).unwrap();
659        let presentation = ElementGeometry {
660            layout: bounds,
661            visual: bounds,
662            clip: None,
663        };
664        geometry.update(root, presentation);
665        geometry.update(children[0], presentation);
666
667        let structural = AccessibilityTree::from_retained(&retained, &geometry).unwrap();
668        assert_eq!(structural.nodes().len(), 2);
669        let presented = AccessibilityTree::from_presented(&retained, &geometry).unwrap();
670        assert_eq!(presented.nodes().len(), 1);
671        assert!(
672            presented
673                .find_by_role_and_name("button", "Visible")
674                .next()
675                .is_some()
676        );
677        assert!(
678            presented
679                .find_by_role_and_name("button", "Hidden")
680                .next()
681                .is_none()
682        );
683    }
684
685    #[test]
686    fn semantic_tree_preserves_pressed_expanded_orientation_and_range_values() {
687        let node = crate::UiNode::text("Volume")
688            .with_attribute("role", UiValue::String("slider".to_owned()))
689            .with_attribute("pressed", UiValue::Bool(false))
690            .with_attribute("expanded", UiValue::Bool(true))
691            .with_attribute("orientation", UiValue::String("horizontal".to_owned()))
692            .with_attribute("value", UiValue::Float(40.0))
693            .with_attribute("value_min", UiValue::Float(0.0))
694            .with_attribute("value_max", UiValue::Float(100.0));
695        let mut retained = RetainedUiTree::new();
696        retained.reconcile(node).unwrap();
697        let tree = AccessibilityTree::from_retained(&retained, &GeometryRegistry::new()).unwrap();
698        let node = tree
699            .find_by_role_and_name("slider", "Volume")
700            .next()
701            .unwrap();
702        assert_eq!(node.pressed, Some(false));
703        assert_eq!(node.expanded, Some(true));
704        assert_eq!(node.orientation.as_deref(), Some("horizontal"));
705        assert_eq!(node.value, Some(UiValue::Float(40.0)));
706        assert_eq!(node.value_min, Some(0.0));
707        assert_eq!(node.value_max, Some(100.0));
708    }
709}
gpui_rhai/accessibility.rs

gpui_rhai/
accessibility.rs
