use std::collections::VecDeque;
use crate::accessibility::{self, attributes, AXUIElementRef};
use crate::error::{AXError, AXResult};
use crate::intent_matching::{score_node, MatchContext};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct NodeId(pub usize);
#[derive(Debug, Clone)]
pub struct SceneNode {
pub id: NodeId,
pub parent: Option<NodeId>,
pub children: Vec<NodeId>,
pub role: Option<String>,
pub title: Option<String>,
pub label: Option<String>,
pub value: Option<String>,
pub description: Option<String>,
pub identifier: Option<String>,
pub bounds: Option<(f64, f64, f64, f64)>,
pub enabled: bool,
pub depth: usize,
}
impl SceneNode {
#[must_use]
pub fn text_labels(&self) -> Vec<&str> {
[
self.title.as_deref(),
self.label.as_deref(),
self.description.as_deref(),
self.value.as_deref(),
self.identifier.as_deref(),
]
.into_iter()
.flatten()
.filter(|s| !s.is_empty())
.collect()
}
#[must_use]
pub fn center(&self) -> Option<(f64, f64)> {
self.bounds.map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
}
}
#[derive(Debug, Clone, Default)]
pub struct SceneGraph {
nodes: Vec<SceneNode>,
}
impl SceneGraph {
#[must_use]
pub fn empty() -> Self {
Self { nodes: Vec::new() }
}
#[must_use]
pub fn len(&self) -> usize {
self.nodes.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.nodes.is_empty()
}
#[must_use]
pub fn get(&self, id: NodeId) -> Option<&SceneNode> {
self.nodes.get(id.0)
}
pub fn iter(&self) -> impl Iterator<Item = &SceneNode> {
self.nodes.iter()
}
#[must_use]
pub fn root(&self) -> Option<&SceneNode> {
self.nodes.first()
}
pub(crate) fn push(&mut self, node: SceneNode) -> NodeId {
let id = NodeId(self.nodes.len());
self.nodes.push(node);
id
}
pub(crate) fn get_mut(&mut self, id: NodeId) -> Option<&mut SceneNode> {
self.nodes.get_mut(id.0)
}
#[must_use]
pub fn nodes_by_role(&self, target_role: &str) -> Vec<&SceneNode> {
self.nodes
.iter()
.filter(|n| n.role.as_deref() == Some(target_role))
.collect()
}
}
#[derive(Debug, Clone)]
pub struct IntentMatch {
pub node_id: NodeId,
pub confidence: f64,
pub match_reason: String,
}
pub fn scan_scene(root_element: AXUIElementRef) -> AXResult<SceneGraph> {
scan_scene_bounded(root_element, 2_000)
}
pub fn scan_scene_bounded(root_element: AXUIElementRef, max_nodes: usize) -> AXResult<SceneGraph> {
if !accessibility::check_accessibility_enabled() {
return Err(AXError::AccessibilityNotEnabled);
}
let mut graph = SceneGraph::default();
let mut queue: VecDeque<(AXUIElementRef, Option<NodeId>, usize)> = VecDeque::new();
queue.push_back((root_element, None, 0));
while let Some((elem_ref, parent_id, depth)) = queue.pop_front() {
if graph.len() >= max_nodes {
break;
}
let node = snapshot_element(elem_ref, parent_id, depth);
let node_id = graph.push(node);
if let Some(pid) = parent_id {
if let Some(parent) = graph.get_mut(pid) {
parent.children.push(node_id);
}
}
if let Ok(children) = accessibility::get_children(elem_ref) {
for child_ref in children {
queue.push_back((child_ref, Some(node_id), depth + 1));
}
}
}
Ok(graph)
}
fn snapshot_element(elem_ref: AXUIElementRef, parent: Option<NodeId>, depth: usize) -> SceneNode {
let bounds = read_bounds(elem_ref);
let enabled =
accessibility::get_bool_attribute_value(elem_ref, attributes::AX_ENABLED).unwrap_or(true);
SceneNode {
id: NodeId(0), parent,
children: Vec::new(),
role: accessibility::get_string_attribute_value(elem_ref, attributes::AX_ROLE),
title: accessibility::get_string_attribute_value(elem_ref, attributes::AX_TITLE),
label: accessibility::get_string_attribute_value(elem_ref, attributes::AX_LABEL),
value: accessibility::get_string_attribute_value(elem_ref, attributes::AX_VALUE),
description: accessibility::get_string_attribute_value(
elem_ref,
attributes::AX_DESCRIPTION,
),
identifier: accessibility::get_string_attribute_value(elem_ref, attributes::AX_IDENTIFIER),
bounds,
enabled,
depth,
}
}
fn read_bounds(elem_ref: AXUIElementRef) -> Option<(f64, f64, f64, f64)> {
let pos = accessibility::get_position_attribute(elem_ref)?;
let size = accessibility::get_size_attribute(elem_ref)?;
Some((pos.x, pos.y, size.width, size.height))
}
#[must_use]
pub fn extract_intent(scene: &SceneGraph, query: &str) -> Vec<IntentMatch> {
let ctx = MatchContext::from_query(query);
let mut matches: Vec<IntentMatch> = scene
.iter()
.filter_map(|node| score_to_match(node, &ctx, scene))
.collect();
matches.sort_by(|a, b| {
b.confidence
.partial_cmp(&a.confidence)
.unwrap_or(std::cmp::Ordering::Equal)
});
matches
}
const MIN_CONFIDENCE: f64 = 0.05;
fn score_to_match(node: &SceneNode, ctx: &MatchContext, scene: &SceneGraph) -> Option<IntentMatch> {
let (confidence, reason) = score_node(node, ctx, scene);
if confidence >= MIN_CONFIDENCE {
Some(IntentMatch {
node_id: node.id,
confidence,
match_reason: reason,
})
} else {
None
}
}
pub fn build_scene_from_nodes(nodes: Vec<SceneNode>) -> SceneGraph {
let mut graph = SceneGraph::default();
for mut node in nodes {
let id = NodeId(graph.nodes.len());
node.id = id;
graph.nodes.push(node);
}
graph
}
#[cfg(test)]
mod tests {
use super::*;
fn make_button(id: usize, title: &str) -> SceneNode {
SceneNode {
id: NodeId(id),
parent: None,
children: vec![],
role: Some("AXButton".into()),
title: Some(title.into()),
label: None,
value: None,
description: None,
identifier: None,
bounds: Some((0.0, f64::from(id as u32) * 40.0, 100.0, 30.0)),
enabled: true,
depth: 1,
}
}
fn make_text_field(id: usize, label: &str) -> SceneNode {
SceneNode {
id: NodeId(id),
parent: None,
children: vec![],
role: Some("AXTextField".into()),
title: None,
label: Some(label.into()),
value: None,
description: None,
identifier: None,
bounds: Some((0.0, f64::from(id as u32) * 40.0, 200.0, 25.0)),
enabled: true,
depth: 1,
}
}
#[test]
fn scene_graph_empty_reports_zero_length() {
let graph = SceneGraph::empty();
assert_eq!(graph.len(), 0);
assert!(graph.is_empty());
}
#[test]
fn scene_graph_push_assigns_sequential_ids() {
let mut graph = SceneGraph::empty();
let id0 = graph.push(make_button(0, "OK"));
let id1 = graph.push(make_button(1, "Cancel"));
assert_eq!(id0, NodeId(0));
assert_eq!(id1, NodeId(1));
}
#[test]
fn scene_graph_get_returns_correct_node() {
let mut graph = SceneGraph::empty();
let id = graph.push(make_button(0, "Save"));
let node = graph.get(id).unwrap();
assert_eq!(node.title.as_deref(), Some("Save"));
}
#[test]
fn scene_graph_get_out_of_range_returns_none() {
let graph = SceneGraph::empty();
assert!(graph.get(NodeId(99)).is_none());
}
#[test]
fn scene_graph_nodes_by_role_filters_correctly() {
let nodes = vec![
make_button(0, "OK"),
make_text_field(1, "Email"),
make_button(2, "Cancel"),
];
let graph = build_scene_from_nodes(nodes);
let buttons = graph.nodes_by_role("AXButton");
assert_eq!(buttons.len(), 2);
}
#[test]
fn scene_graph_root_is_first_node() {
let graph = build_scene_from_nodes(vec![make_button(0, "Root"), make_button(1, "Child")]);
assert_eq!(graph.root().unwrap().id, NodeId(0));
}
#[test]
fn scene_node_text_labels_returns_non_empty_fields() {
let node = SceneNode {
id: NodeId(0),
parent: None,
children: vec![],
role: Some("AXButton".into()),
title: Some("Submit".into()),
label: None,
value: None,
description: Some("Submit the form".into()),
identifier: Some("btn_submit".into()),
bounds: None,
enabled: true,
depth: 0,
};
let labels = node.text_labels();
assert!(labels.contains(&"Submit"));
assert!(labels.contains(&"Submit the form"));
assert!(labels.contains(&"btn_submit"));
assert_eq!(labels.len(), 3);
}
#[test]
fn scene_node_center_computed_correctly() {
let node = make_button(0, "OK");
let (cx, cy) = node.center().unwrap();
assert_eq!(cx, 50.0);
assert_eq!(cy, 15.0);
}
#[test]
fn scene_node_center_returns_none_without_bounds() {
let mut node = make_button(0, "OK");
node.bounds = None;
assert!(node.center().is_none());
}
#[test]
fn extract_intent_finds_exact_title_match() {
let graph =
build_scene_from_nodes(vec![make_button(0, "Submit"), make_button(1, "Cancel")]);
let results = extract_intent(&graph, "submit");
assert!(!results.is_empty());
assert_eq!(results[0].node_id, NodeId(0));
}
#[test]
fn extract_intent_returns_sorted_by_confidence_descending() {
let graph = build_scene_from_nodes(vec![
make_button(0, "Submit"),
make_button(1, "Cancel"),
make_button(2, "Submit Form"),
]);
let results = extract_intent(&graph, "submit");
for window in results.windows(2) {
assert!(window[0].confidence >= window[1].confidence);
}
}
#[test]
fn extract_intent_empty_scene_returns_empty_results() {
let graph = SceneGraph::empty();
let results = extract_intent(&graph, "click ok");
assert!(results.is_empty());
}
#[test]
fn extract_intent_match_reason_is_non_empty() {
let graph = build_scene_from_nodes(vec![make_button(0, "OK")]);
let results = extract_intent(&graph, "ok");
assert!(!results.is_empty());
assert!(!results[0].match_reason.is_empty());
}
#[test]
fn extract_intent_role_hint_boosts_button_for_click_query() {
let graph =
build_scene_from_nodes(vec![make_button(0, "Login"), make_text_field(1, "Login")]);
let results = extract_intent(&graph, "click the login button");
assert!(!results.is_empty());
assert_eq!(
results[0].node_id,
NodeId(0),
"Button should beat text field for click-button query"
);
}
#[test]
fn build_scene_from_nodes_assigns_ids_sequentially() {
let nodes: Vec<SceneNode> = (0..3).map(|i| make_button(i, "X")).collect();
let graph = build_scene_from_nodes(nodes);
for i in 0..3 {
assert_eq!(graph.get(NodeId(i)).unwrap().id, NodeId(i));
}
}
}