car-browser 0.12.0

Browser automation and perception pipeline for Common Agent Runtime
Documentation
//! Accessibility tree to UiElement converter.

use super::ui_map::{ElementSource, UiElement, UiRole, UiState};
use crate::models::A11yNode;

/// Converts A11yNodes from the browser's accessibility tree into UiElements.
pub struct AxConverter {
    /// Minimum bounds size to include an element.
    min_size: f64,
}

impl AxConverter {
    pub fn new() -> Self {
        Self { min_size: 1.0 }
    }

    /// Convert a list of A11yNodes to UiElements.
    ///
    /// Assigns stable IDs based on role priority then position.
    pub fn convert(&self, nodes: &[A11yNode]) -> Vec<UiElement> {
        let mut elements: Vec<UiElement> = nodes
            .iter()
            .filter(|n| self.should_include(n))
            .map(|n| self.convert_node(n))
            .collect();

        // Sort by role priority (interactable first) then by position (top-left)
        elements.sort_by(|a, b| {
            let a_priority = if a.role.is_interactable() { 0 } else { 1 };
            let b_priority = if b.role.is_interactable() { 0 } else { 1 };
            a_priority.cmp(&b_priority).then_with(|| {
                a.bounds
                    .y
                    .partial_cmp(&b.bounds.y)
                    .unwrap_or(std::cmp::Ordering::Equal)
                    .then_with(|| {
                        a.bounds
                            .x
                            .partial_cmp(&b.bounds.x)
                            .unwrap_or(std::cmp::Ordering::Equal)
                    })
            })
        });

        // Assign stable IDs
        for (i, element) in elements.iter_mut().enumerate() {
            element.id = format!("el_{}", i);
        }

        elements
    }

    fn should_include(&self, node: &A11yNode) -> bool {
        // Skip zero-size or very small elements
        if node.bounds.width < self.min_size || node.bounds.height < self.min_size {
            return false;
        }
        // Skip generic containers with no name
        let role = node.role.to_lowercase();
        if matches!(
            role.as_str(),
            "generic" | "group" | "div" | "section" | "webarea"
        ) && node.name.is_none()
        {
            return false;
        }
        true
    }

    fn convert_node(&self, node: &A11yNode) -> UiElement {
        let role = UiRole::from_ax_role(&node.role);
        let states = UiState::from_ax_states(node.disabled, node.focused, None, None, None);

        let mut confidence = ElementSource::AccessibilityTree.base_confidence();
        if node.name.is_some() {
            confidence += 0.05;
        }
        if node.focusable {
            confidence += 0.02;
        }
        if node.disabled {
            confidence -= 0.05;
        }
        if node.bounds.width < 10.0 || node.bounds.height < 10.0 {
            confidence -= 0.10;
        }
        if node.bounds.x == 0.0 && node.bounds.y == 0.0 {
            confidence -= 0.05;
        }
        if matches!(role, UiRole::Container | UiRole::Other(_)) {
            confidence -= 0.05;
        }
        confidence = confidence.clamp(0.0, 1.0);

        UiElement {
            id: String::new(), // Assigned later during sorting
            role,
            name: node.name.clone(),
            value: node.value.clone(),
            bounds: node.bounds,
            states,
            confidence,
            source: ElementSource::AccessibilityTree,
            icon_type: None,
            children: node.children.clone(),
            ax_ref: Some(node.node_id.clone()),
        }
    }
}

impl Default for AxConverter {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::models::Bounds;

    #[test]
    fn test_convert_basic() {
        let converter = AxConverter::new();
        let nodes = vec![
            A11yNode {
                node_id: "n0".to_string(),
                role: "button".to_string(),
                name: Some("Submit".to_string()),
                value: None,
                bounds: Bounds::new(100.0, 100.0, 80.0, 30.0),
                children: vec![],
                focusable: true,
                focused: false,
                disabled: false,
            },
            A11yNode {
                node_id: "n1".to_string(),
                role: "link".to_string(),
                name: Some("Home".to_string()),
                value: None,
                bounds: Bounds::new(0.0, 0.0, 50.0, 20.0),
                children: vec![],
                focusable: true,
                focused: false,
                disabled: false,
            },
        ];

        let elements = converter.convert(&nodes);
        assert_eq!(elements.len(), 2);
        assert_eq!(elements[0].id, "el_0"); // Link at (0,0) first by position
        assert_eq!(elements[1].id, "el_1");
        assert!(elements[0].ax_ref.is_some());
    }

    #[test]
    fn test_filters_zero_size() {
        let converter = AxConverter::new();
        let nodes = vec![A11yNode {
            node_id: "n0".to_string(),
            role: "button".to_string(),
            name: Some("Hidden".to_string()),
            value: None,
            bounds: Bounds::new(0.0, 0.0, 0.0, 0.0),
            children: vec![],
            focusable: false,
            focused: false,
            disabled: false,
        }];

        let elements = converter.convert(&nodes);
        assert!(elements.is_empty());
    }

    #[test]
    fn test_filters_nameless_containers() {
        let converter = AxConverter::new();
        let nodes = vec![A11yNode {
            node_id: "n0".to_string(),
            role: "generic".to_string(),
            name: None,
            value: None,
            bounds: Bounds::new(0.0, 0.0, 1280.0, 720.0),
            children: vec![],
            focusable: false,
            focused: false,
            disabled: false,
        }];

        let elements = converter.convert(&nodes);
        assert!(elements.is_empty());
    }
}