car_browser/perception/
pipeline.rs

1//! Perception pipeline trait and basic implementation.
2
3use async_trait::async_trait;
4use thiserror::Error;
5
6use super::ax_converter::AxConverter;
7use super::signals::SignalDetector;
8use super::ui_map::{TextBlock, UiMap};
9use crate::models::{A11yNode, Viewport};
10
11#[derive(Error, Debug)]
12pub enum PerceptionError {
13    #[error("Conversion failed: {0}")]
14    ConversionFailed(String),
15    #[error("Signal detection failed: {0}")]
16    SignalDetectionFailed(String),
17}
18
19/// Trait for perception pipelines that convert raw browser state to UiMap.
20#[async_trait]
21pub trait PerceptionPipeline: Send + Sync {
22    async fn perceive(
23        &self,
24        screenshot: &[u8],
25        a11y_nodes: &[A11yNode],
26        url: &str,
27        viewport: Viewport,
28    ) -> Result<UiMap, PerceptionError>;
29}
30
31/// Basic perception pipeline using accessibility tree only (no visual analysis).
32pub struct BasicPerceptionPipeline {
33    converter: AxConverter,
34    signal_detector: SignalDetector,
35}
36
37impl BasicPerceptionPipeline {
38    pub fn new() -> Self {
39        Self {
40            converter: AxConverter::new(),
41            signal_detector: SignalDetector::new(),
42        }
43    }
44}
45
46impl Default for BasicPerceptionPipeline {
47    fn default() -> Self {
48        Self::new()
49    }
50}
51
52#[async_trait]
53impl PerceptionPipeline for BasicPerceptionPipeline {
54    async fn perceive(
55        &self,
56        _screenshot: &[u8],
57        a11y_nodes: &[A11yNode],
58        url: &str,
59        viewport: Viewport,
60    ) -> Result<UiMap, PerceptionError> {
61        let elements = self.converter.convert(a11y_nodes);
62        let text_blocks = self.extract_text_blocks(a11y_nodes);
63        let page_signals = self.signal_detector.detect(a11y_nodes);
64
65        Ok(UiMap::new(
66            url.to_string(),
67            elements,
68            text_blocks,
69            page_signals,
70            viewport,
71            String::new(),
72        ))
73    }
74}
75
76impl BasicPerceptionPipeline {
77    fn extract_text_blocks(&self, nodes: &[A11yNode]) -> Vec<TextBlock> {
78        extract_ax_text_blocks(nodes)
79    }
80}
81
82/// Extract visible text blocks from the accessibility tree (static text,
83/// labels, headings with a name and non-zero bounds). Shared by the AX-only
84/// and vision-augmented pipelines.
85pub(crate) fn extract_ax_text_blocks(nodes: &[A11yNode]) -> Vec<TextBlock> {
86    nodes
87        .iter()
88        .filter(|n| {
89            let role = n.role.to_lowercase();
90            matches!(role.as_str(), "statictext" | "label" | "heading")
91                && n.name.is_some()
92                && n.bounds.width > 0.0
93                && n.bounds.height > 0.0
94        })
95        .map(|n| TextBlock::from_ax(n.name.clone().unwrap_or_default(), n.bounds))
96        .collect()
97}
98
99#[cfg(test)]
100mod tests {
101    use super::*;
102    use crate::models::Bounds;
103
104    #[tokio::test]
105    async fn test_basic_pipeline() {
106        let pipeline = BasicPerceptionPipeline::new();
107        let nodes = vec![
108            A11yNode {
109                node_id: "n0".to_string(),
110                role: "button".to_string(),
111                name: Some("Submit".to_string()),
112                value: None,
113                bounds: Bounds::new(100.0, 100.0, 80.0, 30.0),
114                children: vec![],
115                focusable: true,
116                focused: false,
117                disabled: false,
118            },
119            A11yNode {
120                node_id: "n1".to_string(),
121                role: "statictext".to_string(),
122                name: Some("Welcome".to_string()),
123                value: None,
124                bounds: Bounds::new(0.0, 0.0, 200.0, 20.0),
125                children: vec![],
126                focusable: false,
127                focused: false,
128                disabled: false,
129            },
130        ];
131        let viewport = Viewport {
132            width: 1280,
133            height: 720,
134            device_pixel_ratio: 2.0,
135        };
136
137        let ui_map = pipeline
138            .perceive(&[], &nodes, "https://example.com", viewport)
139            .await
140            .unwrap();
141        assert_eq!(ui_map.elements.len(), 2);
142        assert_eq!(ui_map.text_blocks.len(), 1);
143        assert_eq!(ui_map.text_blocks[0].text, "Welcome");
144    }
145}
car_browser/perception/pipeline.rs

car_browser/perception/
pipeline.rs
