agent_tui/detection/
mod.rs

1mod framework;
2mod frameworks;
3pub mod pattern;
4mod registry;
5mod traits;
6
7use crate::terminal::ScreenBuffer;
8use regex::Regex;
9use std::collections::HashSet;
10use std::sync::OnceLock;
11
12fn legacy_ref_regex() -> &'static Regex {
13    static RE: OnceLock<Regex> = OnceLock::new();
14    RE.get_or_init(|| Regex::new(r"^@([a-z]+)(\d+)$").unwrap())
15}
16
17pub use framework::{detect_framework, Framework};
18pub use registry::FrameworkDetector;
19pub use traits::{DetectionContext, ElementDetectorImpl};
20
21#[derive(Debug, Clone, PartialEq, Eq, Hash)]
22pub enum ElementType {
23    Button,
24    Input,
25    Checkbox,
26    Radio,
27    Select,
28    MenuItem,
29    ListItem,
30    Spinner,
31    Progress,
32}
33
34impl ElementType {
35    pub fn as_str(&self) -> &'static str {
36        match self {
37            ElementType::Button => "button",
38            ElementType::Input => "input",
39            ElementType::Checkbox => "checkbox",
40            ElementType::Radio => "radio",
41            ElementType::Select => "select",
42            ElementType::MenuItem => "menuitem",
43            ElementType::ListItem => "listitem",
44            ElementType::Spinner => "spinner",
45            ElementType::Progress => "progress",
46        }
47    }
48}
49
50#[derive(Debug, Clone)]
51pub struct Position {
52    pub row: u16,
53    pub col: u16,
54    pub width: Option<u16>,
55    pub height: Option<u16>,
56}
57
58#[derive(Debug, Clone)]
59pub struct Element {
60    pub element_ref: String,
61    pub element_type: ElementType,
62    pub label: Option<String>,
63    pub value: Option<String>,
64    pub position: Position,
65    pub focused: bool,
66    pub selected: bool,
67    pub checked: Option<bool>,
68    pub disabled: Option<bool>,
69    pub hint: Option<String>,
70}
71
72impl Element {
73    pub fn new(
74        element_ref: String,
75        element_type: ElementType,
76        row: u16,
77        col: u16,
78        width: u16,
79    ) -> Self {
80        Self {
81            element_ref,
82            element_type,
83            label: None,
84            value: None,
85            position: Position {
86                row,
87                col,
88                width: Some(width),
89                height: Some(1),
90            },
91            focused: false,
92            selected: false,
93            checked: None,
94            disabled: None,
95            hint: None,
96        }
97    }
98
99    pub fn is_interactive(&self) -> bool {
100        matches!(
101            self.element_type,
102            ElementType::Button
103                | ElementType::Input
104                | ElementType::Checkbox
105                | ElementType::Radio
106                | ElementType::Select
107                | ElementType::MenuItem
108        )
109    }
110
111    pub fn has_content(&self) -> bool {
112        self.label
113            .as_ref()
114            .map(|l| !l.trim().is_empty())
115            .unwrap_or(false)
116            || self
117                .value
118                .as_ref()
119                .map(|v| !v.trim().is_empty())
120                .unwrap_or(false)
121    }
122}
123
124pub struct ElementDetector {
125    ref_counter: usize,
126    used_refs: HashSet<String>,
127}
128
129impl Default for ElementDetector {
130    fn default() -> Self {
131        Self::new()
132    }
133}
134
135impl ElementDetector {
136    pub fn new() -> Self {
137        Self {
138            ref_counter: 0,
139            used_refs: HashSet::new(),
140        }
141    }
142
143    pub fn detect(
144        &mut self,
145        screen_text: &str,
146        screen_buffer: Option<&ScreenBuffer>,
147    ) -> Vec<Element> {
148        self.detect_with_framework(screen_text, screen_buffer, None)
149    }
150
151    pub fn detect_with_framework(
152        &mut self,
153        screen_text: &str,
154        screen_buffer: Option<&ScreenBuffer>,
155        framework_detector: Option<&FrameworkDetector>,
156    ) -> Vec<Element> {
157        use traits::ElementDetectorImpl;
158
159        self.ref_counter = 0;
160        self.used_refs.clear();
161
162        let mut elements = Vec::new();
163
164        let ctx = DetectionContext::new(screen_text, screen_buffer);
165
166        let default_detector;
167        let detector = match framework_detector {
168            Some(d) => d,
169            None => {
170                default_detector = FrameworkDetector::detect(&ctx);
171                &default_detector
172            }
173        };
174
175        let pattern_matches = detector.detect_patterns(&ctx);
176
177        for m in pattern_matches {
178            let focused = if let Some(buffer) = screen_buffer {
179                self.is_focused_by_style(buffer, m.row, m.col, m.width)
180            } else {
181                false
182            };
183
184            let element_ref = self.generate_ref(
185                &m.element_type,
186                m.label.as_deref(),
187                m.value.as_deref(),
188                m.row,
189                m.col,
190            );
191
192            let mut element = Element::new(element_ref, m.element_type, m.row, m.col, m.width);
193            element.label = m.label;
194            element.value = m.value;
195            element.focused = focused;
196            element.selected = m.checked.unwrap_or(false);
197            element.checked = m.checked;
198
199            elements.push(element);
200        }
201
202        elements.sort_by(|a, b| {
203            if a.position.row != b.position.row {
204                a.position.row.cmp(&b.position.row)
205            } else {
206                a.position.col.cmp(&b.position.col)
207            }
208        });
209
210        elements
211    }
212
213    fn is_focused_by_style(&self, buffer: &ScreenBuffer, row: u16, col: u16, width: u16) -> bool {
214        use crate::terminal::Color;
215
216        let row_idx = row as usize;
217        if row_idx >= buffer.cells.len() {
218            return false;
219        }
220
221        let row_cells = &buffer.cells[row_idx];
222        let start = col as usize;
223        let end = (col + width) as usize;
224
225        row_cells
226            .iter()
227            .take(end.min(row_cells.len()))
228            .skip(start)
229            .any(|cell| {
230                let style = &cell.style;
231
232                if style.inverse {
233                    return true;
234                }
235
236                if style.bold {
237                    let has_colored_fg = match &style.fg_color {
238                        Some(Color::Indexed(idx)) => *idx != 7 && *idx != 15,
239                        Some(Color::Rgb(_, _, _)) => true,
240                        Some(Color::Default) | None => false,
241                    };
242                    if has_colored_fg {
243                        return true;
244                    }
245                }
246
247                let has_highlight_bg = match &style.bg_color {
248                    Some(Color::Indexed(idx)) => *idx != 0 && *idx != 16,
249                    Some(Color::Rgb(r, g, b)) => *r > 20 || *g > 20 || *b > 20,
250                    Some(Color::Default) | None => false,
251                };
252                if has_highlight_bg && cell.char != ' ' {
253                    return true;
254                }
255
256                if style.underline && cell.char != ' ' && cell.char != '_' {
257                    return true;
258                }
259
260                false
261            })
262    }
263
264    fn generate_ref(
265        &mut self,
266        _element_type: &ElementType,
267        _label: Option<&str>,
268        _value: Option<&str>,
269        _row: u16,
270        _col: u16,
271    ) -> String {
272        self.ref_counter += 1;
273        let seq_ref = format!("@e{}", self.ref_counter);
274        self.used_refs.insert(seq_ref.clone());
275        seq_ref
276    }
277
278    pub fn find_by_ref<'a>(&self, elements: &'a [Element], ref_str: &str) -> Option<&'a Element> {
279        let normalized = if ref_str.starts_with('@') {
280            ref_str.to_string()
281        } else {
282            format!("@{}", ref_str)
283        };
284
285        if let Some(el) = elements.iter().find(|e| e.element_ref == normalized) {
286            return Some(el);
287        }
288
289        if let Some(caps) = legacy_ref_regex().captures(&normalized) {
290            let prefix = caps.get(1).map(|m| m.as_str()).unwrap_or("");
291            let index: usize = caps
292                .get(2)
293                .and_then(|m| m.as_str().parse().ok())
294                .unwrap_or(0);
295
296            if index > 0 && prefix != "e" {
297                let target_type = match prefix {
298                    "btn" => Some("button"),
299                    "inp" => Some("input"),
300                    "cb" => Some("checkbox"),
301                    "rb" => Some("radio"),
302                    "sel" => Some("select"),
303                    "mi" => Some("menuitem"),
304                    "li" => Some("listitem"),
305                    "lnk" => Some("link"),
306                    _ => None,
307                };
308
309                if let Some(type_str) = target_type {
310                    let matching: Vec<_> = elements
311                        .iter()
312                        .filter(|e| e.element_type.as_str() == type_str)
313                        .collect();
314
315                    if index <= matching.len() {
316                        return Some(matching[index - 1]);
317                    }
318                }
319            }
320        }
321
322        None
323    }
324}
325
326#[cfg(test)]
327mod tests {
328    use super::*;
329
330    #[test]
331    fn test_generate_sequential_ref() {
332        let mut detector = ElementDetector::new();
333        let ref1 = detector.generate_ref(&ElementType::Button, Some("Submit"), None, 5, 10);
334        let ref2 = detector.generate_ref(&ElementType::Input, Some("Name"), None, 6, 10);
335        let ref3 = detector.generate_ref(&ElementType::Button, Some("Cancel"), None, 7, 10);
336
337        assert_eq!(ref1, "@e1");
338        assert_eq!(ref2, "@e2");
339        assert_eq!(ref3, "@e3");
340    }
341
342    #[test]
343    fn test_refs_reset_on_detect() {
344        let mut detector = ElementDetector::new();
345
346        let elements1 = detector.detect("[Submit] [Cancel]", None);
347        assert!(elements1.iter().any(|e| e.element_ref == "@e1"));
348        assert!(elements1.iter().any(|e| e.element_ref == "@e2"));
349
350        let elements2 = detector.detect("[OK]", None);
351        assert!(elements2.iter().any(|e| e.element_ref == "@e1"));
352    }
353
354    #[test]
355    fn test_find_by_sequential_ref() {
356        let mut detector = ElementDetector::new();
357        let elements = detector.detect("[Submit] [Cancel]", None);
358
359        assert!(detector.find_by_ref(&elements, "@e1").is_some());
360        assert!(detector.find_by_ref(&elements, "@e2").is_some());
361        assert!(detector.find_by_ref(&elements, "@e3").is_none());
362    }
363
364    #[test]
365    fn test_find_by_legacy_ref() {
366        let mut detector = ElementDetector::new();
367        let elements = detector.detect("[Submit] [Cancel]", None);
368
369        let btn1 = detector.find_by_ref(&elements, "@btn1");
370        assert!(btn1.is_some());
371        assert_eq!(btn1.unwrap().element_type.as_str(), "button");
372
373        let btn2 = detector.find_by_ref(&elements, "@btn2");
374        assert!(btn2.is_some());
375        assert_eq!(btn2.unwrap().element_type.as_str(), "button");
376    }
377}
agent_tui/detection/mod.rs

agent_tui/detection/
mod.rs
