agent_tui/detection/
mod.rs

1//! Element detection for TUI applications
2//!
3//! This module provides pattern-based and style-based detection of
4//! interactive UI elements in terminal output.
5
6mod framework;
7mod ink;
8mod pattern;
9mod region;
10
11use crate::terminal::ScreenBuffer;
12use regex::Regex;
13use std::collections::HashSet;
14
15// Framework detection (available for external use)
16#[allow(unused_imports)]
17pub use framework::{detect_framework, Framework};
18// Ink-specific detection (available for external use)
19#[allow(unused_imports)]
20pub use ink::detect_ink_elements;
21pub use pattern::detect_by_pattern;
22// Region/modal detection (available for external use)
23#[allow(unused_imports)]
24pub use region::{detect_regions, find_modals, find_region_at, BorderStyle, Region};
25
26/// Element types that can be detected
27#[derive(Debug, Clone, PartialEq, Eq, Hash)]
28pub enum ElementType {
29    Button,
30    Input,
31    Checkbox,
32    Radio,
33    Select,
34    MenuItem,
35    ListItem,
36    Link,
37    Spinner,
38    Progress,
39    Text,
40    Container,
41    Unknown,
42}
43
44impl ElementType {
45    pub fn as_str(&self) -> &'static str {
46        match self {
47            ElementType::Button => "button",
48            ElementType::Input => "input",
49            ElementType::Checkbox => "checkbox",
50            ElementType::Radio => "radio",
51            ElementType::Select => "select",
52            ElementType::MenuItem => "menuitem",
53            ElementType::ListItem => "listitem",
54            ElementType::Link => "link",
55            ElementType::Spinner => "spinner",
56            ElementType::Progress => "progress",
57            ElementType::Text => "text",
58            ElementType::Container => "container",
59            ElementType::Unknown => "unknown",
60        }
61    }
62
63    /// Get a short prefix for this element type (useful for generating compact refs)
64    pub fn prefix(&self) -> &'static str {
65        match self {
66            ElementType::Button => "btn",
67            ElementType::Input => "inp",
68            ElementType::Checkbox => "cb",
69            ElementType::Radio => "rb",
70            ElementType::Select => "sel",
71            ElementType::MenuItem => "mi",
72            ElementType::ListItem => "li",
73            ElementType::Link => "lnk",
74            ElementType::Spinner => "spn",
75            ElementType::Progress => "prg",
76            ElementType::Text => "txt",
77            ElementType::Container => "cnt",
78            ElementType::Unknown => "el",
79        }
80    }
81}
82
83/// Position in the terminal
84#[derive(Debug, Clone)]
85pub struct Position {
86    pub row: u16,
87    pub col: u16,
88    pub width: Option<u16>,
89    pub height: Option<u16>,
90}
91
92/// A detected element
93#[derive(Debug, Clone)]
94pub struct Element {
95    pub element_ref: String,
96    pub element_type: ElementType,
97    pub label: Option<String>,
98    pub value: Option<String>,
99    pub position: Position,
100    pub focused: bool,
101    pub selected: bool,
102    pub checked: Option<bool>,
103    pub disabled: Option<bool>,
104    pub hint: Option<String>,
105    pub options: Option<Vec<String>>,
106}
107
108impl Element {
109    pub fn new(
110        element_ref: String,
111        element_type: ElementType,
112        row: u16,
113        col: u16,
114        width: u16,
115    ) -> Self {
116        Self {
117            element_ref,
118            element_type,
119            label: None,
120            value: None,
121            position: Position {
122                row,
123                col,
124                width: Some(width),
125                height: Some(1),
126            },
127            focused: false,
128            selected: false,
129            checked: None,
130            disabled: None,
131            hint: None,
132            options: None,
133        }
134    }
135
136    /// Returns true if this element is interactive (can be clicked, filled, toggled, etc.)
137    /// Used for snapshot filtering with -i/--interactive-only flag.
138    pub fn is_interactive(&self) -> bool {
139        matches!(
140            self.element_type,
141            ElementType::Button
142                | ElementType::Input
143                | ElementType::Checkbox
144                | ElementType::Radio
145                | ElementType::Select
146                | ElementType::MenuItem
147                | ElementType::Link
148        )
149    }
150
151    /// Returns true if this element has meaningful content (non-empty text)
152    /// Used for snapshot filtering with -c/--compact flag.
153    pub fn has_content(&self) -> bool {
154        self.label
155            .as_ref()
156            .map(|l| !l.trim().is_empty())
157            .unwrap_or(false)
158            || self
159                .value
160                .as_ref()
161                .map(|v| !v.trim().is_empty())
162                .unwrap_or(false)
163    }
164}
165
166/// Element detector
167pub struct ElementDetector {
168    /// Counter for sequential element refs
169    ref_counter: usize,
170    used_refs: HashSet<String>,
171}
172
173impl Default for ElementDetector {
174    fn default() -> Self {
175        Self::new()
176    }
177}
178
179impl ElementDetector {
180    pub fn new() -> Self {
181        Self {
182            ref_counter: 0,
183            used_refs: HashSet::new(),
184        }
185    }
186
187    /// Detect elements in the screen
188    pub fn detect(
189        &mut self,
190        screen_text: &str,
191        screen_buffer: Option<&ScreenBuffer>,
192    ) -> Vec<Element> {
193        // Reset for each snapshot (agent-browser pattern: refs are per-snapshot)
194        self.ref_counter = 0;
195        self.used_refs.clear();
196
197        let mut elements = Vec::new();
198        let _lines: Vec<&str> = screen_text.lines().collect();
199
200        // Pattern-based detection
201        let pattern_matches = detect_by_pattern(screen_text);
202
203        for m in pattern_matches {
204            let focused = if let Some(buffer) = screen_buffer {
205                self.is_focused_by_style(buffer, m.row, m.col, m.width)
206            } else {
207                false
208            };
209
210            let element_ref = self.generate_ref(
211                &m.element_type,
212                m.label.as_deref(),
213                m.value.as_deref(),
214                m.row,
215                m.col,
216            );
217
218            let mut element = Element::new(element_ref, m.element_type, m.row, m.col, m.width);
219            element.label = m.label;
220            element.value = m.value;
221            element.focused = focused;
222            element.selected = m.checked.unwrap_or(false);
223            element.checked = m.checked;
224
225            elements.push(element);
226        }
227
228        // Sort elements by position (top-to-bottom, left-to-right)
229        elements.sort_by(|a, b| {
230            if a.position.row != b.position.row {
231                a.position.row.cmp(&b.position.row)
232            } else {
233                a.position.col.cmp(&b.position.col)
234            }
235        });
236
237        elements
238    }
239
240    /// Check if a region has styling that indicates focus
241    ///
242    /// Checks multiple style indicators:
243    /// - Inverse (most common focus indicator)
244    /// - Bold (often used for highlighting)
245    /// - Underline (sometimes used for focus)
246    /// - Non-default foreground color (highlighting)
247    /// - Non-default background color (highlighting)
248    fn is_focused_by_style(&self, buffer: &ScreenBuffer, row: u16, col: u16, width: u16) -> bool {
249        use crate::terminal::Color;
250
251        let row_idx = row as usize;
252        if row_idx >= buffer.cells.len() {
253            return false;
254        }
255
256        let row_cells = &buffer.cells[row_idx];
257        let start = col as usize;
258        let end = (col + width) as usize;
259
260        // Check if any cell in the region has focus-indicating styles
261        row_cells
262            .iter()
263            .take(end.min(row_cells.len()))
264            .skip(start)
265            .any(|cell| {
266                let style = &cell.style;
267
268                // Inverse is the strongest focus indicator
269                if style.inverse {
270                    return true;
271                }
272
273                // Bold combined with non-default color often indicates focus
274                if style.bold {
275                    // Check if we have a non-default foreground color
276                    let has_colored_fg = match &style.fg_color {
277                        Some(Color::Indexed(idx)) => *idx != 7 && *idx != 15, // Not white/bright white
278                        Some(Color::Rgb(_, _, _)) => true,
279                        Some(Color::Default) | None => false,
280                    };
281                    if has_colored_fg {
282                        return true;
283                    }
284                }
285
286                // Non-default background color (not just black) with content
287                let has_highlight_bg = match &style.bg_color {
288                    Some(Color::Indexed(idx)) => *idx != 0 && *idx != 16, // Not black
289                    Some(Color::Rgb(r, g, b)) => *r > 20 || *g > 20 || *b > 20, // Not near-black
290                    Some(Color::Default) | None => false,
291                };
292                if has_highlight_bg && cell.char != ' ' {
293                    return true;
294                }
295
296                // Underline on interactive element text
297                if style.underline && cell.char != ' ' && cell.char != '_' {
298                    return true;
299                }
300
301                false
302            })
303    }
304
305    /// Generate a ref for an element
306    ///
307    /// Uses simple sequential refs like agent-browser: @e1, @e2, @e3
308    /// This makes refs deterministic and easy for AI to reason about.
309    /// Refs reset on each snapshot (expected behavior).
310    fn generate_ref(
311        &mut self,
312        _element_type: &ElementType,
313        _label: Option<&str>,
314        _value: Option<&str>,
315        _row: u16,
316        _col: u16,
317    ) -> String {
318        self.ref_counter += 1;
319        let seq_ref = format!("@e{}", self.ref_counter);
320        self.used_refs.insert(seq_ref.clone());
321        seq_ref
322    }
323
324    /// Find element by ref
325    ///
326    /// Supports both new sequential refs (@e1, @e2) and legacy type-prefixed refs (@btn1, @inp1)
327    pub fn find_by_ref<'a>(&self, elements: &'a [Element], ref_str: &str) -> Option<&'a Element> {
328        let normalized = if ref_str.starts_with('@') {
329            ref_str.to_string()
330        } else {
331            format!("@{}", ref_str)
332        };
333
334        // Exact match first (handles @e1, @e2, etc.)
335        if let Some(el) = elements.iter().find(|e| e.element_ref == normalized) {
336            return Some(el);
337        }
338
339        // Support legacy type-prefixed format (@btn1, @inp2, etc.) for backwards compatibility
340        let legacy_re = Regex::new(r"^@([a-z]+)(\d+)$").unwrap();
341        if let Some(caps) = legacy_re.captures(&normalized) {
342            let prefix = caps.get(1).map(|m| m.as_str()).unwrap_or("");
343            let index: usize = caps
344                .get(2)
345                .and_then(|m| m.as_str().parse().ok())
346                .unwrap_or(0);
347
348            if index > 0 && prefix != "e" {
349                // Map legacy prefix to element type
350                let target_type = match prefix {
351                    "btn" => Some("button"),
352                    "inp" => Some("input"),
353                    "cb" => Some("checkbox"),
354                    "rb" => Some("radio"),
355                    "sel" => Some("select"),
356                    "mi" => Some("menuitem"),
357                    "li" => Some("listitem"),
358                    "lnk" => Some("link"),
359                    _ => None,
360                };
361
362                if let Some(type_str) = target_type {
363                    // Find nth element of this type
364                    let matching: Vec<_> = elements
365                        .iter()
366                        .filter(|e| e.element_type.as_str() == type_str)
367                        .collect();
368
369                    if index <= matching.len() {
370                        return Some(matching[index - 1]);
371                    }
372                }
373            }
374        }
375
376        None
377    }
378
379    /// Find focused element
380    pub fn find_focused<'a>(&self, elements: &'a [Element]) -> Option<&'a Element> {
381        elements.iter().find(|e| e.focused)
382    }
383}
384
385#[cfg(test)]
386mod tests {
387    use super::*;
388
389    #[test]
390    fn test_element_type_prefix() {
391        assert_eq!(ElementType::Button.prefix(), "btn");
392        assert_eq!(ElementType::Input.prefix(), "inp");
393        assert_eq!(ElementType::Checkbox.prefix(), "cb");
394    }
395
396    #[test]
397    fn test_generate_sequential_ref() {
398        let mut detector = ElementDetector::new();
399        let ref1 = detector.generate_ref(&ElementType::Button, Some("Submit"), None, 5, 10);
400        let ref2 = detector.generate_ref(&ElementType::Input, Some("Name"), None, 6, 10);
401        let ref3 = detector.generate_ref(&ElementType::Button, Some("Cancel"), None, 7, 10);
402
403        // Sequential refs: @e1, @e2, @e3 (agent-browser pattern)
404        assert_eq!(ref1, "@e1");
405        assert_eq!(ref2, "@e2");
406        assert_eq!(ref3, "@e3");
407    }
408
409    #[test]
410    fn test_refs_reset_on_detect() {
411        let mut detector = ElementDetector::new();
412
413        // First detection
414        let elements1 = detector.detect("[Submit] [Cancel]", None);
415        assert!(elements1.iter().any(|e| e.element_ref == "@e1"));
416        assert!(elements1.iter().any(|e| e.element_ref == "@e2"));
417
418        // Second detection should reset refs
419        let elements2 = detector.detect("[OK]", None);
420        assert!(elements2.iter().any(|e| e.element_ref == "@e1"));
421    }
422
423    #[test]
424    fn test_find_by_sequential_ref() {
425        let mut detector = ElementDetector::new();
426        let elements = detector.detect("[Submit] [Cancel]", None);
427
428        // Find by sequential ref
429        assert!(detector.find_by_ref(&elements, "@e1").is_some());
430        assert!(detector.find_by_ref(&elements, "@e2").is_some());
431        assert!(detector.find_by_ref(&elements, "@e3").is_none());
432    }
433
434    #[test]
435    fn test_find_by_legacy_ref() {
436        let mut detector = ElementDetector::new();
437        let elements = detector.detect("[Submit] [Cancel]", None);
438
439        // Legacy refs (@btn1, @btn2) should still work via type lookup
440        let btn1 = detector.find_by_ref(&elements, "@btn1");
441        assert!(btn1.is_some());
442        assert_eq!(btn1.unwrap().element_type.as_str(), "button");
443
444        let btn2 = detector.find_by_ref(&elements, "@btn2");
445        assert!(btn2.is_some());
446        assert_eq!(btn2.unwrap().element_type.as_str(), "button");
447    }
448}
agent_tui/detection/mod.rs

agent_tui/detection/
mod.rs
