agent_tui/detection/
pattern.rs

1//! Pattern-based element detection
2//!
3//! Detects TUI elements using regex patterns for common UI patterns.
4
5use super::ElementType;
6use regex::Regex;
7use std::collections::HashSet;
8use std::sync::OnceLock;
9
10/// Cached regex patterns for element detection
11struct PatternRegexes {
12    button: [Regex; 2],
13    input: [Regex; 2],
14    checkbox: [Regex; 2],
15    radio: [Regex; 1],
16    select: [Regex; 2],
17    menu_item: [Regex; 2],
18    list_item: [Regex; 2],
19    spinner: [Regex; 2],
20    progress: [Regex; 2],
21}
22
23fn get_patterns() -> &'static PatternRegexes {
24    static PATTERNS: OnceLock<PatternRegexes> = OnceLock::new();
25    PATTERNS.get_or_init(|| PatternRegexes {
26        button: [
27            // [Button Text] - square brackets
28            Regex::new(r"\[\s*([^\[\]_]+?)\s*\]").unwrap(),
29            // <Button Text> - angle brackets
30            Regex::new(r"<\s*([^<>]+?)\s*>").unwrap(),
31        ],
32        input: [
33            // Label: [value___] - input with label and underscores
34            Regex::new(r"([A-Za-z\s]+):\s*\[([^\]]*?)_*\]").unwrap(),
35            // [value___] - input with underscores
36            Regex::new(r"\[([^\]]*?)_{3,}\]").unwrap(),
37        ],
38        checkbox: [
39            // [x] Label or [ ] Label
40            Regex::new(r"\[([xX✓✔\s])\]\s*(.+?)(?:\s{2,}|$)").unwrap(),
41            // ◉ Label or ◯ Label (filled/empty circle)
42            Regex::new(r"([◉◯●○✓✔])\s+(.+?)(?:\s{2,}|$)").unwrap(),
43        ],
44        radio: [
45            // ( ) Label or (•) Label
46            Regex::new(r"\(([•o*\s])\)\s*(.+?)(?:\s{2,}|$)").unwrap(),
47        ],
48        select: [
49            // Label: value ▼
50            Regex::new(r"([A-Za-z\s]+):\s*\[?([^\]▼▾\n]+?)\s*[▼▾]\]?").unwrap(),
51            // value ▼
52            Regex::new(r"([^\s]+)\s+[▼▾]").unwrap(),
53        ],
54        menu_item: [
55            // > Item or ❯ Item or › Item or ▸ Item
56            Regex::new(r"^\s*([>❯›▸►●•])\s+(.+?)$").unwrap(),
57            // 1. Item (numbered menu)
58            Regex::new(r"^\s*(\d+)\.\s+(.+?)$").unwrap(),
59        ],
60        list_item: [
61            // - Item or * Item
62            Regex::new(r"^\s*[-*]\s+(.+?)$").unwrap(),
63            // • Item
64            Regex::new(r"^\s*•\s+(.+?)$").unwrap(),
65        ],
66        spinner: [
67            // Braille spinners
68            Regex::new(r"[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]").unwrap(),
69            // ASCII spinners
70            Regex::new(r"[\|/\-\\]\s+(.+)").unwrap(),
71        ],
72        progress: [
73            // [████░░░░░░] or [====>    ]
74            Regex::new(r"\[([█▓▒░=\->#\s]+)\]").unwrap(),
75            // 50% or 50 %
76            Regex::new(r"(\d+)\s*%").unwrap(),
77        ],
78    })
79}
80
81/// A pattern match result
82#[derive(Debug, Clone)]
83pub struct PatternMatch {
84    pub element_type: ElementType,
85    pub text: String,
86    pub label: Option<String>,
87    pub value: Option<String>,
88    pub row: u16,
89    pub col: u16,
90    pub width: u16,
91    pub checked: Option<bool>,
92}
93
94/// Detect elements using pattern matching
95pub fn detect_by_pattern(screen_text: &str) -> Vec<PatternMatch> {
96    let mut matches = Vec::new();
97    let lines: Vec<&str> = screen_text.lines().collect();
98
99    let patterns = get_patterns();
100
101    for (row_idx, line) in lines.iter().enumerate() {
102        let row = row_idx as u16;
103
104        // Detect buttons (but not inputs or checkboxes)
105        for pattern in &patterns.button {
106            for cap in pattern.captures_iter(line) {
107                let full_match = cap.get(0).unwrap();
108                let label = cap.get(1).map(|m| m.as_str().trim().to_string());
109
110                // Skip if it looks like an input (has underscores) or checkbox
111                let text = full_match.as_str();
112                if text.contains('_')
113                    || text.starts_with("[x]")
114                    || text.starts_with("[ ]")
115                    || text.starts_with("[X]")
116                    || text.starts_with("[✓]")
117                    || text.starts_with("[✔]")
118                {
119                    continue;
120                }
121
122                // Skip very short "buttons" that are likely just formatting
123                if let Some(ref l) = label {
124                    if l.len() < 2 {
125                        continue;
126                    }
127                }
128
129                matches.push(PatternMatch {
130                    element_type: ElementType::Button,
131                    text: text.to_string(),
132                    label,
133                    value: None,
134                    row,
135                    col: full_match.start() as u16,
136                    width: full_match.as_str().len() as u16,
137                    checked: None,
138                });
139            }
140        }
141
142        // Detect inputs
143        for pattern in &patterns.input {
144            for cap in pattern.captures_iter(line) {
145                let full_match = cap.get(0).unwrap();
146                let label = cap.get(1).map(|m| m.as_str().trim().to_string());
147                let value = cap
148                    .get(2)
149                    .map(|m| m.as_str().trim_end_matches('_').trim().to_string())
150                    .filter(|v| !v.is_empty());
151
152                matches.push(PatternMatch {
153                    element_type: ElementType::Input,
154                    text: full_match.as_str().to_string(),
155                    label,
156                    value,
157                    row,
158                    col: full_match.start() as u16,
159                    width: full_match.as_str().len() as u16,
160                    checked: None,
161                });
162            }
163        }
164
165        // Detect checkboxes
166        for pattern in &patterns.checkbox {
167            for cap in pattern.captures_iter(line) {
168                let full_match = cap.get(0).unwrap();
169                let marker = cap.get(1).map(|m| m.as_str()).unwrap_or("");
170                let label = cap.get(2).map(|m| m.as_str().trim().to_string());
171
172                let is_checked = matches!(marker, "x" | "X" | "✓" | "✔" | "◉" | "●");
173
174                matches.push(PatternMatch {
175                    element_type: ElementType::Checkbox,
176                    text: full_match.as_str().to_string(),
177                    label,
178                    value: Some(if is_checked { "checked" } else { "unchecked" }.to_string()),
179                    row,
180                    col: full_match.start() as u16,
181                    width: full_match.as_str().len() as u16,
182                    checked: Some(is_checked),
183                });
184            }
185        }
186
187        // Detect radio buttons
188        for pattern in &patterns.radio {
189            for cap in pattern.captures_iter(line) {
190                let full_match = cap.get(0).unwrap();
191                let marker = cap.get(1).map(|m| m.as_str()).unwrap_or("");
192                let label = cap.get(2).map(|m| m.as_str().trim().to_string());
193
194                let is_selected = marker != " ";
195
196                matches.push(PatternMatch {
197                    element_type: ElementType::Radio,
198                    text: full_match.as_str().to_string(),
199                    label,
200                    value: Some(
201                        if is_selected {
202                            "selected"
203                        } else {
204                            "unselected"
205                        }
206                        .to_string(),
207                    ),
208                    row,
209                    col: full_match.start() as u16,
210                    width: full_match.as_str().len() as u16,
211                    checked: Some(is_selected),
212                });
213            }
214        }
215
216        // Detect selects/dropdowns
217        for pattern in &patterns.select {
218            for cap in pattern.captures_iter(line) {
219                let full_match = cap.get(0).unwrap();
220                let label = cap.get(1).map(|m| m.as_str().trim().to_string());
221                let value = cap.get(2).map(|m| m.as_str().trim().to_string());
222
223                matches.push(PatternMatch {
224                    element_type: ElementType::Select,
225                    text: full_match.as_str().to_string(),
226                    label,
227                    value,
228                    row,
229                    col: full_match.start() as u16,
230                    width: full_match.as_str().len() as u16,
231                    checked: None,
232                });
233            }
234        }
235
236        // Detect menu items
237        for pattern in &patterns.menu_item {
238            for cap in pattern.captures_iter(line) {
239                let full_match = cap.get(0).unwrap();
240                let label = cap
241                    .get(2)
242                    .or_else(|| cap.get(1))
243                    .map(|m| m.as_str().trim().to_string());
244
245                matches.push(PatternMatch {
246                    element_type: ElementType::MenuItem,
247                    text: full_match.as_str().to_string(),
248                    label,
249                    value: None,
250                    row,
251                    col: full_match.start() as u16,
252                    width: full_match.as_str().len() as u16,
253                    checked: None,
254                });
255            }
256        }
257
258        // Detect list items
259        for pattern in &patterns.list_item {
260            for cap in pattern.captures_iter(line) {
261                let full_match = cap.get(0).unwrap();
262                let label = cap.get(1).map(|m| m.as_str().trim().to_string());
263
264                matches.push(PatternMatch {
265                    element_type: ElementType::ListItem,
266                    text: full_match.as_str().to_string(),
267                    label,
268                    value: None,
269                    row,
270                    col: full_match.start() as u16,
271                    width: full_match.as_str().len() as u16,
272                    checked: None,
273                });
274            }
275        }
276
277        // Detect spinners
278        for pattern in &patterns.spinner {
279            for cap in pattern.captures_iter(line) {
280                let full_match = cap.get(0).unwrap();
281
282                matches.push(PatternMatch {
283                    element_type: ElementType::Spinner,
284                    text: full_match.as_str().to_string(),
285                    label: None,
286                    value: None,
287                    row,
288                    col: full_match.start() as u16,
289                    width: full_match.as_str().len() as u16,
290                    checked: None,
291                });
292            }
293        }
294
295        // Detect progress bars
296        for pattern in &patterns.progress {
297            for cap in pattern.captures_iter(line) {
298                let full_match = cap.get(0).unwrap();
299                let value = cap.get(1).map(|m| m.as_str().trim().to_string());
300
301                matches.push(PatternMatch {
302                    element_type: ElementType::Progress,
303                    text: full_match.as_str().to_string(),
304                    label: None,
305                    value,
306                    row,
307                    col: full_match.start() as u16,
308                    width: full_match.as_str().len() as u16,
309                    checked: None,
310                });
311            }
312        }
313    }
314
315    // Deduplicate overlapping matches
316    deduplicate_matches(matches)
317}
318
319/// Type priority for deduplication
320fn type_priority(t: &ElementType) -> i32 {
321    match t {
322        ElementType::Input => 10,
323        ElementType::Checkbox => 9,
324        ElementType::Radio => 9,
325        ElementType::Select => 8,
326        ElementType::Button => 7,
327        ElementType::MenuItem => 6,
328        ElementType::ListItem => 5,
329        ElementType::Spinner => 4,
330        ElementType::Progress => 3,
331        ElementType::Link => 2,
332        ElementType::Text => 1,
333        ElementType::Container => 1,
334        ElementType::Unknown => 0,
335    }
336}
337
338/// Remove overlapping matches, preferring higher-priority types
339fn deduplicate_matches(mut matches: Vec<PatternMatch>) -> Vec<PatternMatch> {
340    // Sort by priority (higher first), then by position
341    matches.sort_by(|a, b| {
342        let priority_cmp = type_priority(&b.element_type).cmp(&type_priority(&a.element_type));
343        if priority_cmp != std::cmp::Ordering::Equal {
344            return priority_cmp;
345        }
346        if a.row != b.row {
347            return a.row.cmp(&b.row);
348        }
349        a.col.cmp(&b.col)
350    });
351
352    let mut result = Vec::new();
353    let mut occupied: HashSet<(u16, u16)> = HashSet::new();
354
355    for m in matches {
356        // Check if any position is already occupied
357        let mut overlaps = false;
358        for c in m.col..(m.col + m.width) {
359            if occupied.contains(&(m.row, c)) {
360                overlaps = true;
361                break;
362            }
363        }
364
365        if !overlaps {
366            // Mark positions as occupied
367            for c in m.col..(m.col + m.width) {
368                occupied.insert((m.row, c));
369            }
370            result.push(m);
371        }
372    }
373
374    result
375}
376
377#[cfg(test)]
378mod tests {
379    use super::*;
380
381    #[test]
382    fn test_detect_button() {
383        let screen = "[Submit] [Cancel]";
384        let matches = detect_by_pattern(screen);
385
386        assert_eq!(matches.len(), 2);
387        assert!(matches
388            .iter()
389            .all(|m| m.element_type == ElementType::Button));
390    }
391
392    #[test]
393    fn test_detect_checkbox() {
394        let screen = "[x] Accept terms\n[ ] Subscribe to newsletter";
395        let matches = detect_by_pattern(screen);
396
397        let checkboxes: Vec<_> = matches
398            .iter()
399            .filter(|m| m.element_type == ElementType::Checkbox)
400            .collect();
401
402        assert_eq!(checkboxes.len(), 2);
403        assert_eq!(checkboxes[0].checked, Some(true));
404        assert_eq!(checkboxes[1].checked, Some(false));
405    }
406
407    #[test]
408    fn test_detect_input() {
409        let screen = "Name: [John Doe___]";
410        let matches = detect_by_pattern(screen);
411
412        let inputs: Vec<_> = matches
413            .iter()
414            .filter(|m| m.element_type == ElementType::Input)
415            .collect();
416
417        assert_eq!(inputs.len(), 1);
418        assert_eq!(inputs[0].label, Some("Name".to_string()));
419        assert_eq!(inputs[0].value, Some("John Doe".to_string()));
420    }
421
422    #[test]
423    fn test_detect_menu_item() {
424        let screen = "  > Option 1\n    Option 2\n    Option 3";
425        let matches = detect_by_pattern(screen);
426
427        let menu_items: Vec<_> = matches
428            .iter()
429            .filter(|m| m.element_type == ElementType::MenuItem)
430            .collect();
431
432        assert!(!menu_items.is_empty());
433    }
434
435    #[test]
436    fn test_deduplication() {
437        // Input pattern should take priority over button pattern
438        let screen = "[value___]";
439        let matches = detect_by_pattern(screen);
440
441        // Should only have one match (input, not button)
442        assert_eq!(matches.len(), 1);
443        assert_eq!(matches[0].element_type, ElementType::Input);
444    }
445}
agent_tui/detection/pattern.rs

agent_tui/detection/
pattern.rs
