1#![allow(missing_docs)]
12use rustc_hash::FxHashMap;
13
14use serde_json::Value;
15
16use super::cdp::client::CdpClient;
17use super::cdp::types::{
18 AXNode, AXProperty, AXValue, EvaluateParams, EvaluateResult, GetFullAXTreeResult,
19};
20use super::element::{resolve_ax_session, RefMap};
21
22const INTERACTIVE_ROLES: &[&str] = &[
23 "button",
24 "link",
25 "textbox",
26 "checkbox",
27 "radio",
28 "combobox",
29 "listbox",
30 "menuitem",
31 "menuitemcheckbox",
32 "menuitemradio",
33 "option",
34 "searchbox",
35 "slider",
36 "spinbutton",
37 "switch",
38 "tab",
39 "treeitem",
40 "Iframe",
41];
42
43const CONTENT_ROLES: &[&str] = &[
44 "heading",
45 "cell",
46 "gridcell",
47 "columnheader",
48 "rowheader",
49 "listitem",
50 "article",
51 "region",
52 "main",
53 "navigation",
54];
55
56const INVISIBLE_CHARS: &[char] = &[
57 '\u{FEFF}', '\u{200B}', '\u{200C}', '\u{200D}', '\u{2060}', '\u{00A0}', ];
64
65#[derive(Default)]
66pub struct SnapshotOptions {
67 pub selector: Option<String>,
68 pub interactive: bool,
69 pub compact: bool,
70 pub depth: Option<usize>,
71 pub urls: bool,
72}
73
74struct TreeNode {
75 role: String,
76 name: String,
77 level: Option<i64>,
78 checked: Option<String>,
79 expanded: Option<bool>,
80 selected: Option<bool>,
81 disabled: Option<bool>,
82 required: Option<bool>,
83 value_text: Option<String>,
84 backend_node_id: Option<i64>,
85 children: Vec<usize>,
86 parent_idx: Option<usize>,
87 has_ref: bool,
88 ref_id: Option<String>,
89 depth: usize,
90 cursor_info: Option<CursorElementInfo>,
91 url: Option<String>,
92}
93
94impl TreeNode {
95 fn empty() -> Self {
97 Self {
98 role: String::new(),
99 name: String::new(),
100 level: None,
101 checked: None,
102 expanded: None,
103 selected: None,
104 disabled: None,
105 required: None,
106 value_text: None,
107 backend_node_id: None,
108 children: Vec::new(),
109 parent_idx: None,
110 has_ref: false,
111 ref_id: None,
112 depth: 0,
113 cursor_info: None,
114 url: None,
115 }
116 }
117
118 fn clear(&mut self) {
119 self.role = String::new();
120 self.name = String::new();
121 self.level = None;
122 self.checked = None;
123 self.expanded = None;
124 self.selected = None;
125 self.disabled = None;
126 self.required = None;
127 self.value_text = None;
128 self.backend_node_id = None;
129 self.children.clear();
130 self.parent_idx = None;
131 self.has_ref = false;
132 self.url = None;
133 self.ref_id = None;
134 self.depth = 0;
135 self.cursor_info = None;
136 }
137}
138
139#[derive(Clone, Copy)]
141enum HiddenInputKind {
142 Radio,
143 Checkbox,
144}
145
146impl HiddenInputKind {
147 fn parse(s: &str) -> Option<Self> {
148 match s {
149 "radio" => Some(Self::Radio),
150 "checkbox" => Some(Self::Checkbox),
151 _ => None,
152 }
153 }
154
155 fn as_role(&self) -> &str {
156 match self {
157 Self::Radio => "radio",
158 Self::Checkbox => "checkbox",
159 }
160 }
161}
162
163#[derive(Clone)]
165struct CursorElementInfo {
166 kind: String, hints: Vec<String>,
168 text: String, hidden_input_kind: Option<HiddenInputKind>,
170 hidden_input_checked: Option<String>, }
172
173struct RoleNameTracker {
174 counts: FxHashMap<String, usize>,
176 entries: Vec<(usize, String)>,
177}
178
179impl RoleNameTracker {
180 fn new() -> Self {
181 Self {
182 counts: FxHashMap::default(),
183 entries: Vec::new(),
184 }
185 }
186
187 fn track(&mut self, role: &str, name: &str, node_idx: usize) -> usize {
188 let key = format!("{}:{}", role, name);
189 let count = self.counts.entry(key.clone()).or_insert(0);
190 let nth = *count;
191 *count += 1;
192 self.entries.push((node_idx, key));
193 nth
194 }
195
196 fn get_duplicates(&self) -> FxHashMap<String, usize> {
197 self.counts
198 .iter()
199 .filter(|(_, &count)| count > 1)
200 .map(|(key, &count)| (key.clone(), count))
201 .collect()
202 }
203}
204
205pub async fn take_snapshot(
206 client: &CdpClient,
207 session_id: &str,
208 options: &SnapshotOptions,
209 ref_map: &mut RefMap,
210 frame_id: Option<&str>,
211 iframe_sessions: &FxHashMap<String, String>,
212) -> Result<String, String> {
213 client
214 .send_command_no_params("DOM.enable", Some(session_id))
215 .await?;
216 client
217 .send_command_no_params("Accessibility.enable", Some(session_id))
218 .await?;
219
220 let selector_backend_ids: Option<std::collections::HashSet<i64>> =
224 if let Some(ref selector) = options.selector {
225 let js = format!(
226 "document.querySelector({})",
227 serde_json::to_string(selector).unwrap_or_default()
228 );
229 let result: EvaluateResult = client
230 .send_command_typed(
231 "Runtime.evaluate",
232 &EvaluateParams {
233 expression: js,
234 return_by_value: Some(false),
235 await_promise: Some(false),
236 },
237 Some(session_id),
238 )
239 .await?;
240
241 if let Some(exception) = result.exception_details {
247 let detail = exception
248 .exception
249 .and_then(|e| e.description)
250 .unwrap_or(exception.text);
251 return Err(format!("Invalid selector '{}': {}", selector, detail));
252 }
253 if result.result.subtype.as_deref() != Some("node") {
254 return Err(format!("Selector '{}' did not match any element", selector));
255 }
256 let object_id = result
257 .result
258 .object_id
259 .ok_or_else(|| format!("Selector '{}' did not match any element", selector))?;
260
261 let describe: Value = client
264 .send_command(
265 "DOM.describeNode",
266 Some(serde_json::json!({ "objectId": object_id, "depth": -1 })),
267 Some(session_id),
268 )
269 .await?;
270
271 let root_node = describe
272 .get("node")
273 .ok_or_else(|| format!("Could not resolve DOM node for selector '{}'", selector))?;
274
275 let mut ids = std::collections::HashSet::new();
276 collect_backend_node_ids(root_node, &mut ids);
277
278 if ids.is_empty() {
279 return Err(format!(
280 "Could not resolve backendNodeId for selector '{}'",
281 selector
282 ));
283 }
284
285 Some(ids)
286 } else {
287 None
288 };
289
290 let (ax_params, effective_session_id) =
291 resolve_ax_session(frame_id, session_id, iframe_sessions);
292 if effective_session_id != session_id {
295 let _ = client
296 .send_command_no_params("DOM.enable", Some(effective_session_id))
297 .await;
298 let _ = client
299 .send_command_no_params("Accessibility.enable", Some(effective_session_id))
300 .await;
301 }
302 let ax_tree: GetFullAXTreeResult = client
303 .send_command_typed(
304 "Accessibility.getFullAXTree",
305 &ax_params,
306 Some(effective_session_id),
307 )
308 .await?;
309
310 let (mut tree_nodes, root_indices) = build_tree(&ax_tree.nodes);
311
312 let effective_roots = if let Some(ref id_set) = selector_backend_ids {
315 let in_subtree: Vec<bool> = tree_nodes
317 .iter()
318 .map(|n| n.backend_node_id.is_some_and(|bid| id_set.contains(&bid)))
319 .collect();
320
321 let mut roots = Vec::new();
324 for (idx, node) in tree_nodes.iter().enumerate() {
325 if !in_subtree[idx] {
326 continue;
327 }
328 let parent_in_subtree = node.parent_idx.is_some_and(|pidx| in_subtree[pidx]);
329 if !parent_in_subtree {
330 roots.push(idx);
331 }
332 }
333
334 if roots.is_empty() {
335 return Err(format!(
336 "No accessibility node found for selector '{}'",
337 options.selector.as_deref().unwrap_or("")
338 ));
339 }
340 roots
341 } else {
342 root_indices
343 };
344
345 let mut tracker = RoleNameTracker::new();
346 let mut next_ref: usize = ref_map.next_ref_num();
347
348 let mut nodes_with_refs: Vec<(usize, usize)> = Vec::new();
349
350 let cursor_elements: FxHashMap<i64, CursorElementInfo> =
352 find_cursor_interactive_elements(client, session_id)
353 .await
354 .unwrap_or_default();
355
356 promote_hidden_inputs(&mut tree_nodes, &cursor_elements);
357
358 for (idx, node) in tree_nodes.iter().enumerate() {
359 let role = node.role.as_str();
360 let mut should_ref = if INTERACTIVE_ROLES.contains(&role) {
361 true
362 } else if CONTENT_ROLES.contains(&role) {
363 !node.name.is_empty()
364 } else {
365 false
366 };
367
368 if node
369 .backend_node_id
370 .is_some_and(|bid| cursor_elements.contains_key(&bid))
371 {
372 should_ref = true;
374 }
375
376 if should_ref {
377 let nth = tracker.track(role, &node.name, idx);
378 nodes_with_refs.push((idx, nth));
379 }
380 }
381
382 let duplicates = tracker.get_duplicates();
383
384 for (idx, nth) in &nodes_with_refs {
385 let node = &tree_nodes[*idx];
386 let key = format!("{}:{}", node.role, node.name);
387 let actual_nth = if duplicates.contains_key(&key) {
388 Some(*nth)
389 } else {
390 None
391 };
392
393 let ref_id = format!("e{}", next_ref);
394 next_ref += 1;
395
396 ref_map.add_with_frame(
397 ref_id.clone(),
398 tree_nodes[*idx].backend_node_id,
399 &tree_nodes[*idx].role,
400 &tree_nodes[*idx].name,
401 actual_nth,
402 frame_id,
403 );
404
405 tree_nodes[*idx].has_ref = true;
406 tree_nodes[*idx].ref_id = Some(ref_id);
407 }
408
409 for (idx, _) in &nodes_with_refs {
411 if let Some(bid) = tree_nodes[*idx].backend_node_id {
412 if let Some(cursor_info) = cursor_elements.get(&bid) {
413 tree_nodes[*idx].cursor_info = Some((*cursor_info).clone());
414 }
415 }
416 }
417
418 ref_map.set_next_ref_num(next_ref);
419
420 if options.urls {
421 let link_nodes: Vec<(usize, i64)> = tree_nodes
422 .iter()
423 .enumerate()
424 .filter(|(_, n)| n.role == "link" && n.has_ref && n.backend_node_id.is_some())
425 .filter_map(|(i, n)| n.backend_node_id.map(|bid| (i, bid)))
426 .collect();
427
428 if !link_nodes.is_empty() {
429 let cdp_limit = crate::concurrency::effective_limit_capped(32);
433 let resolve_futs: Vec<_> = link_nodes
434 .iter()
435 .map(|&(idx, bid)| async move {
436 let resolved = client
437 .send_command(
438 "DOM.resolveNode",
439 Some(serde_json::json!({ "backendNodeId": bid })),
440 Some(session_id),
441 )
442 .await;
443 let obj_id = resolved.ok().and_then(|r| {
444 r.get("object")
445 .and_then(|o| o.get("objectId"))
446 .and_then(|v| v.as_str())
447 .map(|s| s.to_string())
448 });
449 (idx, obj_id)
450 })
451 .collect();
452 let resolved: Vec<(usize, Option<String>)> =
453 crate::concurrency::join_bounded(resolve_futs, cdp_limit).await;
454
455 let href_futs: Vec<_> = resolved
457 .iter()
458 .filter_map(|(idx, obj_id)| {
459 let oid = obj_id.as_ref()?;
460 Some(async move {
461 let result = client
462 .send_command(
463 "Runtime.callFunctionOn",
464 Some(serde_json::json!({
465 "objectId": oid,
466 "functionDeclaration": "function() { return this.href || ''; }",
467 "returnByValue": true,
468 })),
469 Some(session_id),
470 )
471 .await;
472 let href = result.ok().and_then(|r| {
473 r.get("result")
474 .and_then(|r| r.get("value"))
475 .and_then(|v| v.as_str())
476 .filter(|s| !s.is_empty())
477 .map(|s| s.to_string())
478 });
479 (*idx, href)
480 })
481 })
482 .collect();
483 let hrefs: Vec<(usize, Option<String>)> =
484 crate::concurrency::join_bounded(href_futs, cdp_limit).await;
485
486 for (idx, href) in hrefs {
487 if let Some(url) = href {
488 tree_nodes[idx].url = Some(url);
489 }
490 }
491 }
492 }
493
494 let mut output = String::new();
495 for &root_idx in &effective_roots {
496 render_tree(&tree_nodes, root_idx, 0, &mut output, options);
497 }
498
499 if frame_id.is_none() {
504 let mut iframe_snapshots: Vec<(String, String)> = Vec::new(); for node in tree_nodes.iter() {
506 if node.role != "Iframe" || !node.has_ref {
507 continue;
508 }
509 let Some(bid) = node.backend_node_id else {
510 continue;
511 };
512 let ref_id = node.ref_id.as_deref().unwrap_or("");
513 if let Ok(child_fid) = resolve_iframe_frame_id(client, session_id, bid).await {
514 if let Ok(child_text) = Box::pin(take_snapshot(
517 client,
518 session_id,
519 options,
520 ref_map,
521 Some(&child_fid),
522 iframe_sessions,
523 ))
524 .await
525 {
526 if !child_text.is_empty()
527 && child_text != "(empty page)"
528 && child_text != "(no interactive elements)"
529 {
530 iframe_snapshots.push((ref_id.to_string(), child_text));
531 }
532 }
533 }
534 }
535
536 for (ref_id, child_text) in iframe_snapshots {
538 let marker = format!("[ref={}]", ref_id);
539 if let Some(pos) = output.find(&marker) {
540 let line_end = output[pos..]
542 .find('\n')
543 .map(|i| pos + i)
544 .unwrap_or(output.len());
545 let line_start = output[..pos].rfind('\n').map(|i| i + 1).unwrap_or(0);
547 let iframe_line = &output[line_start..line_end];
548 let iframe_indent = iframe_line.len() - iframe_line.trim_start().len();
549 let child_indent = iframe_indent + 2; let prefix = " ".repeat(child_indent);
551
552 let indented_child: String = child_text
553 .lines()
554 .map(|line| format!("{}{}\n", prefix, line))
555 .collect();
556
557 if line_end == output.len() {
559 output.push('\n');
560 output.push_str(&indented_child);
561 } else {
562 output.insert_str(line_end + 1, &indented_child);
563 }
564 }
565 }
566 }
567
568 if options.compact {
569 output = compact_tree(&output, options.interactive);
570 }
571
572 let trimmed = output.trim().to_string();
573
574 if trimmed.is_empty() {
575 if options.interactive {
576 return Ok("(no interactive elements)".to_string());
577 }
578 return Ok("(empty page)".to_string());
579 }
580
581 Ok(trimmed)
582}
583
584async fn resolve_iframe_frame_id(
586 client: &CdpClient,
587 session_id: &str,
588 backend_node_id: i64,
589) -> Result<String, String> {
590 let describe: Value = client
592 .send_command(
593 "DOM.describeNode",
594 Some(serde_json::json!({ "backendNodeId": backend_node_id, "depth": 1 })),
595 Some(session_id),
596 )
597 .await?;
598
599 if let Some(frame_id) = describe
601 .get("node")
602 .and_then(|n| n.get("contentDocument"))
603 .and_then(|cd| cd.get("frameId"))
604 .and_then(|v| v.as_str())
605 {
606 return Ok(frame_id.to_string());
607 }
608
609 describe
611 .get("node")
612 .and_then(|n| n.get("frameId"))
613 .and_then(|v| v.as_str())
614 .map(|s| s.to_string())
615 .ok_or_else(|| "Could not resolve iframe frame ID".to_string())
616}
617
618async fn find_cursor_interactive_elements(
619 client: &CdpClient,
620 session_id: &str,
621) -> Result<FxHashMap<i64, CursorElementInfo>, String> {
622 let js = r#"
632(function() {
633 var results = [];
634 if (!document.body) return results;
635
636 var interactiveRoles = {
637 'button':1, 'link':1, 'textbox':1, 'checkbox':1, 'radio':1, 'combobox':1, 'listbox':1,
638 'menuitem':1, 'menuitemcheckbox':1, 'menuitemradio':1, 'option':1, 'searchbox':1,
639 'slider':1, 'spinbutton':1, 'switch':1, 'tab':1, 'treeitem':1
640 };
641 var interactiveTags = {
642 'a':1, 'button':1, 'input':1, 'select':1, 'textarea':1, 'details':1, 'summary':1
643 };
644
645 var allElements = document.body.querySelectorAll('*');
646 for (var i = 0; i < allElements.length; i++) {
647 var el = allElements[i];
648
649 if (el.closest && el.closest('[hidden], [aria-hidden="true"]')) continue;
650
651 var tagName = el.tagName.toLowerCase();
652 if (interactiveTags[tagName]) continue;
653
654 var role = el.getAttribute('role');
655 if (role && interactiveRoles[role.toLowerCase()]) continue;
656
657 var computedStyle = getComputedStyle(el);
658 var hasCursorPointer = computedStyle.cursor === 'pointer';
659 var hasOnClick = el.hasAttribute('onclick') || el.onclick !== null;
660 var tabIndex = el.getAttribute('tabindex');
661 var hasTabIndex = tabIndex !== null && tabIndex !== '-1';
662 var ce = el.getAttribute('contenteditable');
663 var isEditable = ce === '' || ce === 'true';
664
665 if (!hasCursorPointer && !hasOnClick && !hasTabIndex && !isEditable) continue;
666
667 // Skip elements that only inherit cursor:pointer from an ancestor
668 if (hasCursorPointer && !hasOnClick && !hasTabIndex && !isEditable) {
669 var parent = el.parentElement;
670 if (parent && getComputedStyle(parent).cursor === 'pointer') continue;
671 }
672
673 var text = (el.textContent || '').trim().slice(0, 100);
674
675 var rect = el.getBoundingClientRect();
676 if (rect.width === 0 || rect.height === 0) continue;
677
678 // Detect hidden radio/checkbox inputs inside this element (common pattern:
679 // <label> wrapping a display:none <input type="radio"> styled as a card).
680 // Note: we only check display/visibility/hidden, NOT opacity:0 or sr-only,
681 // because those inputs remain in Chrome's AX tree and already appear as
682 // role="radio" without promotion.
683 var hiddenInputType = null;
684 var hiddenInputChecked = null;
685 var hiddenInput = el.querySelector('input[type="radio"], input[type="checkbox"]');
686 if (hiddenInput) {
687 var hiddenInputStyle = getComputedStyle(hiddenInput);
688 var isInputHidden = hiddenInputStyle.display === 'none' || hiddenInputStyle.visibility === 'hidden' || hiddenInput.hidden;
689 if (isInputHidden) {
690 hiddenInputType = hiddenInput.type;
691 hiddenInputChecked = hiddenInput.indeterminate ? 'mixed' : String(hiddenInput.checked);
692 }
693 }
694
695 el.setAttribute('data-__ab-ci', String(results.length));
696 results.push({
697 text: text,
698 tagName: tagName,
699 hasOnClick: hasOnClick,
700 hasCursorPointer: hasCursorPointer,
701 hasTabIndex: hasTabIndex,
702 isEditable: isEditable,
703 hiddenInputType: hiddenInputType,
704 hiddenInputChecked: hiddenInputChecked
705 });
706 }
707 return results;
708})()
709"#;
710
711 let result: EvaluateResult = client
712 .send_command_typed(
713 "Runtime.evaluate",
714 &EvaluateParams {
715 expression: js.to_string(),
716 return_by_value: Some(true),
717 await_promise: Some(false),
718 },
719 Some(session_id),
720 )
721 .await?;
722
723 let elements: Vec<Value> = result
724 .result
725 .value
726 .and_then(|v| serde_json::from_value::<Vec<Value>>(v).ok())
727 .unwrap_or_default();
728
729 if elements.is_empty() {
730 return Ok(FxHashMap::default());
731 }
732
733 let doc: Value = client
736 .send_command(
737 "DOM.getDocument",
738 Some(serde_json::json!({ "depth": 0 })),
739 Some(session_id),
740 )
741 .await?;
742
743 let root_node_id = doc
744 .get("root")
745 .and_then(|r| r.get("nodeId"))
746 .and_then(|v| v.as_i64())
747 .ok_or("DOM.getDocument did not return root nodeId")?;
748
749 let query_result: Value = client
750 .send_command(
751 "DOM.querySelectorAll",
752 Some(serde_json::json!({
753 "nodeId": root_node_id,
754 "selector": "[data-__ab-ci]"
755 })),
756 Some(session_id),
757 )
758 .await?;
759
760 let node_ids: Vec<i64> = query_result
761 .get("nodeIds")
762 .and_then(|v| v.as_array())
763 .map(|arr| arr.iter().filter_map(|v| v.as_i64()).collect())
764 .unwrap_or_default();
765
766 let cdp_limit = crate::concurrency::effective_limit_capped(32);
768 let describe_futures: Vec<_> = node_ids
769 .iter()
770 .map(|&node_id| {
771 client.send_command(
772 "DOM.describeNode",
773 Some(serde_json::json!({ "nodeId": node_id })),
774 Some(session_id),
775 )
776 })
777 .collect();
778
779 let describe_results =
780 crate::concurrency::join_bounded_ordered(describe_futures, cdp_limit).await;
781
782 let mut idx_to_backend: FxHashMap<usize, i64> = FxHashMap::default();
784 for desc in describe_results.into_iter().flatten() {
785 let backend_id = desc
786 .get("node")
787 .and_then(|n| n.get("backendNodeId"))
788 .and_then(|v| v.as_i64());
789 let ci_attr = desc
790 .get("node")
791 .and_then(|n| n.get("attributes"))
792 .and_then(|a| a.as_array())
793 .and_then(|attrs| {
794 attrs
796 .iter()
797 .enumerate()
798 .find(|(_, v)| v.as_str() == Some("data-__ab-ci"))
799 .and_then(|(i, _)| attrs.get(i + 1))
800 .and_then(|v| v.as_str())
801 .and_then(|s| s.parse::<usize>().ok())
802 });
803 if let (Some(bid), Some(idx)) = (backend_id, ci_attr) {
804 idx_to_backend.insert(idx, bid);
805 }
806 }
807
808 let cleanup_js =
810 r#"(function(){ var els = document.querySelectorAll('[data-__ab-ci]'); for (var i = 0; i < els.length; i++) els[i].removeAttribute('data-__ab-ci'); return els.length; })()"#.to_string();
811 if let Err(e) = client
812 .send_command_typed::<EvaluateParams, EvaluateResult>(
813 "Runtime.evaluate",
814 &EvaluateParams {
815 expression: cleanup_js,
816 return_by_value: Some(true),
817 await_promise: Some(false),
818 },
819 Some(session_id),
820 )
821 .await
822 {
823 tracing::warn!(
824 target: "browser_automation_cli::native::snapshot",
825 error = %e,
826 "failed to clean up data-__ab-ci attributes"
827 );
828 }
829
830 let mut map: FxHashMap<i64, CursorElementInfo> = FxHashMap::default();
832 for (i, elem) in elements.iter().enumerate() {
833 let backend_node_id = idx_to_backend.get(&i).copied();
834
835 let has_cursor_pointer = elem
838 .get("hasCursorPointer")
839 .and_then(|v| v.as_bool())
840 .unwrap_or(false);
841 let has_on_click = elem
842 .get("hasOnClick")
843 .and_then(|v| v.as_bool())
844 .unwrap_or(false);
845 let has_tab_index = elem
846 .get("hasTabIndex")
847 .and_then(|v| v.as_bool())
848 .unwrap_or(false);
849 let is_editable = elem
850 .get("isEditable")
851 .and_then(|v| v.as_bool())
852 .unwrap_or(false);
853
854 let kind = if has_cursor_pointer || has_on_click {
855 "clickable"
856 } else if is_editable {
857 "editable"
858 } else {
859 "focusable"
860 };
861
862 let mut hints: Vec<String> = Vec::new();
863 if has_cursor_pointer {
864 hints.push("cursor:pointer".to_string());
865 }
866 if has_on_click {
867 hints.push("onclick".to_string());
868 }
869 if has_tab_index {
870 hints.push("tabindex".to_string());
871 }
872 if is_editable {
873 hints.push("contenteditable".to_string());
874 }
875
876 let text = elem
877 .get("text")
878 .and_then(|v| v.as_str())
879 .unwrap_or("")
880 .trim()
881 .to_string();
882
883 let hidden_input_kind = elem
884 .get("hiddenInputType")
885 .and_then(|v| v.as_str())
886 .and_then(HiddenInputKind::parse);
887 let hidden_input_checked = elem
888 .get("hiddenInputChecked")
889 .and_then(|v| v.as_str())
890 .map(|s| s.to_string());
891
892 if let Some(bid) = backend_node_id {
893 map.insert(
894 bid,
895 CursorElementInfo {
896 kind: kind.to_string(),
897 hints,
898 text,
899 hidden_input_kind,
900 hidden_input_checked,
901 },
902 );
903 }
904 }
905
906 Ok(map)
907}
908
909fn promote_hidden_inputs(
915 tree_nodes: &mut [TreeNode],
916 cursor_elements: &FxHashMap<i64, CursorElementInfo>,
917) {
918 for node in tree_nodes.iter_mut() {
919 if !matches!(node.role.as_str(), "LabelText" | "generic") {
920 continue;
921 }
922 let cursor_info = match node
923 .backend_node_id
924 .and_then(|bid| cursor_elements.get(&bid))
925 {
926 Some(info) => info,
927 None => continue,
928 };
929 if let Some(input_kind) = cursor_info.hidden_input_kind {
930 node.role = input_kind.as_role().to_string();
931 if node.name.is_empty() && !cursor_info.text.is_empty() {
932 node.name = cursor_info.text.clone();
933 }
934 if let Some(ref checked) = cursor_info.hidden_input_checked {
935 node.checked = Some(checked.clone());
936 }
937 }
938 }
939}
940
941fn build_tree(nodes: &[AXNode]) -> (Vec<TreeNode>, Vec<usize>) {
942 let mut tree_nodes: Vec<TreeNode> = Vec::with_capacity(nodes.len());
943 let mut id_to_idx: FxHashMap<String, usize> =
945 FxHashMap::with_capacity_and_hasher(nodes.len(), Default::default());
946
947 for (i, node) in nodes.iter().enumerate() {
948 let role = extract_ax_string(&node.role);
949 let name = extract_ax_string(&node.name);
950 let value_text = extract_ax_string_opt(&node.value);
951
952 let (level, checked, expanded, selected, disabled, required) =
953 extract_properties(&node.properties);
954
955 if (node.ignored.unwrap_or(false) && role != "RootWebArea") || role == "InlineTextBox" {
956 tree_nodes.push(TreeNode::empty());
957 id_to_idx.insert(node.node_id.clone(), i);
958 continue;
959 }
960
961 tree_nodes.push(TreeNode {
962 role,
963 name,
964 level,
965 checked,
966 expanded,
967 selected,
968 disabled,
969 required,
970 value_text,
971 backend_node_id: node.backend_d_o_m_node_id,
972 children: Vec::new(),
973 parent_idx: None,
974 has_ref: false,
975 ref_id: None,
976 depth: 0,
977 cursor_info: None,
978 url: None,
979 });
980 id_to_idx.insert(node.node_id.clone(), i);
981 }
982
983 for (i, node) in nodes.iter().enumerate() {
985 if let Some(ref child_ids) = node.child_ids {
986 for cid in child_ids {
987 if let Some(&child_idx) = id_to_idx.get(cid) {
988 tree_nodes[i].children.push(child_idx);
989 tree_nodes[child_idx].parent_idx = Some(i);
990 }
991 }
992 }
993 }
994
995 for i in 0..tree_nodes.len() {
997 if tree_nodes[i].role.is_empty() || tree_nodes[i].children.is_empty() {
998 continue;
999 }
1000
1001 let children_indices: Vec<usize> = tree_nodes[i].children.clone();
1002
1003 let mut start = 0;
1007 while start < children_indices.len() {
1008 if tree_nodes[children_indices[start]].role != "StaticText" {
1010 start += 1;
1011 continue;
1012 }
1013
1014 let mut end = start + 1;
1016 while end < children_indices.len()
1017 && tree_nodes[children_indices[end]].role == "StaticText"
1018 {
1019 end += 1;
1020 }
1021
1022 if end > start + 1 {
1024 let aggregated_name: String = (start..end)
1026 .map(|idx| tree_nodes[children_indices[idx]].name.clone())
1027 .collect();
1028 tree_nodes[children_indices[start]].name = aggregated_name;
1030 for j in (start + 1)..end {
1032 tree_nodes[children_indices[j]].clear();
1033 }
1034 }
1035 start = end;
1036 }
1037
1038 if children_indices.len() == 1
1040 && tree_nodes[children_indices[0]].role == "StaticText"
1041 && tree_nodes[i].name == tree_nodes[children_indices[0]].name
1042 {
1043 tree_nodes[children_indices[0]].clear();
1044 }
1045 }
1046
1047 let mut root_indices = Vec::new();
1049 let children_exist: Vec<bool> = nodes.iter().map(|_| false).collect();
1050 let mut is_child = children_exist;
1051 for node in &tree_nodes {
1052 for &child in &node.children {
1053 is_child[child] = true;
1054 }
1055 }
1056 for (i, &is_c) in is_child.iter().enumerate() {
1057 if !is_c {
1058 root_indices.push(i);
1059 }
1060 }
1061
1062 fn set_depth(nodes: &mut [TreeNode], idx: usize, depth: usize) {
1063 nodes[idx].depth = depth;
1064 let children: Vec<usize> = nodes[idx].children.clone();
1065 for child_idx in children {
1066 set_depth(nodes, child_idx, depth + 1);
1067 }
1068 }
1069
1070 for &root in &root_indices {
1071 set_depth(&mut tree_nodes, root, 0);
1072 }
1073
1074 (tree_nodes, root_indices)
1075}
1076
1077fn render_tree(
1078 nodes: &[TreeNode],
1079 idx: usize,
1080 indent: usize,
1081 output: &mut String,
1082 options: &SnapshotOptions,
1083) {
1084 let node = &nodes[idx];
1085
1086 if node.role.is_empty()
1088 || (node.role == "generic" && !node.has_ref && node.children.len() <= 1)
1089 || (node.role == "StaticText" && node.name.replace(INVISIBLE_CHARS, "").is_empty())
1090 {
1091 for &child in &node.children {
1093 render_tree(nodes, child, indent, output, options);
1094 }
1095 return;
1096 }
1097
1098 if let Some(max_depth) = options.depth {
1099 if indent > max_depth {
1100 return;
1101 }
1102 }
1103
1104 let role = &node.role;
1105
1106 if role == "RootWebArea" || role == "WebArea" {
1108 for &child in &node.children {
1109 render_tree(nodes, child, indent, output, options);
1110 }
1111 return;
1112 }
1113
1114 if options.interactive && !node.has_ref {
1115 for &child in &node.children {
1117 render_tree(nodes, child, indent, output, options);
1118 }
1119 return;
1120 }
1121
1122 let prefix = " ".repeat(indent);
1123 let mut line = format!("{}- {}", prefix, role);
1124
1125 let unescaped_display_name = if !node.name.is_empty() {
1127 &node.name
1128 } else if options.interactive {
1129 if let Some(ref ci) = node.cursor_info {
1130 &ci.text
1131 } else {
1132 &node.name
1133 }
1134 } else {
1135 &node.name
1136 };
1137 if !unescaped_display_name.is_empty() {
1138 if let Ok(display_name) = serde_json::to_string(&unescaped_display_name) {
1139 line.push_str(&format!(" {}", display_name.replace(INVISIBLE_CHARS, "")));
1140 }
1141 }
1142
1143 let mut attrs = Vec::new();
1145
1146 if let Some(level) = node.level {
1147 attrs.push(format!("level={}", level));
1148 }
1149 if let Some(ref checked) = node.checked {
1150 attrs.push(format!("checked={}", checked));
1151 }
1152 if let Some(expanded) = node.expanded {
1153 attrs.push(format!("expanded={}", expanded));
1154 }
1155 if let Some(selected) = node.selected {
1156 if selected {
1157 attrs.push("selected".to_string());
1158 }
1159 }
1160 if let Some(disabled) = node.disabled {
1161 if disabled {
1162 attrs.push("disabled".to_string());
1163 }
1164 }
1165 if let Some(required) = node.required {
1166 if required {
1167 attrs.push("required".to_string());
1168 }
1169 }
1170
1171 if let Some(ref ref_id) = node.ref_id {
1172 attrs.push(format!("ref={}", ref_id));
1173 }
1174
1175 if let Some(ref url) = node.url {
1176 attrs.push(format!("url={}", url));
1177 }
1178
1179 if !attrs.is_empty() {
1180 line.push_str(&format!(" [{}]", attrs.join(", ")));
1181 }
1182
1183 if let Some(ref cursor_info) = node.cursor_info {
1185 line.push_str(&format!(
1186 " {} [{}]",
1187 cursor_info.kind,
1188 cursor_info.hints.join(", ")
1189 ));
1190 }
1191
1192 if let Some(ref val) = node.value_text {
1194 if !val.is_empty() && val != &node.name {
1195 line.push_str(&format!(": {}", val));
1196 }
1197 }
1198
1199 output.push_str(&line);
1200 output.push('\n');
1201
1202 for &child in &node.children {
1203 render_tree(nodes, child, indent + 1, output, options);
1204 }
1205}
1206
1207fn compact_tree(tree: &str, interactive: bool) -> String {
1208 let lines: Vec<&str> = tree.lines().collect();
1209 if lines.is_empty() {
1210 return String::new();
1211 }
1212
1213 let mut keep = vec![false; lines.len()];
1214
1215 for (i, line) in lines.iter().enumerate() {
1216 if line.contains("ref=") || line.contains(": ") {
1217 keep[i] = true;
1218 let my_indent = count_indent(line);
1220 for j in (0..i).rev() {
1221 let ancestor_indent = count_indent(lines[j]);
1222 if ancestor_indent < my_indent {
1223 keep[j] = true;
1224 if ancestor_indent == 0 {
1225 break;
1226 }
1227 }
1228 }
1229 }
1230 }
1231
1232 let result: Vec<&str> = lines
1233 .iter()
1234 .enumerate()
1235 .filter(|(i, _)| keep[*i])
1236 .map(|(_, line)| *line)
1237 .collect();
1238
1239 let output = result.join("\n");
1240 if output.trim().is_empty() && interactive {
1241 return "(no interactive elements)".to_string();
1242 }
1243 output
1244}
1245
1246fn count_indent(line: &str) -> usize {
1247 let trimmed = line.trim_start();
1248 (line.len() - trimmed.len()) / 2
1249}
1250
1251fn extract_ax_string(value: &Option<AXValue>) -> String {
1252 match value {
1253 Some(v) => match &v.value {
1254 Some(Value::String(s)) => s.clone(),
1255 Some(Value::Number(n)) => n.to_string(),
1256 Some(Value::Bool(b)) => b.to_string(),
1257 _ => String::new(),
1258 },
1259 None => String::new(),
1260 }
1261}
1262
1263fn extract_ax_string_opt(value: &Option<AXValue>) -> Option<String> {
1264 match value {
1265 Some(v) => match &v.value {
1266 Some(Value::String(s)) if !s.is_empty() => Some(s.clone()),
1267 Some(Value::Number(n)) => Some(n.to_string()),
1268 _ => None,
1269 },
1270 None => None,
1271 }
1272}
1273
1274type NodeProperties = (
1275 Option<i64>, Option<String>, Option<bool>, Option<bool>, Option<bool>, Option<bool>, );
1282
1283fn extract_properties(props: &Option<Vec<AXProperty>>) -> NodeProperties {
1284 let mut level = None;
1285 let mut checked = None;
1286 let mut expanded = None;
1287 let mut selected = None;
1288 let mut disabled = None;
1289 let mut required = None;
1290
1291 if let Some(properties) = props {
1292 for prop in properties {
1293 match prop.name.as_str() {
1294 "level" => {
1295 level = prop.value.value.as_ref().and_then(|v| v.as_i64());
1296 }
1297 "checked" => {
1298 checked = prop.value.value.as_ref().map(|v| match v {
1299 Value::String(s) => s.clone(),
1300 Value::Bool(b) => b.to_string(),
1301 _ => "false".to_string(),
1302 });
1303 }
1304 "expanded" => {
1305 expanded = prop.value.value.as_ref().and_then(|v| v.as_bool());
1306 }
1307 "selected" => {
1308 selected = prop.value.value.as_ref().and_then(|v| v.as_bool());
1309 }
1310 "disabled" => {
1311 disabled = prop.value.value.as_ref().and_then(|v| v.as_bool());
1312 }
1313 "required" => {
1314 required = prop.value.value.as_ref().and_then(|v| v.as_bool());
1315 }
1316 _ => {}
1317 }
1318 }
1319 }
1320
1321 (level, checked, expanded, selected, disabled, required)
1322}
1323
1324#[cfg(test)]
1330fn build_dedup_set(ref_map: &RefMap) -> std::collections::HashSet<String> {
1331 ref_map
1332 .entries_sorted()
1333 .into_iter()
1334 .filter(|(_, entry)| !entry.name.is_empty())
1335 .map(|(_, entry)| entry.name.to_lowercase())
1336 .collect()
1337}
1338
1339fn collect_backend_node_ids(node: &Value, ids: &mut std::collections::HashSet<i64>) {
1342 if let Some(id) = node.get("backendNodeId").and_then(|v| v.as_i64()) {
1343 ids.insert(id);
1344 }
1345 if let Some(children) = node.get("children").and_then(|v| v.as_array()) {
1346 for child in children {
1347 collect_backend_node_ids(child, ids);
1348 }
1349 }
1350 if let Some(shadow) = node.get("shadowRoots").and_then(|v| v.as_array()) {
1352 for child in shadow {
1353 collect_backend_node_ids(child, ids);
1354 }
1355 }
1356 if let Some(doc) = node.get("contentDocument") {
1357 collect_backend_node_ids(doc, ids);
1358 }
1359}
1360
1361#[cfg(test)]
1362mod tests {
1363 use super::*;
1364
1365 #[test]
1366 fn test_interactive_roles() {
1367 assert!(INTERACTIVE_ROLES.contains(&"button"));
1368 assert!(INTERACTIVE_ROLES.contains(&"textbox"));
1369 assert!(!INTERACTIVE_ROLES.contains(&"heading"));
1370 }
1371
1372 #[test]
1373 fn test_content_roles() {
1374 assert!(CONTENT_ROLES.contains(&"heading"));
1375 assert!(!CONTENT_ROLES.contains(&"button"));
1376 }
1377
1378 #[test]
1379 fn test_compact_tree_basic() {
1380 let tree = "- navigation\n - link \"Home\" [ref=e1]\n - link \"About\" [ref=e2]\n- main\n - heading \"Title\"\n - paragraph\n - text: Hello\n";
1381 let result = compact_tree(tree, false);
1382 assert!(result.contains("[ref=e1]"));
1383 assert!(result.contains("[ref=e2]"));
1384 assert!(result.contains("Hello"));
1385 }
1386
1387 #[test]
1388 fn test_compact_tree_radio_checkbox() {
1389 let tree = "- form\n - radio \"Single unit\" [checked=false, ref=e1]\n - checkbox \"I agree\" [checked=false, ref=e2]\n - button \"Submit\" [ref=e3]\n";
1392 let result = compact_tree(tree, true);
1393 assert!(
1394 result.contains("radio \"Single unit\""),
1395 "radio should be kept"
1396 );
1397 assert!(
1398 result.contains("checkbox \"I agree\""),
1399 "checkbox should be kept"
1400 );
1401 assert!(
1402 result.contains("button \"Submit\""),
1403 "button should be kept"
1404 );
1405 }
1406
1407 #[test]
1408 fn test_compact_tree_empty_interactive() {
1409 let result = compact_tree("- generic\n", true);
1410 assert_eq!(result, "(no interactive elements)");
1411 }
1412
1413 #[test]
1414 fn test_count_indent() {
1415 assert_eq!(count_indent("- heading"), 0);
1416 assert_eq!(count_indent(" - link"), 1);
1417 assert_eq!(count_indent(" - text"), 2);
1418 }
1419
1420 #[test]
1421 fn test_role_name_tracker() {
1422 let mut tracker = RoleNameTracker::new();
1423 assert_eq!(tracker.track("button", "Submit", 0), 0);
1424 assert_eq!(tracker.track("button", "Submit", 1), 1);
1425 assert_eq!(tracker.track("button", "Cancel", 2), 0);
1426
1427 let dups = tracker.get_duplicates();
1428 assert!(dups.contains_key("button:Submit"));
1429 assert!(!dups.contains_key("button:Cancel"));
1430 }
1431
1432 #[test]
1437 fn test_dedup_set_from_ref_map_names() {
1438 let mut ref_map = RefMap::new();
1439 ref_map.add("e1".to_string(), Some(1), "link", "Example Link", None);
1440 ref_map.add("e2".to_string(), Some(2), "button", "Submit", None);
1441
1442 let set = build_dedup_set(&ref_map);
1443 assert!(set.contains("example link"));
1444 assert!(set.contains("submit"));
1445 assert!(!set.contains("other text"));
1446 }
1447
1448 #[test]
1449 fn test_dedup_set_case_insensitive() {
1450 let mut ref_map = RefMap::new();
1451 ref_map.add("e1".to_string(), Some(1), "button", "Submit Form", None);
1452
1453 let set = build_dedup_set(&ref_map);
1454 assert!(set.contains("submit form"));
1455 assert!(!set.contains("Submit Form"));
1456 }
1457
1458 #[test]
1459 fn test_dedup_set_empty_inputs() {
1460 let ref_map = RefMap::new();
1461 let set = build_dedup_set(&ref_map);
1462 assert!(set.is_empty());
1463 }
1464
1465 #[test]
1466 fn test_dedup_set_skips_empty_names() {
1467 let mut ref_map = RefMap::new();
1468 ref_map.add("e1".to_string(), Some(1), "generic", "", None);
1469 ref_map.add("e2".to_string(), Some(2), "button", "OK", None);
1470
1471 let set = build_dedup_set(&ref_map);
1472 assert_eq!(set.len(), 1);
1473 assert!(set.contains("ok"));
1474 }
1475
1476 #[test]
1483 fn test_cross_origin_iframe_uses_dedicated_session() {
1484 let parent_session = "parent-session";
1485 let iframe_frame_id = "cross-origin-iframe-frame";
1486 let iframe_session = "cross-origin-iframe-session";
1487
1488 let mut iframe_sessions = FxHashMap::default();
1489 iframe_sessions.insert(iframe_frame_id.to_string(), iframe_session.to_string());
1490
1491 let (params, session) =
1492 resolve_ax_session(Some(iframe_frame_id), parent_session, &iframe_sessions);
1493
1494 assert_eq!(session, iframe_session);
1495 assert_eq!(params, serde_json::json!({}));
1496 }
1497
1498 #[test]
1499 fn test_same_origin_iframe_uses_parent_session_with_frame_id() {
1500 let parent_session = "parent-session";
1501 let iframe_frame_id = "same-origin-iframe-frame";
1502 let iframe_sessions = FxHashMap::default();
1503
1504 let (params, session) =
1505 resolve_ax_session(Some(iframe_frame_id), parent_session, &iframe_sessions);
1506
1507 assert_eq!(session, parent_session);
1508 assert_eq!(params, serde_json::json!({ "frameId": iframe_frame_id }));
1509 }
1510
1511 #[test]
1512 fn test_main_frame_uses_parent_session() {
1513 let parent_session = "parent-session";
1514 let iframe_sessions = FxHashMap::default();
1515
1516 let (params, session) = resolve_ax_session(None, parent_session, &iframe_sessions);
1517
1518 assert_eq!(session, parent_session);
1519 assert_eq!(params, serde_json::json!({}));
1520 }
1521
1522 fn make_node(role: &str, name: &str, backend_node_id: Option<i64>) -> TreeNode {
1527 let mut node = TreeNode::empty();
1528 node.role = role.to_string();
1529 node.name = name.to_string();
1530 node.backend_node_id = backend_node_id;
1531 node
1532 }
1533
1534 fn make_cursor_info(
1535 hidden_kind: Option<HiddenInputKind>,
1536 hidden_checked: Option<&str>,
1537 text: &str,
1538 ) -> CursorElementInfo {
1539 CursorElementInfo {
1540 kind: "clickable".to_string(),
1541 hints: vec!["cursor:pointer".to_string()],
1542 text: text.to_string(),
1543 hidden_input_kind: hidden_kind,
1544 hidden_input_checked: hidden_checked.map(|s| s.to_string()),
1545 }
1546 }
1547
1548 #[test]
1549 fn test_promote_label_with_hidden_radio() {
1550 let mut nodes = vec![
1551 make_node("LabelText", "", Some(1)),
1552 make_node("LabelText", "", Some(2)),
1553 make_node("button", "Submit", Some(3)),
1554 ];
1555 let mut cursor_elements = FxHashMap::default();
1556 cursor_elements.insert(
1557 1,
1558 make_cursor_info(Some(HiddenInputKind::Radio), Some("false"), "Option A"),
1559 );
1560 cursor_elements.insert(
1561 2,
1562 make_cursor_info(Some(HiddenInputKind::Radio), Some("true"), "Option B"),
1563 );
1564
1565 promote_hidden_inputs(&mut nodes, &cursor_elements);
1566
1567 assert_eq!(nodes[0].role, "radio");
1568 assert_eq!(nodes[0].name, "Option A");
1569 assert_eq!(nodes[0].checked, Some("false".to_string()));
1570 assert_eq!(nodes[1].role, "radio");
1571 assert_eq!(nodes[1].name, "Option B");
1572 assert_eq!(nodes[1].checked, Some("true".to_string()));
1573 assert_eq!(nodes[2].role, "button");
1575 }
1576
1577 #[test]
1578 fn test_promote_preserves_existing_name() {
1579 let mut nodes = vec![make_node("LabelText", "AX Name", Some(1))];
1581 let mut cursor_elements = FxHashMap::default();
1582 cursor_elements.insert(
1583 1,
1584 make_cursor_info(Some(HiddenInputKind::Radio), Some("false"), "Text Content"),
1585 );
1586
1587 promote_hidden_inputs(&mut nodes, &cursor_elements);
1588
1589 assert_eq!(nodes[0].role, "radio");
1590 assert_eq!(nodes[0].name, "AX Name"); }
1592
1593 #[test]
1594 fn test_promote_skips_without_hidden_input() {
1595 let mut nodes = vec![make_node("LabelText", "", Some(1))];
1597 let mut cursor_elements = FxHashMap::default();
1598 cursor_elements.insert(1, make_cursor_info(None, None, "Click me"));
1599
1600 promote_hidden_inputs(&mut nodes, &cursor_elements);
1601
1602 assert_eq!(nodes[0].role, "LabelText"); }
1604}