1use crate::robot_assertions::{Bounds, SemanticElementLike};
7use cranpose::SemanticElement;
8use std::collections::HashMap;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::Arc;
11use std::time::Duration;
12
13impl SemanticElementLike for SemanticElement {
16 fn text(&self) -> Option<&str> {
17 self.text.as_deref()
18 }
19
20 fn role(&self) -> &str {
21 &self.role
22 }
23
24 fn clickable(&self) -> bool {
25 self.clickable
26 }
27
28 fn bounds(&self) -> Bounds {
29 Bounds {
30 x: self.bounds.x,
31 y: self.bounds.y,
32 width: self.bounds.width,
33 height: self.bounds.height,
34 }
35 }
36
37 fn children(&self) -> &[Self] {
38 &self.children
39 }
40}
41
42pub fn find_text(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
49 if let Some(ref t) = elem.text {
50 if t.contains(text) {
51 return Some((
52 elem.bounds.x,
53 elem.bounds.y,
54 elem.bounds.width,
55 elem.bounds.height,
56 ));
57 }
58 }
59 for child in &elem.children {
60 if let Some(pos) = find_text(child, text) {
61 return Some(pos);
62 }
63 }
64 None
65}
66
67pub fn find_text_exact(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
71 if let Some(ref t) = elem.text {
72 if t == text {
73 return Some((
74 elem.bounds.x,
75 elem.bounds.y,
76 elem.bounds.width,
77 elem.bounds.height,
78 ));
79 }
80 }
81 for child in &elem.children {
82 if let Some(pos) = find_text_exact(child, text) {
83 return Some(pos);
84 }
85 }
86 None
87}
88
89pub fn find_text_center(elem: &SemanticElement, text: &str) -> Option<(f32, f32)> {
93 find_text(elem, text).map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
94}
95
96pub fn has_text(elem: &SemanticElement, text: &str) -> bool {
98 has_text_by(elem, text, text_contains)
99}
100
101pub fn find_button(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
106 find_button_by(elem, text, text_contains)
107}
108
109pub fn find_button_center(elem: &SemanticElement, text: &str) -> Option<(f32, f32)> {
112 find_button(elem, text).map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
113}
114
115pub fn find_in_semantics<F>(robot: &cranpose::Robot, finder: F) -> Option<(f32, f32, f32, f32)>
123where
124 F: Fn(&SemanticElement) -> Option<(f32, f32, f32, f32)>,
125{
126 match robot.get_semantics() {
127 Ok(semantics) => {
128 for root in semantics.iter() {
129 if let Some(result) = finder(root) {
130 return Some(result);
131 }
132 }
133 None
134 }
135 Err(e) => {
136 eprintln!(" ✗ Failed to get semantics: {}", e);
137 None
138 }
139 }
140}
141
142pub fn find_text_in_semantics(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32, f32, f32)> {
147 match robot.find_text_bounds(text) {
148 Ok(bounds) => bounds,
149 Err(e) => {
150 eprintln!(" ✗ Failed to query text semantics: {}", e);
151 None
152 }
153 }
154}
155
156pub fn find_text_by_prefix(
159 elem: &SemanticElement,
160 prefix: &str,
161) -> Option<(f32, f32, f32, f32, String)> {
162 if let Some(ref t) = elem.text {
163 if t.starts_with(prefix) {
164 return Some((
165 elem.bounds.x,
166 elem.bounds.y,
167 elem.bounds.width,
168 elem.bounds.height,
169 t.clone(),
170 ));
171 }
172 }
173 for child in &elem.children {
174 if let Some(result) = find_text_by_prefix(child, prefix) {
175 return Some(result);
176 }
177 }
178 None
179}
180
181pub fn find_text_by_prefix_in_semantics(
185 robot: &cranpose::Robot,
186 prefix: &str,
187) -> Option<(f32, f32, f32, f32, String)> {
188 match robot.find_text_by_prefix(prefix) {
189 Ok(bounds) => bounds,
190 Err(e) => {
191 eprintln!(" ✗ Failed to query text prefix semantics: {}", e);
192 None
193 }
194 }
195}
196
197pub fn find_button_in_semantics(
200 robot: &cranpose::Robot,
201 text: &str,
202) -> Option<(f32, f32, f32, f32)> {
203 find_button_in_semantics_by(robot, text, TextMatchMode::Contains)
204}
205
206pub fn find_button_exact_in_semantics(
208 robot: &cranpose::Robot,
209 text: &str,
210) -> Option<(f32, f32, f32, f32)> {
211 find_button_in_semantics_by(robot, text, TextMatchMode::Exact)
212}
213
214fn find_button_in_semantics_by(
215 robot: &cranpose::Robot,
216 text: &str,
217 match_mode: TextMatchMode,
218) -> Option<(f32, f32, f32, f32)> {
219 let text_owned = text.to_string();
220 let mut bounds = find_button_bounds_for_mode(robot, &text_owned, match_mode);
221
222 let Some(root) = root_bounds(robot) else {
223 return bounds;
224 };
225
226 if let Some(current) = bounds {
227 if is_fully_visible(current, root) {
228 return bounds;
229 }
230 } else {
231 return None;
232 }
233
234 for _ in 0..8 {
235 let Some(current) = bounds else {
236 break;
237 };
238
239 let Some((axis, dir)) = overflow_axis_direction(current, root) else {
240 break;
241 };
242
243 let semantics = match robot.get_semantics() {
244 Ok(semantics) => semantics,
245 Err(e) => {
246 eprintln!(" ✗ Failed to get semantics: {}", e);
247 break;
248 }
249 };
250
251 let Some((sx, sy, sw, sh)) = find_scroll_anchor(&semantics, current, root, axis) else {
252 break;
253 };
254
255 let start_x = sx + sw / 2.0;
256 let start_y = sy + sh / 2.0;
257 let (end_x, end_y) = match axis {
258 TabAxis::Horizontal => (start_x + dir * SCROLL_STEP, start_y),
259 TabAxis::Vertical => (start_x, start_y + dir * SCROLL_STEP),
260 };
261 let _ = robot.drag(start_x, start_y, end_x, end_y);
262 std::thread::sleep(Duration::from_millis(SCROLL_SETTLE_MS));
263 let _ = robot.wait_for_idle();
264
265 bounds = find_button_bounds_for_mode(robot, &text_owned, match_mode);
266 if let Some(current) = bounds {
267 if is_fully_visible(current, root) {
268 break;
269 }
270 }
271 }
272
273 bounds
274}
275
276pub fn find_by_text_recursive(elements: &[SemanticElement], text: &str) -> Option<SemanticElement> {
279 for elem in elements {
280 if let Some(ref elem_text) = elem.text {
281 if elem_text.contains(text) {
282 return Some(elem.clone());
283 }
284 }
285 if let Some(found) = find_by_text_recursive(&elem.children, text) {
286 return Some(found);
287 }
288 }
289 None
290}
291
292pub fn find_clickables_in_range(
295 elements: &[SemanticElement],
296 min_y: f32,
297 max_y: f32,
298) -> Vec<(String, f32, f32)> {
299 fn search(elem: &SemanticElement, tabs: &mut Vec<(String, f32, f32)>, min_y: f32, max_y: f32) {
300 if elem.role == "Layout" && elem.clickable && elem.bounds.y > min_y && elem.bounds.y < max_y
301 {
302 let label = elem
303 .children
304 .iter()
305 .find(|child| child.role == "Text")
306 .and_then(|text_elem| text_elem.text.clone())
307 .unwrap_or_else(|| "Unknown".to_string());
308 tabs.push((label, elem.bounds.x, elem.bounds.y));
309 }
310 for child in &elem.children {
311 search(child, tabs, min_y, max_y);
312 }
313 }
314
315 let mut tabs = Vec::new();
316 for elem in elements {
317 search(elem, &mut tabs, min_y, max_y);
318 }
319 tabs.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
320 tabs
321}
322
323pub fn find_element_by_text_exact<'a>(
324 elements: &'a [SemanticElement],
325 text: &str,
326) -> Option<&'a SemanticElement> {
327 for elem in elements {
328 if elem.text.as_deref() == Some(text) {
329 return Some(elem);
330 }
331 if let Some(found) = find_element_by_text_exact(&elem.children, text) {
332 return Some(found);
333 }
334 }
335 None
336}
337
338pub fn find_bounds_by_text(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32, f32, f32)> {
339 let semantics = robot.get_semantics().ok()?;
340 let elem = find_element_by_text_exact(&semantics, text)?;
341 Some((
342 elem.bounds.x,
343 elem.bounds.y,
344 elem.bounds.width,
345 elem.bounds.height,
346 ))
347}
348
349pub fn visible_bounds_in_viewport(
350 robot: &cranpose::Robot,
351 bounds: (f32, f32, f32, f32),
352 padding: f32,
353) -> Option<(f32, f32, f32, f32)> {
354 let semantics = robot.get_semantics().ok()?;
355 let mut viewport = None;
356 for elem in semantics.iter() {
357 let elem_bounds = (
358 elem.bounds.x,
359 elem.bounds.y,
360 elem.bounds.width,
361 elem.bounds.height,
362 );
363 viewport = Some(match viewport {
364 Some(existing) => union_bounds(existing, Some(elem_bounds)),
365 None => elem_bounds,
366 });
367 }
368 let (viewport_x, viewport_y, viewport_width, viewport_height) = viewport?;
369 let min_x = viewport_x + padding;
370 let min_y = viewport_y + padding;
371 let max_x = viewport_x + viewport_width - padding;
372 let max_y = viewport_y + viewport_height - padding;
373
374 let left = bounds.0.max(min_x);
375 let top = bounds.1.max(min_y);
376 let right = (bounds.0 + bounds.2).min(max_x);
377 let bottom = (bounds.1 + bounds.3).min(max_y);
378
379 if right <= left || bottom <= top {
380 None
381 } else {
382 Some((left, top, right - left, bottom - top))
383 }
384}
385
386pub fn find_center_by_text(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32)> {
387 let (x, y, w, h) = find_bounds_by_text(robot, text)?;
388 Some((x + w / 2.0, y + h / 2.0))
389}
390
391pub fn find_in_subtree_by_text<'a>(
392 elem: &'a SemanticElement,
393 text: &str,
394) -> Option<&'a SemanticElement> {
395 if elem.text.as_deref() == Some(text) {
396 return Some(elem);
397 }
398 for child in &elem.children {
399 if let Some(found) = find_in_subtree_by_text(child, text) {
400 return Some(found);
401 }
402 }
403 None
404}
405
406pub fn print_semantics_with_bounds(elements: &[SemanticElement], indent: usize) {
407 for elem in elements {
408 let prefix = " ".repeat(indent);
409 let text = elem.text.as_deref().unwrap_or("");
410 println!(
411 "{}role={} text=\"{}\" bounds=({:.1},{:.1},{:.1},{:.1}){}",
412 prefix,
413 elem.role,
414 text,
415 elem.bounds.x,
416 elem.bounds.y,
417 elem.bounds.width,
418 elem.bounds.height,
419 if elem.clickable { " [CLICKABLE]" } else { "" }
420 );
421 print_semantics_with_bounds(&elem.children, indent + 1);
422 }
423}
424
425pub fn union_bounds(
426 base: (f32, f32, f32, f32),
427 other: Option<(f32, f32, f32, f32)>,
428) -> (f32, f32, f32, f32) {
429 let (x, y, w, h) = base;
430 let mut min_x = x;
431 let mut min_y = y;
432 let mut max_x = x + w;
433 let mut max_y = y + h;
434 if let Some((ox, oy, ow, oh)) = other {
435 min_x = min_x.min(ox);
436 min_y = min_y.min(oy);
437 max_x = max_x.max(ox + ow);
438 max_y = max_y.max(oy + oh);
439 }
440 (min_x, min_y, max_x - min_x, max_y - min_y)
441}
442
443pub fn count_text_in_tree(elements: &[SemanticElement], text: &str) -> usize {
444 let mut count = 0;
445 for elem in elements {
446 if elem.text.as_deref() == Some(text) {
447 count += 1;
448 }
449 count += count_text_in_tree(&elem.children, text);
450 }
451 count
452}
453
454pub fn collect_by_text_exact<'a>(
455 elements: &'a [SemanticElement],
456 text: &str,
457 results: &mut Vec<&'a SemanticElement>,
458) {
459 for elem in elements {
460 if elem.text.as_deref() == Some(text) {
461 results.push(elem);
462 }
463 collect_by_text_exact(&elem.children, text, results);
464 }
465}
466
467pub fn collect_text_prefix_counts(
468 elements: &[SemanticElement],
469 prefix: &str,
470 counts: &mut HashMap<String, usize>,
471) {
472 for elem in elements {
473 if let Some(text) = elem.text.as_deref() {
474 if text.starts_with(prefix) {
475 *counts.entry(text.to_string()).or_insert(0) += 1;
476 }
477 }
478 collect_text_prefix_counts(&elem.children, prefix, counts);
479 }
480}
481
482pub fn exit_with_timeout(robot: &cranpose::Robot, timeout: Duration) {
483 let done = Arc::new(AtomicBool::new(false));
484 let done_thread = Arc::clone(&done);
485 std::thread::spawn(move || {
486 std::thread::sleep(timeout);
487 if !done_thread.load(Ordering::Relaxed) {
488 std::process::exit(0);
489 }
490 });
491
492 let _ = robot.exit();
493 done.store(true, Ordering::Relaxed);
494}
495
496#[derive(Clone, Copy, Debug, PartialEq, Eq)]
497pub enum TabAxis {
498 Horizontal,
499 Vertical,
500}
501
502type RectBounds = (f32, f32, f32, f32);
503type LabeledRect = (String, RectBounds);
504
505pub fn collect_tab_bounds(robot: &cranpose::Robot, labels: &[&str]) -> Vec<LabeledRect> {
506 let mut tabs = Vec::new();
507 for label in labels {
508 if let Some(bounds) = find_in_semantics(robot, |elem| find_button(elem, label)) {
509 tabs.push(((*label).to_string(), bounds));
510 }
511 }
512 tabs
513}
514
515pub fn bounds_span(bounds: &[LabeledRect]) -> Option<RectBounds> {
516 let mut iter = bounds.iter();
517 let (_, (x, y, w, h)) = iter.next()?;
518 let mut min_x = *x;
519 let mut min_y = *y;
520 let mut max_x = x + w;
521 let mut max_y = y + h;
522 for (_, (bx, by, bw, bh)) in iter {
523 min_x = min_x.min(*bx);
524 min_y = min_y.min(*by);
525 max_x = max_x.max(*bx + *bw);
526 max_y = max_y.max(*by + *bh);
527 }
528 Some((min_x, min_y, max_x, max_y))
529}
530
531pub fn detect_tab_axis(bounds: &[LabeledRect]) -> Option<TabAxis> {
532 let (min_x, min_y, max_x, max_y) = bounds_span(bounds)?;
533 let span_x = max_x - min_x;
534 let span_y = max_y - min_y;
535 if span_x >= span_y {
536 Some(TabAxis::Horizontal)
537 } else {
538 Some(TabAxis::Vertical)
539 }
540}
541
542pub fn root_bounds(robot: &cranpose::Robot) -> Option<RectBounds> {
543 let semantics = robot.get_semantics().ok()?;
544 let root = semantics.first()?;
545 Some((
546 root.bounds.x,
547 root.bounds.y,
548 root.bounds.width,
549 root.bounds.height,
550 ))
551}
552
553const VISIBILITY_PADDING: f32 = 4.0;
554const SCROLL_STEP: f32 = 240.0;
555const SCROLL_SETTLE_MS: u64 = 140;
556type TextMatcher = fn(&str, &str) -> bool;
557
558#[derive(Clone, Copy, Debug, PartialEq, Eq)]
559enum TextMatchMode {
560 Contains,
561 Exact,
562}
563
564fn text_contains(actual: &str, needle: &str) -> bool {
565 actual.contains(needle)
566}
567
568#[cfg(test)]
569fn text_equals(actual: &str, needle: &str) -> bool {
570 actual == needle
571}
572
573fn find_button_bounds_for_mode(
574 robot: &cranpose::Robot,
575 text: &str,
576 match_mode: TextMatchMode,
577) -> Option<(f32, f32, f32, f32)> {
578 let result = match match_mode {
579 TextMatchMode::Contains => robot.find_button_bounds(text),
580 TextMatchMode::Exact => robot.find_button_bounds_exact(text),
581 };
582 match result {
583 Ok(bounds) => bounds,
584 Err(e) => {
585 eprintln!(" ✗ Failed to query button semantics: {}", e);
586 None
587 }
588 }
589}
590
591fn has_text_by(elem: &SemanticElement, text: &str, matcher: TextMatcher) -> bool {
592 if elem
593 .text
594 .as_deref()
595 .is_some_and(|actual| matcher(actual, text))
596 {
597 return true;
598 }
599 elem.children
600 .iter()
601 .any(|child| has_text_by(child, text, matcher))
602}
603
604fn find_button_by(
605 elem: &SemanticElement,
606 text: &str,
607 matcher: TextMatcher,
608) -> Option<(f32, f32, f32, f32)> {
609 if elem.clickable && has_text_by(elem, text, matcher) {
610 return Some((
611 elem.bounds.x,
612 elem.bounds.y,
613 elem.bounds.width,
614 elem.bounds.height,
615 ));
616 }
617 elem.children
618 .iter()
619 .find_map(|child| find_button_by(child, text, matcher))
620}
621
622fn is_axis_visible(bounds: RectBounds, root: RectBounds, axis: TabAxis) -> bool {
623 let (x, y, w, h) = bounds;
624 let (rx, ry, rw, rh) = root;
625 match axis {
626 TabAxis::Horizontal => {
627 x >= rx + VISIBILITY_PADDING && x + w <= rx + rw - VISIBILITY_PADDING
628 }
629 TabAxis::Vertical => y >= ry + VISIBILITY_PADDING && y + h <= ry + rh - VISIBILITY_PADDING,
630 }
631}
632
633fn is_fully_visible(bounds: RectBounds, root: RectBounds) -> bool {
634 is_axis_visible(bounds, root, TabAxis::Horizontal)
635 && is_axis_visible(bounds, root, TabAxis::Vertical)
636}
637
638fn overflow_axis_direction(bounds: RectBounds, root: RectBounds) -> Option<(TabAxis, f32)> {
639 let (x, y, w, h) = bounds;
640 let (rx, ry, rw, rh) = root;
641
642 let overflow_left = (rx + VISIBILITY_PADDING - x).max(0.0);
643 let overflow_right = (x + w - (rx + rw - VISIBILITY_PADDING)).max(0.0);
644 let overflow_top = (ry + VISIBILITY_PADDING - y).max(0.0);
645 let overflow_bottom = (y + h - (ry + rh - VISIBILITY_PADDING)).max(0.0);
646
647 let horizontal_overflow = overflow_left.max(overflow_right);
648 let vertical_overflow = overflow_top.max(overflow_bottom);
649
650 if horizontal_overflow <= 0.0 && vertical_overflow <= 0.0 {
651 return None;
652 }
653
654 if horizontal_overflow >= vertical_overflow {
655 if overflow_left > 0.0 {
656 Some((TabAxis::Horizontal, 1.0))
657 } else {
658 Some((TabAxis::Horizontal, -1.0))
659 }
660 } else if overflow_top > 0.0 {
661 Some((TabAxis::Vertical, 1.0))
662 } else {
663 Some((TabAxis::Vertical, -1.0))
664 }
665}
666
667fn intersects_root(bounds: RectBounds, root: RectBounds) -> bool {
668 let (x, y, w, h) = bounds;
669 let (rx, ry, rw, rh) = root;
670 let left = x.max(rx);
671 let top = y.max(ry);
672 let right = (x + w).min(rx + rw);
673 let bottom = (y + h).min(ry + rh);
674 right > left && bottom > top
675}
676
677fn overlap_len(a_start: f32, a_len: f32, b_start: f32, b_len: f32) -> f32 {
678 let a_end = a_start + a_len;
679 let b_end = b_start + b_len;
680 (a_end.min(b_end) - a_start.max(b_start)).max(0.0)
681}
682
683fn cross_axis_overlap(bounds: RectBounds, target: RectBounds, axis: TabAxis) -> f32 {
684 match axis {
685 TabAxis::Horizontal => overlap_len(bounds.1, bounds.3, target.1, target.3),
686 TabAxis::Vertical => overlap_len(bounds.0, bounds.2, target.0, target.2),
687 }
688}
689
690fn primary_axis_distance(bounds: RectBounds, target: RectBounds, axis: TabAxis) -> f32 {
691 let center = match axis {
692 TabAxis::Horizontal => bounds.0 + bounds.2 / 2.0,
693 TabAxis::Vertical => bounds.1 + bounds.3 / 2.0,
694 };
695 let target_center = match axis {
696 TabAxis::Horizontal => target.0 + target.2 / 2.0,
697 TabAxis::Vertical => target.1 + target.3 / 2.0,
698 };
699 (center - target_center).abs()
700}
701
702fn find_scroll_anchor(
703 elements: &[SemanticElement],
704 target: RectBounds,
705 root: RectBounds,
706 axis: TabAxis,
707) -> Option<RectBounds> {
708 let mut best: Option<(RectBounds, f32, f32)> = None;
709
710 for elem in elements {
711 if elem.clickable {
712 let bounds = (
713 elem.bounds.x,
714 elem.bounds.y,
715 elem.bounds.width,
716 elem.bounds.height,
717 );
718 if is_fully_visible(bounds, root) && intersects_root(bounds, root) {
719 let overlap = cross_axis_overlap(bounds, target, axis);
720 if overlap > 0.0 {
721 let distance = primary_axis_distance(bounds, target, axis);
722 match best {
723 None => best = Some((bounds, overlap, distance)),
724 Some((_, best_overlap, best_distance)) => {
725 let is_better_overlap = overlap > best_overlap + f32::EPSILON;
726 let is_better_distance = (overlap - best_overlap).abs() <= f32::EPSILON
727 && distance < best_distance;
728 if is_better_overlap || is_better_distance {
729 best = Some((bounds, overlap, distance));
730 }
731 }
732 }
733 }
734 }
735 }
736
737 if let Some(found) = find_scroll_anchor(&elem.children, target, root, axis) {
738 let overlap = cross_axis_overlap(found, target, axis);
739 let distance = primary_axis_distance(found, target, axis);
740 match best {
741 None => best = Some((found, overlap, distance)),
742 Some((_, best_overlap, best_distance)) => {
743 let is_better_overlap = overlap > best_overlap + f32::EPSILON;
744 let is_better_distance =
745 (overlap - best_overlap).abs() <= f32::EPSILON && distance < best_distance;
746 if is_better_overlap || is_better_distance {
747 best = Some((found, overlap, distance));
748 }
749 }
750 }
751 }
752 }
753
754 best.map(|(bounds, _, _)| bounds)
755}
756
757pub fn capture_screenshot(robot: &cranpose::Robot) -> Option<cranpose::RobotScreenshot> {
759 robot.screenshot().ok()
760}
761
762fn screenshot_logical_width(screenshot: &cranpose::RobotScreenshot) -> f32 {
763 if screenshot.logical_width.is_finite() && screenshot.logical_width > 0.0 {
764 screenshot.logical_width
765 } else {
766 screenshot.width.max(1) as f32
767 }
768}
769
770fn screenshot_logical_height(screenshot: &cranpose::RobotScreenshot) -> f32 {
771 if screenshot.logical_height.is_finite() && screenshot.logical_height > 0.0 {
772 screenshot.logical_height
773 } else {
774 screenshot.height.max(1) as f32
775 }
776}
777
778fn screenshot_scale_x(screenshot: &cranpose::RobotScreenshot) -> f32 {
779 screenshot.width.max(1) as f32 / screenshot_logical_width(screenshot)
780}
781
782fn screenshot_scale_y(screenshot: &cranpose::RobotScreenshot) -> f32 {
783 screenshot.height.max(1) as f32 / screenshot_logical_height(screenshot)
784}
785
786fn logical_to_screenshot_x(screenshot: &cranpose::RobotScreenshot, x: f32) -> f32 {
787 x * screenshot_scale_x(screenshot)
788}
789
790fn logical_to_screenshot_y(screenshot: &cranpose::RobotScreenshot, y: f32) -> f32 {
791 y * screenshot_scale_y(screenshot)
792}
793
794pub fn screenshot_logical_size(screenshot: &cranpose::RobotScreenshot) -> (f32, f32) {
795 (
796 screenshot_logical_width(screenshot),
797 screenshot_logical_height(screenshot),
798 )
799}
800
801pub fn screenshot_pixel(screenshot: &cranpose::RobotScreenshot, x: u32, y: u32) -> Option<[u8; 4]> {
803 if x >= screenshot.width || y >= screenshot.height {
804 return None;
805 }
806 let index = ((y * screenshot.width + x) * 4) as usize;
807 Some([
808 screenshot.pixels[index],
809 screenshot.pixels[index + 1],
810 screenshot.pixels[index + 2],
811 screenshot.pixels[index + 3],
812 ])
813}
814
815pub fn sample_screenshot_pixel_logical(
816 screenshot: &cranpose::RobotScreenshot,
817 x: f32,
818 y: f32,
819) -> Option<[u8; 4]> {
820 let logical_width = screenshot_logical_width(screenshot);
821 let logical_height = screenshot_logical_height(screenshot);
822 if x < 0.0 || y < 0.0 || x > logical_width || y > logical_height {
823 return None;
824 }
825
826 Some(sample_screenshot_pixel_bilinear(
827 screenshot,
828 logical_to_screenshot_x(screenshot, x),
829 logical_to_screenshot_y(screenshot, y),
830 ))
831}
832
833pub fn logical_region_to_pixel_bounds(
834 screenshot: &cranpose::RobotScreenshot,
835 region: (f32, f32, f32, f32),
836) -> Option<(u32, u32, u32, u32)> {
837 if region.2 <= 0.0 || region.3 <= 0.0 || screenshot.width == 0 || screenshot.height == 0 {
838 return None;
839 }
840
841 let left = logical_to_screenshot_x(screenshot, region.0.max(0.0))
842 .floor()
843 .max(0.0) as u32;
844 let top = logical_to_screenshot_y(screenshot, region.1.max(0.0))
845 .floor()
846 .max(0.0) as u32;
847 let right = logical_to_screenshot_x(
848 screenshot,
849 (region.0 + region.2).min(screenshot_logical_width(screenshot)),
850 )
851 .ceil()
852 .min(screenshot.width as f32) as u32;
853 let bottom = logical_to_screenshot_y(
854 screenshot,
855 (region.1 + region.3).min(screenshot_logical_height(screenshot)),
856 )
857 .ceil()
858 .min(screenshot.height as f32) as u32;
859
860 if right <= left || bottom <= top {
861 return None;
862 }
863
864 Some((left, top, right, bottom))
865}
866
867pub fn crop_screenshot(
869 screenshot: &cranpose::RobotScreenshot,
870 x: u32,
871 y: u32,
872 width: u32,
873 height: u32,
874) -> Option<cranpose::RobotScreenshot> {
875 if width == 0 || height == 0 {
876 return None;
877 }
878 let end_x = x.checked_add(width)?;
879 let end_y = y.checked_add(height)?;
880 if end_x > screenshot.width || end_y > screenshot.height {
881 return None;
882 }
883
884 let mut pixels = vec![0u8; (width * height * 4) as usize];
885 for row in 0..height {
886 let src_start = (((y + row) * screenshot.width + x) * 4) as usize;
887 let src_end = src_start + (width * 4) as usize;
888 let dst_start = (row * width * 4) as usize;
889 let dst_end = dst_start + (width * 4) as usize;
890 pixels[dst_start..dst_end].copy_from_slice(&screenshot.pixels[src_start..src_end]);
891 }
892
893 Some(cranpose::RobotScreenshot {
894 width,
895 height,
896 logical_width: width as f32 / screenshot_scale_x(screenshot),
897 logical_height: height as f32 / screenshot_scale_y(screenshot),
898 pixels,
899 })
900}
901
902pub fn crop_screenshot_logical(
903 screenshot: &cranpose::RobotScreenshot,
904 x: f32,
905 y: f32,
906 width: f32,
907 height: f32,
908) -> Option<cranpose::RobotScreenshot> {
909 let (left, top, right, bottom) =
910 logical_region_to_pixel_bounds(screenshot, (x, y, width, height))?;
911 crop_screenshot(screenshot, left, top, right - left, bottom - top)
912}
913
914#[derive(Debug, Clone, PartialEq, Eq)]
915pub struct ScreenshotPixelDifference {
916 pub x: u32,
917 pub y: u32,
918 pub before: [u8; 4],
919 pub after: [u8; 4],
920 pub difference: u32,
921}
922
923#[derive(Debug, Clone, PartialEq, Eq)]
924pub struct ScreenshotDifferenceStats {
925 pub differing_pixels: usize,
926 pub max_difference: u32,
927 pub first_difference: Option<ScreenshotPixelDifference>,
928}
929
930pub fn normalize_screenshot_region(
935 screenshot: &cranpose::RobotScreenshot,
936 region: (f32, f32, f32, f32),
937 output_width: u32,
938 output_height: u32,
939) -> Option<cranpose::RobotScreenshot> {
940 if output_width == 0
941 || output_height == 0
942 || region.2 <= 0.0
943 || region.3 <= 0.0
944 || screenshot.width == 0
945 || screenshot.height == 0
946 {
947 return None;
948 }
949
950 let mut pixels = Vec::with_capacity((output_width * output_height * 4) as usize);
951 for y in 0..output_height {
952 for x in 0..output_width {
953 let sample_x = region.0 + ((x as f32 + 0.5) * region.2 / output_width as f32);
954 let sample_y = region.1 + ((y as f32 + 0.5) * region.3 / output_height as f32);
955 pixels.extend_from_slice(&sample_screenshot_pixel_logical(
956 screenshot, sample_x, sample_y,
957 )?);
958 }
959 }
960
961 Some(cranpose::RobotScreenshot {
962 width: output_width,
963 height: output_height,
964 logical_width: output_width as f32,
965 logical_height: output_height as f32,
966 pixels,
967 })
968}
969
970pub fn screenshot_difference_stats(
973 before: &cranpose::RobotScreenshot,
974 after: &cranpose::RobotScreenshot,
975 difference_tolerance: u32,
976) -> Option<ScreenshotDifferenceStats> {
977 if before.width != after.width || before.height != after.height {
978 return None;
979 }
980
981 let mut differing_pixels = 0usize;
982 let mut max_difference = 0u32;
983 let mut first_difference = None;
984
985 for y in 0..before.height {
986 for x in 0..before.width {
987 let before_pixel =
988 screenshot_pixel(before, x, y).expect("screenshot bounds checked by loop");
989 let after_pixel =
990 screenshot_pixel(after, x, y).expect("screenshot bounds checked by loop");
991 let difference = pixel_difference(before_pixel, after_pixel);
992 if difference > difference_tolerance {
993 differing_pixels += 1;
994 max_difference = max_difference.max(difference);
995 if first_difference.is_none() {
996 first_difference = Some(ScreenshotPixelDifference {
997 x,
998 y,
999 before: before_pixel,
1000 after: after_pixel,
1001 difference,
1002 });
1003 }
1004 }
1005 }
1006 }
1007
1008 Some(ScreenshotDifferenceStats {
1009 differing_pixels,
1010 max_difference,
1011 first_difference,
1012 })
1013}
1014
1015pub fn changed_pixel_count(
1018 before: &cranpose::RobotScreenshot,
1019 after: &cranpose::RobotScreenshot,
1020 channel_threshold: u8,
1021) -> usize {
1022 if before.width != after.width || before.height != after.height {
1023 return usize::MAX;
1024 }
1025
1026 before
1027 .pixels
1028 .chunks_exact(4)
1029 .zip(after.pixels.chunks_exact(4))
1030 .filter(|(a, b)| {
1031 a[0].abs_diff(b[0]) > channel_threshold
1032 || a[1].abs_diff(b[1]) > channel_threshold
1033 || a[2].abs_diff(b[2]) > channel_threshold
1034 || a[3].abs_diff(b[3]) > channel_threshold
1035 })
1036 .count()
1037}
1038
1039pub fn changed_pixel_count_in_region(
1041 before: &cranpose::RobotScreenshot,
1042 after: &cranpose::RobotScreenshot,
1043 region: (f32, f32, f32, f32),
1044 channel_threshold: u8,
1045) -> usize {
1046 if before.width != after.width
1047 || before.height != after.height
1048 || (screenshot_scale_x(before) - screenshot_scale_x(after)).abs() > f32::EPSILON
1049 || (screenshot_scale_y(before) - screenshot_scale_y(after)).abs() > f32::EPSILON
1050 {
1051 return usize::MAX;
1052 }
1053
1054 let Some((left, top, right, bottom)) = logical_region_to_pixel_bounds(before, region) else {
1055 return 0;
1056 };
1057
1058 let width = before.width as usize;
1059 let mut changed = 0usize;
1060 for y in top..bottom {
1061 for x in left..right {
1062 let idx = ((y as usize) * width + x as usize) * 4;
1063 if before.pixels[idx].abs_diff(after.pixels[idx]) > channel_threshold
1064 || before.pixels[idx + 1].abs_diff(after.pixels[idx + 1]) > channel_threshold
1065 || before.pixels[idx + 2].abs_diff(after.pixels[idx + 2]) > channel_threshold
1066 || before.pixels[idx + 3].abs_diff(after.pixels[idx + 3]) > channel_threshold
1067 {
1068 changed += 1;
1069 }
1070 }
1071 }
1072
1073 changed
1074}
1075
1076fn sample_screenshot_pixel_bilinear(
1077 screenshot: &cranpose::RobotScreenshot,
1078 x: f32,
1079 y: f32,
1080) -> [u8; 4] {
1081 let max_x = screenshot.width.saturating_sub(1) as f32;
1082 let max_y = screenshot.height.saturating_sub(1) as f32;
1083 let source_x = (x - 0.5).clamp(0.0, max_x);
1084 let source_y = (y - 0.5).clamp(0.0, max_y);
1085 let x0 = source_x.floor() as u32;
1086 let y0 = source_y.floor() as u32;
1087 let x1 = (x0 + 1).min(screenshot.width.saturating_sub(1));
1088 let y1 = (y0 + 1).min(screenshot.height.saturating_sub(1));
1089 let tx = source_x - x0 as f32;
1090 let ty = source_y - y0 as f32;
1091 let top_left = screenshot_pixel(screenshot, x0, y0).expect("bilinear x0/y0 in bounds");
1092 let top_right = screenshot_pixel(screenshot, x1, y0).expect("bilinear x1/y0 in bounds");
1093 let bottom_left = screenshot_pixel(screenshot, x0, y1).expect("bilinear x0/y1 in bounds");
1094 let bottom_right = screenshot_pixel(screenshot, x1, y1).expect("bilinear x1/y1 in bounds");
1095
1096 let lerp_channel = |index: usize| {
1097 let top = top_left[index] as f32 * (1.0 - tx) + top_right[index] as f32 * tx;
1098 let bottom = bottom_left[index] as f32 * (1.0 - tx) + bottom_right[index] as f32 * tx;
1099 (top * (1.0 - ty) + bottom * ty).round() as u8
1100 };
1101
1102 [
1103 lerp_channel(0),
1104 lerp_channel(1),
1105 lerp_channel(2),
1106 lerp_channel(3),
1107 ]
1108}
1109
1110fn pixel_difference(before: [u8; 4], after: [u8; 4]) -> u32 {
1111 before
1112 .into_iter()
1113 .zip(after)
1114 .map(|(lhs, rhs)| lhs.abs_diff(rhs) as u32)
1115 .sum()
1116}
1117
1118pub fn parse_slider_value(text: &str) -> Option<f32> {
1120 text.split_once(':')
1121 .and_then(|(_, value)| value.trim().parse::<f32>().ok())
1122}
1123
1124pub fn scroll_down(robot: &cranpose::Robot, center_x: f32, from_y: f32, to_y: f32) {
1126 let _ = robot.drag(center_x, from_y, center_x, to_y);
1127 std::thread::sleep(std::time::Duration::from_millis(180));
1128 let _ = robot.wait_for_idle();
1129}
1130
1131pub fn scroll_up(robot: &cranpose::Robot, center_x: f32, from_y: f32, to_y: f32) {
1133 let _ = robot.drag(center_x, from_y, center_x, to_y);
1134 std::thread::sleep(std::time::Duration::from_millis(180));
1135 let _ = robot.wait_for_idle();
1136}
1137
1138pub fn y_is_visible(robot: &cranpose::Robot, y: f32) -> bool {
1141 let Some((_, root_y, _, root_h)) = root_bounds(robot) else {
1142 return true;
1143 };
1144 let top = root_y + 28.0;
1145 let bottom = root_y + root_h - 28.0;
1146 y >= top && y <= bottom
1147}
1148
1149#[derive(Clone, Copy, Debug)]
1151pub struct ScrollConfig {
1152 pub center_x: f32,
1153 pub down_from_y: f32,
1154 pub down_to_y: f32,
1155 pub up_from_y: f32,
1156 pub up_to_y: f32,
1157}
1158
1159pub fn scroll_prefix_into_view(
1162 robot: &cranpose::Robot,
1163 prefix: &str,
1164 max_attempts: usize,
1165 cfg: ScrollConfig,
1166) -> Option<(f32, f32, f32, f32, String)> {
1167 for attempt in 0..max_attempts {
1168 if let Some(bounds) = find_text_by_prefix_in_semantics(robot, prefix) {
1169 let center_y = bounds.1 + bounds.3 * 0.5;
1170 if y_is_visible(robot, center_y) {
1171 return Some(bounds);
1172 }
1173 let Some((_, root_y, _, root_h)) = root_bounds(robot) else {
1174 return Some(bounds);
1175 };
1176 let viewport_mid = root_y + root_h * 0.5;
1177 if center_y > viewport_mid {
1178 scroll_down(robot, cfg.center_x, cfg.down_from_y, cfg.down_to_y);
1179 } else {
1180 scroll_up(robot, cfg.center_x, cfg.up_from_y, cfg.up_to_y);
1181 }
1182 } else {
1183 if attempt % 2 == 0 {
1185 scroll_down(robot, cfg.center_x, cfg.down_from_y, cfg.down_to_y);
1186 } else {
1187 scroll_up(robot, cfg.center_x, cfg.up_from_y, cfg.up_to_y);
1188 }
1189 }
1190 }
1191 None
1192}
1193
1194pub fn set_slider_fraction(
1196 robot: &cranpose::Robot,
1197 prefix: &str,
1198 fraction: f32,
1199 slider_width: f32,
1200 slider_touch_offset_y: f32,
1201 cfg: ScrollConfig,
1202) -> Option<f32> {
1203 let (x, y, _w, h, _) = scroll_prefix_into_view(robot, prefix, 18, cfg)?;
1204 let slider_y = y + h + slider_touch_offset_y;
1205 let left_x = x + 2.0;
1206 let target_x = x + slider_width * fraction.clamp(0.0, 1.0);
1207 let _ = robot.drag(left_x, slider_y, target_x, slider_y);
1208 std::thread::sleep(std::time::Duration::from_millis(120));
1209 let _ = robot.wait_for_idle();
1210 find_text_by_prefix_in_semantics(robot, prefix)
1211 .and_then(|(_, _, _, _, t)| parse_slider_value(&t))
1212}
1213
1214#[cfg(test)]
1215mod tests {
1216 use super::*;
1217 use cranpose::{RobotScreenshot, SemanticRect};
1218
1219 fn semantic_element(
1220 role: &str,
1221 text: Option<&str>,
1222 clickable: bool,
1223 bounds: RectBounds,
1224 children: Vec<SemanticElement>,
1225 ) -> SemanticElement {
1226 SemanticElement {
1227 role: role.to_string(),
1228 text: text.map(ToString::to_string),
1229 clickable,
1230 bounds: SemanticRect {
1231 x: bounds.0,
1232 y: bounds.1,
1233 width: bounds.2,
1234 height: bounds.3,
1235 },
1236 children,
1237 }
1238 }
1239
1240 #[test]
1241 fn overflow_axis_direction_detects_horizontal_overflow() {
1242 let root = (0.0, 0.0, 300.0, 200.0);
1243 let target = (320.0, 20.0, 80.0, 30.0);
1244 assert_eq!(
1245 overflow_axis_direction(target, root),
1246 Some((TabAxis::Horizontal, -1.0))
1247 );
1248 }
1249
1250 #[test]
1251 fn overflow_axis_direction_detects_vertical_overflow() {
1252 let root = (0.0, 0.0, 300.0, 200.0);
1253 let target = (20.0, -60.0, 80.0, 30.0);
1254 assert_eq!(
1255 overflow_axis_direction(target, root),
1256 Some((TabAxis::Vertical, 1.0))
1257 );
1258 }
1259
1260 #[test]
1261 fn find_scroll_anchor_prefers_cross_axis_overlap() {
1262 let root = (0.0, 0.0, 300.0, 200.0);
1263 let target = (320.0, 22.0, 80.0, 28.0);
1264
1265 let same_row = semantic_element(
1266 "Layout",
1267 Some("same-row"),
1268 true,
1269 (120.0, 20.0, 80.0, 30.0),
1270 vec![],
1271 );
1272 let other_row = semantic_element(
1273 "Layout",
1274 Some("other-row"),
1275 true,
1276 (120.0, 130.0, 80.0, 30.0),
1277 vec![],
1278 );
1279 let root_elem = semantic_element("Layout", None, false, root, vec![same_row, other_row]);
1280
1281 let anchor = find_scroll_anchor(&[root_elem], target, root, TabAxis::Horizontal)
1282 .expect("expected anchor");
1283
1284 assert_eq!(anchor, (120.0, 20.0, 80.0, 30.0));
1285 }
1286
1287 #[test]
1288 fn find_button_exact_requires_full_text_match() {
1289 let exact_button = semantic_element(
1290 "Button",
1291 None,
1292 true,
1293 (10.0, 20.0, 90.0, 28.0),
1294 vec![
1295 semantic_element(
1296 "Text",
1297 Some("Text"),
1298 false,
1299 (14.0, 24.0, 30.0, 20.0),
1300 vec![],
1301 ),
1302 semantic_element(
1303 "Text",
1304 Some("Text Input"),
1305 false,
1306 (46.0, 24.0, 48.0, 20.0),
1307 vec![],
1308 ),
1309 ],
1310 );
1311 let partial_only_button = semantic_element(
1312 "Button",
1313 None,
1314 true,
1315 (120.0, 20.0, 90.0, 28.0),
1316 vec![semantic_element(
1317 "Text",
1318 Some("Text Input"),
1319 false,
1320 (124.0, 24.0, 48.0, 20.0),
1321 vec![],
1322 )],
1323 );
1324
1325 assert_eq!(
1326 find_button_by(&exact_button, "Text", text_equals),
1327 Some((10.0, 20.0, 90.0, 28.0))
1328 );
1329 assert_eq!(
1330 find_button_by(&partial_only_button, "Text", text_equals),
1331 None
1332 );
1333 }
1334
1335 #[test]
1336 fn screenshot_pixel_reads_expected_value() {
1337 let screenshot = RobotScreenshot {
1338 width: 2,
1339 height: 1,
1340 logical_width: 2.0,
1341 logical_height: 1.0,
1342 pixels: vec![1, 2, 3, 4, 5, 6, 7, 8],
1343 };
1344 assert_eq!(screenshot_pixel(&screenshot, 1, 0), Some([5, 6, 7, 8]));
1345 }
1346
1347 #[test]
1348 fn crop_screenshot_extracts_region() {
1349 let screenshot = RobotScreenshot {
1350 width: 3,
1351 height: 2,
1352 logical_width: 3.0,
1353 logical_height: 2.0,
1354 pixels: vec![
1355 1, 2, 3, 255, 4, 5, 6, 255, 7, 8, 9, 255, 10, 11, 12, 255, 13, 14, 15, 255, 16, 17,
1356 18, 255,
1357 ],
1358 };
1359 let cropped = crop_screenshot(&screenshot, 1, 0, 2, 2).expect("crop");
1360 assert_eq!(cropped.width, 2);
1361 assert_eq!(cropped.height, 2);
1362 assert_eq!(cropped.logical_width, 2.0);
1363 assert_eq!(cropped.logical_height, 2.0);
1364 assert_eq!(
1365 cropped.pixels,
1366 vec![4, 5, 6, 255, 7, 8, 9, 255, 13, 14, 15, 255, 16, 17, 18, 255]
1367 );
1368 }
1369
1370 #[test]
1371 fn normalize_screenshot_region_preserves_pixel_grid_at_native_size() {
1372 let screenshot = RobotScreenshot {
1373 width: 2,
1374 height: 2,
1375 logical_width: 2.0,
1376 logical_height: 2.0,
1377 pixels: vec![1, 2, 3, 255, 4, 5, 6, 255, 7, 8, 9, 255, 10, 11, 12, 255],
1378 };
1379
1380 let normalized =
1381 normalize_screenshot_region(&screenshot, (0.0, 0.0, 2.0, 2.0), 2, 2).expect("norm");
1382
1383 assert_eq!(normalized.width, screenshot.width);
1384 assert_eq!(normalized.height, screenshot.height);
1385 assert_eq!(normalized.logical_width, screenshot.logical_width);
1386 assert_eq!(normalized.logical_height, screenshot.logical_height);
1387 assert_eq!(normalized.pixels, screenshot.pixels);
1388 }
1389
1390 #[test]
1391 fn changed_pixel_count_in_region_uses_logical_coordinates() {
1392 let before = RobotScreenshot {
1393 width: 4,
1394 height: 4,
1395 logical_width: 2.0,
1396 logical_height: 2.0,
1397 pixels: vec![
1398 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255,
1399 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255,
1400 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255,
1401 ],
1402 };
1403 let mut after = before.clone();
1404 for y in 2..4 {
1405 for x in 2..4 {
1406 let idx = ((y * after.width + x) * 4) as usize;
1407 after.pixels[idx] = 255;
1408 }
1409 }
1410
1411 assert_eq!(
1412 changed_pixel_count_in_region(&before, &after, (1.0, 1.0, 1.0, 1.0), 1),
1413 4
1414 );
1415 }
1416
1417 #[test]
1418 fn sample_screenshot_pixel_logical_maps_scaled_capture() {
1419 let screenshot = RobotScreenshot {
1420 width: 4,
1421 height: 4,
1422 logical_width: 2.0,
1423 logical_height: 2.0,
1424 pixels: vec![
1425 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255,
1426 7, 0, 0, 255, 8, 0, 0, 255, 9, 0, 0, 255, 10, 0, 0, 255, 11, 0, 0, 255, 12, 0, 0,
1427 255, 13, 0, 0, 255, 14, 0, 0, 255, 15, 0, 0, 255, 16, 0, 0, 255,
1428 ],
1429 };
1430
1431 assert_eq!(
1432 sample_screenshot_pixel_logical(&screenshot, 1.25, 1.25),
1433 Some([11, 0, 0, 255])
1434 );
1435 }
1436
1437 #[test]
1438 fn screenshot_difference_stats_reports_first_difference() {
1439 let before = RobotScreenshot {
1440 width: 2,
1441 height: 1,
1442 logical_width: 2.0,
1443 logical_height: 1.0,
1444 pixels: vec![10, 20, 30, 255, 1, 2, 3, 255],
1445 };
1446 let after = RobotScreenshot {
1447 width: 2,
1448 height: 1,
1449 logical_width: 2.0,
1450 logical_height: 1.0,
1451 pixels: vec![10, 20, 30, 255, 4, 8, 3, 200],
1452 };
1453
1454 let stats = screenshot_difference_stats(&before, &after, 3).expect("stats");
1455
1456 assert_eq!(stats.differing_pixels, 1);
1457 assert_eq!(stats.max_difference, 64);
1458 assert_eq!(
1459 stats.first_difference,
1460 Some(ScreenshotPixelDifference {
1461 x: 1,
1462 y: 0,
1463 before: [1, 2, 3, 255],
1464 after: [4, 8, 3, 200],
1465 difference: 64,
1466 })
1467 );
1468 }
1469}