1use crate::robot_assertions::{Bounds, SemanticElementLike};
7use cranpose::SemanticElement;
8use std::collections::HashMap;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::Arc;
11use std::time::Duration;
12
13impl SemanticElementLike for SemanticElement {
16 fn text(&self) -> Option<&str> {
17 self.text.as_deref()
18 }
19
20 fn role(&self) -> &str {
21 &self.role
22 }
23
24 fn clickable(&self) -> bool {
25 self.clickable
26 }
27
28 fn bounds(&self) -> Bounds {
29 Bounds {
30 x: self.bounds.x,
31 y: self.bounds.y,
32 width: self.bounds.width,
33 height: self.bounds.height,
34 }
35 }
36
37 fn children(&self) -> &[Self] {
38 &self.children
39 }
40}
41
42pub fn find_text(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
49 if let Some(ref t) = elem.text {
50 if t.contains(text) {
51 return Some((
52 elem.bounds.x,
53 elem.bounds.y,
54 elem.bounds.width,
55 elem.bounds.height,
56 ));
57 }
58 }
59 for child in &elem.children {
60 if let Some(pos) = find_text(child, text) {
61 return Some(pos);
62 }
63 }
64 None
65}
66
67pub fn find_text_exact(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
71 if let Some(ref t) = elem.text {
72 if t == text {
73 return Some((
74 elem.bounds.x,
75 elem.bounds.y,
76 elem.bounds.width,
77 elem.bounds.height,
78 ));
79 }
80 }
81 for child in &elem.children {
82 if let Some(pos) = find_text_exact(child, text) {
83 return Some(pos);
84 }
85 }
86 None
87}
88
89pub fn find_text_center(elem: &SemanticElement, text: &str) -> Option<(f32, f32)> {
93 find_text(elem, text).map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
94}
95
96pub fn has_text(elem: &SemanticElement, text: &str) -> bool {
98 has_text_by(elem, text, text_contains)
99}
100
101pub fn find_button(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
106 find_button_by(elem, text, text_contains)
107}
108
109pub fn find_button_center(elem: &SemanticElement, text: &str) -> Option<(f32, f32)> {
112 find_button(elem, text).map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
113}
114
115pub fn find_in_semantics<F>(robot: &cranpose::Robot, finder: F) -> Option<(f32, f32, f32, f32)>
123where
124 F: Fn(&SemanticElement) -> Option<(f32, f32, f32, f32)>,
125{
126 match robot.get_semantics() {
127 Ok(semantics) => {
128 for root in semantics.iter() {
129 if let Some(result) = finder(root) {
130 return Some(result);
131 }
132 }
133 None
134 }
135 Err(e) => {
136 eprintln!(" ✗ Failed to get semantics: {}", e);
137 None
138 }
139 }
140}
141
142pub fn find_text_in_semantics(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32, f32, f32)> {
147 let text = text.to_string();
148 find_in_semantics(robot, |elem| find_text(elem, &text))
149}
150
151pub fn find_text_by_prefix(
154 elem: &SemanticElement,
155 prefix: &str,
156) -> Option<(f32, f32, f32, f32, String)> {
157 if let Some(ref t) = elem.text {
158 if t.starts_with(prefix) {
159 return Some((
160 elem.bounds.x,
161 elem.bounds.y,
162 elem.bounds.width,
163 elem.bounds.height,
164 t.clone(),
165 ));
166 }
167 }
168 for child in &elem.children {
169 if let Some(result) = find_text_by_prefix(child, prefix) {
170 return Some(result);
171 }
172 }
173 None
174}
175
176pub fn find_text_by_prefix_in_semantics(
180 robot: &cranpose::Robot,
181 prefix: &str,
182) -> Option<(f32, f32, f32, f32, String)> {
183 let prefix = prefix.to_string();
184 match robot.get_semantics() {
185 Ok(semantics) => {
186 for root in semantics.iter() {
187 if let Some(result) = find_text_by_prefix(root, &prefix) {
188 return Some(result);
189 }
190 }
191 None
192 }
193 Err(e) => {
194 eprintln!(" ✗ Failed to get semantics: {}", e);
195 None
196 }
197 }
198}
199
200pub fn find_button_in_semantics(
203 robot: &cranpose::Robot,
204 text: &str,
205) -> Option<(f32, f32, f32, f32)> {
206 find_button_in_semantics_by(robot, text, text_contains)
207}
208
209pub fn find_button_exact_in_semantics(
211 robot: &cranpose::Robot,
212 text: &str,
213) -> Option<(f32, f32, f32, f32)> {
214 find_button_in_semantics_by(robot, text, text_equals)
215}
216
217fn find_button_in_semantics_by(
218 robot: &cranpose::Robot,
219 text: &str,
220 matcher: TextMatcher,
221) -> Option<(f32, f32, f32, f32)> {
222 let text_owned = text.to_string();
223 let mut bounds = find_in_semantics(robot, |elem| find_button_by(elem, &text_owned, matcher));
224
225 let Some(root) = root_bounds(robot) else {
226 return bounds;
227 };
228
229 if let Some(current) = bounds {
230 if is_fully_visible(current, root) {
231 return bounds;
232 }
233 } else {
234 return None;
235 }
236
237 for _ in 0..8 {
238 let Some(current) = bounds else {
239 break;
240 };
241
242 let Some((axis, dir)) = overflow_axis_direction(current, root) else {
243 break;
244 };
245
246 let semantics = match robot.get_semantics() {
247 Ok(semantics) => semantics,
248 Err(e) => {
249 eprintln!(" ✗ Failed to get semantics: {}", e);
250 break;
251 }
252 };
253
254 let Some((sx, sy, sw, sh)) = find_scroll_anchor(&semantics, current, root, axis) else {
255 break;
256 };
257
258 let start_x = sx + sw / 2.0;
259 let start_y = sy + sh / 2.0;
260 let (end_x, end_y) = match axis {
261 TabAxis::Horizontal => (start_x + dir * SCROLL_STEP, start_y),
262 TabAxis::Vertical => (start_x, start_y + dir * SCROLL_STEP),
263 };
264 let _ = robot.drag(start_x, start_y, end_x, end_y);
265 std::thread::sleep(Duration::from_millis(SCROLL_SETTLE_MS));
266 let _ = robot.wait_for_idle();
267
268 bounds = find_in_semantics(robot, |elem| find_button_by(elem, &text_owned, matcher));
269 if let Some(current) = bounds {
270 if is_fully_visible(current, root) {
271 break;
272 }
273 }
274 }
275
276 bounds
277}
278
279pub fn find_by_text_recursive(elements: &[SemanticElement], text: &str) -> Option<SemanticElement> {
282 for elem in elements {
283 if let Some(ref elem_text) = elem.text {
284 if elem_text.contains(text) {
285 return Some(elem.clone());
286 }
287 }
288 if let Some(found) = find_by_text_recursive(&elem.children, text) {
289 return Some(found);
290 }
291 }
292 None
293}
294
295pub fn find_clickables_in_range(
298 elements: &[SemanticElement],
299 min_y: f32,
300 max_y: f32,
301) -> Vec<(String, f32, f32)> {
302 fn search(elem: &SemanticElement, tabs: &mut Vec<(String, f32, f32)>, min_y: f32, max_y: f32) {
303 if elem.role == "Layout" && elem.clickable && elem.bounds.y > min_y && elem.bounds.y < max_y
304 {
305 let label = elem
306 .children
307 .iter()
308 .find(|child| child.role == "Text")
309 .and_then(|text_elem| text_elem.text.clone())
310 .unwrap_or_else(|| "Unknown".to_string());
311 tabs.push((label, elem.bounds.x, elem.bounds.y));
312 }
313 for child in &elem.children {
314 search(child, tabs, min_y, max_y);
315 }
316 }
317
318 let mut tabs = Vec::new();
319 for elem in elements {
320 search(elem, &mut tabs, min_y, max_y);
321 }
322 tabs.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
323 tabs
324}
325
326pub fn find_element_by_text_exact<'a>(
327 elements: &'a [SemanticElement],
328 text: &str,
329) -> Option<&'a SemanticElement> {
330 for elem in elements {
331 if elem.text.as_deref() == Some(text) {
332 return Some(elem);
333 }
334 if let Some(found) = find_element_by_text_exact(&elem.children, text) {
335 return Some(found);
336 }
337 }
338 None
339}
340
341pub fn find_bounds_by_text(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32, f32, f32)> {
342 let semantics = robot.get_semantics().ok()?;
343 let elem = find_element_by_text_exact(&semantics, text)?;
344 Some((
345 elem.bounds.x,
346 elem.bounds.y,
347 elem.bounds.width,
348 elem.bounds.height,
349 ))
350}
351
352pub fn visible_bounds_in_viewport(
353 robot: &cranpose::Robot,
354 bounds: (f32, f32, f32, f32),
355 padding: f32,
356) -> Option<(f32, f32, f32, f32)> {
357 let semantics = robot.get_semantics().ok()?;
358 let mut viewport = None;
359 for elem in semantics.iter() {
360 let elem_bounds = (
361 elem.bounds.x,
362 elem.bounds.y,
363 elem.bounds.width,
364 elem.bounds.height,
365 );
366 viewport = Some(match viewport {
367 Some(existing) => union_bounds(existing, Some(elem_bounds)),
368 None => elem_bounds,
369 });
370 }
371 let (viewport_x, viewport_y, viewport_width, viewport_height) = viewport?;
372 let min_x = viewport_x + padding;
373 let min_y = viewport_y + padding;
374 let max_x = viewport_x + viewport_width - padding;
375 let max_y = viewport_y + viewport_height - padding;
376
377 let left = bounds.0.max(min_x);
378 let top = bounds.1.max(min_y);
379 let right = (bounds.0 + bounds.2).min(max_x);
380 let bottom = (bounds.1 + bounds.3).min(max_y);
381
382 if right <= left || bottom <= top {
383 None
384 } else {
385 Some((left, top, right - left, bottom - top))
386 }
387}
388
389pub fn find_center_by_text(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32)> {
390 let (x, y, w, h) = find_bounds_by_text(robot, text)?;
391 Some((x + w / 2.0, y + h / 2.0))
392}
393
394pub fn find_in_subtree_by_text<'a>(
395 elem: &'a SemanticElement,
396 text: &str,
397) -> Option<&'a SemanticElement> {
398 if elem.text.as_deref() == Some(text) {
399 return Some(elem);
400 }
401 for child in &elem.children {
402 if let Some(found) = find_in_subtree_by_text(child, text) {
403 return Some(found);
404 }
405 }
406 None
407}
408
409pub fn print_semantics_with_bounds(elements: &[SemanticElement], indent: usize) {
410 for elem in elements {
411 let prefix = " ".repeat(indent);
412 let text = elem.text.as_deref().unwrap_or("");
413 println!(
414 "{}role={} text=\"{}\" bounds=({:.1},{:.1},{:.1},{:.1}){}",
415 prefix,
416 elem.role,
417 text,
418 elem.bounds.x,
419 elem.bounds.y,
420 elem.bounds.width,
421 elem.bounds.height,
422 if elem.clickable { " [CLICKABLE]" } else { "" }
423 );
424 print_semantics_with_bounds(&elem.children, indent + 1);
425 }
426}
427
428pub fn union_bounds(
429 base: (f32, f32, f32, f32),
430 other: Option<(f32, f32, f32, f32)>,
431) -> (f32, f32, f32, f32) {
432 let (x, y, w, h) = base;
433 let mut min_x = x;
434 let mut min_y = y;
435 let mut max_x = x + w;
436 let mut max_y = y + h;
437 if let Some((ox, oy, ow, oh)) = other {
438 min_x = min_x.min(ox);
439 min_y = min_y.min(oy);
440 max_x = max_x.max(ox + ow);
441 max_y = max_y.max(oy + oh);
442 }
443 (min_x, min_y, max_x - min_x, max_y - min_y)
444}
445
446pub fn count_text_in_tree(elements: &[SemanticElement], text: &str) -> usize {
447 let mut count = 0;
448 for elem in elements {
449 if elem.text.as_deref() == Some(text) {
450 count += 1;
451 }
452 count += count_text_in_tree(&elem.children, text);
453 }
454 count
455}
456
457pub fn collect_by_text_exact<'a>(
458 elements: &'a [SemanticElement],
459 text: &str,
460 results: &mut Vec<&'a SemanticElement>,
461) {
462 for elem in elements {
463 if elem.text.as_deref() == Some(text) {
464 results.push(elem);
465 }
466 collect_by_text_exact(&elem.children, text, results);
467 }
468}
469
470pub fn collect_text_prefix_counts(
471 elements: &[SemanticElement],
472 prefix: &str,
473 counts: &mut HashMap<String, usize>,
474) {
475 for elem in elements {
476 if let Some(text) = elem.text.as_deref() {
477 if text.starts_with(prefix) {
478 *counts.entry(text.to_string()).or_insert(0) += 1;
479 }
480 }
481 collect_text_prefix_counts(&elem.children, prefix, counts);
482 }
483}
484
485pub fn exit_with_timeout(robot: &cranpose::Robot, timeout: Duration) {
486 let done = Arc::new(AtomicBool::new(false));
487 let done_thread = Arc::clone(&done);
488 std::thread::spawn(move || {
489 std::thread::sleep(timeout);
490 if !done_thread.load(Ordering::Relaxed) {
491 std::process::exit(0);
492 }
493 });
494
495 let _ = robot.exit();
496 done.store(true, Ordering::Relaxed);
497}
498
499#[derive(Clone, Copy, Debug, PartialEq, Eq)]
500pub enum TabAxis {
501 Horizontal,
502 Vertical,
503}
504
505type RectBounds = (f32, f32, f32, f32);
506type LabeledRect = (String, RectBounds);
507
508pub fn collect_tab_bounds(robot: &cranpose::Robot, labels: &[&str]) -> Vec<LabeledRect> {
509 let mut tabs = Vec::new();
510 for label in labels {
511 if let Some(bounds) = find_in_semantics(robot, |elem| find_button(elem, label)) {
512 tabs.push(((*label).to_string(), bounds));
513 }
514 }
515 tabs
516}
517
518pub fn bounds_span(bounds: &[LabeledRect]) -> Option<RectBounds> {
519 let mut iter = bounds.iter();
520 let (_, (x, y, w, h)) = iter.next()?;
521 let mut min_x = *x;
522 let mut min_y = *y;
523 let mut max_x = x + w;
524 let mut max_y = y + h;
525 for (_, (bx, by, bw, bh)) in iter {
526 min_x = min_x.min(*bx);
527 min_y = min_y.min(*by);
528 max_x = max_x.max(*bx + *bw);
529 max_y = max_y.max(*by + *bh);
530 }
531 Some((min_x, min_y, max_x, max_y))
532}
533
534pub fn detect_tab_axis(bounds: &[LabeledRect]) -> Option<TabAxis> {
535 let (min_x, min_y, max_x, max_y) = bounds_span(bounds)?;
536 let span_x = max_x - min_x;
537 let span_y = max_y - min_y;
538 if span_x >= span_y {
539 Some(TabAxis::Horizontal)
540 } else {
541 Some(TabAxis::Vertical)
542 }
543}
544
545pub fn root_bounds(robot: &cranpose::Robot) -> Option<RectBounds> {
546 let semantics = robot.get_semantics().ok()?;
547 let root = semantics.first()?;
548 Some((
549 root.bounds.x,
550 root.bounds.y,
551 root.bounds.width,
552 root.bounds.height,
553 ))
554}
555
556const VISIBILITY_PADDING: f32 = 4.0;
557const SCROLL_STEP: f32 = 240.0;
558const SCROLL_SETTLE_MS: u64 = 140;
559type TextMatcher = fn(&str, &str) -> bool;
560
561fn text_contains(actual: &str, needle: &str) -> bool {
562 actual.contains(needle)
563}
564
565fn text_equals(actual: &str, needle: &str) -> bool {
566 actual == needle
567}
568
569fn has_text_by(elem: &SemanticElement, text: &str, matcher: TextMatcher) -> bool {
570 if elem
571 .text
572 .as_deref()
573 .is_some_and(|actual| matcher(actual, text))
574 {
575 return true;
576 }
577 elem.children
578 .iter()
579 .any(|child| has_text_by(child, text, matcher))
580}
581
582fn find_button_by(
583 elem: &SemanticElement,
584 text: &str,
585 matcher: TextMatcher,
586) -> Option<(f32, f32, f32, f32)> {
587 if elem.clickable && has_text_by(elem, text, matcher) {
588 return Some((
589 elem.bounds.x,
590 elem.bounds.y,
591 elem.bounds.width,
592 elem.bounds.height,
593 ));
594 }
595 elem.children
596 .iter()
597 .find_map(|child| find_button_by(child, text, matcher))
598}
599
600fn is_axis_visible(bounds: RectBounds, root: RectBounds, axis: TabAxis) -> bool {
601 let (x, y, w, h) = bounds;
602 let (rx, ry, rw, rh) = root;
603 match axis {
604 TabAxis::Horizontal => {
605 x >= rx + VISIBILITY_PADDING && x + w <= rx + rw - VISIBILITY_PADDING
606 }
607 TabAxis::Vertical => y >= ry + VISIBILITY_PADDING && y + h <= ry + rh - VISIBILITY_PADDING,
608 }
609}
610
611fn is_fully_visible(bounds: RectBounds, root: RectBounds) -> bool {
612 is_axis_visible(bounds, root, TabAxis::Horizontal)
613 && is_axis_visible(bounds, root, TabAxis::Vertical)
614}
615
616fn overflow_axis_direction(bounds: RectBounds, root: RectBounds) -> Option<(TabAxis, f32)> {
617 let (x, y, w, h) = bounds;
618 let (rx, ry, rw, rh) = root;
619
620 let overflow_left = (rx + VISIBILITY_PADDING - x).max(0.0);
621 let overflow_right = (x + w - (rx + rw - VISIBILITY_PADDING)).max(0.0);
622 let overflow_top = (ry + VISIBILITY_PADDING - y).max(0.0);
623 let overflow_bottom = (y + h - (ry + rh - VISIBILITY_PADDING)).max(0.0);
624
625 let horizontal_overflow = overflow_left.max(overflow_right);
626 let vertical_overflow = overflow_top.max(overflow_bottom);
627
628 if horizontal_overflow <= 0.0 && vertical_overflow <= 0.0 {
629 return None;
630 }
631
632 if horizontal_overflow >= vertical_overflow {
633 if overflow_left > 0.0 {
634 Some((TabAxis::Horizontal, 1.0))
635 } else {
636 Some((TabAxis::Horizontal, -1.0))
637 }
638 } else if overflow_top > 0.0 {
639 Some((TabAxis::Vertical, 1.0))
640 } else {
641 Some((TabAxis::Vertical, -1.0))
642 }
643}
644
645fn intersects_root(bounds: RectBounds, root: RectBounds) -> bool {
646 let (x, y, w, h) = bounds;
647 let (rx, ry, rw, rh) = root;
648 let left = x.max(rx);
649 let top = y.max(ry);
650 let right = (x + w).min(rx + rw);
651 let bottom = (y + h).min(ry + rh);
652 right > left && bottom > top
653}
654
655fn overlap_len(a_start: f32, a_len: f32, b_start: f32, b_len: f32) -> f32 {
656 let a_end = a_start + a_len;
657 let b_end = b_start + b_len;
658 (a_end.min(b_end) - a_start.max(b_start)).max(0.0)
659}
660
661fn cross_axis_overlap(bounds: RectBounds, target: RectBounds, axis: TabAxis) -> f32 {
662 match axis {
663 TabAxis::Horizontal => overlap_len(bounds.1, bounds.3, target.1, target.3),
664 TabAxis::Vertical => overlap_len(bounds.0, bounds.2, target.0, target.2),
665 }
666}
667
668fn primary_axis_distance(bounds: RectBounds, target: RectBounds, axis: TabAxis) -> f32 {
669 let center = match axis {
670 TabAxis::Horizontal => bounds.0 + bounds.2 / 2.0,
671 TabAxis::Vertical => bounds.1 + bounds.3 / 2.0,
672 };
673 let target_center = match axis {
674 TabAxis::Horizontal => target.0 + target.2 / 2.0,
675 TabAxis::Vertical => target.1 + target.3 / 2.0,
676 };
677 (center - target_center).abs()
678}
679
680fn find_scroll_anchor(
681 elements: &[SemanticElement],
682 target: RectBounds,
683 root: RectBounds,
684 axis: TabAxis,
685) -> Option<RectBounds> {
686 let mut best: Option<(RectBounds, f32, f32)> = None;
687
688 for elem in elements {
689 if elem.clickable {
690 let bounds = (
691 elem.bounds.x,
692 elem.bounds.y,
693 elem.bounds.width,
694 elem.bounds.height,
695 );
696 if is_fully_visible(bounds, root) && intersects_root(bounds, root) {
697 let overlap = cross_axis_overlap(bounds, target, axis);
698 if overlap > 0.0 {
699 let distance = primary_axis_distance(bounds, target, axis);
700 match best {
701 None => best = Some((bounds, overlap, distance)),
702 Some((_, best_overlap, best_distance)) => {
703 let is_better_overlap = overlap > best_overlap + f32::EPSILON;
704 let is_better_distance = (overlap - best_overlap).abs() <= f32::EPSILON
705 && distance < best_distance;
706 if is_better_overlap || is_better_distance {
707 best = Some((bounds, overlap, distance));
708 }
709 }
710 }
711 }
712 }
713 }
714
715 if let Some(found) = find_scroll_anchor(&elem.children, target, root, axis) {
716 let overlap = cross_axis_overlap(found, target, axis);
717 let distance = primary_axis_distance(found, target, axis);
718 match best {
719 None => best = Some((found, overlap, distance)),
720 Some((_, best_overlap, best_distance)) => {
721 let is_better_overlap = overlap > best_overlap + f32::EPSILON;
722 let is_better_distance =
723 (overlap - best_overlap).abs() <= f32::EPSILON && distance < best_distance;
724 if is_better_overlap || is_better_distance {
725 best = Some((found, overlap, distance));
726 }
727 }
728 }
729 }
730 }
731
732 best.map(|(bounds, _, _)| bounds)
733}
734
735pub fn capture_screenshot(robot: &cranpose::Robot) -> Option<cranpose::RobotScreenshot> {
737 robot.screenshot().ok()
738}
739
740fn screenshot_logical_width(screenshot: &cranpose::RobotScreenshot) -> f32 {
741 if screenshot.logical_width.is_finite() && screenshot.logical_width > 0.0 {
742 screenshot.logical_width
743 } else {
744 screenshot.width.max(1) as f32
745 }
746}
747
748fn screenshot_logical_height(screenshot: &cranpose::RobotScreenshot) -> f32 {
749 if screenshot.logical_height.is_finite() && screenshot.logical_height > 0.0 {
750 screenshot.logical_height
751 } else {
752 screenshot.height.max(1) as f32
753 }
754}
755
756fn screenshot_scale_x(screenshot: &cranpose::RobotScreenshot) -> f32 {
757 screenshot.width.max(1) as f32 / screenshot_logical_width(screenshot)
758}
759
760fn screenshot_scale_y(screenshot: &cranpose::RobotScreenshot) -> f32 {
761 screenshot.height.max(1) as f32 / screenshot_logical_height(screenshot)
762}
763
764fn logical_to_screenshot_x(screenshot: &cranpose::RobotScreenshot, x: f32) -> f32 {
765 x * screenshot_scale_x(screenshot)
766}
767
768fn logical_to_screenshot_y(screenshot: &cranpose::RobotScreenshot, y: f32) -> f32 {
769 y * screenshot_scale_y(screenshot)
770}
771
772pub fn screenshot_logical_size(screenshot: &cranpose::RobotScreenshot) -> (f32, f32) {
773 (
774 screenshot_logical_width(screenshot),
775 screenshot_logical_height(screenshot),
776 )
777}
778
779pub fn screenshot_pixel(screenshot: &cranpose::RobotScreenshot, x: u32, y: u32) -> Option<[u8; 4]> {
781 if x >= screenshot.width || y >= screenshot.height {
782 return None;
783 }
784 let index = ((y * screenshot.width + x) * 4) as usize;
785 Some([
786 screenshot.pixels[index],
787 screenshot.pixels[index + 1],
788 screenshot.pixels[index + 2],
789 screenshot.pixels[index + 3],
790 ])
791}
792
793pub fn sample_screenshot_pixel_logical(
794 screenshot: &cranpose::RobotScreenshot,
795 x: f32,
796 y: f32,
797) -> Option<[u8; 4]> {
798 let logical_width = screenshot_logical_width(screenshot);
799 let logical_height = screenshot_logical_height(screenshot);
800 if x < 0.0 || y < 0.0 || x > logical_width || y > logical_height {
801 return None;
802 }
803
804 Some(sample_screenshot_pixel_bilinear(
805 screenshot,
806 logical_to_screenshot_x(screenshot, x),
807 logical_to_screenshot_y(screenshot, y),
808 ))
809}
810
811pub fn logical_region_to_pixel_bounds(
812 screenshot: &cranpose::RobotScreenshot,
813 region: (f32, f32, f32, f32),
814) -> Option<(u32, u32, u32, u32)> {
815 if region.2 <= 0.0 || region.3 <= 0.0 || screenshot.width == 0 || screenshot.height == 0 {
816 return None;
817 }
818
819 let left = logical_to_screenshot_x(screenshot, region.0.max(0.0))
820 .floor()
821 .max(0.0) as u32;
822 let top = logical_to_screenshot_y(screenshot, region.1.max(0.0))
823 .floor()
824 .max(0.0) as u32;
825 let right = logical_to_screenshot_x(
826 screenshot,
827 (region.0 + region.2).min(screenshot_logical_width(screenshot)),
828 )
829 .ceil()
830 .min(screenshot.width as f32) as u32;
831 let bottom = logical_to_screenshot_y(
832 screenshot,
833 (region.1 + region.3).min(screenshot_logical_height(screenshot)),
834 )
835 .ceil()
836 .min(screenshot.height as f32) as u32;
837
838 if right <= left || bottom <= top {
839 return None;
840 }
841
842 Some((left, top, right, bottom))
843}
844
845pub fn crop_screenshot(
847 screenshot: &cranpose::RobotScreenshot,
848 x: u32,
849 y: u32,
850 width: u32,
851 height: u32,
852) -> Option<cranpose::RobotScreenshot> {
853 if width == 0 || height == 0 {
854 return None;
855 }
856 let end_x = x.checked_add(width)?;
857 let end_y = y.checked_add(height)?;
858 if end_x > screenshot.width || end_y > screenshot.height {
859 return None;
860 }
861
862 let mut pixels = vec![0u8; (width * height * 4) as usize];
863 for row in 0..height {
864 let src_start = (((y + row) * screenshot.width + x) * 4) as usize;
865 let src_end = src_start + (width * 4) as usize;
866 let dst_start = (row * width * 4) as usize;
867 let dst_end = dst_start + (width * 4) as usize;
868 pixels[dst_start..dst_end].copy_from_slice(&screenshot.pixels[src_start..src_end]);
869 }
870
871 Some(cranpose::RobotScreenshot {
872 width,
873 height,
874 logical_width: width as f32 / screenshot_scale_x(screenshot),
875 logical_height: height as f32 / screenshot_scale_y(screenshot),
876 pixels,
877 })
878}
879
880pub fn crop_screenshot_logical(
881 screenshot: &cranpose::RobotScreenshot,
882 x: f32,
883 y: f32,
884 width: f32,
885 height: f32,
886) -> Option<cranpose::RobotScreenshot> {
887 let (left, top, right, bottom) =
888 logical_region_to_pixel_bounds(screenshot, (x, y, width, height))?;
889 crop_screenshot(screenshot, left, top, right - left, bottom - top)
890}
891
892#[derive(Debug, Clone, PartialEq, Eq)]
893pub struct ScreenshotPixelDifference {
894 pub x: u32,
895 pub y: u32,
896 pub before: [u8; 4],
897 pub after: [u8; 4],
898 pub difference: u32,
899}
900
901#[derive(Debug, Clone, PartialEq, Eq)]
902pub struct ScreenshotDifferenceStats {
903 pub differing_pixels: usize,
904 pub max_difference: u32,
905 pub first_difference: Option<ScreenshotPixelDifference>,
906}
907
908pub fn normalize_screenshot_region(
913 screenshot: &cranpose::RobotScreenshot,
914 region: (f32, f32, f32, f32),
915 output_width: u32,
916 output_height: u32,
917) -> Option<cranpose::RobotScreenshot> {
918 if output_width == 0
919 || output_height == 0
920 || region.2 <= 0.0
921 || region.3 <= 0.0
922 || screenshot.width == 0
923 || screenshot.height == 0
924 {
925 return None;
926 }
927
928 let mut pixels = Vec::with_capacity((output_width * output_height * 4) as usize);
929 for y in 0..output_height {
930 for x in 0..output_width {
931 let sample_x = region.0 + ((x as f32 + 0.5) * region.2 / output_width as f32);
932 let sample_y = region.1 + ((y as f32 + 0.5) * region.3 / output_height as f32);
933 pixels.extend_from_slice(&sample_screenshot_pixel_logical(
934 screenshot, sample_x, sample_y,
935 )?);
936 }
937 }
938
939 Some(cranpose::RobotScreenshot {
940 width: output_width,
941 height: output_height,
942 logical_width: output_width as f32,
943 logical_height: output_height as f32,
944 pixels,
945 })
946}
947
948pub fn screenshot_difference_stats(
951 before: &cranpose::RobotScreenshot,
952 after: &cranpose::RobotScreenshot,
953 difference_tolerance: u32,
954) -> Option<ScreenshotDifferenceStats> {
955 if before.width != after.width || before.height != after.height {
956 return None;
957 }
958
959 let mut differing_pixels = 0usize;
960 let mut max_difference = 0u32;
961 let mut first_difference = None;
962
963 for y in 0..before.height {
964 for x in 0..before.width {
965 let before_pixel =
966 screenshot_pixel(before, x, y).expect("screenshot bounds checked by loop");
967 let after_pixel =
968 screenshot_pixel(after, x, y).expect("screenshot bounds checked by loop");
969 let difference = pixel_difference(before_pixel, after_pixel);
970 if difference > difference_tolerance {
971 differing_pixels += 1;
972 max_difference = max_difference.max(difference);
973 if first_difference.is_none() {
974 first_difference = Some(ScreenshotPixelDifference {
975 x,
976 y,
977 before: before_pixel,
978 after: after_pixel,
979 difference,
980 });
981 }
982 }
983 }
984 }
985
986 Some(ScreenshotDifferenceStats {
987 differing_pixels,
988 max_difference,
989 first_difference,
990 })
991}
992
993pub fn changed_pixel_count(
996 before: &cranpose::RobotScreenshot,
997 after: &cranpose::RobotScreenshot,
998 channel_threshold: u8,
999) -> usize {
1000 if before.width != after.width || before.height != after.height {
1001 return usize::MAX;
1002 }
1003
1004 before
1005 .pixels
1006 .chunks_exact(4)
1007 .zip(after.pixels.chunks_exact(4))
1008 .filter(|(a, b)| {
1009 a[0].abs_diff(b[0]) > channel_threshold
1010 || a[1].abs_diff(b[1]) > channel_threshold
1011 || a[2].abs_diff(b[2]) > channel_threshold
1012 || a[3].abs_diff(b[3]) > channel_threshold
1013 })
1014 .count()
1015}
1016
1017pub fn changed_pixel_count_in_region(
1019 before: &cranpose::RobotScreenshot,
1020 after: &cranpose::RobotScreenshot,
1021 region: (f32, f32, f32, f32),
1022 channel_threshold: u8,
1023) -> usize {
1024 if before.width != after.width
1025 || before.height != after.height
1026 || (screenshot_scale_x(before) - screenshot_scale_x(after)).abs() > f32::EPSILON
1027 || (screenshot_scale_y(before) - screenshot_scale_y(after)).abs() > f32::EPSILON
1028 {
1029 return usize::MAX;
1030 }
1031
1032 let Some((left, top, right, bottom)) = logical_region_to_pixel_bounds(before, region) else {
1033 return 0;
1034 };
1035
1036 let width = before.width as usize;
1037 let mut changed = 0usize;
1038 for y in top..bottom {
1039 for x in left..right {
1040 let idx = ((y as usize) * width + x as usize) * 4;
1041 if before.pixels[idx].abs_diff(after.pixels[idx]) > channel_threshold
1042 || before.pixels[idx + 1].abs_diff(after.pixels[idx + 1]) > channel_threshold
1043 || before.pixels[idx + 2].abs_diff(after.pixels[idx + 2]) > channel_threshold
1044 || before.pixels[idx + 3].abs_diff(after.pixels[idx + 3]) > channel_threshold
1045 {
1046 changed += 1;
1047 }
1048 }
1049 }
1050
1051 changed
1052}
1053
1054fn sample_screenshot_pixel_bilinear(
1055 screenshot: &cranpose::RobotScreenshot,
1056 x: f32,
1057 y: f32,
1058) -> [u8; 4] {
1059 let max_x = screenshot.width.saturating_sub(1) as f32;
1060 let max_y = screenshot.height.saturating_sub(1) as f32;
1061 let source_x = (x - 0.5).clamp(0.0, max_x);
1062 let source_y = (y - 0.5).clamp(0.0, max_y);
1063 let x0 = source_x.floor() as u32;
1064 let y0 = source_y.floor() as u32;
1065 let x1 = (x0 + 1).min(screenshot.width.saturating_sub(1));
1066 let y1 = (y0 + 1).min(screenshot.height.saturating_sub(1));
1067 let tx = source_x - x0 as f32;
1068 let ty = source_y - y0 as f32;
1069 let top_left = screenshot_pixel(screenshot, x0, y0).expect("bilinear x0/y0 in bounds");
1070 let top_right = screenshot_pixel(screenshot, x1, y0).expect("bilinear x1/y0 in bounds");
1071 let bottom_left = screenshot_pixel(screenshot, x0, y1).expect("bilinear x0/y1 in bounds");
1072 let bottom_right = screenshot_pixel(screenshot, x1, y1).expect("bilinear x1/y1 in bounds");
1073
1074 let lerp_channel = |index: usize| {
1075 let top = top_left[index] as f32 * (1.0 - tx) + top_right[index] as f32 * tx;
1076 let bottom = bottom_left[index] as f32 * (1.0 - tx) + bottom_right[index] as f32 * tx;
1077 (top * (1.0 - ty) + bottom * ty).round() as u8
1078 };
1079
1080 [
1081 lerp_channel(0),
1082 lerp_channel(1),
1083 lerp_channel(2),
1084 lerp_channel(3),
1085 ]
1086}
1087
1088fn pixel_difference(before: [u8; 4], after: [u8; 4]) -> u32 {
1089 before
1090 .into_iter()
1091 .zip(after)
1092 .map(|(lhs, rhs)| lhs.abs_diff(rhs) as u32)
1093 .sum()
1094}
1095
1096pub fn parse_slider_value(text: &str) -> Option<f32> {
1098 text.split_once(':')
1099 .and_then(|(_, value)| value.trim().parse::<f32>().ok())
1100}
1101
1102pub fn scroll_down(robot: &cranpose::Robot, center_x: f32, from_y: f32, to_y: f32) {
1104 let _ = robot.drag(center_x, from_y, center_x, to_y);
1105 std::thread::sleep(std::time::Duration::from_millis(180));
1106 let _ = robot.wait_for_idle();
1107}
1108
1109pub fn scroll_up(robot: &cranpose::Robot, center_x: f32, from_y: f32, to_y: f32) {
1111 let _ = robot.drag(center_x, from_y, center_x, to_y);
1112 std::thread::sleep(std::time::Duration::from_millis(180));
1113 let _ = robot.wait_for_idle();
1114}
1115
1116pub fn y_is_visible(robot: &cranpose::Robot, y: f32) -> bool {
1119 let Some((_, root_y, _, root_h)) = root_bounds(robot) else {
1120 return true;
1121 };
1122 let top = root_y + 28.0;
1123 let bottom = root_y + root_h - 28.0;
1124 y >= top && y <= bottom
1125}
1126
1127#[derive(Clone, Copy, Debug)]
1129pub struct ScrollConfig {
1130 pub center_x: f32,
1131 pub down_from_y: f32,
1132 pub down_to_y: f32,
1133 pub up_from_y: f32,
1134 pub up_to_y: f32,
1135}
1136
1137pub fn scroll_prefix_into_view(
1140 robot: &cranpose::Robot,
1141 prefix: &str,
1142 max_attempts: usize,
1143 cfg: ScrollConfig,
1144) -> Option<(f32, f32, f32, f32, String)> {
1145 for attempt in 0..max_attempts {
1146 if let Some(bounds) = find_text_by_prefix_in_semantics(robot, prefix) {
1147 let center_y = bounds.1 + bounds.3 * 0.5;
1148 if y_is_visible(robot, center_y) {
1149 return Some(bounds);
1150 }
1151 let Some((_, root_y, _, root_h)) = root_bounds(robot) else {
1152 return Some(bounds);
1153 };
1154 let viewport_mid = root_y + root_h * 0.5;
1155 if center_y > viewport_mid {
1156 scroll_down(robot, cfg.center_x, cfg.down_from_y, cfg.down_to_y);
1157 } else {
1158 scroll_up(robot, cfg.center_x, cfg.up_from_y, cfg.up_to_y);
1159 }
1160 } else {
1161 if attempt % 2 == 0 {
1163 scroll_down(robot, cfg.center_x, cfg.down_from_y, cfg.down_to_y);
1164 } else {
1165 scroll_up(robot, cfg.center_x, cfg.up_from_y, cfg.up_to_y);
1166 }
1167 }
1168 }
1169 None
1170}
1171
1172pub fn set_slider_fraction(
1174 robot: &cranpose::Robot,
1175 prefix: &str,
1176 fraction: f32,
1177 slider_width: f32,
1178 slider_touch_offset_y: f32,
1179 cfg: ScrollConfig,
1180) -> Option<f32> {
1181 let (x, y, _w, h, _) = scroll_prefix_into_view(robot, prefix, 18, cfg)?;
1182 let slider_y = y + h + slider_touch_offset_y;
1183 let left_x = x + 2.0;
1184 let target_x = x + slider_width * fraction.clamp(0.0, 1.0);
1185 let _ = robot.drag(left_x, slider_y, target_x, slider_y);
1186 std::thread::sleep(std::time::Duration::from_millis(120));
1187 let _ = robot.wait_for_idle();
1188 find_text_by_prefix_in_semantics(robot, prefix)
1189 .and_then(|(_, _, _, _, t)| parse_slider_value(&t))
1190}
1191
1192#[cfg(test)]
1193mod tests {
1194 use super::*;
1195 use cranpose::{RobotScreenshot, SemanticRect};
1196
1197 fn semantic_element(
1198 role: &str,
1199 text: Option<&str>,
1200 clickable: bool,
1201 bounds: RectBounds,
1202 children: Vec<SemanticElement>,
1203 ) -> SemanticElement {
1204 SemanticElement {
1205 role: role.to_string(),
1206 text: text.map(ToString::to_string),
1207 clickable,
1208 bounds: SemanticRect {
1209 x: bounds.0,
1210 y: bounds.1,
1211 width: bounds.2,
1212 height: bounds.3,
1213 },
1214 children,
1215 }
1216 }
1217
1218 #[test]
1219 fn overflow_axis_direction_detects_horizontal_overflow() {
1220 let root = (0.0, 0.0, 300.0, 200.0);
1221 let target = (320.0, 20.0, 80.0, 30.0);
1222 assert_eq!(
1223 overflow_axis_direction(target, root),
1224 Some((TabAxis::Horizontal, -1.0))
1225 );
1226 }
1227
1228 #[test]
1229 fn overflow_axis_direction_detects_vertical_overflow() {
1230 let root = (0.0, 0.0, 300.0, 200.0);
1231 let target = (20.0, -60.0, 80.0, 30.0);
1232 assert_eq!(
1233 overflow_axis_direction(target, root),
1234 Some((TabAxis::Vertical, 1.0))
1235 );
1236 }
1237
1238 #[test]
1239 fn find_scroll_anchor_prefers_cross_axis_overlap() {
1240 let root = (0.0, 0.0, 300.0, 200.0);
1241 let target = (320.0, 22.0, 80.0, 28.0);
1242
1243 let same_row = semantic_element(
1244 "Layout",
1245 Some("same-row"),
1246 true,
1247 (120.0, 20.0, 80.0, 30.0),
1248 vec![],
1249 );
1250 let other_row = semantic_element(
1251 "Layout",
1252 Some("other-row"),
1253 true,
1254 (120.0, 130.0, 80.0, 30.0),
1255 vec![],
1256 );
1257 let root_elem = semantic_element("Layout", None, false, root, vec![same_row, other_row]);
1258
1259 let anchor = find_scroll_anchor(&[root_elem], target, root, TabAxis::Horizontal)
1260 .expect("expected anchor");
1261
1262 assert_eq!(anchor, (120.0, 20.0, 80.0, 30.0));
1263 }
1264
1265 #[test]
1266 fn find_button_exact_requires_full_text_match() {
1267 let exact_button = semantic_element(
1268 "Button",
1269 None,
1270 true,
1271 (10.0, 20.0, 90.0, 28.0),
1272 vec![
1273 semantic_element(
1274 "Text",
1275 Some("Text"),
1276 false,
1277 (14.0, 24.0, 30.0, 20.0),
1278 vec![],
1279 ),
1280 semantic_element(
1281 "Text",
1282 Some("Text Input"),
1283 false,
1284 (46.0, 24.0, 48.0, 20.0),
1285 vec![],
1286 ),
1287 ],
1288 );
1289 let partial_only_button = semantic_element(
1290 "Button",
1291 None,
1292 true,
1293 (120.0, 20.0, 90.0, 28.0),
1294 vec![semantic_element(
1295 "Text",
1296 Some("Text Input"),
1297 false,
1298 (124.0, 24.0, 48.0, 20.0),
1299 vec![],
1300 )],
1301 );
1302
1303 assert_eq!(
1304 find_button_by(&exact_button, "Text", text_equals),
1305 Some((10.0, 20.0, 90.0, 28.0))
1306 );
1307 assert_eq!(
1308 find_button_by(&partial_only_button, "Text", text_equals),
1309 None
1310 );
1311 }
1312
1313 #[test]
1314 fn screenshot_pixel_reads_expected_value() {
1315 let screenshot = RobotScreenshot {
1316 width: 2,
1317 height: 1,
1318 logical_width: 2.0,
1319 logical_height: 1.0,
1320 pixels: vec![1, 2, 3, 4, 5, 6, 7, 8],
1321 };
1322 assert_eq!(screenshot_pixel(&screenshot, 1, 0), Some([5, 6, 7, 8]));
1323 }
1324
1325 #[test]
1326 fn crop_screenshot_extracts_region() {
1327 let screenshot = RobotScreenshot {
1328 width: 3,
1329 height: 2,
1330 logical_width: 3.0,
1331 logical_height: 2.0,
1332 pixels: vec![
1333 1, 2, 3, 255, 4, 5, 6, 255, 7, 8, 9, 255, 10, 11, 12, 255, 13, 14, 15, 255, 16, 17,
1334 18, 255,
1335 ],
1336 };
1337 let cropped = crop_screenshot(&screenshot, 1, 0, 2, 2).expect("crop");
1338 assert_eq!(cropped.width, 2);
1339 assert_eq!(cropped.height, 2);
1340 assert_eq!(cropped.logical_width, 2.0);
1341 assert_eq!(cropped.logical_height, 2.0);
1342 assert_eq!(
1343 cropped.pixels,
1344 vec![4, 5, 6, 255, 7, 8, 9, 255, 13, 14, 15, 255, 16, 17, 18, 255]
1345 );
1346 }
1347
1348 #[test]
1349 fn normalize_screenshot_region_preserves_pixel_grid_at_native_size() {
1350 let screenshot = RobotScreenshot {
1351 width: 2,
1352 height: 2,
1353 logical_width: 2.0,
1354 logical_height: 2.0,
1355 pixels: vec![1, 2, 3, 255, 4, 5, 6, 255, 7, 8, 9, 255, 10, 11, 12, 255],
1356 };
1357
1358 let normalized =
1359 normalize_screenshot_region(&screenshot, (0.0, 0.0, 2.0, 2.0), 2, 2).expect("norm");
1360
1361 assert_eq!(normalized.width, screenshot.width);
1362 assert_eq!(normalized.height, screenshot.height);
1363 assert_eq!(normalized.logical_width, screenshot.logical_width);
1364 assert_eq!(normalized.logical_height, screenshot.logical_height);
1365 assert_eq!(normalized.pixels, screenshot.pixels);
1366 }
1367
1368 #[test]
1369 fn changed_pixel_count_in_region_uses_logical_coordinates() {
1370 let before = RobotScreenshot {
1371 width: 4,
1372 height: 4,
1373 logical_width: 2.0,
1374 logical_height: 2.0,
1375 pixels: vec![
1376 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255,
1377 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255,
1378 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255,
1379 ],
1380 };
1381 let mut after = before.clone();
1382 for y in 2..4 {
1383 for x in 2..4 {
1384 let idx = ((y * after.width + x) * 4) as usize;
1385 after.pixels[idx] = 255;
1386 }
1387 }
1388
1389 assert_eq!(
1390 changed_pixel_count_in_region(&before, &after, (1.0, 1.0, 1.0, 1.0), 1),
1391 4
1392 );
1393 }
1394
1395 #[test]
1396 fn sample_screenshot_pixel_logical_maps_scaled_capture() {
1397 let screenshot = RobotScreenshot {
1398 width: 4,
1399 height: 4,
1400 logical_width: 2.0,
1401 logical_height: 2.0,
1402 pixels: vec![
1403 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255,
1404 7, 0, 0, 255, 8, 0, 0, 255, 9, 0, 0, 255, 10, 0, 0, 255, 11, 0, 0, 255, 12, 0, 0,
1405 255, 13, 0, 0, 255, 14, 0, 0, 255, 15, 0, 0, 255, 16, 0, 0, 255,
1406 ],
1407 };
1408
1409 assert_eq!(
1410 sample_screenshot_pixel_logical(&screenshot, 1.25, 1.25),
1411 Some([11, 0, 0, 255])
1412 );
1413 }
1414
1415 #[test]
1416 fn screenshot_difference_stats_reports_first_difference() {
1417 let before = RobotScreenshot {
1418 width: 2,
1419 height: 1,
1420 logical_width: 2.0,
1421 logical_height: 1.0,
1422 pixels: vec![10, 20, 30, 255, 1, 2, 3, 255],
1423 };
1424 let after = RobotScreenshot {
1425 width: 2,
1426 height: 1,
1427 logical_width: 2.0,
1428 logical_height: 1.0,
1429 pixels: vec![10, 20, 30, 255, 4, 8, 3, 200],
1430 };
1431
1432 let stats = screenshot_difference_stats(&before, &after, 3).expect("stats");
1433
1434 assert_eq!(stats.differing_pixels, 1);
1435 assert_eq!(stats.max_difference, 64);
1436 assert_eq!(
1437 stats.first_difference,
1438 Some(ScreenshotPixelDifference {
1439 x: 1,
1440 y: 0,
1441 before: [1, 2, 3, 255],
1442 after: [4, 8, 3, 200],
1443 difference: 64,
1444 })
1445 );
1446 }
1447}