1use crate::robot_assertions::{Bounds, SemanticElementLike};
7use cranpose::SemanticElement;
8use std::collections::HashMap;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::Arc;
11use std::time::Duration;
12
13impl SemanticElementLike for SemanticElement {
16 fn text(&self) -> Option<&str> {
17 self.text.as_deref()
18 }
19
20 fn role(&self) -> &str {
21 &self.role
22 }
23
24 fn clickable(&self) -> bool {
25 self.clickable
26 }
27
28 fn bounds(&self) -> Bounds {
29 Bounds {
30 x: self.bounds.x,
31 y: self.bounds.y,
32 width: self.bounds.width,
33 height: self.bounds.height,
34 }
35 }
36
37 fn children(&self) -> &[Self] {
38 &self.children
39 }
40}
41
42pub fn find_text(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
49 if let Some(ref t) = elem.text {
50 if t.contains(text) {
51 return Some((
52 elem.bounds.x,
53 elem.bounds.y,
54 elem.bounds.width,
55 elem.bounds.height,
56 ));
57 }
58 }
59 for child in &elem.children {
60 if let Some(pos) = find_text(child, text) {
61 return Some(pos);
62 }
63 }
64 None
65}
66
67pub fn find_text_exact(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
71 if let Some(ref t) = elem.text {
72 if t == text {
73 return Some((
74 elem.bounds.x,
75 elem.bounds.y,
76 elem.bounds.width,
77 elem.bounds.height,
78 ));
79 }
80 }
81 for child in &elem.children {
82 if let Some(pos) = find_text_exact(child, text) {
83 return Some(pos);
84 }
85 }
86 None
87}
88
89pub fn find_text_center(elem: &SemanticElement, text: &str) -> Option<(f32, f32)> {
93 find_text(elem, text).map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
94}
95
96pub fn has_text(elem: &SemanticElement, text: &str) -> bool {
98 if let Some(ref t) = elem.text {
99 if t.contains(text) {
100 return true;
101 }
102 }
103 elem.children.iter().any(|c| has_text(c, text))
104}
105
106pub fn find_button(elem: &SemanticElement, text: &str) -> Option<(f32, f32, f32, f32)> {
111 if elem.clickable && has_text(elem, text) {
112 return Some((
113 elem.bounds.x,
114 elem.bounds.y,
115 elem.bounds.width,
116 elem.bounds.height,
117 ));
118 }
119 for child in &elem.children {
120 if let Some(pos) = find_button(child, text) {
121 return Some(pos);
122 }
123 }
124 None
125}
126
127pub fn find_button_center(elem: &SemanticElement, text: &str) -> Option<(f32, f32)> {
130 find_button(elem, text).map(|(x, y, w, h)| (x + w / 2.0, y + h / 2.0))
131}
132
133pub fn find_in_semantics<F>(robot: &cranpose::Robot, finder: F) -> Option<(f32, f32, f32, f32)>
141where
142 F: Fn(&SemanticElement) -> Option<(f32, f32, f32, f32)>,
143{
144 match robot.get_semantics() {
145 Ok(semantics) => {
146 for root in semantics.iter() {
147 if let Some(result) = finder(root) {
148 return Some(result);
149 }
150 }
151 None
152 }
153 Err(e) => {
154 eprintln!(" ✗ Failed to get semantics: {}", e);
155 None
156 }
157 }
158}
159
160pub fn find_text_in_semantics(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32, f32, f32)> {
165 let text = text.to_string();
166 find_in_semantics(robot, |elem| find_text(elem, &text))
167}
168
169pub fn find_text_by_prefix(
172 elem: &SemanticElement,
173 prefix: &str,
174) -> Option<(f32, f32, f32, f32, String)> {
175 if let Some(ref t) = elem.text {
176 if t.starts_with(prefix) {
177 return Some((
178 elem.bounds.x,
179 elem.bounds.y,
180 elem.bounds.width,
181 elem.bounds.height,
182 t.clone(),
183 ));
184 }
185 }
186 for child in &elem.children {
187 if let Some(result) = find_text_by_prefix(child, prefix) {
188 return Some(result);
189 }
190 }
191 None
192}
193
194pub fn find_text_by_prefix_in_semantics(
198 robot: &cranpose::Robot,
199 prefix: &str,
200) -> Option<(f32, f32, f32, f32, String)> {
201 let prefix = prefix.to_string();
202 match robot.get_semantics() {
203 Ok(semantics) => {
204 for root in semantics.iter() {
205 if let Some(result) = find_text_by_prefix(root, &prefix) {
206 return Some(result);
207 }
208 }
209 None
210 }
211 Err(e) => {
212 eprintln!(" ✗ Failed to get semantics: {}", e);
213 None
214 }
215 }
216}
217
218pub fn find_button_in_semantics(
221 robot: &cranpose::Robot,
222 text: &str,
223) -> Option<(f32, f32, f32, f32)> {
224 let text_owned = text.to_string();
225 let mut bounds = find_in_semantics(robot, |elem| find_button(elem, &text_owned));
226
227 let Some(root) = root_bounds(robot) else {
228 return bounds;
229 };
230
231 if let Some(current) = bounds {
232 if is_fully_visible(current, root) {
233 return bounds;
234 }
235 } else {
236 return None;
237 }
238
239 for _ in 0..8 {
240 let Some(current) = bounds else {
241 break;
242 };
243
244 let Some((axis, dir)) = overflow_axis_direction(current, root) else {
245 break;
246 };
247
248 let semantics = match robot.get_semantics() {
249 Ok(semantics) => semantics,
250 Err(e) => {
251 eprintln!(" ✗ Failed to get semantics: {}", e);
252 break;
253 }
254 };
255
256 let Some((sx, sy, sw, sh)) = find_scroll_anchor(&semantics, current, root, axis) else {
257 break;
258 };
259
260 let start_x = sx + sw / 2.0;
261 let start_y = sy + sh / 2.0;
262 let (end_x, end_y) = match axis {
263 TabAxis::Horizontal => (start_x + dir * SCROLL_STEP, start_y),
264 TabAxis::Vertical => (start_x, start_y + dir * SCROLL_STEP),
265 };
266 let _ = robot.drag(start_x, start_y, end_x, end_y);
267 std::thread::sleep(Duration::from_millis(SCROLL_SETTLE_MS));
268 let _ = robot.wait_for_idle();
269
270 bounds = find_in_semantics(robot, |elem| find_button(elem, &text_owned));
271 if let Some(current) = bounds {
272 if is_fully_visible(current, root) {
273 break;
274 }
275 }
276 }
277
278 bounds
279}
280
281pub fn find_by_text_recursive(elements: &[SemanticElement], text: &str) -> Option<SemanticElement> {
284 for elem in elements {
285 if let Some(ref elem_text) = elem.text {
286 if elem_text.contains(text) {
287 return Some(elem.clone());
288 }
289 }
290 if let Some(found) = find_by_text_recursive(&elem.children, text) {
291 return Some(found);
292 }
293 }
294 None
295}
296
297pub fn find_clickables_in_range(
300 elements: &[SemanticElement],
301 min_y: f32,
302 max_y: f32,
303) -> Vec<(String, f32, f32)> {
304 fn search(elem: &SemanticElement, tabs: &mut Vec<(String, f32, f32)>, min_y: f32, max_y: f32) {
305 if elem.role == "Layout" && elem.clickable && elem.bounds.y > min_y && elem.bounds.y < max_y
306 {
307 let label = elem
308 .children
309 .iter()
310 .find(|child| child.role == "Text")
311 .and_then(|text_elem| text_elem.text.clone())
312 .unwrap_or_else(|| "Unknown".to_string());
313 tabs.push((label, elem.bounds.x, elem.bounds.y));
314 }
315 for child in &elem.children {
316 search(child, tabs, min_y, max_y);
317 }
318 }
319
320 let mut tabs = Vec::new();
321 for elem in elements {
322 search(elem, &mut tabs, min_y, max_y);
323 }
324 tabs.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap());
325 tabs
326}
327
328pub fn find_element_by_text_exact<'a>(
329 elements: &'a [SemanticElement],
330 text: &str,
331) -> Option<&'a SemanticElement> {
332 for elem in elements {
333 if elem.text.as_deref() == Some(text) {
334 return Some(elem);
335 }
336 if let Some(found) = find_element_by_text_exact(&elem.children, text) {
337 return Some(found);
338 }
339 }
340 None
341}
342
343pub fn find_bounds_by_text(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32, f32, f32)> {
344 let semantics = robot.get_semantics().ok()?;
345 let elem = find_element_by_text_exact(&semantics, text)?;
346 Some((
347 elem.bounds.x,
348 elem.bounds.y,
349 elem.bounds.width,
350 elem.bounds.height,
351 ))
352}
353
354pub fn visible_bounds_in_viewport(
355 robot: &cranpose::Robot,
356 bounds: (f32, f32, f32, f32),
357 padding: f32,
358) -> Option<(f32, f32, f32, f32)> {
359 let semantics = robot.get_semantics().ok()?;
360 let mut viewport = None;
361 for elem in semantics.iter() {
362 let elem_bounds = (
363 elem.bounds.x,
364 elem.bounds.y,
365 elem.bounds.width,
366 elem.bounds.height,
367 );
368 viewport = Some(match viewport {
369 Some(existing) => union_bounds(existing, Some(elem_bounds)),
370 None => elem_bounds,
371 });
372 }
373 let (viewport_x, viewport_y, viewport_width, viewport_height) = viewport?;
374 let min_x = viewport_x + padding;
375 let min_y = viewport_y + padding;
376 let max_x = viewport_x + viewport_width - padding;
377 let max_y = viewport_y + viewport_height - padding;
378
379 let left = bounds.0.max(min_x);
380 let top = bounds.1.max(min_y);
381 let right = (bounds.0 + bounds.2).min(max_x);
382 let bottom = (bounds.1 + bounds.3).min(max_y);
383
384 if right <= left || bottom <= top {
385 None
386 } else {
387 Some((left, top, right - left, bottom - top))
388 }
389}
390
391pub fn find_center_by_text(robot: &cranpose::Robot, text: &str) -> Option<(f32, f32)> {
392 let (x, y, w, h) = find_bounds_by_text(robot, text)?;
393 Some((x + w / 2.0, y + h / 2.0))
394}
395
396pub fn find_in_subtree_by_text<'a>(
397 elem: &'a SemanticElement,
398 text: &str,
399) -> Option<&'a SemanticElement> {
400 if elem.text.as_deref() == Some(text) {
401 return Some(elem);
402 }
403 for child in &elem.children {
404 if let Some(found) = find_in_subtree_by_text(child, text) {
405 return Some(found);
406 }
407 }
408 None
409}
410
411pub fn print_semantics_with_bounds(elements: &[SemanticElement], indent: usize) {
412 for elem in elements {
413 let prefix = " ".repeat(indent);
414 let text = elem.text.as_deref().unwrap_or("");
415 println!(
416 "{}role={} text=\"{}\" bounds=({:.1},{:.1},{:.1},{:.1}){}",
417 prefix,
418 elem.role,
419 text,
420 elem.bounds.x,
421 elem.bounds.y,
422 elem.bounds.width,
423 elem.bounds.height,
424 if elem.clickable { " [CLICKABLE]" } else { "" }
425 );
426 print_semantics_with_bounds(&elem.children, indent + 1);
427 }
428}
429
430pub fn union_bounds(
431 base: (f32, f32, f32, f32),
432 other: Option<(f32, f32, f32, f32)>,
433) -> (f32, f32, f32, f32) {
434 let (x, y, w, h) = base;
435 let mut min_x = x;
436 let mut min_y = y;
437 let mut max_x = x + w;
438 let mut max_y = y + h;
439 if let Some((ox, oy, ow, oh)) = other {
440 min_x = min_x.min(ox);
441 min_y = min_y.min(oy);
442 max_x = max_x.max(ox + ow);
443 max_y = max_y.max(oy + oh);
444 }
445 (min_x, min_y, max_x - min_x, max_y - min_y)
446}
447
448pub fn count_text_in_tree(elements: &[SemanticElement], text: &str) -> usize {
449 let mut count = 0;
450 for elem in elements {
451 if elem.text.as_deref() == Some(text) {
452 count += 1;
453 }
454 count += count_text_in_tree(&elem.children, text);
455 }
456 count
457}
458
459pub fn collect_by_text_exact<'a>(
460 elements: &'a [SemanticElement],
461 text: &str,
462 results: &mut Vec<&'a SemanticElement>,
463) {
464 for elem in elements {
465 if elem.text.as_deref() == Some(text) {
466 results.push(elem);
467 }
468 collect_by_text_exact(&elem.children, text, results);
469 }
470}
471
472pub fn collect_text_prefix_counts(
473 elements: &[SemanticElement],
474 prefix: &str,
475 counts: &mut HashMap<String, usize>,
476) {
477 for elem in elements {
478 if let Some(text) = elem.text.as_deref() {
479 if text.starts_with(prefix) {
480 *counts.entry(text.to_string()).or_insert(0) += 1;
481 }
482 }
483 collect_text_prefix_counts(&elem.children, prefix, counts);
484 }
485}
486
487pub fn exit_with_timeout(robot: &cranpose::Robot, timeout: Duration) {
488 let done = Arc::new(AtomicBool::new(false));
489 let done_thread = Arc::clone(&done);
490 std::thread::spawn(move || {
491 std::thread::sleep(timeout);
492 if !done_thread.load(Ordering::Relaxed) {
493 std::process::exit(0);
494 }
495 });
496
497 let _ = robot.exit();
498 done.store(true, Ordering::Relaxed);
499}
500
501#[derive(Clone, Copy, Debug, PartialEq, Eq)]
502pub enum TabAxis {
503 Horizontal,
504 Vertical,
505}
506
507type RectBounds = (f32, f32, f32, f32);
508type LabeledRect = (String, RectBounds);
509
510pub fn collect_tab_bounds(robot: &cranpose::Robot, labels: &[&str]) -> Vec<LabeledRect> {
511 let mut tabs = Vec::new();
512 for label in labels {
513 if let Some(bounds) = find_in_semantics(robot, |elem| find_button(elem, label)) {
514 tabs.push(((*label).to_string(), bounds));
515 }
516 }
517 tabs
518}
519
520pub fn bounds_span(bounds: &[LabeledRect]) -> Option<RectBounds> {
521 let mut iter = bounds.iter();
522 let (_, (x, y, w, h)) = iter.next()?;
523 let mut min_x = *x;
524 let mut min_y = *y;
525 let mut max_x = x + w;
526 let mut max_y = y + h;
527 for (_, (bx, by, bw, bh)) in iter {
528 min_x = min_x.min(*bx);
529 min_y = min_y.min(*by);
530 max_x = max_x.max(*bx + *bw);
531 max_y = max_y.max(*by + *bh);
532 }
533 Some((min_x, min_y, max_x, max_y))
534}
535
536pub fn detect_tab_axis(bounds: &[LabeledRect]) -> Option<TabAxis> {
537 let (min_x, min_y, max_x, max_y) = bounds_span(bounds)?;
538 let span_x = max_x - min_x;
539 let span_y = max_y - min_y;
540 if span_x >= span_y {
541 Some(TabAxis::Horizontal)
542 } else {
543 Some(TabAxis::Vertical)
544 }
545}
546
547pub fn root_bounds(robot: &cranpose::Robot) -> Option<RectBounds> {
548 let semantics = robot.get_semantics().ok()?;
549 let root = semantics.first()?;
550 Some((
551 root.bounds.x,
552 root.bounds.y,
553 root.bounds.width,
554 root.bounds.height,
555 ))
556}
557
558const VISIBILITY_PADDING: f32 = 4.0;
559const SCROLL_STEP: f32 = 240.0;
560const SCROLL_SETTLE_MS: u64 = 140;
561
562fn is_axis_visible(bounds: RectBounds, root: RectBounds, axis: TabAxis) -> bool {
563 let (x, y, w, h) = bounds;
564 let (rx, ry, rw, rh) = root;
565 match axis {
566 TabAxis::Horizontal => {
567 x >= rx + VISIBILITY_PADDING && x + w <= rx + rw - VISIBILITY_PADDING
568 }
569 TabAxis::Vertical => y >= ry + VISIBILITY_PADDING && y + h <= ry + rh - VISIBILITY_PADDING,
570 }
571}
572
573fn is_fully_visible(bounds: RectBounds, root: RectBounds) -> bool {
574 is_axis_visible(bounds, root, TabAxis::Horizontal)
575 && is_axis_visible(bounds, root, TabAxis::Vertical)
576}
577
578fn overflow_axis_direction(bounds: RectBounds, root: RectBounds) -> Option<(TabAxis, f32)> {
579 let (x, y, w, h) = bounds;
580 let (rx, ry, rw, rh) = root;
581
582 let overflow_left = (rx + VISIBILITY_PADDING - x).max(0.0);
583 let overflow_right = (x + w - (rx + rw - VISIBILITY_PADDING)).max(0.0);
584 let overflow_top = (ry + VISIBILITY_PADDING - y).max(0.0);
585 let overflow_bottom = (y + h - (ry + rh - VISIBILITY_PADDING)).max(0.0);
586
587 let horizontal_overflow = overflow_left.max(overflow_right);
588 let vertical_overflow = overflow_top.max(overflow_bottom);
589
590 if horizontal_overflow <= 0.0 && vertical_overflow <= 0.0 {
591 return None;
592 }
593
594 if horizontal_overflow >= vertical_overflow {
595 if overflow_left > 0.0 {
596 Some((TabAxis::Horizontal, 1.0))
597 } else {
598 Some((TabAxis::Horizontal, -1.0))
599 }
600 } else if overflow_top > 0.0 {
601 Some((TabAxis::Vertical, 1.0))
602 } else {
603 Some((TabAxis::Vertical, -1.0))
604 }
605}
606
607fn intersects_root(bounds: RectBounds, root: RectBounds) -> bool {
608 let (x, y, w, h) = bounds;
609 let (rx, ry, rw, rh) = root;
610 let left = x.max(rx);
611 let top = y.max(ry);
612 let right = (x + w).min(rx + rw);
613 let bottom = (y + h).min(ry + rh);
614 right > left && bottom > top
615}
616
617fn overlap_len(a_start: f32, a_len: f32, b_start: f32, b_len: f32) -> f32 {
618 let a_end = a_start + a_len;
619 let b_end = b_start + b_len;
620 (a_end.min(b_end) - a_start.max(b_start)).max(0.0)
621}
622
623fn cross_axis_overlap(bounds: RectBounds, target: RectBounds, axis: TabAxis) -> f32 {
624 match axis {
625 TabAxis::Horizontal => overlap_len(bounds.1, bounds.3, target.1, target.3),
626 TabAxis::Vertical => overlap_len(bounds.0, bounds.2, target.0, target.2),
627 }
628}
629
630fn primary_axis_distance(bounds: RectBounds, target: RectBounds, axis: TabAxis) -> f32 {
631 let center = match axis {
632 TabAxis::Horizontal => bounds.0 + bounds.2 / 2.0,
633 TabAxis::Vertical => bounds.1 + bounds.3 / 2.0,
634 };
635 let target_center = match axis {
636 TabAxis::Horizontal => target.0 + target.2 / 2.0,
637 TabAxis::Vertical => target.1 + target.3 / 2.0,
638 };
639 (center - target_center).abs()
640}
641
642fn find_scroll_anchor(
643 elements: &[SemanticElement],
644 target: RectBounds,
645 root: RectBounds,
646 axis: TabAxis,
647) -> Option<RectBounds> {
648 let mut best: Option<(RectBounds, f32, f32)> = None;
649
650 for elem in elements {
651 if elem.clickable {
652 let bounds = (
653 elem.bounds.x,
654 elem.bounds.y,
655 elem.bounds.width,
656 elem.bounds.height,
657 );
658 if is_fully_visible(bounds, root) && intersects_root(bounds, root) {
659 let overlap = cross_axis_overlap(bounds, target, axis);
660 if overlap > 0.0 {
661 let distance = primary_axis_distance(bounds, target, axis);
662 match best {
663 None => best = Some((bounds, overlap, distance)),
664 Some((_, best_overlap, best_distance)) => {
665 let is_better_overlap = overlap > best_overlap + f32::EPSILON;
666 let is_better_distance = (overlap - best_overlap).abs() <= f32::EPSILON
667 && distance < best_distance;
668 if is_better_overlap || is_better_distance {
669 best = Some((bounds, overlap, distance));
670 }
671 }
672 }
673 }
674 }
675 }
676
677 if let Some(found) = find_scroll_anchor(&elem.children, target, root, axis) {
678 let overlap = cross_axis_overlap(found, target, axis);
679 let distance = primary_axis_distance(found, target, axis);
680 match best {
681 None => best = Some((found, overlap, distance)),
682 Some((_, best_overlap, best_distance)) => {
683 let is_better_overlap = overlap > best_overlap + f32::EPSILON;
684 let is_better_distance =
685 (overlap - best_overlap).abs() <= f32::EPSILON && distance < best_distance;
686 if is_better_overlap || is_better_distance {
687 best = Some((found, overlap, distance));
688 }
689 }
690 }
691 }
692 }
693
694 best.map(|(bounds, _, _)| bounds)
695}
696
697pub fn capture_screenshot(robot: &cranpose::Robot) -> Option<cranpose::RobotScreenshot> {
699 robot.screenshot().ok()
700}
701
702pub fn screenshot_pixel(screenshot: &cranpose::RobotScreenshot, x: u32, y: u32) -> Option<[u8; 4]> {
704 if x >= screenshot.width || y >= screenshot.height {
705 return None;
706 }
707 let index = ((y * screenshot.width + x) * 4) as usize;
708 Some([
709 screenshot.pixels[index],
710 screenshot.pixels[index + 1],
711 screenshot.pixels[index + 2],
712 screenshot.pixels[index + 3],
713 ])
714}
715
716pub fn crop_screenshot(
718 screenshot: &cranpose::RobotScreenshot,
719 x: u32,
720 y: u32,
721 width: u32,
722 height: u32,
723) -> Option<cranpose::RobotScreenshot> {
724 if width == 0 || height == 0 {
725 return None;
726 }
727 let end_x = x.checked_add(width)?;
728 let end_y = y.checked_add(height)?;
729 if end_x > screenshot.width || end_y > screenshot.height {
730 return None;
731 }
732
733 let mut pixels = vec![0u8; (width * height * 4) as usize];
734 for row in 0..height {
735 let src_start = (((y + row) * screenshot.width + x) * 4) as usize;
736 let src_end = src_start + (width * 4) as usize;
737 let dst_start = (row * width * 4) as usize;
738 let dst_end = dst_start + (width * 4) as usize;
739 pixels[dst_start..dst_end].copy_from_slice(&screenshot.pixels[src_start..src_end]);
740 }
741
742 Some(cranpose::RobotScreenshot {
743 width,
744 height,
745 pixels,
746 })
747}
748
749pub fn changed_pixel_count(
752 before: &cranpose::RobotScreenshot,
753 after: &cranpose::RobotScreenshot,
754 channel_threshold: u8,
755) -> usize {
756 if before.width != after.width || before.height != after.height {
757 return usize::MAX;
758 }
759
760 before
761 .pixels
762 .chunks_exact(4)
763 .zip(after.pixels.chunks_exact(4))
764 .filter(|(a, b)| {
765 a[0].abs_diff(b[0]) > channel_threshold
766 || a[1].abs_diff(b[1]) > channel_threshold
767 || a[2].abs_diff(b[2]) > channel_threshold
768 || a[3].abs_diff(b[3]) > channel_threshold
769 })
770 .count()
771}
772
773pub fn changed_pixel_count_in_region(
775 before: &cranpose::RobotScreenshot,
776 after: &cranpose::RobotScreenshot,
777 region: (f32, f32, f32, f32),
778 channel_threshold: u8,
779) -> usize {
780 if before.width != after.width || before.height != after.height {
781 return usize::MAX;
782 }
783
784 let left = region.0.max(0.0).floor() as u32;
785 let top = region.1.max(0.0).floor() as u32;
786 let right = (region.0 + region.2).min(before.width as f32).ceil() as u32;
787 let bottom = (region.1 + region.3).min(before.height as f32).ceil() as u32;
788
789 if right <= left || bottom <= top {
790 return 0;
791 }
792
793 let width = before.width as usize;
794 let mut changed = 0usize;
795 for y in top..bottom {
796 for x in left..right {
797 let idx = ((y as usize) * width + x as usize) * 4;
798 if before.pixels[idx].abs_diff(after.pixels[idx]) > channel_threshold
799 || before.pixels[idx + 1].abs_diff(after.pixels[idx + 1]) > channel_threshold
800 || before.pixels[idx + 2].abs_diff(after.pixels[idx + 2]) > channel_threshold
801 || before.pixels[idx + 3].abs_diff(after.pixels[idx + 3]) > channel_threshold
802 {
803 changed += 1;
804 }
805 }
806 }
807
808 changed
809}
810
811pub fn parse_slider_value(text: &str) -> Option<f32> {
813 text.split_once(':')
814 .and_then(|(_, value)| value.trim().parse::<f32>().ok())
815}
816
817pub fn scroll_down(robot: &cranpose::Robot, center_x: f32, from_y: f32, to_y: f32) {
819 let _ = robot.drag(center_x, from_y, center_x, to_y);
820 std::thread::sleep(std::time::Duration::from_millis(180));
821 let _ = robot.wait_for_idle();
822}
823
824pub fn scroll_up(robot: &cranpose::Robot, center_x: f32, from_y: f32, to_y: f32) {
826 let _ = robot.drag(center_x, from_y, center_x, to_y);
827 std::thread::sleep(std::time::Duration::from_millis(180));
828 let _ = robot.wait_for_idle();
829}
830
831pub fn y_is_visible(robot: &cranpose::Robot, y: f32) -> bool {
834 let Some((_, root_y, _, root_h)) = root_bounds(robot) else {
835 return true;
836 };
837 let top = root_y + 28.0;
838 let bottom = root_y + root_h - 28.0;
839 y >= top && y <= bottom
840}
841
842#[derive(Clone, Copy, Debug)]
844pub struct ScrollConfig {
845 pub center_x: f32,
846 pub down_from_y: f32,
847 pub down_to_y: f32,
848 pub up_from_y: f32,
849 pub up_to_y: f32,
850}
851
852pub fn scroll_prefix_into_view(
855 robot: &cranpose::Robot,
856 prefix: &str,
857 max_attempts: usize,
858 cfg: ScrollConfig,
859) -> Option<(f32, f32, f32, f32, String)> {
860 for attempt in 0..max_attempts {
861 if let Some(bounds) = find_text_by_prefix_in_semantics(robot, prefix) {
862 let center_y = bounds.1 + bounds.3 * 0.5;
863 if y_is_visible(robot, center_y) {
864 return Some(bounds);
865 }
866 let Some((_, root_y, _, root_h)) = root_bounds(robot) else {
867 return Some(bounds);
868 };
869 let viewport_mid = root_y + root_h * 0.5;
870 if center_y > viewport_mid {
871 scroll_down(robot, cfg.center_x, cfg.down_from_y, cfg.down_to_y);
872 } else {
873 scroll_up(robot, cfg.center_x, cfg.up_from_y, cfg.up_to_y);
874 }
875 } else {
876 if attempt % 2 == 0 {
878 scroll_down(robot, cfg.center_x, cfg.down_from_y, cfg.down_to_y);
879 } else {
880 scroll_up(robot, cfg.center_x, cfg.up_from_y, cfg.up_to_y);
881 }
882 }
883 }
884 None
885}
886
887pub fn set_slider_fraction(
889 robot: &cranpose::Robot,
890 prefix: &str,
891 fraction: f32,
892 slider_width: f32,
893 slider_touch_offset_y: f32,
894 cfg: ScrollConfig,
895) -> Option<f32> {
896 let (x, y, _w, h, _) = scroll_prefix_into_view(robot, prefix, 18, cfg)?;
897 let slider_y = y + h + slider_touch_offset_y;
898 let left_x = x + 2.0;
899 let target_x = x + slider_width * fraction.clamp(0.0, 1.0);
900 let _ = robot.drag(left_x, slider_y, target_x, slider_y);
901 std::thread::sleep(std::time::Duration::from_millis(120));
902 let _ = robot.wait_for_idle();
903 find_text_by_prefix_in_semantics(robot, prefix)
904 .and_then(|(_, _, _, _, t)| parse_slider_value(&t))
905}
906
907#[cfg(test)]
908mod tests {
909 use super::*;
910 use cranpose::{RobotScreenshot, SemanticRect};
911
912 fn semantic_element(
913 role: &str,
914 text: Option<&str>,
915 clickable: bool,
916 bounds: RectBounds,
917 children: Vec<SemanticElement>,
918 ) -> SemanticElement {
919 SemanticElement {
920 role: role.to_string(),
921 text: text.map(ToString::to_string),
922 clickable,
923 bounds: SemanticRect {
924 x: bounds.0,
925 y: bounds.1,
926 width: bounds.2,
927 height: bounds.3,
928 },
929 children,
930 }
931 }
932
933 #[test]
934 fn overflow_axis_direction_detects_horizontal_overflow() {
935 let root = (0.0, 0.0, 300.0, 200.0);
936 let target = (320.0, 20.0, 80.0, 30.0);
937 assert_eq!(
938 overflow_axis_direction(target, root),
939 Some((TabAxis::Horizontal, -1.0))
940 );
941 }
942
943 #[test]
944 fn overflow_axis_direction_detects_vertical_overflow() {
945 let root = (0.0, 0.0, 300.0, 200.0);
946 let target = (20.0, -60.0, 80.0, 30.0);
947 assert_eq!(
948 overflow_axis_direction(target, root),
949 Some((TabAxis::Vertical, 1.0))
950 );
951 }
952
953 #[test]
954 fn find_scroll_anchor_prefers_cross_axis_overlap() {
955 let root = (0.0, 0.0, 300.0, 200.0);
956 let target = (320.0, 22.0, 80.0, 28.0);
957
958 let same_row = semantic_element(
959 "Layout",
960 Some("same-row"),
961 true,
962 (120.0, 20.0, 80.0, 30.0),
963 vec![],
964 );
965 let other_row = semantic_element(
966 "Layout",
967 Some("other-row"),
968 true,
969 (120.0, 130.0, 80.0, 30.0),
970 vec![],
971 );
972 let root_elem = semantic_element("Layout", None, false, root, vec![same_row, other_row]);
973
974 let anchor = find_scroll_anchor(&[root_elem], target, root, TabAxis::Horizontal)
975 .expect("expected anchor");
976
977 assert_eq!(anchor, (120.0, 20.0, 80.0, 30.0));
978 }
979
980 #[test]
981 fn screenshot_pixel_reads_expected_value() {
982 let screenshot = RobotScreenshot {
983 width: 2,
984 height: 1,
985 pixels: vec![1, 2, 3, 4, 5, 6, 7, 8],
986 };
987 assert_eq!(screenshot_pixel(&screenshot, 1, 0), Some([5, 6, 7, 8]));
988 }
989
990 #[test]
991 fn crop_screenshot_extracts_region() {
992 let screenshot = RobotScreenshot {
993 width: 3,
994 height: 2,
995 pixels: vec![
996 1, 2, 3, 255, 4, 5, 6, 255, 7, 8, 9, 255, 10, 11, 12, 255, 13, 14, 15, 255, 16, 17,
997 18, 255,
998 ],
999 };
1000 let cropped = crop_screenshot(&screenshot, 1, 0, 2, 2).expect("crop");
1001 assert_eq!(cropped.width, 2);
1002 assert_eq!(cropped.height, 2);
1003 assert_eq!(
1004 cropped.pixels,
1005 vec![4, 5, 6, 255, 7, 8, 9, 255, 13, 14, 15, 255, 16, 17, 18, 255]
1006 );
1007 }
1008}