1use std::collections::VecDeque;
23use std::path::PathBuf;
24use std::sync::Mutex;
25use std::sync::atomic::{AtomicU64, Ordering};
26
27use anyhow::{Context, Result};
28use base64::{Engine as _, engine::general_purpose};
29use tokio::process::Command;
30use tokio_util::sync::CancellationToken;
31
32use mermaid_model::constants::{SCREENSHOT_MAX_WIDTH, SCREENSHOT_REGISTRY_CAPACITY};
33
34use super::Backend;
35
36#[derive(Debug, Clone)]
39pub struct ScreenshotMetadata {
40 pub id: u64,
41 pub scale_factor: f64,
42 pub offset_x: i32,
43 pub offset_y: i32,
44 pub width: u32,
49 pub height: u32,
50 pub kind: String,
54}
55
56#[derive(Debug, Default)]
62pub struct ScreenshotRegistry {
63 entries: VecDeque<ScreenshotMetadata>,
64}
65
66impl ScreenshotRegistry {
67 #[must_use]
68 pub fn new() -> Self {
69 Self {
70 entries: VecDeque::new(),
71 }
72 }
73
74 pub fn push(&mut self, meta: ScreenshotMetadata) {
75 if self.entries.len() >= SCREENSHOT_REGISTRY_CAPACITY {
76 self.entries.pop_front();
77 }
78 self.entries.push_back(meta);
79 }
80
81 #[must_use]
82 pub fn get(&self, id: u64) -> Option<&ScreenshotMetadata> {
83 self.entries.iter().find(|m| m.id == id)
84 }
85
86 #[must_use]
87 pub fn latest(&self) -> Option<&ScreenshotMetadata> {
88 self.entries.back()
89 }
90
91 #[must_use]
92 pub fn len(&self) -> usize {
93 self.entries.len()
94 }
95
96 #[must_use]
97 pub fn is_empty(&self) -> bool {
98 self.entries.is_empty()
99 }
100}
101
102#[derive(Debug, Clone)]
105pub enum ScreenshotSpec {
106 Fullscreen,
107 Focused,
108 Monitor(String),
109 Region(i32, i32, u32, u32),
111 Window(String),
112}
113
114#[derive(Debug)]
117pub struct CaptureResult {
118 pub id: u64,
119 pub base64_png: String,
120 pub raw_bytes: Vec<u8>,
121 pub width: u32,
122 pub height: u32,
123 pub scale_factor: f64,
124 pub offset_x: i32,
125 pub offset_y: i32,
126 pub summary: String,
127}
128
129pub struct ComputerUseDriver {
131 backend: Backend,
132 registry: Mutex<ScreenshotRegistry>,
133 file_counter: AtomicU64,
137 id_counter: AtomicU64,
140}
141
142impl ComputerUseDriver {
143 #[must_use]
144 pub fn new(backend: Backend) -> Self {
145 Self {
146 backend,
147 registry: Mutex::new(ScreenshotRegistry::new()),
148 file_counter: AtomicU64::new(0),
149 id_counter: AtomicU64::new(0),
150 }
151 }
152
153 pub fn backend(&self) -> Backend {
154 self.backend
155 }
156
157 pub fn ensure_alive(&self) -> Result<(), String> {
168 if super::display_is_reachable(self.backend) {
169 Ok(())
170 } else {
171 Err(format!(
172 "Display unreachable (backend={:?}). Was the session \
173 detached, or did `DISPLAY` change?",
174 self.backend
175 ))
176 }
177 }
178
179 pub async fn ensure_alive_async(&self) -> Result<(), String> {
189 let backend = self.backend;
190 match tokio::task::spawn_blocking(move || super::display_is_reachable(backend)).await {
191 Ok(true) => Ok(()),
192 Ok(false) => Err(format!(
193 "Display unreachable (backend={:?}). Was the session \
194 detached, or did `DISPLAY` change?",
195 self.backend
196 )),
197 Err(_) => Err("display liveness probe failed to run".to_string()),
198 }
199 }
200
201 pub fn scale_coords(
212 &self,
213 x: i32,
214 y: i32,
215 screenshot_id: Option<u64>,
216 ) -> Result<(i32, i32), String> {
217 let reg = self.registry.lock().map_err(|e| e.to_string())?;
218 let meta = match screenshot_id {
219 Some(id) => reg.get(id).cloned().ok_or_else(|| {
220 format!(
221 "Screenshot id {id} not found in registry (likely evicted — capacity {SCREENSHOT_REGISTRY_CAPACITY}). \
222 Take a fresh screenshot and retry with the new id."
223 )
224 })?,
225 None => reg.latest().cloned().ok_or_else(|| {
226 "No screenshots registered yet — call `screenshot` before \
227 `click` / `mouse_move`."
228 .to_string()
229 })?,
230 };
231 let cx = if meta.width > 0 {
235 x.clamp(0, meta.width as i32 - 1)
236 } else {
237 x.max(0)
238 };
239 let cy = if meta.height > 0 {
240 y.clamp(0, meta.height as i32 - 1)
241 } else {
242 y.max(0)
243 };
244 Ok((
251 ((cx as f64 * meta.scale_factor) as i32).saturating_add(meta.offset_x),
252 ((cy as f64 * meta.scale_factor) as i32).saturating_add(meta.offset_y),
253 ))
254 }
255
256 pub fn register_screenshot(
258 &self,
259 scale_factor: f64,
260 offset_x: i32,
261 offset_y: i32,
262 width: u32,
263 height: u32,
264 kind: String,
265 ) -> u64 {
266 let id = self.id_counter.fetch_add(1, Ordering::Relaxed);
267 if let Ok(mut reg) = self.registry.lock() {
268 reg.push(ScreenshotMetadata {
269 id,
270 scale_factor,
271 offset_x,
272 offset_y,
273 width,
274 height,
275 kind,
276 });
277 }
278 id
279 }
280
281 pub async fn capture(
292 &self,
293 spec: ScreenshotSpec,
294 token: &CancellationToken,
295 ) -> Result<CaptureResult> {
296 self.ensure_alive_async()
297 .await
298 .map_err(|error| anyhow::anyhow!(error))?;
299
300 let seq = self.file_counter.fetch_add(1, Ordering::Relaxed);
301 let temp_path =
305 mermaid_model::utils::private_temp_dir()?.join(format!("mermaid-screenshot-{seq}.png"));
306 let temp_str = temp_path.to_string_lossy().to_string();
307 let _guard = TempFileGuard(temp_path.clone());
308
309 let (offset_x, offset_y, kind) =
310 dispatch_capture(self.backend, &spec, &temp_str, token).await?;
311
312 let scale_factor =
319 cancellable(token, downscale_if_needed(&temp_str, SCREENSHOT_MAX_WIDTH)).await?;
320
321 let raw_bytes = cancellable(token, async {
322 tokio::fs::read(&temp_path)
323 .await
324 .context("reading captured screenshot")
325 })
326 .await?;
327 let width = read_png_width(&raw_bytes).unwrap_or(0);
328 let height = read_png_height(&raw_bytes).unwrap_or(0);
329
330 let id = self.register_screenshot(
333 scale_factor,
334 offset_x,
335 offset_y,
336 width,
337 height,
338 kind.clone(),
339 );
340
341 let base64_png = general_purpose::STANDARD.encode(&raw_bytes);
342
343 let offset_info = if offset_x != 0 || offset_y != 0 {
344 format!(", offset: +{offset_x}+{offset_y}")
345 } else {
346 String::new()
347 };
348 let summary = format!(
349 "Screenshot captured (id: {id}, {kind}, {width}x{height}, scale: {scale_factor:.2}x{offset_info})"
350 );
351
352 Ok(CaptureResult {
353 id,
354 base64_png,
355 raw_bytes,
356 width,
357 height,
358 scale_factor,
359 offset_x,
360 offset_y,
361 summary,
362 })
363 }
364
365 pub async fn capture_focused_for_autoshot(
371 &self,
372 token: &CancellationToken,
373 ) -> Option<(String, String)> {
374 let cap = self.capture(ScreenshotSpec::Focused, token).await.ok()?;
375 Some((cap.summary, cap.base64_png))
376 }
377
378 pub async fn check_cursor_landed(&self, sx: i32, sy: i32) -> Option<String> {
385 if !matches!(self.backend, Backend::X11) {
386 return None;
387 }
388 let out = run_cmd_stdout(Command::new("xdotool").arg("getmouselocation"))
389 .await
390 .ok()?;
391 let mut actual_x: Option<i32> = None;
392 let mut actual_y: Option<i32> = None;
393 for tok in out.split_whitespace() {
394 if let Some(v) = tok.strip_prefix("X:") {
395 actual_x = v.parse().ok();
396 } else if let Some(v) = tok.strip_prefix("Y:") {
397 actual_y = v.parse().ok();
398 }
399 }
400 let (ax, ay) = (actual_x?, actual_y?);
401 if (ax - sx).abs() > CURSOR_LANDED_TOLERANCE_PX
402 || (ay - sy).abs() > CURSOR_LANDED_TOLERANCE_PX
403 {
404 Some(format!(
405 "WARNING: cursor at ({ax}, {ay}), expected ({sx}, {sy}). Window may have moved \
406 or focus changed before the click landed."
407 ))
408 } else {
409 None
410 }
411 }
412}
413
414const CURSOR_LANDED_TOLERANCE_PX: i32 = 5;
417
418impl ComputerUseDriver {
421 pub async fn click(
431 &self,
432 sx: i32,
433 sy: i32,
434 button: &str,
435 token: &CancellationToken,
436 ) -> Result<()> {
437 let code = match button {
438 "middle" => "2",
439 "right" => "3",
440 _ => "1",
441 };
442 match self.backend {
443 Backend::X11 => {
444 run_cmd_cancellable(
445 Command::new("xdotool").args([
446 "mousemove",
447 "--sync",
448 &sx.to_string(),
449 &sy.to_string(),
450 "click",
451 "--clearmodifiers",
452 code,
453 ]),
454 token,
455 )
456 .await
457 },
458 Backend::Wayland => {
459 if !super::has_command("ydotool") {
460 anyhow::bail!("ydotool required for Wayland mouse control")
461 }
462 run_cmd_cancellable(
463 Command::new("ydotool").args([
464 "mousemove",
465 "--absolute",
466 "-x",
467 &sx.to_string(),
468 "-y",
469 &sy.to_string(),
470 ]),
471 token,
472 )
473 .await?;
474 run_cmd_cancellable(
475 Command::new("ydotool").args(["click", &format!("0x{code}")]),
476 token,
477 )
478 .await
479 },
480 _ => anyhow::bail!("click not supported on this platform"),
481 }
482 }
483
484 pub async fn type_text(&self, text: &str, token: &CancellationToken) -> Result<()> {
495 let delay = mermaid_model::constants::TYPE_KEY_DELAY_MS.to_string();
496 match self.backend {
497 Backend::X11 => {
498 run_cmd_cancellable(
499 Command::new("xdotool").args([
500 "type",
501 "--clearmodifiers",
502 "--delay",
503 &delay,
504 text,
505 ]),
506 token,
507 )
508 .await
509 },
510 Backend::Wayland => {
511 if super::has_command("wtype") {
512 run_cmd_cancellable(Command::new("wtype").arg(text), token).await
513 } else if super::has_command("ydotool") {
514 run_cmd_cancellable(
515 Command::new("ydotool").args(["type", "--delay", &delay, text]),
516 token,
517 )
518 .await
519 } else {
520 anyhow::bail!("wtype or ydotool required for Wayland text input")
521 }
522 },
523 _ => anyhow::bail!("type_text not supported on this platform"),
524 }
525 }
526
527 pub async fn press_key(&self, key: &str, token: &CancellationToken) -> Result<()> {
536 match self.backend {
537 Backend::X11 => {
538 run_cmd_cancellable(Command::new("xdotool").args(["key", key]), token).await
539 },
540 Backend::Wayland => {
541 if super::has_command("wtype") {
542 let parts: Vec<&str> = key.split('+').collect();
544 let mut args: Vec<String> = Vec::new();
545 for (i, part) in parts.iter().enumerate() {
546 if i < parts.len() - 1 {
547 args.push("-M".to_string());
548 args.push(part.to_string());
549 } else {
550 args.push("-k".to_string());
551 args.push(part.to_string());
552 }
553 }
554 for part in parts.iter().take(parts.len().saturating_sub(1)) {
555 args.push("-m".to_string());
556 args.push(part.to_string());
557 }
558 run_cmd_cancellable(Command::new("wtype").args(&args), token).await
559 } else if super::has_command("ydotool") {
560 run_cmd_cancellable(Command::new("ydotool").args(["key", key]), token).await
561 } else {
562 anyhow::bail!("wtype or ydotool required for Wayland key input")
563 }
564 },
565 _ => anyhow::bail!("press_key not supported on this platform"),
566 }
567 }
568
569 pub async fn scroll(
578 &self,
579 direction: &str,
580 amount: i32,
581 token: &CancellationToken,
582 ) -> Result<()> {
583 match self.backend {
584 Backend::X11 => {
585 let button = if direction == "up" { "4" } else { "5" };
587 let mut args: Vec<String> = Vec::new();
588 for _ in 0..amount {
589 args.push("click".to_string());
590 args.push(button.to_string());
591 }
592 run_cmd_cancellable(Command::new("xdotool").args(&args), token).await
593 },
594 Backend::Wayland => {
595 if !super::has_command("ydotool") {
596 anyhow::bail!("ydotool required for Wayland scroll")
597 }
598 let wheel_amount = if direction == "up" { -amount } else { amount };
599 run_cmd_cancellable(
600 Command::new("ydotool").args([
601 "mousemove",
602 "--wheel",
603 &wheel_amount.to_string(),
604 ]),
605 token,
606 )
607 .await
608 },
609 _ => anyhow::bail!("scroll not supported on this platform"),
610 }
611 }
612
613 pub async fn mouse_move(&self, sx: i32, sy: i32, token: &CancellationToken) -> Result<()> {
622 match self.backend {
623 Backend::X11 => {
624 run_cmd_cancellable(
625 Command::new("xdotool").args([
626 "mousemove",
627 "--sync",
628 &sx.to_string(),
629 &sy.to_string(),
630 ]),
631 token,
632 )
633 .await
634 },
635 Backend::Wayland => {
636 if !super::has_command("ydotool") {
637 anyhow::bail!("ydotool required for Wayland mouse control")
638 }
639 run_cmd_cancellable(
640 Command::new("ydotool").args([
641 "mousemove",
642 "--absolute",
643 "-x",
644 &sx.to_string(),
645 "-y",
646 &sy.to_string(),
647 ]),
648 token,
649 )
650 .await
651 },
652 _ => anyhow::bail!("mouse_move not supported on this platform"),
653 }
654 }
655
656 pub async fn list_windows(&self, _token: &CancellationToken) -> Result<Vec<String>> {
666 if !matches!(self.backend, Backend::X11) {
667 anyhow::bail!(
668 "list_windows requires X11. Wayland has no portable window-enumeration \
669 primitive. Run mermaid from an X11 session."
670 );
671 }
672 let wids =
673 run_cmd_stdout(Command::new("xdotool").args(["search", "--onlyvisible", "--name", ""]))
674 .await?;
675 let mut windows = Vec::new();
676 for wid in wids.lines() {
677 let wid = wid.trim();
678 if wid.is_empty() {
679 continue;
680 }
681 if let Ok(name) =
682 run_cmd_stdout(Command::new("xdotool").args(["getwindowname", wid])).await
683 {
684 let name = name.trim().to_string();
685 if !name.is_empty() && !windows.contains(&name) {
686 windows.push(name);
687 }
688 }
689 }
690 Ok(windows)
691 }
692}
693
694struct TempFileGuard(PathBuf);
697
698impl Drop for TempFileGuard {
699 fn drop(&mut self) {
700 let _ = std::fs::remove_file(&self.0);
701 }
702}
703
704#[expect(
707 clippy::too_many_lines,
708 reason = "predates the lint; see .github/baselines/expect_budget.txt"
709)]
710async fn dispatch_capture(
711 backend: Backend,
712 spec: &ScreenshotSpec,
713 out_path: &str,
714 token: &CancellationToken,
715) -> Result<(i32, i32, String)> {
716 match (backend, spec) {
720 (Backend::X11, ScreenshotSpec::Fullscreen) => {
721 run_cmd_cancellable(Command::new("scrot").args(["-o", out_path]), token).await?;
722 Ok((0, 0, "fullscreen".to_string()))
723 },
724 (Backend::Wayland, ScreenshotSpec::Fullscreen) => {
725 run_cmd_cancellable(Command::new("grim").arg(out_path), token).await?;
726 Ok((0, 0, "fullscreen".to_string()))
727 },
728 (Backend::MacOS, ScreenshotSpec::Fullscreen) => {
729 run_cmd_cancellable(Command::new("screencapture").args(["-x", out_path]), token)
730 .await?;
731 Ok((0, 0, "fullscreen".to_string()))
732 },
733 (Backend::X11, ScreenshotSpec::Focused) => {
734 let (wx, wy) = get_focused_window_geometry_x11()
735 .await
736 .map(|(x, y, _, _)| (x, y))
737 .unwrap_or((0, 0));
738 run_cmd_cancellable(Command::new("scrot").args(["-u", "-o", out_path]), token).await?;
739 Ok((wx, wy, "focused window".to_string()))
740 },
741 (Backend::Wayland, ScreenshotSpec::Focused) => anyhow::bail!(
742 "Mode 'focused' not supported on Wayland (grim has no focused-window \
743 primitive). Use mode: 'fullscreen' or mode: 'monitor' with a specific \
744 output name."
745 ),
746 (Backend::MacOS, ScreenshotSpec::Focused) => {
747 run_cmd_cancellable(Command::new("screencapture").args(["-x", out_path]), token)
756 .await?;
757 Ok((0, 0, "focused window (full display on macOS)".to_string()))
758 },
759 (Backend::X11, ScreenshotSpec::Region(x, y, w, h)) => {
760 run_cmd_cancellable(
761 Command::new("scrot").args(["-a", &format!("{x},{y},{w},{h}"), "-o", out_path]),
762 token,
763 )
764 .await?;
765 Ok((*x, *y, format!("region {w}x{h}+{x}+{y}")))
766 },
767 (Backend::Wayland, ScreenshotSpec::Region(x, y, w, h)) => {
768 run_cmd_cancellable(
769 Command::new("grim").args(["-g", &format!("{x},{y} {w}x{h}"), out_path]),
770 token,
771 )
772 .await?;
773 Ok((*x, *y, format!("region {w}x{h}+{x}+{y}")))
774 },
775 (Backend::X11, ScreenshotSpec::Monitor(name)) => {
776 let (mx, my, mw, mh) = parse_monitor_geometry_x11(name).await.ok_or_else(|| {
777 anyhow::anyhow!("Monitor '{name}' not found. Run `xrandr --query` to list outputs.")
778 })?;
779 run_cmd_cancellable(
780 Command::new("scrot").args(["-a", &format!("{mx},{my},{mw},{mh}"), "-o", out_path]),
781 token,
782 )
783 .await?;
784 Ok((mx, my, format!("monitor {name}")))
785 },
786 (Backend::Wayland, ScreenshotSpec::Monitor(name)) => {
787 run_cmd_cancellable(Command::new("grim").args(["-o", name, out_path]), token).await?;
788 Ok((0, 0, format!("monitor {name}")))
789 },
790 (Backend::X11, ScreenshotSpec::Window(title)) => {
791 let wid = run_cmd_stdout(Command::new("xdotool").args(["search", "--name", title]))
794 .await?
795 .lines()
796 .next()
797 .map(str::trim)
798 .filter(|s| !s.is_empty())
799 .map(str::to_string)
800 .ok_or_else(|| {
801 anyhow::anyhow!(
802 "No window found matching '{title}'. Use list_windows to see available \
803 windows."
804 )
805 })?;
806 run_cmd_cancellable(
807 Command::new("xdotool").args(["windowactivate", "--sync", &wid]),
808 token,
809 )
810 .await?;
811 tokio::time::sleep(std::time::Duration::from_millis(
812 mermaid_model::constants::WINDOW_FOCUS_DELAY_MS,
813 ))
814 .await;
815 let (wx, wy) = get_window_geometry_x11(&wid)
816 .await
817 .map(|(x, y, _, _)| (x, y))
818 .unwrap_or((0, 0));
819 run_cmd_cancellable(Command::new("scrot").args(["-u", "-o", out_path]), token).await?;
820 Ok((wx, wy, format!("window \"{title}\"")))
821 },
822 (Backend::Wayland, ScreenshotSpec::Window(_)) => anyhow::bail!(
823 "Mode 'window' not supported on Wayland (grim has no window-by-name capture). \
824 Use mode: 'fullscreen' or mode: 'monitor' with a specific output name."
825 ),
826 (Backend::MacOS, _) => anyhow::bail!(
827 "This screenshot mode is not yet ported to macOS. Use mode: 'fullscreen' for now."
828 ),
829 (Backend::Windows, _) | (Backend::Unsupported, _) => {
830 anyhow::bail!("Unsupported platform for computer-use capture")
831 },
832 }
833}
834
835async fn cancellable<F, T>(token: &CancellationToken, fut: F) -> Result<T>
843where
844 F: std::future::Future<Output = Result<T>>,
845{
846 tokio::select! {
847 biased;
848 _ = token.cancelled() => anyhow::bail!("cancelled"),
849 r = fut => r,
850 }
851}
852
853pub(crate) async fn run_cmd_cancellable(
857 cmd: &mut Command,
858 token: &CancellationToken,
859) -> Result<()> {
860 run_cmd_cancellable_with_timeout(
861 cmd,
862 token,
863 std::time::Duration::from_secs(mermaid_model::constants::COMPUTER_USE_CMD_TIMEOUT_SECS),
864 )
865 .await
866}
867
868async fn run_cmd_cancellable_with_timeout(
873 cmd: &mut Command,
874 token: &CancellationToken,
875 timeout: std::time::Duration,
876) -> Result<()> {
877 cmd.kill_on_drop(true);
878 tokio::select! {
879 biased;
880 _ = token.cancelled() => anyhow::bail!("cancelled"),
881 _ = tokio::time::sleep(timeout) => {
882 anyhow::bail!("subprocess timed out after {timeout:?}")
883 }
884 res = cmd.output() => {
885 let out = res.context("subprocess spawn")?;
886 if !out.status.success() {
887 anyhow::bail!(
888 "subprocess failed: {}",
889 String::from_utf8_lossy(&out.stderr).trim()
890 );
891 }
892 Ok(())
893 }
894 }
895}
896
897async fn run_cmd_stdout(cmd: &mut Command) -> Result<String> {
898 run_cmd_stdout_with_timeout(
899 cmd,
900 std::time::Duration::from_secs(mermaid_model::constants::COMPUTER_USE_CMD_TIMEOUT_SECS),
901 )
902 .await
903}
904
905async fn run_cmd_stdout_with_timeout(
909 cmd: &mut Command,
910 timeout: std::time::Duration,
911) -> Result<String> {
912 cmd.kill_on_drop(true);
913 let out = match tokio::time::timeout(timeout, cmd.output()).await {
914 Ok(res) => res.context("subprocess spawn")?,
915 Err(_) => anyhow::bail!("subprocess timed out after {timeout:?}"),
916 };
917 if !out.status.success() {
918 anyhow::bail!(
919 "subprocess failed: {}",
920 String::from_utf8_lossy(&out.stderr).trim()
921 );
922 }
923 Ok(String::from_utf8_lossy(&out.stdout).to_string())
924}
925
926async fn get_focused_window_geometry_x11() -> Option<(i32, i32, u32, u32)> {
929 let wid = run_cmd_stdout(Command::new("xdotool").arg("getactivewindow"))
930 .await
931 .ok()?;
932 let wid = wid.trim();
933 if wid.is_empty() {
934 return None;
935 }
936 get_window_geometry_x11(wid).await
937}
938
939async fn get_window_geometry_x11(wid: &str) -> Option<(i32, i32, u32, u32)> {
940 let out = run_cmd_stdout(Command::new("xdotool").args(["getwindowgeometry", "--shell", wid]))
941 .await
942 .ok()?;
943 let mut x = None;
944 let mut y = None;
945 let mut width = None;
946 let mut height = None;
947 for line in out.lines() {
948 if let Some(v) = line.strip_prefix("X=") {
949 x = v.parse().ok();
950 } else if let Some(v) = line.strip_prefix("Y=") {
951 y = v.parse().ok();
952 } else if let Some(v) = line.strip_prefix("WIDTH=") {
953 width = v.parse().ok();
954 } else if let Some(v) = line.strip_prefix("HEIGHT=") {
955 height = v.parse().ok();
956 }
957 }
958 Some((x?, y?, width?, height?))
959}
960
961async fn parse_monitor_geometry_x11(name: &str) -> Option<(i32, i32, u32, u32)> {
962 let out = run_cmd_stdout(Command::new("xrandr").arg("--query"))
963 .await
964 .ok()?;
965 out.lines()
966 .find_map(|line| parse_xrandr_monitor_line(line, name))
967}
968
969fn parse_xrandr_monitor_line(line: &str, name: &str) -> Option<(i32, i32, u32, u32)> {
973 if !line.contains(" connected") {
974 return None;
975 }
976 let parts: Vec<&str> = line.split_whitespace().collect();
977 if parts.first() != Some(&name) {
978 return None;
979 }
980 for part in parts.get(2..).unwrap_or(&[]) {
987 if let Some((res, offsets)) = part.split_once('+')
988 && let Some((w, h)) = res.split_once('x')
989 {
990 let width = w.parse::<u32>().ok()?;
991 let height = h.parse::<u32>().ok()?;
992 let mut off = offsets.splitn(2, '+');
993 let x = off.next()?.parse::<i32>().ok()?;
994 let y = off.next()?.parse::<i32>().ok()?;
995 return Some((x, y, width, height));
996 }
997 }
998 None
999}
1000
1001fn read_png_width(bytes: &[u8]) -> Option<u32> {
1004 if bytes.len() > 24 && &bytes[0..8] == b"\x89PNG\r\n\x1a\n" {
1005 Some(u32::from_be_bytes([
1006 bytes[16], bytes[17], bytes[18], bytes[19],
1007 ]))
1008 } else {
1009 None
1010 }
1011}
1012
1013fn read_png_height(bytes: &[u8]) -> Option<u32> {
1014 if bytes.len() > 28 && &bytes[0..8] == b"\x89PNG\r\n\x1a\n" {
1015 Some(u32::from_be_bytes([
1016 bytes[20], bytes[21], bytes[22], bytes[23],
1017 ]))
1018 } else {
1019 None
1020 }
1021}
1022
1023async fn downscale_if_needed(path: &str, max_width: u32) -> Result<f64> {
1028 let bytes = tokio::fs::read(path).await?;
1029 let original_width = read_png_width(&bytes).unwrap_or(1920);
1030 if original_width <= max_width {
1031 return Ok(1.0);
1032 }
1033 let scale_factor = original_width as f64 / max_width as f64;
1034 let scaled = format!("{path}.scaled.png");
1035 let _scaled_guard = TempFileGuard(PathBuf::from(&scaled));
1041 let downscale_timeout =
1046 std::time::Duration::from_secs(mermaid_model::constants::SCREENSHOT_DOWNSCALE_TIMEOUT_SECS);
1047
1048 let convert = tokio::time::timeout(
1049 downscale_timeout,
1050 Command::new("convert")
1051 .args([path, "-resize", &format!("{max_width}x"), &scaled])
1052 .kill_on_drop(true)
1053 .output(),
1054 )
1055 .await;
1056 if let Ok(Ok(o)) = convert
1057 && o.status.success()
1058 {
1059 tokio::fs::rename(&scaled, path).await?;
1060 return Ok(scale_factor);
1061 }
1062
1063 let ffmpeg = tokio::time::timeout(
1064 downscale_timeout,
1065 Command::new("ffmpeg")
1066 .args([
1067 "-y",
1068 "-i",
1069 path,
1070 "-vf",
1071 &format!("scale={max_width}:-1"),
1072 &scaled,
1073 ])
1074 .kill_on_drop(true)
1075 .output(),
1076 )
1077 .await;
1078 if let Ok(Ok(o)) = ffmpeg
1079 && o.status.success()
1080 {
1081 tokio::fs::rename(&scaled, path).await?;
1082 return Ok(scale_factor);
1083 }
1084
1085 tracing::warn!(
1089 original_width,
1090 "neither ImageMagick nor ffmpeg available; sending full-resolution screenshot"
1091 );
1092 Ok(1.0)
1093}
1094
1095#[cfg(test)]
1096mod tests {
1097 use super::*;
1098
1099 #[test]
1100 fn registry_lru_evicts_oldest_past_capacity() {
1101 let mut r = ScreenshotRegistry::new();
1102 for i in 0..(SCREENSHOT_REGISTRY_CAPACITY as u64 + 3) {
1103 r.push(ScreenshotMetadata {
1104 id: i,
1105 scale_factor: 1.0,
1106 offset_x: 0,
1107 offset_y: 0,
1108 width: 0,
1109 height: 0,
1110 kind: "fullscreen".to_string(),
1111 });
1112 }
1113 assert_eq!(r.len(), SCREENSHOT_REGISTRY_CAPACITY);
1114 assert!(r.get(0).is_none());
1116 assert!(r.get(1).is_none());
1117 assert!(r.get(2).is_none());
1118 assert_eq!(
1120 r.latest().unwrap().id,
1121 SCREENSHOT_REGISTRY_CAPACITY as u64 + 2
1122 );
1123 }
1124
1125 #[test]
1126 fn scale_coords_applies_scale_and_offset() {
1127 let d = ComputerUseDriver::new(Backend::X11);
1128 let id = d.register_screenshot(2.0, 100, 50, 640, 480, "fullscreen".to_string());
1129 let (sx, sy) = d.scale_coords(10, 20, Some(id)).unwrap();
1130 assert_eq!(sx, 100 + 20);
1131 assert_eq!(sy, 50 + 40);
1132 }
1133
1134 #[test]
1135 fn scale_coords_saturates_on_offset_overflow() {
1136 let d = ComputerUseDriver::new(Backend::X11);
1140 let id = d.register_screenshot(1.0, 100, 100, 0, 0, "fullscreen".to_string());
1141 let (sx, sy) = d.scale_coords(i32::MAX, i32::MAX, Some(id)).unwrap();
1142 assert_eq!(sx, i32::MAX);
1143 assert_eq!(sy, i32::MAX);
1144 }
1145
1146 #[test]
1147 fn scale_coords_clamps_negative_into_region() {
1148 let d = ComputerUseDriver::new(Backend::X11);
1150 let id = d.register_screenshot(2.0, 100, 50, 640, 480, "region".to_string());
1151 assert_eq!(d.scale_coords(-9999, -1, Some(id)).unwrap(), (100, 50));
1152 }
1153
1154 #[test]
1155 fn scale_coords_clamps_over_max_into_region() {
1156 let d = ComputerUseDriver::new(Backend::X11);
1159 let id = d.register_screenshot(2.0, 100, 50, 640, 480, "region".to_string());
1160 let (sx, sy) = d.scale_coords(100_000, 100_000, Some(id)).unwrap();
1161 assert_eq!((sx, sy), (1378, 1008));
1163 assert!(sx < 100 + 1280 && sy < 50 + 960);
1165 }
1166
1167 #[test]
1168 fn scale_coords_errors_on_evicted_id() {
1169 let d = ComputerUseDriver::new(Backend::X11);
1170 for _ in 0..(SCREENSHOT_REGISTRY_CAPACITY + 1) {
1171 d.register_screenshot(1.0, 0, 0, 0, 0, "fullscreen".to_string());
1172 }
1173 let err = d.scale_coords(0, 0, Some(0)).unwrap_err();
1175 assert!(
1176 err.contains("evicted"),
1177 "expected eviction message, got: {err}"
1178 );
1179 }
1180
1181 #[test]
1182 fn scale_coords_errors_with_no_screenshots_yet() {
1183 let d = ComputerUseDriver::new(Backend::X11);
1184 let err = d.scale_coords(10, 20, None).unwrap_err();
1185 assert!(err.contains("No screenshots"));
1186 }
1187
1188 #[test]
1189 fn ensure_alive_fails_on_unsupported_backend() {
1190 let d = ComputerUseDriver::new(Backend::Unsupported);
1191 assert!(d.ensure_alive().is_err());
1192 }
1193
1194 #[cfg(unix)]
1195 #[tokio::test]
1196 async fn run_cmd_stdout_times_out_on_slow_command() {
1197 let mut cmd = Command::new("sleep");
1200 cmd.arg("5");
1201 let err = run_cmd_stdout_with_timeout(&mut cmd, std::time::Duration::from_millis(50))
1202 .await
1203 .unwrap_err();
1204 assert!(err.to_string().contains("timed out"), "got: {err}");
1205 }
1206
1207 #[cfg(unix)]
1208 #[tokio::test]
1209 async fn run_cmd_stdout_returns_output_for_fast_command() {
1210 let mut cmd = Command::new("echo");
1211 cmd.arg("hi");
1212 assert_eq!(run_cmd_stdout(&mut cmd).await.unwrap().trim(), "hi");
1213 }
1214
1215 #[cfg(unix)]
1216 #[tokio::test]
1217 async fn run_cmd_cancellable_times_out_on_wedged_backend() {
1218 let token = tokio_util::sync::CancellationToken::new();
1221 let mut cmd = Command::new("sleep");
1222 cmd.arg("5");
1223 let err = run_cmd_cancellable_with_timeout(
1224 &mut cmd,
1225 &token,
1226 std::time::Duration::from_millis(50),
1227 )
1228 .await
1229 .unwrap_err();
1230 assert!(err.to_string().contains("timed out"), "got: {err}");
1231 }
1232
1233 #[test]
1236 fn parse_xrandr_monitor_line_short_line_does_not_panic() {
1237 assert_eq!(parse_xrandr_monitor_line(" connected", "connected"), None);
1243 assert_eq!(
1245 parse_xrandr_monitor_line("HDMI-1 connected", "HDMI-1"),
1246 None
1247 );
1248 }
1249
1250 #[test]
1251 fn parse_xrandr_monitor_line_parses_geometry_and_skips_others() {
1252 let line = "HDMI-1 connected primary 2560x1440+1920+0 \
1253 (normal left inverted right) 597mm x 336mm";
1254 assert_eq!(
1255 parse_xrandr_monitor_line(line, "HDMI-1"),
1256 Some((1920, 0, 2560, 1440))
1257 );
1258 assert_eq!(parse_xrandr_monitor_line(line, "DP-2"), None);
1260 assert_eq!(
1263 parse_xrandr_monitor_line("DP-3 disconnected (normal left inverted right)", "DP-3"),
1264 None
1265 );
1266 }
1267
1268 #[test]
1271 fn temp_file_guard_removes_scaled_sibling_on_drop() {
1272 let scaled = std::env::temp_dir().join(format!(
1278 "mermaid-f58-guard-{}.png.scaled.png",
1279 std::process::id()
1280 ));
1281 std::fs::write(&scaled, b"x").unwrap();
1282 assert!(scaled.exists());
1283 {
1284 let _guard = TempFileGuard(scaled.clone());
1285 }
1286 assert!(
1287 !scaled.exists(),
1288 "scaled sibling must be removed when its guard drops"
1289 );
1290 }
1291
1292 #[tokio::test]
1293 async fn downscale_skips_and_leaves_no_scaled_sibling_when_within_max() {
1294 let path =
1297 std::env::temp_dir().join(format!("mermaid-f58-skip-{}.png", std::process::id()));
1298 let path_str = path.to_string_lossy().to_string();
1299 let _cleanup = TempFileGuard(path.clone());
1300 let mut png = vec![0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n'];
1302 png.extend_from_slice(&[0, 0, 0, 13]); png.extend_from_slice(b"IHDR");
1304 png.extend_from_slice(&16u32.to_be_bytes()); png.extend_from_slice(&16u32.to_be_bytes()); png.extend_from_slice(&[8, 6, 0, 0, 0]); std::fs::write(&path, &png).unwrap();
1308
1309 let scale = downscale_if_needed(&path_str, 1920).await.unwrap();
1310 assert_eq!(scale, 1.0);
1311 assert!(
1312 !std::path::Path::new(&format!("{path_str}.scaled.png")).exists(),
1313 "no scaled sibling for an already-small capture"
1314 );
1315 }
1316
1317 #[tokio::test]
1320 async fn cancellable_returns_cancelled_when_token_already_cancelled() {
1321 let token = tokio_util::sync::CancellationToken::new();
1325 token.cancel();
1326 let slow = async {
1327 tokio::time::sleep(std::time::Duration::from_secs(30)).await;
1328 Ok::<(), anyhow::Error>(())
1329 };
1330 let err = cancellable(&token, slow).await.unwrap_err();
1331 assert!(err.to_string().contains("cancelled"), "got: {err}");
1332 }
1333
1334 #[tokio::test]
1335 async fn cancellable_passes_through_result_when_not_cancelled() {
1336 let token = tokio_util::sync::CancellationToken::new();
1337 let v = cancellable(&token, async { Ok::<u32, anyhow::Error>(7) })
1338 .await
1339 .unwrap();
1340 assert_eq!(v, 7);
1341 }
1342
1343 #[tokio::test]
1344 async fn cancellable_aborts_inflight_future_on_cancel() {
1345 let token = tokio_util::sync::CancellationToken::new();
1348 let t2 = token.clone();
1349 tokio::spawn(async move {
1350 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1351 t2.cancel();
1352 });
1353 let slow = async {
1354 tokio::time::sleep(std::time::Duration::from_secs(30)).await;
1355 Ok::<(), anyhow::Error>(())
1356 };
1357 let start = std::time::Instant::now();
1358 let err = cancellable(&token, slow).await.unwrap_err();
1359 assert!(err.to_string().contains("cancelled"), "got: {err}");
1360 assert!(
1361 start.elapsed() < std::time::Duration::from_secs(5),
1362 "must abort promptly, not wait out the slow future"
1363 );
1364 }
1365}