1use serde_json::{json, Value};
21
22use crate::error::ComputerError;
23use crate::types::{DisplayInfo, Region};
24
25pub const VIRTUAL_SELECTOR: &str = "virtual";
27
28pub const XVFB_DISPLAY: &str = ":99";
30
31pub const XVFB_GEOMETRY: &str = "1440x900x24";
33
34#[derive(Debug, Clone, PartialEq, Eq)]
36pub enum DisplaySelector {
37 Id(u32),
39 Virtual,
41}
42
43#[derive(Debug, Clone, PartialEq)]
45pub struct DisplayTarget {
46 pub display: DisplayInfo,
48 pub region: Region,
51}
52
53impl DisplayTarget {
54 pub fn bounds(&self) -> String {
56 format!(
57 "{}x{}+{}+{}",
58 self.region.width, self.region.height, self.region.x, self.region.y
59 )
60 }
61
62 pub fn known_ids(displays: &[DisplayInfo]) -> String {
64 displays
65 .iter()
66 .map(|display| display.id.to_string())
67 .collect::<Vec<_>>()
68 .join(", ")
69 }
70}
71
72pub fn parse_selector(spec: &str) -> Result<DisplaySelector, ComputerError> {
74 let trimmed = spec.trim();
75 if trimmed.eq_ignore_ascii_case(VIRTUAL_SELECTOR) {
76 return Ok(DisplaySelector::Virtual);
77 }
78 trimmed
79 .parse::<u32>()
80 .map(DisplaySelector::Id)
81 .map_err(|_| {
82 ComputerError::InvalidArguments(format!(
83 "display must be a monitor id from list_displays or `{VIRTUAL_SELECTOR}` (got {spec})"
84 ))
85 })
86}
87
88pub fn resolve(spec: &str, displays: &[DisplayInfo]) -> Result<DisplayTarget, ComputerError> {
94 match parse_selector(spec)? {
95 DisplaySelector::Id(id) => find(displays, id).ok_or_else(|| {
96 let known = DisplayTarget::known_ids(displays);
97 if known.is_empty() {
98 ComputerError::InvalidArguments(format!(
99 "display {id} was not found: no display is reported by this platform"
100 ))
101 } else {
102 ComputerError::InvalidArguments(format!(
103 "display {id} was not found; list_displays reports {known}"
104 ))
105 }
106 }),
107 DisplaySelector::Virtual => Err(ComputerError::Unsupported(
108 "display `virtual` prepares an environment and is not a capture target; \
109 run `computer-use --display virtual` on Linux to print the Xvfb recipe"
110 .into(),
111 )),
112 }
113}
114
115pub fn find(displays: &[DisplayInfo], id: u32) -> Option<DisplayTarget> {
117 displays
118 .iter()
119 .find(|display| display.id == id)
120 .and_then(display_target)
121}
122
123pub fn display_target(display: &DisplayInfo) -> Option<DisplayTarget> {
125 if display.width == 0 || display.height == 0 {
126 return None;
127 }
128 let region = Region::new(display.x, display.y, display.width, display.height).ok()?;
129 Some(DisplayTarget {
130 display: display.clone(),
131 region,
132 })
133}
134
135pub fn point_inside(target: &DisplayTarget, x: i32, y: i32) -> bool {
140 let (rx, ry, rw, rh) = (
141 i64::from(target.region.x),
142 i64::from(target.region.y),
143 i64::from(target.region.width),
144 i64::from(target.region.height),
145 );
146 let (px, py) = (i64::from(x), i64::from(y));
147 px >= rx && py >= ry && px < rx + rw && py < ry + rh
148}
149
150pub fn ensure_point_inside(target: &DisplayTarget, x: i32, y: i32) -> Result<(), ComputerError> {
152 if point_inside(target, x, y) {
153 return Ok(());
154 }
155 Err(ComputerError::InvalidArguments(format!(
156 "point {x},{y} is outside display {} ({} covers {})",
157 target.display.id,
158 target.display.name,
159 target.bounds()
160 )))
161}
162
163pub fn ensure_region_inside(target: &DisplayTarget, region: Region) -> Result<(), ComputerError> {
168 if region_inside(target, region) {
169 return Ok(());
170 }
171 Err(ComputerError::InvalidArguments(format!(
172 "region {},{},{}x{} is not inside display {} ({} covers {})",
173 region.x,
174 region.y,
175 region.width,
176 region.height,
177 target.display.id,
178 target.display.name,
179 target.bounds()
180 )))
181}
182
183pub fn region_inside(target: &DisplayTarget, region: Region) -> bool {
185 let (rx, ry, rw, rh) = (
186 i64::from(target.region.x),
187 i64::from(target.region.y),
188 i64::from(target.region.width),
189 i64::from(target.region.height),
190 );
191 let x = i64::from(region.x);
192 let y = i64::from(region.y);
193 let x2 = x + i64::from(region.width);
194 let y2 = y + i64::from(region.height);
195 x >= rx && y >= ry && x2 <= rx + rw && y2 <= ry + rh
196}
197
198pub fn window_inside(target: &DisplayTarget, bounds: crate::target::WindowBounds) -> bool {
200 region_inside(
201 target,
202 Region {
203 x: bounds.x,
204 y: bounds.y,
205 width: bounds.width,
206 height: bounds.height,
207 },
208 )
209}
210
211pub fn xvfb_command() -> String {
217 format!("Xvfb {XVFB_DISPLAY} -screen 0 {XVFB_GEOMETRY}")
218}
219
220pub fn prepared_env() -> Vec<(&'static str, String)> {
222 vec![("DISPLAY", XVFB_DISPLAY.to_string())]
223}
224
225pub fn xvfb_recipe() -> Value {
231 json!({
232 "method": "xvfb",
233 "platform": "linux",
234 "display": XVFB_DISPLAY,
235 "geometry": XVFB_GEOMETRY,
236 "server_command": xvfb_command(),
237 "client_prefix": format!("DISPLAY={XVFB_DISPLAY}"),
238 "example": "computer-use --display virtual --exec -- computer-use screenshot",
239 "starts_server": false,
240 "env": prepared_env()
241 .into_iter()
242 .map(|(key, value)| json!({"key": key, "value": value}))
243 .collect::<Vec<_>>(),
244 "notes": [
245 "Start Xvfb yourself, in another terminal. --display virtual never starts one.",
246 "When DISPLAY already points at a separate server, that server is the isolation; no new protocol is involved.",
247 "The existing X11 backend does the capture and the input. Nothing new is needed.",
248 ],
249 })
250}
251
252pub fn exec_arguments(raw: &[String]) -> Vec<String> {
258 match raw.first() {
259 Some(first) if first == "--" => raw[1..].to_vec(),
260 _ => raw.to_vec(),
261 }
262}
263
264#[cfg(target_os = "linux")]
267pub fn exec_prepared(command: &[String]) -> Result<std::convert::Infallible, ComputerError> {
268 use std::os::unix::process::CommandExt;
269
270 if command.is_empty() {
271 return Err(ComputerError::InvalidArguments(
272 "--exec needs a command; pass it after `--`, for example \
273 `--display virtual --exec -- computer-use screenshot`"
274 .into(),
275 ));
276 }
277 let mut child = std::process::Command::new(&command[0]);
278 child.args(&command[1..]);
279 for (key, value) in prepared_env() {
280 child.env(key, value);
281 }
282 let error = child.exec();
285 Err(ComputerError::ExecutionFailed(format!(
286 "could not execute {} in the prepared environment: {error}",
287 command[0]
288 )))
289}
290
291#[cfg(not(target_os = "linux"))]
293pub fn exec_prepared(_command: &[String]) -> Result<std::convert::Infallible, ComputerError> {
294 Err(ComputerError::Unsupported(format!(
295 "--display virtual needs an X server and only Linux has a supported recipe \
296 (current platform: {}); see docs/ISOLATION.md",
297 platform()
298 )))
299}
300
301pub const fn platform() -> &'static str {
307 if cfg!(target_os = "macos") {
308 "macos"
309 } else if cfg!(target_os = "linux") {
310 "linux"
311 } else if cfg!(target_os = "windows") {
312 "windows"
313 } else {
314 "unknown"
315 }
316}
317
318pub fn virtual_supported() -> bool {
323 cfg!(target_os = "linux")
324}
325
326pub fn virtual_display_compiled() -> bool {
328 cfg!(feature = "virtual-display")
329}
330
331#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
333pub struct VirtualDisplayProbe {
334 pub descriptor_create: bool,
336 pub display_create: bool,
338 pub action: &'static str,
340 pub note: String,
342}
343
344impl VirtualDisplayProbe {
345 pub fn available(&self) -> bool {
347 self.descriptor_create && self.display_create
348 }
349
350 pub fn status(&self) -> &'static str {
352 if self.available() {
353 "experimental"
354 } else {
355 "unsupported"
356 }
357 }
358}
359
360#[cfg(all(target_os = "macos", feature = "virtual-display"))]
361fn probe_virtual_display() -> VirtualDisplayProbe {
362 use self::virtual_display::resolve_symbols;
363 let symbols = resolve_symbols();
364 let available = symbols.descriptor_create && symbols.display_create;
365 VirtualDisplayProbe {
366 descriptor_create: symbols.descriptor_create,
367 display_create: symbols.display_create,
368 action: "probe only",
369 note: if available {
370 "CGVirtualDisplayDescriptorCreate and CGVirtualDisplayCreate resolved in \
371 CoreGraphics. The descriptor is an Objective-C object whose remaining \
372 properties need private class setup this build does not do, so no display \
373 is ever created here. The symbols are system-visible: TCC, the menu bar, \
374 and input focus stay shared with the host session. This is not isolation."
375 .into()
376 } else {
377 "CGVirtualDisplay is not exported by this CoreGraphics, so macOS reports \
378 `unsupported` for the experimental virtual display."
379 .into()
380 },
381 }
382}
383
384#[cfg(not(all(target_os = "macos", feature = "virtual-display")))]
385fn probe_virtual_display() -> VirtualDisplayProbe {
386 let (descriptor_create, display_create, note) = if cfg!(target_os = "macos") {
387 (
388 false,
389 false,
390 "this binary was built without `--features virtual-display`, so the private \
391 CoreGraphics symbols are never probed",
392 )
393 } else if cfg!(feature = "virtual-display") {
394 (
395 false,
396 false,
397 "CGVirtualDisplay is a macOS private API; the feature compiles on every target \
398 and only probes on macOS",
399 )
400 } else {
401 (
402 false,
403 false,
404 "macOS CGVirtualDisplay is unavailable on this platform",
405 )
406 };
407 VirtualDisplayProbe {
408 descriptor_create,
409 display_create,
410 action: "none",
411 note: note.into(),
412 }
413}
414
415pub fn probe() -> VirtualDisplayProbe {
417 probe_virtual_display()
418}
419
420pub fn infeasible() -> Value {
424 json!({
425 "macos_vm_per_agent": "operator setup, not shipped: run this binary in a UTM or Virtualization.framework guest and pipe JSON lines in over SSH or a virtio socket",
426 "share_host_app_into_guest": "infeasible: a host application cannot be driven from inside a guest",
427 "hide_virtual_display_from_user": "infeasible with public APIs: a created CGVirtualDisplay is composited by WindowServer and visible to the user",
428 "windows_virtual_display": "not implemented here: use a Windows VM or Windows Sandbox running the same binary",
429 "nested_wayland_input_without_portal": "infeasible by design of the XDG RemoteDesktop portal: a grant is required",
430 })
431}
432
433pub fn matrix() -> Value {
435 let probe = probe();
436 let cg_status = if cfg!(target_os = "macos") {
437 probe.status()
438 } else {
439 "unsupported"
440 };
441 json!({
442 "platform": platform(),
443 "display_filter": true,
444 "display_filter_note": "--display <id> restricts capture and coordinate conversion to one monitor from list_displays; points stay virtual-desktop points",
445 "xvfb": "linux only",
446 "xvfb_recipe": xvfb_recipe(),
447 "cg_virtual_display": cg_status,
448 "cg_virtual_display_detail": probe,
449 "vm": "documented, not shipped",
450 "vm_note": "docs/ISOLATION.md describes the JSON-over-SSH proxy; the crate does not grow a hypervisor dependency",
451 "infeasible": infeasible(),
452 })
453}
454
455#[cfg(all(target_os = "macos", feature = "virtual-display"))]
456mod virtual_display {
457 #![allow(unsafe_code)]
466
467 use std::ffi::{c_void, CStr};
468
469 #[allow(dead_code)]
474 pub(super) type DescriptorCreate = unsafe extern "C" fn() -> *mut c_void;
475
476 #[allow(dead_code)]
478 pub(super) type DisplayCreate = unsafe extern "C" fn(*const c_void, *mut u32) -> i32;
479
480 const CORE_GRAPHICS: &CStr = c"/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics";
481
482 #[derive(Debug, Clone, Copy)]
484 pub(super) struct Symbols {
485 pub(super) descriptor_create: bool,
486 pub(super) display_create: bool,
487 }
488
489 pub(super) fn resolve_symbols() -> Symbols {
496 let handle = unsafe { libc::dlopen(CORE_GRAPHICS.as_ptr(), libc::RTLD_LAZY) };
500 if handle.is_null() {
501 return Symbols {
502 descriptor_create: false,
503 display_create: false,
504 };
505 }
506 let found = Symbols {
507 descriptor_create: symbol(handle, c"CGVirtualDisplayDescriptorCreate"),
508 display_create: symbol(handle, c"CGVirtualDisplayCreate"),
509 };
510 unsafe { libc::dlclose(handle) };
512 found
513 }
514
515 fn symbol(handle: *mut c_void, name: &CStr) -> bool {
516 !unsafe { libc::dlsym(handle, name.as_ptr()) }.is_null()
520 }
521}
522
523#[cfg(test)]
524mod tests {
525 use super::*;
526
527 fn fake_displays() -> Vec<DisplayInfo> {
528 vec![
529 DisplayInfo {
530 id: 1,
531 name: "Left".into(),
532 x: 0,
533 y: 0,
534 width: 1000,
535 height: 600,
536 scale: 2.0,
537 is_primary: true,
538 },
539 DisplayInfo {
540 id: 2,
541 name: "Right".into(),
542 x: 1000,
543 y: 200,
544 width: 800,
545 height: 400,
546 scale: 1.0,
547 is_primary: false,
548 },
549 ]
550 }
551
552 #[test]
553 fn parses_ids_and_the_virtual_keyword() {
554 assert_eq!(parse_selector("2").unwrap(), DisplaySelector::Id(2));
555 assert_eq!(parse_selector(" 1 ").unwrap(), DisplaySelector::Id(1));
556 assert_eq!(parse_selector("virtual").unwrap(), DisplaySelector::Virtual);
557 assert_eq!(parse_selector("VIRTUAL").unwrap(), DisplaySelector::Virtual);
558 for bad in ["", "left", "-1", "1.5", "id:1", "99999999999999999999"] {
559 let error = parse_selector(bad).unwrap_err();
560 assert_eq!(error.code(), "invalid_arguments", "{bad}");
561 }
562 }
563
564 #[test]
565 fn unknown_id_is_invalid_arguments_and_lists_what_exists() {
566 let error = resolve("7", &fake_displays()).unwrap_err();
567 assert_eq!(error.code(), "invalid_arguments");
568 assert!(error.to_string().contains("list_displays reports 1, 2"));
569
570 let error = resolve("3", &[]).unwrap_err();
571 assert_eq!(error.code(), "invalid_arguments");
572 assert!(error.to_string().contains("no display is reported"));
573 }
574
575 #[test]
576 fn virtual_is_unsupported_in_the_json_path() {
577 let error = resolve("virtual", &fake_displays()).unwrap_err();
578 assert_eq!(error.code(), "unsupported");
579 }
580
581 #[test]
582 fn rejects_a_point_outside_the_chosen_monitor() {
583 let target = resolve("2", &fake_displays()).unwrap();
584 assert!(ensure_point_inside(&target, 1000, 200).is_ok());
586 assert!(ensure_point_inside(&target, 1799, 599).is_ok());
587
588 for (x, y) in [(0, 0), (999, 199), (1800, 200), (1000, 199), (1000, 600)] {
590 let error = ensure_point_inside(&target, x, y).unwrap_err();
591 assert_eq!(error.code(), "invalid_arguments", "{x},{y}");
592 }
593 }
594
595 #[test]
596 fn the_right_and_bottom_edges_are_exclusive() {
597 let target = resolve("1", &fake_displays()).unwrap();
598 assert!(point_inside(&target, 999, 599));
599 assert!(!point_inside(&target, 1000, 599));
600 assert!(!point_inside(&target, 999, 600));
601 let other = resolve("2", &fake_displays()).unwrap();
603 assert!(point_inside(&other, 1000, 200));
604 }
605
606 #[test]
607 fn the_map_origin_is_the_monitor_origin() {
608 let target = resolve("2", &fake_displays()).unwrap();
609 assert_eq!((target.region.x, target.region.y), (1000, 200));
610 assert_eq!((target.region.width, target.region.height), (800, 400));
611 assert_eq!(target.bounds(), "800x400+1000+200");
612 }
613
614 #[test]
615 fn a_region_must_fit_inside_the_monitor() {
616 let target = resolve("2", &fake_displays()).unwrap();
617 assert!(ensure_region_inside(&target, Region::new(1000, 200, 800, 400).unwrap()).is_ok());
618 assert!(ensure_region_inside(&target, Region::new(1000, 200, 801, 400).unwrap()).is_err());
619 assert!(ensure_region_inside(&target, Region::new(999, 200, 800, 400).unwrap()).is_err());
620 let error = ensure_region_inside(&target, Region::new(0, 0, 10, 10).unwrap()).unwrap_err();
621 assert_eq!(error.code(), "invalid_arguments");
622 assert!(error.to_string().contains("800x400+1000+200"));
623 }
624
625 #[test]
626 fn a_window_must_fit_inside_the_monitor() {
627 let target = resolve("1", &fake_displays()).unwrap();
628 let inside = crate::target::WindowBounds {
629 x: 10,
630 y: 10,
631 width: 100,
632 height: 100,
633 };
634 let straddling = crate::target::WindowBounds {
635 x: 990,
636 y: 10,
637 width: 100,
638 height: 100,
639 };
640 assert!(window_inside(&target, inside));
641 assert!(!window_inside(&target, straddling));
642 }
643
644 #[test]
645 fn a_zero_sized_monitor_is_not_a_target() {
646 let mut broken = fake_displays();
647 broken[0].width = 0;
648 assert!(find(&broken, 1).is_none());
649 }
650
651 #[test]
652 fn exec_arguments_drop_the_separator() {
653 assert_eq!(
654 exec_arguments(&["--".into(), "env".into(), "A=1".into()]),
655 vec!["env".to_string(), "A=1".to_string()]
656 );
657 assert_eq!(
658 exec_arguments(&["env".into(), "A=1".into()]),
659 vec!["env".to_string(), "A=1".to_string()]
660 );
661 assert!(exec_arguments(&["--".into()]).is_empty());
662 }
663
664 #[test]
665 fn the_recipe_documented_is_the_recipe_returned() {
666 let recipe = xvfb_recipe();
667 assert_eq!(recipe["server_command"], "Xvfb :99 -screen 0 1440x900x24");
668 assert_eq!(recipe["display"], ":99");
669 assert_eq!(recipe["starts_server"], false);
670 assert_eq!(recipe["env"][0]["key"], "DISPLAY");
671 assert_eq!(recipe["env"][0]["value"], ":99");
672 }
673
674 #[test]
676 fn isolation_matrix_is_a_stable_snapshot() {
677 let matrix = matrix();
678 let infeasible = infeasible();
679
680 assert_eq!(matrix["platform"], platform());
681 assert_eq!(matrix["display_filter"], true);
682 assert_eq!(matrix["xvfb"], "linux only");
683 assert_eq!(matrix["vm"], "documented, not shipped");
684 assert_eq!(matrix["cg_virtual_display"], probe().status());
685 assert_eq!(
686 matrix["cg_virtual_display_detail"]["action"],
687 probe().action
688 );
689
690 let mut expected = vec![
693 "macos_vm_per_agent",
694 "share_host_app_into_guest",
695 "hide_virtual_display_from_user",
696 "windows_virtual_display",
697 "nested_wayland_input_without_portal",
698 ];
699 expected.sort_unstable();
700 let mut actual: Vec<String> = infeasible
701 .as_object()
702 .expect("object")
703 .keys()
704 .cloned()
705 .collect();
706 actual.sort();
707 assert_eq!(actual, expected);
708 for key in &expected {
709 assert!(
710 infeasible[*key].as_str().expect("reason").len() > 20,
711 "{key} needs a reason"
712 );
713 }
714 assert_eq!(matrix["infeasible"], infeasible);
715 }
716
717 #[test]
718 fn the_virtual_display_claim_never_outruns_the_probe() {
719 let matrix = matrix();
720 let detail = &matrix["cg_virtual_display_detail"];
721 let resolved = detail["descriptor_create"].as_bool().unwrap_or(false)
722 && detail["display_create"].as_bool().unwrap_or(false);
723 let probed = cfg!(all(target_os = "macos", feature = "virtual-display"));
724
725 assert_eq!(detail["action"], if probed { "probe only" } else { "none" });
727 if !probed {
728 assert!(!resolved);
729 }
730
731 if resolved {
735 assert_eq!(matrix["cg_virtual_display"], "experimental");
736 assert!(
737 detail["note"]
738 .as_str()
739 .unwrap_or_default()
740 .contains("not isolation"),
741 "the note must not describe this as isolation"
742 );
743 } else {
744 assert_eq!(matrix["cg_virtual_display"], "unsupported");
745 assert!(
746 detail["note"].as_str().unwrap_or_default().len() > 20,
747 "an unsupported claim needs a reason"
748 );
749 }
750 if !cfg!(target_os = "macos") {
751 assert!(!resolved);
752 assert_eq!(matrix["cg_virtual_display"], "unsupported");
753 }
754 assert_eq!(detail["action"], if probed { "probe only" } else { "none" });
756 }
757}