1use std::fs;
4use std::panic::{catch_unwind, AssertUnwindSafe};
5use std::path::{Path, PathBuf};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::thread;
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use serde::de::{self, Deserializer};
11use serde::ser::{Serialize, Serializer};
12use serde::{Deserialize, Serialize as SerializeDerive};
13use serde_json::{json, Map, Value};
14
15use crate::attempt::{self, AttemptFlags};
16use crate::backend::clipboard::ClipboardContents;
17use crate::backend::create_desktop_backend;
18use crate::backend::{validate_key, DesktopBackend, WindowTarget};
19use crate::cache;
20use crate::element::{self, ElementError, ElementOp, ElementOutcome, ElementRequest};
21use crate::error::{ComputerError, ErrorBody};
22use crate::image_util::{self, encode_base64};
23use crate::input_session::{self, InputSession};
24use crate::keys::{parse_key_combo, parse_key_sequence};
25use crate::policy;
26use crate::state::collect_state_for;
27use crate::target::{
28 filter_windows_for_pid, parse_app_selector, parse_window_selector, resolve_app, resolve_window,
29 WindowSelector,
30};
31use crate::tier::{self, InputKind, KeyStroke, TierName};
32use crate::tree::TreeQuery;
33use crate::types::{
34 parse_path, region_inside, Frame, ImageFormat, Modifier, MouseButton, Point, Region,
35 ScrollDirection,
36};
37use crate::vision::{OcrBlock, OcrResult};
38
39const MAX_WAIT_MS: u128 = 24 * 60 * 60 * 1000;
40
41pub const MENU_DEFAULT_DEPTH: u32 = 4;
44
45pub const MENU_MAX_DEPTH: u32 = 8;
48
49pub const SHORTCUTS_TIMEOUT_SECS: u64 = 60;
51
52pub const APPLE_EVENT_TIMEOUT_SECS: u64 = 30;
54
55#[derive(Debug, Clone, PartialEq, SerializeDerive, Deserialize)]
61#[serde(tag = "action", rename_all = "snake_case")]
62pub enum Action {
63 Screenshot {
65 #[serde(default, skip_serializing_if = "Option::is_none")]
67 output: Option<String>,
68 #[serde(default, skip_serializing_if = "Option::is_none")]
70 region: Option<Region>,
71 #[serde(default, skip_serializing_if = "Option::is_none")]
73 format: Option<ImageFormat>,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
78 max_width: Option<u32>,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
81 canvas: Option<String>,
82 #[serde(default, skip_serializing_if = "is_false")]
84 base64: bool,
85 #[serde(default, skip_serializing_if = "is_false")]
89 marks: bool,
90 },
91 Click {
93 #[serde(default, skip_serializing_if = "Option::is_none")]
95 x: Option<i32>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
98 y: Option<i32>,
99 #[serde(default)]
101 button: MouseButton,
102 #[serde(default, skip_serializing_if = "is_false")]
104 double: bool,
105 #[serde(default = "default_press_count", skip_serializing_if = "is_one")]
107 count: u8,
108 },
109 #[serde(rename = "triple_click", alias = "triple-click")]
111 TripleClick {
112 #[serde(default, skip_serializing_if = "Option::is_none")]
114 x: Option<i32>,
115 #[serde(default, skip_serializing_if = "Option::is_none")]
117 y: Option<i32>,
118 #[serde(default)]
120 button: MouseButton,
121 },
122 #[serde(rename = "double_click")]
124 DoubleClick {
125 #[serde(default, skip_serializing_if = "Option::is_none")]
127 x: Option<i32>,
128 #[serde(default, skip_serializing_if = "Option::is_none")]
130 y: Option<i32>,
131 #[serde(default)]
133 button: MouseButton,
134 },
135 Press {
137 #[serde(default = "default_press_count")]
139 count: u8,
140 #[serde(default)]
142 button: MouseButton,
143 },
144 Drag {
146 #[serde(alias = "points", deserialize_with = "deserialize_path")]
148 path: Vec<Point>,
149 #[serde(default)]
151 button: MouseButton,
152 },
153 #[serde(alias = "mouse-down")]
155 MouseDown {
156 x: i32,
158 y: i32,
160 #[serde(default)]
162 button: MouseButton,
163 },
164 #[serde(alias = "mouse-up")]
166 MouseUp {
167 x: i32,
169 y: i32,
171 #[serde(default)]
173 button: MouseButton,
174 },
175 #[serde(alias = "hold-key")]
177 HoldKey {
178 keys: String,
180 ms: u64,
182 },
183 ClipboardGet,
185 #[serde(alias = "clipboard-set")]
187 ClipboardSet {
188 #[serde(default, skip_serializing_if = "Option::is_none")]
190 text: Option<String>,
191 #[serde(default, skip_serializing_if = "Option::is_none")]
193 image: Option<String>,
194 },
195 Zoom {
197 #[serde(default, skip_serializing_if = "Option::is_none")]
199 output: Option<String>,
200 region: Region,
202 #[serde(default, skip_serializing_if = "Option::is_none")]
204 format: Option<ImageFormat>,
205 #[serde(default, skip_serializing_if = "Option::is_none")]
207 max_width: Option<u32>,
208 #[serde(default, skip_serializing_if = "Option::is_none")]
210 canvas: Option<String>,
211 #[serde(default, skip_serializing_if = "is_false")]
213 base64: bool,
214 },
215 Focus,
217 #[serde(alias = "set-value")]
219 SetValue {
220 value: String,
222 },
223 #[serde(alias = "select-text")]
225 SelectText {
226 start: i64,
228 end: i64,
230 },
231 #[serde(alias = "perform-action")]
233 PerformAction {
234 name: String,
236 },
237 #[serde(
239 alias = "scroll-to",
240 alias = "scroll_to_visible",
241 alias = "scroll-to-visible"
242 )]
243 ScrollTo,
244 #[serde(alias = "move_mouse")]
246 Move {
247 x: i32,
249 y: i32,
251 },
252 Type {
254 text: String,
256 },
257 Key {
259 keys: String,
261 },
262 Scroll {
264 #[serde(default, skip_serializing_if = "Option::is_none")]
266 x: Option<i32>,
267 #[serde(default, skip_serializing_if = "Option::is_none")]
269 y: Option<i32>,
270 #[serde(default, alias = "scroll_x", skip_serializing_if = "Option::is_none")]
272 dx: Option<i32>,
273 #[serde(default, alias = "scroll_y", skip_serializing_if = "Option::is_none")]
275 dy: Option<i32>,
276 #[serde(default, skip_serializing_if = "Option::is_none")]
278 direction: Option<ScrollDirection>,
279 #[serde(default, skip_serializing_if = "Option::is_none")]
281 amount: Option<i32>,
282 },
283 #[serde(alias = "get-state")]
285 GetState,
286 Wait {
288 #[serde(default, skip_serializing_if = "Option::is_none")]
290 ms: Option<u64>,
291 #[serde(default, skip_serializing_if = "Option::is_none")]
293 seconds: Option<f64>,
294 },
295 Displays,
297 Isolation,
299 ListApps {
301 #[serde(default, rename = "all")]
303 all: bool,
304 },
305 ListWindows,
307 Launch,
309 Activate,
311 Quit {
313 #[serde(default, skip_serializing_if = "is_false")]
315 force: bool,
316 },
317 #[serde(rename = "policy_check")]
319 PolicyCheck,
320 Pause,
322 Resume,
324 Doctor {
330 #[serde(default, alias = "request")]
332 prompt: bool,
333 #[serde(default)]
335 open_settings: bool,
336 #[serde(default)]
340 x11: bool,
341 },
342 Tree,
344 SessionEnd,
346 CursorPosition,
348 Batch {
350 #[serde(default = "default_stop_on_error")]
352 stop_on_error: bool,
353 actions: Vec<Value>,
355 },
356 #[serde(alias = "session-prune")]
358 SessionPrune {
359 #[serde(default)]
361 dry_run: bool,
362 #[serde(default, skip_serializing_if = "Option::is_none")]
364 ttl_secs: Option<u64>,
365 },
366 #[serde(alias = "menu_list", alias = "menu-list")]
371 MenuList {
372 #[serde(default, skip_serializing_if = "Option::is_none")]
374 app: Option<String>,
375 #[serde(default, skip_serializing_if = "Option::is_none")]
377 depth: Option<u32>,
378 },
379 #[serde(alias = "menu_click", alias = "menu-click")]
385 MenuClick {
386 #[serde(default, skip_serializing_if = "Option::is_none")]
388 app: Option<String>,
389 path: String,
391 },
392 #[serde(alias = "apple_event", alias = "apple-event")]
398 AppleEvent {
399 #[serde(default, skip_serializing_if = "Option::is_none")]
401 app: Option<String>,
402 script: String,
404 #[serde(default, skip_serializing_if = "Option::is_none")]
406 language: Option<AppleEventLanguage>,
407 },
408 #[serde(alias = "shortcuts_run", alias = "shortcuts-run")]
413 ShortcutsRun {
414 name: String,
416 },
417 #[serde(alias = "dock_list", alias = "dock-list")]
419 DockList {},
420 #[serde(alias = "dock_click", alias = "dock-click")]
422 DockClick {
423 title: String,
425 },
426 #[serde(alias = "spaces")]
432 Spaces {},
433 #[serde(rename = "browser_tabs", alias = "browser-tabs")]
436 BrowserTabs {},
437 #[serde(rename = "browser_activate", alias = "browser-activate")]
439 BrowserActivate {
440 #[serde(default, skip_serializing_if = "Option::is_none")]
442 target_id: Option<String>,
443 },
444 #[serde(rename = "browser_navigate", alias = "browser-navigate")]
447 BrowserNavigate {
448 url: String,
450 #[serde(default, skip_serializing_if = "Option::is_none")]
452 target_id: Option<String>,
453 },
454 #[serde(rename = "browser_click", alias = "browser-click")]
456 BrowserClick {
457 selector: String,
459 #[serde(default, skip_serializing_if = "Option::is_none")]
461 target_id: Option<String>,
462 },
463 #[serde(rename = "browser_text", alias = "browser-text")]
468 BrowserText {
469 selector: String,
471 #[serde(default, skip_serializing_if = "Option::is_none")]
473 target_id: Option<String>,
474 },
475 #[serde(rename = "browser_type", alias = "browser-type")]
481 BrowserType {
482 text: String,
484 #[serde(default, skip_serializing_if = "Option::is_none")]
486 selector: Option<String>,
487 #[serde(default, skip_serializing_if = "Option::is_none")]
489 target_id: Option<String>,
490 },
491 #[serde(rename = "browser_eval", alias = "browser-eval")]
495 BrowserEval {
496 script: String,
499 #[serde(default, skip_serializing_if = "Option::is_none")]
501 target_id: Option<String>,
502 },
503 #[serde(rename = "ocr")]
506 Ocr {
507 #[serde(default, skip_serializing_if = "is_false")]
509 force: bool,
510 },
511}
512
513fn is_false(value: &bool) -> bool {
514 !*value
515}
516
517fn is_one(value: &u8) -> bool {
518 *value == 1
519}
520
521fn default_press_count() -> u8 {
522 1
523}
524
525#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDerive, Deserialize)]
530#[serde(rename_all = "snake_case")]
531pub enum AppleEventLanguage {
532 #[default]
534 #[serde(alias = "applescript", alias = "osa")]
535 AppleScript,
536 #[serde(alias = "javascript", alias = "js", alias = "JavaScript")]
538 JavaScript,
539}
540
541impl AppleEventLanguage {
542 pub fn as_str(&self) -> &'static str {
544 match self {
545 Self::AppleScript => "AppleScript",
546 Self::JavaScript => "JavaScript",
547 }
548 }
549}
550
551fn deserialize_path<'de, D>(deserializer: D) -> Result<Vec<Point>, D::Error>
552where
553 D: Deserializer<'de>,
554{
555 let value = Value::deserialize(deserializer)?;
556 match value {
557 Value::String(text) => parse_path(&text).map_err(de::Error::custom),
558 Value::Array(_) => serde_json::from_value(value).map_err(de::Error::custom),
559 _ => Err(de::Error::custom(
560 "path must be an array of points or an x,y,x,y string",
561 )),
562 }
563}
564
565fn default_stop_on_error() -> bool {
566 true
567}
568
569#[derive(Debug, Clone, PartialEq, Default)]
571pub struct CallContext {
572 pub app: Option<String>,
574 pub window: Option<String>,
576 pub element: Option<u32>,
578 pub diff: bool,
580 pub full: bool,
582 pub return_state: bool,
584 pub include_screenshot: bool,
586 pub coord: Option<String>,
588 pub session: Option<String>,
590 pub tier: Option<String>,
592 pub adapter: Option<String>,
594 pub display: Option<String>,
596 pub require_visible: bool,
598 pub policy: Option<String>,
600 pub confirm_cmd: Option<String>,
602 pub allow_foreground: bool,
604 pub require_background: bool,
606 pub no_overlay: bool,
608 pub cdp: bool,
612 pub cdp_url: Option<String>,
615 pub browser_blocked_urls: Vec<String>,
619}
620
621#[derive(Debug, Clone, PartialEq)]
623pub struct Request {
624 pub id: Option<Value>,
626 pub action: Action,
628 pub context: CallContext,
630}
631
632pub fn parse_request(mut value: Value) -> Result<Request, ComputerError> {
634 if !value.is_object() {
635 return Err(ComputerError::InvalidArguments(
636 "request must be a JSON object".into(),
637 ));
638 }
639 let object = value
640 .as_object_mut()
641 .expect("object check above guarantees a map");
642 let id = object.remove("id");
643 let context = take_context(object)?;
644 let action = serde_json::from_value(value)
645 .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
646 Ok(Request {
647 id,
648 action,
649 context,
650 })
651}
652
653fn take_context(object: &mut Map<String, Value>) -> Result<CallContext, ComputerError> {
654 Ok(CallContext {
655 app: take_stringish(object, "app")?,
656 window: take_stringish(object, "window")?,
657 element: take_optional_u32(object, "element")?,
658 diff: take_bool(object, "diff")?,
659 full: take_bool(object, "full")?,
660 return_state: take_bool(object, "return_state")?,
661 include_screenshot: take_bool(object, "include_screenshot")?,
662 coord: take_stringish(object, "coord")?,
663 session: take_stringish(object, "session")?,
664 tier: take_stringish(object, "tier")?,
665 adapter: take_stringish(object, "adapter")?,
666 display: take_stringish(object, "display")?,
667 require_visible: take_bool(object, "require_visible")?,
668 policy: None,
669 confirm_cmd: None,
670 allow_foreground: take_bool(object, "allow_foreground")?,
671 require_background: take_bool(object, "require_background")?,
672 no_overlay: take_bool(object, "no_overlay")?,
673 cdp: take_bool(object, "cdp")?,
674 cdp_url: take_stringish(object, "cdp_url")?,
675 browser_blocked_urls: take_string_list(object, "browser_blocked_urls")?,
676 })
677}
678
679fn take_stringish(
680 object: &mut Map<String, Value>,
681 key: &str,
682) -> Result<Option<String>, ComputerError> {
683 match object.remove(key) {
684 None | Some(Value::Null) => Ok(None),
685 Some(Value::String(text)) => Ok(Some(text)),
686 Some(Value::Number(number)) => Ok(Some(number.to_string())),
687 Some(_) => Err(ComputerError::InvalidArguments(format!(
688 "{key} must be a string or number"
689 ))),
690 }
691}
692
693fn take_bool(object: &mut Map<String, Value>, key: &str) -> Result<bool, ComputerError> {
694 match object.remove(key) {
695 None | Some(Value::Null) => Ok(false),
696 Some(Value::Bool(value)) => Ok(value),
697 Some(_) => Err(ComputerError::InvalidArguments(format!(
698 "{key} must be a boolean"
699 ))),
700 }
701}
702
703fn take_string_list(
704 object: &mut Map<String, Value>,
705 key: &str,
706) -> Result<Vec<String>, ComputerError> {
707 match object.remove(key) {
708 None | Some(Value::Null) => Ok(Vec::new()),
709 Some(Value::Array(values)) => values
710 .into_iter()
711 .map(|value| match value {
712 Value::String(text) => Ok(text),
713 _ => Err(ComputerError::InvalidArguments(format!(
714 "{key} entries must be strings"
715 ))),
716 })
717 .collect(),
718 Some(_) => Err(ComputerError::InvalidArguments(format!(
719 "{key} must be a list of strings"
720 ))),
721 }
722}
723
724fn take_optional_u32(
725 object: &mut Map<String, Value>,
726 key: &str,
727) -> Result<Option<u32>, ComputerError> {
728 match object.remove(key) {
729 None | Some(Value::Null) => Ok(None),
730 Some(Value::Number(number)) => number
731 .as_u64()
732 .and_then(|value| u32::try_from(value).ok())
733 .map(Some)
734 .ok_or_else(|| ComputerError::InvalidArguments(format!("{key} is out of range"))),
735 Some(Value::String(text)) => text
736 .parse::<u32>()
737 .map(Some)
738 .map_err(|_| ComputerError::InvalidArguments(format!("{key} must be an integer"))),
739 Some(_) => Err(ComputerError::InvalidArguments(format!(
740 "{key} must be an integer"
741 ))),
742 }
743}
744
745pub fn handle_request_line(line: &str) -> Response {
750 handle_request_line_with(line, None)
751}
752
753#[derive(Debug, Clone, Copy, Default)]
755pub struct LineOptions<'a> {
756 pub adapter: Option<&'a str>,
758 pub policy: Option<&'a str>,
760 pub confirm_cmd: Option<&'a str>,
762 pub tier: Option<&'a str>,
764 pub allow_foreground: bool,
766 pub require_background: bool,
768 pub no_overlay: bool,
770 pub cdp: bool,
772 pub cdp_url: Option<&'a str>,
774}
775
776pub fn handle_request_line_with(line: &str, cli_adapter: Option<&str>) -> Response {
781 handle_request_line_opts(
782 line,
783 LineOptions {
784 adapter: cli_adapter,
785 ..LineOptions::default()
786 },
787 )
788}
789
790pub fn handle_request_line_opts(line: &str, options: LineOptions<'_>) -> Response {
792 let value = match serde_json::from_str::<Value>(line) {
793 Ok(value) => value,
794 Err(error) => {
795 return Response::failure("invalid_arguments", format!("invalid JSON: {error}"))
796 }
797 };
798 let id = value.get("id").cloned();
799 let echo = match effective_adapter(&value, options.adapter) {
800 Ok(name) => name,
801 Err(error) => return Response::from_error(error).with_id(id),
802 };
803 match build_request(value, echo.as_deref()) {
804 Ok(mut request) => {
805 if let Some(policy) = options.policy {
806 request.context.policy = Some(policy.to_string());
807 }
808 if let Some(confirm) = options.confirm_cmd {
809 request.context.confirm_cmd = Some(confirm.to_string());
810 }
811 if request.context.tier.is_none() {
812 if let Some(tier) = options.tier {
813 request.context.tier = Some(tier.to_string());
814 }
815 }
816 request.context.allow_foreground |= options.allow_foreground;
817 request.context.require_background |= options.require_background;
818 request.context.no_overlay |= options.no_overlay;
819 request.context.cdp |= options.cdp;
820 if request.context.cdp_url.is_none() {
821 if let Some(url) = options.cdp_url {
822 request.context.cdp_url = Some(url.to_string());
823 }
824 }
825 dispatch_context(request.action, &request.context).with_id(request.id)
826 }
827 Err(error) => {
828 let mut response = Response::from_error(error).with_id(id);
829 stamp_adapter_name(&mut response, echo.as_deref());
830 response
831 }
832 }
833}
834
835fn effective_adapter(
836 value: &Value,
837 cli_adapter: Option<&str>,
838) -> Result<Option<String>, ComputerError> {
839 if !value.is_object() {
840 return Ok(None);
841 }
842 let from_request = match value.get("adapter") {
843 None | Some(Value::Null) => None,
844 Some(Value::String(text)) => Some(text.as_str()),
845 Some(_) => {
846 return Err(ComputerError::InvalidArguments(
847 "adapter must be a string".into(),
848 ))
849 }
850 };
851 let name = from_request.or(cli_adapter);
852 match name {
853 None | Some("native") => Ok(None),
854 Some("openai" | "anthropic") => Ok(name.map(str::to_string)),
855 Some(other) => Err(ComputerError::InvalidArguments(format!(
856 "unknown adapter {other:?}"
857 ))),
858 }
859}
860
861fn build_request(value: Value, adapter: Option<&str>) -> Result<Request, ComputerError> {
862 let Some(name) = adapter else {
863 return parse_request(value);
864 };
865 if !value.is_object() {
866 return Err(ComputerError::InvalidArguments(
867 "request must be a JSON object".into(),
868 ));
869 }
870 let mut value = value;
871 let object = value
872 .as_object_mut()
873 .expect("object check above guarantees a map");
874 let id = object.remove("id");
875 let mut context = take_context(object)?;
876 context.coord = Some("image".into());
877 context.adapter = Some(name.to_string());
878 let action = crate::adapters::translate(name, &value)?;
879 Ok(Request {
880 id,
881 action,
882 context,
883 })
884}
885
886fn stamp_adapter_name(response: &mut Response, adapter: Option<&str>) {
887 if let Some(name @ ("openai" | "anthropic")) = adapter {
888 response.fields.insert("adapter".into(), json!(name));
889 }
890}
891
892pub fn dispatch(action: Action) -> Response {
894 dispatch_context(action, &CallContext::default())
895}
896
897pub fn dispatch_context(action: Action, context: &CallContext) -> Response {
899 let backend = create_desktop_backend();
900 dispatch_with_context(backend.as_ref(), action, context)
901}
902
903pub fn dispatch_with(backend: &dyn DesktopBackend, action: Action) -> Response {
905 dispatch_with_context(backend, action, &CallContext::default())
906}
907
908pub fn dispatch_with_context(
910 backend: &dyn DesktopBackend,
911 action: Action,
912 context: &CallContext,
913) -> Response {
914 let mut response = match catch_unwind(AssertUnwindSafe(|| {
915 dispatch_inner(backend, action, context)
916 })) {
917 Ok(response) => response,
918 Err(_) => Response::failure("execution_failed", "action panicked and was recovered"),
919 };
920 stamp_adapter_name(&mut response, context.adapter.as_deref());
921 if !input_session::session_open() {
924 let _ = backend.release_held();
925 }
926 response
927}
928
929enum PostKind {
930 Mutating,
932 ReadOnly,
934 Screenshot,
936}
937
938fn dispatch_inner(backend: &dyn DesktopBackend, action: Action, context: &CallContext) -> Response {
939 let label = action_label(&action);
940 let chars = clipboard_chars(&action);
941 let action = match remap_coordinates(context, action) {
942 Ok(action) => action,
943 Err(error) => {
944 return finish_action(
945 context,
946 label,
947 &policy::Subject::default(),
948 None,
949 "allow",
950 "default",
951 chars,
952 Response::from_error(error),
953 )
954 }
955 };
956 if let Err(error) = preflight_action(&action) {
957 return finish_action(
958 context,
959 label,
960 &policy::Subject::default(),
961 None,
962 "allow",
963 "default",
964 chars,
965 Response::from_error(error),
966 );
967 }
968 if let Err(error) = gate_display(backend, context, &action) {
969 return finish_action(
970 context,
971 label,
972 &policy::Subject::default(),
973 None,
974 "allow",
975 "default",
976 chars,
977 Response::from_error(error),
978 );
979 }
980 let prepared = prepare_action(backend, &action, context);
981 if let Some(response) = prepared.stop {
982 return finish_action(
983 context,
984 label,
985 &prepared.subject,
986 prepared.warning.as_deref(),
987 prepared.decision,
988 prepared.source,
989 chars,
990 response,
991 );
992 }
993 let kind = post_state_kind(&action);
994 let response = match action {
995 Action::Batch {
996 actions,
997 stop_on_error,
998 } => {
999 return finish_action(
1000 context,
1001 label,
1002 &prepared.subject,
1003 prepared.warning.as_deref(),
1004 prepared.decision,
1005 prepared.source,
1006 chars,
1007 run_batch(backend, context, actions, stop_on_error),
1008 )
1009 }
1010 Action::Screenshot {
1011 output,
1012 region,
1013 format,
1014 max_width,
1015 canvas,
1016 base64,
1017 marks,
1018 } => result_response(screenshot(
1019 backend,
1020 context,
1021 ScreenshotArgs {
1022 output,
1023 region,
1024 format,
1025 max_width,
1026 canvas,
1027 include_base64: base64,
1028 window_crop: false,
1029 marks,
1030 },
1031 )),
1032 Action::Zoom {
1033 output,
1034 region,
1035 format,
1036 max_width,
1037 canvas,
1038 base64,
1039 } => result_response(screenshot(
1040 backend,
1041 context,
1042 ScreenshotArgs {
1043 output,
1044 region: Some(region),
1045 format,
1046 max_width,
1047 canvas,
1048 include_base64: base64,
1049 window_crop: true,
1050 marks: false,
1051 },
1052 )),
1053 Action::Click {
1054 x,
1055 y,
1056 button,
1057 double,
1058 count,
1059 } => click(backend, context, x, y, button, double, count),
1060 Action::DoubleClick { x, y, button } => click(backend, context, x, y, button, true, 1),
1061 Action::TripleClick { x, y, button } => click(backend, context, x, y, button, false, 3),
1062 Action::Press { count, button } => {
1063 element_command(backend, context, ElementOp::Press { count, button })
1064 }
1065 Action::Drag { path, button } => drag(backend, context, path, button),
1066 Action::MouseDown { x, y, button } => pointer_half(backend, context, x, y, button, true),
1067 Action::MouseUp { x, y, button } => pointer_half(backend, context, x, y, button, false),
1068 Action::HoldKey { keys, ms } => hold_key(backend, context, &keys, ms),
1069 Action::ClipboardGet => result_response(clipboard_get(backend, context)),
1070 Action::ClipboardSet { text, image } => {
1071 result_response(clipboard_set(backend, text, image))
1072 }
1073 Action::Focus => element_command(backend, context, ElementOp::Focus),
1074 Action::SetValue { value } => {
1075 element_command(backend, context, ElementOp::SetValue { value })
1076 }
1077 Action::SelectText { start, end } => {
1078 element_command(backend, context, ElementOp::SelectText { start, end })
1079 }
1080 Action::PerformAction { name } => {
1081 element_command(backend, context, ElementOp::Perform { name })
1082 }
1083 Action::ScrollTo => element_command(backend, context, ElementOp::ScrollTo),
1084 Action::Move { x, y } => move_mouse(backend, context, x, y),
1085 Action::Type { text } => type_command(backend, context, text),
1086 Action::Key { keys } => press_key(backend, context, &keys),
1087 Action::Scroll {
1088 x,
1089 y,
1090 dx,
1091 dy,
1092 direction,
1093 amount,
1094 } => scroll(backend, context, (x, y), (dx, dy), direction, amount),
1095 Action::GetState => result_response(Ok(collect_state_for(backend, &tree_query(context)))),
1096 Action::Tree => result_response(tree_action(backend, context)),
1097 Action::Ocr { force } => result_response(ocr_action(backend, context, force)),
1098 Action::SessionEnd => result_response(session_end(backend, context)),
1099 Action::Wait { ms, seconds } => result_response(wait(ms, seconds)),
1100 Action::Displays => result_response(displays(backend)),
1101 Action::Isolation => result_response(isolation()),
1102 Action::ListApps { all } => result_response(list_apps(backend, all)),
1103 Action::ListWindows => result_response(list_windows(backend, context.app.as_deref())),
1104 Action::Launch => result_response(launch(backend, context.app.as_deref())),
1105 Action::Activate => result_response(activate(
1106 backend,
1107 context.app.as_deref(),
1108 context.window.as_deref(),
1109 )),
1110 Action::Quit { force } => result_response(quit(backend, context.app.as_deref(), force)),
1111 Action::Doctor {
1112 prompt,
1113 open_settings,
1114 x11,
1115 } => result_response(doctor(
1116 backend,
1117 crate::doctor::DoctorOptions {
1118 prompt,
1119 open_settings,
1120 x11,
1121 },
1122 )),
1123 Action::CursorPosition => result_response(cursor_position(backend)),
1124 Action::SessionPrune { dry_run, ttl_secs } => {
1125 result_response(session_prune(dry_run, ttl_secs))
1126 }
1127 Action::MenuList { app, depth } => {
1128 let app = match app.or_else(|| context.app.clone()) {
1129 Some(app) => app,
1130 None => return Response::from_error(invalid_arguments("menu list requires --app")),
1131 };
1132 let depth = depth.unwrap_or(MENU_DEFAULT_DEPTH);
1133 result_response(menu_list(backend, &app, depth))
1134 }
1135 Action::MenuClick { app, path } => {
1136 let app = match app.or_else(|| context.app.clone()) {
1137 Some(app) => app,
1138 None => {
1139 return Response::from_error(invalid_arguments("menu click requires --app"))
1140 }
1141 };
1142 if path.trim().is_empty() {
1143 return Response::from_error(invalid_arguments("menu click requires --path"));
1144 }
1145 result_response(menu_click(backend, &app, &path))
1146 }
1147 Action::AppleEvent {
1148 app,
1149 script,
1150 language,
1151 } => {
1152 let app = match app.or_else(|| context.app.clone()) {
1153 Some(app) => app,
1154 None => {
1155 return Response::from_error(invalid_arguments("apple-event requires --app"))
1156 }
1157 };
1158 if script.is_empty() {
1159 return Response::from_error(invalid_arguments("apple-event requires --script"));
1160 }
1161 let language = language.unwrap_or_default();
1162 result_response(apple_event_action(backend, &app, &script, language))
1163 }
1164 Action::ShortcutsRun { name } => {
1165 if !cfg!(target_os = "macos") {
1166 return Response::from_error(unsupported_error("shortcuts_run"));
1167 }
1168 if name.trim().is_empty() {
1169 return Response::from_error(invalid_arguments(
1170 "shortcuts run requires a workflow name",
1171 ));
1172 }
1173 result_response(shortcuts_run_action(name))
1174 }
1175 Action::DockList {} => result_response(dock_list_action(backend)),
1176 Action::DockClick { title } => {
1177 if title.trim().is_empty() {
1178 return Response::from_error(invalid_arguments("dock click requires --title"));
1179 }
1180 result_response(dock_click_action(backend, &title))
1181 }
1182 Action::Spaces {} => result_response(spaces_action(backend)),
1183 Action::BrowserTabs {} => result_response(browser_tabs(context, &prepared.subject)),
1184 Action::BrowserActivate { target_id } => result_response(browser_activate(
1185 context,
1186 &prepared.subject,
1187 target_id.clone(),
1188 )),
1189 Action::BrowserNavigate { url, target_id } => result_response(browser_navigate(
1190 context,
1191 &prepared.subject,
1192 url.clone(),
1193 target_id.clone(),
1194 )),
1195 Action::BrowserClick {
1196 selector,
1197 target_id,
1198 } => result_response(browser_click(
1199 context,
1200 &prepared.subject,
1201 selector.clone(),
1202 target_id.clone(),
1203 )),
1204 Action::BrowserText {
1205 selector,
1206 target_id,
1207 } => result_response(browser_text(
1208 context,
1209 &prepared.subject,
1210 selector.clone(),
1211 target_id.clone(),
1212 )),
1213 Action::BrowserType {
1214 text,
1215 selector,
1216 target_id,
1217 } => result_response(browser_type(
1218 context,
1219 &prepared.subject,
1220 text.clone(),
1221 selector.clone(),
1222 target_id.clone(),
1223 )),
1224 Action::BrowserEval { script, target_id } => result_response(browser_eval(
1225 context,
1226 &prepared.subject,
1227 script.clone(),
1228 target_id.clone(),
1229 )),
1230 Action::PolicyCheck => {
1231 let (response, subject) = policy_check(backend, context);
1232 let response = attach_post_state(backend, context, kind, response);
1233 return finish_action(
1234 context,
1235 label,
1236 &subject,
1237 prepared.warning.as_deref(),
1238 prepared.decision,
1239 prepared.source,
1240 chars,
1241 response,
1242 );
1243 }
1244 Action::Pause => pause_response(true),
1245 Action::Resume => pause_response(false),
1246 };
1247 let response = attach_post_state(backend, context, kind, response);
1248 finish_action(
1249 context,
1250 label,
1251 &prepared.subject,
1252 prepared.warning.as_deref(),
1253 prepared.decision,
1254 prepared.source,
1255 chars,
1256 response,
1257 )
1258}
1259
1260#[allow(clippy::too_many_arguments)]
1261fn finish_action(
1262 context: &CallContext,
1263 label: &str,
1264 subject: &policy::Subject,
1265 warning: Option<&str>,
1266 decision: &str,
1267 source: &str,
1268 chars: Option<u64>,
1269 mut response: Response,
1270) -> Response {
1271 if let Some(warning) = warning {
1272 push_warning(&mut response, warning);
1273 }
1274 let code = if response.ok {
1275 "ok"
1276 } else {
1277 response
1278 .error
1279 .as_ref()
1280 .map(|error| error.code.as_str())
1281 .unwrap_or("execution_failed")
1282 };
1283 let mut decision = decision.to_string();
1284 let mut source = source.to_string();
1285 if label == "policy_check" {
1286 if let Some(value) = response
1287 .fields
1288 .get("decision")
1289 .and_then(|value| value.as_str())
1290 {
1291 decision = value.to_string();
1292 }
1293 if let Some(value) = response
1294 .fields
1295 .get("source")
1296 .and_then(|value| value.as_str())
1297 {
1298 source = value.to_string();
1299 }
1300 }
1301 let tier = response
1302 .fields
1303 .get("tier")
1304 .and_then(|value| value.as_str())
1305 .map(str::to_string);
1306 let session = context
1307 .session
1308 .clone()
1309 .unwrap_or_else(|| std::process::id().to_string());
1310 if let Err(error) = crate::audit::record(&crate::audit::AuditRecord {
1311 session: &session,
1312 action: label,
1313 bundle_id: subject.bundle_id.as_deref(),
1314 pid: subject.pid,
1315 window_id: subject.window_id,
1316 element: context.element,
1317 code,
1318 decision: &decision,
1319 source: &source,
1320 tier: tier.as_deref(),
1321 chars,
1322 }) {
1323 push_warning(
1324 &mut response,
1325 &format!("audit log was not written: {error}"),
1326 );
1327 }
1328 response
1329}
1330
1331fn push_warning(response: &mut Response, warning: &str) {
1332 let warnings = response
1333 .fields
1334 .entry("warnings")
1335 .or_insert_with(|| json!([]));
1336 if let Some(items) = warnings.as_array_mut() {
1337 items.push(json!(warning));
1338 }
1339}
1340
1341fn action_label(action: &Action) -> &'static str {
1342 match action {
1343 Action::Screenshot { .. } => "screenshot",
1344 Action::Zoom { .. } => "zoom",
1345 Action::Click { .. } => "click",
1346 Action::DoubleClick { .. } => "double_click",
1347 Action::TripleClick { .. } => "triple_click",
1348 Action::Press { .. } => "press",
1349 Action::Drag { .. } => "drag",
1350 Action::MouseDown { .. } => "mouse_down",
1351 Action::MouseUp { .. } => "mouse_up",
1352 Action::HoldKey { .. } => "hold_key",
1353 Action::ClipboardGet => "clipboard_get",
1354 Action::ClipboardSet { .. } => "clipboard_set",
1355 Action::Focus => "focus",
1356 Action::SetValue { .. } => "set_value",
1357 Action::SelectText { .. } => "select_text",
1358 Action::PerformAction { .. } => "perform_action",
1359 Action::ScrollTo => "scroll_to",
1360 Action::Move { .. } => "move",
1361 Action::Type { .. } => "type",
1362 Action::Key { .. } => "key",
1363 Action::Scroll { .. } => "scroll",
1364 Action::GetState => "get_state",
1365 Action::Wait { .. } => "wait",
1366 Action::Displays => "displays",
1367 Action::Isolation => "isolation",
1368 Action::ListApps { .. } => "list_apps",
1369 Action::ListWindows => "list_windows",
1370 Action::Launch => "launch",
1371 Action::Activate => "activate",
1372 Action::Quit { .. } => "quit",
1373 Action::Doctor { .. } => "doctor",
1374 Action::Tree => "tree",
1375 Action::SessionEnd => "session_end",
1376 Action::CursorPosition => "cursor_position",
1377 Action::Batch { .. } => "batch",
1378 Action::SessionPrune { .. } => "session_prune",
1379 Action::PolicyCheck => "policy_check",
1380 Action::Pause => "pause",
1381 Action::Resume => "resume",
1382 Action::MenuList { .. } => "menu_list",
1383 Action::MenuClick { .. } => "menu_click",
1384 Action::AppleEvent { .. } => "apple_event",
1385 Action::ShortcutsRun { .. } => "shortcuts_run",
1386 Action::DockList { .. } => "dock_list",
1387 Action::DockClick { .. } => "dock_click",
1388 Action::Spaces { .. } => "spaces",
1389 Action::BrowserTabs { .. } => "browser_tabs",
1390 Action::BrowserActivate { .. } => "browser_activate",
1391 Action::BrowserNavigate { .. } => "browser_navigate",
1392 Action::BrowserClick { .. } => "browser_click",
1393 Action::BrowserText { .. } => "browser_text",
1394 Action::BrowserType { .. } => "browser_type",
1395 Action::BrowserEval { .. } => "browser_eval",
1396 Action::Ocr { .. } => "ocr",
1397 }
1398}
1399
1400#[derive(Clone, Copy)]
1401enum PolicyClass {
1402 Always,
1403 Benign,
1404 Observe,
1405 Mutate,
1406 Quit,
1407 Key,
1408 ClipboardSet,
1409}
1410
1411fn policy_class(action: &Action) -> PolicyClass {
1412 match action {
1413 Action::Wait { .. }
1414 | Action::Doctor { .. }
1415 | Action::Displays
1416 | Action::Isolation
1417 | Action::ListApps { .. }
1418 | Action::ListWindows
1419 | Action::Pause
1420 | Action::Resume => PolicyClass::Always,
1421 Action::SessionEnd | Action::SessionPrune { .. } | Action::CursorPosition => {
1422 PolicyClass::Benign
1423 }
1424 Action::Screenshot { .. }
1425 | Action::Zoom { .. }
1426 | Action::Tree
1427 | Action::GetState
1428 | Action::PolicyCheck
1429 | Action::MenuList { .. }
1430 | Action::DockList { .. }
1431 | Action::Spaces { .. }
1432 | Action::BrowserTabs { .. }
1433 | Action::BrowserText { .. }
1434 | Action::BrowserActivate { .. }
1435 | Action::Ocr { .. } => PolicyClass::Observe,
1436 Action::Quit { .. } => PolicyClass::Quit,
1437 Action::Key { .. } | Action::HoldKey { .. } => PolicyClass::Key,
1438 Action::ClipboardGet => PolicyClass::Benign,
1439 Action::ClipboardSet { .. } => PolicyClass::ClipboardSet,
1440 _ => PolicyClass::Mutate,
1441 }
1442}
1443
1444struct Prepared {
1445 stop: Option<Response>,
1446 warning: Option<String>,
1447 subject: policy::Subject,
1448 decision: &'static str,
1449 source: &'static str,
1450}
1451
1452fn early_stop(warning: Option<String>, subject: policy::Subject, response: Response) -> Prepared {
1453 Prepared {
1454 stop: Some(response),
1455 warning,
1456 subject,
1457 decision: "allow",
1458 source: "default",
1459 }
1460}
1461
1462fn prepare_action(
1463 backend: &dyn DesktopBackend,
1464 action: &Action,
1465 context: &CallContext,
1466) -> Prepared {
1467 let class = policy_class(action);
1468 let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1469 Ok(policy) => policy,
1470 Err(error) => {
1471 if matches!(class, PolicyClass::Always) {
1472 policy::LoadedPolicy::default()
1473 } else {
1474 return early_stop(
1475 None,
1476 policy::Subject::default(),
1477 Response::from_error(error),
1478 );
1479 }
1480 }
1481 };
1482 let warning = if loaded.pause_on_user_input {
1483 backend.user_input_pause_warning().map(str::to_string)
1484 } else {
1485 None
1486 };
1487 if matches!(action, Action::Pause | Action::Resume | Action::PolicyCheck) {
1488 return Prepared {
1489 stop: None,
1490 warning,
1491 subject: policy::Subject::default(),
1492 decision: "allow",
1493 source: "default",
1494 };
1495 }
1496 let subject = match resolve_subject(backend, action, context) {
1497 Ok(subject) => subject,
1498 Err(error) => {
1499 return early_stop(
1500 warning,
1501 policy::Subject::default(),
1502 Response::from_error(error),
1503 )
1504 }
1505 };
1506 let policy_action = match action {
1507 Action::Quit { .. } => policy::PolicyAction::Quit,
1508 Action::Key { keys } | Action::HoldKey { keys, .. } => policy::PolicyAction::Key {
1509 combo: keys.clone(),
1510 },
1511 Action::Screenshot { .. } | Action::Zoom { .. } | Action::Tree | Action::GetState => {
1512 policy::PolicyAction::Observe
1513 }
1514 Action::Ocr { .. } => policy::PolicyAction::Observe,
1515 Action::ClipboardSet { .. } => policy::PolicyAction::ClipboardSet,
1516 _ => match class {
1517 PolicyClass::Always => policy::PolicyAction::Always,
1518 PolicyClass::Benign => policy::PolicyAction::Benign,
1519 PolicyClass::Mutate => policy::PolicyAction::Mutate,
1520 PolicyClass::Observe => policy::PolicyAction::Observe,
1521 PolicyClass::Quit => policy::PolicyAction::Quit,
1522 PolicyClass::Key => policy::PolicyAction::Key {
1523 combo: String::new(),
1524 },
1525 PolicyClass::ClipboardSet => policy::PolicyAction::ClipboardSet,
1526 },
1527 };
1528 let decision = policy::decide(&loaded, &subject, &policy_action);
1529 if let policy::Decision::Deny { message, source } = &decision {
1530 return Prepared {
1531 stop: Some(Response::from_error(ComputerError::PolicyDenied(
1532 message.clone(),
1533 ))),
1534 warning,
1535 subject,
1536 decision: "deny",
1537 source,
1538 };
1539 }
1540 if let Err(error) = refuse_blocked_url(backend, &loaded, &subject) {
1541 let (decision_name, source) = decision_record(&decision);
1542 return Prepared {
1543 stop: Some(Response::from_error(error)),
1544 warning,
1545 subject,
1546 decision: decision_name,
1547 source,
1548 };
1549 }
1550 let mutating = matches!(
1551 class,
1552 PolicyClass::Mutate | PolicyClass::Quit | PolicyClass::Key | PolicyClass::ClipboardSet
1553 );
1554 if mutating && policy::is_paused() {
1555 let (decision_name, source) = decision_record(&decision);
1556 return Prepared {
1557 stop: Some(Response::from_error(ComputerError::Paused(
1558 "local input paused mutating actions; run computer-use resume".into(),
1559 ))),
1560 warning,
1561 subject,
1562 decision: decision_name,
1563 source,
1564 };
1565 }
1566 if let Action::Key { .. } | Action::HoldKey { .. } = action {
1567 if let Some(index) = context.element {
1568 if let Ok(Some(peek)) = backend.peek_element(
1569 context.app.as_deref(),
1570 context.window.as_deref(),
1571 context.session.as_deref(),
1572 index,
1573 ) {
1574 if peek.secure {
1575 return Prepared {
1576 stop: Some(Response::from_error(ComputerError::PolicyDenied(
1577 "refusing a key event aimed at a secure field".into(),
1578 ))),
1579 warning,
1580 subject,
1581 decision: "deny",
1582 source: "builtin",
1583 };
1584 }
1585 }
1586 }
1587 }
1588 if let policy::Decision::Confirm { reason } = &decision {
1589 if let Err(error) = confirm_action(context, &loaded, &subject, action_label(action), reason)
1590 {
1591 return Prepared {
1592 stop: Some(Response::from_error(error)),
1593 warning,
1594 subject,
1595 decision: "confirm",
1596 source: "builtin",
1597 };
1598 }
1599 }
1600 let (decision_name, source) = decision_record(&decision);
1601 Prepared {
1602 stop: None,
1603 warning,
1604 subject,
1605 decision: decision_name,
1606 source,
1607 }
1608}
1609
1610fn decision_record(decision: &policy::Decision) -> (&'static str, &'static str) {
1611 match decision {
1612 policy::Decision::Allow { source } => ("allow", *source),
1613 policy::Decision::Deny { source, .. } => ("deny", *source),
1614 policy::Decision::Confirm { .. } => ("confirm", "builtin"),
1615 }
1616}
1617
1618fn confirm_action(
1619 context: &CallContext,
1620 loaded: &policy::LoadedPolicy,
1621 subject: &policy::Subject,
1622 action: &str,
1623 reason: &str,
1624) -> Result<(), ComputerError> {
1625 let command = policy::confirm_program(loaded, context.confirm_cmd.as_deref());
1626 policy::run_confirm(
1627 command.as_deref(),
1628 loaded.timeout_secs(),
1629 &policy::ConfirmDocument {
1630 action,
1631 bundle_id: subject.bundle_id.as_deref(),
1632 path: subject.path.as_deref(),
1633 pid: subject.pid,
1634 window_id: subject.window_id,
1635 element: context.element,
1636 reason,
1637 },
1638 )
1639}
1640
1641fn refuse_blocked_url(
1642 backend: &dyn DesktopBackend,
1643 loaded: &policy::LoadedPolicy,
1644 subject: &policy::Subject,
1645) -> Result<(), ComputerError> {
1646 if loaded.blocked_urls().is_empty() {
1647 return Ok(());
1648 }
1649 if !policy::is_browser(
1650 subject.bundle_id.as_deref(),
1651 subject.path.as_deref().map(Path::new),
1652 subject.name.as_deref(),
1653 ) {
1654 return Ok(());
1655 }
1656 let from_title = subject
1657 .window_title
1658 .as_deref()
1659 .and_then(policy::url_in_text)
1660 .map(str::to_string);
1661 let url = if let Some(url) = from_title {
1662 Some(url)
1663 } else if let Some(pid) = subject.pid {
1664 backend.browser_url(pid)?
1665 } else {
1666 None
1667 };
1668 if let Some(url) = url.as_deref() {
1669 if let Some(pattern) = policy::url_blocked(url, loaded.blocked_urls()) {
1670 return Err(ComputerError::BlockedUrl(format!("matched {pattern}")));
1671 }
1672 }
1673 Ok(())
1674}
1675
1676fn resolve_subject(
1677 backend: &dyn DesktopBackend,
1678 action: &Action,
1679 context: &CallContext,
1680) -> Result<policy::Subject, ComputerError> {
1681 let class = policy_class(action);
1682 if matches!(class, PolicyClass::Always | PolicyClass::Benign) {
1683 return Ok(policy::Subject::default());
1684 }
1685 let screenshot = matches!(action, Action::Screenshot { .. } | Action::Zoom { .. });
1686 let frontmost_observe =
1687 matches!(action, Action::Tree | Action::GetState) && context.app.is_none();
1688 let app_text = context.app.clone().or_else(|| {
1689 if frontmost_observe {
1690 backend.frontmost().app
1691 } else {
1692 None
1693 }
1694 });
1695 if app_text.is_none() && context.window.is_none() {
1696 return Ok(policy::Subject::default());
1697 }
1698 if app_text.is_none() && screenshot {
1699 return subject_from_window(backend, context);
1700 }
1701 let Some(app_text) = app_text else {
1702 return Ok(policy::Subject::default());
1703 };
1704 let selector = parse_app_selector(&app_text)?;
1705 let fallback = subject_from_selector(&selector);
1706 let apps = match backend.list_apps(true) {
1707 Ok(apps) => apps,
1708 Err(error) => {
1709 if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1710 return Ok(fallback);
1711 }
1712 return Err(error);
1713 }
1714 };
1715 let info = match resolve_app(&apps, &selector) {
1716 Ok(info) => info.clone(),
1717 Err(error) => {
1718 if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1719 return Ok(fallback);
1720 }
1721 if context.app.is_none() {
1724 return Ok(policy::Subject::default());
1725 }
1726 return Err(error);
1727 }
1728 };
1729 let mut subject = policy::Subject {
1730 name: Some(info.name.clone()),
1731 bundle_id: info.bundle_id.clone(),
1732 path: info.path.clone(),
1733 pid: Some(info.pid),
1734 window_id: None,
1735 window_title: None,
1736 ax_identifier: None,
1737 };
1738 let want_window = context.window.is_some()
1739 || matches!(
1740 action,
1741 Action::Tree | Action::GetState | Action::Screenshot { .. } | Action::Zoom { .. }
1742 );
1743 if want_window {
1744 if let Ok(listed) = backend.list_windows() {
1745 let owned = filter_windows_for_pid(&listed.windows, info.pid);
1746 let chosen = match context.window.as_deref() {
1747 Some(window) => {
1748 let selector = parse_window_selector(window)?;
1749 Some(resolve_window(&owned, &selector)?.clone())
1750 }
1751 None => owned.first().cloned(),
1752 };
1753 if let Some(window) = chosen {
1754 subject.window_id = Some(window.id);
1755 if context.window.is_some()
1756 || matches!(
1757 action,
1758 Action::Tree
1759 | Action::GetState
1760 | Action::Screenshot { .. }
1761 | Action::Zoom { .. }
1762 )
1763 {
1764 subject.window_title = Some(window.title);
1765 }
1766 }
1767 }
1768 }
1769 Ok(subject)
1770}
1771
1772fn subject_from_window(
1773 backend: &dyn DesktopBackend,
1774 context: &CallContext,
1775) -> Result<policy::Subject, ComputerError> {
1776 let Some(window) = context.window.as_deref() else {
1777 return Ok(policy::Subject::default());
1778 };
1779 let listed = backend.list_windows()?;
1780 let selector = parse_window_selector(window)?;
1781 let window = resolve_window(&listed.windows, &selector)?.clone();
1782 let apps = backend.list_apps(true)?;
1783 let info = apps.into_iter().find(|app| app.pid == window.pid);
1784 Ok(policy::Subject {
1785 name: info
1786 .as_ref()
1787 .map(|app| app.name.clone())
1788 .or(Some(window.app.clone())),
1789 bundle_id: info.as_ref().and_then(|app| app.bundle_id.clone()),
1790 path: info.and_then(|app| app.path),
1791 pid: Some(window.pid),
1792 window_id: Some(window.id),
1793 window_title: Some(window.title),
1794 ax_identifier: None,
1795 })
1796}
1797
1798fn subject_from_selector(selector: &crate::target::AppSelector) -> policy::Subject {
1799 use crate::target::AppSelector;
1800 match selector {
1801 AppSelector::BundleId(bundle) => policy::Subject {
1802 bundle_id: Some(bundle.clone()),
1803 ..policy::Subject::default()
1804 },
1805 AppSelector::Name(name) => policy::Subject {
1806 name: Some(name.clone()),
1807 ..policy::Subject::default()
1808 },
1809 AppSelector::Path(path) => policy::Subject {
1810 path: Some(path.display().to_string()),
1811 ..policy::Subject::default()
1812 },
1813 AppSelector::Pid(pid) => policy::Subject {
1814 pid: Some(*pid),
1815 ..policy::Subject::default()
1816 },
1817 }
1818}
1819
1820fn policy_check(
1821 backend: &dyn DesktopBackend,
1822 context: &CallContext,
1823) -> (Response, policy::Subject) {
1824 let Some(app) = context
1825 .app
1826 .as_deref()
1827 .map(str::trim)
1828 .filter(|text| !text.is_empty())
1829 else {
1830 return (
1831 Response::from_error(ComputerError::InvalidArguments("app is required".into())),
1832 policy::Subject::default(),
1833 );
1834 };
1835 let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1836 Ok(policy) => policy,
1837 Err(error) => return (Response::from_error(error), policy::Subject::default()),
1838 };
1839 let selector = match parse_app_selector(app) {
1840 Ok(selector) => selector,
1841 Err(error) => return (Response::from_error(error), policy::Subject::default()),
1842 };
1843 let mut subject = subject_from_selector(&selector);
1844 if let Ok(apps) = backend.list_apps(true) {
1845 if let Ok(info) = resolve_app(&apps, &selector) {
1846 subject.name = Some(info.name.clone());
1847 subject.bundle_id = info.bundle_id.clone();
1848 subject.path = info.path.clone();
1849 subject.pid = Some(info.pid);
1850 }
1851 }
1852 let decision = policy::app_verdict(&loaded, &subject);
1853 let (name, source) = match decision {
1854 policy::Decision::Allow { source } => ("allow", source),
1855 policy::Decision::Deny { source, .. } => ("deny", source),
1856 policy::Decision::Confirm { .. } => ("allow", "default"),
1857 };
1858 (
1859 Response::success(json!({
1860 "decision": name,
1861 "source": source,
1862 "bundle_id": subject.bundle_id,
1863 })),
1864 subject,
1865 )
1866}
1867
1868fn pause_response(paused: bool) -> Response {
1869 match policy::set_paused(paused) {
1870 Ok(()) => Response::success(json!({
1871 "action": if paused { "pause" } else { "resume" },
1872 "paused": paused,
1873 })),
1874 Err(error) => Response::from_error(error),
1875 }
1876}
1877
1878fn result_response(result: Result<Value, ComputerError>) -> Response {
1879 match result {
1880 Ok(value) => Response::success(value),
1881 Err(error) => Response::from_error(error),
1882 }
1883}
1884
1885fn post_state_kind(action: &Action) -> PostKind {
1886 match action {
1887 Action::Screenshot { .. } | Action::Zoom { .. } => PostKind::Screenshot,
1888 Action::Click { .. }
1889 | Action::DoubleClick { .. }
1890 | Action::TripleClick { .. }
1891 | Action::Drag { .. }
1892 | Action::MouseDown { .. }
1893 | Action::MouseUp { .. }
1894 | Action::HoldKey { .. }
1895 | Action::ClipboardSet { .. }
1896 | Action::Press { .. }
1897 | Action::Focus
1898 | Action::SetValue { .. }
1899 | Action::SelectText { .. }
1900 | Action::PerformAction { .. }
1901 | Action::ScrollTo
1902 | Action::Move { .. }
1903 | Action::Type { .. }
1904 | Action::Key { .. }
1905 | Action::Scroll { .. }
1906 | Action::Launch
1907 | Action::Activate
1908 | Action::Quit { .. }
1909 | Action::Batch { .. }
1910 | Action::MenuClick { .. }
1911 | Action::AppleEvent { .. }
1912 | Action::ShortcutsRun { .. }
1913 | Action::DockClick { .. }
1914 | Action::BrowserNavigate { .. }
1915 | Action::BrowserClick { .. }
1916 | Action::BrowserText { .. }
1917 | Action::BrowserType { .. }
1918 | Action::BrowserEval { .. }
1919 | Action::BrowserActivate { .. }
1920 | Action::Ocr { .. } => PostKind::Mutating,
1921 Action::GetState
1922 | Action::Wait { .. }
1923 | Action::Displays
1924 | Action::Isolation
1925 | Action::ListApps { .. }
1926 | Action::ListWindows
1927 | Action::Doctor { .. }
1928 | Action::Tree
1929 | Action::SessionEnd
1930 | Action::SessionPrune { .. }
1931 | Action::CursorPosition
1932 | Action::ClipboardGet
1933 | Action::PolicyCheck
1934 | Action::Pause
1935 | Action::Resume
1936 | Action::MenuList { .. }
1937 | Action::DockList { .. }
1938 | Action::Spaces { .. }
1939 | Action::BrowserTabs { .. } => PostKind::ReadOnly,
1940 }
1941}
1942
1943fn remap_coordinates(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1944 match context.coord.as_deref() {
1945 None | Some("screen") => Ok(action),
1946 Some("image") => remap_image_action(context, action),
1947 Some(other) => Err(ComputerError::InvalidArguments(format!(
1948 "coord must be screen or image (got {other})"
1949 ))),
1950 }
1951}
1952
1953fn remap_image_action(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1954 match action {
1955 Action::Click {
1956 x,
1957 y,
1958 button,
1959 double,
1960 count,
1961 } => {
1962 let (x, y) = remap_optional_point(context, x, y)?;
1963 Ok(Action::Click {
1964 x,
1965 y,
1966 button,
1967 double,
1968 count,
1969 })
1970 }
1971 Action::DoubleClick { x, y, button } => {
1972 let (x, y) = remap_optional_point(context, x, y)?;
1973 Ok(Action::DoubleClick { x, y, button })
1974 }
1975 Action::TripleClick { x, y, button } => {
1976 let (x, y) = remap_optional_point(context, x, y)?;
1977 Ok(Action::TripleClick { x, y, button })
1978 }
1979 Action::Drag { path, button } => {
1980 let mut mapped = Vec::with_capacity(path.len());
1981 for point in path {
1982 let (x, y) = remap_point(context, point.x, point.y)?;
1983 mapped.push(Point { x, y });
1984 }
1985 Ok(Action::Drag {
1986 path: mapped,
1987 button,
1988 })
1989 }
1990 Action::MouseDown { x, y, button } => {
1991 let (x, y) = remap_point(context, x, y)?;
1992 Ok(Action::MouseDown { x, y, button })
1993 }
1994 Action::MouseUp { x, y, button } => {
1995 let (x, y) = remap_point(context, x, y)?;
1996 Ok(Action::MouseUp { x, y, button })
1997 }
1998 Action::Move { x, y } => {
1999 let (x, y) = remap_point(context, x, y)?;
2000 Ok(Action::Move { x, y })
2001 }
2002 Action::Scroll {
2003 x,
2004 y,
2005 dx,
2006 dy,
2007 direction,
2008 amount,
2009 } => {
2010 let (x, y) = remap_optional_point(context, x, y)?;
2011 Ok(Action::Scroll {
2012 x,
2013 y,
2014 dx,
2015 dy,
2016 direction,
2017 amount,
2018 })
2019 }
2020 Action::Screenshot {
2021 output,
2022 region,
2023 format,
2024 max_width,
2025 canvas,
2026 base64,
2027 marks,
2028 } => {
2029 let region = match region {
2030 Some(region) => Some(remap_region(context, region)?),
2031 None => None,
2032 };
2033 Ok(Action::Screenshot {
2034 output,
2035 region,
2036 format,
2037 max_width,
2038 canvas,
2039 base64,
2040 marks,
2041 })
2042 }
2043 Action::Zoom {
2044 output,
2045 region,
2046 format,
2047 max_width,
2048 canvas,
2049 base64,
2050 } => {
2051 let region = remap_region(context, region)?;
2052 Ok(Action::Zoom {
2053 output,
2054 region,
2055 format,
2056 max_width,
2057 canvas,
2058 base64,
2059 })
2060 }
2061 other => Ok(other),
2062 }
2063}
2064
2065fn gate_display(
2072 backend: &dyn DesktopBackend,
2073 context: &CallContext,
2074 action: &Action,
2075) -> Result<(), ComputerError> {
2076 let Some(spec) = context.display.as_deref() else {
2077 return Ok(());
2078 };
2079 crate::isolation::parse_selector(spec)?;
2084 if matches!(action, Action::Isolation | Action::Displays) {
2086 return Ok(());
2087 }
2088 let displays = backend.list_displays()?;
2089 let target = crate::isolation::resolve(spec, &displays)?;
2090 match action {
2091 Action::Click { x, y, .. }
2094 | Action::DoubleClick { x, y, .. }
2095 | Action::TripleClick { x, y, .. }
2096 | Action::Scroll { x, y, .. } => match (x, y) {
2097 (Some(x), Some(y)) => crate::isolation::ensure_point_inside(&target, *x, *y),
2098 _ => Ok(()),
2099 },
2100 Action::Move { x, y } | Action::MouseDown { x, y, .. } | Action::MouseUp { x, y, .. } => {
2101 crate::isolation::ensure_point_inside(&target, *x, *y)
2102 }
2103 Action::Drag { path, .. } => {
2104 for point in path {
2105 crate::isolation::ensure_point_inside(&target, point.x, point.y)?;
2106 }
2107 Ok(())
2108 }
2109 Action::Screenshot {
2110 region: Some(region),
2111 ..
2112 } => crate::isolation::ensure_region_inside(&target, *region),
2113 Action::Screenshot { .. } => Ok(()),
2114 Action::Zoom { region, .. } => crate::isolation::ensure_region_inside(&target, *region),
2115 _ => Ok(()),
2116 }
2117}
2118
2119fn remap_optional_point(
2120 context: &CallContext,
2121 x: Option<i32>,
2122 y: Option<i32>,
2123) -> Result<(Option<i32>, Option<i32>), ComputerError> {
2124 match (x, y) {
2125 (Some(x), Some(y)) => {
2126 let (x, y) = remap_point(context, x, y)?;
2127 Ok((Some(x), Some(y)))
2128 }
2129 other => Ok(other),
2130 }
2131}
2132
2133fn remap_point(context: &CallContext, x: i32, y: i32) -> Result<(i32, i32), ComputerError> {
2134 let map = require_screenshot_map(context)?;
2135 ensure_point_inside(&map, x, y)?;
2136 let (screen_x, screen_y) = image_util::image_to_screen(&map, f64::from(x), f64::from(y));
2137 Ok((round_screen(screen_x)?, round_screen(screen_y)?))
2138}
2139
2140fn remap_region(context: &CallContext, region: Region) -> Result<Region, ComputerError> {
2141 let map = require_screenshot_map(context)?;
2142 ensure_region_inside(&map, region)?;
2143 let x2 = i64::from(region.x) + i64::from(region.width);
2144 let y2 = i64::from(region.y) + i64::from(region.height);
2145 let (screen_x1, screen_y1) =
2146 image_util::image_to_screen(&map, f64::from(region.x), f64::from(region.y));
2147 let (screen_x2, screen_y2) = image_util::image_to_screen(&map, x2 as f64, y2 as f64);
2148 let x = round_screen(screen_x1)?;
2149 let y = round_screen(screen_y1)?;
2150 let far_x = round_screen(screen_x2)?;
2151 let far_y = round_screen(screen_y2)?;
2152 let width = u32::try_from(i64::from(far_x) - i64::from(x))
2153 .unwrap_or(0)
2154 .max(1);
2155 let height = u32::try_from(i64::from(far_y) - i64::from(y))
2156 .unwrap_or(0)
2157 .max(1);
2158 Region::new(x, y, width, height)
2159}
2160
2161fn require_screenshot_map(
2162 context: &CallContext,
2163) -> Result<image_util::ScreenshotMap, ComputerError> {
2164 match cache::load_screenshot_map(context.session.as_deref())? {
2165 Some(map) => Ok(map),
2166 None => Err(ComputerError::InvalidArguments(
2167 "take a screenshot first".into(),
2168 )),
2169 }
2170}
2171
2172fn content_box(map: &image_util::ScreenshotMap) -> (i64, i64, i64, i64) {
2173 let left = i64::from(map.pad.x);
2174 let top = i64::from(map.pad.y);
2175 let right = left + i64::from(map.content_width);
2176 let bottom = top + i64::from(map.content_height);
2177 (left, top, right, bottom)
2178}
2179
2180fn ensure_point_inside(
2181 map: &image_util::ScreenshotMap,
2182 x: i32,
2183 y: i32,
2184) -> Result<(), ComputerError> {
2185 let (left, top, right, bottom) = content_box(map);
2186 let x = i64::from(x);
2187 let y = i64::from(y);
2188 if x < left || y < top || x >= right || y >= bottom {
2189 return Err(ComputerError::InvalidArguments(format!(
2190 "coordinate ({x}, {y}) is outside the screenshot"
2191 )));
2192 }
2193 Ok(())
2194}
2195
2196fn ensure_region_inside(
2197 map: &image_util::ScreenshotMap,
2198 region: Region,
2199) -> Result<(), ComputerError> {
2200 let (left, top, right, bottom) = content_box(map);
2201 let x1 = i64::from(region.x);
2202 let y1 = i64::from(region.y);
2203 let x2 = x1 + i64::from(region.width);
2204 let y2 = y1 + i64::from(region.height);
2205 if x1 < left || y1 < top || x1 >= right || y1 >= bottom || x2 > right || y2 > bottom {
2206 return Err(ComputerError::InvalidArguments(format!(
2207 "region {},{} {}x{} is outside the screenshot",
2208 region.x, region.y, region.width, region.height
2209 )));
2210 }
2211 Ok(())
2212}
2213
2214fn round_screen(value: f64) -> Result<i32, ComputerError> {
2215 if !value.is_finite() {
2216 return Err(ComputerError::InvalidArguments(
2217 "coordinate remap was not finite".into(),
2218 ));
2219 }
2220 let rounded = value.round();
2221 if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) {
2222 return Err(ComputerError::InvalidArguments(
2223 "coordinate remap is out of range".into(),
2224 ));
2225 }
2226 Ok(rounded as i32)
2227}
2228
2229fn run_batch(
2230 backend: &dyn DesktopBackend,
2231 context: &CallContext,
2232 actions: Vec<Value>,
2233 stop_on_error: bool,
2234) -> Response {
2235 let _session = InputSession::open(backend);
2237 let mut results = Vec::with_capacity(actions.len());
2238 let mut first_error: Option<(usize, ErrorBody)> = None;
2239 for (index, step) in actions.into_iter().enumerate() {
2240 let response = dispatch_batch_step(backend, context, step);
2241 let failed = !response.ok;
2242 if failed && first_error.is_none() {
2243 if let Some(error) = response.error.clone() {
2244 first_error = Some((index, error));
2245 }
2246 }
2247 results.push(response.to_value());
2248 if failed && stop_on_error {
2249 return attach_post_state(
2250 backend,
2251 context,
2252 PostKind::Mutating,
2253 batch_response(results, Some(index), first_error),
2254 );
2255 }
2256 }
2257 attach_post_state(
2258 backend,
2259 context,
2260 PostKind::Mutating,
2261 batch_response(results, None, first_error),
2262 )
2263}
2264
2265fn dispatch_batch_step(
2266 backend: &dyn DesktopBackend,
2267 outer: &CallContext,
2268 value: Value,
2269) -> Response {
2270 match prepare_batch_step(outer, value) {
2271 Ok((action, context)) => dispatch_with_context(backend, action, &context),
2272 Err(error) => Response::from_error(error),
2273 }
2274}
2275
2276fn prepare_batch_step(
2277 outer: &CallContext,
2278 mut value: Value,
2279) -> Result<(Action, CallContext), ComputerError> {
2280 let Some(object) = value.as_object_mut() else {
2281 return Err(ComputerError::InvalidArguments(
2282 "batch step must be a JSON object".into(),
2283 ));
2284 };
2285 if object.get("action").and_then(Value::as_str) == Some("batch") {
2286 return Err(ComputerError::InvalidArguments(
2287 "nested batch is not allowed".into(),
2288 ));
2289 }
2290 let mut context = outer.clone();
2291 overlay_context(&mut context, object)?;
2292 context.return_state = false;
2293 context.include_screenshot = false;
2294 let action = serde_json::from_value(value)
2295 .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
2296 Ok((action, context))
2297}
2298
2299fn overlay_context(
2300 context: &mut CallContext,
2301 object: &mut Map<String, Value>,
2302) -> Result<(), ComputerError> {
2303 if object.contains_key("app") {
2304 context.app = take_stringish(object, "app")?;
2305 }
2306 if object.contains_key("window") {
2307 context.window = take_stringish(object, "window")?;
2308 }
2309 if object.contains_key("element") {
2310 context.element = take_optional_u32(object, "element")?;
2311 }
2312 if object.contains_key("diff") {
2313 context.diff = take_bool(object, "diff")?;
2314 }
2315 if object.contains_key("full") {
2316 context.full = take_bool(object, "full")?;
2317 }
2318 let _ = object.remove("return_state");
2319 let _ = object.remove("include_screenshot");
2320 let _ = object.remove("id");
2321 let _ = object.remove("stop_on_error");
2322 if object.contains_key("coord") {
2323 context.coord = take_stringish(object, "coord")?;
2324 }
2325 if object.contains_key("session") {
2326 context.session = take_stringish(object, "session")?;
2327 }
2328 if object.contains_key("tier") {
2329 context.tier = take_stringish(object, "tier")?;
2330 }
2331 if object.contains_key("adapter") {
2332 context.adapter = take_stringish(object, "adapter")?;
2333 }
2334 if object.contains_key("display") {
2335 context.display = take_stringish(object, "display")?;
2336 }
2337 if object.contains_key("require_visible") {
2338 context.require_visible = take_bool(object, "require_visible")?;
2339 }
2340 if object.contains_key("allow_foreground") {
2341 context.allow_foreground = take_bool(object, "allow_foreground")?;
2342 }
2343 if object.contains_key("require_background") {
2344 context.require_background = take_bool(object, "require_background")?;
2345 }
2346 if object.contains_key("no_overlay") {
2347 context.no_overlay = take_bool(object, "no_overlay")?;
2348 }
2349 Ok(())
2350}
2351
2352fn batch_response(
2353 results: Vec<Value>,
2354 stopped_at: Option<usize>,
2355 first_error: Option<(usize, ErrorBody)>,
2356) -> Response {
2357 let mut fields = Map::new();
2358 fields.insert("action".into(), json!("batch"));
2359 fields.insert("results".into(), Value::Array(results));
2360 if let Some(index) = stopped_at {
2361 fields.insert("stopped_at".into(), json!(index));
2362 }
2363 match first_error {
2364 Some((index, error)) => {
2365 let mut response = Response::failure(
2366 error.code,
2367 format!("batch failed at action {index}: {}", error.message),
2368 );
2369 response.fields = fields;
2370 response
2371 }
2372 None => Response {
2373 ok: true,
2374 id: None,
2375 error: None,
2376 fields,
2377 },
2378 }
2379}
2380
2381fn attach_post_state(
2382 backend: &dyn DesktopBackend,
2383 context: &CallContext,
2384 kind: PostKind,
2385 mut response: Response,
2386) -> Response {
2387 if !context.return_state || matches!(kind, PostKind::ReadOnly) {
2388 return response;
2389 }
2390 let mut warnings = Vec::new();
2391 let mut state = match backend.tree(&tree_query(context)) {
2392 Ok(report) => json!({
2393 "app": report.app,
2394 "pid": report.pid,
2395 "window_id": report.window_id,
2396 "revision": report.revision,
2397 "tree": report.full_text,
2398 "diff": report.diff,
2399 "truncated": report.truncated,
2400 }),
2401 Err(error) => {
2402 if !response.ok {
2403 return response;
2404 }
2405 warnings.push(format!("tree snapshot failed: {error}"));
2406 json!({
2407 "app": null,
2408 "pid": null,
2409 "window_id": null,
2410 "revision": null,
2411 "tree": null,
2412 "diff": null,
2413 "truncated": null,
2414 })
2415 }
2416 };
2417 if context.include_screenshot {
2418 let shot = if response.ok && matches!(kind, PostKind::Screenshot) {
2419 Ok(Value::Object(response.fields.clone()))
2420 } else if matches!(kind, PostKind::Screenshot) {
2421 Err(ComputerError::ExecutionFailed(
2422 response
2423 .error
2424 .as_ref()
2425 .map(|error| error.message.clone())
2426 .unwrap_or_else(|| "screenshot failed".into()),
2427 ))
2428 } else {
2429 screenshot(
2430 backend,
2431 context,
2432 ScreenshotArgs {
2433 output: None,
2434 region: None,
2435 format: None,
2436 max_width: None,
2437 canvas: None,
2438 include_base64: false,
2439 window_crop: false,
2440 marks: false,
2441 },
2442 )
2443 };
2444 match shot {
2445 Ok(value) => state["screenshot"] = value,
2446 Err(error) => {
2447 state["screenshot"] = Value::Null;
2448 warnings.push(format!("screenshot failed: {error}"));
2449 }
2450 }
2451 }
2452 response.fields.insert("state".into(), state);
2453 push_warnings(&mut response, warnings);
2454 response
2455}
2456
2457fn push_warnings(response: &mut Response, extra: Vec<String>) {
2458 if extra.is_empty() {
2459 return;
2460 }
2461 let mut warnings = match response.fields.remove("warnings") {
2462 Some(Value::Array(items)) => items,
2463 Some(other) => vec![other],
2464 None => Vec::new(),
2465 };
2466 for warning in extra {
2467 warnings.push(json!(warning));
2468 }
2469 response
2470 .fields
2471 .insert("warnings".into(), Value::Array(warnings));
2472}
2473
2474fn cursor_position(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2475 let (x, y) = backend.cursor_position()?;
2476 Ok(json!({"action": "cursor_position", "x": x, "y": y}))
2477}
2478
2479fn invalid_arguments(message: &str) -> ComputerError {
2480 ComputerError::InvalidArguments(message.to_string())
2481}
2482
2483fn unsupported_error(operation: &str) -> ComputerError {
2484 ComputerError::Unsupported(format!(
2485 "{operation} is not available on {}",
2486 std::env::consts::OS
2487 ))
2488}
2489
2490fn menu_list(backend: &dyn DesktopBackend, app: &str, depth: u32) -> Result<Value, ComputerError> {
2491 let report = backend.menu_list(app, depth)?;
2492 Ok(json!({
2493 "action": "menu_list",
2494 "app": report.app,
2495 "sections": report.sections,
2496 }))
2497}
2498
2499fn menu_click(backend: &dyn DesktopBackend, app: &str, path: &str) -> Result<Value, ComputerError> {
2500 let parsed = crate::menu_path::parse(path)?;
2501 let result = backend.menu_click(app, &parsed)?;
2502 Ok(json!({
2503 "action": "menu_click",
2504 "app": app,
2505 "path": parsed.segments,
2506 "pressed": result.pressed,
2507 }))
2508}
2509
2510fn apple_event_action(
2511 backend: &dyn DesktopBackend,
2512 app: &str,
2513 script: &str,
2514 language: AppleEventLanguage,
2515) -> Result<Value, ComputerError> {
2516 let report = backend.apple_event(app, script, language)?;
2517 Ok(json!({
2518 "action": "apple_event",
2519 "app": app,
2520 "language": language.as_str(),
2521 "script_bytes": report.script_bytes,
2522 "script_hash": report.script_hash,
2523 "stdout": report.stdout,
2524 "stderr": report.stderr,
2525 "exit_status": report.exit_status,
2526 }))
2527}
2528
2529fn shortcuts_run_action(name: String) -> Result<Value, ComputerError> {
2530 let report = backend_shortcuts_run(&name)?;
2531 Ok(json!({
2532 "action": "shortcuts_run",
2533 "name": name,
2534 "exit_status": report.exit_status,
2535 "stderr": report.stderr,
2536 }))
2537}
2538
2539fn dock_list_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2540 let report = backend.dock_list()?;
2541 Ok(json!({
2542 "action": "dock_list",
2543 "items": report.items,
2544 }))
2545}
2546
2547fn dock_click_action(backend: &dyn DesktopBackend, title: &str) -> Result<Value, ComputerError> {
2548 let report = backend.dock_click(title)?;
2549 Ok(json!({
2550 "action": "dock_click",
2551 "title": title,
2552 "pressed": report.pressed,
2553 }))
2554}
2555
2556fn spaces_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2557 let report = backend.spaces()?;
2558 Ok(json!({
2559 "action": "spaces",
2560 "screens": report.screens,
2561 "note": report.note,
2562 }))
2563}
2564
2565fn backend_shortcuts_run(name: &str) -> Result<crate::shortcuts::ShortcutsReport, ComputerError> {
2568 crate::shortcuts::run(name)
2569}
2570
2571fn session_prune(dry_run: bool, ttl_secs: Option<u64>) -> Result<Value, ComputerError> {
2572 let ttl = match ttl_secs {
2573 Some(seconds) => Duration::from_secs(seconds),
2574 None => cache::DEFAULT_SESSION_TTL,
2575 };
2576 let report = cache::prune_sessions(
2577 &cache::sessions_root(),
2578 &cache::PruneOptions {
2579 dry_run,
2580 ttl,
2581 now: SystemTime::now(),
2582 },
2583 &cache::pid_alive,
2584 )?;
2585 Ok(json!({
2586 "action": "session_prune",
2587 "dry_run": dry_run,
2588 "ttl_secs": ttl.as_secs(),
2589 "removed": report.removed,
2590 "kept": report.kept,
2591 }))
2592}
2593
2594struct ScreenshotArgs {
2595 output: Option<String>,
2596 region: Option<Region>,
2597 format: Option<ImageFormat>,
2598 max_width: Option<u32>,
2599 canvas: Option<String>,
2600 include_base64: bool,
2601 window_crop: bool,
2604 marks: bool,
2606}
2607
2608fn screenshot(
2609 backend: &dyn DesktopBackend,
2610 context: &CallContext,
2611 args: ScreenshotArgs,
2612) -> Result<Value, ComputerError> {
2613 let ScreenshotArgs {
2614 output,
2615 region,
2616 format,
2617 max_width,
2618 canvas,
2619 include_base64,
2620 window_crop,
2621 marks,
2622 } = args;
2623 let targeted = context.app.is_some() || context.window.is_some();
2624 if region.is_some() && targeted && !window_crop {
2625 return Err(ComputerError::InvalidArguments(
2626 "pass a region or a window target, not both".into(),
2627 ));
2628 }
2629 let format = format.unwrap_or_default();
2630 let max_width = max_width.unwrap_or(image_util::DEFAULT_SCREENSHOT_MAX_WIDTH);
2631 let canvas = canvas
2632 .as_deref()
2633 .map(image_util::parse_canvas)
2634 .transpose()?;
2635 let raw = if targeted {
2636 if let Some(error) = backend.display_block() {
2637 return Err(error);
2638 }
2639 let target = resolve_screenshot_window(backend, context)?;
2640 if let Some(filter) = display_filter(backend, context)? {
2643 if !crate::isolation::window_inside(&filter, target.bounds) {
2644 return Err(ComputerError::InvalidArguments(format!(
2645 "window {}x{}+{}+{} is not inside display {} ({} covers {})",
2646 target.bounds.width,
2647 target.bounds.height,
2648 target.bounds.x,
2649 target.bounds.y,
2650 filter.display.id,
2651 filter.display.name,
2652 filter.bounds()
2653 )));
2654 }
2655 }
2656 let captured = backend.capture_window(&target)?;
2657 match region {
2658 Some(region) if window_crop => crop_window_region(captured, region)?,
2659 _ => captured,
2660 }
2661 } else {
2662 let region = match display_filter(backend, context)? {
2666 Some(filter) => Some(filter.region),
2667 None => region,
2668 };
2669 backend.capture_screen(region)?
2670 };
2671 let prepared = image_util::prepare_shot(
2672 raw.frame.image(),
2673 image_util::ShotFit {
2674 scale: raw.frame.scale(),
2675 origin_x: raw.origin_x,
2676 origin_y: raw.origin_y,
2677 logical_width: raw.logical_width,
2678 logical_height: raw.logical_height,
2679 max_width,
2680 canvas,
2681 },
2682 )?;
2683 let mut image = prepared.image;
2684 let mut warnings = raw.warnings;
2685 if marks {
2686 let applied = apply_marks(&mut image, &prepared.map, context, backend)?;
2687 if applied == 0 {
2688 warnings.push("marks: no accessibility element frame fell inside this capture".into());
2691 }
2692 }
2693 let encoded = image_util::encode_rgba(&image, format)?;
2694 let path = match output {
2695 Some(path) => PathBuf::from(path),
2696 None => default_screenshot_path(format),
2697 };
2698 if let Some(parent) = path.parent() {
2699 if !parent.as_os_str().is_empty() {
2700 fs::create_dir_all(parent)?;
2701 }
2702 }
2703 fs::write(&path, &encoded.bytes)?;
2704 if let Err(error) = cache::store_screenshot_map(context.session.as_deref(), &prepared.map) {
2705 warnings.push(format!("screenshot map was not cached: {error}"));
2706 }
2707 let mut body = serde_json::to_value(&prepared.map).map_err(|error| {
2708 ComputerError::ExecutionFailed(format!("screenshot map encode failed: {error}"))
2709 })?;
2710 body["path"] = json!(path_string(&path));
2711 body["format"] = json!(format.as_str());
2712 body["capture_source"] = json!(raw.capture_source);
2713 if raw.occluded {
2714 body["occluded"] = json!(true);
2715 }
2716 if !warnings.is_empty() {
2717 body["warnings"] = json!(warnings);
2718 }
2719 if include_base64 {
2720 body["data"] = json!(encode_base64(&encoded.bytes));
2721 }
2722 Ok(body)
2723}
2724
2725fn resolve_screenshot_window(
2726 backend: &dyn DesktopBackend,
2727 context: &CallContext,
2728) -> Result<WindowTarget, ComputerError> {
2729 let listed = backend.list_windows()?;
2730 if let Some(app) = context.app.as_deref() {
2731 let apps = backend.list_apps(true)?;
2732 let selector = parse_app_selector(app)?;
2733 let app = resolve_app(&apps, &selector)?;
2734 let owned = filter_windows_for_pid(&listed.windows, app.pid);
2735 let selector = match context.window.as_deref() {
2736 Some(window) => parse_window_selector(window)?,
2737 None => WindowSelector::Index(0),
2738 };
2739 let window = resolve_window(&owned, &selector)?;
2740 return Ok(window_target(window));
2741 }
2742 let selector = parse_window_selector(context.window.as_deref().unwrap_or(""))?;
2743 let window = resolve_window(&listed.windows, &selector)?;
2744 Ok(window_target(window))
2745}
2746
2747fn window_target(window: &crate::target::WindowInfo) -> WindowTarget {
2748 WindowTarget {
2749 window_id: window.id,
2750 pid: window.pid,
2751 bounds: window.bounds,
2752 }
2753}
2754
2755fn crop_window_region(
2757 raw: crate::types::RawCapture,
2758 region: Region,
2759) -> Result<crate::types::RawCapture, ComputerError> {
2760 region_inside(region, raw.logical_width, raw.logical_height)?;
2761 let scale = raw.frame.scale();
2762 let x = logical_to_device(i64::from(region.x), scale)?;
2763 let y = logical_to_device(i64::from(region.y), scale)?;
2764 let width = logical_len_to_device(region.width, scale)?;
2765 let height = logical_len_to_device(region.height, scale)?;
2766 let image = raw.frame.image();
2767 if x >= image.width() || y >= image.height() {
2768 return Err(ComputerError::InvalidArguments(
2769 "region is outside the capture".into(),
2770 ));
2771 }
2772 let width = width.min(image.width() - x).max(1);
2773 let height = height.min(image.height() - y).max(1);
2774 let cropped = image::imageops::crop_imm(image, x, y, width, height).to_image();
2775 Ok(crate::types::RawCapture {
2776 frame: Frame::from_image(cropped, scale),
2777 origin_x: raw.origin_x.saturating_add(region.x),
2778 origin_y: raw.origin_y.saturating_add(region.y),
2779 logical_width: region.width,
2780 logical_height: region.height,
2781 capture_source: raw.capture_source,
2782 occluded: raw.occluded,
2783 warnings: raw.warnings,
2784 })
2785}
2786
2787fn logical_to_device(value: i64, scale: f64) -> Result<u32, ComputerError> {
2788 if value < 0 {
2789 return Err(ComputerError::InvalidArguments(
2790 "region is outside the capture".into(),
2791 ));
2792 }
2793 let pixels = (value as f64) * scale;
2794 if !pixels.is_finite() || pixels < 0.0 || pixels > f64::from(u32::MAX) {
2795 return Err(ComputerError::ExecutionFailed(
2796 "region pixel coordinate is out of range".into(),
2797 ));
2798 }
2799 Ok(pixels.round() as u32)
2800}
2801
2802fn logical_len_to_device(value: u32, scale: f64) -> Result<u32, ComputerError> {
2803 let pixels = logical_to_device(i64::from(value), scale)?;
2804 Ok(pixels.max(1))
2805}
2806
2807fn default_screenshot_path(format: ImageFormat) -> PathBuf {
2808 static SEQUENCE: AtomicU64 = AtomicU64::new(0);
2809 let nanos = SystemTime::now()
2810 .duration_since(UNIX_EPOCH)
2811 .map(|duration| duration.as_nanos())
2812 .unwrap_or(0);
2813 let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed);
2814 let name = format!(
2815 "computer-use-{}-{nanos}-{sequence}.{}",
2816 std::process::id(),
2817 format.extension()
2818 );
2819 std::env::temp_dir().join(name)
2820}
2821
2822fn path_string(path: &Path) -> String {
2823 path.to_string_lossy().into_owned()
2824}
2825
2826fn click(
2827 backend: &dyn DesktopBackend,
2828 context: &CallContext,
2829 x: Option<i32>,
2830 y: Option<i32>,
2831 button: MouseButton,
2832 double: bool,
2833 count: u8,
2834) -> Response {
2835 let count = match effective_click_count(double, count) {
2836 Ok(count) => count,
2837 Err(error) => return Response::from_error(error),
2838 };
2839 match (context.element, x, y) {
2840 (Some(_), Some(_), _) | (Some(_), _, Some(_)) => {
2841 Response::failure("invalid_arguments", "pass element or x and y, not both")
2842 }
2843 (None, None, None) => Response::failure(
2844 "invalid_arguments",
2845 "click requires element, or both x and y",
2846 ),
2847 (None, Some(_), None) | (None, None, Some(_)) => {
2848 Response::failure("invalid_arguments", "x and y must be passed together")
2849 }
2850 (Some(_index), None, None) => {
2851 element_command(backend, context, ElementOp::Press { count, button })
2852 }
2853 (None, Some(x), Some(y)) => coordinate_click(backend, context, x, y, button, count),
2854 }
2855}
2856
2857fn effective_click_count(double: bool, count: u8) -> Result<u8, ComputerError> {
2858 if count == 0 || count > 3 {
2859 return Err(ComputerError::InvalidArguments(
2860 "click count must be 1, 2, or 3".into(),
2861 ));
2862 }
2863 if double && count != 1 && count != 2 {
2864 return Err(ComputerError::InvalidArguments(
2865 "double and count disagree".into(),
2866 ));
2867 }
2868 if double {
2869 Ok(2)
2870 } else {
2871 Ok(count)
2872 }
2873}
2874
2875fn coordinate_click(
2876 backend: &dyn DesktopBackend,
2877 context: &CallContext,
2878 x: i32,
2879 y: i32,
2880 button: MouseButton,
2881 count: u8,
2882) -> Response {
2883 let kind = InputKind::Click {
2884 x,
2885 y,
2886 button,
2887 count,
2888 };
2889 if let Err(error) = planned_tiers(backend, context) {
2891 return Response::from_error(error);
2892 }
2893 let mut warnings = Vec::new();
2894 let hit = match backend.hit_test(x, y) {
2895 Ok(hit) => hit,
2896 Err(error) => {
2897 warnings.push(format!("hit test failed: {error}"));
2898 None
2899 }
2900 };
2901 if let Some(title) = hit.as_ref().and_then(|hit| hit.title.as_deref()) {
2902 if policy::sensitive_title(title) {
2903 if let Err(error) = confirm_title(context, "click", "sensitive control title") {
2904 return Response::from_error(error);
2905 }
2906 }
2907 }
2908 match run_attempts(backend, context, kind) {
2909 Ok(report) => {
2910 let action_name = match count {
2911 2 => "double_click",
2912 3 => "triple_click",
2913 _ => "click",
2914 };
2915 let body = json!({
2916 "action": action_name,
2917 "x": x,
2918 "y": y,
2919 "button": button.as_str(),
2920 "double": count == 2,
2921 "count": count,
2922 "hit": hit_value(hit),
2923 });
2924 Response::success(finish_input(body, report, warnings))
2925 }
2926 Err(error) => Response::from_error(error),
2927 }
2928}
2929
2930fn hit_value(hit: Option<crate::element::HitInfo>) -> Value {
2931 match hit {
2932 Some(hit) => json!({"role": hit.role, "title": hit.title}),
2933 None => Value::Null,
2934 }
2935}
2936
2937fn type_command(backend: &dyn DesktopBackend, context: &CallContext, text: String) -> Response {
2938 if context.element.is_some() {
2939 return element_command(backend, context, ElementOp::Type { text });
2940 }
2941 if text.is_empty() {
2942 return Response::from_error(ComputerError::InvalidArguments(
2943 "text must not be empty".into(),
2944 ));
2945 }
2946 if text.contains('\0') {
2947 return Response::from_error(ComputerError::InvalidArguments(
2948 "text must not contain a null byte".into(),
2949 ));
2950 }
2951 let chars = text.chars().count();
2952 match run_attempts(backend, context, InputKind::Type { text: text.clone() }) {
2953 Ok(report) => {
2954 let body = json!({
2955 "action": "type",
2956 "text": text,
2957 "chars": chars,
2958 });
2959 Response::success(finish_input(body, report, Vec::new()))
2960 }
2961 Err(error) => Response::from_error(error),
2962 }
2963}
2964
2965fn element_command(backend: &dyn DesktopBackend, context: &CallContext, op: ElementOp) -> Response {
2966 if let Err(error) = precheck(&op) {
2967 return Response::from_error(error);
2968 }
2969 if let Err(error) = confirm_element_title(backend, context, &op) {
2970 return Response::from_error(error);
2971 }
2972 let index = match context.element {
2973 Some(index) => index,
2974 None => {
2975 return Response::from_error(ComputerError::InvalidArguments(
2976 "element is required".into(),
2977 ))
2978 }
2979 };
2980 let request = ElementRequest {
2981 op,
2982 index,
2983 require_visible: context.require_visible,
2984 app: context.app.clone(),
2985 window: context.window.clone(),
2986 session: context.session.clone(),
2987 tier: context.tier.clone(),
2988 allow_foreground: context.allow_foreground,
2989 require_background: context.require_background,
2990 no_overlay: context.no_overlay,
2991 };
2992 match backend.element_action(&request) {
2993 Ok(outcome) => Response::success(outcome_value(&outcome)),
2994 Err(error) => element_error_response(error),
2995 }
2996}
2997
2998fn confirm_element_title(
2999 backend: &dyn DesktopBackend,
3000 context: &CallContext,
3001 op: &ElementOp,
3002) -> Result<(), ComputerError> {
3003 let press = matches!(op, ElementOp::Press { .. })
3004 || matches!(op, ElementOp::Perform { name } if name.eq_ignore_ascii_case("AXPress"));
3005 if !press {
3006 return Ok(());
3007 }
3008 let Some(index) = context.element else {
3009 return Ok(());
3010 };
3011 let Ok(Some(peek)) = backend.peek_element(
3012 context.app.as_deref(),
3013 context.window.as_deref(),
3014 context.session.as_deref(),
3015 index,
3016 ) else {
3017 return Ok(());
3018 };
3019 if peek.title.as_deref().is_some_and(policy::sensitive_title) {
3020 confirm_title(context, "press", "sensitive control title")?;
3021 }
3022 Ok(())
3023}
3024
3025fn confirm_title(context: &CallContext, action: &str, reason: &str) -> Result<(), ComputerError> {
3026 let loaded = policy::load_policy(context.policy.as_deref().map(Path::new))?;
3027 let command = policy::confirm_program(&loaded, context.confirm_cmd.as_deref());
3028 policy::run_confirm(
3029 command.as_deref(),
3030 loaded.timeout_secs(),
3031 &policy::ConfirmDocument {
3032 action,
3033 bundle_id: None,
3034 path: None,
3035 pid: None,
3036 window_id: None,
3037 element: context.element,
3038 reason,
3039 },
3040 )
3041}
3042
3043fn precheck(op: &ElementOp) -> Result<(), ComputerError> {
3044 match op {
3045 ElementOp::Press { count, .. } => element::validate_count(*count),
3046 ElementOp::SetValue { value } => element::validate_text(value, true),
3047 ElementOp::Type { text } => element::validate_text(text, false),
3048 ElementOp::SelectText { start, end } => element::validate_range(*start, *end, None),
3049 ElementOp::Perform { name } => element::validate_perform_name(name),
3050 ElementOp::Focus | ElementOp::ScrollTo => Ok(()),
3051 }
3052}
3053
3054fn outcome_value(outcome: &ElementOutcome) -> Value {
3055 let mut body = json!({
3056 "action": outcome.action,
3057 "element": outcome.element,
3058 "revision": outcome.revision,
3059 "tier": outcome.tier,
3060 "cursor_moved": outcome.cursor_moved,
3061 });
3062 if let Some(restored) = outcome.focus_restored {
3063 body["focus_restored"] = json!(restored);
3064 }
3065 if !outcome.warnings.is_empty() {
3066 body["warnings"] = json!(outcome.warnings);
3067 }
3068 if let Some(via) = &outcome.via {
3069 body["via"] = json!(via);
3070 }
3071 if let Some(name) = &outcome.name {
3072 body["name"] = json!(name);
3073 }
3074 if let Some(count) = outcome.count {
3075 body["count"] = json!(count);
3076 }
3077 body
3078}
3079
3080fn element_error_response(error: ElementError) -> Response {
3081 let mut response = Response::from_error(error.error);
3082 if let Some(tree) = error.tree {
3083 response.fields.insert("tree".into(), json!(tree));
3084 }
3085 response
3086}
3087
3088fn move_mouse(backend: &dyn DesktopBackend, context: &CallContext, x: i32, y: i32) -> Response {
3089 match run_attempts(backend, context, InputKind::Move { x, y }) {
3090 Ok(report) => Response::success(finish_input(
3091 json!({"action": "move", "x": x, "y": y}),
3092 report,
3093 Vec::new(),
3094 )),
3095 Err(error) => Response::from_error(error),
3096 }
3097}
3098
3099fn press_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str) -> Response {
3100 let strokes = match key_strokes(keys) {
3101 Ok(strokes) => strokes,
3102 Err(error) => return Response::from_error(error),
3103 };
3104 let mut body = json!({
3105 "action": "key",
3106 "keys": keys,
3107 });
3108 if strokes.len() == 1 {
3109 let modifiers: Vec<&str> = strokes[0]
3110 .modifiers
3111 .iter()
3112 .copied()
3113 .map(Modifier::as_str)
3114 .collect();
3115 body["modifiers"] = json!(modifiers);
3116 body["key"] = json!(strokes[0].key);
3117 } else {
3118 body["strokes"] = json!(strokes.len());
3119 }
3120 match run_attempts(backend, context, InputKind::Key { strokes }) {
3121 Ok(report) => Response::success(finish_input(body, report, Vec::new())),
3122 Err(error) => Response::from_error(error),
3123 }
3124}
3125
3126fn key_strokes(keys: &str) -> Result<Vec<KeyStroke>, ComputerError> {
3127 let combos = parse_key_sequence(keys)?;
3128 let mut strokes = Vec::with_capacity(combos.len());
3129 for combo in combos {
3130 validate_key(&combo.key)?;
3131 strokes.push(KeyStroke {
3132 modifiers: combo.modifiers,
3133 key: combo.key,
3134 });
3135 }
3136 Ok(strokes)
3137}
3138
3139fn drag(
3140 backend: &dyn DesktopBackend,
3141 context: &CallContext,
3142 path: Vec<Point>,
3143 button: MouseButton,
3144) -> Response {
3145 if path.len() < 2 {
3146 return Response::from_error(ComputerError::InvalidArguments(
3147 "drag needs at least 2 points".into(),
3148 ));
3149 }
3150 let points: Vec<(i32, i32)> = path.iter().map(|point| (point.x, point.y)).collect();
3151 let moves = points.len() - 1;
3152 match run_attempts(backend, context, InputKind::Drag { points, button }) {
3153 Ok(report) => Response::success(finish_input(
3154 json!({
3155 "action": "drag",
3156 "button": button.as_str(),
3157 "points": path.len(),
3158 "moves": moves,
3159 }),
3160 report,
3161 Vec::new(),
3162 )),
3163 Err(error) => Response::from_error(error),
3164 }
3165}
3166
3167fn pointer_half(
3168 backend: &dyn DesktopBackend,
3169 context: &CallContext,
3170 x: i32,
3171 y: i32,
3172 button: MouseButton,
3173 down: bool,
3174) -> Response {
3175 let kind = if down {
3176 InputKind::MouseDown { x, y, button }
3177 } else {
3178 InputKind::MouseUp { x, y, button }
3179 };
3180 let name = if down { "mouse_down" } else { "mouse_up" };
3181 match run_attempts(backend, context, kind) {
3182 Ok(report) => Response::success(finish_input(
3183 json!({
3184 "action": name,
3185 "x": x,
3186 "y": y,
3187 "button": button.as_str(),
3188 }),
3189 report,
3190 Vec::new(),
3191 )),
3192 Err(error) => Response::from_error(error),
3193 }
3194}
3195
3196fn hold_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str, ms: u64) -> Response {
3197 let stroke = match one_hold(keys, ms) {
3198 Ok(stroke) => stroke,
3199 Err(error) => return Response::from_error(error),
3200 };
3201 match run_attempts(
3202 backend,
3203 context,
3204 InputKind::HoldKey {
3205 modifiers: stroke.modifiers.clone(),
3206 key: stroke.key.clone(),
3207 ms,
3208 },
3209 ) {
3210 Ok(report) => Response::success(finish_input(
3211 json!({
3212 "action": "hold_key",
3213 "keys": keys,
3214 "key": stroke.key,
3215 "ms": ms,
3216 }),
3217 report,
3218 Vec::new(),
3219 )),
3220 Err(error) => Response::from_error(error),
3221 }
3222}
3223
3224fn one_hold(keys: &str, ms: u64) -> Result<KeyStroke, ComputerError> {
3225 if ms > tier::MAX_HOLD_MS {
3226 return Err(ComputerError::InvalidArguments(format!(
3227 "hold_key ms must be at most {}",
3228 tier::MAX_HOLD_MS
3229 )));
3230 }
3231 if keys.split_whitespace().count() != 1 {
3232 return Err(ComputerError::InvalidArguments(
3233 "hold_key accepts one key combination".into(),
3234 ));
3235 }
3236 let combo = parse_key_combo(keys)?;
3237 validate_key(&combo.key)?;
3238 Ok(KeyStroke {
3239 modifiers: combo.modifiers,
3240 key: combo.key,
3241 })
3242}
3243
3244fn clipboard_chars(action: &Action) -> Option<u64> {
3245 match action {
3246 Action::ClipboardSet {
3247 text: Some(text), ..
3248 } => Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX)),
3249 Action::BrowserType { text, .. } => {
3250 Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX))
3251 }
3252 _ => None,
3253 }
3254}
3255
3256fn preflight_action(action: &Action) -> Result<(), ComputerError> {
3257 match action {
3258 Action::Click { double, count, .. } => {
3259 effective_click_count(*double, *count)?;
3260 }
3261 Action::Press { count, .. } => element::validate_count(*count)?,
3262 Action::Drag { path, .. } if path.len() < 2 => {
3263 return Err(ComputerError::InvalidArguments(
3264 "drag needs at least 2 points".into(),
3265 ));
3266 }
3267 Action::HoldKey { keys, ms } => {
3268 one_hold(keys, *ms)?;
3269 }
3270 Action::Key { keys } => {
3271 key_strokes(keys)?;
3272 }
3273 Action::ClipboardSet { text, image } => match (text, image) {
3274 (Some(text), None) if text.is_empty() => {
3275 return Err(ComputerError::InvalidArguments(
3276 "clipboard text must not be empty".into(),
3277 ));
3278 }
3279 (None, None) => {
3280 return Err(ComputerError::InvalidArguments(
3281 "clipboard set requires text or image".into(),
3282 ));
3283 }
3284 (Some(_), Some(_)) => {
3285 return Err(ComputerError::InvalidArguments(
3286 "pass clipboard text or image, not both".into(),
3287 ));
3288 }
3289 _ => {}
3290 },
3291 _ => {}
3292 }
3293 Ok(())
3294}
3295
3296fn clipboard_get(
3297 backend: &dyn DesktopBackend,
3298 context: &CallContext,
3299) -> Result<serde_json::Value, ComputerError> {
3300 match backend.clipboard_get()? {
3301 ClipboardContents::Empty => Ok(json!({
3302 "action": "clipboard_get",
3303 "kind": "empty",
3304 })),
3305 ClipboardContents::Text(text) => {
3306 let chars = text.chars().count();
3307 Ok(json!({
3308 "action": "clipboard_get",
3309 "kind": "text",
3310 "text": text,
3311 "chars": chars,
3312 }))
3313 }
3314 ClipboardContents::Image {
3315 width,
3316 height,
3317 rgba,
3318 } => {
3319 let frame = Frame::new(width, height, 1.0, rgba)?;
3320 let encoded = image_util::encode_rgba(frame.image(), ImageFormat::Png)?;
3321 let dir = cache::session_dir(context.session.as_deref())?;
3322 let path = dir.join("clipboard.png");
3323 cache::write_secret_bytes(&path, &encoded.bytes)?;
3324 Ok(json!({
3325 "action": "clipboard_get",
3326 "kind": "image",
3327 "path": path_string(&path),
3328 "width": width,
3329 "height": height,
3330 }))
3331 }
3332 }
3333}
3334
3335fn clipboard_set(
3336 backend: &dyn DesktopBackend,
3337 text: Option<String>,
3338 image: Option<String>,
3339) -> Result<serde_json::Value, ComputerError> {
3340 match (text, image) {
3341 (Some(text), None) => {
3342 let chars = text.chars().count();
3343 backend.clipboard_set_text(&text)?;
3344 Ok(json!({
3345 "action": "clipboard_set",
3346 "kind": "text",
3347 "chars": chars,
3348 }))
3349 }
3350 (None, Some(path)) => {
3351 let (width, height, rgba) = decode_clipboard_image(&path)?;
3352 backend.clipboard_set_image(width, height, &rgba)?;
3353 Ok(json!({
3354 "action": "clipboard_set",
3355 "kind": "image",
3356 "width": width,
3357 "height": height,
3358 }))
3359 }
3360 _ => Err(ComputerError::InvalidArguments(
3361 "clipboard set requires text or image".into(),
3362 )),
3363 }
3364}
3365
3366fn decode_clipboard_image(path: &str) -> Result<(u32, u32, Vec<u8>), ComputerError> {
3367 let image = image::ImageReader::open(path)
3368 .map_err(|error| {
3369 ComputerError::InvalidArguments(format!("clipboard image could not be opened: {error}"))
3370 })?
3371 .decode()
3372 .map_err(|error| {
3373 ComputerError::InvalidArguments(format!(
3374 "clipboard image could not be decoded: {error}"
3375 ))
3376 })?
3377 .to_rgba8();
3378 let width = image.width();
3379 let height = image.height();
3380 Ok((width, height, image.into_raw()))
3381}
3382
3383fn scroll(
3384 backend: &dyn DesktopBackend,
3385 context: &CallContext,
3386 point: (Option<i32>, Option<i32>),
3387 delta: (Option<i32>, Option<i32>),
3388 direction: Option<ScrollDirection>,
3389 amount: Option<i32>,
3390) -> Response {
3391 let (x, y) = point;
3392 let (dx, dy) = delta;
3393 if x.is_some() != y.is_some() {
3394 return Response::from_error(ComputerError::InvalidArguments(
3395 "scroll x and y must be set together".into(),
3396 ));
3397 }
3398 let (scroll_dx, scroll_dy) = match resolve_scroll(dx, dy, direction, amount) {
3399 Ok(delta) => delta,
3400 Err(error) => return Response::from_error(error),
3401 };
3402 match run_attempts(
3403 backend,
3404 context,
3405 InputKind::Scroll {
3406 x,
3407 y,
3408 dx: scroll_dx,
3409 dy: scroll_dy,
3410 },
3411 ) {
3412 Ok(report) => {
3413 let body = json!({
3414 "action": "scroll",
3415 "x": x,
3416 "y": y,
3417 "dx": scroll_dx,
3418 "dy": scroll_dy,
3419 });
3420 Response::success(finish_input(scroll_body(body, &report), report, Vec::new()))
3421 }
3422 Err(error) => Response::from_error(error),
3423 }
3424}
3425
3426fn scroll_body(mut body: Value, report: &crate::tier::InputReport) -> Value {
3435 let Some(outcome) = &report.scroll else {
3436 return body;
3437 };
3438 body["via"] = json!(outcome.via.as_str());
3439 body["scrolled"] = json!(outcome.scrolled);
3440 body["before"] = json!(outcome.before);
3441 body["after"] = json!(outcome.after);
3442 body["step"] = json!(outcome.step);
3443 body
3444}
3445
3446fn planned_tiers(
3447 backend: &dyn DesktopBackend,
3448 context: &CallContext,
3449) -> Result<Vec<TierName>, ComputerError> {
3450 let requested = crate::tier::parse_tier(context.tier.as_deref())?;
3451 crate::tier::plan(
3452 requested,
3453 crate::tier::InputClass::Background,
3454 backend.input_support(),
3455 crate::tier::skylight_compiled(),
3456 )
3457}
3458
3459fn run_attempts(
3460 backend: &dyn DesktopBackend,
3461 context: &CallContext,
3462 kind: InputKind,
3463) -> Result<crate::tier::InputReport, ComputerError> {
3464 attempt::run(backend, &attempt_flags(context), kind)
3465}
3466
3467fn attempt_flags(context: &CallContext) -> AttemptFlags {
3468 AttemptFlags {
3469 app: context.app.clone(),
3470 window: context.window.clone(),
3471 tier: context.tier.clone(),
3472 allow_foreground: context.allow_foreground,
3473 require_background: context.require_background,
3474 no_overlay: context.no_overlay,
3475 session: context.session.clone(),
3476 }
3477}
3478
3479fn finish_input(
3480 mut body: Value,
3481 report: crate::tier::InputReport,
3482 mut warnings: Vec<String>,
3483) -> Value {
3484 warnings.extend(report.warnings);
3485 body["tier"] = json!(report.tier);
3486 body["cursor_moved"] = json!(report.cursor_moved);
3487 if let Some(restored) = report.focus_restored {
3488 body["focus_restored"] = json!(restored);
3489 }
3490 if !warnings.is_empty() {
3491 body["warnings"] = json!(warnings);
3492 }
3493 body
3494}
3495
3496pub(crate) fn resolve_scroll(
3497 dx: Option<i32>,
3498 dy: Option<i32>,
3499 direction: Option<ScrollDirection>,
3500 amount: Option<i32>,
3501) -> Result<(i32, i32), ComputerError> {
3502 let has_delta = dx.is_some() || dy.is_some();
3503 let has_direction = direction.is_some() || amount.is_some();
3504 if has_delta && has_direction {
3505 return Err(ComputerError::InvalidArguments(
3506 "use either dx/dy or direction/amount, not both".into(),
3507 ));
3508 }
3509 if let Some(direction) = direction {
3510 let amount = amount.unwrap_or(1);
3511 return Ok(match direction {
3512 ScrollDirection::Up => (0, -amount),
3513 ScrollDirection::Down => (0, amount),
3514 ScrollDirection::Left => (-amount, 0),
3515 ScrollDirection::Right => (amount, 0),
3516 });
3517 }
3518 if amount.is_some() {
3519 return Err(ComputerError::InvalidArguments(
3520 "amount requires direction".into(),
3521 ));
3522 }
3523 if !has_delta {
3524 return Err(ComputerError::InvalidArguments(
3525 "scroll requires dx/dy or direction".into(),
3526 ));
3527 }
3528 Ok((dx.unwrap_or(0), dy.unwrap_or(0)))
3529}
3530
3531fn wait(ms: Option<u64>, seconds: Option<f64>) -> Result<Value, ComputerError> {
3532 let waited = resolve_wait(ms, seconds)?;
3533 thread::sleep(Duration::from_millis(waited));
3534 Ok(json!({"waited_ms": waited}))
3535}
3536
3537pub(crate) fn resolve_wait(ms: Option<u64>, seconds: Option<f64>) -> Result<u64, ComputerError> {
3538 if ms.is_none() && seconds.is_none() {
3539 return Err(ComputerError::InvalidArguments(
3540 "wait requires ms, seconds, or both".into(),
3541 ));
3542 }
3543 let mut total: u128 = u128::from(ms.unwrap_or(0));
3544 if let Some(seconds) = seconds {
3545 if !seconds.is_finite() || seconds < 0.0 {
3546 return Err(ComputerError::InvalidArguments(
3547 "seconds must be a finite number greater than or equal to 0".into(),
3548 ));
3549 }
3550 let extra = (seconds * 1000.0).round();
3551 if extra > u128::MAX as f64 {
3552 return Err(ComputerError::InvalidArguments("wait is too long".into()));
3553 }
3554 total = total.saturating_add(extra as u128);
3555 }
3556 if total > MAX_WAIT_MS {
3557 return Err(ComputerError::InvalidArguments(
3558 "wait must be at most 24 hours".into(),
3559 ));
3560 }
3561 u64::try_from(total).map_err(|_| ComputerError::InvalidArguments("wait is too long".into()))
3562}
3563
3564fn displays(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
3565 let displays = backend.list_displays()?;
3566 Ok(json!({"displays": displays}))
3567}
3568
3569fn display_filter(
3571 backend: &dyn DesktopBackend,
3572 context: &CallContext,
3573) -> Result<Option<crate::isolation::DisplayTarget>, ComputerError> {
3574 match context.display.as_deref() {
3575 None => Ok(None),
3576 Some(spec) => Ok(Some(crate::isolation::resolve(
3577 spec,
3578 &backend.list_displays()?,
3579 )?)),
3580 }
3581}
3582
3583fn isolation() -> Result<Value, ComputerError> {
3585 Ok(crate::isolation::matrix())
3586}
3587
3588fn tree_query(context: &CallContext) -> TreeQuery {
3589 TreeQuery {
3590 app: context.app.clone(),
3591 window: context.window.clone(),
3592 full: context.full,
3593 session: context.session.clone(),
3594 }
3595}
3596
3597fn ocr_action(
3601 backend: &dyn DesktopBackend,
3602 context: &CallContext,
3603 _force: bool,
3604) -> Result<Value, ComputerError> {
3605 let targeted = context.app.is_some() || context.window.is_some();
3606 if !targeted {
3607 return Err(ComputerError::InvalidArguments(
3608 "ocr requires --app or --window".into(),
3609 ));
3610 }
3611 let prepared = latest_prepared_ocr_capture(backend, context)?;
3612 let OcrResult { blocks, note } = backend.recognize_text(&prepared.image, &prepared.map)?;
3613 let redacted = redact_secure_ocr_blocks(blocks, backend, context, &prepared.map)?;
3614 let blocks_json: Vec<Value> = redacted
3615 .iter()
3616 .map(|block| {
3617 let image_rect = crate::vision::ocr::ImageRect {
3618 x: block.x,
3619 y: block.y,
3620 width: block.width,
3621 height: block.height,
3622 };
3623 let screen = image_rect.to_screen(&prepared.map);
3624 let screen_json = match screen {
3625 Some((x, y)) => json!({
3626 "x": x.round() as i64,
3627 "y": y.round() as i64,
3628 }),
3629 None => Value::Null,
3630 };
3631 json!({
3632 "text": block.text,
3633 "x": block.x,
3634 "y": block.y,
3635 "width": block.width,
3636 "height": block.height,
3637 "confidence": block.confidence,
3638 "screen": screen_json,
3639 })
3640 })
3641 .collect();
3642 let mut body = json!({
3643 "action": "ocr",
3644 "blocks": blocks_json,
3645 });
3646 if let Some(note) = note {
3647 body["note"] = json!(note);
3648 }
3649 Ok(body)
3650}
3651
3652fn redact_secure_ocr_blocks(
3657 blocks: Vec<OcrBlock>,
3658 backend: &dyn DesktopBackend,
3659 context: &CallContext,
3660 map: &crate::image_util::ScreenshotMap,
3661) -> Result<Vec<OcrBlock>, ComputerError> {
3662 let secure_frames = backend.secure_ocr_frames(&tree_query(context), map)?;
3667 Ok(crate::vision::redact_blocks(blocks, secure_frames))
3668}
3669
3670struct OcrPreparedCapture {
3672 image: image::RgbaImage,
3673 map: crate::image_util::ScreenshotMap,
3674}
3675
3676fn latest_prepared_ocr_capture(
3680 backend: &dyn DesktopBackend,
3681 context: &CallContext,
3682) -> Result<OcrPreparedCapture, ComputerError> {
3683 let targeted = context.app.is_some() || context.window.is_some();
3684 if targeted {
3685 if let Some(error) = backend.display_block() {
3686 return Err(error);
3687 }
3688 let target = resolve_screenshot_window(backend, context)?;
3689 let captured = backend.capture_window(&target)?;
3690 let prepared = image_util::prepare_shot(
3691 captured.frame.image(),
3692 image_util::ShotFit {
3693 scale: captured.frame.scale(),
3694 origin_x: captured.origin_x,
3695 origin_y: captured.origin_y,
3696 logical_width: captured.logical_width,
3697 logical_height: captured.logical_height,
3698 max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3699 canvas: None,
3700 },
3701 )?;
3702 Ok(OcrPreparedCapture {
3703 image: prepared.image,
3704 map: prepared.map,
3705 })
3706 } else {
3707 let raw = backend.capture_screen(None)?;
3709 let prepared = image_util::prepare_shot(
3710 raw.frame.image(),
3711 image_util::ShotFit {
3712 scale: raw.frame.scale(),
3713 origin_x: raw.origin_x,
3714 origin_y: raw.origin_y,
3715 logical_width: raw.logical_width,
3716 logical_height: raw.logical_height,
3717 max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3718 canvas: None,
3719 },
3720 )?;
3721 Ok(OcrPreparedCapture {
3722 image: prepared.image,
3723 map: prepared.map,
3724 })
3725 }
3726}
3727
3728fn apply_marks(
3734 image: &mut image::RgbaImage,
3735 map: &crate::image_util::ScreenshotMap,
3736 context: &CallContext,
3737 backend: &dyn DesktopBackend,
3738) -> Result<usize, ComputerError> {
3739 let boxes = backend.marks_for_tree(&tree_query(context), map)?;
3740 if boxes.is_empty() {
3741 return Ok(0);
3742 }
3743 let marked = crate::vision::draw_marks(image, &boxes);
3744 *image = marked;
3745 Ok(boxes.len())
3746}
3747
3748fn tree_action(
3749 backend: &dyn DesktopBackend,
3750 context: &CallContext,
3751) -> Result<Value, ComputerError> {
3752 let report = backend.tree(&tree_query(context))?;
3753 let mut body = json!({
3754 "pid": report.pid,
3755 "window_id": report.window_id,
3756 "revision": report.revision,
3757 "source": report.source,
3758 "truncated": report.truncated,
3759 "tree": report.tree,
3760 });
3761 if let Some(note) = report.note {
3762 body["note"] = json!(note);
3763 }
3764 if let Some(legacy) = report.legacy.as_deref() {
3765 body["accessibility_legacy"] = serde_json::from_str(legacy).unwrap_or(Value::Null);
3766 }
3767 let tree_was_empty = report.full_text.trim().is_empty();
3768 if tree_was_empty {
3769 if context.app.is_some() || context.window.is_some() {
3772 body["ocr_fallback_available"] = json!(true);
3773 }
3774 }
3775 Ok(body)
3776}
3777
3778fn session_end(
3779 backend: &dyn DesktopBackend,
3780 context: &CallContext,
3781) -> Result<Value, ComputerError> {
3782 let restored = backend.end_session(context.session.as_deref())?;
3783 let mut body = json!({
3784 "action": "session_end",
3785 "restored": restored,
3786 });
3787 if let Err(error) = cache::prune_empty_sessions() {
3788 body["warnings"] = json!([format!("session prune failed: {error}")]);
3789 }
3790 Ok(body)
3791}
3792
3793fn list_apps(backend: &dyn DesktopBackend, all: bool) -> Result<Value, ComputerError> {
3794 let apps = backend.list_apps(all)?;
3795 Ok(json!({"apps": apps}))
3796}
3797
3798fn list_windows(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3799 let listed = backend.list_windows()?;
3800 let windows = match app {
3801 None => listed.windows,
3802 Some(app) => {
3803 let selector = parse_app_selector(app)?;
3804 let apps = backend.list_apps(true)?;
3805 let info = resolve_app(&apps, &selector)?;
3806 filter_windows_for_pid(&listed.windows, info.pid)
3807 }
3808 };
3809 let mut body = json!({"windows": windows});
3810 if let Some(note) = listed.note {
3811 body["note"] = json!(note);
3812 }
3813 Ok(body)
3814}
3815
3816fn launch(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3817 let app = require_app(app)?;
3818 let result = backend.launch(app)?;
3819 Ok(json!({
3820 "action": "launch",
3821 "name": result.name,
3822 "bundle_id": result.bundle_id,
3823 "pid": result.pid,
3824 "path": result.path,
3825 }))
3826}
3827
3828fn activate(
3829 backend: &dyn DesktopBackend,
3830 app: Option<&str>,
3831 window: Option<&str>,
3832) -> Result<Value, ComputerError> {
3833 let app = require_app(app)?;
3834 let result = backend.activate(app, window)?;
3835 Ok(json!({
3836 "action": "activate",
3837 "name": result.name,
3838 "bundle_id": result.bundle_id,
3839 "pid": result.pid,
3840 "path": result.path,
3841 "window_id": result.window_id,
3842 "window_title": result.window_title,
3843 }))
3844}
3845
3846fn quit(
3847 backend: &dyn DesktopBackend,
3848 app: Option<&str>,
3849 force: bool,
3850) -> Result<Value, ComputerError> {
3851 let app = require_app(app)?;
3852 let result = backend.quit(app, force)?;
3853 Ok(json!({
3854 "action": "quit",
3855 "name": result.name,
3856 "bundle_id": result.bundle_id,
3857 "pid": result.pid,
3858 "accepted": result.accepted,
3859 "force": force,
3860 }))
3861}
3862
3863fn doctor(
3864 backend: &dyn DesktopBackend,
3865 options: crate::doctor::DoctorOptions,
3866) -> Result<Value, ComputerError> {
3867 let report = backend.doctor(options)?;
3868 serde_json::to_value(report).map_err(|error| {
3869 ComputerError::ExecutionFailed(format!("failed to encode doctor report: {error}"))
3870 })
3871}
3872
3873fn require_app(app: Option<&str>) -> Result<&str, ComputerError> {
3874 match app.map(str::trim).filter(|text| !text.is_empty()) {
3875 Some(app) => Ok(app),
3876 None => Err(ComputerError::InvalidArguments("app is required".into())),
3877 }
3878}
3879
3880#[derive(Debug, Clone, PartialEq)]
3882pub struct Response {
3883 pub ok: bool,
3885 pub id: Option<Value>,
3887 pub error: Option<ErrorBody>,
3889 pub fields: Map<String, Value>,
3891}
3892
3893impl Response {
3894 pub fn success(data: Value) -> Self {
3896 let fields = match data {
3897 Value::Object(map) => map,
3898 other => {
3899 let mut map = Map::new();
3900 map.insert("result".into(), other);
3901 map
3902 }
3903 };
3904 Self {
3905 ok: true,
3906 id: None,
3907 error: None,
3908 fields,
3909 }
3910 }
3911
3912 pub fn failure(code: impl Into<String>, message: impl Into<String>) -> Self {
3914 Self {
3915 ok: false,
3916 id: None,
3917 error: Some(ErrorBody {
3918 code: code.into(),
3919 message: message.into(),
3920 }),
3921 fields: Map::new(),
3922 }
3923 }
3924
3925 pub fn from_error(error: ComputerError) -> Self {
3927 Self::failure(error.code(), error.to_string())
3928 }
3929
3930 pub fn with_id(mut self, id: Option<Value>) -> Self {
3932 self.id = id;
3933 self
3934 }
3935
3936 pub fn to_value(&self) -> Value {
3938 let mut map = Map::new();
3939 map.insert("ok".into(), Value::Bool(self.ok));
3940 if let Some(id) = &self.id {
3941 map.insert("id".into(), id.clone());
3942 }
3943 if let Some(error) = &self.error {
3944 map.insert(
3945 "error".into(),
3946 serde_json::to_value(error).unwrap_or_else(
3947 |_| json!({"code": "execution_failed", "message": "failed to encode error"}),
3948 ),
3949 );
3950 }
3951 for (key, value) in &self.fields {
3952 if key != "ok" && key != "id" && key != "error" {
3953 map.insert(key.clone(), value.clone());
3954 }
3955 }
3956 Value::Object(map)
3957 }
3958
3959 pub fn to_json_line(&self, pretty: bool) -> String {
3961 let value = self.to_value();
3962 let body = if pretty {
3963 serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3964 } else {
3965 serde_json::to_string(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3966 };
3967 format!("{body}\n")
3968 }
3969}
3970
3971impl From<ComputerError> for Response {
3972 fn from(error: ComputerError) -> Self {
3973 Self::from_error(error)
3974 }
3975}
3976
3977impl Serialize for Response {
3978 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
3979 self.to_value().serialize(serializer)
3980 }
3981}
3982
3983impl<'de> Deserialize<'de> for Response {
3984 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
3985 let value = Value::deserialize(deserializer)?;
3986 let Value::Object(mut object) = value else {
3987 return Err(de::Error::custom("response must be a JSON object"));
3988 };
3989 let ok = object
3990 .remove("ok")
3991 .ok_or_else(|| de::Error::missing_field("ok"))?;
3992 let ok = ok
3993 .as_bool()
3994 .ok_or_else(|| de::Error::custom("ok must be a boolean"))?;
3995 let id = object.remove("id");
3996 let error = match object.remove("error") {
3997 Some(value) => Some(serde_json::from_value(value).map_err(de::Error::custom)?),
3998 None => None,
3999 };
4000 Ok(Self {
4001 ok,
4002 id,
4003 error,
4004 fields: object,
4005 })
4006 }
4007}
4008
4009#[cfg(test)]
4010mod tests {
4011 use super::*;
4012 use crate::test_backend::StubBackend;
4013 use std::sync::atomic::Ordering;
4014
4015 #[test]
4016 fn action_roundtrip_and_aliases() {
4017 let click: Action = serde_json::from_str(
4018 r#"{"action":"click","x":10,"y":20,"button":"right","double":true}"#,
4019 )
4020 .unwrap();
4021 match &click {
4022 Action::Click {
4023 x,
4024 y,
4025 button,
4026 double,
4027 count,
4028 } => {
4029 assert_eq!(
4030 (*x, *y, *button, *double, *count),
4031 (Some(10), Some(20), MouseButton::Right, true, 1)
4032 );
4033 }
4034 other => panic!("unexpected {other:?}"),
4035 }
4036 let encoded = serde_json::to_string(&click).unwrap();
4037 let decoded: Action = serde_json::from_str(&encoded).unwrap();
4038 assert_eq!(decoded, click);
4039
4040 let moved: Action = serde_json::from_str(r#"{"action":"move_mouse","x":1,"y":2}"#).unwrap();
4041 assert!(matches!(moved, Action::Move { x: 1, y: 2 }));
4042
4043 let state: Action = serde_json::from_str(r#"{"action":"get-state"}"#).unwrap();
4044 assert!(matches!(state, Action::GetState));
4045 let state: Action = serde_json::from_str(r#"{"action":"get_state"}"#).unwrap();
4046 assert!(matches!(state, Action::GetState));
4047
4048 let shot: Action = serde_json::from_str(
4049 r#"{"action":"screenshot","region":"0,0,8,9","format":"jpeg","max_width":640,"base64":true}"#,
4050 )
4051 .unwrap();
4052 match shot {
4053 Action::Screenshot {
4054 region,
4055 format,
4056 max_width,
4057 base64,
4058 ..
4059 } => {
4060 assert_eq!(region.unwrap().width, 8);
4061 assert_eq!(format, Some(ImageFormat::Jpeg));
4062 assert_eq!(max_width, Some(640));
4063 assert!(base64);
4064 }
4065 other => panic!("unexpected {other:?}"),
4066 }
4067 let shot: Action = serde_json::from_str(
4068 r#"{"action":"screenshot","region":{"x":0,"y":1,"width":2,"height":3}}"#,
4069 )
4070 .unwrap();
4071 match shot {
4072 Action::Screenshot { region, .. } => {
4073 assert_eq!(region.unwrap().height, 3);
4074 }
4075 other => panic!("unexpected {other:?}"),
4076 }
4077
4078 assert!(serde_json::from_str::<Action>(r#"{"action":"fly"}"#).is_err());
4079 let bare: Action = serde_json::from_str(r#"{"action":"click"}"#).unwrap();
4080 assert!(matches!(
4081 bare,
4082 Action::Click {
4083 x: None,
4084 y: None,
4085 ..
4086 }
4087 ));
4088 let press: Action = serde_json::from_str(r#"{"action":"press"}"#).unwrap();
4089 assert!(matches!(
4090 press,
4091 Action::Press {
4092 count: 1,
4093 button: MouseButton::Left
4094 }
4095 ));
4096 let renamed: Action =
4097 serde_json::from_str(r#"{"action":"set-value","value":"hi"}"#).unwrap();
4098 assert!(matches!(renamed, Action::SetValue { .. }));
4099 let scrolled: Action = serde_json::from_str(r#"{"action":"scroll-to-visible"}"#).unwrap();
4100 assert!(matches!(scrolled, Action::ScrollTo));
4101 }
4102
4103 #[test]
4104 fn response_shape_and_roundtrip() {
4105 let ok = Response::success(json!({"waited_ms": 1})).with_id(Some(json!(7)));
4106 let value = ok.to_value();
4107 assert_eq!(value["ok"], true);
4108 assert!(value.get("error").is_none());
4109 assert_eq!(value["waited_ms"], 1);
4110 assert_eq!(value["id"], 7);
4111 let text = serde_json::to_string(&ok).unwrap();
4112 let back: Response = serde_json::from_str(&text).unwrap();
4113 assert_eq!(back.to_value(), value);
4114
4115 let err = Response::failure("invalid_arguments", "bad region");
4116 let value = err.to_value();
4117 assert_eq!(value["ok"], false);
4118 assert_eq!(value["error"]["code"], "invalid_arguments");
4119 assert_eq!(value["error"]["message"], "bad region");
4120 assert!(value.get("waited_ms").is_none());
4121 let line = err.to_json_line(false);
4122 assert!(line.ends_with('\n'));
4123 assert!(!line[..line.len() - 1].contains('\n'));
4124 }
4125
4126 #[test]
4127 fn scroll_and_wait_resolution() {
4128 assert_eq!(
4129 resolve_scroll(Some(1), Some(-2), None, None).unwrap(),
4130 (1, -2)
4131 );
4132 assert_eq!(
4133 resolve_scroll(None, None, Some(ScrollDirection::Up), Some(3)).unwrap(),
4134 (0, -3)
4135 );
4136 assert_eq!(
4137 resolve_scroll(None, None, Some(ScrollDirection::Right), None).unwrap(),
4138 (1, 0)
4139 );
4140 assert_eq!(
4141 resolve_scroll(None, None, Some(ScrollDirection::Left), Some(4)).unwrap(),
4142 (-4, 0)
4143 );
4144 assert_eq!(
4145 resolve_scroll(None, None, Some(ScrollDirection::Down), Some(2)).unwrap(),
4146 (0, 2)
4147 );
4148 assert!(resolve_scroll(Some(1), None, Some(ScrollDirection::Up), None).is_err());
4149 assert!(resolve_scroll(None, None, None, Some(2)).is_err());
4150 assert!(resolve_scroll(None, None, None, None).is_err());
4151
4152 assert_eq!(resolve_wait(Some(10), None).unwrap(), 10);
4153 assert_eq!(resolve_wait(Some(10), Some(0.5)).unwrap(), 510);
4154 assert_eq!(resolve_wait(None, Some(1.0)).unwrap(), 1000);
4155 assert!(resolve_wait(None, None).is_err());
4156 assert!(resolve_wait(None, Some(-1.0)).is_err());
4157 assert!(resolve_wait(None, Some(f64::NAN)).is_err());
4158 }
4159
4160 #[test]
4161 fn json_line_dispatch_and_stub_actions() {
4162 let wait = handle_request_line(r#"{"id":1,"action":"wait","ms":0}"#);
4163 assert!(wait.ok);
4164 assert_eq!(wait.to_value()["id"], 1);
4165 assert_eq!(wait.to_value()["waited_ms"], 0);
4166
4167 let bad = handle_request_line("not-json");
4168 assert!(!bad.ok);
4169 assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
4170
4171 let missing = handle_request_line(r#"{"id":"x","action":"nope"}"#);
4172 assert!(!missing.ok);
4173 assert_eq!(missing.to_value()["id"], "x");
4174
4175 let request = parse_request(json!({"id": 3, "action": "displays"})).unwrap();
4176 assert_eq!(request.id, Some(json!(3)));
4177 assert!(matches!(request.action, Action::Displays));
4178
4179 let backend = StubBackend::default();
4180 let response = dispatch_with(
4181 &backend,
4182 Action::Click {
4183 x: Some(3),
4184 y: Some(4),
4185 button: MouseButton::Left,
4186 double: true,
4187 count: 1,
4188 },
4189 );
4190 assert!(response.ok);
4191 assert_eq!(response.to_value()["action"], "double_click");
4192 assert_eq!(backend.clicks.load(Ordering::SeqCst), 2);
4193
4194 let response = dispatch_with(&backend, Action::Move { x: 8, y: 9 });
4195 assert!(response.ok);
4196 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4197
4198 let response = dispatch_with(&backend, Action::Type { text: "hi".into() });
4199 assert_eq!(response.to_value()["chars"], 2);
4200 assert_eq!(backend.typed.lock().unwrap().as_slice(), ["hi"]);
4201
4202 let response = dispatch_with(
4203 &backend,
4204 Action::Key {
4205 keys: "cmd+shift+4".into(),
4206 },
4207 );
4208 assert!(response.ok);
4209 assert_eq!(response.to_value()["key"], "4");
4210 let keys = backend.keys.lock().unwrap();
4211 assert_eq!(keys[0].1, "4");
4212 assert_eq!(keys[0].0, vec![Modifier::Command, Modifier::Shift]);
4213 drop(keys);
4214
4215 let response = dispatch_with(
4216 &backend,
4217 Action::Key {
4218 keys: "not+a+key".into(),
4219 },
4220 );
4221 assert!(!response.ok);
4222
4223 let response = dispatch_with(
4224 &backend,
4225 Action::Scroll {
4226 x: Some(1),
4227 y: Some(2),
4228 dx: None,
4229 dy: None,
4230 direction: Some(ScrollDirection::Down),
4231 amount: Some(3),
4232 },
4233 );
4234 assert!(response.ok);
4235 assert_eq!(backend.scrolls.lock().unwrap().as_slice(), [(0, 3)]);
4236 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4237 assert_eq!(response.to_value()["tier"], "event");
4238 assert_eq!(response.to_value()["cursor_moved"], false);
4239
4240 let response = dispatch_with(&backend, Action::Displays);
4241 assert_eq!(response.to_value()["displays"][0]["name"], "Stub");
4242
4243 let response = dispatch_with(&backend, Action::GetState);
4244 assert_eq!(response.to_value()["cursor"]["x"], 4);
4245 assert!(response.ok);
4246
4247 let _cache = crate::cache::CacheEnv::acquire();
4248 let shot = dispatch_with(
4249 &backend,
4250 Action::Screenshot {
4251 output: None,
4252 region: None,
4253 format: Some(ImageFormat::Png),
4254 max_width: Some(1),
4255 canvas: None,
4256 base64: true,
4257 marks: false,
4258 },
4259 );
4260 assert!(shot.ok, "{shot:?}");
4261 let value = shot.to_value();
4262 assert_eq!(value["width"], 1);
4263 assert_eq!(value["height"], 1);
4264 assert_eq!(value["scale"], 2.0);
4265 assert_eq!(value["format"], "png");
4266 let path = value["path"].as_str().unwrap();
4267 let bytes = fs::read(path).unwrap();
4268 assert_eq!(
4269 &bytes[..8],
4270 &[0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n']
4271 );
4272 assert!(value["data"].as_str().unwrap().starts_with("iVBOR"));
4273 assert_eq!(value["capture_source"], "stub");
4274 assert_eq!(value["origin"]["x"], 0);
4275 assert_eq!(value["origin"]["y"], 0);
4276 assert_eq!(value["fit"], "width");
4277 assert!(value["map"]["image_to_screen"].is_string());
4278 let _ = fs::remove_file(path);
4279 if let Ok(dir) = crate::cache::session_dir(None) {
4280 let _ = fs::remove_file(dir.join("screenshot-map.json"));
4281 }
4282
4283 let failed = StubBackend {
4284 fail_click: true,
4285 ..StubBackend::default()
4286 };
4287 let response = dispatch_with(
4288 &failed,
4289 Action::Click {
4290 x: Some(0),
4291 y: Some(0),
4292 button: MouseButton::Left,
4293 double: false,
4294 count: 1,
4295 },
4296 );
4297 assert!(!response.ok);
4298 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4299 }
4300
4301 #[test]
4302 fn dispatch_wait_does_not_need_a_display() {
4303 let response = dispatch(Action::Wait {
4304 ms: Some(0),
4305 seconds: None,
4306 });
4307 assert!(response.ok);
4308 assert_eq!(response.to_value()["waited_ms"], 0);
4309 }
4310
4311 fn shot(max_width: Option<u32>, canvas: Option<&str>) -> Action {
4312 Action::Screenshot {
4313 output: None,
4314 region: None,
4315 format: Some(ImageFormat::Png),
4316 max_width,
4317 canvas: canvas.map(str::to_string),
4318 base64: false,
4319 marks: false,
4320 }
4321 }
4322
4323 #[test]
4324 fn dispatch_applies_default_max_width_and_keeps_explicit_zero() {
4325 let _cache = crate::cache::CacheEnv::acquire();
4326 let backend = StubBackend {
4327 capture_pixels: Some((1600, 1000)),
4328 ..StubBackend::default()
4329 };
4330 let response = dispatch_with_context(
4331 &backend,
4332 shot(None, None),
4333 &CallContext {
4334 session: Some("p4-default-width".into()),
4335 ..CallContext::default()
4336 },
4337 );
4338 assert!(response.ok, "{response:?}");
4339 let value = response.to_value();
4340 assert_eq!(value["width"], 1440);
4341 assert_eq!(value["height"], 900);
4342 assert_eq!(value["scale"], 2.0);
4343 assert_eq!(value["logical_width"], 800);
4344 assert_eq!(value["logical_height"], 500);
4345 assert_eq!(value["fit"], "width");
4346 assert_eq!(value["capture_source"], "stub");
4347 assert_eq!(value["content_width"], 1440);
4348 let path = value["path"].as_str().unwrap();
4349 let _ = fs::remove_file(path);
4350 let map = crate::cache::load_screenshot_map(Some("p4-default-width"))
4351 .unwrap()
4352 .expect("map");
4353 assert_eq!(map.width, 1440);
4354 let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-default-width")).unwrap());
4355
4356 let full = dispatch_with_context(
4357 &backend,
4358 shot(Some(0), None),
4359 &CallContext {
4360 session: Some("p4-zero-width".into()),
4361 ..CallContext::default()
4362 },
4363 );
4364 assert!(full.ok, "{full:?}");
4365 assert_eq!(full.to_value()["width"], 1600);
4366 assert_eq!(full.to_value()["height"], 1000);
4367 assert_eq!(full.to_value()["fit"], "none");
4368 let path = full.to_value()["path"].as_str().unwrap().to_string();
4369 let _ = fs::remove_file(path);
4370 let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-zero-width")).unwrap());
4371 }
4372
4373 #[test]
4374 fn dispatch_window_capture_records_origin_and_faults() {
4375 let _cache = crate::cache::CacheEnv::acquire();
4376 let backend = StubBackend::default();
4377 let response = dispatch_with_context(
4378 &backend,
4379 shot(Some(0), None),
4380 &CallContext {
4381 app: Some("Notes".into()),
4382 window: Some("0".into()),
4383 session: Some("p4-window".into()),
4384 ..CallContext::default()
4385 },
4386 );
4387 assert!(response.ok, "{response:?}");
4388 let value = response.to_value();
4389 assert_eq!(value["capture_source"], "stub");
4390 assert_eq!(value["origin"]["x"], 10);
4391 assert_eq!(value["origin"]["y"], 20);
4392 assert_eq!(value["logical_width"], 400);
4393 assert_eq!(value["logical_height"], 300);
4394 assert_eq!(value["width"], 800);
4395 assert_eq!(value["fit"], "none");
4396 let path = value["path"].as_str().unwrap();
4397 let _ = fs::remove_file(path);
4398 let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-window")).unwrap());
4399
4400 let denied = StubBackend {
4401 capture_fault: crate::test_backend::StubCaptureFault::Permission,
4402 ..StubBackend::default()
4403 };
4404 let response = dispatch_with_context(
4405 &denied,
4406 shot(Some(0), None),
4407 &CallContext {
4408 app: Some("Notes".into()),
4409 ..CallContext::default()
4410 },
4411 );
4412 assert!(!response.ok);
4413 assert_eq!(response.to_value()["error"]["code"], "permission_required");
4414 assert!(response.to_value()["error"]["message"]
4415 .as_str()
4416 .unwrap()
4417 .contains("Screen Recording"));
4418
4419 let missing = StubBackend {
4420 capture_fault: crate::test_backend::StubCaptureFault::MissingWindow,
4421 ..StubBackend::default()
4422 };
4423 let response = dispatch_with_context(
4424 &missing,
4425 shot(Some(0), None),
4426 &CallContext {
4427 window: Some("id:100".into()),
4428 ..CallContext::default()
4429 },
4430 );
4431 assert!(!response.ok);
4432 assert_eq!(response.to_value()["error"]["code"], "window_not_found");
4433
4434 let both = dispatch_with_context(
4435 &backend,
4436 Action::Screenshot {
4437 output: None,
4438 region: Some(Region::new(0, 0, 8, 9).unwrap()),
4439 format: None,
4440 max_width: Some(0),
4441 canvas: None,
4442 base64: false,
4443 marks: false,
4444 },
4445 &CallContext {
4446 app: Some("Notes".into()),
4447 ..CallContext::default()
4448 },
4449 );
4450 assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
4451
4452 let locked = StubBackend {
4453 block_display: true,
4454 windows: Vec::new(),
4455 ..StubBackend::default()
4456 };
4457 let response = dispatch_with_context(
4458 &locked,
4459 shot(Some(0), None),
4460 &CallContext {
4461 app: Some("Notes".into()),
4462 ..CallContext::default()
4463 },
4464 );
4465 assert_eq!(response.to_value()["error"]["code"], "display_unavailable");
4466 assert!(response.to_value()["error"]["message"]
4467 .as_str()
4468 .unwrap()
4469 .contains("locked"));
4470 }
4471
4472 #[test]
4473 fn shared_flags_are_stripped_before_the_action_decodes() {
4474 let request = parse_request(json!({
4475 "id": 4,
4476 "action": "list_windows",
4477 "app": "Foo \"Bar\"",
4478 "window": 0,
4479 "element": "3",
4480 "diff": true,
4481 "full": false,
4482 "return_state": true,
4483 "require_visible": true
4484 }))
4485 .unwrap();
4486 assert_eq!(request.id, Some(json!(4)));
4487 assert!(matches!(request.action, Action::ListWindows));
4488 assert_eq!(request.context.app.as_deref(), Some("Foo \"Bar\""));
4489 assert_eq!(request.context.window.as_deref(), Some("0"));
4490 assert_eq!(request.context.element, Some(3));
4491 assert!(request.context.diff);
4492 assert!(!request.context.full);
4493 assert!(request.context.return_state);
4494 assert!(request.context.require_visible);
4495 }
4496
4497 fn ctx(app: &str) -> CallContext {
4498 CallContext {
4499 app: Some(app.to_string()),
4500 ..CallContext::default()
4501 }
4502 }
4503
4504 #[test]
4505 fn stub_inventory_launch_and_doctor() {
4506 let backend = StubBackend::default();
4507
4508 let response = dispatch_with(&backend, Action::ListApps { all: false });
4509 assert!(response.ok);
4510 let value = response.to_value();
4511 let apps = value["apps"].as_array().unwrap();
4512 assert!(apps.iter().any(|app| app["name"] == "Notes"));
4513 assert!(apps.iter().all(|app| app["name"] != "Widget"));
4514
4515 let response = dispatch_with(&backend, Action::ListApps { all: true });
4516 let value = response.to_value();
4517 assert!(value["apps"]
4518 .as_array()
4519 .unwrap()
4520 .iter()
4521 .any(|app| app["pid"] == 77));
4522
4523 let response = dispatch_with_context(&backend, Action::ListWindows, &ctx("Notes"));
4524 assert!(response.ok, "{response:?}");
4525 let value = response.to_value();
4526 let windows = &value["windows"];
4527 assert_eq!(windows.as_array().unwrap().len(), 2);
4528 assert_eq!(windows[0]["z"], 0);
4529 assert_eq!(windows[0]["id"], 100);
4530 assert_eq!(windows[1]["z"], 1);
4531 assert_eq!(windows[1]["title"], "Welcome");
4532 assert_eq!(windows[0]["ax_matched"], true);
4533 assert!(value.get("note").is_none());
4534
4535 let response = dispatch_with_context(&backend, Action::Launch, &ctx("Notes"));
4536 assert!(response.ok, "{response:?}");
4537 assert_eq!(response.to_value()["pid"], 10);
4538 assert_eq!(response.to_value()["action"], "launch");
4539 assert_eq!(backend.launched.lock().unwrap().as_slice(), ["Notes"]);
4540
4541 let response = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
4542 assert!(!response.ok);
4543 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4544 assert_eq!(backend.launched.lock().unwrap().len(), 1);
4545
4546 let response = dispatch_with_context(&backend, Action::Activate, &ctx("10"));
4547 assert!(response.ok, "{response:?}");
4548 assert!(response.to_value()["window_id"].is_null());
4549 assert_eq!(backend.activated.lock().unwrap().len(), 1);
4550
4551 let mut context = ctx("Notes");
4552 context.window = Some("Welcome".into());
4553 let response = dispatch_with_context(&backend, Action::Activate, &context);
4554 assert_eq!(response.to_value()["window_id"], 101);
4555 assert_eq!(response.to_value()["window_title"], "Welcome");
4556
4557 let response =
4558 dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("Terminal"));
4559 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4560 assert!(backend.quit.lock().unwrap().is_empty());
4561
4562 let response = dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("412"));
4563 assert_eq!(response.to_value()["error"]["code"], "confirm_required");
4564 assert!(backend.quit.lock().unwrap().is_empty());
4565
4566 let response = dispatch_with(&backend, Action::Launch);
4567 assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4568
4569 let response = dispatch_with(
4570 &backend,
4571 Action::Doctor {
4572 prompt: false,
4573 open_settings: false,
4574 x11: false,
4575 },
4576 );
4577 let value = response.to_value();
4578 assert_eq!(value["ok"], true);
4579 assert_eq!(value["permissions_ok"], false);
4580 assert_eq!(value["accessibility"], true);
4581 assert_eq!(value["screen_capture"], false);
4582 assert_eq!(value["screen_recording"], false);
4583 assert!(value["input_monitor"].is_null());
4584 assert_eq!(value["missing"][0], "screen_recording");
4585 assert_eq!(value["responsible_app"]["name"], "Terminal");
4586 assert!(value["fixes"][0]
4587 .as_str()
4588 .unwrap()
4589 .contains("Screen Recording"));
4590 assert_eq!(value["fix"][0]["target"], "Terminal");
4591
4592 let prompted: Action =
4593 serde_json::from_str(r#"{"action":"doctor","request":true}"#).unwrap();
4594 assert!(matches!(
4595 prompted,
4596 Action::Doctor {
4597 prompt: true,
4598 open_settings: false,
4599 ..
4600 }
4601 ));
4602 let plain: Action = serde_json::from_str(r#"{"action":"doctor"}"#).unwrap();
4603 assert!(matches!(
4604 plain,
4605 Action::Doctor {
4606 prompt: false,
4607 open_settings: false,
4608 ..
4609 }
4610 ));
4611
4612 let response = dispatch_with(&backend, Action::GetState);
4613 assert!(response.ok);
4614 let value = response.to_value();
4615 assert_eq!(value["pid"], 10);
4616 assert_eq!(value["source"], "stub");
4617 assert!(value["tree"].as_str().unwrap().contains("AXWindow"));
4618 assert!(value["accessibility_legacy"].is_null());
4619
4620 let response = dispatch_with_context(&backend, Action::GetState, &ctx("412"));
4621 assert_eq!(response.to_value()["pid"], 412);
4622 assert_eq!(response.to_value()["source"], "stub");
4623 assert!(response.to_value()["tree"]
4624 .as_str()
4625 .unwrap()
4626 .contains("AXWindow"));
4627 let seen = backend.accessibility_apps.lock().unwrap().clone();
4628 assert_eq!(seen.last().and_then(|name| name.as_deref()), Some("412"));
4629
4630 backend.baselines.lock().unwrap().clear();
4633 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4634 assert!(first.ok, "{first:?}");
4635 let full = first.to_value()["tree"].as_str().unwrap().to_string();
4636 assert!(full.contains("value=\"a\""), "{full}");
4637 assert_eq!(first.to_value()["source"], "stub");
4638 *backend.stub_value.lock().unwrap() = "ab".into();
4639 let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4640 let diff_text = diff.to_value()["tree"].as_str().unwrap().to_string();
4641 assert!(
4642 diff_text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4643 "{diff_text}"
4644 );
4645 let mut full_context = ctx("Notes");
4646 full_context.full = true;
4647 let replaced = dispatch_with_context(&backend, Action::Tree, &full_context);
4648 let replaced_text = replaced.to_value()["tree"].as_str().unwrap().to_string();
4649 assert!(replaced_text.contains("value=\"ab\""), "{replaced_text}");
4650 assert!(!replaced_text.contains("~ [1]"));
4651 let after = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4652 assert_eq!(after.to_value()["tree"].as_str().unwrap(), "");
4653
4654 let _cache = crate::cache::CacheEnv::acquire();
4655 let ended = dispatch_with(&backend, Action::SessionEnd);
4656 assert!(ended.ok, "{ended:?}");
4657 assert_eq!(ended.to_value()["action"], "session_end");
4658 assert!(ended.to_value()["restored"].as_array().unwrap().is_empty());
4659 }
4660
4661 #[test]
4662 fn stub_refuses_privacy_window_and_self_quit() {
4663 let mut backend = StubBackend::default();
4664 backend.apps.push(crate::target::AppInfo {
4665 name: "System Settings".into(),
4666 bundle_id: Some("com.apple.systempreferences".into()),
4667 pid: 50,
4668 path: Some(
4669 "/System/Applications/System Settings.app/Contents/MacOS/System Settings".into(),
4670 ),
4671 active: false,
4672 hidden: false,
4673 activation_policy: crate::target::ActivationPolicy::Regular,
4674 });
4675 backend.windows.push(crate::target::WindowInfo {
4676 id: 9,
4677 pid: 50,
4678 app: "System Settings".into(),
4679 title: "Privacy & Security".into(),
4680 z: 4,
4681 bounds: crate::target::WindowBounds {
4682 x: 0,
4683 y: 0,
4684 width: 10,
4685 height: 10,
4686 },
4687 layer: 0,
4688 on_screen: true,
4689 ax_matched: false,
4690 });
4691 let mut context = ctx("com.apple.systempreferences");
4692 context.window = Some("Privacy".into());
4693 let response = dispatch_with_context(&backend, Action::Activate, &context);
4694 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4695 assert!(backend.activated.lock().unwrap().is_empty());
4696
4697 let response = dispatch_with_context(&backend, Action::Activate, &ctx("System Settings"));
4698 assert!(response.ok, "{response:?}");
4699
4700 let self_pid = std::process::id() as i32;
4701 backend.apps.retain(|app| app.pid != self_pid);
4702 backend.apps.push(crate::target::AppInfo {
4703 name: "computer-use".into(),
4704 bundle_id: None,
4705 pid: self_pid,
4706 path: None,
4707 active: false,
4708 hidden: false,
4709 activation_policy: crate::target::ActivationPolicy::Regular,
4710 });
4711 let response = dispatch_with_context(
4712 &backend,
4713 Action::Quit { force: false },
4714 &ctx(&self_pid.to_string()),
4715 );
4716 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4717 }
4718
4719 fn notes_button() -> crate::tree::UiNode {
4720 let mut button = crate::tree::UiNode::bare("AXButton");
4721 button.title = Some("Save".into());
4722 button.actions = vec!["AXPress".into(), "AXShowMenu".into()];
4723 button.frame = Some(crate::tree::NodeFrame {
4724 x: 12.0,
4725 y: 10.0,
4726 width: 80.0,
4727 height: 24.0,
4728 });
4729 let mut root = crate::tree::UiNode::bare("AXWindow");
4730 root.title = Some("Notes".into());
4731 root.frame = Some(crate::tree::NodeFrame {
4732 x: 0.0,
4733 y: 0.0,
4734 width: 800.0,
4735 height: 600.0,
4736 });
4737 root.children = vec![button];
4738 root
4739 }
4740
4741 fn element_context(action_element: u32) -> CallContext {
4742 CallContext {
4743 app: Some("Notes".into()),
4744 element: Some(action_element),
4745 ..CallContext::default()
4746 }
4747 }
4748
4749 #[test]
4750 fn element_json_shape_revalidation_and_hit_test() {
4751 let backend = StubBackend::default();
4752 backend.seed_element(notes_button());
4753 let request = parse_request(json!({
4754 "action": "press",
4755 "app": "Notes",
4756 "element": 1,
4757 "count": 1
4758 }))
4759 .unwrap();
4760 let response = dispatch_with_context(&backend, request.action, &request.context);
4761 assert!(response.ok, "{response:?}");
4762 let value = response.to_value();
4763 assert_eq!(value["action"], "press");
4764 assert_eq!(value["element"], 1);
4765 assert_eq!(value["tier"], "ax");
4766 assert_eq!(value["count"], 1);
4767 assert!(value["revision"].is_number());
4768 assert!(value.get("warnings").is_none());
4769
4770 let request = parse_request(json!({
4771 "action": "perform-action",
4772 "app": "Notes",
4773 "element": 1,
4774 "name": "AXShowMenu"
4775 }))
4776 .unwrap();
4777 let response = dispatch_with_context(&backend, request.action, &request.context);
4778 assert!(response.ok, "{response:?}");
4779 assert_eq!(response.to_value()["action"], "perform_action");
4780 assert_eq!(response.to_value()["name"], "AXShowMenu");
4781 assert_eq!(response.to_value()["tier"], "ax");
4782
4783 let before = backend.performs.lock().unwrap().len();
4784 let request = parse_request(json!({
4785 "action": "perform_action",
4786 "app": "Notes",
4787 "element": 1,
4788 "name": "AXDelete"
4789 }))
4790 .unwrap();
4791 let response = dispatch_with_context(&backend, request.action, &request.context);
4792 assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4793 assert_eq!(backend.performs.lock().unwrap().len(), before);
4794
4795 let request = parse_request(json!({
4796 "action": "set_value",
4797 "app": "Notes",
4798 "element": 1,
4799 "value": "hello"
4800 }))
4801 .unwrap();
4802 let response = dispatch_with_context(&backend, request.action, &request.context);
4803 assert!(response.ok, "{response:?}");
4804 assert_eq!(response.to_value()["action"], "set_value");
4805
4806 let request = parse_request(json!({
4807 "action": "select_text",
4808 "app": "Notes",
4809 "element": 1,
4810 "start": 0,
4811 "end": 5
4812 }))
4813 .unwrap();
4814 let response = dispatch_with_context(&backend, request.action, &request.context);
4815 assert!(response.ok, "{response:?}");
4816 assert_eq!(response.to_value()["action"], "select_text");
4817
4818 let request = parse_request(json!({
4819 "action": "scroll_to_visible",
4820 "app": "Notes",
4821 "element": 1
4822 }))
4823 .unwrap();
4824 assert!(matches!(request.action, Action::ScrollTo));
4825 let response = dispatch_with_context(&backend, request.action, &request.context);
4826 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4827 assert!(response.to_value()["error"]["message"]
4828 .as_str()
4829 .unwrap()
4830 .contains("AXScrollToVisible"));
4831
4832 let clicks = backend.clicks.load(Ordering::SeqCst);
4833 let request = parse_request(json!({
4834 "action": "click",
4835 "app": "Notes",
4836 "element": 1,
4837 "double": true
4838 }))
4839 .unwrap();
4840 let response = dispatch_with_context(&backend, request.action, &request.context);
4841 assert!(response.ok, "{response:?}");
4842 assert_eq!(response.to_value()["action"], "press");
4843 assert_eq!(response.to_value()["count"], 2);
4844 assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks);
4845
4846 *backend.hit.lock().unwrap() = Some(crate::element::HitInfo {
4847 role: "AXButton".into(),
4848 title: Some("Save".into()),
4849 });
4850 let response = dispatch_with(
4851 &backend,
4852 Action::Click {
4853 x: Some(12),
4854 y: Some(10),
4855 button: MouseButton::Left,
4856 double: false,
4857 count: 1,
4858 },
4859 );
4860 assert!(response.ok, "{response:?}");
4861 assert_eq!(response.to_value()["hit"]["role"], "AXButton");
4862 assert_eq!(response.to_value()["hit"]["title"], "Save");
4863 assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks + 1);
4864
4865 let mut changed = notes_button();
4866 changed.children[0].role = "AXTextArea".into();
4867 backend.set_live(changed);
4868 let response = dispatch_with_context(&backend, Action::Focus, &element_context(1));
4869 assert_eq!(response.to_value()["error"]["code"], "stale_element");
4870 assert!(response.to_value()["tree"]
4871 .as_str()
4872 .unwrap()
4873 .contains("AXTextArea"));
4874
4875 let empty = StubBackend::default();
4876 let response = dispatch_with_context(&empty, Action::Focus, &element_context(0));
4877 assert_eq!(response.to_value()["error"]["code"], "element_not_found");
4878 assert!(response.to_value()["error"]["message"]
4879 .as_str()
4880 .unwrap()
4881 .contains("refresh indices"));
4882
4883 let bare = dispatch_with(
4884 &empty,
4885 Action::Click {
4886 x: None,
4887 y: None,
4888 button: MouseButton::Left,
4889 double: false,
4890 count: 1,
4891 },
4892 );
4893 assert_eq!(bare.to_value()["error"]["code"], "invalid_arguments");
4894 }
4895
4896 #[test]
4897 fn element_action_does_not_replace_the_tree_diff_baseline() {
4898 let backend = StubBackend::default();
4899 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4900 assert!(first.ok, "{first:?}");
4901 let key = (String::new(), 10, 100);
4902 let baseline = backend.baselines.lock().unwrap().get(&key).cloned();
4903 let response = dispatch_with_context(&backend, Action::Focus, &element_context(0));
4904 assert!(response.ok, "{response:?}");
4905 assert_eq!(
4906 backend.baselines.lock().unwrap().get(&key),
4907 baseline.as_ref()
4908 );
4909 *backend.stub_value.lock().unwrap() = "ab".into();
4910 let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4911 let text = diff.to_value()["tree"].as_str().unwrap().to_string();
4912 assert!(
4913 text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4914 "{text}"
4915 );
4916 }
4917
4918 fn sample_map() -> crate::image_util::ScreenshotMap {
4919 crate::image_util::ScreenshotMap {
4920 width: 1440,
4921 height: 900,
4922 scale: 2.0,
4923 logical_width: 800,
4924 logical_height: 500,
4925 origin: crate::image_util::MapPoint { x: 120, y: 40 },
4926 fit: "width".into(),
4927 pad: crate::image_util::MapPoint { x: 0, y: 0 },
4928 content_width: 1440,
4929 content_height: 900,
4930 map: crate::image_util::MapFormulas {
4931 image_to_screen:
4932 "screen = origin + (image - pad) * logical_size / image_content_size".into(),
4933 screen_to_image:
4934 "image = pad + (screen - origin) * image_content_size / logical_size".into(),
4935 },
4936 }
4937 }
4938
4939 #[test]
4940 fn secure_ocr_frames_receives_the_map_of_the_ocr_capture() {
4941 let cache = crate::cache::CacheEnv::acquire();
4945 assert!(cache.dir().is_dir());
4946 let backend = StubBackend::default();
4947 *backend.secure_frames.lock().unwrap() = Some(vec![crate::vision::ocr::ImageRect {
4948 x: 10,
4949 y: 20,
4950 width: 100,
4951 height: 40,
4952 }]);
4953 let context = CallContext {
4954 app: Some("Finder".into()),
4955 session: Some("p12b-redact".into()),
4956 ..CallContext::default()
4957 };
4958 let blocks = vec![OcrBlock {
4959 text: "hunter2".into(),
4960 x: 20,
4961 y: 30,
4962 width: 30,
4963 height: 10,
4964 confidence: 0.9,
4965 }];
4966 let map = sample_map();
4967 let kept = redact_secure_ocr_blocks(blocks.clone(), &backend, &context, &map).unwrap();
4968 assert!(kept.is_empty(), "the block sits inside the secure frame");
4969
4970 let seen = backend
4971 .secure_frames_map
4972 .lock()
4973 .unwrap()
4974 .clone()
4975 .expect("secure_ocr_frames was called");
4976 assert_eq!(seen.origin, map.origin);
4977 assert_eq!(seen.logical_width, map.logical_width);
4978 assert_eq!(seen.content_width, map.content_width);
4979
4980 let outside = vec![OcrBlock {
4981 text: "visible".into(),
4982 x: 900,
4983 y: 700,
4984 width: 30,
4985 height: 10,
4986 confidence: 0.9,
4987 }];
4988 let kept = redact_secure_ocr_blocks(outside, &backend, &context, &map).unwrap();
4989 assert_eq!(kept.len(), 1);
4990 }
4991
4992 #[test]
4996 fn marks_change_the_pixels_the_screenshot_writes() {
4997 let cache = crate::cache::CacheEnv::acquire();
4998 let backend = StubBackend::default();
4999 let context = CallContext {
5000 app: Some("Notes".into()),
5001 session: Some("marks".into()),
5002 ..CallContext::default()
5003 };
5004 let plain = cache.dir().join("plain.png");
5005 let marked = cache.dir().join("marked.png");
5006
5007 let shot = |path: std::path::PathBuf, marks: bool| {
5008 screenshot(
5009 &backend,
5010 &context,
5011 ScreenshotArgs {
5012 output: Some(path.to_string_lossy().into_owned()),
5013 region: None,
5014 format: Some(ImageFormat::Png),
5015 max_width: None,
5016 canvas: None,
5017 include_base64: false,
5018 window_crop: false,
5019 marks,
5020 },
5021 )
5022 };
5023
5024 let clean = shot(plain.clone(), false).expect("plain screenshot");
5025 let marked_body = shot(marked.clone(), true).expect("marked screenshot");
5026
5027 let clean_bytes = std::fs::read(&plain).expect("plain png");
5028 let marked_bytes = std::fs::read(&marked).expect("marked png");
5029 assert_ne!(
5030 clean_bytes, marked_bytes,
5031 "--marks wrote the same image as a plain screenshot"
5032 );
5033
5034 let seen = backend
5037 .marks_map
5038 .lock()
5039 .unwrap()
5040 .clone()
5041 .expect("marks_for_tree was called");
5042 assert_eq!(
5043 i64::from(seen.origin.x),
5044 clean["origin"]["x"].as_i64().unwrap()
5045 );
5046 assert_eq!(
5047 i64::from(seen.origin.y),
5048 clean["origin"]["y"].as_i64().unwrap()
5049 );
5050 assert_eq!(
5051 seen.content_width,
5052 clean["content_width"].as_u64().unwrap() as u32
5053 );
5054 assert_eq!(
5055 seen.logical_width,
5056 clean["logical_width"].as_u64().unwrap() as u32
5057 );
5058
5059 let warnings = marked_body["warnings"].as_array();
5061 assert!(
5062 !warnings
5063 .map(|items| items
5064 .iter()
5065 .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))))
5066 .unwrap_or(false),
5067 "unexpected marks warning: {warnings:?}"
5068 );
5069 }
5070
5071 #[test]
5072 fn marks_warn_when_no_frame_is_inside_the_capture() {
5073 let cache = crate::cache::CacheEnv::acquire();
5075 let backend = StubBackend::default();
5076 backend.set_live({
5079 let mut window = crate::tree::UiNode::bare("AXWindow");
5080 window.frame = Some(crate::tree::NodeFrame {
5081 x: 9_000.0,
5082 y: 9_000.0,
5083 width: 100.0,
5084 height: 100.0,
5085 });
5086 window
5087 });
5088 let context = CallContext {
5089 session: Some("marks-empty".into()),
5090 ..CallContext::default()
5091 };
5092 let body = screenshot(
5093 &backend,
5094 &context,
5095 ScreenshotArgs {
5096 output: Some(
5097 cache
5098 .dir()
5099 .join("marks-empty.png")
5100 .to_string_lossy()
5101 .into_owned(),
5102 ),
5103 region: None,
5104 format: Some(ImageFormat::Png),
5105 max_width: None,
5106 canvas: None,
5107 include_base64: false,
5108 window_crop: false,
5109 marks: true,
5110 },
5111 )
5112 .expect("screenshot");
5113 let warnings = body["warnings"].as_array().expect("warnings");
5114 assert!(
5115 warnings
5116 .iter()
5117 .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))),
5118 "expected a marks warning, got {warnings:?}"
5119 );
5120 }
5121
5122 #[test]
5123 fn no_secure_frames_keeps_every_ocr_block() {
5124 let cache = crate::cache::CacheEnv::acquire();
5125 assert!(cache.dir().is_dir());
5126 let backend = StubBackend::default();
5127 let context = CallContext {
5128 app: Some("Finder".into()),
5129 session: Some("p12b-redact-none".into()),
5130 ..CallContext::default()
5131 };
5132 let blocks = vec![OcrBlock {
5133 text: "hello".into(),
5134 x: 1,
5135 y: 2,
5136 width: 3,
5137 height: 4,
5138 confidence: 0.9,
5139 }];
5140 let kept = redact_secure_ocr_blocks(blocks, &backend, &context, &sample_map()).unwrap();
5141 assert_eq!(kept.len(), 1);
5142 }
5143
5144 #[test]
5145 fn image_coordinate_on_a_stored_map_becomes_the_screen_point() {
5146 let cache = crate::cache::CacheEnv::acquire();
5147 assert!(cache.dir().is_dir());
5148 let map = sample_map();
5149 cache::store_screenshot_map(Some("p5-map"), &map).unwrap();
5150 let backend = StubBackend::default();
5151 let context = CallContext {
5152 coord: Some("image".into()),
5153 session: Some("p5-map".into()),
5154 ..CallContext::default()
5155 };
5156 let response = dispatch_with_context(&backend, Action::Move { x: 100, y: 50 }, &context);
5157 assert!(response.ok, "{response:?}");
5158 let (screen_x, screen_y) = image_util::image_to_screen(&map, 100.0, 50.0);
5159 let value = response.to_value();
5160 assert_eq!(value["x"], json!(screen_x.round() as i64));
5161 assert_eq!(value["y"], json!(screen_y.round() as i64));
5162 assert_eq!(value["x"], 176);
5163 assert_eq!(value["y"], 68);
5164 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5165
5166 let outside = dispatch_with_context(&backend, Action::Move { x: 1440, y: 50 }, &context);
5167 assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
5168 assert!(outside.to_value()["error"]["message"]
5169 .as_str()
5170 .unwrap()
5171 .contains("outside"));
5172 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5173
5174 let missing = dispatch_with_context(
5175 &backend,
5176 Action::Move { x: 1, y: 2 },
5177 &CallContext {
5178 coord: Some("image".into()),
5179 session: Some("no-map".into()),
5180 ..CallContext::default()
5181 },
5182 );
5183 assert_eq!(missing.to_value()["error"]["code"], "invalid_arguments");
5184 assert!(missing.to_value()["error"]["message"]
5185 .as_str()
5186 .unwrap()
5187 .contains("take a screenshot first"));
5188
5189 let action =
5190 crate::adapters::translate("openai", &json!({"type": "move", "x": 100, "y": 50}))
5191 .unwrap();
5192 let adapted = dispatch_with_context(
5193 &backend,
5194 action,
5195 &CallContext {
5196 coord: Some("image".into()),
5197 session: Some("p5-map".into()),
5198 adapter: Some("openai".into()),
5199 ..CallContext::default()
5200 },
5201 );
5202 assert!(adapted.ok, "{adapted:?}");
5203 assert_eq!(adapted.to_value()["adapter"], "openai");
5204 assert_eq!(adapted.to_value()["x"], 176);
5205 assert_eq!(adapted.to_value()["y"], 68);
5206 assert!(adapted.to_value().get("state").is_none());
5207
5208 let missing_adapter = handle_request_line_with(
5209 r#"{"adapter":"anthropic","action":"mouse_move","coordinate":[1,2],"session":"no-map"}"#,
5210 None,
5211 );
5212 assert!(!missing_adapter.ok);
5213 assert_eq!(
5214 missing_adapter.to_value()["error"]["code"],
5215 "invalid_arguments"
5216 );
5217 assert!(missing_adapter.to_value()["error"]["message"]
5218 .as_str()
5219 .unwrap()
5220 .contains("take a screenshot first"));
5221 assert_eq!(missing_adapter.to_value()["adapter"], "anthropic");
5222
5223 let waited = handle_request_line_with(r#"{"adapter":"openai","type":"wait"}"#, None);
5224 assert!(waited.ok, "{waited:?}");
5225 assert_eq!(waited.to_value()["adapter"], "openai");
5226 assert_eq!(waited.to_value()["waited_ms"], 0);
5227 }
5228
5229 #[test]
5230 fn return_state_includes_the_numbered_tree_and_ignores_read_only() {
5231 let backend = StubBackend::default();
5232 let before = backend.accessibility_apps.lock().unwrap().len();
5233 let response = dispatch_with_context(
5234 &backend,
5235 Action::Click {
5236 x: Some(3),
5237 y: Some(4),
5238 button: MouseButton::Left,
5239 double: false,
5240 count: 1,
5241 },
5242 &CallContext {
5243 app: Some("Notes".into()),
5244 return_state: true,
5245 ..CallContext::default()
5246 },
5247 );
5248 assert!(response.ok, "{response:?}");
5249 let value = response.to_value();
5250 assert_eq!(value["action"], "click");
5251 let tree = value["state"]["tree"].as_str().unwrap();
5252 assert!(tree.contains("[0] AXWindow"), "{tree}");
5253 assert!(tree.contains("AXTextArea"), "{tree}");
5254 assert!(value["state"]["diff"].is_null());
5255 assert_eq!(value["state"]["app"], "com.apple.Notes");
5256 assert_eq!(value["state"]["pid"], 10);
5257 assert_eq!(value["state"]["window_id"], 100);
5258 assert!(value.get("warnings").is_none());
5259 assert!(backend.accessibility_apps.lock().unwrap().len() > before);
5260
5261 let displays = dispatch_with_context(
5262 &backend,
5263 Action::Displays,
5264 &CallContext {
5265 return_state: true,
5266 include_screenshot: true,
5267 ..CallContext::default()
5268 },
5269 );
5270 assert!(displays.ok, "{displays:?}");
5271 assert!(displays.to_value().get("state").is_none());
5272
5273 let cursor = dispatch_with(&backend, Action::CursorPosition);
5274 assert_eq!(cursor.to_value()["x"], 4);
5275 assert_eq!(cursor.to_value()["y"], 5);
5276 assert!(cursor.to_value().get("adapter").is_none());
5277 }
5278
5279 #[test]
5280 fn return_state_refreshes_the_tree_diff_baseline() {
5281 let backend = StubBackend::default();
5282 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5283 assert!(first.ok, "{first:?}");
5284 *backend.stub_value.lock().unwrap() = "ab".into();
5285 let mut context = element_context(0);
5286 context.return_state = true;
5287 let response = dispatch_with_context(&backend, Action::Focus, &context);
5288 assert!(response.ok, "{response:?}");
5289 let value = response.to_value();
5290 let diff = value["state"]["diff"].as_str().unwrap();
5291 assert!(
5292 diff.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
5293 "{diff}"
5294 );
5295 assert!(value["state"]["tree"]
5296 .as_str()
5297 .unwrap()
5298 .contains("value=\"ab\""));
5299 *backend.stub_value.lock().unwrap() = "abc".into();
5300 let next = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5301 let value = next.to_value();
5302 let text = value["tree"].as_str().unwrap();
5303 assert!(
5304 text.contains("~ [1] AXTextArea value: \"ab\" -> \"abc\""),
5305 "{text}"
5306 );
5307 assert!(!text.contains("\"a\" -> \"abc\""), "{text}");
5308 }
5309
5310 #[test]
5311 fn include_screenshot_failure_keeps_the_action_result() {
5312 let backend = StubBackend {
5313 capture_fault: crate::test_backend::StubCaptureFault::Permission,
5314 ..StubBackend::default()
5315 };
5316 let response = dispatch_with_context(
5317 &backend,
5318 Action::Click {
5319 x: Some(1),
5320 y: Some(2),
5321 button: MouseButton::Left,
5322 double: false,
5323 count: 1,
5324 },
5325 &CallContext {
5326 app: Some("Notes".into()),
5327 return_state: true,
5328 include_screenshot: true,
5329 ..CallContext::default()
5330 },
5331 );
5332 assert!(response.ok, "{response:?}");
5333 let value = response.to_value();
5334 assert_eq!(value["action"], "click");
5335 assert_eq!(value["x"], 1);
5336 assert_eq!(value["y"], 2);
5337 assert!(value["state"]["tree"]
5338 .as_str()
5339 .unwrap()
5340 .contains("AXWindow"));
5341 assert!(value["state"]["screenshot"].is_null());
5342 let warnings = value["warnings"].as_array().unwrap();
5343 assert!(warnings
5344 .iter()
5345 .any(|item| { item.as_str().unwrap().contains("screenshot failed") }));
5346 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5347 }
5348
5349 fn batch_clicks(stop_on_error: bool) -> Value {
5350 json!({
5351 "action": "batch",
5352 "stop_on_error": stop_on_error,
5353 "actions": [
5354 {"action": "click", "x": 1, "y": 2},
5355 {"action": "click", "x": 3, "y": 4}
5356 ]
5357 })
5358 }
5359
5360 #[test]
5361 fn batch_stop_on_error_controls_the_second_attempt() {
5362 let stopped = StubBackend {
5363 fail_click: true,
5364 ..StubBackend::default()
5365 };
5366 let request = parse_request(batch_clicks(true)).unwrap();
5367 let response = dispatch_with_context(&stopped, request.action, &request.context);
5368 assert!(!response.ok);
5369 let value = response.to_value();
5370 assert_eq!(value["error"]["code"], "execution_failed");
5371 assert_eq!(value["stopped_at"], 0);
5372 assert_eq!(value["results"].as_array().unwrap().len(), 1);
5373 assert!(!value["results"][0]["ok"].as_bool().unwrap());
5374 assert_eq!(stopped.click_attempts.load(Ordering::SeqCst), 2);
5375 assert!(value["error"]["message"]
5376 .as_str()
5377 .unwrap()
5378 .contains("batch failed at action 0"));
5379
5380 let both = StubBackend {
5381 fail_click: true,
5382 ..StubBackend::default()
5383 };
5384 let request = parse_request(batch_clicks(false)).unwrap();
5385 let response = dispatch_with_context(&both, request.action, &request.context);
5386 assert!(!response.ok);
5387 let value = response.to_value();
5388 assert!(value.get("stopped_at").is_none(), "{value}");
5389 assert_eq!(value["results"].as_array().unwrap().len(), 2);
5390 assert!(!value["results"][1]["ok"].as_bool().unwrap());
5391 assert_eq!(both.click_attempts.load(Ordering::SeqCst), 4);
5392
5393 let nested = parse_request(json!({
5394 "action": "batch",
5395 "actions": [{"action": "batch", "actions": []}]
5396 }))
5397 .unwrap();
5398 let response = dispatch_with(&StubBackend::default(), nested.action);
5399 assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
5400 assert_eq!(response.to_value()["stopped_at"], 0);
5401 assert!(response.to_value()["error"]["message"]
5402 .as_str()
5403 .unwrap()
5404 .contains("nested batch"));
5405
5406 let backend = StubBackend::default();
5407 backend.seed_element(notes_button());
5408 let request = parse_request(json!({
5409 "action": "batch",
5410 "app": "Notes",
5411 "actions": [
5412 {"action": "press", "element": 1},
5413 {"action": "focus", "element": 1}
5414 ]
5415 }))
5416 .unwrap();
5417 assert!(matches!(
5418 request.action,
5419 Action::Batch {
5420 stop_on_error: true,
5421 ..
5422 }
5423 ));
5424 let response = dispatch_with_context(&backend, request.action, &request.context);
5425 assert!(response.ok, "{response:?}");
5426 let results = response.to_value()["results"].clone();
5427 assert_eq!(results[0]["action"], "press");
5428 assert_eq!(results[0]["element"], 1);
5429 assert_eq!(results[1]["action"], "focus");
5430 assert!(response.to_value().get("state").is_none());
5431 }
5432
5433 #[test]
5434 fn p9_menu_list_round_trip_records_no_args() {
5435 let backend = StubBackend::default();
5436 backend
5437 .menu_sections
5438 .lock()
5439 .unwrap()
5440 .push(crate::backend::MenuNode {
5441 title: "File".into(),
5442 enabled: true,
5443 children: vec![crate::backend::MenuNode {
5444 title: "Save".into(),
5445 enabled: true,
5446 children: Vec::new(),
5447 }],
5448 });
5449 let response = dispatch_with_context(
5450 &backend,
5451 Action::MenuList {
5452 app: Some("Notes".into()),
5453 depth: Some(2),
5454 },
5455 &ctx("Notes"),
5456 );
5457 assert!(response.ok, "{:?}", response.to_value());
5458 let value = response.to_value();
5459 assert_eq!(value["action"], "menu_list");
5460 assert_eq!(value["app"], "Notes");
5461 assert_eq!(value["sections"][0]["title"], "File");
5462 assert_eq!(value["sections"][0]["children"][0]["title"], "Save");
5463 }
5464
5465 #[test]
5466 fn p9_menu_click_records_path_and_reports_disabled() {
5467 let mut backend = StubBackend::default();
5468 let response = dispatch_with_context(
5469 &backend,
5470 Action::MenuClick {
5471 app: Some("Notes".into()),
5472 path: "File > Save".into(),
5473 },
5474 &ctx("Notes"),
5475 );
5476 assert!(response.ok, "{:?}", response.to_value());
5477 let value = response.to_value();
5478 assert_eq!(value["path"], serde_json::json!(["File", "Save"]));
5479 assert_eq!(value["pressed"], true);
5480 let recorded = backend.menu_clicks.lock().unwrap().clone();
5481 assert_eq!(recorded.len(), 1);
5482 assert_eq!(
5483 recorded[0].segments,
5484 vec!["File".to_string(), "Save".to_string()]
5485 );
5486
5487 backend.menu_disabled = true;
5488 let response = dispatch_with_context(
5489 &backend,
5490 Action::MenuClick {
5491 app: Some("Notes".into()),
5492 path: "File > Save".into(),
5493 },
5494 &ctx("Notes"),
5495 );
5496 assert!(!response.ok, "disabled menu item must fail");
5497 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5498 }
5499
5500 #[test]
5501 fn p9_apple_event_records_only_length_and_language() {
5502 let backend = StubBackend::default();
5503 *backend.apple_event_stdout.lock().unwrap() = "Notes".into();
5504 let secret = "system prompt with token ABCDEFG";
5505 let response = dispatch_with_context(
5506 &backend,
5507 Action::AppleEvent {
5508 app: Some("Notes".into()),
5509 script: secret.into(),
5510 language: Some(AppleEventLanguage::AppleScript),
5511 },
5512 &ctx("Notes"),
5513 );
5514 assert!(response.ok, "{:?}", response.to_value());
5515 let value = response.to_value();
5516 assert_eq!(value["language"], "AppleScript");
5517 assert_eq!(value["script_bytes"], secret.len());
5518 assert_eq!(value["stdout"], "Notes");
5519 assert!(value["script_hash"].is_string());
5520 let recorded = backend.apple_event_calls.lock().unwrap().clone();
5521 assert_eq!(recorded.len(), 1);
5522 assert_eq!(recorded[0].1, AppleEventLanguage::AppleScript);
5523 assert_eq!(recorded[0].2, secret.len());
5525 }
5526
5527 #[test]
5528 fn p9_apple_event_records_language_javascript() {
5529 let backend = StubBackend::default();
5530 let response = dispatch_with_context(
5531 &backend,
5532 Action::AppleEvent {
5533 app: Some("Notes".into()),
5534 script: "Application('Notes').name()".into(),
5535 language: Some(AppleEventLanguage::JavaScript),
5536 },
5537 &ctx("Notes"),
5538 );
5539 assert!(response.ok);
5540 assert_eq!(response.to_value()["language"], "JavaScript");
5541 }
5542
5543 #[test]
5544 fn p9_apple_event_failure_surfaces_execution_failed() {
5545 let backend = StubBackend {
5546 apple_event_fail: true,
5547 ..Default::default()
5548 };
5549 *backend.apple_event_stderr.lock().unwrap() = "execution error: 42".into();
5550 let response = dispatch_with_context(
5551 &backend,
5552 Action::AppleEvent {
5553 app: Some("Notes".into()),
5554 script: "1/0".into(),
5555 language: None,
5556 },
5557 &ctx("Notes"),
5558 );
5559 assert!(!response.ok);
5560 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5561 }
5562
5563 #[test]
5564 fn p9_dock_list_and_click_round_trip() {
5565 let backend = StubBackend::default();
5566 backend.dock_items.lock().unwrap().extend([
5567 "Finder".to_string(),
5568 "Terminal".to_string(),
5569 "Mail".to_string(),
5570 ]);
5571 let response = dispatch_with(&backend, Action::DockList {});
5572 assert!(response.ok);
5573 let value = response.to_value();
5574 let items = value["items"].as_array().unwrap();
5575 assert_eq!(items.len(), 3);
5576 assert_eq!(items[0]["title"], "Finder");
5577
5578 let response = dispatch_with(
5579 &backend,
5580 Action::DockClick {
5581 title: "terminal".into(),
5582 },
5583 );
5584 assert!(response.ok, "{:?}", response.to_value());
5585 assert_eq!(response.to_value()["pressed"], true);
5586 assert_eq!(
5587 backend.dock_clicks.lock().unwrap().clone(),
5588 vec!["terminal".to_string()]
5589 );
5590
5591 let response = dispatch_with(
5592 &backend,
5593 Action::DockClick {
5594 title: "missing".into(),
5595 },
5596 );
5597 assert!(!response.ok);
5598 assert_eq!(response.to_value()["error"]["code"], "element_not_found");
5599 }
5600
5601 #[test]
5602 fn p9_spaces_reports_screen_groups_from_fake_list() {
5603 use crate::target::{WindowBounds, WindowInfo};
5604 use crate::types::DisplayInfo;
5605 use crate::window_groups::SpacesReport;
5606
5607 let mut spaces_report = SpacesReport {
5608 screens: Vec::new(),
5609 note: String::new(),
5610 };
5611 spaces_report
5612 .screens
5613 .push(crate::window_groups::SpaceScreen {
5614 index: 0,
5615 name: "Display 0".into(),
5616 x: 0,
5617 y: 0,
5618 width: 1000,
5619 height: 1000,
5620 is_primary: true,
5621 windows: vec![WindowInfo {
5622 id: 1,
5623 pid: 100,
5624 app: "Notes".into(),
5625 title: "Note 1".into(),
5626 bounds: WindowBounds {
5627 x: 10,
5628 y: 10,
5629 width: 200,
5630 height: 200,
5631 },
5632 z: 0,
5633 layer: 0,
5634 on_screen: true,
5635 ax_matched: false,
5636 }],
5637 });
5638
5639 let backend = StubBackend::default();
5640 *backend.spaces_report.lock().unwrap() = spaces_report;
5641 let response = dispatch_with(&backend, Action::Spaces {});
5642 assert!(response.ok);
5643 let value = response.to_value();
5644 assert_eq!(value["action"], "spaces");
5645 assert_eq!(value["screens"][0]["name"], "Display 0");
5646 assert_eq!(value["screens"][0]["windows"][0]["title"], "Note 1");
5647 assert_eq!(value["screens"][0]["is_primary"], true);
5648 let _ = DisplayInfo {
5650 id: 0,
5651 name: String::new(),
5652 x: 0,
5653 y: 0,
5654 width: 0,
5655 height: 0,
5656 scale: 1.0,
5657 is_primary: false,
5658 };
5659 }
5660
5661 #[test]
5662 fn p9_shortcuts_run_invokes_with_only_the_name() {
5663 let backend = StubBackend::default();
5666 let response = dispatch_with(
5667 &backend,
5668 Action::ShortcutsRun {
5669 name: "Open URL".into(),
5670 },
5671 );
5672 if !response.ok {
5678 let value = response.to_value();
5679 let code = value["error"]["code"].as_str().unwrap_or_default();
5680 assert!(
5681 matches!(code, "execution_failed" | "unsupported"),
5682 "{value:?}"
5683 );
5684 }
5685 }
5686
5687 #[test]
5688 fn session_prune_action_removes_an_empty_temp_session() {
5689 let _cache = crate::cache::CacheEnv::acquire();
5690 let dir = cache::session_dir(Some("empty-session")).unwrap();
5691 assert!(dir.join("owner.json").is_file());
5692 let response = dispatch(Action::SessionPrune {
5693 dry_run: true,
5694 ttl_secs: Some(0),
5695 });
5696 assert!(response.ok, "{response:?}");
5697 assert_eq!(response.to_value()["dry_run"], true);
5698 assert!(response.to_value()["removed"]
5699 .as_array()
5700 .unwrap()
5701 .iter()
5702 .any(|name| name == "empty-session"));
5703 assert!(dir.exists());
5704 let response = dispatch(Action::SessionPrune {
5705 dry_run: false,
5706 ttl_secs: Some(0),
5707 });
5708 assert!(response.ok, "{response:?}");
5709 assert!(!dir.exists());
5710 }
5711
5712 #[test]
5713 fn pause_short_circuits_before_the_stub_clicks() {
5714 let backend = StubBackend::default();
5715 let _paused = policy::override_paused(true);
5716 let response = dispatch_with(
5717 &backend,
5718 Action::Click {
5719 x: Some(1),
5720 y: Some(2),
5721 button: MouseButton::Left,
5722 double: false,
5723 count: 1,
5724 },
5725 );
5726 assert_eq!(response.to_value()["error"]["code"], "paused");
5727 assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
5728 assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5729 let waited = dispatch_with(
5730 &backend,
5731 Action::Wait {
5732 ms: Some(0),
5733 seconds: None,
5734 },
5735 );
5736 assert!(waited.ok, "{waited:?}");
5737 }
5738
5739 #[test]
5740 fn confirm_hook_allows_quit_and_a_failure_does_not() {
5741 let backend = StubBackend::default();
5742 let dir = std::env::temp_dir().join(format!(
5743 "computer-use-hook-{}-{}",
5744 std::process::id(),
5745 std::time::SystemTime::now()
5746 .duration_since(std::time::UNIX_EPOCH)
5747 .map(|duration| duration.as_nanos())
5748 .unwrap_or(0)
5749 ));
5750 let _ = std::fs::remove_dir_all(&dir);
5751 std::fs::create_dir_all(&dir).unwrap();
5752 #[cfg(unix)]
5753 let (allow, deny) = {
5754 let allow = dir.join("allow.sh");
5755 let deny = dir.join("deny.sh");
5756 std::fs::write(&allow, "#!/bin/sh\ncat >/dev/null\nexit 0\n").unwrap();
5757 std::fs::write(&deny, "#!/bin/sh\ncat >/dev/null\nexit 1\n").unwrap();
5758 use std::os::unix::fs::PermissionsExt;
5759 std::fs::set_permissions(&allow, std::fs::Permissions::from_mode(0o700)).unwrap();
5760 std::fs::set_permissions(&deny, std::fs::Permissions::from_mode(0o700)).unwrap();
5761 (allow.display().to_string(), deny.display().to_string())
5762 };
5763 #[cfg(windows)]
5764 let (allow, deny) = ("cmd /C exit 0".to_string(), "cmd /C exit 1".to_string());
5765 let mut context = ctx("412");
5766 context.confirm_cmd = Some(deny);
5767 let denied = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5768 assert_eq!(denied.to_value()["error"]["code"], "confirm_denied");
5769 assert!(backend.quit.lock().unwrap().is_empty());
5770 context.confirm_cmd = Some(allow);
5771 let allowed = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5772 assert!(allowed.ok, "{allowed:?}");
5773 assert_eq!(backend.quit.lock().unwrap().len(), 1);
5774 let _ = std::fs::remove_dir_all(&dir);
5775 }
5776
5777 #[test]
5778 fn blocked_url_prevents_the_click() {
5779 let mut backend = StubBackend::default();
5780 backend.apps.push(crate::target::AppInfo {
5781 name: "Safari".into(),
5782 bundle_id: Some("com.apple.Safari".into()),
5783 pid: 80,
5784 path: Some("/Applications/Safari.app/Contents/MacOS/Safari".into()),
5785 active: false,
5786 hidden: false,
5787 activation_policy: crate::target::ActivationPolicy::Regular,
5788 });
5789 backend
5790 .browser_urls
5791 .lock()
5792 .unwrap()
5793 .insert(80, "https://accounts.example.com/login".into());
5794 let dir = std::env::temp_dir().join(format!(
5795 "computer-use-url-{}-{}",
5796 std::process::id(),
5797 std::time::SystemTime::now()
5798 .duration_since(std::time::UNIX_EPOCH)
5799 .map(|duration| duration.as_nanos())
5800 .unwrap_or(0)
5801 ));
5802 let _ = std::fs::remove_dir_all(&dir);
5803 std::fs::create_dir_all(&dir).unwrap();
5804 let policy_path = dir.join("policy.toml");
5805 std::fs::write(
5806 &policy_path,
5807 "version = 1\n[browser]\nblocked_urls = [\"https://accounts.example.com/*\"]\n",
5808 )
5809 .unwrap();
5810 let mut context = ctx("80");
5811 context.policy = Some(policy_path.display().to_string());
5812 let response = dispatch_with_context(
5813 &backend,
5814 Action::Click {
5815 x: Some(3),
5816 y: Some(4),
5817 button: MouseButton::Left,
5818 double: false,
5819 count: 1,
5820 },
5821 &context,
5822 );
5823 assert_eq!(response.to_value()["error"]["code"], "blocked_url");
5824 assert!(response.to_value()["error"]["message"]
5825 .as_str()
5826 .unwrap()
5827 .contains("https://accounts.example.com/*"));
5828 assert!(!response.to_value()["error"]["message"]
5829 .as_str()
5830 .unwrap()
5831 .contains("login"));
5832 assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5833 backend
5834 .browser_urls
5835 .lock()
5836 .unwrap()
5837 .insert(80, "https://example.com/".into());
5838 let allowed = dispatch_with_context(
5839 &backend,
5840 Action::Click {
5841 x: Some(3),
5842 y: Some(4),
5843 button: MouseButton::Left,
5844 double: false,
5845 count: 1,
5846 },
5847 &context,
5848 );
5849 assert!(allowed.ok, "{allowed:?}");
5850 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5851 let _ = std::fs::remove_dir_all(&dir);
5852 }
5853
5854 fn temp_dir(label: &str) -> std::path::PathBuf {
5855 let dir = std::env::temp_dir().join(format!(
5856 "computer-use-{label}-{}-{}",
5857 std::process::id(),
5858 std::time::SystemTime::now()
5859 .duration_since(std::time::UNIX_EPOCH)
5860 .map(|duration| duration.as_nanos())
5861 .unwrap_or(0)
5862 ));
5863 let _ = std::fs::remove_dir_all(&dir);
5864 std::fs::create_dir_all(&dir).unwrap();
5865 dir
5866 }
5867
5868 fn click_at(x: i32, y: i32) -> Action {
5869 Action::Click {
5870 x: Some(x),
5871 y: Some(y),
5872 button: MouseButton::Left,
5873 double: false,
5874 count: 1,
5875 }
5876 }
5877
5878 fn last_audit(dir: &std::path::Path) -> Value {
5879 let text = std::fs::read_to_string(dir.join("audit.jsonl")).unwrap();
5880 serde_json::from_str(text.lines().last().unwrap()).unwrap()
5881 }
5882
5883 #[test]
5884 fn auto_event_falls_back_to_activate_unless_blocked() {
5885 let backend = StubBackend {
5886 fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5887 ..StubBackend::default()
5888 };
5889 let response = dispatch_with(&backend, click_at(3, 4));
5890 assert!(response.ok, "{response:?}");
5891 let value = response.to_value();
5892 assert_eq!(value["tier"], "activate");
5893 assert_eq!(value["cursor_moved"], true);
5894 assert_eq!(value["focus_restored"], true);
5895 assert_eq!(
5896 backend.tiers_run.lock().unwrap().as_slice(),
5897 ["event", "activate"]
5898 );
5899 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5900
5901 let held = StubBackend {
5902 fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5903 ..StubBackend::default()
5904 };
5905 let dir = temp_dir("lock-fallback");
5906 let path = dir.join("foreground.lock");
5907 let _slot = crate::foreground::push_lock_path(&path);
5908 let _lock = crate::foreground::try_lock(&path).unwrap();
5909 let response = dispatch_with(&held, click_at(3, 4));
5910 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5911 assert!(response.to_value()["error"]["message"]
5912 .as_str()
5913 .unwrap()
5914 .contains("stub tier event failed"));
5915 assert_eq!(held.tiers_run.lock().unwrap().as_slice(), ["event"]);
5916 assert_eq!(held.clicks.load(Ordering::SeqCst), 0);
5917 drop(_lock);
5918 let _ = std::fs::remove_dir_all(&dir);
5919
5920 let background = StubBackend {
5921 fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5922 ..StubBackend::default()
5923 };
5924 let response = dispatch_with_context(
5925 &background,
5926 click_at(1, 2),
5927 &CallContext {
5928 require_background: true,
5929 ..CallContext::default()
5930 },
5931 );
5932 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5933 assert_eq!(background.tiers_run.lock().unwrap().as_slice(), ["event"]);
5934 assert_eq!(background.clicks.load(Ordering::SeqCst), 0);
5935
5936 let refused = StubBackend::default();
5937 let response = dispatch_with_context(
5938 &refused,
5939 click_at(1, 2),
5940 &CallContext {
5941 tier: Some("activate".into()),
5942 require_background: true,
5943 ..CallContext::default()
5944 },
5945 );
5946 assert!(response.to_value()["error"]["message"]
5947 .as_str()
5948 .unwrap()
5949 .contains("--require-background"));
5950 assert!(refused.tiers_run.lock().unwrap().is_empty());
5951 assert_eq!(refused.clicks.load(Ordering::SeqCst), 0);
5952 }
5953
5954 #[test]
5955 fn activate_names_the_lock_holder_before_it_clicks() {
5956 let backend = StubBackend::default();
5957 let dir = temp_dir("lock-held");
5958 let path = dir.join("foreground.lock");
5959 let _slot = crate::foreground::push_lock_path(&path);
5960 let _lock = crate::foreground::try_lock(&path).unwrap();
5961 let response = dispatch_with_context(
5962 &backend,
5963 click_at(9, 8),
5964 &CallContext {
5965 tier: Some("activate".into()),
5966 ..CallContext::default()
5967 },
5968 );
5969 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5970 let message = response.to_value()["error"]["message"]
5971 .as_str()
5972 .unwrap()
5973 .to_string();
5974 assert!(
5975 message.contains(&format!(
5976 "foreground.lock is held by pid {}",
5977 std::process::id()
5978 )),
5979 "{message}"
5980 );
5981 assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
5982 assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 1);
5983 assert!(backend.tiers_run.lock().unwrap().is_empty());
5984 drop(_lock);
5985 let _ = std::fs::remove_dir_all(&dir);
5986 }
5987
5988 #[test]
5989 fn synthetic_without_the_feature_does_not_hit_test_or_click() {
5990 let backend = StubBackend::default();
5991 let response = dispatch_with_context(
5992 &backend,
5993 click_at(1, 1),
5994 &CallContext {
5995 tier: Some("synthetic".into()),
5996 ..CallContext::default()
5997 },
5998 );
5999 if crate::tier::skylight_compiled() {
6000 assert!(response.ok, "{response:?}");
6001 assert_eq!(response.to_value()["tier"], "synthetic");
6002 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
6003 assert_eq!(backend.tiers_run.lock().unwrap().as_slice(), ["synthetic"]);
6004 } else {
6005 assert_eq!(response.to_value()["error"]["code"], "unsupported");
6006 assert!(response.to_value()["error"]["message"]
6007 .as_str()
6008 .unwrap()
6009 .contains("built without skylight feature"));
6010 assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 0);
6011 assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
6012 assert!(backend.tiers_run.lock().unwrap().is_empty());
6013 }
6014 }
6015
6016 #[test]
6017 fn missing_overlay_helper_warns_and_no_overlay_skips_it() {
6018 let dir = temp_dir("overlay-warn");
6019 let _slot = crate::overlay::push_socket_path(dir.join("missing.sock"));
6020 let backend = StubBackend::default();
6021 let response = dispatch_with_context(
6022 &backend,
6023 click_at(4, 5),
6024 &CallContext {
6025 session: Some("agent".into()),
6026 ..CallContext::default()
6027 },
6028 );
6029 assert!(response.ok, "{response:?}");
6030 assert_eq!(response.to_value()["tier"], "event");
6031 assert_eq!(response.to_value()["cursor_moved"], false);
6032 assert!(response.to_value().get("focus_restored").is_none());
6033 let warnings = response.to_value()["warnings"].as_array().unwrap().clone();
6034 assert!(warnings
6035 .iter()
6036 .any(|item| item == "overlay helper not running"));
6037
6038 let quiet = dispatch_with_context(
6039 &backend,
6040 click_at(4, 5),
6041 &CallContext {
6042 no_overlay: true,
6043 ..CallContext::default()
6044 },
6045 );
6046 assert!(quiet.ok, "{quiet:?}");
6047 assert!(quiet.to_value().get("warnings").is_none());
6048 let _ = std::fs::remove_dir_all(&dir);
6049 }
6050
6051 #[test]
6052 fn audit_records_decision_source_and_the_real_code() {
6053 let dir = temp_dir("audit");
6054 let _state = crate::audit::push_state_dir(&dir);
6055 let backend = StubBackend::default();
6056 let response = dispatch_with(&backend, click_at(2, 3));
6057 assert!(response.ok, "{response:?}");
6058 let line = last_audit(&dir);
6059 assert_eq!(line["action"], "click");
6060 assert_eq!(line["code"], "ok");
6061 assert_eq!(line["decision"], "allow");
6062 assert_eq!(line["source"], "default");
6063 assert_eq!(line["tier"], "event");
6064 assert!(line.get("text").is_none());
6065
6066 let denied = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
6067 assert_eq!(denied.to_value()["error"]["code"], "policy_denied");
6068 let line = last_audit(&dir);
6069 assert_eq!(line["action"], "launch");
6070 assert_eq!(line["code"], "policy_denied");
6071 assert_eq!(line["decision"], "deny");
6072 assert_eq!(line["source"], "builtin");
6073
6074 let checked =
6075 dispatch_with_context(&backend, Action::PolicyCheck, &ctx("com.apple.Terminal"));
6076 assert!(checked.ok, "{checked:?}");
6077 assert_eq!(checked.to_value()["decision"], "deny");
6078 let line = last_audit(&dir);
6079 assert_eq!(line["action"], "policy_check");
6080 assert_eq!(line["code"], "ok");
6081 assert_eq!(line["decision"], "deny");
6082 assert_eq!(line["source"], "builtin");
6083
6084 let bad = dispatch_with(
6085 &backend,
6086 Action::Click {
6087 x: None,
6088 y: None,
6089 button: MouseButton::Left,
6090 double: false,
6091 count: 1,
6092 },
6093 );
6094 assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
6095 let line = last_audit(&dir);
6096 assert_eq!(line["code"], "invalid_arguments");
6097 assert_eq!(line["decision"], "allow");
6098 assert_eq!(line["source"], "default");
6099 let _ = std::fs::remove_dir_all(&dir);
6100 }
6101
6102 #[test]
6103 fn drag_hold_click_and_keys_use_the_stub() {
6104 let backend = StubBackend::default();
6105 let short = dispatch_with(
6106 &backend,
6107 Action::Drag {
6108 path: vec![crate::types::Point { x: 1, y: 1 }],
6109 button: MouseButton::Left,
6110 },
6111 );
6112 assert_eq!(short.to_value()["error"]["code"], "invalid_arguments");
6113 assert_eq!(backend.moves.load(Ordering::SeqCst), 0);
6114
6115 let dragged = dispatch_with(
6116 &backend,
6117 Action::Drag {
6118 path: vec![
6119 crate::types::Point { x: 1, y: 2 },
6120 crate::types::Point { x: 3, y: 4 },
6121 crate::types::Point { x: 5, y: 6 },
6122 ],
6123 button: MouseButton::Left,
6124 },
6125 );
6126 assert!(dragged.ok, "{dragged:?}");
6127 assert_eq!(dragged.to_value()["moves"], 2);
6128 assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6129 assert_eq!(dragged.to_value()["tier"], "event");
6130
6131 let ax = dispatch_with_context(
6132 &backend,
6133 Action::Drag {
6134 path: vec![
6135 crate::types::Point { x: 1, y: 2 },
6136 crate::types::Point { x: 3, y: 4 },
6137 ],
6138 button: MouseButton::Left,
6139 },
6140 &CallContext {
6141 tier: Some("ax".into()),
6142 ..CallContext::default()
6143 },
6144 );
6145 assert_eq!(ax.to_value()["error"]["code"], "unsupported");
6146 assert!(ax.to_value()["error"]["message"]
6147 .as_str()
6148 .unwrap()
6149 .contains("pass --element"));
6150 assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6151
6152 let triple = dispatch_with(
6153 &backend,
6154 Action::TripleClick {
6155 x: Some(8),
6156 y: Some(9),
6157 button: MouseButton::Left,
6158 },
6159 );
6160 assert!(triple.ok, "{triple:?}");
6161 assert_eq!(triple.to_value()["action"], "triple_click");
6162 assert_eq!(triple.to_value()["count"], 3);
6163 assert_eq!(backend.clicks.load(Ordering::SeqCst), 3);
6164
6165 let held = dispatch_with(
6166 &backend,
6167 Action::HoldKey {
6168 keys: "shift".into(),
6169 ms: 40_000,
6170 },
6171 );
6172 assert_eq!(held.to_value()["error"]["code"], "invalid_arguments");
6173 assert!(backend.holds_ms.lock().unwrap().is_empty());
6174
6175 let shift = dispatch_with(
6176 &backend,
6177 Action::HoldKey {
6178 keys: "shift".into(),
6179 ms: 400,
6180 },
6181 );
6182 assert!(shift.ok, "{shift:?}");
6183 assert_eq!(backend.holds_ms.lock().unwrap().as_slice(), [400]);
6184
6185 let sequence = dispatch_with(
6186 &backend,
6187 Action::Key {
6188 keys: "super+c Return".into(),
6189 },
6190 );
6191 assert!(sequence.ok, "{sequence:?}");
6192 assert_eq!(sequence.to_value()["strokes"], 2);
6193 let keys = backend.keys.lock().unwrap();
6194 let sequence = &keys[keys.len() - 2..];
6195 assert_eq!(sequence[0].1, "c");
6196 assert_eq!(sequence[0].0, vec![Modifier::Meta]);
6197 assert_eq!(sequence[1].1, "enter");
6198 drop(keys);
6199
6200 let unknown = dispatch_with(
6201 &backend,
6202 Action::Key {
6203 keys: "bogus".into(),
6204 },
6205 );
6206 assert_eq!(unknown.to_value()["error"]["code"], "invalid_arguments");
6207 }
6208
6209 #[test]
6210 fn mouse_down_releases_when_the_session_ends() {
6211 let backend = StubBackend::default();
6212 let down = Action::MouseDown {
6213 x: 4,
6214 y: 5,
6215 button: MouseButton::Left,
6216 };
6217 let alone = dispatch_with(&backend, down.clone());
6218 assert!(alone.ok, "{alone:?}");
6219 assert!(backend.held_buttons.lock().unwrap().is_empty());
6220 assert_eq!(
6221 backend.releases.lock().unwrap().as_slice(),
6222 [MouseButton::Left]
6223 );
6224
6225 backend.releases.lock().unwrap().clear();
6226 let _session = crate::InputSession::open(&backend);
6227 let kept = dispatch_with(&backend, down);
6228 assert!(kept.ok, "{kept:?}");
6229 assert_eq!(
6230 backend.held_buttons.lock().unwrap().as_slice(),
6231 [MouseButton::Left]
6232 );
6233 assert!(backend.releases.lock().unwrap().is_empty());
6234 drop(_session);
6235 assert!(backend.held_buttons.lock().unwrap().is_empty());
6236 assert_eq!(
6237 backend.releases.lock().unwrap().as_slice(),
6238 [MouseButton::Left]
6239 );
6240
6241 let batch = dispatch_with(
6242 &backend,
6243 Action::Batch {
6244 stop_on_error: true,
6245 actions: vec![
6246 json!({"action": "mouse_down", "x": 1, "y": 2}),
6247 json!({"action": "mouse_up", "x": 1, "y": 2}),
6248 ],
6249 },
6250 );
6251 assert!(batch.ok, "{batch:?}");
6252 assert!(backend.held_buttons.lock().unwrap().is_empty());
6253 }
6254
6255 #[test]
6256 fn clipboard_confirms_and_omits_the_text_from_the_audit() {
6257 let dir = temp_dir("clipboard-audit");
6258 let _state = crate::audit::push_state_dir(&dir);
6259 let _cache = crate::cache::CacheEnv::acquire();
6260 let backend = StubBackend::default();
6261 let secret = "do-not-log";
6262 let response = dispatch_with(
6263 &backend,
6264 Action::ClipboardSet {
6265 text: Some(secret.into()),
6266 image: None,
6267 },
6268 );
6269 assert_eq!(response.to_value()["error"]["code"], "confirm_required");
6270 assert!(backend.clipboard_text.lock().unwrap().is_none());
6271 let line = last_audit(&dir);
6272 assert_eq!(line["action"], "clipboard_set");
6273 assert_eq!(line["decision"], "confirm");
6274 assert_eq!(line["chars"], secret.chars().count() as u64);
6275 let encoded = line.to_string();
6276 assert!(!encoded.contains(secret));
6277 assert!(line.get("text").is_none());
6278
6279 *backend.clipboard_text.lock().unwrap() = Some("ready".into());
6280 let got = dispatch_with(&backend, Action::ClipboardGet);
6281 assert!(got.ok, "{got:?}");
6282 assert_eq!(got.to_value()["kind"], "text");
6283 assert_eq!(got.to_value()["text"], "ready");
6284
6285 let empty = dispatch_with(
6286 &backend,
6287 Action::ClipboardSet {
6288 text: Some(String::new()),
6289 image: None,
6290 },
6291 );
6292 assert_eq!(empty.to_value()["error"]["code"], "invalid_arguments");
6293 let _ = std::fs::remove_dir_all(&dir);
6294 }
6295
6296 #[test]
6297 fn zoom_crops_a_window_region_and_rejects_the_far_edge() {
6298 let _cache = crate::cache::CacheEnv::acquire();
6299 let backend = StubBackend::default();
6300 let response = dispatch_with_context(
6301 &backend,
6302 Action::Zoom {
6303 output: None,
6304 region: Region::new(0, 0, 10, 10).unwrap(),
6305 format: Some(ImageFormat::Png),
6306 max_width: Some(0),
6307 canvas: None,
6308 base64: false,
6309 },
6310 &CallContext {
6311 app: Some("Notes".into()),
6312 session: Some("p8-zoom".into()),
6313 ..CallContext::default()
6314 },
6315 );
6316 assert!(response.ok, "{response:?}");
6317 let value = response.to_value();
6318 assert_eq!(value["width"], 20);
6319 assert_eq!(value["height"], 20);
6320 assert_eq!(value["logical_width"], 10);
6321 assert_eq!(value["logical_height"], 10);
6322
6323 let outside = dispatch_with_context(
6324 &backend,
6325 Action::Zoom {
6326 output: None,
6327 region: Region::new(0, 0, 401, 300).unwrap(),
6328 format: None,
6329 max_width: Some(0),
6330 canvas: None,
6331 base64: false,
6332 },
6333 &CallContext {
6334 app: Some("Notes".into()),
6335 session: Some("p8-zoom-out".into()),
6336 ..CallContext::default()
6337 },
6338 );
6339 assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
6340
6341 let both = dispatch_with_context(
6342 &backend,
6343 Action::Screenshot {
6344 output: None,
6345 region: Some(Region::new(0, 0, 10, 10).unwrap()),
6346 format: None,
6347 max_width: None,
6348 canvas: None,
6349 base64: false,
6350 marks: false,
6351 },
6352 &CallContext {
6353 app: Some("Notes".into()),
6354 ..CallContext::default()
6355 },
6356 );
6357 assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
6358 assert!(both.to_value()["error"]["message"]
6359 .as_str()
6360 .unwrap()
6361 .contains("not both"));
6362 }
6363
6364 #[test]
6367 fn browser_text_action_round_trips_with_only_selector() {
6368 let text: Action =
6369 serde_json::from_str(r#"{"action":"browser_text","selector":"h1"}"#).unwrap();
6370 match &text {
6371 Action::BrowserText {
6372 selector,
6373 target_id,
6374 } => {
6375 assert_eq!(selector, "h1");
6376 assert!(target_id.is_none());
6377 }
6378 other => panic!("unexpected {other:?}"),
6379 }
6380 let encoded = serde_json::to_string(&text).unwrap();
6381 let decoded: Action = serde_json::from_str(&encoded).unwrap();
6382 assert_eq!(decoded, text);
6383 }
6384
6385 #[test]
6386 fn browser_type_action_round_trips_with_text_and_optional_selector() {
6387 let action: Action =
6388 serde_json::from_str(r##"{"action":"browser_type","text":"hi","selector":"#name"}"##)
6389 .unwrap();
6390 match &action {
6391 Action::BrowserType {
6392 text,
6393 selector,
6394 target_id,
6395 } => {
6396 assert_eq!(text, "hi");
6397 assert_eq!(selector.as_deref(), Some("#name"));
6398 assert!(target_id.is_none());
6399 }
6400 other => panic!("unexpected {other:?}"),
6401 }
6402 let encoded = serde_json::to_string(&action).unwrap();
6403 let decoded: Action = serde_json::from_str(&encoded).unwrap();
6404 assert_eq!(decoded, action);
6405 }
6406
6407 #[test]
6408 fn browser_text_label_is_browser_text_and_class_is_observe() {
6409 let action = Action::BrowserText {
6411 selector: "h1".into(),
6412 target_id: None,
6413 };
6414 assert_eq!(action_label(&action), "browser_text");
6415 assert!(matches!(policy_class(&action), PolicyClass::Observe));
6416 }
6417
6418 #[test]
6419 fn browser_type_label_is_browser_type_and_class_is_mutate() {
6420 let action = Action::BrowserType {
6423 text: "abc".into(),
6424 selector: Some("#name".into()),
6425 target_id: None,
6426 };
6427 assert_eq!(action_label(&action), "browser_type");
6428 assert!(matches!(policy_class(&action), PolicyClass::Mutate));
6429 assert_eq!(clipboard_chars(&action), Some(3));
6430 }
6431
6432 #[test]
6433 fn browser_text_does_not_record_chars_in_audit() {
6434 let action = Action::BrowserText {
6436 selector: "h1".into(),
6437 target_id: None,
6438 };
6439 assert_eq!(clipboard_chars(&action), None);
6440 }
6441
6442 #[test]
6443 #[cfg(feature = "cdp")]
6444 fn browser_navigate_blocks_url_before_connecting() {
6445 let backend = StubBackend::default();
6449 let context = CallContext {
6450 cdp: true,
6451 cdp_url: Some("ws://127.0.0.1:1".into()),
6452 browser_blocked_urls: vec!["https://blocked.example.com".into()],
6453 ..CallContext::default()
6454 };
6455 let response = dispatch_with_context(
6456 &backend,
6457 Action::BrowserNavigate {
6458 url: "https://blocked.example.com".into(),
6459 target_id: None,
6460 },
6461 &context,
6462 );
6463 assert!(!response.ok);
6464 let value = response.to_value();
6465 assert_eq!(value["error"]["code"], "blocked_url");
6466 }
6467
6468 #[test]
6469 #[cfg(feature = "cdp")]
6470 fn browser_navigate_unblocked_url_still_attempts_connect() {
6471 let backend = StubBackend::default();
6474 let context = CallContext {
6475 cdp: true,
6476 cdp_url: Some("ws://127.0.0.1:1".into()),
6477 ..CallContext::default()
6478 };
6479 let response = dispatch_with_context(
6480 &backend,
6481 Action::BrowserNavigate {
6482 url: "https://example.com".into(),
6483 target_id: None,
6484 },
6485 &context,
6486 );
6487 assert!(!response.ok);
6488 let value = response.to_value();
6489 assert_eq!(value["error"]["code"], "cdp_unavailable");
6490 }
6491
6492 fn filtered(spec: &str) -> CallContext {
6497 CallContext {
6498 display: Some(spec.into()),
6499 ..CallContext::default()
6500 }
6501 }
6502
6503 fn code_of(response: &Response) -> String {
6504 response.to_value()["error"]["code"]
6505 .as_str()
6506 .unwrap_or("")
6507 .to_string()
6508 }
6509
6510 #[test]
6511 fn display_filter_rejects_a_point_outside_the_chosen_monitor() {
6512 let backend = StubBackend::with_two_displays();
6514 let context = filtered("2");
6515
6516 let response = dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &context);
6518 assert!(response.ok, "{response:?}");
6519
6520 let before = backend.moves.load(Ordering::SeqCst);
6523 for (x, y) in [(0, 0), (799, 599), (1440, 580), (800, 99), (900, 90)] {
6524 let response = dispatch_with_context(&backend, Action::Move { x, y }, &context);
6525 assert_eq!(code_of(&response), "invalid_arguments", "{x},{y}");
6526 assert_eq!(backend.moves.load(Ordering::SeqCst), before);
6527 }
6528 let message = dispatch_with_context(&backend, Action::Move { x: 0, y: 0 }, &context)
6529 .to_value()["error"]["message"]
6530 .as_str()
6531 .expect("message")
6532 .to_string();
6533 assert!(message.contains("640x480+800+100"), "{message}");
6534 }
6535
6536 #[test]
6537 fn display_filter_covers_click_move_drag_scroll_and_mouse_halves() {
6538 let backend = StubBackend::with_two_displays();
6539 let context = filtered("2");
6540 let inside = Action::Click {
6541 x: Some(900),
6542 y: Some(300),
6543 button: MouseButton::Left,
6544 double: false,
6545 count: 1,
6546 };
6547 let outside = Action::Click {
6548 x: Some(10),
6549 y: Some(10),
6550 button: MouseButton::Left,
6551 double: false,
6552 count: 1,
6553 };
6554
6555 assert!(dispatch_with_context(&backend, inside.clone(), &context).ok);
6556 assert_eq!(
6557 code_of(&dispatch_with_context(&backend, outside.clone(), &context)),
6558 "invalid_arguments"
6559 );
6560 assert_eq!(
6561 code_of(&dispatch_with_context(
6562 &backend,
6563 Action::DoubleClick {
6564 x: Some(10),
6565 y: Some(10),
6566 button: MouseButton::Left
6567 },
6568 &context
6569 )),
6570 "invalid_arguments"
6571 );
6572 assert_eq!(
6573 code_of(&dispatch_with_context(
6574 &backend,
6575 Action::MouseDown {
6576 x: 10,
6577 y: 10,
6578 button: MouseButton::Left
6579 },
6580 &context
6581 )),
6582 "invalid_arguments"
6583 );
6584 assert_eq!(
6585 code_of(&dispatch_with_context(
6586 &backend,
6587 Action::Scroll {
6588 x: Some(10),
6589 y: Some(10),
6590 dx: Some(0),
6591 dy: Some(1),
6592 direction: None,
6593 amount: None,
6594 },
6595 &context
6596 )),
6597 "invalid_arguments"
6598 );
6599 let path = vec![Point { x: 900, y: 300 }, Point { x: 10, y: 10 }];
6600 assert_eq!(
6601 code_of(&dispatch_with_context(
6602 &backend,
6603 Action::Drag {
6604 path,
6605 button: MouseButton::Left
6606 },
6607 &context
6608 )),
6609 "invalid_arguments"
6610 );
6611 }
6612
6613 #[test]
6614 fn a_coordinate_action_without_a_point_passes_the_filter() {
6615 let backend = StubBackend::with_two_displays();
6619 let response = dispatch_with_context(
6620 &backend,
6621 Action::Scroll {
6622 x: None,
6623 y: None,
6624 dx: Some(0),
6625 dy: Some(2),
6626 direction: None,
6627 amount: None,
6628 },
6629 &filtered("2"),
6630 );
6631 assert!(response.ok, "{response:?}");
6632 assert_eq!(backend.scrolls.lock().expect("scrolls").len(), 1);
6633 }
6634
6635 #[test]
6636 fn an_unknown_display_id_is_invalid_arguments() {
6637 let backend = StubBackend::with_two_displays();
6638 let response =
6639 dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &filtered("9"));
6640 assert_eq!(code_of(&response), "invalid_arguments");
6641
6642 let response = dispatch_with_context(&backend, Action::Displays, &filtered("99999"));
6645 assert!(response.ok, "{response:?}");
6646 let response = dispatch_with_context(&backend, Action::Isolation, &filtered("99999"));
6647 assert!(response.ok, "{response:?}");
6648 assert_eq!(backend.list_displays().unwrap().len(), 2);
6649
6650 let response = dispatch_with_context(&backend, Action::Displays, &filtered("nope"));
6652 assert_eq!(code_of(&response), "invalid_arguments");
6653 let response = dispatch_with_context(&backend, Action::Displays, &filtered("virtual"));
6657 assert!(response.ok, "{response:?}");
6658 }
6659
6660 #[test]
6661 fn virtual_is_unsupported_over_the_json_path() {
6662 let backend = StubBackend::default();
6665 let response = dispatch_with_context(
6666 &backend,
6667 Action::Move { x: 10, y: 10 },
6668 &filtered("virtual"),
6669 );
6670 assert_eq!(code_of(&response), "unsupported");
6671
6672 let response = dispatch_with_context(
6673 &backend,
6674 Action::Screenshot {
6675 output: None,
6676 region: None,
6677 format: None,
6678 max_width: None,
6679 canvas: None,
6680 base64: false,
6681 marks: false,
6682 },
6683 &filtered("virtual"),
6684 );
6685 assert_eq!(code_of(&response), "unsupported");
6686 }
6687
6688 #[test]
6689 fn the_display_filter_never_leaks_into_displays_or_isolation() {
6690 let backend = StubBackend::with_two_displays();
6691 let context = filtered("9");
6692 assert!(dispatch_with_context(&backend, Action::Displays, &context).ok);
6693 assert!(dispatch_with_context(&backend, Action::Isolation, &context).ok);
6694 }
6695
6696 #[test]
6697 fn a_screenshot_region_must_fit_inside_the_filtered_monitor() {
6698 let _cache = crate::cache::CacheEnv::acquire();
6699 let backend = StubBackend::with_two_displays();
6700 let context = filtered("2");
6701 let inside = Action::Screenshot {
6702 output: None,
6703 region: Some(Region::new(800, 100, 640, 480).unwrap()),
6704 format: None,
6705 max_width: None,
6706 canvas: None,
6707 base64: false,
6708 marks: false,
6709 };
6710 let response = dispatch_with_context(&backend, inside, &context);
6711 assert!(response.ok, "{response:?}");
6712
6713 let map = response.to_value();
6715 assert_eq!(map["origin"]["x"], 800);
6716 assert_eq!(map["origin"]["y"], 100);
6717 let _ = fs::remove_file(
6718 map["path"]
6719 .as_str()
6720 .expect("path")
6721 .parse::<std::path::PathBuf>()
6722 .expect("path"),
6723 );
6724
6725 let straddling = Action::Screenshot {
6726 output: None,
6727 region: Some(Region::new(700, 100, 200, 200).unwrap()),
6728 format: None,
6729 max_width: None,
6730 canvas: None,
6731 base64: false,
6732 marks: false,
6733 };
6734 assert_eq!(
6735 code_of(&dispatch_with_context(&backend, straddling, &context)),
6736 "invalid_arguments"
6737 );
6738 }
6739
6740 #[test]
6741 fn isolation_reports_the_matrix_and_the_infeasible_keys() {
6742 let response = dispatch_with(&StubBackend::default(), Action::Isolation);
6743 assert!(response.ok, "{response:?}");
6744 let value = response.to_value();
6745 assert_eq!(value["platform"], crate::isolation::platform());
6746 assert_eq!(value["display_filter"], true);
6747 assert_eq!(value["xvfb"], "linux only");
6748 assert_eq!(value["vm"], "documented, not shipped");
6749 assert_eq!(
6750 value["xvfb_recipe"]["server_command"],
6751 "Xvfb :99 -screen 0 1440x900x24"
6752 );
6753 for key in [
6754 "macos_vm_per_agent",
6755 "share_host_app_into_guest",
6756 "hide_virtual_display_from_user",
6757 "windows_virtual_display",
6758 "nested_wayland_input_without_portal",
6759 ] {
6760 assert!(
6761 value["infeasible"][key]
6762 .as_str()
6763 .is_some_and(|s| !s.is_empty()),
6764 "infeasible.{key} missing"
6765 );
6766 }
6767 }
6768
6769 #[test]
6770 fn isolation_is_read_only_and_ignores_the_state_flags() {
6771 assert_eq!(action_label(&Action::Isolation), "isolation");
6772 assert!(matches!(
6773 post_state_kind(&Action::Isolation),
6774 PostKind::ReadOnly
6775 ));
6776 let backend = StubBackend::default();
6777 let response = dispatch_with_context(
6778 &backend,
6779 Action::Isolation,
6780 &CallContext {
6781 return_state: true,
6782 include_screenshot: true,
6783 ..CallContext::default()
6784 },
6785 );
6786 assert!(response.ok);
6787 assert!(response.to_value().get("state").is_none());
6788 }
6789
6790 #[test]
6791 fn the_isolation_action_parses_from_json() {
6792 let request = parse_request(json!({"id": 7, "action": "isolation"})).unwrap();
6793 assert_eq!(request.id, Some(json!(7)));
6794 assert!(matches!(request.action, Action::Isolation));
6795 let request = parse_request(json!({"action": "isolation", "display": "1"})).unwrap();
6797 assert_eq!(request.context.display.as_deref(), Some("1"));
6798 }
6799
6800 fn scroll_action(dx: Option<i32>, dy: Option<i32>) -> Action {
6801 Action::Scroll {
6802 x: None,
6803 y: None,
6804 dx,
6805 dy,
6806 direction: None,
6807 amount: None,
6808 }
6809 }
6810
6811 fn near(value: f64, expected: f64) -> bool {
6813 (value - expected).abs() < 1e-9
6814 }
6815
6816 fn tier_context(tier: &str) -> CallContext {
6817 CallContext {
6818 tier: Some(tier.to_string()),
6819 ..CallContext::default()
6820 }
6821 }
6822
6823 #[test]
6824 fn a_scroll_reports_the_tier_that_actually_moved_the_scroll_bar() {
6825 let backend = StubBackend::default();
6826 let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6827 assert_eq!(value["ok"], json!(true));
6828 assert_eq!(value["via"], json!("event"));
6829 assert_eq!(value["scrolled"], json!(true));
6830 assert_eq!(value["before"], json!(0.0));
6831 assert!(near(value["after"].as_f64().unwrap_or_default(), 0.15));
6832 assert_eq!(value["tier"], json!("event"));
6833 assert!(value.get("warnings").is_none());
6835 }
6836
6837 #[test]
6838 fn a_dropped_wheel_event_falls_back_to_the_ax_scroll_bar_and_says_so() {
6839 let backend = StubBackend {
6840 scroll_event_delivered: false,
6841 ..StubBackend::default()
6842 };
6843 let value = dispatch_with(&backend, scroll_action(None, Some(2))).to_value();
6844 assert_eq!(value["ok"], json!(true));
6845 assert_eq!(value["via"], json!("ax_scrollbar"));
6846 assert_eq!(value["scrolled"], json!(true));
6847 assert_eq!(value["before"], json!(0.0));
6848 assert!(near(value["after"].as_f64().unwrap_or_default(), 0.1));
6849 assert_eq!(
6850 value["warnings"][0],
6851 json!("wheel event not delivered to background app; used AX scroll bar")
6852 );
6853 }
6854
6855 #[test]
6856 fn a_bar_already_at_its_limit_is_ok_and_never_claims_a_scroll() {
6857 let backend = StubBackend {
6858 scroll_event_delivered: false,
6859 scroll_value: std::sync::Mutex::new(1.0),
6860 ..StubBackend::default()
6861 };
6862 let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6863 assert_eq!(value["ok"], json!(true));
6864 assert_eq!(value["scrolled"], json!(false));
6865 assert_eq!(value["before"], json!(1.0));
6866 assert_eq!(value["after"], json!(1.0));
6867 let warning = value["warnings"][0].as_str().unwrap_or_default();
6868 assert!(warning.contains("bottom"), "{warning}");
6869 }
6870
6871 #[test]
6872 fn a_scroll_that_cannot_be_verified_is_never_reported_as_a_scroll() {
6873 let backend = StubBackend {
6874 scroll_area_missing: true,
6875 ..StubBackend::default()
6876 };
6877 let response = dispatch_with_context(
6878 &backend,
6879 scroll_action(None, Some(3)),
6880 &tier_context("event"),
6881 );
6882 assert!(!response.ok);
6883 assert_eq!(code_of(&response), "execution_failed");
6884 let message = response.to_value()["error"]["message"]
6885 .as_str()
6886 .unwrap_or_default()
6887 .to_string();
6888 assert!(message.contains("cannot be verified"), "{message}");
6889 assert!(response.to_value().get("scrolled").is_none());
6890 }
6891
6892 #[test]
6893 fn an_ax_refusal_is_a_failure_and_not_a_scroll() {
6894 let backend = StubBackend {
6895 scroll_event_delivered: false,
6896 scroll_ax_refused: true,
6897 ..StubBackend::default()
6898 };
6899 let response = dispatch_with_context(
6900 &backend,
6901 scroll_action(None, Some(3)),
6902 &tier_context("event"),
6903 );
6904 assert!(!response.ok);
6905 assert_eq!(code_of(&response), "execution_failed");
6906 assert!(
6907 response.to_value()["error"]["message"]
6908 .as_str()
6909 .unwrap_or_default()
6910 .contains("refused"),
6911 "{:?}",
6912 response.to_value()
6913 );
6914 }
6915
6916 #[test]
6917 fn a_zero_delta_scroll_reports_no_scroll_and_does_not_panic() {
6918 for tier in ["event", "ax"] {
6921 let backend = StubBackend::default();
6922 let value = dispatch_with_context(
6923 &backend,
6924 scroll_action(Some(0), Some(0)),
6925 &tier_context(tier),
6926 )
6927 .to_value();
6928 assert_eq!(value["ok"], json!(true), "tier {tier}");
6929 assert_eq!(value["scrolled"], json!(false), "tier {tier}");
6930 assert_eq!(value["before"], json!(null), "tier {tier}");
6931 assert_eq!(value["after"], json!(null), "tier {tier}");
6932 assert!(value.get("warnings").is_none(), "tier {tier}");
6933 }
6934 }
6935
6936 #[test]
6937 fn an_explicit_ax_scroll_runs_the_scroll_bar_path_without_posting_an_event() {
6938 let backend = StubBackend::default();
6939 let response =
6940 dispatch_with_context(&backend, scroll_action(None, Some(3)), &tier_context("ax"));
6941 let value = response.to_value();
6942 assert_eq!(value["ok"], json!(true));
6943 assert_eq!(value["tier"], json!("ax"));
6944 assert_eq!(value["via"], json!("ax_scrollbar"));
6945 assert_eq!(value["scrolled"], json!(true));
6946 assert!(
6948 value.get("warnings").is_none(),
6949 "unexpected warnings: {:?}",
6950 value["warnings"]
6951 );
6952 assert_eq!(
6953 *backend.tiers_run.lock().unwrap(),
6954 vec!["ax".to_string()],
6955 "tier ax must not fall back to activate"
6956 );
6957 }
6958
6959 #[test]
6960 fn require_background_never_reaches_the_activate_scroll_tier() {
6961 let backend = StubBackend {
6964 scroll_event_delivered: false,
6965 scroll_area_missing: true,
6966 ..StubBackend::default()
6967 };
6968 let context = CallContext {
6969 tier: Some("event".into()),
6970 require_background: true,
6971 ..CallContext::default()
6972 };
6973 let response = dispatch_with_context(&backend, scroll_action(None, Some(3)), &context);
6974 assert!(!response.ok);
6975 assert_eq!(code_of(&response), "execution_failed");
6976 let tiers = backend.tiers_run.lock().unwrap().clone();
6977 assert!(
6978 !tiers.iter().any(|tier| tier == "activate"),
6979 "activate must never run: {tiers:?}"
6980 );
6981 }
6982
6983 #[test]
6984 fn a_horizontal_scroll_reports_the_horizontal_bar() {
6985 let backend = StubBackend {
6986 scroll_event_delivered: false,
6987 scroll_value: std::sync::Mutex::new(0.5),
6988 ..StubBackend::default()
6989 };
6990 let value = dispatch_with(&backend, scroll_action(Some(-2), None)).to_value();
6991 assert_eq!(value["via"], json!("ax_scrollbar"));
6992 assert_eq!(value["before"], json!(0.5));
6993 assert!(near(value["after"].as_f64().unwrap_or_default(), 0.4));
6994 assert_eq!(value["dx"], json!(-2));
6995 }
6996}
6997
6998#[allow(clippy::items_after_test_module)]
7006fn browser_action_options(
7007 context: &CallContext,
7008 subject: &policy::Subject,
7009 target_id: Option<String>,
7010) -> crate::browser::ActionOptions {
7011 let mut policy_blocked_urls = context.browser_blocked_urls.clone();
7016 if policy_blocked_urls.is_empty() {
7017 if let Ok(loaded) = policy::load_policy(context.policy.as_deref().map(Path::new)) {
7018 policy_blocked_urls = loaded.blocked_urls().to_vec();
7019 }
7020 }
7021 crate::browser::ActionOptions {
7022 debug_url: context.cdp_url.clone(),
7023 app: context.app.clone(),
7024 app_pid: subject.pid,
7025 target_id,
7026 url: None,
7027 selector: None,
7028 text: None,
7029 script: None,
7030 command_timeout: Duration::from_secs(30),
7031 policy_blocked_urls,
7032 policy_subject: Some(subject.clone()),
7033 }
7034}
7035
7036fn browser_require_cdp(context: &CallContext) -> Result<(), ComputerError> {
7037 if !context.cdp {
7038 return Err(ComputerError::Unsupported(
7039 "browser commands require --cdp (or `cdp: true` in the JSON request)".into(),
7040 ));
7041 }
7042 Ok(())
7043}
7044
7045fn browser_tabs(context: &CallContext, subject: &policy::Subject) -> Result<Value, ComputerError> {
7046 browser_require_cdp(context)?;
7047 let opts = browser_action_options(context, subject, None);
7048 let tabs = crate::browser::cdp::list_tabs(&opts)?;
7049 serde_json::to_value(&tabs)
7050 .map_err(|error| ComputerError::ExecutionFailed(format!("encode tabs: {error}")))
7051}
7052
7053fn browser_activate(
7054 context: &CallContext,
7055 subject: &policy::Subject,
7056 target_id: Option<String>,
7057) -> Result<Value, ComputerError> {
7058 browser_require_cdp(context)?;
7059 let target = target_id.ok_or_else(|| {
7060 ComputerError::InvalidArguments("browser activate requires --target".into())
7061 })?;
7062 let opts = browser_action_options(context, subject, Some(target));
7063 let active = crate::browser::cdp::activate_tab(&opts)?;
7064 Ok(json!({ "url": active }))
7065}
7066
7067fn browser_navigate(
7068 context: &CallContext,
7069 subject: &policy::Subject,
7070 url: String,
7071 target_id: Option<String>,
7072) -> Result<Value, ComputerError> {
7073 browser_require_cdp(context)?;
7074 let opts = browser_action_options(context, subject, target_id);
7075 let landed = crate::browser::cdp::navigate(&crate::browser::ActionOptions {
7076 url: Some(url.clone()),
7077 ..opts
7078 })?;
7079 Ok(json!({ "url": landed }))
7080}
7081
7082fn browser_click(
7083 context: &CallContext,
7084 subject: &policy::Subject,
7085 selector: String,
7086 target_id: Option<String>,
7087) -> Result<Value, ComputerError> {
7088 browser_require_cdp(context)?;
7089 let opts = browser_action_options(context, subject, target_id);
7090 crate::browser::cdp::click(&crate::browser::ActionOptions {
7091 selector: Some(selector.clone()),
7092 ..opts
7093 })?;
7094 Ok(json!({ "selector": selector }))
7095}
7096
7097fn browser_text(
7098 context: &CallContext,
7099 subject: &policy::Subject,
7100 selector: String,
7101 target_id: Option<String>,
7102) -> Result<Value, ComputerError> {
7103 browser_require_cdp(context)?;
7104 let opts = browser_action_options(context, subject, target_id);
7105 let text = crate::browser::cdp::read_inner_text(&crate::browser::ActionOptions {
7106 selector: Some(selector.clone()),
7107 ..opts
7108 })?;
7109 Ok(json!({ "selector": selector, "text": text }))
7110}
7111
7112fn browser_type(
7113 context: &CallContext,
7114 subject: &policy::Subject,
7115 text: String,
7116 selector: Option<String>,
7117 target_id: Option<String>,
7118) -> Result<Value, ComputerError> {
7119 browser_require_cdp(context)?;
7120 let opts = browser_action_options(context, subject, target_id);
7121 crate::browser::cdp::type_text(&crate::browser::ActionOptions {
7122 selector,
7123 text: Some(text.clone()),
7124 ..opts
7125 })?;
7126 Ok(json!({ "chars": text.chars().count() as u64 }))
7129}
7130
7131fn browser_eval(
7132 context: &CallContext,
7133 subject: &policy::Subject,
7134 script: String,
7135 target_id: Option<String>,
7136) -> Result<Value, ComputerError> {
7137 browser_require_cdp(context)?;
7138 let opts = browser_action_options(context, subject, target_id);
7139 let value = crate::browser::cdp::eval(&crate::browser::ActionOptions {
7140 script: Some(script.clone()),
7141 ..opts
7142 })?;
7143 let length = script.chars().count() as u64;
7147 use std::hash::{Hash, Hasher};
7148 let mut hasher = std::collections::hash_map::DefaultHasher::new();
7149 script.hash(&mut hasher);
7150 let hash = hasher.finish();
7151 Ok(json!({
7152 "result": value,
7153 "script_chars": length,
7154 "script_hash": format!("{hash:016x}"),
7155 }))
7156}