1use std::fs;
4use std::panic::{catch_unwind, AssertUnwindSafe};
5use std::path::{Path, PathBuf};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::thread;
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use serde::de::{self, Deserializer};
11use serde::ser::{Serialize, Serializer};
12use serde::{Deserialize, Serialize as SerializeDerive};
13use serde_json::{json, Map, Value};
14
15use crate::attempt::{self, AttemptFlags};
16use crate::backend::clipboard::ClipboardContents;
17use crate::backend::create_desktop_backend;
18use crate::backend::{validate_key, DesktopBackend, WindowTarget};
19use crate::cache;
20use crate::element::{self, ElementError, ElementOp, ElementOutcome, ElementRequest};
21use crate::error::{ComputerError, ErrorBody};
22use crate::image_util::{self, encode_base64};
23use crate::input_session::{self, InputSession};
24use crate::keys::{parse_key_combo, parse_key_sequence};
25use crate::policy;
26use crate::state::collect_state_for;
27use crate::target::{
28 filter_windows_for_pid, parse_app_selector, parse_window_selector, resolve_app, resolve_window,
29 WindowSelector,
30};
31use crate::tier::{self, InputKind, KeyStroke, TierName};
32use crate::tree::TreeQuery;
33use crate::types::{
34 parse_path, region_inside, Frame, ImageFormat, Modifier, MouseButton, Point, Region,
35 ScrollDirection,
36};
37use crate::vision::{OcrBlock, OcrResult};
38
39const MAX_WAIT_MS: u128 = 24 * 60 * 60 * 1000;
40
41pub const MENU_DEFAULT_DEPTH: u32 = 4;
44
45pub const MENU_MAX_DEPTH: u32 = 8;
48
49pub const SHORTCUTS_TIMEOUT_SECS: u64 = 60;
51
52pub const APPLE_EVENT_TIMEOUT_SECS: u64 = 30;
54
55#[derive(Debug, Clone, PartialEq, SerializeDerive, Deserialize)]
61#[serde(tag = "action", rename_all = "snake_case")]
62pub enum Action {
63 Screenshot {
65 #[serde(default, skip_serializing_if = "Option::is_none")]
67 output: Option<String>,
68 #[serde(default, skip_serializing_if = "Option::is_none")]
70 region: Option<Region>,
71 #[serde(default, skip_serializing_if = "Option::is_none")]
73 format: Option<ImageFormat>,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
78 max_width: Option<u32>,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
81 canvas: Option<String>,
82 #[serde(default, skip_serializing_if = "is_false")]
84 base64: bool,
85 #[serde(default, skip_serializing_if = "is_false")]
89 marks: bool,
90 },
91 Click {
93 #[serde(default, skip_serializing_if = "Option::is_none")]
95 x: Option<i32>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
98 y: Option<i32>,
99 #[serde(default)]
101 button: MouseButton,
102 #[serde(default, skip_serializing_if = "is_false")]
104 double: bool,
105 #[serde(default = "default_press_count", skip_serializing_if = "is_one")]
107 count: u8,
108 },
109 #[serde(rename = "triple_click", alias = "triple-click")]
111 TripleClick {
112 #[serde(default, skip_serializing_if = "Option::is_none")]
114 x: Option<i32>,
115 #[serde(default, skip_serializing_if = "Option::is_none")]
117 y: Option<i32>,
118 #[serde(default)]
120 button: MouseButton,
121 },
122 #[serde(rename = "double_click")]
124 DoubleClick {
125 #[serde(default, skip_serializing_if = "Option::is_none")]
127 x: Option<i32>,
128 #[serde(default, skip_serializing_if = "Option::is_none")]
130 y: Option<i32>,
131 #[serde(default)]
133 button: MouseButton,
134 },
135 Press {
137 #[serde(default = "default_press_count")]
139 count: u8,
140 #[serde(default)]
142 button: MouseButton,
143 },
144 Drag {
146 #[serde(alias = "points", deserialize_with = "deserialize_path")]
148 path: Vec<Point>,
149 #[serde(default)]
151 button: MouseButton,
152 },
153 #[serde(alias = "mouse-down")]
155 MouseDown {
156 x: i32,
158 y: i32,
160 #[serde(default)]
162 button: MouseButton,
163 },
164 #[serde(alias = "mouse-up")]
166 MouseUp {
167 x: i32,
169 y: i32,
171 #[serde(default)]
173 button: MouseButton,
174 },
175 #[serde(alias = "hold-key")]
177 HoldKey {
178 keys: String,
180 ms: u64,
182 },
183 ClipboardGet,
185 #[serde(alias = "clipboard-set")]
187 ClipboardSet {
188 #[serde(default, skip_serializing_if = "Option::is_none")]
190 text: Option<String>,
191 #[serde(default, skip_serializing_if = "Option::is_none")]
193 image: Option<String>,
194 },
195 Zoom {
197 #[serde(default, skip_serializing_if = "Option::is_none")]
199 output: Option<String>,
200 region: Region,
202 #[serde(default, skip_serializing_if = "Option::is_none")]
204 format: Option<ImageFormat>,
205 #[serde(default, skip_serializing_if = "Option::is_none")]
207 max_width: Option<u32>,
208 #[serde(default, skip_serializing_if = "Option::is_none")]
210 canvas: Option<String>,
211 #[serde(default, skip_serializing_if = "is_false")]
213 base64: bool,
214 },
215 Focus,
217 #[serde(alias = "set-value")]
219 SetValue {
220 value: String,
222 },
223 #[serde(alias = "select-text")]
225 SelectText {
226 start: i64,
228 end: i64,
230 },
231 #[serde(alias = "perform-action")]
233 PerformAction {
234 name: String,
236 },
237 #[serde(
239 alias = "scroll-to",
240 alias = "scroll_to_visible",
241 alias = "scroll-to-visible"
242 )]
243 ScrollTo,
244 #[serde(alias = "move_mouse")]
246 Move {
247 x: i32,
249 y: i32,
251 },
252 Type {
254 text: String,
256 },
257 Key {
259 keys: String,
261 },
262 Scroll {
264 #[serde(default, skip_serializing_if = "Option::is_none")]
266 x: Option<i32>,
267 #[serde(default, skip_serializing_if = "Option::is_none")]
269 y: Option<i32>,
270 #[serde(default, alias = "scroll_x", skip_serializing_if = "Option::is_none")]
272 dx: Option<i32>,
273 #[serde(default, alias = "scroll_y", skip_serializing_if = "Option::is_none")]
275 dy: Option<i32>,
276 #[serde(default, skip_serializing_if = "Option::is_none")]
278 direction: Option<ScrollDirection>,
279 #[serde(default, skip_serializing_if = "Option::is_none")]
281 amount: Option<i32>,
282 },
283 #[serde(alias = "get-state")]
285 GetState,
286 Wait {
288 #[serde(default, skip_serializing_if = "Option::is_none")]
290 ms: Option<u64>,
291 #[serde(default, skip_serializing_if = "Option::is_none")]
293 seconds: Option<f64>,
294 },
295 Displays,
297 Isolation,
299 ListApps {
301 #[serde(default, rename = "all")]
303 all: bool,
304 },
305 ListWindows,
307 Launch,
309 Activate,
311 Quit {
313 #[serde(default, skip_serializing_if = "is_false")]
315 force: bool,
316 },
317 #[serde(rename = "policy_check")]
319 PolicyCheck,
320 Pause,
322 Resume,
324 Doctor {
330 #[serde(default, alias = "request")]
332 prompt: bool,
333 #[serde(default)]
335 open_settings: bool,
336 #[serde(default)]
340 x11: bool,
341 },
342 Tree,
344 SessionEnd,
346 CursorPosition,
348 Batch {
350 #[serde(default = "default_stop_on_error")]
352 stop_on_error: bool,
353 actions: Vec<Value>,
355 },
356 #[serde(alias = "session-prune")]
358 SessionPrune {
359 #[serde(default)]
361 dry_run: bool,
362 #[serde(default, skip_serializing_if = "Option::is_none")]
364 ttl_secs: Option<u64>,
365 },
366 #[serde(alias = "menu_list", alias = "menu-list")]
371 MenuList {
372 #[serde(default, skip_serializing_if = "Option::is_none")]
374 app: Option<String>,
375 #[serde(default, skip_serializing_if = "Option::is_none")]
377 depth: Option<u32>,
378 },
379 #[serde(alias = "menu_click", alias = "menu-click")]
385 MenuClick {
386 #[serde(default, skip_serializing_if = "Option::is_none")]
388 app: Option<String>,
389 path: String,
391 },
392 #[serde(alias = "apple_event", alias = "apple-event")]
398 AppleEvent {
399 #[serde(default, skip_serializing_if = "Option::is_none")]
401 app: Option<String>,
402 script: String,
404 #[serde(default, skip_serializing_if = "Option::is_none")]
406 language: Option<AppleEventLanguage>,
407 },
408 #[serde(alias = "shortcuts_run", alias = "shortcuts-run")]
413 ShortcutsRun {
414 name: String,
416 },
417 #[serde(alias = "dock_list", alias = "dock-list")]
419 DockList {},
420 #[serde(alias = "dock_click", alias = "dock-click")]
422 DockClick {
423 title: String,
425 },
426 #[serde(alias = "spaces")]
432 Spaces {},
433 #[serde(rename = "browser_tabs", alias = "browser-tabs")]
436 BrowserTabs {},
437 #[serde(rename = "browser_activate", alias = "browser-activate")]
439 BrowserActivate {
440 #[serde(default, skip_serializing_if = "Option::is_none")]
442 target_id: Option<String>,
443 },
444 #[serde(rename = "browser_navigate", alias = "browser-navigate")]
447 BrowserNavigate {
448 url: String,
450 #[serde(default, skip_serializing_if = "Option::is_none")]
452 target_id: Option<String>,
453 },
454 #[serde(rename = "browser_click", alias = "browser-click")]
456 BrowserClick {
457 selector: String,
459 #[serde(default, skip_serializing_if = "Option::is_none")]
461 target_id: Option<String>,
462 },
463 #[serde(rename = "browser_text", alias = "browser-text")]
468 BrowserText {
469 selector: String,
471 #[serde(default, skip_serializing_if = "Option::is_none")]
473 target_id: Option<String>,
474 },
475 #[serde(rename = "browser_type", alias = "browser-type")]
481 BrowserType {
482 text: String,
484 #[serde(default, skip_serializing_if = "Option::is_none")]
486 selector: Option<String>,
487 #[serde(default, skip_serializing_if = "Option::is_none")]
489 target_id: Option<String>,
490 },
491 #[serde(rename = "browser_eval", alias = "browser-eval")]
495 BrowserEval {
496 script: String,
499 #[serde(default, skip_serializing_if = "Option::is_none")]
501 target_id: Option<String>,
502 },
503 #[serde(rename = "ocr")]
506 Ocr {
507 #[serde(default, skip_serializing_if = "is_false")]
509 force: bool,
510 },
511}
512
513fn is_false(value: &bool) -> bool {
514 !*value
515}
516
517fn is_one(value: &u8) -> bool {
518 *value == 1
519}
520
521fn default_press_count() -> u8 {
522 1
523}
524
525#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDerive, Deserialize)]
530#[serde(rename_all = "snake_case")]
531pub enum AppleEventLanguage {
532 #[default]
534 #[serde(alias = "applescript", alias = "osa")]
535 AppleScript,
536 #[serde(alias = "javascript", alias = "js", alias = "JavaScript")]
538 JavaScript,
539}
540
541impl AppleEventLanguage {
542 pub fn as_str(&self) -> &'static str {
544 match self {
545 Self::AppleScript => "AppleScript",
546 Self::JavaScript => "JavaScript",
547 }
548 }
549}
550
551fn deserialize_path<'de, D>(deserializer: D) -> Result<Vec<Point>, D::Error>
552where
553 D: Deserializer<'de>,
554{
555 let value = Value::deserialize(deserializer)?;
556 match value {
557 Value::String(text) => parse_path(&text).map_err(de::Error::custom),
558 Value::Array(_) => serde_json::from_value(value).map_err(de::Error::custom),
559 _ => Err(de::Error::custom(
560 "path must be an array of points or an x,y,x,y string",
561 )),
562 }
563}
564
565fn default_stop_on_error() -> bool {
566 true
567}
568
569#[derive(Debug, Clone, PartialEq, Default)]
571pub struct CallContext {
572 pub app: Option<String>,
574 pub window: Option<String>,
576 pub element: Option<u32>,
578 pub diff: bool,
580 pub full: bool,
582 pub return_state: bool,
584 pub state_diff: bool,
586 pub include_screenshot: bool,
588 pub coord: Option<String>,
590 pub session: Option<String>,
592 pub tier: Option<String>,
594 pub adapter: Option<String>,
596 pub display: Option<String>,
598 pub require_visible: bool,
600 pub policy: Option<String>,
602 pub confirm_cmd: Option<String>,
604 pub allow_foreground: bool,
606 pub require_background: bool,
608 pub no_overlay: bool,
610 pub cdp: bool,
614 pub cdp_url: Option<String>,
617 pub browser_blocked_urls: Vec<String>,
621}
622
623#[derive(Debug, Clone, PartialEq)]
625pub struct Request {
626 pub id: Option<Value>,
628 pub action: Action,
630 pub context: CallContext,
632}
633
634pub fn parse_request(mut value: Value) -> Result<Request, ComputerError> {
636 if !value.is_object() {
637 return Err(ComputerError::InvalidArguments(
638 "request must be a JSON object".into(),
639 ));
640 }
641 let object = value
642 .as_object_mut()
643 .expect("object check above guarantees a map");
644 let id = object.remove("id");
645 let context = take_context(object)?;
646 let action = serde_json::from_value(value)
647 .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
648 Ok(Request {
649 id,
650 action,
651 context,
652 })
653}
654
655fn take_context(object: &mut Map<String, Value>) -> Result<CallContext, ComputerError> {
656 let state_diff = take_bool(object, "state_diff")?;
657 Ok(CallContext {
658 app: take_stringish(object, "app")?,
659 window: take_stringish(object, "window")?,
660 element: take_optional_u32(object, "element")?,
661 diff: take_bool(object, "diff")?,
662 full: take_bool(object, "full")?,
663 return_state: take_bool(object, "return_state")? || state_diff,
664 state_diff,
665 include_screenshot: take_bool(object, "include_screenshot")?,
666 coord: take_stringish(object, "coord")?,
667 session: take_stringish(object, "session")?,
668 tier: take_stringish(object, "tier")?,
669 adapter: take_stringish(object, "adapter")?,
670 display: take_stringish(object, "display")?,
671 require_visible: take_bool(object, "require_visible")?,
672 policy: None,
673 confirm_cmd: None,
674 allow_foreground: take_bool(object, "allow_foreground")?,
675 require_background: take_bool(object, "require_background")?,
676 no_overlay: take_bool(object, "no_overlay")?,
677 cdp: take_bool(object, "cdp")?,
678 cdp_url: take_stringish(object, "cdp_url")?,
679 browser_blocked_urls: take_string_list(object, "browser_blocked_urls")?,
680 })
681}
682
683fn take_stringish(
684 object: &mut Map<String, Value>,
685 key: &str,
686) -> Result<Option<String>, ComputerError> {
687 match object.remove(key) {
688 None | Some(Value::Null) => Ok(None),
689 Some(Value::String(text)) => Ok(Some(text)),
690 Some(Value::Number(number)) => Ok(Some(number.to_string())),
691 Some(_) => Err(ComputerError::InvalidArguments(format!(
692 "{key} must be a string or number"
693 ))),
694 }
695}
696
697fn take_bool(object: &mut Map<String, Value>, key: &str) -> Result<bool, ComputerError> {
698 match object.remove(key) {
699 None | Some(Value::Null) => Ok(false),
700 Some(Value::Bool(value)) => Ok(value),
701 Some(_) => Err(ComputerError::InvalidArguments(format!(
702 "{key} must be a boolean"
703 ))),
704 }
705}
706
707fn take_string_list(
708 object: &mut Map<String, Value>,
709 key: &str,
710) -> Result<Vec<String>, ComputerError> {
711 match object.remove(key) {
712 None | Some(Value::Null) => Ok(Vec::new()),
713 Some(Value::Array(values)) => values
714 .into_iter()
715 .map(|value| match value {
716 Value::String(text) => Ok(text),
717 _ => Err(ComputerError::InvalidArguments(format!(
718 "{key} entries must be strings"
719 ))),
720 })
721 .collect(),
722 Some(_) => Err(ComputerError::InvalidArguments(format!(
723 "{key} must be a list of strings"
724 ))),
725 }
726}
727
728fn take_optional_u32(
729 object: &mut Map<String, Value>,
730 key: &str,
731) -> Result<Option<u32>, ComputerError> {
732 match object.remove(key) {
733 None | Some(Value::Null) => Ok(None),
734 Some(Value::Number(number)) => number
735 .as_u64()
736 .and_then(|value| u32::try_from(value).ok())
737 .map(Some)
738 .ok_or_else(|| ComputerError::InvalidArguments(format!("{key} is out of range"))),
739 Some(Value::String(text)) => text
740 .parse::<u32>()
741 .map(Some)
742 .map_err(|_| ComputerError::InvalidArguments(format!("{key} must be an integer"))),
743 Some(_) => Err(ComputerError::InvalidArguments(format!(
744 "{key} must be an integer"
745 ))),
746 }
747}
748
749pub fn handle_request_line(line: &str) -> Response {
754 handle_request_line_with(line, None)
755}
756
757#[derive(Debug, Clone, Copy, Default)]
759pub struct LineOptions<'a> {
760 pub adapter: Option<&'a str>,
762 pub policy: Option<&'a str>,
764 pub confirm_cmd: Option<&'a str>,
766 pub tier: Option<&'a str>,
768 pub allow_foreground: bool,
770 pub require_background: bool,
772 pub no_overlay: bool,
774 pub cdp: bool,
776 pub cdp_url: Option<&'a str>,
778}
779
780pub fn handle_request_line_with(line: &str, cli_adapter: Option<&str>) -> Response {
785 handle_request_line_opts(
786 line,
787 LineOptions {
788 adapter: cli_adapter,
789 ..LineOptions::default()
790 },
791 )
792}
793
794pub fn handle_request_line_opts(line: &str, options: LineOptions<'_>) -> Response {
796 let value = match serde_json::from_str::<Value>(line) {
797 Ok(value) => value,
798 Err(error) => {
799 return Response::failure("invalid_arguments", format!("invalid JSON: {error}"))
800 }
801 };
802 let id = value.get("id").cloned();
803 let echo = match effective_adapter(&value, options.adapter) {
804 Ok(name) => name,
805 Err(error) => return Response::from_error(error).with_id(id),
806 };
807 match build_request(value, echo.as_deref()) {
808 Ok(mut request) => {
809 if let Some(policy) = options.policy {
810 request.context.policy = Some(policy.to_string());
811 }
812 if let Some(confirm) = options.confirm_cmd {
813 request.context.confirm_cmd = Some(confirm.to_string());
814 }
815 if request.context.tier.is_none() {
816 if let Some(tier) = options.tier {
817 request.context.tier = Some(tier.to_string());
818 }
819 }
820 request.context.allow_foreground |= options.allow_foreground;
821 request.context.require_background |= options.require_background;
822 request.context.no_overlay |= options.no_overlay;
823 request.context.cdp |= options.cdp;
824 if request.context.cdp_url.is_none() {
825 if let Some(url) = options.cdp_url {
826 request.context.cdp_url = Some(url.to_string());
827 }
828 }
829 dispatch_context(request.action, &request.context).with_id(request.id)
830 }
831 Err(error) => {
832 let mut response = Response::from_error(error).with_id(id);
833 stamp_adapter_name(&mut response, echo.as_deref());
834 response
835 }
836 }
837}
838
839fn effective_adapter(
840 value: &Value,
841 cli_adapter: Option<&str>,
842) -> Result<Option<String>, ComputerError> {
843 if !value.is_object() {
844 return Ok(None);
845 }
846 let from_request = match value.get("adapter") {
847 None | Some(Value::Null) => None,
848 Some(Value::String(text)) => Some(text.as_str()),
849 Some(_) => {
850 return Err(ComputerError::InvalidArguments(
851 "adapter must be a string".into(),
852 ))
853 }
854 };
855 let name = from_request.or(cli_adapter);
856 match name {
857 None | Some("native") => Ok(None),
858 Some("openai" | "anthropic") => Ok(name.map(str::to_string)),
859 Some(other) => Err(ComputerError::InvalidArguments(format!(
860 "unknown adapter {other:?}"
861 ))),
862 }
863}
864
865fn build_request(value: Value, adapter: Option<&str>) -> Result<Request, ComputerError> {
866 let Some(name) = adapter else {
867 return parse_request(value);
868 };
869 if !value.is_object() {
870 return Err(ComputerError::InvalidArguments(
871 "request must be a JSON object".into(),
872 ));
873 }
874 let mut value = value;
875 let object = value
876 .as_object_mut()
877 .expect("object check above guarantees a map");
878 let id = object.remove("id");
879 let mut context = take_context(object)?;
880 context.coord = Some("image".into());
881 context.adapter = Some(name.to_string());
882 let action = crate::adapters::translate(name, &value)?;
883 Ok(Request {
884 id,
885 action,
886 context,
887 })
888}
889
890fn stamp_adapter_name(response: &mut Response, adapter: Option<&str>) {
891 if let Some(name @ ("openai" | "anthropic")) = adapter {
892 response.fields.insert("adapter".into(), json!(name));
893 }
894}
895
896pub fn dispatch(action: Action) -> Response {
898 dispatch_context(action, &CallContext::default())
899}
900
901pub fn dispatch_context(action: Action, context: &CallContext) -> Response {
903 let backend = create_desktop_backend();
904 dispatch_with_context(backend.as_ref(), action, context)
905}
906
907pub fn dispatch_with(backend: &dyn DesktopBackend, action: Action) -> Response {
909 dispatch_with_context(backend, action, &CallContext::default())
910}
911
912pub fn dispatch_with_context(
914 backend: &dyn DesktopBackend,
915 action: Action,
916 context: &CallContext,
917) -> Response {
918 let mut response = match catch_unwind(AssertUnwindSafe(|| {
919 dispatch_inner(backend, action, context)
920 })) {
921 Ok(response) => response,
922 Err(_) => Response::failure("execution_failed", "action panicked and was recovered"),
923 };
924 stamp_adapter_name(&mut response, context.adapter.as_deref());
925 if !input_session::session_open() {
928 let _ = backend.release_held();
929 }
930 response
931}
932
933enum PostKind {
934 Mutating,
936 ReadOnly,
938 Screenshot,
940}
941
942fn dispatch_inner(backend: &dyn DesktopBackend, action: Action, context: &CallContext) -> Response {
943 let label = action_label(&action);
944 let chars = clipboard_chars(&action);
945 let action = match remap_coordinates(context, action) {
946 Ok(action) => action,
947 Err(error) => {
948 return finish_action(
949 context,
950 label,
951 &policy::Subject::default(),
952 None,
953 "allow",
954 "default",
955 chars,
956 Response::from_error(error),
957 )
958 }
959 };
960 if let Err(error) = preflight_action(&action) {
961 return finish_action(
962 context,
963 label,
964 &policy::Subject::default(),
965 None,
966 "allow",
967 "default",
968 chars,
969 Response::from_error(error),
970 );
971 }
972 if let Err(error) = gate_display(backend, context, &action) {
973 return finish_action(
974 context,
975 label,
976 &policy::Subject::default(),
977 None,
978 "allow",
979 "default",
980 chars,
981 Response::from_error(error),
982 );
983 }
984 let prepared = prepare_action(backend, &action, context);
985 if let Some(response) = prepared.stop {
986 return finish_action(
987 context,
988 label,
989 &prepared.subject,
990 prepared.warning.as_deref(),
991 prepared.decision,
992 prepared.source,
993 chars,
994 response,
995 );
996 }
997 let kind = post_state_kind(&action);
998 let response = match action {
999 Action::Batch {
1000 actions,
1001 stop_on_error,
1002 } => {
1003 return finish_action(
1004 context,
1005 label,
1006 &prepared.subject,
1007 prepared.warning.as_deref(),
1008 prepared.decision,
1009 prepared.source,
1010 chars,
1011 run_batch(backend, context, actions, stop_on_error),
1012 )
1013 }
1014 Action::Screenshot {
1015 output,
1016 region,
1017 format,
1018 max_width,
1019 canvas,
1020 base64,
1021 marks,
1022 } => result_response(screenshot(
1023 backend,
1024 context,
1025 ScreenshotArgs {
1026 output,
1027 region,
1028 format,
1029 max_width,
1030 canvas,
1031 include_base64: base64,
1032 window_crop: false,
1033 marks,
1034 },
1035 )),
1036 Action::Zoom {
1037 output,
1038 region,
1039 format,
1040 max_width,
1041 canvas,
1042 base64,
1043 } => result_response(screenshot(
1044 backend,
1045 context,
1046 ScreenshotArgs {
1047 output,
1048 region: Some(region),
1049 format,
1050 max_width,
1051 canvas,
1052 include_base64: base64,
1053 window_crop: true,
1054 marks: false,
1055 },
1056 )),
1057 Action::Click {
1058 x,
1059 y,
1060 button,
1061 double,
1062 count,
1063 } => click(backend, context, x, y, button, double, count),
1064 Action::DoubleClick { x, y, button } => click(backend, context, x, y, button, true, 1),
1065 Action::TripleClick { x, y, button } => click(backend, context, x, y, button, false, 3),
1066 Action::Press { count, button } => {
1067 element_command(backend, context, ElementOp::Press { count, button })
1068 }
1069 Action::Drag { path, button } => drag(backend, context, path, button),
1070 Action::MouseDown { x, y, button } => pointer_half(backend, context, x, y, button, true),
1071 Action::MouseUp { x, y, button } => pointer_half(backend, context, x, y, button, false),
1072 Action::HoldKey { keys, ms } => hold_key(backend, context, &keys, ms),
1073 Action::ClipboardGet => result_response(clipboard_get(backend, context)),
1074 Action::ClipboardSet { text, image } => {
1075 result_response(clipboard_set(backend, text, image))
1076 }
1077 Action::Focus => element_command(backend, context, ElementOp::Focus),
1078 Action::SetValue { value } => {
1079 element_command(backend, context, ElementOp::SetValue { value })
1080 }
1081 Action::SelectText { start, end } => {
1082 element_command(backend, context, ElementOp::SelectText { start, end })
1083 }
1084 Action::PerformAction { name } => {
1085 element_command(backend, context, ElementOp::Perform { name })
1086 }
1087 Action::ScrollTo => element_command(backend, context, ElementOp::ScrollTo),
1088 Action::Move { x, y } => move_mouse(backend, context, x, y),
1089 Action::Type { text } => type_command(backend, context, text),
1090 Action::Key { keys } => press_key(backend, context, &keys),
1091 Action::Scroll {
1092 x,
1093 y,
1094 dx,
1095 dy,
1096 direction,
1097 amount,
1098 } => scroll(backend, context, (x, y), (dx, dy), direction, amount),
1099 Action::GetState => result_response(Ok(collect_state_for(backend, &tree_query(context)))),
1100 Action::Tree => result_response(tree_action(backend, context)),
1101 Action::Ocr { force } => result_response(ocr_action(backend, context, force)),
1102 Action::SessionEnd => result_response(session_end(backend, context)),
1103 Action::Wait { ms, seconds } => result_response(wait(ms, seconds)),
1104 Action::Displays => result_response(displays(backend)),
1105 Action::Isolation => result_response(isolation()),
1106 Action::ListApps { all } => result_response(list_apps(backend, all)),
1107 Action::ListWindows => result_response(list_windows(backend, context.app.as_deref())),
1108 Action::Launch => result_response(launch(backend, context.app.as_deref())),
1109 Action::Activate => result_response(activate(
1110 backend,
1111 context.app.as_deref(),
1112 context.window.as_deref(),
1113 )),
1114 Action::Quit { force } => result_response(quit(backend, context.app.as_deref(), force)),
1115 Action::Doctor {
1116 prompt,
1117 open_settings,
1118 x11,
1119 } => result_response(doctor(
1120 backend,
1121 crate::doctor::DoctorOptions {
1122 prompt,
1123 open_settings,
1124 x11,
1125 },
1126 )),
1127 Action::CursorPosition => result_response(cursor_position(backend)),
1128 Action::SessionPrune { dry_run, ttl_secs } => {
1129 result_response(session_prune(dry_run, ttl_secs))
1130 }
1131 Action::MenuList { app, depth } => {
1132 let app = match app.or_else(|| context.app.clone()) {
1133 Some(app) => app,
1134 None => return Response::from_error(invalid_arguments("menu list requires --app")),
1135 };
1136 let depth = depth.unwrap_or(MENU_DEFAULT_DEPTH);
1137 result_response(menu_list(backend, &app, depth))
1138 }
1139 Action::MenuClick { app, path } => {
1140 let app = match app.or_else(|| context.app.clone()) {
1141 Some(app) => app,
1142 None => {
1143 return Response::from_error(invalid_arguments("menu click requires --app"))
1144 }
1145 };
1146 if path.trim().is_empty() {
1147 return Response::from_error(invalid_arguments("menu click requires --path"));
1148 }
1149 result_response(menu_click(backend, &app, &path))
1150 }
1151 Action::AppleEvent {
1152 app,
1153 script,
1154 language,
1155 } => {
1156 let app = match app.or_else(|| context.app.clone()) {
1157 Some(app) => app,
1158 None => {
1159 return Response::from_error(invalid_arguments("apple-event requires --app"))
1160 }
1161 };
1162 if script.is_empty() {
1163 return Response::from_error(invalid_arguments("apple-event requires --script"));
1164 }
1165 let language = language.unwrap_or_default();
1166 result_response(apple_event_action(backend, &app, &script, language))
1167 }
1168 Action::ShortcutsRun { name } => {
1169 if !cfg!(target_os = "macos") {
1170 return Response::from_error(unsupported_error("shortcuts_run"));
1171 }
1172 if name.trim().is_empty() {
1173 return Response::from_error(invalid_arguments(
1174 "shortcuts run requires a workflow name",
1175 ));
1176 }
1177 result_response(shortcuts_run_action(name))
1178 }
1179 Action::DockList {} => result_response(dock_list_action(backend)),
1180 Action::DockClick { title } => {
1181 if title.trim().is_empty() {
1182 return Response::from_error(invalid_arguments("dock click requires --title"));
1183 }
1184 result_response(dock_click_action(backend, &title))
1185 }
1186 Action::Spaces {} => result_response(spaces_action(backend)),
1187 Action::BrowserTabs {} => result_response(browser_tabs(context, &prepared.subject)),
1188 Action::BrowserActivate { target_id } => result_response(browser_activate(
1189 context,
1190 &prepared.subject,
1191 target_id.clone(),
1192 )),
1193 Action::BrowserNavigate { url, target_id } => result_response(browser_navigate(
1194 context,
1195 &prepared.subject,
1196 url.clone(),
1197 target_id.clone(),
1198 )),
1199 Action::BrowserClick {
1200 selector,
1201 target_id,
1202 } => result_response(browser_click(
1203 context,
1204 &prepared.subject,
1205 selector.clone(),
1206 target_id.clone(),
1207 )),
1208 Action::BrowserText {
1209 selector,
1210 target_id,
1211 } => result_response(browser_text(
1212 context,
1213 &prepared.subject,
1214 selector.clone(),
1215 target_id.clone(),
1216 )),
1217 Action::BrowserType {
1218 text,
1219 selector,
1220 target_id,
1221 } => result_response(browser_type(
1222 context,
1223 &prepared.subject,
1224 text.clone(),
1225 selector.clone(),
1226 target_id.clone(),
1227 )),
1228 Action::BrowserEval { script, target_id } => result_response(browser_eval(
1229 context,
1230 &prepared.subject,
1231 script.clone(),
1232 target_id.clone(),
1233 )),
1234 Action::PolicyCheck => {
1235 let (response, subject) = policy_check(backend, context);
1236 let response = attach_post_state(backend, context, kind, response);
1237 return finish_action(
1238 context,
1239 label,
1240 &subject,
1241 prepared.warning.as_deref(),
1242 prepared.decision,
1243 prepared.source,
1244 chars,
1245 response,
1246 );
1247 }
1248 Action::Pause => pause_response(true),
1249 Action::Resume => pause_response(false),
1250 };
1251 let response = attach_post_state(backend, context, kind, response);
1252 finish_action(
1253 context,
1254 label,
1255 &prepared.subject,
1256 prepared.warning.as_deref(),
1257 prepared.decision,
1258 prepared.source,
1259 chars,
1260 response,
1261 )
1262}
1263
1264#[allow(clippy::too_many_arguments)]
1265fn finish_action(
1266 context: &CallContext,
1267 label: &str,
1268 subject: &policy::Subject,
1269 warning: Option<&str>,
1270 decision: &str,
1271 source: &str,
1272 chars: Option<u64>,
1273 mut response: Response,
1274) -> Response {
1275 if let Some(warning) = warning {
1276 push_warning(&mut response, warning);
1277 }
1278 let code = if response.ok {
1279 "ok"
1280 } else {
1281 response
1282 .error
1283 .as_ref()
1284 .map(|error| error.code.as_str())
1285 .unwrap_or("execution_failed")
1286 };
1287 let mut decision = decision.to_string();
1288 let mut source = source.to_string();
1289 if label == "policy_check" {
1290 if let Some(value) = response
1291 .fields
1292 .get("decision")
1293 .and_then(|value| value.as_str())
1294 {
1295 decision = value.to_string();
1296 }
1297 if let Some(value) = response
1298 .fields
1299 .get("source")
1300 .and_then(|value| value.as_str())
1301 {
1302 source = value.to_string();
1303 }
1304 }
1305 let tier = response
1306 .fields
1307 .get("tier")
1308 .and_then(|value| value.as_str())
1309 .map(str::to_string);
1310 let session = context
1311 .session
1312 .clone()
1313 .unwrap_or_else(|| std::process::id().to_string());
1314 if let Err(error) = crate::audit::record(&crate::audit::AuditRecord {
1315 session: &session,
1316 action: label,
1317 bundle_id: subject.bundle_id.as_deref(),
1318 pid: subject.pid,
1319 window_id: subject.window_id,
1320 element: context.element,
1321 code,
1322 decision: &decision,
1323 source: &source,
1324 tier: tier.as_deref(),
1325 chars,
1326 }) {
1327 push_warning(
1328 &mut response,
1329 &format!("audit log was not written: {error}"),
1330 );
1331 }
1332 response
1333}
1334
1335fn push_warning(response: &mut Response, warning: &str) {
1336 let warnings = response
1337 .fields
1338 .entry("warnings")
1339 .or_insert_with(|| json!([]));
1340 if let Some(items) = warnings.as_array_mut() {
1341 items.push(json!(warning));
1342 }
1343}
1344
1345fn action_label(action: &Action) -> &'static str {
1346 match action {
1347 Action::Screenshot { .. } => "screenshot",
1348 Action::Zoom { .. } => "zoom",
1349 Action::Click { .. } => "click",
1350 Action::DoubleClick { .. } => "double_click",
1351 Action::TripleClick { .. } => "triple_click",
1352 Action::Press { .. } => "press",
1353 Action::Drag { .. } => "drag",
1354 Action::MouseDown { .. } => "mouse_down",
1355 Action::MouseUp { .. } => "mouse_up",
1356 Action::HoldKey { .. } => "hold_key",
1357 Action::ClipboardGet => "clipboard_get",
1358 Action::ClipboardSet { .. } => "clipboard_set",
1359 Action::Focus => "focus",
1360 Action::SetValue { .. } => "set_value",
1361 Action::SelectText { .. } => "select_text",
1362 Action::PerformAction { .. } => "perform_action",
1363 Action::ScrollTo => "scroll_to",
1364 Action::Move { .. } => "move",
1365 Action::Type { .. } => "type",
1366 Action::Key { .. } => "key",
1367 Action::Scroll { .. } => "scroll",
1368 Action::GetState => "get_state",
1369 Action::Wait { .. } => "wait",
1370 Action::Displays => "displays",
1371 Action::Isolation => "isolation",
1372 Action::ListApps { .. } => "list_apps",
1373 Action::ListWindows => "list_windows",
1374 Action::Launch => "launch",
1375 Action::Activate => "activate",
1376 Action::Quit { .. } => "quit",
1377 Action::Doctor { .. } => "doctor",
1378 Action::Tree => "tree",
1379 Action::SessionEnd => "session_end",
1380 Action::CursorPosition => "cursor_position",
1381 Action::Batch { .. } => "batch",
1382 Action::SessionPrune { .. } => "session_prune",
1383 Action::PolicyCheck => "policy_check",
1384 Action::Pause => "pause",
1385 Action::Resume => "resume",
1386 Action::MenuList { .. } => "menu_list",
1387 Action::MenuClick { .. } => "menu_click",
1388 Action::AppleEvent { .. } => "apple_event",
1389 Action::ShortcutsRun { .. } => "shortcuts_run",
1390 Action::DockList { .. } => "dock_list",
1391 Action::DockClick { .. } => "dock_click",
1392 Action::Spaces { .. } => "spaces",
1393 Action::BrowserTabs { .. } => "browser_tabs",
1394 Action::BrowserActivate { .. } => "browser_activate",
1395 Action::BrowserNavigate { .. } => "browser_navigate",
1396 Action::BrowserClick { .. } => "browser_click",
1397 Action::BrowserText { .. } => "browser_text",
1398 Action::BrowserType { .. } => "browser_type",
1399 Action::BrowserEval { .. } => "browser_eval",
1400 Action::Ocr { .. } => "ocr",
1401 }
1402}
1403
1404#[derive(Clone, Copy)]
1405enum PolicyClass {
1406 Always,
1407 Benign,
1408 Observe,
1409 Mutate,
1410 Quit,
1411 Key,
1412 ClipboardSet,
1413}
1414
1415fn policy_class(action: &Action) -> PolicyClass {
1416 match action {
1417 Action::Wait { .. }
1418 | Action::Doctor { .. }
1419 | Action::Displays
1420 | Action::Isolation
1421 | Action::ListApps { .. }
1422 | Action::ListWindows
1423 | Action::Pause
1424 | Action::Resume => PolicyClass::Always,
1425 Action::SessionEnd | Action::SessionPrune { .. } | Action::CursorPosition => {
1426 PolicyClass::Benign
1427 }
1428 Action::Screenshot { .. }
1429 | Action::Zoom { .. }
1430 | Action::Tree
1431 | Action::GetState
1432 | Action::PolicyCheck
1433 | Action::MenuList { .. }
1434 | Action::DockList { .. }
1435 | Action::Spaces { .. }
1436 | Action::BrowserTabs { .. }
1437 | Action::BrowserText { .. }
1438 | Action::BrowserActivate { .. }
1439 | Action::Ocr { .. } => PolicyClass::Observe,
1440 Action::Quit { .. } => PolicyClass::Quit,
1441 Action::Key { .. } | Action::HoldKey { .. } => PolicyClass::Key,
1442 Action::ClipboardGet => PolicyClass::Benign,
1443 Action::ClipboardSet { .. } => PolicyClass::ClipboardSet,
1444 _ => PolicyClass::Mutate,
1445 }
1446}
1447
1448struct Prepared {
1449 stop: Option<Response>,
1450 warning: Option<String>,
1451 subject: policy::Subject,
1452 decision: &'static str,
1453 source: &'static str,
1454}
1455
1456fn early_stop(warning: Option<String>, subject: policy::Subject, response: Response) -> Prepared {
1457 Prepared {
1458 stop: Some(response),
1459 warning,
1460 subject,
1461 decision: "allow",
1462 source: "default",
1463 }
1464}
1465
1466fn prepare_action(
1467 backend: &dyn DesktopBackend,
1468 action: &Action,
1469 context: &CallContext,
1470) -> Prepared {
1471 let class = policy_class(action);
1472 let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1473 Ok(policy) => policy,
1474 Err(error) => {
1475 if matches!(class, PolicyClass::Always) {
1476 policy::LoadedPolicy::default()
1477 } else {
1478 return early_stop(
1479 None,
1480 policy::Subject::default(),
1481 Response::from_error(error),
1482 );
1483 }
1484 }
1485 };
1486 policy::set_builtin_exemptions(loaded.builtin_exempt());
1489 let warning = if loaded.pause_on_user_input {
1490 backend.user_input_pause_warning().map(str::to_string)
1491 } else {
1492 None
1493 };
1494 if matches!(action, Action::Pause | Action::Resume | Action::PolicyCheck) {
1495 return Prepared {
1496 stop: None,
1497 warning,
1498 subject: policy::Subject::default(),
1499 decision: "allow",
1500 source: "default",
1501 };
1502 }
1503 let subject = match resolve_subject(backend, action, context) {
1504 Ok(subject) => subject,
1505 Err(error) => {
1506 return early_stop(
1507 warning,
1508 policy::Subject::default(),
1509 Response::from_error(error),
1510 )
1511 }
1512 };
1513 let policy_action = match action {
1514 Action::Quit { .. } => policy::PolicyAction::Quit,
1515 Action::Key { keys } | Action::HoldKey { keys, .. } => policy::PolicyAction::Key {
1516 combo: keys.clone(),
1517 },
1518 Action::Screenshot { .. } | Action::Zoom { .. } | Action::Tree | Action::GetState => {
1519 policy::PolicyAction::Observe
1520 }
1521 Action::Ocr { .. } => policy::PolicyAction::Observe,
1522 Action::ClipboardSet { .. } => policy::PolicyAction::ClipboardSet,
1523 _ => match class {
1524 PolicyClass::Always => policy::PolicyAction::Always,
1525 PolicyClass::Benign => policy::PolicyAction::Benign,
1526 PolicyClass::Mutate => policy::PolicyAction::Mutate,
1527 PolicyClass::Observe => policy::PolicyAction::Observe,
1528 PolicyClass::Quit => policy::PolicyAction::Quit,
1529 PolicyClass::Key => policy::PolicyAction::Key {
1530 combo: String::new(),
1531 },
1532 PolicyClass::ClipboardSet => policy::PolicyAction::ClipboardSet,
1533 },
1534 };
1535 let decision = policy::decide(&loaded, &subject, &policy_action);
1536 if let policy::Decision::Deny { message, source } = &decision {
1537 return Prepared {
1538 stop: Some(Response::from_error(ComputerError::PolicyDenied(
1539 message.clone(),
1540 ))),
1541 warning,
1542 subject,
1543 decision: "deny",
1544 source,
1545 };
1546 }
1547 if let Err(error) = refuse_blocked_url(backend, &loaded, &subject) {
1548 let (decision_name, source) = decision_record(&decision);
1549 return Prepared {
1550 stop: Some(Response::from_error(error)),
1551 warning,
1552 subject,
1553 decision: decision_name,
1554 source,
1555 };
1556 }
1557 let mutating = matches!(
1558 class,
1559 PolicyClass::Mutate | PolicyClass::Quit | PolicyClass::Key | PolicyClass::ClipboardSet
1560 );
1561 if mutating && policy::is_paused() {
1562 let (decision_name, source) = decision_record(&decision);
1563 return Prepared {
1564 stop: Some(Response::from_error(ComputerError::Paused(
1565 "local input paused mutating actions; run computer-use resume".into(),
1566 ))),
1567 warning,
1568 subject,
1569 decision: decision_name,
1570 source,
1571 };
1572 }
1573 if let Action::Key { .. } | Action::HoldKey { .. } = action {
1574 if let Some(index) = context.element {
1575 if let Ok(Some(peek)) = backend.peek_element(
1576 context.app.as_deref(),
1577 context.window.as_deref(),
1578 context.session.as_deref(),
1579 index,
1580 ) {
1581 if peek.secure {
1582 return Prepared {
1583 stop: Some(Response::from_error(ComputerError::PolicyDenied(
1584 "refusing a key event aimed at a secure field".into(),
1585 ))),
1586 warning,
1587 subject,
1588 decision: "deny",
1589 source: "builtin",
1590 };
1591 }
1592 }
1593 }
1594 }
1595 if let policy::Decision::Confirm { reason } = &decision {
1596 if let Err(error) = confirm_action(context, &loaded, &subject, action_label(action), reason)
1597 {
1598 return Prepared {
1599 stop: Some(Response::from_error(error)),
1600 warning,
1601 subject,
1602 decision: "confirm",
1603 source: "builtin",
1604 };
1605 }
1606 }
1607 let (decision_name, source) = decision_record(&decision);
1608 Prepared {
1609 stop: None,
1610 warning,
1611 subject,
1612 decision: decision_name,
1613 source,
1614 }
1615}
1616
1617fn decision_record(decision: &policy::Decision) -> (&'static str, &'static str) {
1618 match decision {
1619 policy::Decision::Allow { source } => ("allow", *source),
1620 policy::Decision::Deny { source, .. } => ("deny", *source),
1621 policy::Decision::Confirm { .. } => ("confirm", "builtin"),
1622 }
1623}
1624
1625fn confirm_action(
1626 context: &CallContext,
1627 loaded: &policy::LoadedPolicy,
1628 subject: &policy::Subject,
1629 action: &str,
1630 reason: &str,
1631) -> Result<(), ComputerError> {
1632 let command = policy::confirm_program(loaded, context.confirm_cmd.as_deref());
1633 policy::run_confirm(
1634 command.as_deref(),
1635 loaded.timeout_secs(),
1636 &policy::ConfirmDocument {
1637 action,
1638 bundle_id: subject.bundle_id.as_deref(),
1639 path: subject.path.as_deref(),
1640 pid: subject.pid,
1641 window_id: subject.window_id,
1642 element: context.element,
1643 reason,
1644 },
1645 )
1646}
1647
1648fn refuse_blocked_url(
1649 backend: &dyn DesktopBackend,
1650 loaded: &policy::LoadedPolicy,
1651 subject: &policy::Subject,
1652) -> Result<(), ComputerError> {
1653 if loaded.blocked_urls().is_empty() {
1654 return Ok(());
1655 }
1656 if !policy::is_browser(
1657 subject.bundle_id.as_deref(),
1658 subject.path.as_deref().map(Path::new),
1659 subject.name.as_deref(),
1660 ) {
1661 return Ok(());
1662 }
1663 let from_title = subject
1664 .window_title
1665 .as_deref()
1666 .and_then(policy::url_in_text)
1667 .map(str::to_string);
1668 let url = if let Some(url) = from_title {
1669 Some(url)
1670 } else if let Some(pid) = subject.pid {
1671 backend.browser_url(pid)?
1672 } else {
1673 None
1674 };
1675 if let Some(url) = url.as_deref() {
1676 if let Some(pattern) = policy::url_blocked(url, loaded.blocked_urls()) {
1677 return Err(ComputerError::BlockedUrl(format!("matched {pattern}")));
1678 }
1679 }
1680 Ok(())
1681}
1682
1683fn resolve_subject(
1684 backend: &dyn DesktopBackend,
1685 action: &Action,
1686 context: &CallContext,
1687) -> Result<policy::Subject, ComputerError> {
1688 let class = policy_class(action);
1689 if matches!(class, PolicyClass::Always | PolicyClass::Benign) {
1690 return Ok(policy::Subject::default());
1691 }
1692 let screenshot = matches!(action, Action::Screenshot { .. } | Action::Zoom { .. });
1693 let frontmost_observe =
1694 matches!(action, Action::Tree | Action::GetState) && context.app.is_none();
1695 let app_text = context.app.clone().or_else(|| {
1696 if frontmost_observe {
1697 backend.frontmost().app
1698 } else {
1699 None
1700 }
1701 });
1702 if app_text.is_none() && context.window.is_none() {
1703 return Ok(policy::Subject::default());
1704 }
1705 if app_text.is_none() && screenshot {
1706 return subject_from_window(backend, context);
1707 }
1708 let Some(app_text) = app_text else {
1709 return Ok(policy::Subject::default());
1710 };
1711 let selector = parse_app_selector(&app_text)?;
1712 let fallback = subject_from_selector(&selector);
1713 let apps = match backend.list_apps(true) {
1714 Ok(apps) => apps,
1715 Err(error) => {
1716 if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1717 return Ok(fallback);
1718 }
1719 return Err(error);
1720 }
1721 };
1722 let info = match resolve_app(&apps, &selector) {
1723 Ok(info) => info.clone(),
1724 Err(error) => {
1725 if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1726 return Ok(fallback);
1727 }
1728 if context.app.is_none() {
1731 return Ok(policy::Subject::default());
1732 }
1733 return Err(error);
1734 }
1735 };
1736 let mut subject = policy::Subject {
1737 name: Some(info.name.clone()),
1738 bundle_id: info.bundle_id.clone(),
1739 path: info.path.clone(),
1740 pid: Some(info.pid),
1741 window_id: None,
1742 window_title: None,
1743 ax_identifier: None,
1744 };
1745 let want_window = context.window.is_some()
1746 || matches!(
1747 action,
1748 Action::Tree | Action::GetState | Action::Screenshot { .. } | Action::Zoom { .. }
1749 );
1750 if want_window {
1751 if let Ok(listed) = backend.list_windows() {
1752 let owned = filter_windows_for_pid(&listed.windows, info.pid);
1753 let chosen = match context.window.as_deref() {
1754 Some(window) => {
1755 let selector = parse_window_selector(window)?;
1756 Some(resolve_window(&owned, &selector)?.clone())
1757 }
1758 None => owned.first().cloned(),
1759 };
1760 if let Some(window) = chosen {
1761 subject.window_id = Some(window.id);
1762 if context.window.is_some()
1763 || matches!(
1764 action,
1765 Action::Tree
1766 | Action::GetState
1767 | Action::Screenshot { .. }
1768 | Action::Zoom { .. }
1769 )
1770 {
1771 subject.window_title = Some(window.title);
1772 }
1773 }
1774 }
1775 }
1776 Ok(subject)
1777}
1778
1779fn subject_from_window(
1780 backend: &dyn DesktopBackend,
1781 context: &CallContext,
1782) -> Result<policy::Subject, ComputerError> {
1783 let Some(window) = context.window.as_deref() else {
1784 return Ok(policy::Subject::default());
1785 };
1786 let listed = backend.list_windows()?;
1787 let selector = parse_window_selector(window)?;
1788 let window = resolve_window(&listed.windows, &selector)?.clone();
1789 let apps = backend.list_apps(true)?;
1790 let info = apps.into_iter().find(|app| app.pid == window.pid);
1791 Ok(policy::Subject {
1792 name: info
1793 .as_ref()
1794 .map(|app| app.name.clone())
1795 .or(Some(window.app.clone())),
1796 bundle_id: info.as_ref().and_then(|app| app.bundle_id.clone()),
1797 path: info.and_then(|app| app.path),
1798 pid: Some(window.pid),
1799 window_id: Some(window.id),
1800 window_title: Some(window.title),
1801 ax_identifier: None,
1802 })
1803}
1804
1805fn subject_from_selector(selector: &crate::target::AppSelector) -> policy::Subject {
1806 use crate::target::AppSelector;
1807 match selector {
1808 AppSelector::BundleId(bundle) => policy::Subject {
1809 bundle_id: Some(bundle.clone()),
1810 ..policy::Subject::default()
1811 },
1812 AppSelector::Name(name) => policy::Subject {
1813 name: Some(name.clone()),
1814 ..policy::Subject::default()
1815 },
1816 AppSelector::Path(path) => policy::Subject {
1817 path: Some(path.display().to_string()),
1818 ..policy::Subject::default()
1819 },
1820 AppSelector::Pid(pid) => policy::Subject {
1821 pid: Some(*pid),
1822 ..policy::Subject::default()
1823 },
1824 }
1825}
1826
1827fn policy_check(
1828 backend: &dyn DesktopBackend,
1829 context: &CallContext,
1830) -> (Response, policy::Subject) {
1831 let Some(app) = context
1832 .app
1833 .as_deref()
1834 .map(str::trim)
1835 .filter(|text| !text.is_empty())
1836 else {
1837 return (
1838 Response::from_error(ComputerError::InvalidArguments("app is required".into())),
1839 policy::Subject::default(),
1840 );
1841 };
1842 let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1843 Ok(policy) => policy,
1844 Err(error) => return (Response::from_error(error), policy::Subject::default()),
1845 };
1846 policy::set_builtin_exemptions(loaded.builtin_exempt());
1847 let selector = match parse_app_selector(app) {
1848 Ok(selector) => selector,
1849 Err(error) => return (Response::from_error(error), policy::Subject::default()),
1850 };
1851 let mut subject = subject_from_selector(&selector);
1852 if let Ok(apps) = backend.list_apps(true) {
1853 if let Ok(info) = resolve_app(&apps, &selector) {
1854 subject.name = Some(info.name.clone());
1855 subject.bundle_id = info.bundle_id.clone();
1856 subject.path = info.path.clone();
1857 subject.pid = Some(info.pid);
1858 }
1859 }
1860 let decision = policy::app_verdict(&loaded, &subject);
1861 let (name, source, code) = match decision {
1862 policy::Decision::Allow { source } => ("allow", source, "ok"),
1863 policy::Decision::Deny { source, .. } => ("deny", source, "policy_denied"),
1864 policy::Decision::Confirm { .. } => ("allow", "default", "ok"),
1865 };
1866 (
1867 Response::success(json!({
1868 "decision": name,
1869 "source": source,
1870 "code": code,
1871 "bundle_id": subject.bundle_id,
1872 })),
1873 subject,
1874 )
1875}
1876
1877fn pause_response(paused: bool) -> Response {
1878 match policy::set_paused(paused) {
1879 Ok(()) => Response::success(json!({
1880 "action": if paused { "pause" } else { "resume" },
1881 "paused": paused,
1882 })),
1883 Err(error) => Response::from_error(error),
1884 }
1885}
1886
1887fn result_response(result: Result<Value, ComputerError>) -> Response {
1888 match result {
1889 Ok(value) => Response::success(value),
1890 Err(error) => Response::from_error(error),
1891 }
1892}
1893
1894fn post_state_kind(action: &Action) -> PostKind {
1895 match action {
1896 Action::Screenshot { .. } | Action::Zoom { .. } => PostKind::Screenshot,
1897 Action::Click { .. }
1898 | Action::DoubleClick { .. }
1899 | Action::TripleClick { .. }
1900 | Action::Drag { .. }
1901 | Action::MouseDown { .. }
1902 | Action::MouseUp { .. }
1903 | Action::HoldKey { .. }
1904 | Action::ClipboardSet { .. }
1905 | Action::Press { .. }
1906 | Action::Focus
1907 | Action::SetValue { .. }
1908 | Action::SelectText { .. }
1909 | Action::PerformAction { .. }
1910 | Action::ScrollTo
1911 | Action::Move { .. }
1912 | Action::Type { .. }
1913 | Action::Key { .. }
1914 | Action::Scroll { .. }
1915 | Action::Launch
1916 | Action::Activate
1917 | Action::Quit { .. }
1918 | Action::Batch { .. }
1919 | Action::MenuClick { .. }
1920 | Action::AppleEvent { .. }
1921 | Action::ShortcutsRun { .. }
1922 | Action::DockClick { .. }
1923 | Action::BrowserNavigate { .. }
1924 | Action::BrowserClick { .. }
1925 | Action::BrowserText { .. }
1926 | Action::BrowserType { .. }
1927 | Action::BrowserEval { .. }
1928 | Action::BrowserActivate { .. }
1929 | Action::Ocr { .. } => PostKind::Mutating,
1930 Action::GetState
1931 | Action::Wait { .. }
1932 | Action::Displays
1933 | Action::Isolation
1934 | Action::ListApps { .. }
1935 | Action::ListWindows
1936 | Action::Doctor { .. }
1937 | Action::Tree
1938 | Action::SessionEnd
1939 | Action::SessionPrune { .. }
1940 | Action::CursorPosition
1941 | Action::ClipboardGet
1942 | Action::PolicyCheck
1943 | Action::Pause
1944 | Action::Resume
1945 | Action::MenuList { .. }
1946 | Action::DockList { .. }
1947 | Action::Spaces { .. }
1948 | Action::BrowserTabs { .. } => PostKind::ReadOnly,
1949 }
1950}
1951
1952fn remap_coordinates(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1953 match context.coord.as_deref() {
1954 None | Some("screen") => Ok(action),
1955 Some("image") => remap_image_action(context, action),
1956 Some(other) => Err(ComputerError::InvalidArguments(format!(
1957 "coord must be screen or image (got {other})"
1958 ))),
1959 }
1960}
1961
1962fn remap_image_action(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1963 match action {
1964 Action::Click {
1965 x,
1966 y,
1967 button,
1968 double,
1969 count,
1970 } => {
1971 let (x, y) = remap_optional_point(context, x, y)?;
1972 Ok(Action::Click {
1973 x,
1974 y,
1975 button,
1976 double,
1977 count,
1978 })
1979 }
1980 Action::DoubleClick { x, y, button } => {
1981 let (x, y) = remap_optional_point(context, x, y)?;
1982 Ok(Action::DoubleClick { x, y, button })
1983 }
1984 Action::TripleClick { x, y, button } => {
1985 let (x, y) = remap_optional_point(context, x, y)?;
1986 Ok(Action::TripleClick { x, y, button })
1987 }
1988 Action::Drag { path, button } => {
1989 let mut mapped = Vec::with_capacity(path.len());
1990 for point in path {
1991 let (x, y) = remap_point(context, point.x, point.y)?;
1992 mapped.push(Point { x, y });
1993 }
1994 Ok(Action::Drag {
1995 path: mapped,
1996 button,
1997 })
1998 }
1999 Action::MouseDown { x, y, button } => {
2000 let (x, y) = remap_point(context, x, y)?;
2001 Ok(Action::MouseDown { x, y, button })
2002 }
2003 Action::MouseUp { x, y, button } => {
2004 let (x, y) = remap_point(context, x, y)?;
2005 Ok(Action::MouseUp { x, y, button })
2006 }
2007 Action::Move { x, y } => {
2008 let (x, y) = remap_point(context, x, y)?;
2009 Ok(Action::Move { x, y })
2010 }
2011 Action::Scroll {
2012 x,
2013 y,
2014 dx,
2015 dy,
2016 direction,
2017 amount,
2018 } => {
2019 let (x, y) = remap_optional_point(context, x, y)?;
2020 Ok(Action::Scroll {
2021 x,
2022 y,
2023 dx,
2024 dy,
2025 direction,
2026 amount,
2027 })
2028 }
2029 Action::Screenshot {
2030 output,
2031 region,
2032 format,
2033 max_width,
2034 canvas,
2035 base64,
2036 marks,
2037 } => {
2038 let region = match region {
2039 Some(region) => Some(remap_region(context, region)?),
2040 None => None,
2041 };
2042 Ok(Action::Screenshot {
2043 output,
2044 region,
2045 format,
2046 max_width,
2047 canvas,
2048 base64,
2049 marks,
2050 })
2051 }
2052 Action::Zoom {
2053 output,
2054 region,
2055 format,
2056 max_width,
2057 canvas,
2058 base64,
2059 } => {
2060 let region = remap_region(context, region)?;
2061 Ok(Action::Zoom {
2062 output,
2063 region,
2064 format,
2065 max_width,
2066 canvas,
2067 base64,
2068 })
2069 }
2070 other => Ok(other),
2071 }
2072}
2073
2074fn gate_display(
2081 backend: &dyn DesktopBackend,
2082 context: &CallContext,
2083 action: &Action,
2084) -> Result<(), ComputerError> {
2085 let Some(spec) = context.display.as_deref() else {
2086 return Ok(());
2087 };
2088 crate::isolation::parse_selector(spec)?;
2093 if matches!(action, Action::Isolation | Action::Displays) {
2095 return Ok(());
2096 }
2097 let displays = backend.list_displays()?;
2098 let target = crate::isolation::resolve(spec, &displays)?;
2099 match action {
2100 Action::Click { x, y, .. }
2103 | Action::DoubleClick { x, y, .. }
2104 | Action::TripleClick { x, y, .. }
2105 | Action::Scroll { x, y, .. } => match (x, y) {
2106 (Some(x), Some(y)) => crate::isolation::ensure_point_inside(&target, *x, *y),
2107 _ => Ok(()),
2108 },
2109 Action::Move { x, y } | Action::MouseDown { x, y, .. } | Action::MouseUp { x, y, .. } => {
2110 crate::isolation::ensure_point_inside(&target, *x, *y)
2111 }
2112 Action::Drag { path, .. } => {
2113 for point in path {
2114 crate::isolation::ensure_point_inside(&target, point.x, point.y)?;
2115 }
2116 Ok(())
2117 }
2118 Action::Screenshot {
2119 region: Some(region),
2120 ..
2121 } => crate::isolation::ensure_region_inside(&target, *region),
2122 Action::Screenshot { .. } => Ok(()),
2123 Action::Zoom { region, .. } => crate::isolation::ensure_region_inside(&target, *region),
2124 _ => Ok(()),
2125 }
2126}
2127
2128fn remap_optional_point(
2129 context: &CallContext,
2130 x: Option<i32>,
2131 y: Option<i32>,
2132) -> Result<(Option<i32>, Option<i32>), ComputerError> {
2133 match (x, y) {
2134 (Some(x), Some(y)) => {
2135 let (x, y) = remap_point(context, x, y)?;
2136 Ok((Some(x), Some(y)))
2137 }
2138 other => Ok(other),
2139 }
2140}
2141
2142fn remap_point(context: &CallContext, x: i32, y: i32) -> Result<(i32, i32), ComputerError> {
2143 let map = require_screenshot_map(context)?;
2144 ensure_point_inside(&map, x, y)?;
2145 let (screen_x, screen_y) = image_util::image_to_screen(&map, f64::from(x), f64::from(y));
2146 Ok((round_screen(screen_x)?, round_screen(screen_y)?))
2147}
2148
2149fn remap_region(context: &CallContext, region: Region) -> Result<Region, ComputerError> {
2150 let map = require_screenshot_map(context)?;
2151 ensure_region_inside(&map, region)?;
2152 let x2 = i64::from(region.x) + i64::from(region.width);
2153 let y2 = i64::from(region.y) + i64::from(region.height);
2154 let (screen_x1, screen_y1) =
2155 image_util::image_to_screen(&map, f64::from(region.x), f64::from(region.y));
2156 let (screen_x2, screen_y2) = image_util::image_to_screen(&map, x2 as f64, y2 as f64);
2157 let x = round_screen(screen_x1)?;
2158 let y = round_screen(screen_y1)?;
2159 let far_x = round_screen(screen_x2)?;
2160 let far_y = round_screen(screen_y2)?;
2161 let width = u32::try_from(i64::from(far_x) - i64::from(x))
2162 .unwrap_or(0)
2163 .max(1);
2164 let height = u32::try_from(i64::from(far_y) - i64::from(y))
2165 .unwrap_or(0)
2166 .max(1);
2167 Region::new(x, y, width, height)
2168}
2169
2170fn require_screenshot_map(
2171 context: &CallContext,
2172) -> Result<image_util::ScreenshotMap, ComputerError> {
2173 match cache::load_screenshot_map(context.session.as_deref())? {
2174 Some(map) => Ok(map),
2175 None => Err(ComputerError::InvalidArguments(
2176 "take a screenshot first".into(),
2177 )),
2178 }
2179}
2180
2181fn content_box(map: &image_util::ScreenshotMap) -> (i64, i64, i64, i64) {
2182 let left = i64::from(map.pad.x);
2183 let top = i64::from(map.pad.y);
2184 let right = left + i64::from(map.content_width);
2185 let bottom = top + i64::from(map.content_height);
2186 (left, top, right, bottom)
2187}
2188
2189fn ensure_point_inside(
2190 map: &image_util::ScreenshotMap,
2191 x: i32,
2192 y: i32,
2193) -> Result<(), ComputerError> {
2194 let (left, top, right, bottom) = content_box(map);
2195 let x = i64::from(x);
2196 let y = i64::from(y);
2197 if x < left || y < top || x >= right || y >= bottom {
2198 return Err(ComputerError::InvalidArguments(format!(
2199 "coordinate ({x}, {y}) is outside the screenshot"
2200 )));
2201 }
2202 Ok(())
2203}
2204
2205fn ensure_region_inside(
2206 map: &image_util::ScreenshotMap,
2207 region: Region,
2208) -> Result<(), ComputerError> {
2209 let (left, top, right, bottom) = content_box(map);
2210 let x1 = i64::from(region.x);
2211 let y1 = i64::from(region.y);
2212 let x2 = x1 + i64::from(region.width);
2213 let y2 = y1 + i64::from(region.height);
2214 if x1 < left || y1 < top || x1 >= right || y1 >= bottom || x2 > right || y2 > bottom {
2215 return Err(ComputerError::InvalidArguments(format!(
2216 "region {},{} {}x{} is outside the screenshot",
2217 region.x, region.y, region.width, region.height
2218 )));
2219 }
2220 Ok(())
2221}
2222
2223fn round_screen(value: f64) -> Result<i32, ComputerError> {
2224 if !value.is_finite() {
2225 return Err(ComputerError::InvalidArguments(
2226 "coordinate remap was not finite".into(),
2227 ));
2228 }
2229 let rounded = value.round();
2230 if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) {
2231 return Err(ComputerError::InvalidArguments(
2232 "coordinate remap is out of range".into(),
2233 ));
2234 }
2235 Ok(rounded as i32)
2236}
2237
2238fn run_batch(
2239 backend: &dyn DesktopBackend,
2240 context: &CallContext,
2241 actions: Vec<Value>,
2242 stop_on_error: bool,
2243) -> Response {
2244 let _session = InputSession::open(backend);
2246 let mut results = Vec::with_capacity(actions.len());
2247 let mut first_error: Option<(usize, ErrorBody)> = None;
2248 for (index, step) in actions.into_iter().enumerate() {
2249 let response = dispatch_batch_step(backend, context, step);
2250 let failed = !response.ok;
2251 if failed && first_error.is_none() {
2252 if let Some(error) = response.error.clone() {
2253 first_error = Some((index, error));
2254 }
2255 }
2256 results.push(response.to_value());
2257 if failed && stop_on_error {
2258 return attach_post_state(
2259 backend,
2260 context,
2261 PostKind::Mutating,
2262 batch_response(results, Some(index), first_error),
2263 );
2264 }
2265 }
2266 attach_post_state(
2267 backend,
2268 context,
2269 PostKind::Mutating,
2270 batch_response(results, None, first_error),
2271 )
2272}
2273
2274fn dispatch_batch_step(
2275 backend: &dyn DesktopBackend,
2276 outer: &CallContext,
2277 value: Value,
2278) -> Response {
2279 match prepare_batch_step(outer, value) {
2280 Ok((action, context)) => dispatch_with_context(backend, action, &context),
2281 Err(error) => Response::from_error(error),
2282 }
2283}
2284
2285fn prepare_batch_step(
2286 outer: &CallContext,
2287 mut value: Value,
2288) -> Result<(Action, CallContext), ComputerError> {
2289 let Some(object) = value.as_object_mut() else {
2290 return Err(ComputerError::InvalidArguments(
2291 "batch step must be a JSON object".into(),
2292 ));
2293 };
2294 if object.get("action").and_then(Value::as_str) == Some("batch") {
2295 return Err(ComputerError::InvalidArguments(
2296 "nested batch is not allowed".into(),
2297 ));
2298 }
2299 let mut context = outer.clone();
2300 overlay_context(&mut context, object)?;
2301 context.return_state = false;
2302 context.state_diff = false;
2303 context.include_screenshot = false;
2304 let action = serde_json::from_value(value)
2305 .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
2306 Ok((action, context))
2307}
2308
2309fn overlay_context(
2310 context: &mut CallContext,
2311 object: &mut Map<String, Value>,
2312) -> Result<(), ComputerError> {
2313 if object.contains_key("app") {
2314 context.app = take_stringish(object, "app")?;
2315 }
2316 if object.contains_key("window") {
2317 context.window = take_stringish(object, "window")?;
2318 }
2319 if object.contains_key("element") {
2320 context.element = take_optional_u32(object, "element")?;
2321 }
2322 if object.contains_key("diff") {
2323 context.diff = take_bool(object, "diff")?;
2324 }
2325 if object.contains_key("full") {
2326 context.full = take_bool(object, "full")?;
2327 }
2328 let _ = object.remove("return_state");
2329 let _ = object.remove("state_diff");
2330 let _ = object.remove("include_screenshot");
2331 let _ = object.remove("id");
2332 let _ = object.remove("stop_on_error");
2333 if object.contains_key("coord") {
2334 context.coord = take_stringish(object, "coord")?;
2335 }
2336 if object.contains_key("session") {
2337 context.session = take_stringish(object, "session")?;
2338 }
2339 if object.contains_key("tier") {
2340 context.tier = take_stringish(object, "tier")?;
2341 }
2342 if object.contains_key("adapter") {
2343 context.adapter = take_stringish(object, "adapter")?;
2344 }
2345 if object.contains_key("display") {
2346 context.display = take_stringish(object, "display")?;
2347 }
2348 if object.contains_key("require_visible") {
2349 context.require_visible = take_bool(object, "require_visible")?;
2350 }
2351 if object.contains_key("allow_foreground") {
2352 context.allow_foreground = take_bool(object, "allow_foreground")?;
2353 }
2354 if object.contains_key("require_background") {
2355 context.require_background = take_bool(object, "require_background")?;
2356 }
2357 if object.contains_key("no_overlay") {
2358 context.no_overlay = take_bool(object, "no_overlay")?;
2359 }
2360 Ok(())
2361}
2362
2363fn batch_response(
2364 results: Vec<Value>,
2365 stopped_at: Option<usize>,
2366 first_error: Option<(usize, ErrorBody)>,
2367) -> Response {
2368 let mut fields = Map::new();
2369 fields.insert("action".into(), json!("batch"));
2370 fields.insert("results".into(), Value::Array(results));
2371 if let Some(index) = stopped_at {
2372 fields.insert("stopped_at".into(), json!(index));
2373 }
2374 match first_error {
2375 Some((index, error)) => {
2376 let mut response = Response::failure(
2377 error.code,
2378 format!("batch failed at action {index}: {}", error.message),
2379 );
2380 response.fields = fields;
2381 response
2382 }
2383 None => Response {
2384 ok: true,
2385 id: None,
2386 error: None,
2387 fields,
2388 },
2389 }
2390}
2391
2392fn attach_post_state(
2393 backend: &dyn DesktopBackend,
2394 context: &CallContext,
2395 kind: PostKind,
2396 mut response: Response,
2397) -> Response {
2398 if !(context.return_state || context.state_diff) || matches!(kind, PostKind::ReadOnly) {
2401 return response;
2402 }
2403 let mut warnings = Vec::new();
2404 let mut state = match backend.tree(&tree_query(context)) {
2405 Ok(report) => {
2406 let baseline = if report.diff.is_some() {
2408 "existing"
2409 } else {
2410 "created"
2411 };
2412 let mut state = json!({
2413 "app": report.app,
2414 "pid": report.pid,
2415 "window_id": report.window_id,
2416 "revision": report.revision,
2417 "diff": report.diff,
2418 "truncated": report.truncated,
2419 "baseline": baseline,
2420 });
2421 if !context.state_diff {
2424 state["tree"] = json!(report.full_text);
2425 }
2426 state
2427 }
2428 Err(error) => {
2429 if !response.ok {
2430 return response;
2431 }
2432 warnings.push(format!("tree snapshot failed: {error}"));
2433 json!({
2434 "app": null,
2435 "pid": null,
2436 "window_id": null,
2437 "revision": null,
2438 "tree": null,
2439 "diff": null,
2440 "truncated": null,
2441 "baseline": null,
2442 })
2443 }
2444 };
2445 if context.include_screenshot {
2446 let shot = if response.ok && matches!(kind, PostKind::Screenshot) {
2447 Ok(Value::Object(response.fields.clone()))
2448 } else if matches!(kind, PostKind::Screenshot) {
2449 Err(ComputerError::ExecutionFailed(
2450 response
2451 .error
2452 .as_ref()
2453 .map(|error| error.message.clone())
2454 .unwrap_or_else(|| "screenshot failed".into()),
2455 ))
2456 } else {
2457 screenshot(
2458 backend,
2459 context,
2460 ScreenshotArgs {
2461 output: None,
2462 region: None,
2463 format: None,
2464 max_width: None,
2465 canvas: None,
2466 include_base64: false,
2467 window_crop: false,
2468 marks: false,
2469 },
2470 )
2471 };
2472 match shot {
2473 Ok(value) => state["screenshot"] = value,
2474 Err(error) => {
2475 state["screenshot"] = Value::Null;
2476 warnings.push(format!("screenshot failed: {error}"));
2477 }
2478 }
2479 }
2480 response.fields.insert("state".into(), state);
2481 push_warnings(&mut response, warnings);
2482 response
2483}
2484
2485fn push_warnings(response: &mut Response, extra: Vec<String>) {
2486 if extra.is_empty() {
2487 return;
2488 }
2489 let mut warnings = match response.fields.remove("warnings") {
2490 Some(Value::Array(items)) => items,
2491 Some(other) => vec![other],
2492 None => Vec::new(),
2493 };
2494 for warning in extra {
2495 warnings.push(json!(warning));
2496 }
2497 response
2498 .fields
2499 .insert("warnings".into(), Value::Array(warnings));
2500}
2501
2502fn cursor_position(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2503 let (x, y) = backend.cursor_position()?;
2504 Ok(json!({"action": "cursor_position", "x": x, "y": y}))
2505}
2506
2507fn invalid_arguments(message: &str) -> ComputerError {
2508 ComputerError::InvalidArguments(message.to_string())
2509}
2510
2511fn unsupported_error(operation: &str) -> ComputerError {
2512 ComputerError::Unsupported(format!(
2513 "{operation} is not available on {}",
2514 std::env::consts::OS
2515 ))
2516}
2517
2518fn menu_list(backend: &dyn DesktopBackend, app: &str, depth: u32) -> Result<Value, ComputerError> {
2519 let report = backend.menu_list(app, depth)?;
2520 Ok(json!({
2521 "action": "menu_list",
2522 "app": report.app,
2523 "sections": report.sections,
2524 }))
2525}
2526
2527fn menu_click(backend: &dyn DesktopBackend, app: &str, path: &str) -> Result<Value, ComputerError> {
2528 let parsed = crate::menu_path::parse(path)?;
2529 let result = backend.menu_click(app, &parsed)?;
2530 Ok(json!({
2531 "action": "menu_click",
2532 "app": app,
2533 "path": parsed.segments,
2534 "pressed": result.pressed,
2535 }))
2536}
2537
2538fn apple_event_action(
2539 backend: &dyn DesktopBackend,
2540 app: &str,
2541 script: &str,
2542 language: AppleEventLanguage,
2543) -> Result<Value, ComputerError> {
2544 let report = backend.apple_event(app, script, language)?;
2545 Ok(json!({
2546 "action": "apple_event",
2547 "app": app,
2548 "language": language.as_str(),
2549 "script_bytes": report.script_bytes,
2550 "script_hash": report.script_hash,
2551 "stdout": report.stdout,
2552 "stderr": report.stderr,
2553 "exit_status": report.exit_status,
2554 }))
2555}
2556
2557fn shortcuts_run_action(name: String) -> Result<Value, ComputerError> {
2558 let report = backend_shortcuts_run(&name)?;
2559 Ok(json!({
2560 "action": "shortcuts_run",
2561 "name": name,
2562 "exit_status": report.exit_status,
2563 "stderr": report.stderr,
2564 }))
2565}
2566
2567fn dock_list_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2568 let report = backend.dock_list()?;
2569 Ok(json!({
2570 "action": "dock_list",
2571 "items": report.items,
2572 }))
2573}
2574
2575fn dock_click_action(backend: &dyn DesktopBackend, title: &str) -> Result<Value, ComputerError> {
2576 let report = backend.dock_click(title)?;
2577 Ok(json!({
2578 "action": "dock_click",
2579 "title": title,
2580 "pressed": report.pressed,
2581 }))
2582}
2583
2584fn spaces_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2585 let report = backend.spaces()?;
2586 Ok(json!({
2587 "action": "spaces",
2588 "screens": report.screens,
2589 "note": report.note,
2590 }))
2591}
2592
2593fn backend_shortcuts_run(name: &str) -> Result<crate::shortcuts::ShortcutsReport, ComputerError> {
2596 crate::shortcuts::run(name)
2597}
2598
2599fn session_prune(dry_run: bool, ttl_secs: Option<u64>) -> Result<Value, ComputerError> {
2600 let ttl = match ttl_secs {
2601 Some(seconds) => Duration::from_secs(seconds),
2602 None => cache::DEFAULT_SESSION_TTL,
2603 };
2604 let report = cache::prune_sessions(
2605 &cache::sessions_root(),
2606 &cache::PruneOptions {
2607 dry_run,
2608 ttl,
2609 now: SystemTime::now(),
2610 },
2611 &cache::pid_alive,
2612 )?;
2613 Ok(json!({
2614 "action": "session_prune",
2615 "dry_run": dry_run,
2616 "ttl_secs": ttl.as_secs(),
2617 "removed": report.removed,
2618 "kept": report.kept,
2619 }))
2620}
2621
2622struct ScreenshotArgs {
2623 output: Option<String>,
2624 region: Option<Region>,
2625 format: Option<ImageFormat>,
2626 max_width: Option<u32>,
2627 canvas: Option<String>,
2628 include_base64: bool,
2629 window_crop: bool,
2632 marks: bool,
2634}
2635
2636fn screenshot(
2637 backend: &dyn DesktopBackend,
2638 context: &CallContext,
2639 args: ScreenshotArgs,
2640) -> Result<Value, ComputerError> {
2641 let ScreenshotArgs {
2642 output,
2643 region,
2644 format,
2645 max_width,
2646 canvas,
2647 include_base64,
2648 window_crop,
2649 marks,
2650 } = args;
2651 let targeted = context.app.is_some() || context.window.is_some();
2652 if region.is_some() && targeted && !window_crop {
2653 return Err(ComputerError::InvalidArguments(
2654 "pass a region or a window target, not both".into(),
2655 ));
2656 }
2657 let format = format.unwrap_or_default();
2658 let max_width = max_width.unwrap_or(image_util::DEFAULT_SCREENSHOT_MAX_WIDTH);
2659 let canvas = canvas
2660 .as_deref()
2661 .map(image_util::parse_canvas)
2662 .transpose()?;
2663 let raw = if targeted {
2664 if let Some(error) = backend.display_block() {
2665 return Err(error);
2666 }
2667 let target = resolve_screenshot_window(backend, context)?;
2668 if let Some(filter) = display_filter(backend, context)? {
2671 if !crate::isolation::window_inside(&filter, target.bounds) {
2672 return Err(ComputerError::InvalidArguments(format!(
2673 "window {}x{}+{}+{} is not inside display {} ({} covers {})",
2674 target.bounds.width,
2675 target.bounds.height,
2676 target.bounds.x,
2677 target.bounds.y,
2678 filter.display.id,
2679 filter.display.name,
2680 filter.bounds()
2681 )));
2682 }
2683 }
2684 let captured = backend.capture_window(&target)?;
2685 match region {
2686 Some(region) if window_crop => crop_window_region(captured, region)?,
2687 _ => captured,
2688 }
2689 } else {
2690 let region = match display_filter(backend, context)? {
2694 Some(filter) => Some(filter.region),
2695 None => region,
2696 };
2697 backend.capture_screen(region)?
2698 };
2699 let prepared = image_util::prepare_shot(
2700 raw.frame.image(),
2701 image_util::ShotFit {
2702 scale: raw.frame.scale(),
2703 origin_x: raw.origin_x,
2704 origin_y: raw.origin_y,
2705 logical_width: raw.logical_width,
2706 logical_height: raw.logical_height,
2707 max_width,
2708 canvas,
2709 },
2710 )?;
2711 let mut image = prepared.image;
2712 let mut warnings = raw.warnings;
2713 if marks {
2714 let applied = apply_marks(&mut image, &prepared.map, context, backend)?;
2715 if applied == 0 {
2716 warnings.push("marks: no accessibility element frame fell inside this capture".into());
2719 }
2720 }
2721 let encoded = image_util::encode_rgba(&image, format)?;
2722 let path = match output {
2723 Some(path) => PathBuf::from(path),
2724 None => default_screenshot_path(format),
2725 };
2726 if let Some(parent) = path.parent() {
2727 if !parent.as_os_str().is_empty() {
2728 fs::create_dir_all(parent)?;
2729 }
2730 }
2731 fs::write(&path, &encoded.bytes)?;
2732 if let Err(error) = cache::store_screenshot_map(context.session.as_deref(), &prepared.map) {
2733 warnings.push(format!("screenshot map was not cached: {error}"));
2734 }
2735 let mut body = serde_json::to_value(&prepared.map).map_err(|error| {
2736 ComputerError::ExecutionFailed(format!("screenshot map encode failed: {error}"))
2737 })?;
2738 body["path"] = json!(path_string(&path));
2739 body["format"] = json!(format.as_str());
2740 body["capture_source"] = json!(raw.capture_source);
2741 if raw.occluded {
2742 body["occluded"] = json!(true);
2743 }
2744 if !warnings.is_empty() {
2745 body["warnings"] = json!(warnings);
2746 }
2747 if include_base64 {
2748 body["data"] = json!(encode_base64(&encoded.bytes));
2749 }
2750 Ok(body)
2751}
2752
2753fn resolve_screenshot_window(
2754 backend: &dyn DesktopBackend,
2755 context: &CallContext,
2756) -> Result<WindowTarget, ComputerError> {
2757 let listed = backend.list_windows()?;
2758 if let Some(app) = context.app.as_deref() {
2759 let apps = backend.list_apps(true)?;
2760 let selector = parse_app_selector(app)?;
2761 let app = resolve_app(&apps, &selector)?;
2762 let owned = filter_windows_for_pid(&listed.windows, app.pid);
2763 let selector = match context.window.as_deref() {
2764 Some(window) => parse_window_selector(window)?,
2765 None => WindowSelector::Index(0),
2766 };
2767 let window = resolve_window(&owned, &selector)?;
2768 return Ok(window_target(window));
2769 }
2770 let selector = parse_window_selector(context.window.as_deref().unwrap_or(""))?;
2771 let window = resolve_window(&listed.windows, &selector)?;
2772 Ok(window_target(window))
2773}
2774
2775fn window_target(window: &crate::target::WindowInfo) -> WindowTarget {
2776 WindowTarget {
2777 window_id: window.id,
2778 pid: window.pid,
2779 bounds: window.bounds,
2780 }
2781}
2782
2783fn crop_window_region(
2785 raw: crate::types::RawCapture,
2786 region: Region,
2787) -> Result<crate::types::RawCapture, ComputerError> {
2788 region_inside(region, raw.logical_width, raw.logical_height)?;
2789 let scale = raw.frame.scale();
2790 let x = logical_to_device(i64::from(region.x), scale)?;
2791 let y = logical_to_device(i64::from(region.y), scale)?;
2792 let width = logical_len_to_device(region.width, scale)?;
2793 let height = logical_len_to_device(region.height, scale)?;
2794 let image = raw.frame.image();
2795 if x >= image.width() || y >= image.height() {
2796 return Err(ComputerError::InvalidArguments(
2797 "region is outside the capture".into(),
2798 ));
2799 }
2800 let width = width.min(image.width() - x).max(1);
2801 let height = height.min(image.height() - y).max(1);
2802 let cropped = image::imageops::crop_imm(image, x, y, width, height).to_image();
2803 Ok(crate::types::RawCapture {
2804 frame: Frame::from_image(cropped, scale),
2805 origin_x: raw.origin_x.saturating_add(region.x),
2806 origin_y: raw.origin_y.saturating_add(region.y),
2807 logical_width: region.width,
2808 logical_height: region.height,
2809 capture_source: raw.capture_source,
2810 occluded: raw.occluded,
2811 warnings: raw.warnings,
2812 })
2813}
2814
2815fn logical_to_device(value: i64, scale: f64) -> Result<u32, ComputerError> {
2816 if value < 0 {
2817 return Err(ComputerError::InvalidArguments(
2818 "region is outside the capture".into(),
2819 ));
2820 }
2821 let pixels = (value as f64) * scale;
2822 if !pixels.is_finite() || pixels < 0.0 || pixels > f64::from(u32::MAX) {
2823 return Err(ComputerError::ExecutionFailed(
2824 "region pixel coordinate is out of range".into(),
2825 ));
2826 }
2827 Ok(pixels.round() as u32)
2828}
2829
2830fn logical_len_to_device(value: u32, scale: f64) -> Result<u32, ComputerError> {
2831 let pixels = logical_to_device(i64::from(value), scale)?;
2832 Ok(pixels.max(1))
2833}
2834
2835fn default_screenshot_path(format: ImageFormat) -> PathBuf {
2836 static SEQUENCE: AtomicU64 = AtomicU64::new(0);
2837 let nanos = SystemTime::now()
2838 .duration_since(UNIX_EPOCH)
2839 .map(|duration| duration.as_nanos())
2840 .unwrap_or(0);
2841 let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed);
2842 let name = format!(
2843 "computer-use-{}-{nanos}-{sequence}.{}",
2844 std::process::id(),
2845 format.extension()
2846 );
2847 std::env::temp_dir().join(name)
2848}
2849
2850fn path_string(path: &Path) -> String {
2851 path.to_string_lossy().into_owned()
2852}
2853
2854fn click(
2855 backend: &dyn DesktopBackend,
2856 context: &CallContext,
2857 x: Option<i32>,
2858 y: Option<i32>,
2859 button: MouseButton,
2860 double: bool,
2861 count: u8,
2862) -> Response {
2863 let count = match effective_click_count(double, count) {
2864 Ok(count) => count,
2865 Err(error) => return Response::from_error(error),
2866 };
2867 match (context.element, x, y) {
2868 (Some(_), Some(_), _) | (Some(_), _, Some(_)) => {
2869 Response::failure("invalid_arguments", "pass element or x and y, not both")
2870 }
2871 (None, None, None) => Response::failure(
2872 "invalid_arguments",
2873 "click requires element, or both x and y",
2874 ),
2875 (None, Some(_), None) | (None, None, Some(_)) => {
2876 Response::failure("invalid_arguments", "x and y must be passed together")
2877 }
2878 (Some(_index), None, None) => {
2879 element_command(backend, context, ElementOp::Press { count, button })
2880 }
2881 (None, Some(x), Some(y)) => coordinate_click(backend, context, x, y, button, count),
2882 }
2883}
2884
2885fn effective_click_count(double: bool, count: u8) -> Result<u8, ComputerError> {
2886 if count == 0 || count > 3 {
2887 return Err(ComputerError::InvalidArguments(
2888 "click count must be 1, 2, or 3".into(),
2889 ));
2890 }
2891 if double && count != 1 && count != 2 {
2892 return Err(ComputerError::InvalidArguments(
2893 "double and count disagree".into(),
2894 ));
2895 }
2896 if double {
2897 Ok(2)
2898 } else {
2899 Ok(count)
2900 }
2901}
2902
2903fn coordinate_click(
2904 backend: &dyn DesktopBackend,
2905 context: &CallContext,
2906 x: i32,
2907 y: i32,
2908 button: MouseButton,
2909 count: u8,
2910) -> Response {
2911 let kind = InputKind::Click {
2912 x,
2913 y,
2914 button,
2915 count,
2916 };
2917 if let Err(error) = planned_tiers(backend, context) {
2919 return Response::from_error(error);
2920 }
2921 let mut warnings = Vec::new();
2922 let hit = match backend.hit_test(x, y) {
2923 Ok(hit) => hit,
2924 Err(error) => {
2925 warnings.push(format!("hit test failed: {error}"));
2926 None
2927 }
2928 };
2929 if let Some(title) = hit.as_ref().and_then(|hit| hit.title.as_deref()) {
2930 if policy::sensitive_title(title) {
2931 if let Err(error) = confirm_title(context, "click", "sensitive control title") {
2932 return Response::from_error(error);
2933 }
2934 }
2935 }
2936 match run_attempts(backend, context, kind) {
2937 Ok(report) => {
2938 let action_name = match count {
2939 2 => "double_click",
2940 3 => "triple_click",
2941 _ => "click",
2942 };
2943 let body = json!({
2944 "action": action_name,
2945 "x": x,
2946 "y": y,
2947 "button": button.as_str(),
2948 "double": count == 2,
2949 "count": count,
2950 "hit": hit_value(hit),
2951 });
2952 Response::success(finish_input(body, report, warnings))
2953 }
2954 Err(error) => Response::from_error(error),
2955 }
2956}
2957
2958fn hit_value(hit: Option<crate::element::HitInfo>) -> Value {
2959 match hit {
2960 Some(hit) => json!({"role": hit.role, "title": hit.title}),
2961 None => Value::Null,
2962 }
2963}
2964
2965fn type_command(backend: &dyn DesktopBackend, context: &CallContext, text: String) -> Response {
2966 if context.element.is_some() {
2967 return element_command(backend, context, ElementOp::Type { text });
2968 }
2969 if text.is_empty() {
2970 return Response::from_error(ComputerError::InvalidArguments(
2971 "text must not be empty".into(),
2972 ));
2973 }
2974 if text.contains('\0') {
2975 return Response::from_error(ComputerError::InvalidArguments(
2976 "text must not contain a null byte".into(),
2977 ));
2978 }
2979 let chars = text.chars().count();
2980 match run_attempts(backend, context, InputKind::Type { text: text.clone() }) {
2981 Ok(report) => {
2982 let body = json!({
2983 "action": "type",
2984 "text": text,
2985 "chars": chars,
2986 });
2987 Response::success(finish_input(body, report, Vec::new()))
2988 }
2989 Err(error) => Response::from_error(error),
2990 }
2991}
2992
2993fn element_command(backend: &dyn DesktopBackend, context: &CallContext, op: ElementOp) -> Response {
2994 if let Err(error) = precheck(&op) {
2995 return Response::from_error(error);
2996 }
2997 if let Err(error) = confirm_element_title(backend, context, &op) {
2998 return Response::from_error(error);
2999 }
3000 let index = match context.element {
3001 Some(index) => index,
3002 None => {
3003 return Response::from_error(ComputerError::InvalidArguments(
3004 "element is required".into(),
3005 ))
3006 }
3007 };
3008 let request = ElementRequest {
3009 op,
3010 index,
3011 require_visible: context.require_visible,
3012 app: context.app.clone(),
3013 window: context.window.clone(),
3014 session: context.session.clone(),
3015 tier: context.tier.clone(),
3016 allow_foreground: context.allow_foreground,
3017 require_background: context.require_background,
3018 no_overlay: context.no_overlay,
3019 };
3020 match backend.element_action(&request) {
3021 Ok(outcome) => Response::success(outcome_value(&outcome)),
3022 Err(error) => element_error_response(error),
3023 }
3024}
3025
3026fn confirm_element_title(
3027 backend: &dyn DesktopBackend,
3028 context: &CallContext,
3029 op: &ElementOp,
3030) -> Result<(), ComputerError> {
3031 let press = matches!(op, ElementOp::Press { .. })
3032 || matches!(op, ElementOp::Perform { name } if name.eq_ignore_ascii_case("AXPress"));
3033 if !press {
3034 return Ok(());
3035 }
3036 let Some(index) = context.element else {
3037 return Ok(());
3038 };
3039 let Ok(Some(peek)) = backend.peek_element(
3040 context.app.as_deref(),
3041 context.window.as_deref(),
3042 context.session.as_deref(),
3043 index,
3044 ) else {
3045 return Ok(());
3046 };
3047 if peek.title.as_deref().is_some_and(policy::sensitive_title) {
3048 confirm_title(context, "press", "sensitive control title")?;
3049 }
3050 Ok(())
3051}
3052
3053fn confirm_title(context: &CallContext, action: &str, reason: &str) -> Result<(), ComputerError> {
3054 let loaded = policy::load_policy(context.policy.as_deref().map(Path::new))?;
3055 let command = policy::confirm_program(&loaded, context.confirm_cmd.as_deref());
3056 policy::run_confirm(
3057 command.as_deref(),
3058 loaded.timeout_secs(),
3059 &policy::ConfirmDocument {
3060 action,
3061 bundle_id: None,
3062 path: None,
3063 pid: None,
3064 window_id: None,
3065 element: context.element,
3066 reason,
3067 },
3068 )
3069}
3070
3071fn precheck(op: &ElementOp) -> Result<(), ComputerError> {
3072 match op {
3073 ElementOp::Press { count, .. } => element::validate_count(*count),
3074 ElementOp::SetValue { value } => element::validate_text(value, true),
3075 ElementOp::Type { text } => element::validate_text(text, false),
3076 ElementOp::SelectText { start, end } => element::validate_range(*start, *end, None),
3077 ElementOp::Perform { name } => element::validate_perform_name(name),
3078 ElementOp::Focus | ElementOp::ScrollTo => Ok(()),
3079 }
3080}
3081
3082fn outcome_value(outcome: &ElementOutcome) -> Value {
3083 let mut body = json!({
3084 "action": outcome.action,
3085 "element": outcome.element,
3086 "revision": outcome.revision,
3087 "tier": outcome.tier,
3088 "cursor_moved": outcome.cursor_moved,
3089 });
3090 if let Some(restored) = outcome.focus_restored {
3091 body["focus_restored"] = json!(restored);
3092 }
3093 if !outcome.warnings.is_empty() {
3094 body["warnings"] = json!(outcome.warnings);
3095 }
3096 if let Some(via) = &outcome.via {
3097 body["via"] = json!(via);
3098 }
3099 if let Some(name) = &outcome.name {
3100 body["name"] = json!(name);
3101 }
3102 if let Some(count) = outcome.count {
3103 body["count"] = json!(count);
3104 }
3105 body
3106}
3107
3108fn element_error_response(error: ElementError) -> Response {
3109 let mut response = Response::from_error(error.error);
3110 if let Some(tree) = error.tree {
3111 response.fields.insert("tree".into(), json!(tree));
3112 }
3113 response
3114}
3115
3116fn move_mouse(backend: &dyn DesktopBackend, context: &CallContext, x: i32, y: i32) -> Response {
3117 match run_attempts(backend, context, InputKind::Move { x, y }) {
3118 Ok(report) => Response::success(finish_input(
3119 json!({"action": "move", "x": x, "y": y}),
3120 report,
3121 Vec::new(),
3122 )),
3123 Err(error) => Response::from_error(error),
3124 }
3125}
3126
3127fn press_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str) -> Response {
3128 let strokes = match key_strokes(keys) {
3129 Ok(strokes) => strokes,
3130 Err(error) => return Response::from_error(error),
3131 };
3132 let mut body = json!({
3133 "action": "key",
3134 "keys": keys,
3135 });
3136 if strokes.len() == 1 {
3137 let modifiers: Vec<&str> = strokes[0]
3138 .modifiers
3139 .iter()
3140 .copied()
3141 .map(Modifier::as_str)
3142 .collect();
3143 body["modifiers"] = json!(modifiers);
3144 body["key"] = json!(strokes[0].key);
3145 } else {
3146 body["strokes"] = json!(strokes.len());
3147 }
3148 match run_attempts(backend, context, InputKind::Key { strokes }) {
3149 Ok(report) => Response::success(finish_input(body, report, Vec::new())),
3150 Err(error) => Response::from_error(error),
3151 }
3152}
3153
3154fn key_strokes(keys: &str) -> Result<Vec<KeyStroke>, ComputerError> {
3155 let combos = parse_key_sequence(keys)?;
3156 let mut strokes = Vec::with_capacity(combos.len());
3157 for combo in combos {
3158 validate_key(&combo.key)?;
3159 strokes.push(KeyStroke {
3160 modifiers: combo.modifiers,
3161 key: combo.key,
3162 });
3163 }
3164 Ok(strokes)
3165}
3166
3167fn drag(
3168 backend: &dyn DesktopBackend,
3169 context: &CallContext,
3170 path: Vec<Point>,
3171 button: MouseButton,
3172) -> Response {
3173 if path.len() < 2 {
3174 return Response::from_error(ComputerError::InvalidArguments(
3175 "drag needs at least 2 points".into(),
3176 ));
3177 }
3178 let points: Vec<(i32, i32)> = path.iter().map(|point| (point.x, point.y)).collect();
3179 let moves = points.len() - 1;
3180 match run_attempts(backend, context, InputKind::Drag { points, button }) {
3181 Ok(report) => Response::success(finish_input(
3182 json!({
3183 "action": "drag",
3184 "button": button.as_str(),
3185 "points": path.len(),
3186 "moves": moves,
3187 }),
3188 report,
3189 Vec::new(),
3190 )),
3191 Err(error) => Response::from_error(error),
3192 }
3193}
3194
3195fn pointer_half(
3196 backend: &dyn DesktopBackend,
3197 context: &CallContext,
3198 x: i32,
3199 y: i32,
3200 button: MouseButton,
3201 down: bool,
3202) -> Response {
3203 let kind = if down {
3204 InputKind::MouseDown { x, y, button }
3205 } else {
3206 InputKind::MouseUp { x, y, button }
3207 };
3208 let name = if down { "mouse_down" } else { "mouse_up" };
3209 match run_attempts(backend, context, kind) {
3210 Ok(report) => Response::success(finish_input(
3211 json!({
3212 "action": name,
3213 "x": x,
3214 "y": y,
3215 "button": button.as_str(),
3216 }),
3217 report,
3218 Vec::new(),
3219 )),
3220 Err(error) => Response::from_error(error),
3221 }
3222}
3223
3224fn hold_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str, ms: u64) -> Response {
3225 let stroke = match one_hold(keys, ms) {
3226 Ok(stroke) => stroke,
3227 Err(error) => return Response::from_error(error),
3228 };
3229 match run_attempts(
3230 backend,
3231 context,
3232 InputKind::HoldKey {
3233 modifiers: stroke.modifiers.clone(),
3234 key: stroke.key.clone(),
3235 ms,
3236 },
3237 ) {
3238 Ok(report) => Response::success(finish_input(
3239 json!({
3240 "action": "hold_key",
3241 "keys": keys,
3242 "key": stroke.key,
3243 "ms": ms,
3244 }),
3245 report,
3246 Vec::new(),
3247 )),
3248 Err(error) => Response::from_error(error),
3249 }
3250}
3251
3252fn one_hold(keys: &str, ms: u64) -> Result<KeyStroke, ComputerError> {
3253 if ms > tier::MAX_HOLD_MS {
3254 return Err(ComputerError::InvalidArguments(format!(
3255 "hold_key ms must be at most {}",
3256 tier::MAX_HOLD_MS
3257 )));
3258 }
3259 if keys.split_whitespace().count() != 1 {
3260 return Err(ComputerError::InvalidArguments(
3261 "hold_key accepts one key combination".into(),
3262 ));
3263 }
3264 let combo = parse_key_combo(keys)?;
3265 validate_key(&combo.key)?;
3266 Ok(KeyStroke {
3267 modifiers: combo.modifiers,
3268 key: combo.key,
3269 })
3270}
3271
3272fn clipboard_chars(action: &Action) -> Option<u64> {
3273 match action {
3274 Action::ClipboardSet {
3275 text: Some(text), ..
3276 } => Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX)),
3277 Action::BrowserType { text, .. } => {
3278 Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX))
3279 }
3280 _ => None,
3281 }
3282}
3283
3284fn preflight_action(action: &Action) -> Result<(), ComputerError> {
3285 match action {
3286 Action::Click { double, count, .. } => {
3287 effective_click_count(*double, *count)?;
3288 }
3289 Action::Press { count, .. } => element::validate_count(*count)?,
3290 Action::Drag { path, .. } if path.len() < 2 => {
3291 return Err(ComputerError::InvalidArguments(
3292 "drag needs at least 2 points".into(),
3293 ));
3294 }
3295 Action::HoldKey { keys, ms } => {
3296 one_hold(keys, *ms)?;
3297 }
3298 Action::Key { keys } => {
3299 key_strokes(keys)?;
3300 }
3301 Action::ClipboardSet { text, image } => match (text, image) {
3302 (Some(text), None) if text.is_empty() => {
3303 return Err(ComputerError::InvalidArguments(
3304 "clipboard text must not be empty".into(),
3305 ));
3306 }
3307 (None, None) => {
3308 return Err(ComputerError::InvalidArguments(
3309 "clipboard set requires text or image".into(),
3310 ));
3311 }
3312 (Some(_), Some(_)) => {
3313 return Err(ComputerError::InvalidArguments(
3314 "pass clipboard text or image, not both".into(),
3315 ));
3316 }
3317 _ => {}
3318 },
3319 _ => {}
3320 }
3321 Ok(())
3322}
3323
3324fn clipboard_get(
3325 backend: &dyn DesktopBackend,
3326 context: &CallContext,
3327) -> Result<serde_json::Value, ComputerError> {
3328 match backend.clipboard_get()? {
3329 ClipboardContents::Empty => Ok(json!({
3330 "action": "clipboard_get",
3331 "kind": "empty",
3332 })),
3333 ClipboardContents::Text(text) => {
3334 let chars = text.chars().count();
3335 Ok(json!({
3336 "action": "clipboard_get",
3337 "kind": "text",
3338 "text": text,
3339 "chars": chars,
3340 }))
3341 }
3342 ClipboardContents::Image {
3343 width,
3344 height,
3345 rgba,
3346 } => {
3347 let frame = Frame::new(width, height, 1.0, rgba)?;
3348 let encoded = image_util::encode_rgba(frame.image(), ImageFormat::Png)?;
3349 let dir = cache::session_dir(context.session.as_deref())?;
3350 let path = dir.join("clipboard.png");
3351 cache::write_secret_bytes(&path, &encoded.bytes)?;
3352 Ok(json!({
3353 "action": "clipboard_get",
3354 "kind": "image",
3355 "path": path_string(&path),
3356 "width": width,
3357 "height": height,
3358 }))
3359 }
3360 }
3361}
3362
3363fn clipboard_set(
3364 backend: &dyn DesktopBackend,
3365 text: Option<String>,
3366 image: Option<String>,
3367) -> Result<serde_json::Value, ComputerError> {
3368 match (text, image) {
3369 (Some(text), None) => {
3370 let chars = text.chars().count();
3371 backend.clipboard_set_text(&text)?;
3372 Ok(json!({
3373 "action": "clipboard_set",
3374 "kind": "text",
3375 "chars": chars,
3376 }))
3377 }
3378 (None, Some(path)) => {
3379 let (width, height, rgba) = decode_clipboard_image(&path)?;
3380 backend.clipboard_set_image(width, height, &rgba)?;
3381 Ok(json!({
3382 "action": "clipboard_set",
3383 "kind": "image",
3384 "width": width,
3385 "height": height,
3386 }))
3387 }
3388 _ => Err(ComputerError::InvalidArguments(
3389 "clipboard set requires text or image".into(),
3390 )),
3391 }
3392}
3393
3394fn decode_clipboard_image(path: &str) -> Result<(u32, u32, Vec<u8>), ComputerError> {
3395 let image = image::ImageReader::open(path)
3396 .map_err(|error| {
3397 ComputerError::InvalidArguments(format!("clipboard image could not be opened: {error}"))
3398 })?
3399 .decode()
3400 .map_err(|error| {
3401 ComputerError::InvalidArguments(format!(
3402 "clipboard image could not be decoded: {error}"
3403 ))
3404 })?
3405 .to_rgba8();
3406 let width = image.width();
3407 let height = image.height();
3408 Ok((width, height, image.into_raw()))
3409}
3410
3411fn scroll(
3412 backend: &dyn DesktopBackend,
3413 context: &CallContext,
3414 point: (Option<i32>, Option<i32>),
3415 delta: (Option<i32>, Option<i32>),
3416 direction: Option<ScrollDirection>,
3417 amount: Option<i32>,
3418) -> Response {
3419 let (x, y) = point;
3420 let (dx, dy) = delta;
3421 if x.is_some() != y.is_some() {
3422 return Response::from_error(ComputerError::InvalidArguments(
3423 "scroll x and y must be set together".into(),
3424 ));
3425 }
3426 let (scroll_dx, scroll_dy) = match resolve_scroll(dx, dy, direction, amount) {
3427 Ok(delta) => delta,
3428 Err(error) => return Response::from_error(error),
3429 };
3430 match run_attempts(
3431 backend,
3432 context,
3433 InputKind::Scroll {
3434 x,
3435 y,
3436 dx: scroll_dx,
3437 dy: scroll_dy,
3438 },
3439 ) {
3440 Ok(report) => {
3441 let body = json!({
3442 "action": "scroll",
3443 "x": x,
3444 "y": y,
3445 "dx": scroll_dx,
3446 "dy": scroll_dy,
3447 });
3448 Response::success(finish_input(scroll_body(body, &report), report, Vec::new()))
3449 }
3450 Err(error) => Response::from_error(error),
3451 }
3452}
3453
3454fn scroll_body(mut body: Value, report: &crate::tier::InputReport) -> Value {
3463 let Some(outcome) = &report.scroll else {
3464 return body;
3465 };
3466 body["via"] = json!(outcome.via.as_str());
3467 body["scrolled"] = json!(outcome.scrolled);
3468 body["before"] = json!(outcome.before);
3469 body["after"] = json!(outcome.after);
3470 body["step"] = json!(outcome.step);
3471 body
3472}
3473
3474fn planned_tiers(
3475 backend: &dyn DesktopBackend,
3476 context: &CallContext,
3477) -> Result<Vec<TierName>, ComputerError> {
3478 let requested = crate::tier::parse_tier(context.tier.as_deref())?;
3479 crate::tier::plan(
3480 requested,
3481 crate::tier::InputClass::Background,
3482 backend.input_support(),
3483 crate::tier::skylight_compiled(),
3484 )
3485}
3486
3487fn run_attempts(
3488 backend: &dyn DesktopBackend,
3489 context: &CallContext,
3490 kind: InputKind,
3491) -> Result<crate::tier::InputReport, ComputerError> {
3492 attempt::run(backend, &attempt_flags(context), kind)
3493}
3494
3495fn attempt_flags(context: &CallContext) -> AttemptFlags {
3496 AttemptFlags {
3497 app: context.app.clone(),
3498 window: context.window.clone(),
3499 tier: context.tier.clone(),
3500 allow_foreground: context.allow_foreground,
3501 require_background: context.require_background,
3502 no_overlay: context.no_overlay,
3503 session: context.session.clone(),
3504 }
3505}
3506
3507fn finish_input(
3508 mut body: Value,
3509 report: crate::tier::InputReport,
3510 mut warnings: Vec<String>,
3511) -> Value {
3512 warnings.extend(report.warnings);
3513 body["tier"] = json!(report.tier);
3514 body["cursor_moved"] = json!(report.cursor_moved);
3515 if let Some(restored) = report.focus_restored {
3516 body["focus_restored"] = json!(restored);
3517 }
3518 if !warnings.is_empty() {
3519 body["warnings"] = json!(warnings);
3520 }
3521 body
3522}
3523
3524pub(crate) fn resolve_scroll(
3525 dx: Option<i32>,
3526 dy: Option<i32>,
3527 direction: Option<ScrollDirection>,
3528 amount: Option<i32>,
3529) -> Result<(i32, i32), ComputerError> {
3530 let has_delta = dx.is_some() || dy.is_some();
3531 let has_direction = direction.is_some() || amount.is_some();
3532 if has_delta && has_direction {
3533 return Err(ComputerError::InvalidArguments(
3534 "use either dx/dy or direction/amount, not both".into(),
3535 ));
3536 }
3537 if let Some(direction) = direction {
3538 let amount = amount.unwrap_or(1);
3539 return Ok(match direction {
3540 ScrollDirection::Up => (0, -amount),
3541 ScrollDirection::Down => (0, amount),
3542 ScrollDirection::Left => (-amount, 0),
3543 ScrollDirection::Right => (amount, 0),
3544 });
3545 }
3546 if amount.is_some() {
3547 return Err(ComputerError::InvalidArguments(
3548 "amount requires direction".into(),
3549 ));
3550 }
3551 if !has_delta {
3552 return Err(ComputerError::InvalidArguments(
3553 "scroll requires dx/dy or direction".into(),
3554 ));
3555 }
3556 Ok((dx.unwrap_or(0), dy.unwrap_or(0)))
3557}
3558
3559fn wait(ms: Option<u64>, seconds: Option<f64>) -> Result<Value, ComputerError> {
3560 let waited = resolve_wait(ms, seconds)?;
3561 thread::sleep(Duration::from_millis(waited));
3562 Ok(json!({"waited_ms": waited}))
3563}
3564
3565pub(crate) fn resolve_wait(ms: Option<u64>, seconds: Option<f64>) -> Result<u64, ComputerError> {
3566 if ms.is_none() && seconds.is_none() {
3567 return Err(ComputerError::InvalidArguments(
3568 "wait requires ms, seconds, or both".into(),
3569 ));
3570 }
3571 let mut total: u128 = u128::from(ms.unwrap_or(0));
3572 if let Some(seconds) = seconds {
3573 if !seconds.is_finite() || seconds < 0.0 {
3574 return Err(ComputerError::InvalidArguments(
3575 "seconds must be a finite number greater than or equal to 0".into(),
3576 ));
3577 }
3578 let extra = (seconds * 1000.0).round();
3579 if extra > u128::MAX as f64 {
3580 return Err(ComputerError::InvalidArguments("wait is too long".into()));
3581 }
3582 total = total.saturating_add(extra as u128);
3583 }
3584 if total > MAX_WAIT_MS {
3585 return Err(ComputerError::InvalidArguments(
3586 "wait must be at most 24 hours".into(),
3587 ));
3588 }
3589 u64::try_from(total).map_err(|_| ComputerError::InvalidArguments("wait is too long".into()))
3590}
3591
3592fn displays(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
3593 let displays = backend.list_displays()?;
3594 Ok(json!({"displays": displays}))
3595}
3596
3597fn display_filter(
3599 backend: &dyn DesktopBackend,
3600 context: &CallContext,
3601) -> Result<Option<crate::isolation::DisplayTarget>, ComputerError> {
3602 match context.display.as_deref() {
3603 None => Ok(None),
3604 Some(spec) => Ok(Some(crate::isolation::resolve(
3605 spec,
3606 &backend.list_displays()?,
3607 )?)),
3608 }
3609}
3610
3611fn isolation() -> Result<Value, ComputerError> {
3613 Ok(crate::isolation::matrix())
3614}
3615
3616fn tree_query(context: &CallContext) -> TreeQuery {
3617 TreeQuery {
3618 app: context.app.clone(),
3619 window: context.window.clone(),
3620 full: context.full,
3621 session: context.session.clone(),
3622 }
3623}
3624
3625fn ocr_action(
3629 backend: &dyn DesktopBackend,
3630 context: &CallContext,
3631 _force: bool,
3632) -> Result<Value, ComputerError> {
3633 let targeted = context.app.is_some() || context.window.is_some();
3634 if !targeted {
3635 return Err(ComputerError::InvalidArguments(
3636 "ocr requires --app or --window".into(),
3637 ));
3638 }
3639 let prepared = latest_prepared_ocr_capture(backend, context)?;
3640 let OcrResult { blocks, note } = backend.recognize_text(&prepared.image, &prepared.map)?;
3641 let redacted = redact_secure_ocr_blocks(blocks, backend, context, &prepared.map)?;
3642 let blocks_json: Vec<Value> = redacted
3643 .iter()
3644 .map(|block| {
3645 let image_rect = crate::vision::ocr::ImageRect {
3646 x: block.x,
3647 y: block.y,
3648 width: block.width,
3649 height: block.height,
3650 };
3651 let screen = image_rect.to_screen(&prepared.map);
3652 let screen_json = match screen {
3653 Some((x, y)) => json!({
3654 "x": x.round() as i64,
3655 "y": y.round() as i64,
3656 }),
3657 None => Value::Null,
3658 };
3659 json!({
3660 "text": block.text,
3661 "x": block.x,
3662 "y": block.y,
3663 "width": block.width,
3664 "height": block.height,
3665 "confidence": block.confidence,
3666 "screen": screen_json,
3667 })
3668 })
3669 .collect();
3670 let mut body = json!({
3671 "action": "ocr",
3672 "blocks": blocks_json,
3673 });
3674 if let Some(note) = note {
3675 body["note"] = json!(note);
3676 }
3677 Ok(body)
3678}
3679
3680fn redact_secure_ocr_blocks(
3685 blocks: Vec<OcrBlock>,
3686 backend: &dyn DesktopBackend,
3687 context: &CallContext,
3688 map: &crate::image_util::ScreenshotMap,
3689) -> Result<Vec<OcrBlock>, ComputerError> {
3690 let secure_frames = backend.secure_ocr_frames(&tree_query(context), map)?;
3695 Ok(crate::vision::redact_blocks(blocks, secure_frames))
3696}
3697
3698struct OcrPreparedCapture {
3700 image: image::RgbaImage,
3701 map: crate::image_util::ScreenshotMap,
3702}
3703
3704fn latest_prepared_ocr_capture(
3708 backend: &dyn DesktopBackend,
3709 context: &CallContext,
3710) -> Result<OcrPreparedCapture, ComputerError> {
3711 let targeted = context.app.is_some() || context.window.is_some();
3712 if targeted {
3713 if let Some(error) = backend.display_block() {
3714 return Err(error);
3715 }
3716 let target = resolve_screenshot_window(backend, context)?;
3717 let captured = backend.capture_window(&target)?;
3718 let prepared = image_util::prepare_shot(
3719 captured.frame.image(),
3720 image_util::ShotFit {
3721 scale: captured.frame.scale(),
3722 origin_x: captured.origin_x,
3723 origin_y: captured.origin_y,
3724 logical_width: captured.logical_width,
3725 logical_height: captured.logical_height,
3726 max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3727 canvas: None,
3728 },
3729 )?;
3730 Ok(OcrPreparedCapture {
3731 image: prepared.image,
3732 map: prepared.map,
3733 })
3734 } else {
3735 let raw = backend.capture_screen(None)?;
3737 let prepared = image_util::prepare_shot(
3738 raw.frame.image(),
3739 image_util::ShotFit {
3740 scale: raw.frame.scale(),
3741 origin_x: raw.origin_x,
3742 origin_y: raw.origin_y,
3743 logical_width: raw.logical_width,
3744 logical_height: raw.logical_height,
3745 max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3746 canvas: None,
3747 },
3748 )?;
3749 Ok(OcrPreparedCapture {
3750 image: prepared.image,
3751 map: prepared.map,
3752 })
3753 }
3754}
3755
3756fn apply_marks(
3762 image: &mut image::RgbaImage,
3763 map: &crate::image_util::ScreenshotMap,
3764 context: &CallContext,
3765 backend: &dyn DesktopBackend,
3766) -> Result<usize, ComputerError> {
3767 let boxes = backend.marks_for_tree(&tree_query(context), map)?;
3768 if boxes.is_empty() {
3769 return Ok(0);
3770 }
3771 let marked = crate::vision::draw_marks(image, &boxes);
3772 *image = marked;
3773 Ok(boxes.len())
3774}
3775
3776fn tree_action(
3777 backend: &dyn DesktopBackend,
3778 context: &CallContext,
3779) -> Result<Value, ComputerError> {
3780 let report = backend.tree(&tree_query(context))?;
3781 let mut body = json!({
3782 "pid": report.pid,
3783 "window_id": report.window_id,
3784 "revision": report.revision,
3785 "source": report.source,
3786 "truncated": report.truncated,
3787 "tree": report.tree,
3788 "baseline": if report.diff.is_some() { "existing" } else { "created" },
3790 });
3791 if let Some(note) = report.note {
3792 body["note"] = json!(note);
3793 }
3794 if let Some(legacy) = report.legacy.as_deref() {
3795 body["accessibility_legacy"] = serde_json::from_str(legacy).unwrap_or(Value::Null);
3796 }
3797 let tree_was_empty = report.full_text.trim().is_empty();
3798 if tree_was_empty {
3799 if context.app.is_some() || context.window.is_some() {
3802 body["ocr_fallback_available"] = json!(true);
3803 }
3804 }
3805 Ok(body)
3806}
3807
3808fn session_end(
3809 backend: &dyn DesktopBackend,
3810 context: &CallContext,
3811) -> Result<Value, ComputerError> {
3812 let restored = backend.end_session(context.session.as_deref())?;
3813 let mut body = json!({
3814 "action": "session_end",
3815 "restored": restored,
3816 });
3817 if let Err(error) = cache::prune_empty_sessions() {
3818 body["warnings"] = json!([format!("session prune failed: {error}")]);
3819 }
3820 Ok(body)
3821}
3822
3823fn list_apps(backend: &dyn DesktopBackend, all: bool) -> Result<Value, ComputerError> {
3824 let apps = backend.list_apps(all)?;
3825 Ok(json!({"apps": apps}))
3826}
3827
3828fn list_windows(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3829 let listed = backend.list_windows()?;
3830 let windows = match app {
3831 None => listed.windows,
3832 Some(app) => {
3833 let selector = parse_app_selector(app)?;
3834 let apps = backend.list_apps(true)?;
3835 let info = resolve_app(&apps, &selector)?;
3836 filter_windows_for_pid(&listed.windows, info.pid)
3837 }
3838 };
3839 let mut body = json!({"windows": windows});
3840 if let Some(note) = listed.note {
3841 body["note"] = json!(note);
3842 }
3843 Ok(body)
3844}
3845
3846fn launch(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3847 let app = require_app(app)?;
3848 let result = backend.launch(app)?;
3849 Ok(json!({
3850 "action": "launch",
3851 "name": result.name,
3852 "bundle_id": result.bundle_id,
3853 "pid": result.pid,
3854 "path": result.path,
3855 }))
3856}
3857
3858fn activate(
3859 backend: &dyn DesktopBackend,
3860 app: Option<&str>,
3861 window: Option<&str>,
3862) -> Result<Value, ComputerError> {
3863 let app = require_app(app)?;
3864 let result = backend.activate(app, window)?;
3865 Ok(json!({
3866 "action": "activate",
3867 "name": result.name,
3868 "bundle_id": result.bundle_id,
3869 "pid": result.pid,
3870 "path": result.path,
3871 "window_id": result.window_id,
3872 "window_title": result.window_title,
3873 }))
3874}
3875
3876fn quit(
3877 backend: &dyn DesktopBackend,
3878 app: Option<&str>,
3879 force: bool,
3880) -> Result<Value, ComputerError> {
3881 let app = require_app(app)?;
3882 let result = backend.quit(app, force)?;
3883 Ok(json!({
3884 "action": "quit",
3885 "name": result.name,
3886 "bundle_id": result.bundle_id,
3887 "pid": result.pid,
3888 "accepted": result.accepted,
3889 "force": force,
3890 }))
3891}
3892
3893fn doctor(
3894 backend: &dyn DesktopBackend,
3895 options: crate::doctor::DoctorOptions,
3896) -> Result<Value, ComputerError> {
3897 let report = backend.doctor(options)?;
3898 serde_json::to_value(report).map_err(|error| {
3899 ComputerError::ExecutionFailed(format!("failed to encode doctor report: {error}"))
3900 })
3901}
3902
3903fn require_app(app: Option<&str>) -> Result<&str, ComputerError> {
3904 match app.map(str::trim).filter(|text| !text.is_empty()) {
3905 Some(app) => Ok(app),
3906 None => Err(ComputerError::InvalidArguments("app is required".into())),
3907 }
3908}
3909
3910#[derive(Debug, Clone, PartialEq)]
3912pub struct Response {
3913 pub ok: bool,
3915 pub id: Option<Value>,
3917 pub error: Option<ErrorBody>,
3919 pub fields: Map<String, Value>,
3921}
3922
3923impl Response {
3924 pub fn success(data: Value) -> Self {
3926 let fields = match data {
3927 Value::Object(map) => map,
3928 other => {
3929 let mut map = Map::new();
3930 map.insert("result".into(), other);
3931 map
3932 }
3933 };
3934 Self {
3935 ok: true,
3936 id: None,
3937 error: None,
3938 fields,
3939 }
3940 }
3941
3942 pub fn failure(code: impl Into<String>, message: impl Into<String>) -> Self {
3944 Self {
3945 ok: false,
3946 id: None,
3947 error: Some(ErrorBody {
3948 code: code.into(),
3949 message: message.into(),
3950 }),
3951 fields: Map::new(),
3952 }
3953 }
3954
3955 pub fn from_error(error: ComputerError) -> Self {
3957 Self::failure(error.code(), error.to_string())
3958 }
3959
3960 pub fn with_id(mut self, id: Option<Value>) -> Self {
3962 self.id = id;
3963 self
3964 }
3965
3966 pub fn to_value(&self) -> Value {
3968 let mut map = Map::new();
3969 map.insert("ok".into(), Value::Bool(self.ok));
3970 if let Some(id) = &self.id {
3971 map.insert("id".into(), id.clone());
3972 }
3973 if let Some(error) = &self.error {
3974 map.insert(
3975 "error".into(),
3976 serde_json::to_value(error).unwrap_or_else(
3977 |_| json!({"code": "execution_failed", "message": "failed to encode error"}),
3978 ),
3979 );
3980 }
3981 for (key, value) in &self.fields {
3982 if key != "ok" && key != "id" && key != "error" {
3983 map.insert(key.clone(), value.clone());
3984 }
3985 }
3986 Value::Object(map)
3987 }
3988
3989 pub fn to_json_line(&self, pretty: bool) -> String {
3991 let value = self.to_value();
3992 let body = if pretty {
3993 serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3994 } else {
3995 serde_json::to_string(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3996 };
3997 format!("{body}\n")
3998 }
3999}
4000
4001impl From<ComputerError> for Response {
4002 fn from(error: ComputerError) -> Self {
4003 Self::from_error(error)
4004 }
4005}
4006
4007impl Serialize for Response {
4008 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
4009 self.to_value().serialize(serializer)
4010 }
4011}
4012
4013impl<'de> Deserialize<'de> for Response {
4014 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
4015 let value = Value::deserialize(deserializer)?;
4016 let Value::Object(mut object) = value else {
4017 return Err(de::Error::custom("response must be a JSON object"));
4018 };
4019 let ok = object
4020 .remove("ok")
4021 .ok_or_else(|| de::Error::missing_field("ok"))?;
4022 let ok = ok
4023 .as_bool()
4024 .ok_or_else(|| de::Error::custom("ok must be a boolean"))?;
4025 let id = object.remove("id");
4026 let error = match object.remove("error") {
4027 Some(value) => Some(serde_json::from_value(value).map_err(de::Error::custom)?),
4028 None => None,
4029 };
4030 Ok(Self {
4031 ok,
4032 id,
4033 error,
4034 fields: object,
4035 })
4036 }
4037}
4038
4039#[cfg(test)]
4040mod tests {
4041 use super::*;
4042 use crate::test_backend::StubBackend;
4043 use std::sync::atomic::Ordering;
4044
4045 #[test]
4046 fn action_roundtrip_and_aliases() {
4047 let click: Action = serde_json::from_str(
4048 r#"{"action":"click","x":10,"y":20,"button":"right","double":true}"#,
4049 )
4050 .unwrap();
4051 match &click {
4052 Action::Click {
4053 x,
4054 y,
4055 button,
4056 double,
4057 count,
4058 } => {
4059 assert_eq!(
4060 (*x, *y, *button, *double, *count),
4061 (Some(10), Some(20), MouseButton::Right, true, 1)
4062 );
4063 }
4064 other => panic!("unexpected {other:?}"),
4065 }
4066 let encoded = serde_json::to_string(&click).unwrap();
4067 let decoded: Action = serde_json::from_str(&encoded).unwrap();
4068 assert_eq!(decoded, click);
4069
4070 let moved: Action = serde_json::from_str(r#"{"action":"move_mouse","x":1,"y":2}"#).unwrap();
4071 assert!(matches!(moved, Action::Move { x: 1, y: 2 }));
4072
4073 let state: Action = serde_json::from_str(r#"{"action":"get-state"}"#).unwrap();
4074 assert!(matches!(state, Action::GetState));
4075 let state: Action = serde_json::from_str(r#"{"action":"get_state"}"#).unwrap();
4076 assert!(matches!(state, Action::GetState));
4077
4078 let shot: Action = serde_json::from_str(
4079 r#"{"action":"screenshot","region":"0,0,8,9","format":"jpeg","max_width":640,"base64":true}"#,
4080 )
4081 .unwrap();
4082 match shot {
4083 Action::Screenshot {
4084 region,
4085 format,
4086 max_width,
4087 base64,
4088 ..
4089 } => {
4090 assert_eq!(region.unwrap().width, 8);
4091 assert_eq!(format, Some(ImageFormat::Jpeg));
4092 assert_eq!(max_width, Some(640));
4093 assert!(base64);
4094 }
4095 other => panic!("unexpected {other:?}"),
4096 }
4097 let shot: Action = serde_json::from_str(
4098 r#"{"action":"screenshot","region":{"x":0,"y":1,"width":2,"height":3}}"#,
4099 )
4100 .unwrap();
4101 match shot {
4102 Action::Screenshot { region, .. } => {
4103 assert_eq!(region.unwrap().height, 3);
4104 }
4105 other => panic!("unexpected {other:?}"),
4106 }
4107
4108 assert!(serde_json::from_str::<Action>(r#"{"action":"fly"}"#).is_err());
4109 let bare: Action = serde_json::from_str(r#"{"action":"click"}"#).unwrap();
4110 assert!(matches!(
4111 bare,
4112 Action::Click {
4113 x: None,
4114 y: None,
4115 ..
4116 }
4117 ));
4118 let press: Action = serde_json::from_str(r#"{"action":"press"}"#).unwrap();
4119 assert!(matches!(
4120 press,
4121 Action::Press {
4122 count: 1,
4123 button: MouseButton::Left
4124 }
4125 ));
4126 let renamed: Action =
4127 serde_json::from_str(r#"{"action":"set-value","value":"hi"}"#).unwrap();
4128 assert!(matches!(renamed, Action::SetValue { .. }));
4129 let scrolled: Action = serde_json::from_str(r#"{"action":"scroll-to-visible"}"#).unwrap();
4130 assert!(matches!(scrolled, Action::ScrollTo));
4131 }
4132
4133 #[test]
4134 fn response_shape_and_roundtrip() {
4135 let ok = Response::success(json!({"waited_ms": 1})).with_id(Some(json!(7)));
4136 let value = ok.to_value();
4137 assert_eq!(value["ok"], true);
4138 assert!(value.get("error").is_none());
4139 assert_eq!(value["waited_ms"], 1);
4140 assert_eq!(value["id"], 7);
4141 let text = serde_json::to_string(&ok).unwrap();
4142 let back: Response = serde_json::from_str(&text).unwrap();
4143 assert_eq!(back.to_value(), value);
4144
4145 let err = Response::failure("invalid_arguments", "bad region");
4146 let value = err.to_value();
4147 assert_eq!(value["ok"], false);
4148 assert_eq!(value["error"]["code"], "invalid_arguments");
4149 assert_eq!(value["error"]["message"], "bad region");
4150 assert!(value.get("waited_ms").is_none());
4151 let line = err.to_json_line(false);
4152 assert!(line.ends_with('\n'));
4153 assert!(!line[..line.len() - 1].contains('\n'));
4154 }
4155
4156 #[test]
4157 fn scroll_and_wait_resolution() {
4158 assert_eq!(
4159 resolve_scroll(Some(1), Some(-2), None, None).unwrap(),
4160 (1, -2)
4161 );
4162 assert_eq!(
4163 resolve_scroll(None, None, Some(ScrollDirection::Up), Some(3)).unwrap(),
4164 (0, -3)
4165 );
4166 assert_eq!(
4167 resolve_scroll(None, None, Some(ScrollDirection::Right), None).unwrap(),
4168 (1, 0)
4169 );
4170 assert_eq!(
4171 resolve_scroll(None, None, Some(ScrollDirection::Left), Some(4)).unwrap(),
4172 (-4, 0)
4173 );
4174 assert_eq!(
4175 resolve_scroll(None, None, Some(ScrollDirection::Down), Some(2)).unwrap(),
4176 (0, 2)
4177 );
4178 assert!(resolve_scroll(Some(1), None, Some(ScrollDirection::Up), None).is_err());
4179 assert!(resolve_scroll(None, None, None, Some(2)).is_err());
4180 assert!(resolve_scroll(None, None, None, None).is_err());
4181
4182 assert_eq!(resolve_wait(Some(10), None).unwrap(), 10);
4183 assert_eq!(resolve_wait(Some(10), Some(0.5)).unwrap(), 510);
4184 assert_eq!(resolve_wait(None, Some(1.0)).unwrap(), 1000);
4185 assert!(resolve_wait(None, None).is_err());
4186 assert!(resolve_wait(None, Some(-1.0)).is_err());
4187 assert!(resolve_wait(None, Some(f64::NAN)).is_err());
4188 }
4189
4190 #[test]
4191 fn json_line_dispatch_and_stub_actions() {
4192 let wait = handle_request_line(r#"{"id":1,"action":"wait","ms":0}"#);
4193 assert!(wait.ok);
4194 assert_eq!(wait.to_value()["id"], 1);
4195 assert_eq!(wait.to_value()["waited_ms"], 0);
4196
4197 let bad = handle_request_line("not-json");
4198 assert!(!bad.ok);
4199 assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
4200
4201 let missing = handle_request_line(r#"{"id":"x","action":"nope"}"#);
4202 assert!(!missing.ok);
4203 assert_eq!(missing.to_value()["id"], "x");
4204
4205 let request = parse_request(json!({"id": 3, "action": "displays"})).unwrap();
4206 assert_eq!(request.id, Some(json!(3)));
4207 assert!(matches!(request.action, Action::Displays));
4208
4209 let backend = StubBackend::default();
4210 let response = dispatch_with(
4211 &backend,
4212 Action::Click {
4213 x: Some(3),
4214 y: Some(4),
4215 button: MouseButton::Left,
4216 double: true,
4217 count: 1,
4218 },
4219 );
4220 assert!(response.ok);
4221 assert_eq!(response.to_value()["action"], "double_click");
4222 assert_eq!(backend.clicks.load(Ordering::SeqCst), 2);
4223
4224 let response = dispatch_with(&backend, Action::Move { x: 8, y: 9 });
4225 assert!(response.ok);
4226 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4227
4228 let response = dispatch_with(&backend, Action::Type { text: "hi".into() });
4229 assert_eq!(response.to_value()["chars"], 2);
4230 assert_eq!(backend.typed.lock().unwrap().as_slice(), ["hi"]);
4231
4232 let response = dispatch_with(
4233 &backend,
4234 Action::Key {
4235 keys: "cmd+shift+4".into(),
4236 },
4237 );
4238 assert!(response.ok);
4239 assert_eq!(response.to_value()["key"], "4");
4240 let keys = backend.keys.lock().unwrap();
4241 assert_eq!(keys[0].1, "4");
4242 assert_eq!(keys[0].0, vec![Modifier::Command, Modifier::Shift]);
4243 drop(keys);
4244
4245 let response = dispatch_with(
4246 &backend,
4247 Action::Key {
4248 keys: "not+a+key".into(),
4249 },
4250 );
4251 assert!(!response.ok);
4252
4253 let response = dispatch_with(
4254 &backend,
4255 Action::Scroll {
4256 x: Some(1),
4257 y: Some(2),
4258 dx: None,
4259 dy: None,
4260 direction: Some(ScrollDirection::Down),
4261 amount: Some(3),
4262 },
4263 );
4264 assert!(response.ok);
4265 assert_eq!(backend.scrolls.lock().unwrap().as_slice(), [(0, 3)]);
4266 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4267 assert_eq!(response.to_value()["tier"], "event");
4268 assert_eq!(response.to_value()["cursor_moved"], false);
4269
4270 let response = dispatch_with(&backend, Action::Displays);
4271 assert_eq!(response.to_value()["displays"][0]["name"], "Stub");
4272
4273 let response = dispatch_with(&backend, Action::GetState);
4274 assert_eq!(response.to_value()["cursor"]["x"], 4);
4275 assert!(response.ok);
4276
4277 let _cache = crate::cache::CacheEnv::acquire();
4278 let shot = dispatch_with(
4279 &backend,
4280 Action::Screenshot {
4281 output: None,
4282 region: None,
4283 format: Some(ImageFormat::Png),
4284 max_width: Some(1),
4285 canvas: None,
4286 base64: true,
4287 marks: false,
4288 },
4289 );
4290 assert!(shot.ok, "{shot:?}");
4291 let value = shot.to_value();
4292 assert_eq!(value["width"], 1);
4293 assert_eq!(value["height"], 1);
4294 assert_eq!(value["scale"], 2.0);
4295 assert_eq!(value["format"], "png");
4296 let path = value["path"].as_str().unwrap();
4297 let bytes = fs::read(path).unwrap();
4298 assert_eq!(
4299 &bytes[..8],
4300 &[0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n']
4301 );
4302 assert!(value["data"].as_str().unwrap().starts_with("iVBOR"));
4303 assert_eq!(value["capture_source"], "stub");
4304 assert_eq!(value["origin"]["x"], 0);
4305 assert_eq!(value["origin"]["y"], 0);
4306 assert_eq!(value["fit"], "width");
4307 assert!(value["map"]["image_to_screen"].is_string());
4308 let _ = fs::remove_file(path);
4309 if let Ok(dir) = crate::cache::session_dir(None) {
4310 let _ = fs::remove_file(dir.join("screenshot-map.json"));
4311 }
4312
4313 let failed = StubBackend {
4314 fail_click: true,
4315 ..StubBackend::default()
4316 };
4317 let response = dispatch_with(
4318 &failed,
4319 Action::Click {
4320 x: Some(0),
4321 y: Some(0),
4322 button: MouseButton::Left,
4323 double: false,
4324 count: 1,
4325 },
4326 );
4327 assert!(!response.ok);
4328 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4329 }
4330
4331 #[test]
4332 fn dispatch_wait_does_not_need_a_display() {
4333 let response = dispatch(Action::Wait {
4334 ms: Some(0),
4335 seconds: None,
4336 });
4337 assert!(response.ok);
4338 assert_eq!(response.to_value()["waited_ms"], 0);
4339 }
4340
4341 fn shot(max_width: Option<u32>, canvas: Option<&str>) -> Action {
4342 Action::Screenshot {
4343 output: None,
4344 region: None,
4345 format: Some(ImageFormat::Png),
4346 max_width,
4347 canvas: canvas.map(str::to_string),
4348 base64: false,
4349 marks: false,
4350 }
4351 }
4352
4353 #[test]
4354 fn dispatch_applies_default_max_width_and_keeps_explicit_zero() {
4355 let _cache = crate::cache::CacheEnv::acquire();
4356 let backend = StubBackend {
4357 capture_pixels: Some((1600, 1000)),
4358 ..StubBackend::default()
4359 };
4360 let response = dispatch_with_context(
4361 &backend,
4362 shot(None, None),
4363 &CallContext {
4364 session: Some("p4-default-width".into()),
4365 ..CallContext::default()
4366 },
4367 );
4368 assert!(response.ok, "{response:?}");
4369 let value = response.to_value();
4370 assert_eq!(value["width"], 1440);
4371 assert_eq!(value["height"], 900);
4372 assert_eq!(value["scale"], 2.0);
4373 assert_eq!(value["logical_width"], 800);
4374 assert_eq!(value["logical_height"], 500);
4375 assert_eq!(value["fit"], "width");
4376 assert_eq!(value["capture_source"], "stub");
4377 assert_eq!(value["content_width"], 1440);
4378 let path = value["path"].as_str().unwrap();
4379 let _ = fs::remove_file(path);
4380 let map = crate::cache::load_screenshot_map(Some("p4-default-width"))
4381 .unwrap()
4382 .expect("map");
4383 assert_eq!(map.width, 1440);
4384 let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-default-width")).unwrap());
4385
4386 let full = dispatch_with_context(
4387 &backend,
4388 shot(Some(0), None),
4389 &CallContext {
4390 session: Some("p4-zero-width".into()),
4391 ..CallContext::default()
4392 },
4393 );
4394 assert!(full.ok, "{full:?}");
4395 assert_eq!(full.to_value()["width"], 1600);
4396 assert_eq!(full.to_value()["height"], 1000);
4397 assert_eq!(full.to_value()["fit"], "none");
4398 let path = full.to_value()["path"].as_str().unwrap().to_string();
4399 let _ = fs::remove_file(path);
4400 let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-zero-width")).unwrap());
4401 }
4402
4403 #[test]
4404 fn dispatch_window_capture_records_origin_and_faults() {
4405 let _cache = crate::cache::CacheEnv::acquire();
4406 let backend = StubBackend::default();
4407 let response = dispatch_with_context(
4408 &backend,
4409 shot(Some(0), None),
4410 &CallContext {
4411 app: Some("Notes".into()),
4412 window: Some("0".into()),
4413 session: Some("p4-window".into()),
4414 ..CallContext::default()
4415 },
4416 );
4417 assert!(response.ok, "{response:?}");
4418 let value = response.to_value();
4419 assert_eq!(value["capture_source"], "stub");
4420 assert_eq!(value["origin"]["x"], 10);
4421 assert_eq!(value["origin"]["y"], 20);
4422 assert_eq!(value["logical_width"], 400);
4423 assert_eq!(value["logical_height"], 300);
4424 assert_eq!(value["width"], 800);
4425 assert_eq!(value["fit"], "none");
4426 let path = value["path"].as_str().unwrap();
4427 let _ = fs::remove_file(path);
4428 let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-window")).unwrap());
4429
4430 let denied = StubBackend {
4431 capture_fault: crate::test_backend::StubCaptureFault::Permission,
4432 ..StubBackend::default()
4433 };
4434 let response = dispatch_with_context(
4435 &denied,
4436 shot(Some(0), None),
4437 &CallContext {
4438 app: Some("Notes".into()),
4439 ..CallContext::default()
4440 },
4441 );
4442 assert!(!response.ok);
4443 assert_eq!(response.to_value()["error"]["code"], "permission_required");
4444 assert!(response.to_value()["error"]["message"]
4445 .as_str()
4446 .unwrap()
4447 .contains("Screen Recording"));
4448
4449 let missing = StubBackend {
4450 capture_fault: crate::test_backend::StubCaptureFault::MissingWindow,
4451 ..StubBackend::default()
4452 };
4453 let response = dispatch_with_context(
4454 &missing,
4455 shot(Some(0), None),
4456 &CallContext {
4457 window: Some("id:100".into()),
4458 ..CallContext::default()
4459 },
4460 );
4461 assert!(!response.ok);
4462 assert_eq!(response.to_value()["error"]["code"], "window_not_found");
4463
4464 let both = dispatch_with_context(
4465 &backend,
4466 Action::Screenshot {
4467 output: None,
4468 region: Some(Region::new(0, 0, 8, 9).unwrap()),
4469 format: None,
4470 max_width: Some(0),
4471 canvas: None,
4472 base64: false,
4473 marks: false,
4474 },
4475 &CallContext {
4476 app: Some("Notes".into()),
4477 ..CallContext::default()
4478 },
4479 );
4480 assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
4481
4482 let locked = StubBackend {
4483 block_display: true,
4484 windows: Vec::new(),
4485 ..StubBackend::default()
4486 };
4487 let response = dispatch_with_context(
4488 &locked,
4489 shot(Some(0), None),
4490 &CallContext {
4491 app: Some("Notes".into()),
4492 ..CallContext::default()
4493 },
4494 );
4495 assert_eq!(response.to_value()["error"]["code"], "display_unavailable");
4496 assert!(response.to_value()["error"]["message"]
4497 .as_str()
4498 .unwrap()
4499 .contains("locked"));
4500 }
4501
4502 #[test]
4503 fn shared_flags_are_stripped_before_the_action_decodes() {
4504 let request = parse_request(json!({
4505 "id": 4,
4506 "action": "list_windows",
4507 "app": "Foo \"Bar\"",
4508 "window": 0,
4509 "element": "3",
4510 "diff": true,
4511 "full": false,
4512 "return_state": true,
4513 "require_visible": true
4514 }))
4515 .unwrap();
4516 assert_eq!(request.id, Some(json!(4)));
4517 assert!(matches!(request.action, Action::ListWindows));
4518 assert_eq!(request.context.app.as_deref(), Some("Foo \"Bar\""));
4519 assert_eq!(request.context.window.as_deref(), Some("0"));
4520 assert_eq!(request.context.element, Some(3));
4521 assert!(request.context.diff);
4522 assert!(!request.context.full);
4523 assert!(request.context.return_state);
4524 assert!(request.context.require_visible);
4525 }
4526
4527 fn ctx(app: &str) -> CallContext {
4528 CallContext {
4529 app: Some(app.to_string()),
4530 ..CallContext::default()
4531 }
4532 }
4533
4534 #[test]
4535 fn stub_inventory_launch_and_doctor() {
4536 let backend = StubBackend::default();
4537
4538 let response = dispatch_with(&backend, Action::ListApps { all: false });
4539 assert!(response.ok);
4540 let value = response.to_value();
4541 let apps = value["apps"].as_array().unwrap();
4542 assert!(apps.iter().any(|app| app["name"] == "Notes"));
4543 assert!(apps.iter().all(|app| app["name"] != "Widget"));
4544
4545 let response = dispatch_with(&backend, Action::ListApps { all: true });
4546 let value = response.to_value();
4547 assert!(value["apps"]
4548 .as_array()
4549 .unwrap()
4550 .iter()
4551 .any(|app| app["pid"] == 77));
4552
4553 let response = dispatch_with_context(&backend, Action::ListWindows, &ctx("Notes"));
4554 assert!(response.ok, "{response:?}");
4555 let value = response.to_value();
4556 let windows = &value["windows"];
4557 assert_eq!(windows.as_array().unwrap().len(), 2);
4558 assert_eq!(windows[0]["z"], 0);
4559 assert_eq!(windows[0]["id"], 100);
4560 assert_eq!(windows[1]["z"], 1);
4561 assert_eq!(windows[1]["title"], "Welcome");
4562 assert_eq!(windows[0]["ax_matched"], true);
4563 assert!(value.get("note").is_none());
4564
4565 let response = dispatch_with_context(&backend, Action::Launch, &ctx("Notes"));
4566 assert!(response.ok, "{response:?}");
4567 assert_eq!(response.to_value()["pid"], 10);
4568 assert_eq!(response.to_value()["action"], "launch");
4569 assert_eq!(backend.launched.lock().unwrap().as_slice(), ["Notes"]);
4570
4571 let response = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
4572 assert!(!response.ok);
4573 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4574 assert_eq!(backend.launched.lock().unwrap().len(), 1);
4575
4576 let response = dispatch_with_context(&backend, Action::Activate, &ctx("10"));
4577 assert!(response.ok, "{response:?}");
4578 assert!(response.to_value()["window_id"].is_null());
4579 assert_eq!(backend.activated.lock().unwrap().len(), 1);
4580
4581 let mut context = ctx("Notes");
4582 context.window = Some("Welcome".into());
4583 let response = dispatch_with_context(&backend, Action::Activate, &context);
4584 assert_eq!(response.to_value()["window_id"], 101);
4585 assert_eq!(response.to_value()["window_title"], "Welcome");
4586
4587 let response =
4588 dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("Terminal"));
4589 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4590 assert!(backend.quit.lock().unwrap().is_empty());
4591
4592 let response = dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("412"));
4593 assert_eq!(response.to_value()["error"]["code"], "confirm_required");
4594 assert!(backend.quit.lock().unwrap().is_empty());
4595
4596 let response = dispatch_with(&backend, Action::Launch);
4597 assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4598
4599 let response = dispatch_with(
4600 &backend,
4601 Action::Doctor {
4602 prompt: false,
4603 open_settings: false,
4604 x11: false,
4605 },
4606 );
4607 let value = response.to_value();
4608 assert_eq!(value["ok"], true);
4609 assert_eq!(value["permissions_ok"], false);
4610 assert_eq!(value["accessibility"], true);
4611 assert_eq!(value["screen_capture"], false);
4612 assert_eq!(value["screen_recording"], false);
4613 assert!(value["input_monitor"].is_null());
4614 assert_eq!(value["missing"][0], "screen_recording");
4615 assert_eq!(value["responsible_app"]["name"], "Terminal");
4616 assert!(value["fixes"][0]
4617 .as_str()
4618 .unwrap()
4619 .contains("Screen Recording"));
4620 assert_eq!(value["fix"][0]["target"], "Terminal");
4621
4622 let prompted: Action =
4623 serde_json::from_str(r#"{"action":"doctor","request":true}"#).unwrap();
4624 assert!(matches!(
4625 prompted,
4626 Action::Doctor {
4627 prompt: true,
4628 open_settings: false,
4629 ..
4630 }
4631 ));
4632 let plain: Action = serde_json::from_str(r#"{"action":"doctor"}"#).unwrap();
4633 assert!(matches!(
4634 plain,
4635 Action::Doctor {
4636 prompt: false,
4637 open_settings: false,
4638 ..
4639 }
4640 ));
4641
4642 let response = dispatch_with(&backend, Action::GetState);
4643 assert!(response.ok);
4644 let value = response.to_value();
4645 assert_eq!(value["pid"], 10);
4646 assert_eq!(value["source"], "stub");
4647 assert!(value["tree"].as_str().unwrap().contains("AXWindow"));
4648 assert!(value["accessibility_legacy"].is_null());
4649
4650 let response = dispatch_with_context(&backend, Action::GetState, &ctx("412"));
4651 assert_eq!(response.to_value()["pid"], 412);
4652 assert_eq!(response.to_value()["source"], "stub");
4653 assert!(response.to_value()["tree"]
4654 .as_str()
4655 .unwrap()
4656 .contains("AXWindow"));
4657 let seen = backend.accessibility_apps.lock().unwrap().clone();
4658 assert_eq!(seen.last().and_then(|name| name.as_deref()), Some("412"));
4659
4660 backend.baselines.lock().unwrap().clear();
4663 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4664 assert!(first.ok, "{first:?}");
4665 let full = first.to_value()["tree"].as_str().unwrap().to_string();
4666 assert!(full.contains("value=\"a\""), "{full}");
4667 assert_eq!(first.to_value()["source"], "stub");
4668 *backend.stub_value.lock().unwrap() = "ab".into();
4669 let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4670 let diff_text = diff.to_value()["tree"].as_str().unwrap().to_string();
4671 assert!(
4672 diff_text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4673 "{diff_text}"
4674 );
4675 let mut full_context = ctx("Notes");
4676 full_context.full = true;
4677 let replaced = dispatch_with_context(&backend, Action::Tree, &full_context);
4678 let replaced_text = replaced.to_value()["tree"].as_str().unwrap().to_string();
4679 assert!(replaced_text.contains("value=\"ab\""), "{replaced_text}");
4680 assert!(!replaced_text.contains("~ [1]"));
4681 let after = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4682 assert_eq!(after.to_value()["tree"].as_str().unwrap(), "");
4683
4684 let _cache = crate::cache::CacheEnv::acquire();
4685 let ended = dispatch_with(&backend, Action::SessionEnd);
4686 assert!(ended.ok, "{ended:?}");
4687 assert_eq!(ended.to_value()["action"], "session_end");
4688 assert!(ended.to_value()["restored"].as_array().unwrap().is_empty());
4689 }
4690
4691 #[test]
4692 fn stub_refuses_privacy_window_and_self_quit() {
4693 let mut backend = StubBackend::default();
4694 backend.apps.push(crate::target::AppInfo {
4695 name: "System Settings".into(),
4696 bundle_id: Some("com.apple.systempreferences".into()),
4697 pid: 50,
4698 path: Some(
4699 "/System/Applications/System Settings.app/Contents/MacOS/System Settings".into(),
4700 ),
4701 active: false,
4702 hidden: false,
4703 activation_policy: crate::target::ActivationPolicy::Regular,
4704 });
4705 backend.windows.push(crate::target::WindowInfo {
4706 id: 9,
4707 pid: 50,
4708 app: "System Settings".into(),
4709 title: "Privacy & Security".into(),
4710 z: 4,
4711 bounds: crate::target::WindowBounds {
4712 x: 0,
4713 y: 0,
4714 width: 10,
4715 height: 10,
4716 },
4717 layer: 0,
4718 on_screen: true,
4719 ax_matched: false,
4720 });
4721 let mut context = ctx("com.apple.systempreferences");
4722 context.window = Some("Privacy".into());
4723 let response = dispatch_with_context(&backend, Action::Activate, &context);
4724 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4725 assert!(backend.activated.lock().unwrap().is_empty());
4726
4727 let response = dispatch_with_context(&backend, Action::Activate, &ctx("System Settings"));
4728 assert!(response.ok, "{response:?}");
4729
4730 let self_pid = std::process::id() as i32;
4731 backend.apps.retain(|app| app.pid != self_pid);
4732 backend.apps.push(crate::target::AppInfo {
4733 name: "computer-use".into(),
4734 bundle_id: None,
4735 pid: self_pid,
4736 path: None,
4737 active: false,
4738 hidden: false,
4739 activation_policy: crate::target::ActivationPolicy::Regular,
4740 });
4741 let response = dispatch_with_context(
4742 &backend,
4743 Action::Quit { force: false },
4744 &ctx(&self_pid.to_string()),
4745 );
4746 assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4747 }
4748
4749 fn notes_button() -> crate::tree::UiNode {
4750 let mut button = crate::tree::UiNode::bare("AXButton");
4751 button.title = Some("Save".into());
4752 button.actions = vec!["AXPress".into(), "AXShowMenu".into()];
4753 button.frame = Some(crate::tree::NodeFrame {
4754 x: 12.0,
4755 y: 10.0,
4756 width: 80.0,
4757 height: 24.0,
4758 });
4759 let mut root = crate::tree::UiNode::bare("AXWindow");
4760 root.title = Some("Notes".into());
4761 root.frame = Some(crate::tree::NodeFrame {
4762 x: 0.0,
4763 y: 0.0,
4764 width: 800.0,
4765 height: 600.0,
4766 });
4767 root.children = vec![button];
4768 root
4769 }
4770
4771 fn element_context(action_element: u32) -> CallContext {
4772 CallContext {
4773 app: Some("Notes".into()),
4774 element: Some(action_element),
4775 ..CallContext::default()
4776 }
4777 }
4778
4779 #[test]
4780 fn element_json_shape_revalidation_and_hit_test() {
4781 let backend = StubBackend::default();
4782 backend.seed_element(notes_button());
4783 let request = parse_request(json!({
4784 "action": "press",
4785 "app": "Notes",
4786 "element": 1,
4787 "count": 1
4788 }))
4789 .unwrap();
4790 let response = dispatch_with_context(&backend, request.action, &request.context);
4791 assert!(response.ok, "{response:?}");
4792 let value = response.to_value();
4793 assert_eq!(value["action"], "press");
4794 assert_eq!(value["element"], 1);
4795 assert_eq!(value["tier"], "ax");
4796 assert_eq!(value["count"], 1);
4797 assert!(value["revision"].is_number());
4798 assert!(value.get("warnings").is_none());
4799
4800 let request = parse_request(json!({
4801 "action": "perform-action",
4802 "app": "Notes",
4803 "element": 1,
4804 "name": "AXShowMenu"
4805 }))
4806 .unwrap();
4807 let response = dispatch_with_context(&backend, request.action, &request.context);
4808 assert!(response.ok, "{response:?}");
4809 assert_eq!(response.to_value()["action"], "perform_action");
4810 assert_eq!(response.to_value()["name"], "AXShowMenu");
4811 assert_eq!(response.to_value()["tier"], "ax");
4812
4813 let before = backend.performs.lock().unwrap().len();
4814 let request = parse_request(json!({
4815 "action": "perform_action",
4816 "app": "Notes",
4817 "element": 1,
4818 "name": "AXDelete"
4819 }))
4820 .unwrap();
4821 let response = dispatch_with_context(&backend, request.action, &request.context);
4822 assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4823 assert_eq!(backend.performs.lock().unwrap().len(), before);
4824
4825 let request = parse_request(json!({
4826 "action": "set_value",
4827 "app": "Notes",
4828 "element": 1,
4829 "value": "hello"
4830 }))
4831 .unwrap();
4832 let response = dispatch_with_context(&backend, request.action, &request.context);
4833 assert!(response.ok, "{response:?}");
4834 assert_eq!(response.to_value()["action"], "set_value");
4835
4836 let request = parse_request(json!({
4837 "action": "select_text",
4838 "app": "Notes",
4839 "element": 1,
4840 "start": 0,
4841 "end": 5
4842 }))
4843 .unwrap();
4844 let response = dispatch_with_context(&backend, request.action, &request.context);
4845 assert!(response.ok, "{response:?}");
4846 assert_eq!(response.to_value()["action"], "select_text");
4847
4848 let request = parse_request(json!({
4849 "action": "scroll_to_visible",
4850 "app": "Notes",
4851 "element": 1
4852 }))
4853 .unwrap();
4854 assert!(matches!(request.action, Action::ScrollTo));
4855 let response = dispatch_with_context(&backend, request.action, &request.context);
4856 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4857 assert!(response.to_value()["error"]["message"]
4858 .as_str()
4859 .unwrap()
4860 .contains("AXScrollToVisible"));
4861
4862 let clicks = backend.clicks.load(Ordering::SeqCst);
4863 let request = parse_request(json!({
4864 "action": "click",
4865 "app": "Notes",
4866 "element": 1,
4867 "double": true
4868 }))
4869 .unwrap();
4870 let response = dispatch_with_context(&backend, request.action, &request.context);
4871 assert!(response.ok, "{response:?}");
4872 assert_eq!(response.to_value()["action"], "press");
4873 assert_eq!(response.to_value()["count"], 2);
4874 assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks);
4875
4876 *backend.hit.lock().unwrap() = Some(crate::element::HitInfo {
4877 role: "AXButton".into(),
4878 title: Some("Save".into()),
4879 });
4880 let response = dispatch_with(
4881 &backend,
4882 Action::Click {
4883 x: Some(12),
4884 y: Some(10),
4885 button: MouseButton::Left,
4886 double: false,
4887 count: 1,
4888 },
4889 );
4890 assert!(response.ok, "{response:?}");
4891 assert_eq!(response.to_value()["hit"]["role"], "AXButton");
4892 assert_eq!(response.to_value()["hit"]["title"], "Save");
4893 assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks + 1);
4894
4895 let mut changed = notes_button();
4896 changed.children[0].role = "AXTextArea".into();
4897 backend.set_live(changed);
4898 let response = dispatch_with_context(&backend, Action::Focus, &element_context(1));
4899 assert_eq!(response.to_value()["error"]["code"], "stale_element");
4900 assert!(response.to_value()["tree"]
4901 .as_str()
4902 .unwrap()
4903 .contains("AXTextArea"));
4904
4905 let empty = StubBackend::default();
4906 let response = dispatch_with_context(&empty, Action::Focus, &element_context(0));
4907 assert_eq!(response.to_value()["error"]["code"], "element_not_found");
4908 assert!(response.to_value()["error"]["message"]
4909 .as_str()
4910 .unwrap()
4911 .contains("refresh indices"));
4912
4913 let bare = dispatch_with(
4914 &empty,
4915 Action::Click {
4916 x: None,
4917 y: None,
4918 button: MouseButton::Left,
4919 double: false,
4920 count: 1,
4921 },
4922 );
4923 assert_eq!(bare.to_value()["error"]["code"], "invalid_arguments");
4924 }
4925
4926 #[test]
4927 fn element_action_does_not_replace_the_tree_diff_baseline() {
4928 let backend = StubBackend::default();
4929 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4930 assert!(first.ok, "{first:?}");
4931 let key = (String::new(), 10, 100);
4932 let baseline = backend.baselines.lock().unwrap().get(&key).cloned();
4933 let response = dispatch_with_context(&backend, Action::Focus, &element_context(0));
4934 assert!(response.ok, "{response:?}");
4935 assert_eq!(
4936 backend.baselines.lock().unwrap().get(&key),
4937 baseline.as_ref()
4938 );
4939 *backend.stub_value.lock().unwrap() = "ab".into();
4940 let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4941 let text = diff.to_value()["tree"].as_str().unwrap().to_string();
4942 assert!(
4943 text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4944 "{text}"
4945 );
4946 }
4947
4948 fn sample_map() -> crate::image_util::ScreenshotMap {
4949 crate::image_util::ScreenshotMap {
4950 width: 1440,
4951 height: 900,
4952 scale: 2.0,
4953 logical_width: 800,
4954 logical_height: 500,
4955 origin: crate::image_util::MapPoint { x: 120, y: 40 },
4956 fit: "width".into(),
4957 pad: crate::image_util::MapPoint { x: 0, y: 0 },
4958 content_width: 1440,
4959 content_height: 900,
4960 map: crate::image_util::MapFormulas {
4961 image_to_screen:
4962 "screen = origin + (image - pad) * logical_size / image_content_size".into(),
4963 screen_to_image:
4964 "image = pad + (screen - origin) * image_content_size / logical_size".into(),
4965 },
4966 }
4967 }
4968
4969 #[test]
4970 fn secure_ocr_frames_receives_the_map_of_the_ocr_capture() {
4971 let cache = crate::cache::CacheEnv::acquire();
4975 assert!(cache.dir().is_dir());
4976 let backend = StubBackend::default();
4977 *backend.secure_frames.lock().unwrap() = Some(vec![crate::vision::ocr::ImageRect {
4978 x: 10,
4979 y: 20,
4980 width: 100,
4981 height: 40,
4982 }]);
4983 let context = CallContext {
4984 app: Some("Finder".into()),
4985 session: Some("p12b-redact".into()),
4986 ..CallContext::default()
4987 };
4988 let blocks = vec![OcrBlock {
4989 text: "hunter2".into(),
4990 x: 20,
4991 y: 30,
4992 width: 30,
4993 height: 10,
4994 confidence: 0.9,
4995 }];
4996 let map = sample_map();
4997 let kept = redact_secure_ocr_blocks(blocks.clone(), &backend, &context, &map).unwrap();
4998 assert!(kept.is_empty(), "the block sits inside the secure frame");
4999
5000 let seen = backend
5001 .secure_frames_map
5002 .lock()
5003 .unwrap()
5004 .clone()
5005 .expect("secure_ocr_frames was called");
5006 assert_eq!(seen.origin, map.origin);
5007 assert_eq!(seen.logical_width, map.logical_width);
5008 assert_eq!(seen.content_width, map.content_width);
5009
5010 let outside = vec![OcrBlock {
5011 text: "visible".into(),
5012 x: 900,
5013 y: 700,
5014 width: 30,
5015 height: 10,
5016 confidence: 0.9,
5017 }];
5018 let kept = redact_secure_ocr_blocks(outside, &backend, &context, &map).unwrap();
5019 assert_eq!(kept.len(), 1);
5020 }
5021
5022 #[test]
5026 fn marks_change_the_pixels_the_screenshot_writes() {
5027 let cache = crate::cache::CacheEnv::acquire();
5028 let backend = StubBackend::default();
5029 let context = CallContext {
5030 app: Some("Notes".into()),
5031 session: Some("marks".into()),
5032 ..CallContext::default()
5033 };
5034 let plain = cache.dir().join("plain.png");
5035 let marked = cache.dir().join("marked.png");
5036
5037 let shot = |path: std::path::PathBuf, marks: bool| {
5038 screenshot(
5039 &backend,
5040 &context,
5041 ScreenshotArgs {
5042 output: Some(path.to_string_lossy().into_owned()),
5043 region: None,
5044 format: Some(ImageFormat::Png),
5045 max_width: None,
5046 canvas: None,
5047 include_base64: false,
5048 window_crop: false,
5049 marks,
5050 },
5051 )
5052 };
5053
5054 let clean = shot(plain.clone(), false).expect("plain screenshot");
5055 let marked_body = shot(marked.clone(), true).expect("marked screenshot");
5056
5057 let clean_bytes = std::fs::read(&plain).expect("plain png");
5058 let marked_bytes = std::fs::read(&marked).expect("marked png");
5059 assert_ne!(
5060 clean_bytes, marked_bytes,
5061 "--marks wrote the same image as a plain screenshot"
5062 );
5063
5064 let seen = backend
5067 .marks_map
5068 .lock()
5069 .unwrap()
5070 .clone()
5071 .expect("marks_for_tree was called");
5072 assert_eq!(
5073 i64::from(seen.origin.x),
5074 clean["origin"]["x"].as_i64().unwrap()
5075 );
5076 assert_eq!(
5077 i64::from(seen.origin.y),
5078 clean["origin"]["y"].as_i64().unwrap()
5079 );
5080 assert_eq!(
5081 seen.content_width,
5082 clean["content_width"].as_u64().unwrap() as u32
5083 );
5084 assert_eq!(
5085 seen.logical_width,
5086 clean["logical_width"].as_u64().unwrap() as u32
5087 );
5088
5089 let warnings = marked_body["warnings"].as_array();
5091 assert!(
5092 !warnings
5093 .map(|items| items
5094 .iter()
5095 .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))))
5096 .unwrap_or(false),
5097 "unexpected marks warning: {warnings:?}"
5098 );
5099 }
5100
5101 #[test]
5102 fn marks_warn_when_no_frame_is_inside_the_capture() {
5103 let cache = crate::cache::CacheEnv::acquire();
5105 let backend = StubBackend::default();
5106 backend.set_live({
5109 let mut window = crate::tree::UiNode::bare("AXWindow");
5110 window.frame = Some(crate::tree::NodeFrame {
5111 x: 9_000.0,
5112 y: 9_000.0,
5113 width: 100.0,
5114 height: 100.0,
5115 });
5116 window
5117 });
5118 let context = CallContext {
5119 session: Some("marks-empty".into()),
5120 ..CallContext::default()
5121 };
5122 let body = screenshot(
5123 &backend,
5124 &context,
5125 ScreenshotArgs {
5126 output: Some(
5127 cache
5128 .dir()
5129 .join("marks-empty.png")
5130 .to_string_lossy()
5131 .into_owned(),
5132 ),
5133 region: None,
5134 format: Some(ImageFormat::Png),
5135 max_width: None,
5136 canvas: None,
5137 include_base64: false,
5138 window_crop: false,
5139 marks: true,
5140 },
5141 )
5142 .expect("screenshot");
5143 let warnings = body["warnings"].as_array().expect("warnings");
5144 assert!(
5145 warnings
5146 .iter()
5147 .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))),
5148 "expected a marks warning, got {warnings:?}"
5149 );
5150 }
5151
5152 #[test]
5153 fn no_secure_frames_keeps_every_ocr_block() {
5154 let cache = crate::cache::CacheEnv::acquire();
5155 assert!(cache.dir().is_dir());
5156 let backend = StubBackend::default();
5157 let context = CallContext {
5158 app: Some("Finder".into()),
5159 session: Some("p12b-redact-none".into()),
5160 ..CallContext::default()
5161 };
5162 let blocks = vec![OcrBlock {
5163 text: "hello".into(),
5164 x: 1,
5165 y: 2,
5166 width: 3,
5167 height: 4,
5168 confidence: 0.9,
5169 }];
5170 let kept = redact_secure_ocr_blocks(blocks, &backend, &context, &sample_map()).unwrap();
5171 assert_eq!(kept.len(), 1);
5172 }
5173
5174 #[test]
5175 fn image_coordinate_on_a_stored_map_becomes_the_screen_point() {
5176 let cache = crate::cache::CacheEnv::acquire();
5177 assert!(cache.dir().is_dir());
5178 let map = sample_map();
5179 cache::store_screenshot_map(Some("p5-map"), &map).unwrap();
5180 let backend = StubBackend::default();
5181 let context = CallContext {
5182 coord: Some("image".into()),
5183 session: Some("p5-map".into()),
5184 ..CallContext::default()
5185 };
5186 let response = dispatch_with_context(&backend, Action::Move { x: 100, y: 50 }, &context);
5187 assert!(response.ok, "{response:?}");
5188 let (screen_x, screen_y) = image_util::image_to_screen(&map, 100.0, 50.0);
5189 let value = response.to_value();
5190 assert_eq!(value["x"], json!(screen_x.round() as i64));
5191 assert_eq!(value["y"], json!(screen_y.round() as i64));
5192 assert_eq!(value["x"], 176);
5193 assert_eq!(value["y"], 68);
5194 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5195
5196 let outside = dispatch_with_context(&backend, Action::Move { x: 1440, y: 50 }, &context);
5197 assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
5198 assert!(outside.to_value()["error"]["message"]
5199 .as_str()
5200 .unwrap()
5201 .contains("outside"));
5202 assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5203
5204 let missing = dispatch_with_context(
5205 &backend,
5206 Action::Move { x: 1, y: 2 },
5207 &CallContext {
5208 coord: Some("image".into()),
5209 session: Some("no-map".into()),
5210 ..CallContext::default()
5211 },
5212 );
5213 assert_eq!(missing.to_value()["error"]["code"], "invalid_arguments");
5214 assert!(missing.to_value()["error"]["message"]
5215 .as_str()
5216 .unwrap()
5217 .contains("take a screenshot first"));
5218
5219 let action =
5220 crate::adapters::translate("openai", &json!({"type": "move", "x": 100, "y": 50}))
5221 .unwrap();
5222 let adapted = dispatch_with_context(
5223 &backend,
5224 action,
5225 &CallContext {
5226 coord: Some("image".into()),
5227 session: Some("p5-map".into()),
5228 adapter: Some("openai".into()),
5229 ..CallContext::default()
5230 },
5231 );
5232 assert!(adapted.ok, "{adapted:?}");
5233 assert_eq!(adapted.to_value()["adapter"], "openai");
5234 assert_eq!(adapted.to_value()["x"], 176);
5235 assert_eq!(adapted.to_value()["y"], 68);
5236 assert!(adapted.to_value().get("state").is_none());
5237
5238 let missing_adapter = handle_request_line_with(
5239 r#"{"adapter":"anthropic","action":"mouse_move","coordinate":[1,2],"session":"no-map"}"#,
5240 None,
5241 );
5242 assert!(!missing_adapter.ok);
5243 assert_eq!(
5244 missing_adapter.to_value()["error"]["code"],
5245 "invalid_arguments"
5246 );
5247 assert!(missing_adapter.to_value()["error"]["message"]
5248 .as_str()
5249 .unwrap()
5250 .contains("take a screenshot first"));
5251 assert_eq!(missing_adapter.to_value()["adapter"], "anthropic");
5252
5253 let waited = handle_request_line_with(r#"{"adapter":"openai","type":"wait"}"#, None);
5254 assert!(waited.ok, "{waited:?}");
5255 assert_eq!(waited.to_value()["adapter"], "openai");
5256 assert_eq!(waited.to_value()["waited_ms"], 0);
5257 }
5258
5259 #[test]
5260 fn return_state_includes_the_numbered_tree_and_ignores_read_only() {
5261 let backend = StubBackend::default();
5262 let before = backend.accessibility_apps.lock().unwrap().len();
5263 let response = dispatch_with_context(
5264 &backend,
5265 Action::Click {
5266 x: Some(3),
5267 y: Some(4),
5268 button: MouseButton::Left,
5269 double: false,
5270 count: 1,
5271 },
5272 &CallContext {
5273 app: Some("Notes".into()),
5274 return_state: true,
5275 ..CallContext::default()
5276 },
5277 );
5278 assert!(response.ok, "{response:?}");
5279 let value = response.to_value();
5280 assert_eq!(value["action"], "click");
5281 let tree = value["state"]["tree"].as_str().unwrap();
5282 assert!(tree.contains("[0] AXWindow"), "{tree}");
5283 assert!(tree.contains("AXTextArea"), "{tree}");
5284 assert!(value["state"]["diff"].is_null());
5285 assert_eq!(value["state"]["app"], "com.apple.Notes");
5286 assert_eq!(value["state"]["pid"], 10);
5287 assert_eq!(value["state"]["window_id"], 100);
5288 assert!(value.get("warnings").is_none());
5289 assert!(backend.accessibility_apps.lock().unwrap().len() > before);
5290
5291 let displays = dispatch_with_context(
5292 &backend,
5293 Action::Displays,
5294 &CallContext {
5295 return_state: true,
5296 include_screenshot: true,
5297 ..CallContext::default()
5298 },
5299 );
5300 assert!(displays.ok, "{displays:?}");
5301 assert!(displays.to_value().get("state").is_none());
5302
5303 let cursor = dispatch_with(&backend, Action::CursorPosition);
5304 assert_eq!(cursor.to_value()["x"], 4);
5305 assert_eq!(cursor.to_value()["y"], 5);
5306 assert!(cursor.to_value().get("adapter").is_none());
5307 }
5308
5309 #[test]
5310 fn return_state_refreshes_the_tree_diff_baseline() {
5311 let backend = StubBackend::default();
5312 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5313 assert!(first.ok, "{first:?}");
5314 *backend.stub_value.lock().unwrap() = "ab".into();
5315 let mut context = element_context(0);
5316 context.return_state = true;
5317 let response = dispatch_with_context(&backend, Action::Focus, &context);
5318 assert!(response.ok, "{response:?}");
5319 let value = response.to_value();
5320 let diff = value["state"]["diff"].as_str().unwrap();
5321 assert!(
5322 diff.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
5323 "{diff}"
5324 );
5325 assert!(value["state"]["tree"]
5326 .as_str()
5327 .unwrap()
5328 .contains("value=\"ab\""));
5329 *backend.stub_value.lock().unwrap() = "abc".into();
5330 let next = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5331 let value = next.to_value();
5332 let text = value["tree"].as_str().unwrap();
5333 assert!(
5334 text.contains("~ [1] AXTextArea value: \"ab\" -> \"abc\""),
5335 "{text}"
5336 );
5337 assert!(!text.contains("\"a\" -> \"abc\""), "{text}");
5338 }
5339
5340 #[test]
5341 fn state_diff_omits_the_full_tree_and_keeps_the_delta() {
5342 let backend = StubBackend::default();
5343 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5344 assert!(first.ok, "{first:?}");
5345 *backend.stub_value.lock().unwrap() = "ab".into();
5346 let mut context = element_context(0);
5347 context.return_state = true;
5348 context.state_diff = true;
5349 let response = dispatch_with_context(&backend, Action::Focus, &context);
5350 assert!(response.ok, "{response:?}");
5351 let value = response.to_value();
5352 assert!(value["state"].get("tree").is_none(), "{value}");
5354 let diff = value["state"]["diff"].as_str().unwrap();
5355 assert!(
5356 diff.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
5357 "{diff}"
5358 );
5359 assert_eq!(value["state"]["baseline"], "existing");
5360 assert_eq!(value["state"]["app"], "com.apple.Notes");
5361 }
5362
5363 #[test]
5364 fn state_diff_without_state_still_attaches_the_post_state() {
5365 let backend = StubBackend::default();
5366 let seeded = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5367 assert!(seeded.ok, "{seeded:?}");
5368 let mut context = element_context(0);
5369 context.state_diff = true;
5370 let response = dispatch_with_context(&backend, Action::Focus, &context);
5371 assert!(response.ok, "{response:?}");
5372 let value = response.to_value();
5373 assert!(value["state"].get("tree").is_none(), "{value}");
5374 assert!(value["state"].get("diff").is_some(), "{value}");
5375 }
5376
5377 #[test]
5378 fn the_tree_response_says_whether_a_baseline_existed() {
5379 let backend = StubBackend::default();
5380 let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5381 assert!(first.ok, "{first:?}");
5382 assert_eq!(first.to_value()["baseline"], "created");
5383 *backend.stub_value.lock().unwrap() = "ab".into();
5384 let second = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5385 assert!(second.ok, "{second:?}");
5386 assert_eq!(second.to_value()["baseline"], "existing");
5387 }
5388
5389 #[test]
5390 fn policy_check_reports_a_code_next_to_the_decision() {
5391 let backend = StubBackend::default();
5392 let response = dispatch_with_context(
5393 &backend,
5394 Action::PolicyCheck,
5395 &CallContext {
5396 app: Some("Notes".into()),
5397 ..CallContext::default()
5398 },
5399 );
5400 assert!(response.ok, "{response:?}");
5401 let value = response.to_value();
5402 assert_eq!(value["decision"], "allow");
5403 assert_eq!(value["code"], "ok");
5404 assert!(value["source"].is_string(), "{value}");
5405 }
5406
5407 #[test]
5408 fn include_screenshot_failure_keeps_the_action_result() {
5409 let backend = StubBackend {
5410 capture_fault: crate::test_backend::StubCaptureFault::Permission,
5411 ..StubBackend::default()
5412 };
5413 let response = dispatch_with_context(
5414 &backend,
5415 Action::Click {
5416 x: Some(1),
5417 y: Some(2),
5418 button: MouseButton::Left,
5419 double: false,
5420 count: 1,
5421 },
5422 &CallContext {
5423 app: Some("Notes".into()),
5424 return_state: true,
5425 include_screenshot: true,
5426 ..CallContext::default()
5427 },
5428 );
5429 assert!(response.ok, "{response:?}");
5430 let value = response.to_value();
5431 assert_eq!(value["action"], "click");
5432 assert_eq!(value["x"], 1);
5433 assert_eq!(value["y"], 2);
5434 assert!(value["state"]["tree"]
5435 .as_str()
5436 .unwrap()
5437 .contains("AXWindow"));
5438 assert!(value["state"]["screenshot"].is_null());
5439 let warnings = value["warnings"].as_array().unwrap();
5440 assert!(warnings
5441 .iter()
5442 .any(|item| { item.as_str().unwrap().contains("screenshot failed") }));
5443 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5444 }
5445
5446 fn batch_clicks(stop_on_error: bool) -> Value {
5447 json!({
5448 "action": "batch",
5449 "stop_on_error": stop_on_error,
5450 "actions": [
5451 {"action": "click", "x": 1, "y": 2},
5452 {"action": "click", "x": 3, "y": 4}
5453 ]
5454 })
5455 }
5456
5457 #[test]
5458 fn batch_stop_on_error_controls_the_second_attempt() {
5459 let stopped = StubBackend {
5460 fail_click: true,
5461 ..StubBackend::default()
5462 };
5463 let request = parse_request(batch_clicks(true)).unwrap();
5464 let response = dispatch_with_context(&stopped, request.action, &request.context);
5465 assert!(!response.ok);
5466 let value = response.to_value();
5467 assert_eq!(value["error"]["code"], "execution_failed");
5468 assert_eq!(value["stopped_at"], 0);
5469 assert_eq!(value["results"].as_array().unwrap().len(), 1);
5470 assert!(!value["results"][0]["ok"].as_bool().unwrap());
5471 assert_eq!(stopped.click_attempts.load(Ordering::SeqCst), 2);
5472 assert!(value["error"]["message"]
5473 .as_str()
5474 .unwrap()
5475 .contains("batch failed at action 0"));
5476
5477 let both = StubBackend {
5478 fail_click: true,
5479 ..StubBackend::default()
5480 };
5481 let request = parse_request(batch_clicks(false)).unwrap();
5482 let response = dispatch_with_context(&both, request.action, &request.context);
5483 assert!(!response.ok);
5484 let value = response.to_value();
5485 assert!(value.get("stopped_at").is_none(), "{value}");
5486 assert_eq!(value["results"].as_array().unwrap().len(), 2);
5487 assert!(!value["results"][1]["ok"].as_bool().unwrap());
5488 assert_eq!(both.click_attempts.load(Ordering::SeqCst), 4);
5489
5490 let nested = parse_request(json!({
5491 "action": "batch",
5492 "actions": [{"action": "batch", "actions": []}]
5493 }))
5494 .unwrap();
5495 let response = dispatch_with(&StubBackend::default(), nested.action);
5496 assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
5497 assert_eq!(response.to_value()["stopped_at"], 0);
5498 assert!(response.to_value()["error"]["message"]
5499 .as_str()
5500 .unwrap()
5501 .contains("nested batch"));
5502
5503 let backend = StubBackend::default();
5504 backend.seed_element(notes_button());
5505 let request = parse_request(json!({
5506 "action": "batch",
5507 "app": "Notes",
5508 "actions": [
5509 {"action": "press", "element": 1},
5510 {"action": "focus", "element": 1}
5511 ]
5512 }))
5513 .unwrap();
5514 assert!(matches!(
5515 request.action,
5516 Action::Batch {
5517 stop_on_error: true,
5518 ..
5519 }
5520 ));
5521 let response = dispatch_with_context(&backend, request.action, &request.context);
5522 assert!(response.ok, "{response:?}");
5523 let results = response.to_value()["results"].clone();
5524 assert_eq!(results[0]["action"], "press");
5525 assert_eq!(results[0]["element"], 1);
5526 assert_eq!(results[1]["action"], "focus");
5527 assert!(response.to_value().get("state").is_none());
5528 }
5529
5530 #[test]
5531 fn p9_menu_list_round_trip_records_no_args() {
5532 let backend = StubBackend::default();
5533 backend
5534 .menu_sections
5535 .lock()
5536 .unwrap()
5537 .push(crate::backend::MenuNode {
5538 title: "File".into(),
5539 enabled: true,
5540 children: vec![crate::backend::MenuNode {
5541 title: "Save".into(),
5542 enabled: true,
5543 children: Vec::new(),
5544 }],
5545 });
5546 let response = dispatch_with_context(
5547 &backend,
5548 Action::MenuList {
5549 app: Some("Notes".into()),
5550 depth: Some(2),
5551 },
5552 &ctx("Notes"),
5553 );
5554 assert!(response.ok, "{:?}", response.to_value());
5555 let value = response.to_value();
5556 assert_eq!(value["action"], "menu_list");
5557 assert_eq!(value["app"], "Notes");
5558 assert_eq!(value["sections"][0]["title"], "File");
5559 assert_eq!(value["sections"][0]["children"][0]["title"], "Save");
5560 }
5561
5562 #[test]
5563 fn p9_menu_click_records_path_and_reports_disabled() {
5564 let mut backend = StubBackend::default();
5565 let response = dispatch_with_context(
5566 &backend,
5567 Action::MenuClick {
5568 app: Some("Notes".into()),
5569 path: "File > Save".into(),
5570 },
5571 &ctx("Notes"),
5572 );
5573 assert!(response.ok, "{:?}", response.to_value());
5574 let value = response.to_value();
5575 assert_eq!(value["path"], serde_json::json!(["File", "Save"]));
5576 assert_eq!(value["pressed"], true);
5577 let recorded = backend.menu_clicks.lock().unwrap().clone();
5578 assert_eq!(recorded.len(), 1);
5579 assert_eq!(
5580 recorded[0].segments,
5581 vec!["File".to_string(), "Save".to_string()]
5582 );
5583
5584 backend.menu_disabled = true;
5585 let response = dispatch_with_context(
5586 &backend,
5587 Action::MenuClick {
5588 app: Some("Notes".into()),
5589 path: "File > Save".into(),
5590 },
5591 &ctx("Notes"),
5592 );
5593 assert!(!response.ok, "disabled menu item must fail");
5594 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5595 }
5596
5597 #[test]
5598 fn p9_apple_event_records_only_length_and_language() {
5599 let backend = StubBackend::default();
5600 *backend.apple_event_stdout.lock().unwrap() = "Notes".into();
5601 let secret = "system prompt with token ABCDEFG";
5602 let response = dispatch_with_context(
5603 &backend,
5604 Action::AppleEvent {
5605 app: Some("Notes".into()),
5606 script: secret.into(),
5607 language: Some(AppleEventLanguage::AppleScript),
5608 },
5609 &ctx("Notes"),
5610 );
5611 assert!(response.ok, "{:?}", response.to_value());
5612 let value = response.to_value();
5613 assert_eq!(value["language"], "AppleScript");
5614 assert_eq!(value["script_bytes"], secret.len());
5615 assert_eq!(value["stdout"], "Notes");
5616 assert!(value["script_hash"].is_string());
5617 let recorded = backend.apple_event_calls.lock().unwrap().clone();
5618 assert_eq!(recorded.len(), 1);
5619 assert_eq!(recorded[0].1, AppleEventLanguage::AppleScript);
5620 assert_eq!(recorded[0].2, secret.len());
5622 }
5623
5624 #[test]
5625 fn p9_apple_event_records_language_javascript() {
5626 let backend = StubBackend::default();
5627 let response = dispatch_with_context(
5628 &backend,
5629 Action::AppleEvent {
5630 app: Some("Notes".into()),
5631 script: "Application('Notes').name()".into(),
5632 language: Some(AppleEventLanguage::JavaScript),
5633 },
5634 &ctx("Notes"),
5635 );
5636 assert!(response.ok);
5637 assert_eq!(response.to_value()["language"], "JavaScript");
5638 }
5639
5640 #[test]
5641 fn p9_apple_event_failure_surfaces_execution_failed() {
5642 let backend = StubBackend {
5643 apple_event_fail: true,
5644 ..Default::default()
5645 };
5646 *backend.apple_event_stderr.lock().unwrap() = "execution error: 42".into();
5647 let response = dispatch_with_context(
5648 &backend,
5649 Action::AppleEvent {
5650 app: Some("Notes".into()),
5651 script: "1/0".into(),
5652 language: None,
5653 },
5654 &ctx("Notes"),
5655 );
5656 assert!(!response.ok);
5657 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5658 }
5659
5660 #[test]
5661 fn p9_dock_list_and_click_round_trip() {
5662 let backend = StubBackend::default();
5663 backend.dock_items.lock().unwrap().extend([
5664 "Finder".to_string(),
5665 "Terminal".to_string(),
5666 "Mail".to_string(),
5667 ]);
5668 let response = dispatch_with(&backend, Action::DockList {});
5669 assert!(response.ok);
5670 let value = response.to_value();
5671 let items = value["items"].as_array().unwrap();
5672 assert_eq!(items.len(), 3);
5673 assert_eq!(items[0]["title"], "Finder");
5674
5675 let response = dispatch_with(
5676 &backend,
5677 Action::DockClick {
5678 title: "terminal".into(),
5679 },
5680 );
5681 assert!(response.ok, "{:?}", response.to_value());
5682 assert_eq!(response.to_value()["pressed"], true);
5683 assert_eq!(
5684 backend.dock_clicks.lock().unwrap().clone(),
5685 vec!["terminal".to_string()]
5686 );
5687
5688 let response = dispatch_with(
5689 &backend,
5690 Action::DockClick {
5691 title: "missing".into(),
5692 },
5693 );
5694 assert!(!response.ok);
5695 assert_eq!(response.to_value()["error"]["code"], "element_not_found");
5696 }
5697
5698 #[test]
5699 fn p9_spaces_reports_screen_groups_from_fake_list() {
5700 use crate::target::{WindowBounds, WindowInfo};
5701 use crate::types::DisplayInfo;
5702 use crate::window_groups::SpacesReport;
5703
5704 let mut spaces_report = SpacesReport {
5705 screens: Vec::new(),
5706 note: String::new(),
5707 };
5708 spaces_report
5709 .screens
5710 .push(crate::window_groups::SpaceScreen {
5711 index: 0,
5712 name: "Display 0".into(),
5713 x: 0,
5714 y: 0,
5715 width: 1000,
5716 height: 1000,
5717 is_primary: true,
5718 windows: vec![WindowInfo {
5719 id: 1,
5720 pid: 100,
5721 app: "Notes".into(),
5722 title: "Note 1".into(),
5723 bounds: WindowBounds {
5724 x: 10,
5725 y: 10,
5726 width: 200,
5727 height: 200,
5728 },
5729 z: 0,
5730 layer: 0,
5731 on_screen: true,
5732 ax_matched: false,
5733 }],
5734 });
5735
5736 let backend = StubBackend::default();
5737 *backend.spaces_report.lock().unwrap() = spaces_report;
5738 let response = dispatch_with(&backend, Action::Spaces {});
5739 assert!(response.ok);
5740 let value = response.to_value();
5741 assert_eq!(value["action"], "spaces");
5742 assert_eq!(value["screens"][0]["name"], "Display 0");
5743 assert_eq!(value["screens"][0]["windows"][0]["title"], "Note 1");
5744 assert_eq!(value["screens"][0]["is_primary"], true);
5745 let _ = DisplayInfo {
5747 id: 0,
5748 name: String::new(),
5749 x: 0,
5750 y: 0,
5751 width: 0,
5752 height: 0,
5753 scale: 1.0,
5754 is_primary: false,
5755 };
5756 }
5757
5758 #[test]
5759 fn p9_shortcuts_run_invokes_with_only_the_name() {
5760 let backend = StubBackend::default();
5763 let response = dispatch_with(
5764 &backend,
5765 Action::ShortcutsRun {
5766 name: "Open URL".into(),
5767 },
5768 );
5769 if !response.ok {
5775 let value = response.to_value();
5776 let code = value["error"]["code"].as_str().unwrap_or_default();
5777 assert!(
5778 matches!(code, "execution_failed" | "unsupported"),
5779 "{value:?}"
5780 );
5781 }
5782 }
5783
5784 #[test]
5785 fn session_prune_action_removes_an_empty_temp_session() {
5786 let _cache = crate::cache::CacheEnv::acquire();
5787 let dir = cache::session_dir(Some("empty-session")).unwrap();
5788 assert!(dir.join("owner.json").is_file());
5789 let response = dispatch(Action::SessionPrune {
5790 dry_run: true,
5791 ttl_secs: Some(0),
5792 });
5793 assert!(response.ok, "{response:?}");
5794 assert_eq!(response.to_value()["dry_run"], true);
5795 assert!(response.to_value()["removed"]
5796 .as_array()
5797 .unwrap()
5798 .iter()
5799 .any(|name| name == "empty-session"));
5800 assert!(dir.exists());
5801 let response = dispatch(Action::SessionPrune {
5802 dry_run: false,
5803 ttl_secs: Some(0),
5804 });
5805 assert!(response.ok, "{response:?}");
5806 assert!(!dir.exists());
5807 }
5808
5809 #[test]
5810 fn pause_short_circuits_before_the_stub_clicks() {
5811 let backend = StubBackend::default();
5812 let _paused = policy::override_paused(true);
5813 let response = dispatch_with(
5814 &backend,
5815 Action::Click {
5816 x: Some(1),
5817 y: Some(2),
5818 button: MouseButton::Left,
5819 double: false,
5820 count: 1,
5821 },
5822 );
5823 assert_eq!(response.to_value()["error"]["code"], "paused");
5824 assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
5825 assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5826 let waited = dispatch_with(
5827 &backend,
5828 Action::Wait {
5829 ms: Some(0),
5830 seconds: None,
5831 },
5832 );
5833 assert!(waited.ok, "{waited:?}");
5834 }
5835
5836 #[test]
5837 fn confirm_hook_allows_quit_and_a_failure_does_not() {
5838 let backend = StubBackend::default();
5839 let dir = std::env::temp_dir().join(format!(
5840 "computer-use-hook-{}-{}",
5841 std::process::id(),
5842 std::time::SystemTime::now()
5843 .duration_since(std::time::UNIX_EPOCH)
5844 .map(|duration| duration.as_nanos())
5845 .unwrap_or(0)
5846 ));
5847 let _ = std::fs::remove_dir_all(&dir);
5848 std::fs::create_dir_all(&dir).unwrap();
5849 #[cfg(unix)]
5850 let (allow, deny) = {
5851 let allow = dir.join("allow.sh");
5852 let deny = dir.join("deny.sh");
5853 std::fs::write(&allow, "#!/bin/sh\ncat >/dev/null\nexit 0\n").unwrap();
5854 std::fs::write(&deny, "#!/bin/sh\ncat >/dev/null\nexit 1\n").unwrap();
5855 use std::os::unix::fs::PermissionsExt;
5856 std::fs::set_permissions(&allow, std::fs::Permissions::from_mode(0o700)).unwrap();
5857 std::fs::set_permissions(&deny, std::fs::Permissions::from_mode(0o700)).unwrap();
5858 (allow.display().to_string(), deny.display().to_string())
5859 };
5860 #[cfg(windows)]
5861 let (allow, deny) = ("cmd /C exit 0".to_string(), "cmd /C exit 1".to_string());
5862 let mut context = ctx("412");
5863 context.confirm_cmd = Some(deny);
5864 let denied = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5865 assert_eq!(denied.to_value()["error"]["code"], "confirm_denied");
5866 assert!(backend.quit.lock().unwrap().is_empty());
5867 context.confirm_cmd = Some(allow);
5868 let allowed = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5869 assert!(allowed.ok, "{allowed:?}");
5870 assert_eq!(backend.quit.lock().unwrap().len(), 1);
5871 let _ = std::fs::remove_dir_all(&dir);
5872 }
5873
5874 #[test]
5875 fn blocked_url_prevents_the_click() {
5876 let mut backend = StubBackend::default();
5877 backend.apps.push(crate::target::AppInfo {
5878 name: "Safari".into(),
5879 bundle_id: Some("com.apple.Safari".into()),
5880 pid: 80,
5881 path: Some("/Applications/Safari.app/Contents/MacOS/Safari".into()),
5882 active: false,
5883 hidden: false,
5884 activation_policy: crate::target::ActivationPolicy::Regular,
5885 });
5886 backend
5887 .browser_urls
5888 .lock()
5889 .unwrap()
5890 .insert(80, "https://accounts.example.com/login".into());
5891 let dir = std::env::temp_dir().join(format!(
5892 "computer-use-url-{}-{}",
5893 std::process::id(),
5894 std::time::SystemTime::now()
5895 .duration_since(std::time::UNIX_EPOCH)
5896 .map(|duration| duration.as_nanos())
5897 .unwrap_or(0)
5898 ));
5899 let _ = std::fs::remove_dir_all(&dir);
5900 std::fs::create_dir_all(&dir).unwrap();
5901 let policy_path = dir.join("policy.toml");
5902 std::fs::write(
5903 &policy_path,
5904 "version = 1\n[browser]\nblocked_urls = [\"https://accounts.example.com/*\"]\n",
5905 )
5906 .unwrap();
5907 let mut context = ctx("80");
5908 context.policy = Some(policy_path.display().to_string());
5909 let response = dispatch_with_context(
5910 &backend,
5911 Action::Click {
5912 x: Some(3),
5913 y: Some(4),
5914 button: MouseButton::Left,
5915 double: false,
5916 count: 1,
5917 },
5918 &context,
5919 );
5920 assert_eq!(response.to_value()["error"]["code"], "blocked_url");
5921 assert!(response.to_value()["error"]["message"]
5922 .as_str()
5923 .unwrap()
5924 .contains("https://accounts.example.com/*"));
5925 assert!(!response.to_value()["error"]["message"]
5926 .as_str()
5927 .unwrap()
5928 .contains("login"));
5929 assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5930 backend
5931 .browser_urls
5932 .lock()
5933 .unwrap()
5934 .insert(80, "https://example.com/".into());
5935 let allowed = dispatch_with_context(
5936 &backend,
5937 Action::Click {
5938 x: Some(3),
5939 y: Some(4),
5940 button: MouseButton::Left,
5941 double: false,
5942 count: 1,
5943 },
5944 &context,
5945 );
5946 assert!(allowed.ok, "{allowed:?}");
5947 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5948 let _ = std::fs::remove_dir_all(&dir);
5949 }
5950
5951 fn temp_dir(label: &str) -> std::path::PathBuf {
5952 let dir = std::env::temp_dir().join(format!(
5953 "computer-use-{label}-{}-{}",
5954 std::process::id(),
5955 std::time::SystemTime::now()
5956 .duration_since(std::time::UNIX_EPOCH)
5957 .map(|duration| duration.as_nanos())
5958 .unwrap_or(0)
5959 ));
5960 let _ = std::fs::remove_dir_all(&dir);
5961 std::fs::create_dir_all(&dir).unwrap();
5962 dir
5963 }
5964
5965 fn click_at(x: i32, y: i32) -> Action {
5966 Action::Click {
5967 x: Some(x),
5968 y: Some(y),
5969 button: MouseButton::Left,
5970 double: false,
5971 count: 1,
5972 }
5973 }
5974
5975 fn last_audit(dir: &std::path::Path) -> Value {
5976 let text = std::fs::read_to_string(dir.join("audit.jsonl")).unwrap();
5977 serde_json::from_str(text.lines().last().unwrap()).unwrap()
5978 }
5979
5980 #[test]
5981 fn auto_event_falls_back_to_activate_unless_blocked() {
5982 let backend = StubBackend {
5983 fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5984 ..StubBackend::default()
5985 };
5986 let response = dispatch_with(&backend, click_at(3, 4));
5987 assert!(response.ok, "{response:?}");
5988 let value = response.to_value();
5989 assert_eq!(value["tier"], "activate");
5990 assert_eq!(value["cursor_moved"], true);
5991 assert_eq!(value["focus_restored"], true);
5992 assert_eq!(
5993 backend.tiers_run.lock().unwrap().as_slice(),
5994 ["event", "activate"]
5995 );
5996 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5997
5998 let held = StubBackend {
5999 fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
6000 ..StubBackend::default()
6001 };
6002 let dir = temp_dir("lock-fallback");
6003 let path = dir.join("foreground.lock");
6004 let _slot = crate::foreground::push_lock_path(&path);
6005 let _lock = crate::foreground::try_lock(&path).unwrap();
6006 let response = dispatch_with(&held, click_at(3, 4));
6007 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
6008 assert!(response.to_value()["error"]["message"]
6009 .as_str()
6010 .unwrap()
6011 .contains("stub tier event failed"));
6012 assert_eq!(held.tiers_run.lock().unwrap().as_slice(), ["event"]);
6013 assert_eq!(held.clicks.load(Ordering::SeqCst), 0);
6014 drop(_lock);
6015 let _ = std::fs::remove_dir_all(&dir);
6016
6017 let background = StubBackend {
6018 fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
6019 ..StubBackend::default()
6020 };
6021 let response = dispatch_with_context(
6022 &background,
6023 click_at(1, 2),
6024 &CallContext {
6025 require_background: true,
6026 ..CallContext::default()
6027 },
6028 );
6029 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
6030 assert_eq!(background.tiers_run.lock().unwrap().as_slice(), ["event"]);
6031 assert_eq!(background.clicks.load(Ordering::SeqCst), 0);
6032
6033 let refused = StubBackend::default();
6034 let response = dispatch_with_context(
6035 &refused,
6036 click_at(1, 2),
6037 &CallContext {
6038 tier: Some("activate".into()),
6039 require_background: true,
6040 ..CallContext::default()
6041 },
6042 );
6043 assert!(response.to_value()["error"]["message"]
6044 .as_str()
6045 .unwrap()
6046 .contains("--require-background"));
6047 assert!(refused.tiers_run.lock().unwrap().is_empty());
6048 assert_eq!(refused.clicks.load(Ordering::SeqCst), 0);
6049 }
6050
6051 #[test]
6052 fn activate_names_the_lock_holder_before_it_clicks() {
6053 let backend = StubBackend::default();
6054 let dir = temp_dir("lock-held");
6055 let path = dir.join("foreground.lock");
6056 let _slot = crate::foreground::push_lock_path(&path);
6057 let _lock = crate::foreground::try_lock(&path).unwrap();
6058 let response = dispatch_with_context(
6059 &backend,
6060 click_at(9, 8),
6061 &CallContext {
6062 tier: Some("activate".into()),
6063 ..CallContext::default()
6064 },
6065 );
6066 assert_eq!(response.to_value()["error"]["code"], "execution_failed");
6067 let message = response.to_value()["error"]["message"]
6068 .as_str()
6069 .unwrap()
6070 .to_string();
6071 assert!(
6072 message.contains(&format!(
6073 "foreground.lock is held by pid {}",
6074 std::process::id()
6075 )),
6076 "{message}"
6077 );
6078 assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
6079 assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 1);
6080 assert!(backend.tiers_run.lock().unwrap().is_empty());
6081 drop(_lock);
6082 let _ = std::fs::remove_dir_all(&dir);
6083 }
6084
6085 #[test]
6086 fn synthetic_without_the_feature_does_not_hit_test_or_click() {
6087 let backend = StubBackend::default();
6088 let response = dispatch_with_context(
6089 &backend,
6090 click_at(1, 1),
6091 &CallContext {
6092 tier: Some("synthetic".into()),
6093 ..CallContext::default()
6094 },
6095 );
6096 if crate::tier::skylight_compiled() {
6097 assert!(response.ok, "{response:?}");
6098 assert_eq!(response.to_value()["tier"], "synthetic");
6099 assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
6100 assert_eq!(backend.tiers_run.lock().unwrap().as_slice(), ["synthetic"]);
6101 } else {
6102 assert_eq!(response.to_value()["error"]["code"], "unsupported");
6103 assert!(response.to_value()["error"]["message"]
6104 .as_str()
6105 .unwrap()
6106 .contains("built without skylight feature"));
6107 assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 0);
6108 assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
6109 assert!(backend.tiers_run.lock().unwrap().is_empty());
6110 }
6111 }
6112
6113 #[test]
6114 fn missing_overlay_helper_warns_and_no_overlay_skips_it() {
6115 let dir = temp_dir("overlay-warn");
6116 let _slot = crate::overlay::push_socket_path(dir.join("missing.sock"));
6117 let backend = StubBackend::default();
6118 let response = dispatch_with_context(
6119 &backend,
6120 click_at(4, 5),
6121 &CallContext {
6122 session: Some("agent".into()),
6123 ..CallContext::default()
6124 },
6125 );
6126 assert!(response.ok, "{response:?}");
6127 assert_eq!(response.to_value()["tier"], "event");
6128 assert_eq!(response.to_value()["cursor_moved"], false);
6129 assert!(response.to_value().get("focus_restored").is_none());
6130 let warnings = response.to_value()["warnings"].as_array().unwrap().clone();
6131 assert!(warnings
6132 .iter()
6133 .any(|item| item == "overlay helper not running"));
6134
6135 let quiet = dispatch_with_context(
6136 &backend,
6137 click_at(4, 5),
6138 &CallContext {
6139 no_overlay: true,
6140 ..CallContext::default()
6141 },
6142 );
6143 assert!(quiet.ok, "{quiet:?}");
6144 assert!(quiet.to_value().get("warnings").is_none());
6145 let _ = std::fs::remove_dir_all(&dir);
6146 }
6147
6148 #[test]
6149 fn audit_records_decision_source_and_the_real_code() {
6150 let dir = temp_dir("audit");
6151 let _state = crate::audit::push_state_dir(&dir);
6152 let backend = StubBackend::default();
6153 let response = dispatch_with(&backend, click_at(2, 3));
6154 assert!(response.ok, "{response:?}");
6155 let line = last_audit(&dir);
6156 assert_eq!(line["action"], "click");
6157 assert_eq!(line["code"], "ok");
6158 assert_eq!(line["decision"], "allow");
6159 assert_eq!(line["source"], "default");
6160 assert_eq!(line["tier"], "event");
6161 assert!(line.get("text").is_none());
6162
6163 let denied = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
6164 assert_eq!(denied.to_value()["error"]["code"], "policy_denied");
6165 let line = last_audit(&dir);
6166 assert_eq!(line["action"], "launch");
6167 assert_eq!(line["code"], "policy_denied");
6168 assert_eq!(line["decision"], "deny");
6169 assert_eq!(line["source"], "builtin");
6170
6171 let checked =
6172 dispatch_with_context(&backend, Action::PolicyCheck, &ctx("com.apple.Terminal"));
6173 assert!(checked.ok, "{checked:?}");
6174 assert_eq!(checked.to_value()["decision"], "deny");
6175 let line = last_audit(&dir);
6176 assert_eq!(line["action"], "policy_check");
6177 assert_eq!(line["code"], "ok");
6178 assert_eq!(line["decision"], "deny");
6179 assert_eq!(line["source"], "builtin");
6180
6181 let bad = dispatch_with(
6182 &backend,
6183 Action::Click {
6184 x: None,
6185 y: None,
6186 button: MouseButton::Left,
6187 double: false,
6188 count: 1,
6189 },
6190 );
6191 assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
6192 let line = last_audit(&dir);
6193 assert_eq!(line["code"], "invalid_arguments");
6194 assert_eq!(line["decision"], "allow");
6195 assert_eq!(line["source"], "default");
6196 let _ = std::fs::remove_dir_all(&dir);
6197 }
6198
6199 #[test]
6200 fn drag_hold_click_and_keys_use_the_stub() {
6201 let backend = StubBackend::default();
6202 let short = dispatch_with(
6203 &backend,
6204 Action::Drag {
6205 path: vec![crate::types::Point { x: 1, y: 1 }],
6206 button: MouseButton::Left,
6207 },
6208 );
6209 assert_eq!(short.to_value()["error"]["code"], "invalid_arguments");
6210 assert_eq!(backend.moves.load(Ordering::SeqCst), 0);
6211
6212 let dragged = dispatch_with(
6213 &backend,
6214 Action::Drag {
6215 path: vec![
6216 crate::types::Point { x: 1, y: 2 },
6217 crate::types::Point { x: 3, y: 4 },
6218 crate::types::Point { x: 5, y: 6 },
6219 ],
6220 button: MouseButton::Left,
6221 },
6222 );
6223 assert!(dragged.ok, "{dragged:?}");
6224 assert_eq!(dragged.to_value()["moves"], 2);
6225 assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6226 assert_eq!(dragged.to_value()["tier"], "event");
6227
6228 let ax = dispatch_with_context(
6229 &backend,
6230 Action::Drag {
6231 path: vec![
6232 crate::types::Point { x: 1, y: 2 },
6233 crate::types::Point { x: 3, y: 4 },
6234 ],
6235 button: MouseButton::Left,
6236 },
6237 &CallContext {
6238 tier: Some("ax".into()),
6239 ..CallContext::default()
6240 },
6241 );
6242 assert_eq!(ax.to_value()["error"]["code"], "unsupported");
6243 assert!(ax.to_value()["error"]["message"]
6244 .as_str()
6245 .unwrap()
6246 .contains("pass --element"));
6247 assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6248
6249 let triple = dispatch_with(
6250 &backend,
6251 Action::TripleClick {
6252 x: Some(8),
6253 y: Some(9),
6254 button: MouseButton::Left,
6255 },
6256 );
6257 assert!(triple.ok, "{triple:?}");
6258 assert_eq!(triple.to_value()["action"], "triple_click");
6259 assert_eq!(triple.to_value()["count"], 3);
6260 assert_eq!(backend.clicks.load(Ordering::SeqCst), 3);
6261
6262 let held = dispatch_with(
6263 &backend,
6264 Action::HoldKey {
6265 keys: "shift".into(),
6266 ms: 40_000,
6267 },
6268 );
6269 assert_eq!(held.to_value()["error"]["code"], "invalid_arguments");
6270 assert!(backend.holds_ms.lock().unwrap().is_empty());
6271
6272 let shift = dispatch_with(
6273 &backend,
6274 Action::HoldKey {
6275 keys: "shift".into(),
6276 ms: 400,
6277 },
6278 );
6279 assert!(shift.ok, "{shift:?}");
6280 assert_eq!(backend.holds_ms.lock().unwrap().as_slice(), [400]);
6281
6282 let sequence = dispatch_with(
6283 &backend,
6284 Action::Key {
6285 keys: "super+c Return".into(),
6286 },
6287 );
6288 assert!(sequence.ok, "{sequence:?}");
6289 assert_eq!(sequence.to_value()["strokes"], 2);
6290 let keys = backend.keys.lock().unwrap();
6291 let sequence = &keys[keys.len() - 2..];
6292 assert_eq!(sequence[0].1, "c");
6293 assert_eq!(sequence[0].0, vec![Modifier::Meta]);
6294 assert_eq!(sequence[1].1, "enter");
6295 drop(keys);
6296
6297 let unknown = dispatch_with(
6298 &backend,
6299 Action::Key {
6300 keys: "bogus".into(),
6301 },
6302 );
6303 assert_eq!(unknown.to_value()["error"]["code"], "invalid_arguments");
6304 }
6305
6306 #[test]
6307 fn mouse_down_releases_when_the_session_ends() {
6308 let backend = StubBackend::default();
6309 let down = Action::MouseDown {
6310 x: 4,
6311 y: 5,
6312 button: MouseButton::Left,
6313 };
6314 let alone = dispatch_with(&backend, down.clone());
6315 assert!(alone.ok, "{alone:?}");
6316 assert!(backend.held_buttons.lock().unwrap().is_empty());
6317 assert_eq!(
6318 backend.releases.lock().unwrap().as_slice(),
6319 [MouseButton::Left]
6320 );
6321
6322 backend.releases.lock().unwrap().clear();
6323 let _session = crate::InputSession::open(&backend);
6324 let kept = dispatch_with(&backend, down);
6325 assert!(kept.ok, "{kept:?}");
6326 assert_eq!(
6327 backend.held_buttons.lock().unwrap().as_slice(),
6328 [MouseButton::Left]
6329 );
6330 assert!(backend.releases.lock().unwrap().is_empty());
6331 drop(_session);
6332 assert!(backend.held_buttons.lock().unwrap().is_empty());
6333 assert_eq!(
6334 backend.releases.lock().unwrap().as_slice(),
6335 [MouseButton::Left]
6336 );
6337
6338 let batch = dispatch_with(
6339 &backend,
6340 Action::Batch {
6341 stop_on_error: true,
6342 actions: vec![
6343 json!({"action": "mouse_down", "x": 1, "y": 2}),
6344 json!({"action": "mouse_up", "x": 1, "y": 2}),
6345 ],
6346 },
6347 );
6348 assert!(batch.ok, "{batch:?}");
6349 assert!(backend.held_buttons.lock().unwrap().is_empty());
6350 }
6351
6352 #[test]
6353 fn clipboard_confirms_and_omits_the_text_from_the_audit() {
6354 let dir = temp_dir("clipboard-audit");
6355 let _state = crate::audit::push_state_dir(&dir);
6356 let _cache = crate::cache::CacheEnv::acquire();
6357 let backend = StubBackend::default();
6358 let secret = "do-not-log";
6359 let response = dispatch_with(
6360 &backend,
6361 Action::ClipboardSet {
6362 text: Some(secret.into()),
6363 image: None,
6364 },
6365 );
6366 assert_eq!(response.to_value()["error"]["code"], "confirm_required");
6367 assert!(backend.clipboard_text.lock().unwrap().is_none());
6368 let line = last_audit(&dir);
6369 assert_eq!(line["action"], "clipboard_set");
6370 assert_eq!(line["decision"], "confirm");
6371 assert_eq!(line["chars"], secret.chars().count() as u64);
6372 let encoded = line.to_string();
6373 assert!(!encoded.contains(secret));
6374 assert!(line.get("text").is_none());
6375
6376 *backend.clipboard_text.lock().unwrap() = Some("ready".into());
6377 let got = dispatch_with(&backend, Action::ClipboardGet);
6378 assert!(got.ok, "{got:?}");
6379 assert_eq!(got.to_value()["kind"], "text");
6380 assert_eq!(got.to_value()["text"], "ready");
6381
6382 let empty = dispatch_with(
6383 &backend,
6384 Action::ClipboardSet {
6385 text: Some(String::new()),
6386 image: None,
6387 },
6388 );
6389 assert_eq!(empty.to_value()["error"]["code"], "invalid_arguments");
6390 let _ = std::fs::remove_dir_all(&dir);
6391 }
6392
6393 #[test]
6394 fn zoom_crops_a_window_region_and_rejects_the_far_edge() {
6395 let _cache = crate::cache::CacheEnv::acquire();
6396 let backend = StubBackend::default();
6397 let response = dispatch_with_context(
6398 &backend,
6399 Action::Zoom {
6400 output: None,
6401 region: Region::new(0, 0, 10, 10).unwrap(),
6402 format: Some(ImageFormat::Png),
6403 max_width: Some(0),
6404 canvas: None,
6405 base64: false,
6406 },
6407 &CallContext {
6408 app: Some("Notes".into()),
6409 session: Some("p8-zoom".into()),
6410 ..CallContext::default()
6411 },
6412 );
6413 assert!(response.ok, "{response:?}");
6414 let value = response.to_value();
6415 assert_eq!(value["width"], 20);
6416 assert_eq!(value["height"], 20);
6417 assert_eq!(value["logical_width"], 10);
6418 assert_eq!(value["logical_height"], 10);
6419
6420 let outside = dispatch_with_context(
6421 &backend,
6422 Action::Zoom {
6423 output: None,
6424 region: Region::new(0, 0, 401, 300).unwrap(),
6425 format: None,
6426 max_width: Some(0),
6427 canvas: None,
6428 base64: false,
6429 },
6430 &CallContext {
6431 app: Some("Notes".into()),
6432 session: Some("p8-zoom-out".into()),
6433 ..CallContext::default()
6434 },
6435 );
6436 assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
6437
6438 let both = dispatch_with_context(
6439 &backend,
6440 Action::Screenshot {
6441 output: None,
6442 region: Some(Region::new(0, 0, 10, 10).unwrap()),
6443 format: None,
6444 max_width: None,
6445 canvas: None,
6446 base64: false,
6447 marks: false,
6448 },
6449 &CallContext {
6450 app: Some("Notes".into()),
6451 ..CallContext::default()
6452 },
6453 );
6454 assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
6455 assert!(both.to_value()["error"]["message"]
6456 .as_str()
6457 .unwrap()
6458 .contains("not both"));
6459 }
6460
6461 #[test]
6464 fn browser_text_action_round_trips_with_only_selector() {
6465 let text: Action =
6466 serde_json::from_str(r#"{"action":"browser_text","selector":"h1"}"#).unwrap();
6467 match &text {
6468 Action::BrowserText {
6469 selector,
6470 target_id,
6471 } => {
6472 assert_eq!(selector, "h1");
6473 assert!(target_id.is_none());
6474 }
6475 other => panic!("unexpected {other:?}"),
6476 }
6477 let encoded = serde_json::to_string(&text).unwrap();
6478 let decoded: Action = serde_json::from_str(&encoded).unwrap();
6479 assert_eq!(decoded, text);
6480 }
6481
6482 #[test]
6483 fn browser_type_action_round_trips_with_text_and_optional_selector() {
6484 let action: Action =
6485 serde_json::from_str(r##"{"action":"browser_type","text":"hi","selector":"#name"}"##)
6486 .unwrap();
6487 match &action {
6488 Action::BrowserType {
6489 text,
6490 selector,
6491 target_id,
6492 } => {
6493 assert_eq!(text, "hi");
6494 assert_eq!(selector.as_deref(), Some("#name"));
6495 assert!(target_id.is_none());
6496 }
6497 other => panic!("unexpected {other:?}"),
6498 }
6499 let encoded = serde_json::to_string(&action).unwrap();
6500 let decoded: Action = serde_json::from_str(&encoded).unwrap();
6501 assert_eq!(decoded, action);
6502 }
6503
6504 #[test]
6505 fn browser_text_label_is_browser_text_and_class_is_observe() {
6506 let action = Action::BrowserText {
6508 selector: "h1".into(),
6509 target_id: None,
6510 };
6511 assert_eq!(action_label(&action), "browser_text");
6512 assert!(matches!(policy_class(&action), PolicyClass::Observe));
6513 }
6514
6515 #[test]
6516 fn browser_type_label_is_browser_type_and_class_is_mutate() {
6517 let action = Action::BrowserType {
6520 text: "abc".into(),
6521 selector: Some("#name".into()),
6522 target_id: None,
6523 };
6524 assert_eq!(action_label(&action), "browser_type");
6525 assert!(matches!(policy_class(&action), PolicyClass::Mutate));
6526 assert_eq!(clipboard_chars(&action), Some(3));
6527 }
6528
6529 #[test]
6530 fn browser_text_does_not_record_chars_in_audit() {
6531 let action = Action::BrowserText {
6533 selector: "h1".into(),
6534 target_id: None,
6535 };
6536 assert_eq!(clipboard_chars(&action), None);
6537 }
6538
6539 #[test]
6540 #[cfg(feature = "cdp")]
6541 fn browser_navigate_blocks_url_before_connecting() {
6542 let backend = StubBackend::default();
6546 let context = CallContext {
6547 cdp: true,
6548 cdp_url: Some("ws://127.0.0.1:1".into()),
6549 browser_blocked_urls: vec!["https://blocked.example.com".into()],
6550 ..CallContext::default()
6551 };
6552 let response = dispatch_with_context(
6553 &backend,
6554 Action::BrowserNavigate {
6555 url: "https://blocked.example.com".into(),
6556 target_id: None,
6557 },
6558 &context,
6559 );
6560 assert!(!response.ok);
6561 let value = response.to_value();
6562 assert_eq!(value["error"]["code"], "blocked_url");
6563 }
6564
6565 #[test]
6566 #[cfg(feature = "cdp")]
6567 fn browser_navigate_unblocked_url_still_attempts_connect() {
6568 let backend = StubBackend::default();
6571 let context = CallContext {
6572 cdp: true,
6573 cdp_url: Some("ws://127.0.0.1:1".into()),
6574 ..CallContext::default()
6575 };
6576 let response = dispatch_with_context(
6577 &backend,
6578 Action::BrowserNavigate {
6579 url: "https://example.com".into(),
6580 target_id: None,
6581 },
6582 &context,
6583 );
6584 assert!(!response.ok);
6585 let value = response.to_value();
6586 assert_eq!(value["error"]["code"], "cdp_unavailable");
6587 }
6588
6589 fn filtered(spec: &str) -> CallContext {
6594 CallContext {
6595 display: Some(spec.into()),
6596 ..CallContext::default()
6597 }
6598 }
6599
6600 fn code_of(response: &Response) -> String {
6601 response.to_value()["error"]["code"]
6602 .as_str()
6603 .unwrap_or("")
6604 .to_string()
6605 }
6606
6607 #[test]
6608 fn display_filter_rejects_a_point_outside_the_chosen_monitor() {
6609 let backend = StubBackend::with_two_displays();
6611 let context = filtered("2");
6612
6613 let response = dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &context);
6615 assert!(response.ok, "{response:?}");
6616
6617 let before = backend.moves.load(Ordering::SeqCst);
6620 for (x, y) in [(0, 0), (799, 599), (1440, 580), (800, 99), (900, 90)] {
6621 let response = dispatch_with_context(&backend, Action::Move { x, y }, &context);
6622 assert_eq!(code_of(&response), "invalid_arguments", "{x},{y}");
6623 assert_eq!(backend.moves.load(Ordering::SeqCst), before);
6624 }
6625 let message = dispatch_with_context(&backend, Action::Move { x: 0, y: 0 }, &context)
6626 .to_value()["error"]["message"]
6627 .as_str()
6628 .expect("message")
6629 .to_string();
6630 assert!(message.contains("640x480+800+100"), "{message}");
6631 }
6632
6633 #[test]
6634 fn display_filter_covers_click_move_drag_scroll_and_mouse_halves() {
6635 let backend = StubBackend::with_two_displays();
6636 let context = filtered("2");
6637 let inside = Action::Click {
6638 x: Some(900),
6639 y: Some(300),
6640 button: MouseButton::Left,
6641 double: false,
6642 count: 1,
6643 };
6644 let outside = Action::Click {
6645 x: Some(10),
6646 y: Some(10),
6647 button: MouseButton::Left,
6648 double: false,
6649 count: 1,
6650 };
6651
6652 assert!(dispatch_with_context(&backend, inside.clone(), &context).ok);
6653 assert_eq!(
6654 code_of(&dispatch_with_context(&backend, outside.clone(), &context)),
6655 "invalid_arguments"
6656 );
6657 assert_eq!(
6658 code_of(&dispatch_with_context(
6659 &backend,
6660 Action::DoubleClick {
6661 x: Some(10),
6662 y: Some(10),
6663 button: MouseButton::Left
6664 },
6665 &context
6666 )),
6667 "invalid_arguments"
6668 );
6669 assert_eq!(
6670 code_of(&dispatch_with_context(
6671 &backend,
6672 Action::MouseDown {
6673 x: 10,
6674 y: 10,
6675 button: MouseButton::Left
6676 },
6677 &context
6678 )),
6679 "invalid_arguments"
6680 );
6681 assert_eq!(
6682 code_of(&dispatch_with_context(
6683 &backend,
6684 Action::Scroll {
6685 x: Some(10),
6686 y: Some(10),
6687 dx: Some(0),
6688 dy: Some(1),
6689 direction: None,
6690 amount: None,
6691 },
6692 &context
6693 )),
6694 "invalid_arguments"
6695 );
6696 let path = vec![Point { x: 900, y: 300 }, Point { x: 10, y: 10 }];
6697 assert_eq!(
6698 code_of(&dispatch_with_context(
6699 &backend,
6700 Action::Drag {
6701 path,
6702 button: MouseButton::Left
6703 },
6704 &context
6705 )),
6706 "invalid_arguments"
6707 );
6708 }
6709
6710 #[test]
6711 fn a_coordinate_action_without_a_point_passes_the_filter() {
6712 let backend = StubBackend::with_two_displays();
6716 let response = dispatch_with_context(
6717 &backend,
6718 Action::Scroll {
6719 x: None,
6720 y: None,
6721 dx: Some(0),
6722 dy: Some(2),
6723 direction: None,
6724 amount: None,
6725 },
6726 &filtered("2"),
6727 );
6728 assert!(response.ok, "{response:?}");
6729 assert_eq!(backend.scrolls.lock().expect("scrolls").len(), 1);
6730 }
6731
6732 #[test]
6733 fn an_unknown_display_id_is_invalid_arguments() {
6734 let backend = StubBackend::with_two_displays();
6735 let response =
6736 dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &filtered("9"));
6737 assert_eq!(code_of(&response), "invalid_arguments");
6738
6739 let response = dispatch_with_context(&backend, Action::Displays, &filtered("99999"));
6742 assert!(response.ok, "{response:?}");
6743 let response = dispatch_with_context(&backend, Action::Isolation, &filtered("99999"));
6744 assert!(response.ok, "{response:?}");
6745 assert_eq!(backend.list_displays().unwrap().len(), 2);
6746
6747 let response = dispatch_with_context(&backend, Action::Displays, &filtered("nope"));
6749 assert_eq!(code_of(&response), "invalid_arguments");
6750 let response = dispatch_with_context(&backend, Action::Displays, &filtered("virtual"));
6754 assert!(response.ok, "{response:?}");
6755 }
6756
6757 #[test]
6758 fn virtual_is_unsupported_over_the_json_path() {
6759 let backend = StubBackend::default();
6762 let response = dispatch_with_context(
6763 &backend,
6764 Action::Move { x: 10, y: 10 },
6765 &filtered("virtual"),
6766 );
6767 assert_eq!(code_of(&response), "unsupported");
6768
6769 let response = dispatch_with_context(
6770 &backend,
6771 Action::Screenshot {
6772 output: None,
6773 region: None,
6774 format: None,
6775 max_width: None,
6776 canvas: None,
6777 base64: false,
6778 marks: false,
6779 },
6780 &filtered("virtual"),
6781 );
6782 assert_eq!(code_of(&response), "unsupported");
6783 }
6784
6785 #[test]
6786 fn the_display_filter_never_leaks_into_displays_or_isolation() {
6787 let backend = StubBackend::with_two_displays();
6788 let context = filtered("9");
6789 assert!(dispatch_with_context(&backend, Action::Displays, &context).ok);
6790 assert!(dispatch_with_context(&backend, Action::Isolation, &context).ok);
6791 }
6792
6793 #[test]
6794 fn a_screenshot_region_must_fit_inside_the_filtered_monitor() {
6795 let _cache = crate::cache::CacheEnv::acquire();
6796 let backend = StubBackend::with_two_displays();
6797 let context = filtered("2");
6798 let inside = Action::Screenshot {
6799 output: None,
6800 region: Some(Region::new(800, 100, 640, 480).unwrap()),
6801 format: None,
6802 max_width: None,
6803 canvas: None,
6804 base64: false,
6805 marks: false,
6806 };
6807 let response = dispatch_with_context(&backend, inside, &context);
6808 assert!(response.ok, "{response:?}");
6809
6810 let map = response.to_value();
6812 assert_eq!(map["origin"]["x"], 800);
6813 assert_eq!(map["origin"]["y"], 100);
6814 let _ = fs::remove_file(
6815 map["path"]
6816 .as_str()
6817 .expect("path")
6818 .parse::<std::path::PathBuf>()
6819 .expect("path"),
6820 );
6821
6822 let straddling = Action::Screenshot {
6823 output: None,
6824 region: Some(Region::new(700, 100, 200, 200).unwrap()),
6825 format: None,
6826 max_width: None,
6827 canvas: None,
6828 base64: false,
6829 marks: false,
6830 };
6831 assert_eq!(
6832 code_of(&dispatch_with_context(&backend, straddling, &context)),
6833 "invalid_arguments"
6834 );
6835 }
6836
6837 #[test]
6838 fn isolation_reports_the_matrix_and_the_infeasible_keys() {
6839 let response = dispatch_with(&StubBackend::default(), Action::Isolation);
6840 assert!(response.ok, "{response:?}");
6841 let value = response.to_value();
6842 assert_eq!(value["platform"], crate::isolation::platform());
6843 assert_eq!(value["display_filter"], true);
6844 assert_eq!(value["xvfb"], "linux only");
6845 assert_eq!(value["vm"], "documented, not shipped");
6846 assert_eq!(
6847 value["xvfb_recipe"]["server_command"],
6848 "Xvfb :99 -screen 0 1440x900x24"
6849 );
6850 for key in [
6851 "macos_vm_per_agent",
6852 "share_host_app_into_guest",
6853 "hide_virtual_display_from_user",
6854 "windows_virtual_display",
6855 "nested_wayland_input_without_portal",
6856 ] {
6857 assert!(
6858 value["infeasible"][key]
6859 .as_str()
6860 .is_some_and(|s| !s.is_empty()),
6861 "infeasible.{key} missing"
6862 );
6863 }
6864 }
6865
6866 #[test]
6867 fn isolation_is_read_only_and_ignores_the_state_flags() {
6868 assert_eq!(action_label(&Action::Isolation), "isolation");
6869 assert!(matches!(
6870 post_state_kind(&Action::Isolation),
6871 PostKind::ReadOnly
6872 ));
6873 let backend = StubBackend::default();
6874 let response = dispatch_with_context(
6875 &backend,
6876 Action::Isolation,
6877 &CallContext {
6878 return_state: true,
6879 include_screenshot: true,
6880 ..CallContext::default()
6881 },
6882 );
6883 assert!(response.ok);
6884 assert!(response.to_value().get("state").is_none());
6885 }
6886
6887 #[test]
6888 fn the_isolation_action_parses_from_json() {
6889 let request = parse_request(json!({"id": 7, "action": "isolation"})).unwrap();
6890 assert_eq!(request.id, Some(json!(7)));
6891 assert!(matches!(request.action, Action::Isolation));
6892 let request = parse_request(json!({"action": "isolation", "display": "1"})).unwrap();
6894 assert_eq!(request.context.display.as_deref(), Some("1"));
6895 }
6896
6897 fn scroll_action(dx: Option<i32>, dy: Option<i32>) -> Action {
6898 Action::Scroll {
6899 x: None,
6900 y: None,
6901 dx,
6902 dy,
6903 direction: None,
6904 amount: None,
6905 }
6906 }
6907
6908 fn near(value: f64, expected: f64) -> bool {
6910 (value - expected).abs() < 1e-9
6911 }
6912
6913 fn tier_context(tier: &str) -> CallContext {
6914 CallContext {
6915 tier: Some(tier.to_string()),
6916 ..CallContext::default()
6917 }
6918 }
6919
6920 #[test]
6921 fn a_scroll_reports_the_tier_that_actually_moved_the_scroll_bar() {
6922 let backend = StubBackend::default();
6923 let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6924 assert_eq!(value["ok"], json!(true));
6925 assert_eq!(value["via"], json!("event"));
6926 assert_eq!(value["scrolled"], json!(true));
6927 assert_eq!(value["before"], json!(0.0));
6928 assert!(near(value["after"].as_f64().unwrap_or_default(), 0.15));
6929 assert_eq!(value["tier"], json!("event"));
6930 assert!(value.get("warnings").is_none());
6932 }
6933
6934 #[test]
6935 fn a_dropped_wheel_event_falls_back_to_the_ax_scroll_bar_and_says_so() {
6936 let backend = StubBackend {
6937 scroll_event_delivered: false,
6938 ..StubBackend::default()
6939 };
6940 let value = dispatch_with(&backend, scroll_action(None, Some(2))).to_value();
6941 assert_eq!(value["ok"], json!(true));
6942 assert_eq!(value["via"], json!("ax_scrollbar"));
6943 assert_eq!(value["scrolled"], json!(true));
6944 assert_eq!(value["before"], json!(0.0));
6945 assert!(near(value["after"].as_f64().unwrap_or_default(), 0.1));
6946 assert_eq!(
6947 value["warnings"][0],
6948 json!("wheel event not delivered to background app; used AX scroll bar")
6949 );
6950 }
6951
6952 #[test]
6953 fn a_bar_already_at_its_limit_is_ok_and_never_claims_a_scroll() {
6954 let backend = StubBackend {
6955 scroll_event_delivered: false,
6956 scroll_value: std::sync::Mutex::new(1.0),
6957 ..StubBackend::default()
6958 };
6959 let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6960 assert_eq!(value["ok"], json!(true));
6961 assert_eq!(value["scrolled"], json!(false));
6962 assert_eq!(value["before"], json!(1.0));
6963 assert_eq!(value["after"], json!(1.0));
6964 let warning = value["warnings"][0].as_str().unwrap_or_default();
6965 assert!(warning.contains("bottom"), "{warning}");
6966 }
6967
6968 #[test]
6969 fn a_scroll_that_cannot_be_verified_is_never_reported_as_a_scroll() {
6970 let backend = StubBackend {
6971 scroll_area_missing: true,
6972 ..StubBackend::default()
6973 };
6974 let response = dispatch_with_context(
6975 &backend,
6976 scroll_action(None, Some(3)),
6977 &tier_context("event"),
6978 );
6979 assert!(!response.ok);
6980 assert_eq!(code_of(&response), "execution_failed");
6981 let message = response.to_value()["error"]["message"]
6982 .as_str()
6983 .unwrap_or_default()
6984 .to_string();
6985 assert!(message.contains("cannot be verified"), "{message}");
6986 assert!(response.to_value().get("scrolled").is_none());
6987 }
6988
6989 #[test]
6990 fn an_ax_refusal_is_a_failure_and_not_a_scroll() {
6991 let backend = StubBackend {
6992 scroll_event_delivered: false,
6993 scroll_ax_refused: true,
6994 ..StubBackend::default()
6995 };
6996 let response = dispatch_with_context(
6997 &backend,
6998 scroll_action(None, Some(3)),
6999 &tier_context("event"),
7000 );
7001 assert!(!response.ok);
7002 assert_eq!(code_of(&response), "execution_failed");
7003 assert!(
7004 response.to_value()["error"]["message"]
7005 .as_str()
7006 .unwrap_or_default()
7007 .contains("refused"),
7008 "{:?}",
7009 response.to_value()
7010 );
7011 }
7012
7013 #[test]
7014 fn a_zero_delta_scroll_reports_no_scroll_and_does_not_panic() {
7015 for tier in ["event", "ax"] {
7018 let backend = StubBackend::default();
7019 let value = dispatch_with_context(
7020 &backend,
7021 scroll_action(Some(0), Some(0)),
7022 &tier_context(tier),
7023 )
7024 .to_value();
7025 assert_eq!(value["ok"], json!(true), "tier {tier}");
7026 assert_eq!(value["scrolled"], json!(false), "tier {tier}");
7027 assert_eq!(value["before"], json!(null), "tier {tier}");
7028 assert_eq!(value["after"], json!(null), "tier {tier}");
7029 assert!(value.get("warnings").is_none(), "tier {tier}");
7030 }
7031 }
7032
7033 #[test]
7034 fn an_explicit_ax_scroll_runs_the_scroll_bar_path_without_posting_an_event() {
7035 let backend = StubBackend::default();
7036 let response =
7037 dispatch_with_context(&backend, scroll_action(None, Some(3)), &tier_context("ax"));
7038 let value = response.to_value();
7039 assert_eq!(value["ok"], json!(true));
7040 assert_eq!(value["tier"], json!("ax"));
7041 assert_eq!(value["via"], json!("ax_scrollbar"));
7042 assert_eq!(value["scrolled"], json!(true));
7043 assert!(
7045 value.get("warnings").is_none(),
7046 "unexpected warnings: {:?}",
7047 value["warnings"]
7048 );
7049 assert_eq!(
7050 *backend.tiers_run.lock().unwrap(),
7051 vec!["ax".to_string()],
7052 "tier ax must not fall back to activate"
7053 );
7054 }
7055
7056 #[test]
7057 fn require_background_never_reaches_the_activate_scroll_tier() {
7058 let backend = StubBackend {
7061 scroll_event_delivered: false,
7062 scroll_area_missing: true,
7063 ..StubBackend::default()
7064 };
7065 let context = CallContext {
7066 tier: Some("event".into()),
7067 require_background: true,
7068 ..CallContext::default()
7069 };
7070 let response = dispatch_with_context(&backend, scroll_action(None, Some(3)), &context);
7071 assert!(!response.ok);
7072 assert_eq!(code_of(&response), "execution_failed");
7073 let tiers = backend.tiers_run.lock().unwrap().clone();
7074 assert!(
7075 !tiers.iter().any(|tier| tier == "activate"),
7076 "activate must never run: {tiers:?}"
7077 );
7078 }
7079
7080 #[test]
7081 fn a_horizontal_scroll_reports_the_horizontal_bar() {
7082 let backend = StubBackend {
7083 scroll_event_delivered: false,
7084 scroll_value: std::sync::Mutex::new(0.5),
7085 ..StubBackend::default()
7086 };
7087 let value = dispatch_with(&backend, scroll_action(Some(-2), None)).to_value();
7088 assert_eq!(value["via"], json!("ax_scrollbar"));
7089 assert_eq!(value["before"], json!(0.5));
7090 assert!(near(value["after"].as_f64().unwrap_or_default(), 0.4));
7091 assert_eq!(value["dx"], json!(-2));
7092 }
7093}
7094
7095#[allow(clippy::items_after_test_module)]
7103fn browser_action_options(
7104 context: &CallContext,
7105 subject: &policy::Subject,
7106 target_id: Option<String>,
7107) -> crate::browser::ActionOptions {
7108 let mut policy_blocked_urls = context.browser_blocked_urls.clone();
7113 if policy_blocked_urls.is_empty() {
7114 if let Ok(loaded) = policy::load_policy(context.policy.as_deref().map(Path::new)) {
7115 policy_blocked_urls = loaded.blocked_urls().to_vec();
7116 }
7117 }
7118 crate::browser::ActionOptions {
7119 debug_url: context.cdp_url.clone(),
7120 app: context.app.clone(),
7121 app_pid: subject.pid,
7122 target_id,
7123 url: None,
7124 selector: None,
7125 text: None,
7126 script: None,
7127 command_timeout: Duration::from_secs(30),
7128 policy_blocked_urls,
7129 policy_subject: Some(subject.clone()),
7130 }
7131}
7132
7133fn browser_require_cdp(context: &CallContext) -> Result<(), ComputerError> {
7134 if !context.cdp {
7135 return Err(ComputerError::Unsupported(
7136 "browser commands require --cdp (or `cdp: true` in the JSON request)".into(),
7137 ));
7138 }
7139 Ok(())
7140}
7141
7142fn browser_tabs(context: &CallContext, subject: &policy::Subject) -> Result<Value, ComputerError> {
7143 browser_require_cdp(context)?;
7144 let opts = browser_action_options(context, subject, None);
7145 let tabs = crate::browser::cdp::list_tabs(&opts)?;
7146 serde_json::to_value(&tabs)
7147 .map_err(|error| ComputerError::ExecutionFailed(format!("encode tabs: {error}")))
7148}
7149
7150fn browser_activate(
7151 context: &CallContext,
7152 subject: &policy::Subject,
7153 target_id: Option<String>,
7154) -> Result<Value, ComputerError> {
7155 browser_require_cdp(context)?;
7156 let target = target_id.ok_or_else(|| {
7157 ComputerError::InvalidArguments("browser activate requires --target".into())
7158 })?;
7159 let opts = browser_action_options(context, subject, Some(target));
7160 let active = crate::browser::cdp::activate_tab(&opts)?;
7161 Ok(json!({ "url": active }))
7162}
7163
7164fn browser_navigate(
7165 context: &CallContext,
7166 subject: &policy::Subject,
7167 url: String,
7168 target_id: Option<String>,
7169) -> Result<Value, ComputerError> {
7170 browser_require_cdp(context)?;
7171 let opts = browser_action_options(context, subject, target_id);
7172 let landed = crate::browser::cdp::navigate(&crate::browser::ActionOptions {
7173 url: Some(url.clone()),
7174 ..opts
7175 })?;
7176 Ok(json!({ "url": landed }))
7177}
7178
7179fn browser_click(
7180 context: &CallContext,
7181 subject: &policy::Subject,
7182 selector: String,
7183 target_id: Option<String>,
7184) -> Result<Value, ComputerError> {
7185 browser_require_cdp(context)?;
7186 let opts = browser_action_options(context, subject, target_id);
7187 crate::browser::cdp::click(&crate::browser::ActionOptions {
7188 selector: Some(selector.clone()),
7189 ..opts
7190 })?;
7191 Ok(json!({ "selector": selector }))
7192}
7193
7194fn browser_text(
7195 context: &CallContext,
7196 subject: &policy::Subject,
7197 selector: String,
7198 target_id: Option<String>,
7199) -> Result<Value, ComputerError> {
7200 browser_require_cdp(context)?;
7201 let opts = browser_action_options(context, subject, target_id);
7202 let text = crate::browser::cdp::read_inner_text(&crate::browser::ActionOptions {
7203 selector: Some(selector.clone()),
7204 ..opts
7205 })?;
7206 Ok(json!({ "selector": selector, "text": text }))
7207}
7208
7209fn browser_type(
7210 context: &CallContext,
7211 subject: &policy::Subject,
7212 text: String,
7213 selector: Option<String>,
7214 target_id: Option<String>,
7215) -> Result<Value, ComputerError> {
7216 browser_require_cdp(context)?;
7217 let opts = browser_action_options(context, subject, target_id);
7218 crate::browser::cdp::type_text(&crate::browser::ActionOptions {
7219 selector,
7220 text: Some(text.clone()),
7221 ..opts
7222 })?;
7223 Ok(json!({ "chars": text.chars().count() as u64 }))
7226}
7227
7228fn browser_eval(
7229 context: &CallContext,
7230 subject: &policy::Subject,
7231 script: String,
7232 target_id: Option<String>,
7233) -> Result<Value, ComputerError> {
7234 browser_require_cdp(context)?;
7235 let opts = browser_action_options(context, subject, target_id);
7236 let value = crate::browser::cdp::eval(&crate::browser::ActionOptions {
7237 script: Some(script.clone()),
7238 ..opts
7239 })?;
7240 let length = script.chars().count() as u64;
7244 use std::hash::{Hash, Hasher};
7245 let mut hasher = std::collections::hash_map::DefaultHasher::new();
7246 script.hash(&mut hasher);
7247 let hash = hasher.finish();
7248 Ok(json!({
7249 "result": value,
7250 "script_chars": length,
7251 "script_hash": format!("{hash:016x}"),
7252 }))
7253}