Skip to main content

computer_use/
protocol.rs

1//! Actions, JSON requests, and the shared dispatcher.
2
3use std::fs;
4use std::panic::{catch_unwind, AssertUnwindSafe};
5use std::path::{Path, PathBuf};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::thread;
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use serde::de::{self, Deserializer};
11use serde::ser::{Serialize, Serializer};
12use serde::{Deserialize, Serialize as SerializeDerive};
13use serde_json::{json, Map, Value};
14
15use crate::attempt::{self, AttemptFlags};
16use crate::backend::clipboard::ClipboardContents;
17use crate::backend::create_desktop_backend;
18use crate::backend::{validate_key, DesktopBackend, WindowTarget};
19use crate::cache;
20use crate::element::{self, ElementError, ElementOp, ElementOutcome, ElementRequest};
21use crate::error::{ComputerError, ErrorBody};
22use crate::image_util::{self, encode_base64};
23use crate::input_session::{self, InputSession};
24use crate::keys::{parse_key_combo, parse_key_sequence};
25use crate::policy;
26use crate::state::collect_state_for;
27use crate::target::{
28    filter_windows_for_pid, parse_app_selector, parse_window_selector, resolve_app, resolve_window,
29    WindowSelector,
30};
31use crate::tier::{self, InputKind, KeyStroke, TierName};
32use crate::tree::TreeQuery;
33use crate::types::{
34    parse_path, region_inside, Frame, ImageFormat, Modifier, MouseButton, Point, Region,
35    ScrollDirection,
36};
37use crate::vision::{OcrBlock, OcrResult};
38
39const MAX_WAIT_MS: u128 = 24 * 60 * 60 * 1000;
40
41/// Default recursion depth for `menu_list`. Matches the AXMenuBar chain
42/// (menu bar -> menu bar item -> menu -> submenu) plus one extra level.
43pub const MENU_DEFAULT_DEPTH: u32 = 4;
44
45/// Maximum recursion depth accepted by `menu_list`. AX nesting beyond this
46/// stops with a `*` placeholder so the report never explodes.
47pub const MENU_MAX_DEPTH: u32 = 8;
48
49/// Default timeout for a `shortcuts run` invocation, in seconds.
50pub const SHORTCUTS_TIMEOUT_SECS: u64 = 60;
51
52/// Default timeout for an `osascript` invocation, in seconds.
53pub const APPLE_EVENT_TIMEOUT_SECS: u64 = 30;
54
55/// One desktop action. Serialized with an `action` tag.
56///
57/// ```json
58/// {"action":"click","x":10,"y":20}
59/// ```
60#[derive(Debug, Clone, PartialEq, SerializeDerive, Deserialize)]
61#[serde(tag = "action", rename_all = "snake_case")]
62pub enum Action {
63    /// Capture the screen or a region.
64    Screenshot {
65        /// File path. A temp path is used when omitted.
66        #[serde(default, skip_serializing_if = "Option::is_none")]
67        output: Option<String>,
68        /// Logical region, as `x,y,width,height` or an object.
69        #[serde(default, skip_serializing_if = "Option::is_none")]
70        region: Option<Region>,
71        /// `png` (default) or `jpeg`.
72        #[serde(default, skip_serializing_if = "Option::is_none")]
73        format: Option<ImageFormat>,
74        /// Scale the image down so its width is at most this many pixels.
75        ///
76        /// Absent means 1440. `0` keeps the source width.
77        #[serde(default, skip_serializing_if = "Option::is_none")]
78        max_width: Option<u32>,
79        /// Letterbox into `WxH` pixels, for example `1440x900`.
80        #[serde(default, skip_serializing_if = "Option::is_none")]
81        canvas: Option<String>,
82        /// Include standard base64 bytes in `data`.
83        #[serde(default, skip_serializing_if = "is_false")]
84        base64: bool,
85        /// Stamp numbered element rectangles on the image. tree indices
86        /// (`n`) on a window target; OCR indices (`on`) when a tree cannot
87        /// be read. `--marks` from the CLI turns this on.
88        #[serde(default, skip_serializing_if = "is_false")]
89        marks: bool,
90    },
91    /// Move to a point and click, or press an element when `element` is set.
92    Click {
93        /// Logical x. Omit it when `element` is set.
94        #[serde(default, skip_serializing_if = "Option::is_none")]
95        x: Option<i32>,
96        /// Logical y. Omit it when `element` is set.
97        #[serde(default, skip_serializing_if = "Option::is_none")]
98        y: Option<i32>,
99        /// Button. Defaults to left.
100        #[serde(default)]
101        button: MouseButton,
102        /// Click twice when true. With `element`, this is `press` count 2.
103        #[serde(default, skip_serializing_if = "is_false")]
104        double: bool,
105        /// Repeat count. 1, 2, or 3. `double: true` means 2 when `count` is 1.
106        #[serde(default = "default_press_count", skip_serializing_if = "is_one")]
107        count: u8,
108    },
109    /// Click three times.
110    #[serde(rename = "triple_click", alias = "triple-click")]
111    TripleClick {
112        /// Logical x. Omit it when `element` is set.
113        #[serde(default, skip_serializing_if = "Option::is_none")]
114        x: Option<i32>,
115        /// Logical y. Omit it when `element` is set.
116        #[serde(default, skip_serializing_if = "Option::is_none")]
117        y: Option<i32>,
118        /// Button. Defaults to left.
119        #[serde(default)]
120        button: MouseButton,
121    },
122    /// Click twice. Equivalent to `click` with `double: true`.
123    #[serde(rename = "double_click")]
124    DoubleClick {
125        /// Logical x. Omit it when `element` is set.
126        #[serde(default, skip_serializing_if = "Option::is_none")]
127        x: Option<i32>,
128        /// Logical y. Omit it when `element` is set.
129        #[serde(default, skip_serializing_if = "Option::is_none")]
130        y: Option<i32>,
131        /// Button. Defaults to left.
132        #[serde(default)]
133        button: MouseButton,
134    },
135    /// Press an element. `count` is 1, 2, or 3.
136    Press {
137        /// Repeat count. Defaults to 1.
138        #[serde(default = "default_press_count")]
139        count: u8,
140        /// Mouse button. Left uses `AXPress`. Right uses `AXShowMenu` when advertised.
141        #[serde(default)]
142        button: MouseButton,
143    },
144    /// Button down at the first point, dragged events through the rest, up at the last.
145    Drag {
146        /// At least two points. A JSON array, or a flat `x,y,x,y` string.
147        #[serde(alias = "points", deserialize_with = "deserialize_path")]
148        path: Vec<Point>,
149        /// Mouse button. Defaults to left.
150        #[serde(default)]
151        button: MouseButton,
152    },
153    /// Button down. A later `mouse_up`, or the end of a one-shot command, releases it.
154    #[serde(alias = "mouse-down")]
155    MouseDown {
156        /// Logical x.
157        x: i32,
158        /// Logical y.
159        y: i32,
160        /// Mouse button. Defaults to left.
161        #[serde(default)]
162        button: MouseButton,
163    },
164    /// Button up.
165    #[serde(alias = "mouse-up")]
166    MouseUp {
167        /// Logical x.
168        x: i32,
169        /// Logical y.
170        y: i32,
171        /// Mouse button. Defaults to left.
172        #[serde(default)]
173        button: MouseButton,
174    },
175    /// Key down, sleep `ms`, key up. One combination. `ms` above 30000 is refused.
176    #[serde(alias = "hold-key")]
177    HoldKey {
178        /// One combo such as `shift` or `ctrl+alt+Delete`.
179        keys: String,
180        /// Milliseconds to hold.
181        ms: u64,
182    },
183    /// Read the clipboard. Text wins over an image. Empty is `kind: empty`.
184    ClipboardGet,
185    /// Write the clipboard. Confirmed. The audit stores the character count, not the text.
186    #[serde(alias = "clipboard-set")]
187    ClipboardSet {
188        /// Text to write. Empty is `invalid_arguments`.
189        #[serde(default, skip_serializing_if = "Option::is_none")]
190        text: Option<String>,
191        /// PNG path. Used when `text` is absent.
192        #[serde(default, skip_serializing_if = "Option::is_none")]
193        image: Option<String>,
194    },
195    /// Screenshot of a region. A window target crops that region in window-local points.
196    Zoom {
197        /// File path. A temp path is used when omitted.
198        #[serde(default, skip_serializing_if = "Option::is_none")]
199        output: Option<String>,
200        /// Logical region, as `x,y,width,height` or an object. Required.
201        region: Region,
202        /// `png` (default) or `jpeg`.
203        #[serde(default, skip_serializing_if = "Option::is_none")]
204        format: Option<ImageFormat>,
205        /// Scale the image down so its width is at most this many pixels.
206        #[serde(default, skip_serializing_if = "Option::is_none")]
207        max_width: Option<u32>,
208        /// Letterbox into `WxH` pixels, for example `1440x900`.
209        #[serde(default, skip_serializing_if = "Option::is_none")]
210        canvas: Option<String>,
211        /// Include standard base64 bytes in `data`.
212        #[serde(default, skip_serializing_if = "is_false")]
213        base64: bool,
214    },
215    /// Set `AXFocused` on an element.
216    Focus,
217    /// Set an element's value. Refused for secure fields.
218    #[serde(alias = "set-value")]
219    SetValue {
220        /// New value.
221        value: String,
222    },
223    /// Set the selected character range. Refused for secure fields.
224    #[serde(alias = "select-text")]
225    SelectText {
226        /// First selected character.
227        start: i64,
228        /// Character after the selection.
229        end: i64,
230    },
231    /// Perform an allowlisted accessibility action.
232    #[serde(alias = "perform-action")]
233    PerformAction {
234        /// Action name such as `AXShowMenu`.
235        name: String,
236    },
237    /// Scroll an element into view.
238    #[serde(
239        alias = "scroll-to",
240        alias = "scroll_to_visible",
241        alias = "scroll-to-visible"
242    )]
243    ScrollTo,
244    /// Move the pointer without clicking.
245    #[serde(alias = "move_mouse")]
246    Move {
247        /// Logical x.
248        x: i32,
249        /// Logical y.
250        y: i32,
251    },
252    /// Type text into the focused control.
253    Type {
254        /// Text to type.
255        text: String,
256    },
257    /// Press a key combination such as `cmd+shift+4`.
258    Key {
259        /// Combo string.
260        keys: String,
261    },
262    /// Scroll the wheel. Positive `dy` is down. Positive `dx` is right.
263    Scroll {
264        /// Optional pointer x. Must be paired with `y`.
265        #[serde(default, skip_serializing_if = "Option::is_none")]
266        x: Option<i32>,
267        /// Optional pointer y. Must be paired with `x`.
268        #[serde(default, skip_serializing_if = "Option::is_none")]
269        y: Option<i32>,
270        /// Horizontal wheel steps.
271        #[serde(default, alias = "scroll_x", skip_serializing_if = "Option::is_none")]
272        dx: Option<i32>,
273        /// Vertical wheel steps.
274        #[serde(default, alias = "scroll_y", skip_serializing_if = "Option::is_none")]
275        dy: Option<i32>,
276        /// Direction used with `amount`.
277        #[serde(default, skip_serializing_if = "Option::is_none")]
278        direction: Option<ScrollDirection>,
279        /// Wheel steps used with `direction`. Defaults to 1.
280        #[serde(default, skip_serializing_if = "Option::is_none")]
281        amount: Option<i32>,
282    },
283    /// Screen, cursor, frontmost window, and a best-effort accessibility tree.
284    #[serde(alias = "get-state")]
285    GetState,
286    /// Sleep, then return.
287    Wait {
288        /// Milliseconds. Added to `seconds` when both are set.
289        #[serde(default, skip_serializing_if = "Option::is_none")]
290        ms: Option<u64>,
291        /// Seconds, including fractions. Added to `ms`.
292        #[serde(default, skip_serializing_if = "Option::is_none")]
293        seconds: Option<f64>,
294    },
295    /// List displays without capturing pixels.
296    Displays,
297    /// Report what this build can isolate and what it cannot.
298    Isolation,
299    /// List running applications. `all` includes accessory and prohibited apps.
300    ListApps {
301        /// Include accessory and prohibited activation policies.
302        #[serde(default, rename = "all")]
303        all: bool,
304    },
305    /// List on-screen windows. `app` comes from the shared request context.
306    ListWindows,
307    /// Launch an application named by the shared `app` field.
308    Launch,
309    /// Activate an application named by the shared `app` field.
310    Activate,
311    /// Ask an application to quit. `force` requests a forced terminate.
312    Quit {
313        /// Force-quit when the platform supports it. Still requires confirmation.
314        #[serde(default, skip_serializing_if = "is_false")]
315        force: bool,
316    },
317    /// Report whether policy would allow `app`. Does not perform an action.
318    #[serde(rename = "policy_check")]
319    PolicyCheck,
320    /// Pause mutating actions until `resume`.
321    Pause,
322    /// Clear the pause flag.
323    Resume,
324    /// Report permissions and the TCC identity.
325    ///
326    /// The process exits 0 when the report is produced. `permissions_ok` in the
327    /// body is the permission aggregate. `prompt` asks macOS to show dialogs.
328    /// `request` is an alias of `prompt`.
329    Doctor {
330        /// Prompt for Accessibility, Screen Recording, Post Event, and Input Monitoring.
331        #[serde(default, alias = "request")]
332        prompt: bool,
333        /// Open the first missing System Settings pane. macOS only.
334        #[serde(default)]
335        open_settings: bool,
336        /// Linux only: force X11 detection even when WAYLAND_DISPLAY is set.
337        /// Other platforms ignore the field so the dispatcher can pass it
338        /// unconditionally.
339        #[serde(default)]
340        x11: bool,
341    },
342    /// Numbered accessibility tree for the key window.
343    Tree,
344    /// Restore Electron accessibility flags and stop observers.
345    SessionEnd,
346    /// Read the pointer. The Anthropic adapter maps `cursor_position` here.
347    CursorPosition,
348    /// Run several actions that share one target and one cache revision sequence.
349    Batch {
350        /// Stop after the first failed step. Defaults to true.
351        #[serde(default = "default_stop_on_error")]
352        stop_on_error: bool,
353        /// Native action objects. A nested `batch` is rejected at dispatch.
354        actions: Vec<Value>,
355    },
356    /// Remove empty session directories and stale ones.
357    #[serde(alias = "session-prune")]
358    SessionPrune {
359        /// Report matches without deleting them.
360        #[serde(default)]
361        dry_run: bool,
362        /// Age in seconds after which a dead owner's session is stale. Absent means 24 hours.
363        #[serde(default, skip_serializing_if = "Option::is_none")]
364        ttl_secs: Option<u64>,
365    },
366    /// Walk an application's menu bar. Reads AXMenuBar -> AXMenuBarItem -> AXMenu.
367    ///
368    /// `--app` selects the running app. The default depth is 4; `depth` caps it.
369    /// Matches are case-insensitive. Disabled items are still reported.
370    #[serde(alias = "menu_list", alias = "menu-list")]
371    MenuList {
372        /// Override the shared `--app` flag with one of these variants: name, bundle id, or pid.
373        #[serde(default, skip_serializing_if = "Option::is_none")]
374        app: Option<String>,
375        /// Maximum depth below the menu bar. Default: 4. Caps recursive submenus.
376        #[serde(default, skip_serializing_if = "Option::is_none")]
377        depth: Option<u32>,
378    },
379    /// Click a menu item by `">"`-separated titles.
380    ///
381    /// `--app` selects the running app. `--path` is a list of titles such as
382    /// `"File > Save As > Save"`. Matches are case-insensitive. Disabled items
383    /// are not pressed.
384    #[serde(alias = "menu_click", alias = "menu-click")]
385    MenuClick {
386        /// Override the shared `--app` flag.
387        #[serde(default, skip_serializing_if = "Option::is_none")]
388        app: Option<String>,
389        /// Path of menu titles separated by `>`.
390        path: String,
391    },
392    /// Send an Apple event via `osascript`. macOS only.
393    ///
394    /// `--app` is recorded and used by the deny policy; the script body decides
395    /// which application the event targets. `--language` is `applescript` (the
396    /// default) or `javascript`. The script body is never logged verbatim.
397    #[serde(alias = "apple_event", alias = "apple-event")]
398    AppleEvent {
399        /// Override the shared `--app` flag.
400        #[serde(default, skip_serializing_if = "Option::is_none")]
401        app: Option<String>,
402        /// Script body passed to `osascript -e`. Never logged verbatim.
403        script: String,
404        /// `applescript` (default) or `javascript`.
405        #[serde(default, skip_serializing_if = "Option::is_none")]
406        language: Option<AppleEventLanguage>,
407    },
408    /// Run a Shortcuts workflow by name. macOS only.
409    ///
410    /// The `shortcuts` binary is called with the workflow name as a single
411    /// argument. A missing binary is `execution_failed`.
412    #[serde(alias = "shortcuts_run", alias = "shortcuts-run")]
413    ShortcutsRun {
414        /// Workflow name.
415        name: String,
416    },
417    /// List the Dock items. macOS only.
418    #[serde(alias = "dock_list", alias = "dock-list")]
419    DockList {},
420    /// Click a Dock item by title. macOS only.
421    #[serde(alias = "dock_click", alias = "dock-click")]
422    DockClick {
423        /// Dock item title.
424        title: String,
425    },
426    /// Group the on-screen windows by the screen that contains each window's center.
427    ///
428    /// Mission Control moves are not attempted; the report only describes the
429    /// current layout. The skylight feature, when active, may move windows
430    /// between spaces; otherwise the action reports the visible state only.
431    #[serde(alias = "spaces")]
432    Spaces {},
433    /// List the tabs the running Chrome exposes. Requires `--features cdp`.
434    /// Without the feature this returns `unsupported`.
435    #[serde(rename = "browser_tabs", alias = "browser-tabs")]
436    BrowserTabs {},
437    /// Activate a tab by `target_id`. Requires `--features cdp`.
438    #[serde(rename = "browser_activate", alias = "browser-activate")]
439    BrowserActivate {
440        /// CDP `targetId` to bring to the front.
441        #[serde(default, skip_serializing_if = "Option::is_none")]
442        target_id: Option<String>,
443    },
444    /// Navigate the named tab (or first page) to `url`. The URL is checked
445    /// against `[browser].blocked_urls` before the websocket is touched.
446    #[serde(rename = "browser_navigate", alias = "browser-navigate")]
447    BrowserNavigate {
448        /// Destination URL.
449        url: String,
450        /// Restrict to one tab. Optional.
451        #[serde(default, skip_serializing_if = "Option::is_none")]
452        target_id: Option<String>,
453    },
454    /// Click an element matched by CSS `selector`. Requires `--features cdp`.
455    #[serde(rename = "browser_click", alias = "browser-click")]
456    BrowserClick {
457        /// CSS selector.
458        selector: String,
459        /// Restrict to one tab. Optional.
460        #[serde(default, skip_serializing_if = "Option::is_none")]
461        target_id: Option<String>,
462    },
463    /// Read the `innerText` of the first element matched by CSS `selector`.
464    /// Requires `--features cdp`. The current page URL is checked against
465    /// `[browser].blocked_urls` first; a selector that matches nothing returns
466    /// `element_not_found`.
467    #[serde(rename = "browser_text", alias = "browser-text")]
468    BrowserText {
469        /// CSS selector whose `innerText` should be returned.
470        selector: String,
471        /// Restrict to one tab. Optional.
472        #[serde(default, skip_serializing_if = "Option::is_none")]
473        target_id: Option<String>,
474    },
475    /// Type `text` into the element matched by `selector` (or the page body
476    /// when no selector is given). Requires `--features cdp`. Classified like
477    /// [`Action::Type`] for audit: the body is never stored, only its
478    /// character count. The current page URL is checked against
479    /// `[browser].blocked_urls` first.
480    #[serde(rename = "browser_type", alias = "browser-type")]
481    BrowserType {
482        /// Text to type.
483        text: String,
484        /// Optional CSS selector. When omitted the page body receives input.
485        #[serde(default, skip_serializing_if = "Option::is_none")]
486        selector: Option<String>,
487        /// Restrict to one tab. Optional.
488        #[serde(default, skip_serializing_if = "Option::is_none")]
489        target_id: Option<String>,
490    },
491    /// Evaluate `script` in the page context and return the JSON value.
492    /// Requires `--features cdp`. The script body is never logged verbatim;
493    /// only its length and a small hash appear in the audit row.
494    #[serde(rename = "browser_eval", alias = "browser-eval")]
495    BrowserEval {
496        /// JavaScript source. Quoted on the CLI via `--eval`; here it is just
497        /// a string field that the caller has already prepared.
498        script: String,
499        /// Restrict to one tab. Optional.
500        #[serde(default, skip_serializing_if = "Option::is_none")]
501        target_id: Option<String>,
502    },
503    /// Run OCR on the most recent capture for `--app`/window target. macOS
504    /// only. Linux returns `unsupported`; Windows is a stub until P12.
505    #[serde(rename = "ocr")]
506    Ocr {
507        /// Re-run OCR even when `--app` already produced it. Defaults to false.
508        #[serde(default, skip_serializing_if = "is_false")]
509        force: bool,
510    },
511}
512
513fn is_false(value: &bool) -> bool {
514    !*value
515}
516
517fn is_one(value: &u8) -> bool {
518    *value == 1
519}
520
521fn default_press_count() -> u8 {
522    1
523}
524
525/// Language of an Apple event script. macOS only.
526///
527/// `applescript` is the default. `javascript` selects the JavaScript for
528/// Automation (`JXA`) runtime in `osascript`.
529#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDerive, Deserialize)]
530#[serde(rename_all = "snake_case")]
531pub enum AppleEventLanguage {
532    /// AppleScript. Default.
533    #[default]
534    #[serde(alias = "applescript", alias = "osa")]
535    AppleScript,
536    /// JavaScript for Automation. `osascript -l JavaScript`.
537    #[serde(alias = "javascript", alias = "js", alias = "JavaScript")]
538    JavaScript,
539}
540
541impl AppleEventLanguage {
542    /// Short tag passed to `osascript -l`.
543    pub fn as_str(&self) -> &'static str {
544        match self {
545            Self::AppleScript => "AppleScript",
546            Self::JavaScript => "JavaScript",
547        }
548    }
549}
550
551fn deserialize_path<'de, D>(deserializer: D) -> Result<Vec<Point>, D::Error>
552where
553    D: Deserializer<'de>,
554{
555    let value = Value::deserialize(deserializer)?;
556    match value {
557        Value::String(text) => parse_path(&text).map_err(de::Error::custom),
558        Value::Array(_) => serde_json::from_value(value).map_err(de::Error::custom),
559        _ => Err(de::Error::custom(
560            "path must be an array of points or an x,y,x,y string",
561        )),
562    }
563}
564
565fn default_stop_on_error() -> bool {
566    true
567}
568
569/// Flags shared by every action. JSON keys are removed before the action is decoded.
570#[derive(Debug, Clone, PartialEq, Default)]
571pub struct CallContext {
572    /// Target application: name, bundle id, or pid.
573    pub app: Option<String>,
574    /// Target window: title, z-order index, or `id:<number>`.
575    pub window: Option<String>,
576    /// Element index in the current revision. Used from P3.
577    pub element: Option<u32>,
578    /// Emit a tree diff when a baseline exists.
579    pub diff: bool,
580    /// Replace the tree baseline with the full tree.
581    pub full: bool,
582    /// Attach post-action state. Used from P5.
583    pub return_state: bool,
584    /// Attach only the post-action delta instead of the full tree.
585    pub state_diff: bool,
586    /// Attach a screenshot with post-action state. Used from P5.
587    pub include_screenshot: bool,
588    /// Coordinate space name. Used from P5.
589    pub coord: Option<String>,
590    /// Cache namespace. Defaults to the process id in later phases.
591    pub session: Option<String>,
592    /// Input tier name. Used from P7.
593    pub tier: Option<String>,
594    /// Adapter name. Used from P5.
595    pub adapter: Option<String>,
596    /// Display filter. Used from P14.
597    pub display: Option<String>,
598    /// Fail when the element is outside the window and scroll-into-view cannot run.
599    pub require_visible: bool,
600    /// User policy file from `--policy`. Absent uses the default path.
601    pub policy: Option<String>,
602    /// Confirm hook from `--confirm-cmd`. Wins over the environment and the policy file.
603    pub confirm_cmd: Option<String>,
604    /// Skip `foreground.lock` for tier `activate`.
605    pub allow_foreground: bool,
606    /// Refuse tier `activate`, and treat cursor motion as failure.
607    pub require_background: bool,
608    /// Do not send the agent point to the overlay helper.
609    pub no_overlay: bool,
610    /// Route browser commands through CDP. The dispatcher refuses the request
611    /// when this flag is off (`invalid_arguments` for the CLI, `unsupported`
612    /// for the JSON path).
613    pub cdp: bool,
614    /// Override the discovered CDP websocket URL. Wins over `--app` argv
615    /// discovery. Accepts `ws://`, `wss://`, `http://`, `https://`.
616    pub cdp_url: Option<String>,
617    /// Cached `[browser].blocked_urls` from the loaded policy. The dispatcher
618    /// refuses the websocket connection (and the page activation) when the
619    /// destination URL is in here.
620    pub browser_blocked_urls: Vec<String>,
621}
622
623/// A JSON request: an optional `id`, shared flags, and an [`Action`].
624#[derive(Debug, Clone, PartialEq)]
625pub struct Request {
626    /// Caller-supplied id, echoed on the response when present.
627    pub id: Option<Value>,
628    /// Action to run.
629    pub action: Action,
630    /// Flags that were siblings of `action`.
631    pub context: CallContext,
632}
633
634/// Parse one JSON object. `id` and the shared flag keys are removed before the action is decoded.
635pub fn parse_request(mut value: Value) -> Result<Request, ComputerError> {
636    if !value.is_object() {
637        return Err(ComputerError::InvalidArguments(
638            "request must be a JSON object".into(),
639        ));
640    }
641    let object = value
642        .as_object_mut()
643        .expect("object check above guarantees a map");
644    let id = object.remove("id");
645    let context = take_context(object)?;
646    let action = serde_json::from_value(value)
647        .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
648    Ok(Request {
649        id,
650        action,
651        context,
652    })
653}
654
655fn take_context(object: &mut Map<String, Value>) -> Result<CallContext, ComputerError> {
656    let state_diff = take_bool(object, "state_diff")?;
657    Ok(CallContext {
658        app: take_stringish(object, "app")?,
659        window: take_stringish(object, "window")?,
660        element: take_optional_u32(object, "element")?,
661        diff: take_bool(object, "diff")?,
662        full: take_bool(object, "full")?,
663        return_state: take_bool(object, "return_state")? || state_diff,
664        state_diff,
665        include_screenshot: take_bool(object, "include_screenshot")?,
666        coord: take_stringish(object, "coord")?,
667        session: take_stringish(object, "session")?,
668        tier: take_stringish(object, "tier")?,
669        adapter: take_stringish(object, "adapter")?,
670        display: take_stringish(object, "display")?,
671        require_visible: take_bool(object, "require_visible")?,
672        policy: None,
673        confirm_cmd: None,
674        allow_foreground: take_bool(object, "allow_foreground")?,
675        require_background: take_bool(object, "require_background")?,
676        no_overlay: take_bool(object, "no_overlay")?,
677        cdp: take_bool(object, "cdp")?,
678        cdp_url: take_stringish(object, "cdp_url")?,
679        browser_blocked_urls: take_string_list(object, "browser_blocked_urls")?,
680    })
681}
682
683fn take_stringish(
684    object: &mut Map<String, Value>,
685    key: &str,
686) -> Result<Option<String>, ComputerError> {
687    match object.remove(key) {
688        None | Some(Value::Null) => Ok(None),
689        Some(Value::String(text)) => Ok(Some(text)),
690        Some(Value::Number(number)) => Ok(Some(number.to_string())),
691        Some(_) => Err(ComputerError::InvalidArguments(format!(
692            "{key} must be a string or number"
693        ))),
694    }
695}
696
697fn take_bool(object: &mut Map<String, Value>, key: &str) -> Result<bool, ComputerError> {
698    match object.remove(key) {
699        None | Some(Value::Null) => Ok(false),
700        Some(Value::Bool(value)) => Ok(value),
701        Some(_) => Err(ComputerError::InvalidArguments(format!(
702            "{key} must be a boolean"
703        ))),
704    }
705}
706
707fn take_string_list(
708    object: &mut Map<String, Value>,
709    key: &str,
710) -> Result<Vec<String>, ComputerError> {
711    match object.remove(key) {
712        None | Some(Value::Null) => Ok(Vec::new()),
713        Some(Value::Array(values)) => values
714            .into_iter()
715            .map(|value| match value {
716                Value::String(text) => Ok(text),
717                _ => Err(ComputerError::InvalidArguments(format!(
718                    "{key} entries must be strings"
719                ))),
720            })
721            .collect(),
722        Some(_) => Err(ComputerError::InvalidArguments(format!(
723            "{key} must be a list of strings"
724        ))),
725    }
726}
727
728fn take_optional_u32(
729    object: &mut Map<String, Value>,
730    key: &str,
731) -> Result<Option<u32>, ComputerError> {
732    match object.remove(key) {
733        None | Some(Value::Null) => Ok(None),
734        Some(Value::Number(number)) => number
735            .as_u64()
736            .and_then(|value| u32::try_from(value).ok())
737            .map(Some)
738            .ok_or_else(|| ComputerError::InvalidArguments(format!("{key} is out of range"))),
739        Some(Value::String(text)) => text
740            .parse::<u32>()
741            .map(Some)
742            .map_err(|_| ComputerError::InvalidArguments(format!("{key} must be an integer"))),
743        Some(_) => Err(ComputerError::InvalidArguments(format!(
744            "{key} must be an integer"
745        ))),
746    }
747}
748
749/// Decode one JSON line and dispatch it.
750///
751/// Parse errors return `ok: false` and do not panic. The adapter is native
752/// unless the object sets `adapter`.
753pub fn handle_request_line(line: &str) -> Response {
754    handle_request_line_with(line, None)
755}
756
757/// Options from the CLI that JSON requests do not carry.
758#[derive(Debug, Clone, Copy, Default)]
759pub struct LineOptions<'a> {
760    /// Process `--adapter` flag. A request field wins.
761    pub adapter: Option<&'a str>,
762    /// Process `--policy` path.
763    pub policy: Option<&'a str>,
764    /// Process `--confirm-cmd` program.
765    pub confirm_cmd: Option<&'a str>,
766    /// Process `--tier`. A request field wins. `--synthetic-focus` arrives as `synthetic`.
767    pub tier: Option<&'a str>,
768    /// Process `--allow-foreground`. Combined with the request by or.
769    pub allow_foreground: bool,
770    /// Process `--require-background`. Combined with the request by or.
771    pub require_background: bool,
772    /// Process `--no-overlay`. Combined with the request by or.
773    pub no_overlay: bool,
774    /// Process `--cdp` flag. Combined with the request by or.
775    pub cdp: bool,
776    /// Process `--cdp-url` flag. A request field wins.
777    pub cdp_url: Option<&'a str>,
778}
779
780/// Decode one JSON line. `cli_adapter` is the process `--adapter` flag.
781///
782/// A request field `adapter` wins. `native` skips translation. There is no
783/// detection when both are absent.
784pub fn handle_request_line_with(line: &str, cli_adapter: Option<&str>) -> Response {
785    handle_request_line_opts(
786        line,
787        LineOptions {
788            adapter: cli_adapter,
789            ..LineOptions::default()
790        },
791    )
792}
793
794/// Decode one JSON line and apply CLI `--policy` and `--confirm-cmd`.
795pub fn handle_request_line_opts(line: &str, options: LineOptions<'_>) -> Response {
796    let value = match serde_json::from_str::<Value>(line) {
797        Ok(value) => value,
798        Err(error) => {
799            return Response::failure("invalid_arguments", format!("invalid JSON: {error}"))
800        }
801    };
802    let id = value.get("id").cloned();
803    let echo = match effective_adapter(&value, options.adapter) {
804        Ok(name) => name,
805        Err(error) => return Response::from_error(error).with_id(id),
806    };
807    match build_request(value, echo.as_deref()) {
808        Ok(mut request) => {
809            if let Some(policy) = options.policy {
810                request.context.policy = Some(policy.to_string());
811            }
812            if let Some(confirm) = options.confirm_cmd {
813                request.context.confirm_cmd = Some(confirm.to_string());
814            }
815            if request.context.tier.is_none() {
816                if let Some(tier) = options.tier {
817                    request.context.tier = Some(tier.to_string());
818                }
819            }
820            request.context.allow_foreground |= options.allow_foreground;
821            request.context.require_background |= options.require_background;
822            request.context.no_overlay |= options.no_overlay;
823            request.context.cdp |= options.cdp;
824            if request.context.cdp_url.is_none() {
825                if let Some(url) = options.cdp_url {
826                    request.context.cdp_url = Some(url.to_string());
827                }
828            }
829            dispatch_context(request.action, &request.context).with_id(request.id)
830        }
831        Err(error) => {
832            let mut response = Response::from_error(error).with_id(id);
833            stamp_adapter_name(&mut response, echo.as_deref());
834            response
835        }
836    }
837}
838
839fn effective_adapter(
840    value: &Value,
841    cli_adapter: Option<&str>,
842) -> Result<Option<String>, ComputerError> {
843    if !value.is_object() {
844        return Ok(None);
845    }
846    let from_request = match value.get("adapter") {
847        None | Some(Value::Null) => None,
848        Some(Value::String(text)) => Some(text.as_str()),
849        Some(_) => {
850            return Err(ComputerError::InvalidArguments(
851                "adapter must be a string".into(),
852            ))
853        }
854    };
855    let name = from_request.or(cli_adapter);
856    match name {
857        None | Some("native") => Ok(None),
858        Some("openai" | "anthropic") => Ok(name.map(str::to_string)),
859        Some(other) => Err(ComputerError::InvalidArguments(format!(
860            "unknown adapter {other:?}"
861        ))),
862    }
863}
864
865fn build_request(value: Value, adapter: Option<&str>) -> Result<Request, ComputerError> {
866    let Some(name) = adapter else {
867        return parse_request(value);
868    };
869    if !value.is_object() {
870        return Err(ComputerError::InvalidArguments(
871            "request must be a JSON object".into(),
872        ));
873    }
874    let mut value = value;
875    let object = value
876        .as_object_mut()
877        .expect("object check above guarantees a map");
878    let id = object.remove("id");
879    let mut context = take_context(object)?;
880    context.coord = Some("image".into());
881    context.adapter = Some(name.to_string());
882    let action = crate::adapters::translate(name, &value)?;
883    Ok(Request {
884        id,
885        action,
886        context,
887    })
888}
889
890fn stamp_adapter_name(response: &mut Response, adapter: Option<&str>) {
891    if let Some(name @ ("openai" | "anthropic")) = adapter {
892        response.fields.insert("adapter".into(), json!(name));
893    }
894}
895
896/// Run `action` on the platform backend.
897pub fn dispatch(action: Action) -> Response {
898    dispatch_context(action, &CallContext::default())
899}
900
901/// Run `action` with shared flags on the platform backend.
902pub fn dispatch_context(action: Action, context: &CallContext) -> Response {
903    let backend = create_desktop_backend();
904    dispatch_with_context(backend.as_ref(), action, context)
905}
906
907/// Run `action` on `backend`. Panics inside the backend become an error response.
908pub fn dispatch_with(backend: &dyn DesktopBackend, action: Action) -> Response {
909    dispatch_with_context(backend, action, &CallContext::default())
910}
911
912/// Run `action` on `backend` with shared flags.
913pub fn dispatch_with_context(
914    backend: &dyn DesktopBackend,
915    action: Action,
916    context: &CallContext,
917) -> Response {
918    let mut response = match catch_unwind(AssertUnwindSafe(|| {
919        dispatch_inner(backend, action, context)
920    })) {
921        Ok(response) => response,
922        Err(_) => Response::failure("execution_failed", "action panicked and was recovered"),
923    };
924    stamp_adapter_name(&mut response, context.adapter.as_deref());
925    // A serve or batch session stays open so a later mouse_up can pair with this down.
926    // A one-shot command has no session, so anything still held is released here.
927    if !input_session::session_open() {
928        let _ = backend.release_held();
929    }
930    response
931}
932
933enum PostKind {
934    /// Successful or failed mutating action. `return_state` snapshots afterward.
935    Mutating,
936    /// `return_state` and `include_screenshot` are ignored.
937    ReadOnly,
938    /// The action already captured. `include_screenshot` reuses that body.
939    Screenshot,
940}
941
942fn dispatch_inner(backend: &dyn DesktopBackend, action: Action, context: &CallContext) -> Response {
943    let label = action_label(&action);
944    let chars = clipboard_chars(&action);
945    let action = match remap_coordinates(context, action) {
946        Ok(action) => action,
947        Err(error) => {
948            return finish_action(
949                context,
950                label,
951                &policy::Subject::default(),
952                None,
953                "allow",
954                "default",
955                chars,
956                Response::from_error(error),
957            )
958        }
959    };
960    if let Err(error) = preflight_action(&action) {
961        return finish_action(
962            context,
963            label,
964            &policy::Subject::default(),
965            None,
966            "allow",
967            "default",
968            chars,
969            Response::from_error(error),
970        );
971    }
972    if let Err(error) = gate_display(backend, context, &action) {
973        return finish_action(
974            context,
975            label,
976            &policy::Subject::default(),
977            None,
978            "allow",
979            "default",
980            chars,
981            Response::from_error(error),
982        );
983    }
984    let prepared = prepare_action(backend, &action, context);
985    if let Some(response) = prepared.stop {
986        return finish_action(
987            context,
988            label,
989            &prepared.subject,
990            prepared.warning.as_deref(),
991            prepared.decision,
992            prepared.source,
993            chars,
994            response,
995        );
996    }
997    let kind = post_state_kind(&action);
998    let response = match action {
999        Action::Batch {
1000            actions,
1001            stop_on_error,
1002        } => {
1003            return finish_action(
1004                context,
1005                label,
1006                &prepared.subject,
1007                prepared.warning.as_deref(),
1008                prepared.decision,
1009                prepared.source,
1010                chars,
1011                run_batch(backend, context, actions, stop_on_error),
1012            )
1013        }
1014        Action::Screenshot {
1015            output,
1016            region,
1017            format,
1018            max_width,
1019            canvas,
1020            base64,
1021            marks,
1022        } => result_response(screenshot(
1023            backend,
1024            context,
1025            ScreenshotArgs {
1026                output,
1027                region,
1028                format,
1029                max_width,
1030                canvas,
1031                include_base64: base64,
1032                window_crop: false,
1033                marks,
1034            },
1035        )),
1036        Action::Zoom {
1037            output,
1038            region,
1039            format,
1040            max_width,
1041            canvas,
1042            base64,
1043        } => result_response(screenshot(
1044            backend,
1045            context,
1046            ScreenshotArgs {
1047                output,
1048                region: Some(region),
1049                format,
1050                max_width,
1051                canvas,
1052                include_base64: base64,
1053                window_crop: true,
1054                marks: false,
1055            },
1056        )),
1057        Action::Click {
1058            x,
1059            y,
1060            button,
1061            double,
1062            count,
1063        } => click(backend, context, x, y, button, double, count),
1064        Action::DoubleClick { x, y, button } => click(backend, context, x, y, button, true, 1),
1065        Action::TripleClick { x, y, button } => click(backend, context, x, y, button, false, 3),
1066        Action::Press { count, button } => {
1067            element_command(backend, context, ElementOp::Press { count, button })
1068        }
1069        Action::Drag { path, button } => drag(backend, context, path, button),
1070        Action::MouseDown { x, y, button } => pointer_half(backend, context, x, y, button, true),
1071        Action::MouseUp { x, y, button } => pointer_half(backend, context, x, y, button, false),
1072        Action::HoldKey { keys, ms } => hold_key(backend, context, &keys, ms),
1073        Action::ClipboardGet => result_response(clipboard_get(backend, context)),
1074        Action::ClipboardSet { text, image } => {
1075            result_response(clipboard_set(backend, text, image))
1076        }
1077        Action::Focus => element_command(backend, context, ElementOp::Focus),
1078        Action::SetValue { value } => {
1079            element_command(backend, context, ElementOp::SetValue { value })
1080        }
1081        Action::SelectText { start, end } => {
1082            element_command(backend, context, ElementOp::SelectText { start, end })
1083        }
1084        Action::PerformAction { name } => {
1085            element_command(backend, context, ElementOp::Perform { name })
1086        }
1087        Action::ScrollTo => element_command(backend, context, ElementOp::ScrollTo),
1088        Action::Move { x, y } => move_mouse(backend, context, x, y),
1089        Action::Type { text } => type_command(backend, context, text),
1090        Action::Key { keys } => press_key(backend, context, &keys),
1091        Action::Scroll {
1092            x,
1093            y,
1094            dx,
1095            dy,
1096            direction,
1097            amount,
1098        } => scroll(backend, context, (x, y), (dx, dy), direction, amount),
1099        Action::GetState => result_response(Ok(collect_state_for(backend, &tree_query(context)))),
1100        Action::Tree => result_response(tree_action(backend, context)),
1101        Action::Ocr { force } => result_response(ocr_action(backend, context, force)),
1102        Action::SessionEnd => result_response(session_end(backend, context)),
1103        Action::Wait { ms, seconds } => result_response(wait(ms, seconds)),
1104        Action::Displays => result_response(displays(backend)),
1105        Action::Isolation => result_response(isolation()),
1106        Action::ListApps { all } => result_response(list_apps(backend, all)),
1107        Action::ListWindows => result_response(list_windows(backend, context.app.as_deref())),
1108        Action::Launch => result_response(launch(backend, context.app.as_deref())),
1109        Action::Activate => result_response(activate(
1110            backend,
1111            context.app.as_deref(),
1112            context.window.as_deref(),
1113        )),
1114        Action::Quit { force } => result_response(quit(backend, context.app.as_deref(), force)),
1115        Action::Doctor {
1116            prompt,
1117            open_settings,
1118            x11,
1119        } => result_response(doctor(
1120            backend,
1121            crate::doctor::DoctorOptions {
1122                prompt,
1123                open_settings,
1124                x11,
1125            },
1126        )),
1127        Action::CursorPosition => result_response(cursor_position(backend)),
1128        Action::SessionPrune { dry_run, ttl_secs } => {
1129            result_response(session_prune(dry_run, ttl_secs))
1130        }
1131        Action::MenuList { app, depth } => {
1132            let app = match app.or_else(|| context.app.clone()) {
1133                Some(app) => app,
1134                None => return Response::from_error(invalid_arguments("menu list requires --app")),
1135            };
1136            let depth = depth.unwrap_or(MENU_DEFAULT_DEPTH);
1137            result_response(menu_list(backend, &app, depth))
1138        }
1139        Action::MenuClick { app, path } => {
1140            let app = match app.or_else(|| context.app.clone()) {
1141                Some(app) => app,
1142                None => {
1143                    return Response::from_error(invalid_arguments("menu click requires --app"))
1144                }
1145            };
1146            if path.trim().is_empty() {
1147                return Response::from_error(invalid_arguments("menu click requires --path"));
1148            }
1149            result_response(menu_click(backend, &app, &path))
1150        }
1151        Action::AppleEvent {
1152            app,
1153            script,
1154            language,
1155        } => {
1156            let app = match app.or_else(|| context.app.clone()) {
1157                Some(app) => app,
1158                None => {
1159                    return Response::from_error(invalid_arguments("apple-event requires --app"))
1160                }
1161            };
1162            if script.is_empty() {
1163                return Response::from_error(invalid_arguments("apple-event requires --script"));
1164            }
1165            let language = language.unwrap_or_default();
1166            result_response(apple_event_action(backend, &app, &script, language))
1167        }
1168        Action::ShortcutsRun { name } => {
1169            if !cfg!(target_os = "macos") {
1170                return Response::from_error(unsupported_error("shortcuts_run"));
1171            }
1172            if name.trim().is_empty() {
1173                return Response::from_error(invalid_arguments(
1174                    "shortcuts run requires a workflow name",
1175                ));
1176            }
1177            result_response(shortcuts_run_action(name))
1178        }
1179        Action::DockList {} => result_response(dock_list_action(backend)),
1180        Action::DockClick { title } => {
1181            if title.trim().is_empty() {
1182                return Response::from_error(invalid_arguments("dock click requires --title"));
1183            }
1184            result_response(dock_click_action(backend, &title))
1185        }
1186        Action::Spaces {} => result_response(spaces_action(backend)),
1187        Action::BrowserTabs {} => result_response(browser_tabs(context, &prepared.subject)),
1188        Action::BrowserActivate { target_id } => result_response(browser_activate(
1189            context,
1190            &prepared.subject,
1191            target_id.clone(),
1192        )),
1193        Action::BrowserNavigate { url, target_id } => result_response(browser_navigate(
1194            context,
1195            &prepared.subject,
1196            url.clone(),
1197            target_id.clone(),
1198        )),
1199        Action::BrowserClick {
1200            selector,
1201            target_id,
1202        } => result_response(browser_click(
1203            context,
1204            &prepared.subject,
1205            selector.clone(),
1206            target_id.clone(),
1207        )),
1208        Action::BrowserText {
1209            selector,
1210            target_id,
1211        } => result_response(browser_text(
1212            context,
1213            &prepared.subject,
1214            selector.clone(),
1215            target_id.clone(),
1216        )),
1217        Action::BrowserType {
1218            text,
1219            selector,
1220            target_id,
1221        } => result_response(browser_type(
1222            context,
1223            &prepared.subject,
1224            text.clone(),
1225            selector.clone(),
1226            target_id.clone(),
1227        )),
1228        Action::BrowserEval { script, target_id } => result_response(browser_eval(
1229            context,
1230            &prepared.subject,
1231            script.clone(),
1232            target_id.clone(),
1233        )),
1234        Action::PolicyCheck => {
1235            let (response, subject) = policy_check(backend, context);
1236            let response = attach_post_state(backend, context, kind, response);
1237            return finish_action(
1238                context,
1239                label,
1240                &subject,
1241                prepared.warning.as_deref(),
1242                prepared.decision,
1243                prepared.source,
1244                chars,
1245                response,
1246            );
1247        }
1248        Action::Pause => pause_response(true),
1249        Action::Resume => pause_response(false),
1250    };
1251    let response = attach_post_state(backend, context, kind, response);
1252    finish_action(
1253        context,
1254        label,
1255        &prepared.subject,
1256        prepared.warning.as_deref(),
1257        prepared.decision,
1258        prepared.source,
1259        chars,
1260        response,
1261    )
1262}
1263
1264#[allow(clippy::too_many_arguments)]
1265fn finish_action(
1266    context: &CallContext,
1267    label: &str,
1268    subject: &policy::Subject,
1269    warning: Option<&str>,
1270    decision: &str,
1271    source: &str,
1272    chars: Option<u64>,
1273    mut response: Response,
1274) -> Response {
1275    if let Some(warning) = warning {
1276        push_warning(&mut response, warning);
1277    }
1278    let code = if response.ok {
1279        "ok"
1280    } else {
1281        response
1282            .error
1283            .as_ref()
1284            .map(|error| error.code.as_str())
1285            .unwrap_or("execution_failed")
1286    };
1287    let mut decision = decision.to_string();
1288    let mut source = source.to_string();
1289    if label == "policy_check" {
1290        if let Some(value) = response
1291            .fields
1292            .get("decision")
1293            .and_then(|value| value.as_str())
1294        {
1295            decision = value.to_string();
1296        }
1297        if let Some(value) = response
1298            .fields
1299            .get("source")
1300            .and_then(|value| value.as_str())
1301        {
1302            source = value.to_string();
1303        }
1304    }
1305    let tier = response
1306        .fields
1307        .get("tier")
1308        .and_then(|value| value.as_str())
1309        .map(str::to_string);
1310    let session = context
1311        .session
1312        .clone()
1313        .unwrap_or_else(|| std::process::id().to_string());
1314    if let Err(error) = crate::audit::record(&crate::audit::AuditRecord {
1315        session: &session,
1316        action: label,
1317        bundle_id: subject.bundle_id.as_deref(),
1318        pid: subject.pid,
1319        window_id: subject.window_id,
1320        element: context.element,
1321        code,
1322        decision: &decision,
1323        source: &source,
1324        tier: tier.as_deref(),
1325        chars,
1326    }) {
1327        push_warning(
1328            &mut response,
1329            &format!("audit log was not written: {error}"),
1330        );
1331    }
1332    response
1333}
1334
1335fn push_warning(response: &mut Response, warning: &str) {
1336    let warnings = response
1337        .fields
1338        .entry("warnings")
1339        .or_insert_with(|| json!([]));
1340    if let Some(items) = warnings.as_array_mut() {
1341        items.push(json!(warning));
1342    }
1343}
1344
1345fn action_label(action: &Action) -> &'static str {
1346    match action {
1347        Action::Screenshot { .. } => "screenshot",
1348        Action::Zoom { .. } => "zoom",
1349        Action::Click { .. } => "click",
1350        Action::DoubleClick { .. } => "double_click",
1351        Action::TripleClick { .. } => "triple_click",
1352        Action::Press { .. } => "press",
1353        Action::Drag { .. } => "drag",
1354        Action::MouseDown { .. } => "mouse_down",
1355        Action::MouseUp { .. } => "mouse_up",
1356        Action::HoldKey { .. } => "hold_key",
1357        Action::ClipboardGet => "clipboard_get",
1358        Action::ClipboardSet { .. } => "clipboard_set",
1359        Action::Focus => "focus",
1360        Action::SetValue { .. } => "set_value",
1361        Action::SelectText { .. } => "select_text",
1362        Action::PerformAction { .. } => "perform_action",
1363        Action::ScrollTo => "scroll_to",
1364        Action::Move { .. } => "move",
1365        Action::Type { .. } => "type",
1366        Action::Key { .. } => "key",
1367        Action::Scroll { .. } => "scroll",
1368        Action::GetState => "get_state",
1369        Action::Wait { .. } => "wait",
1370        Action::Displays => "displays",
1371        Action::Isolation => "isolation",
1372        Action::ListApps { .. } => "list_apps",
1373        Action::ListWindows => "list_windows",
1374        Action::Launch => "launch",
1375        Action::Activate => "activate",
1376        Action::Quit { .. } => "quit",
1377        Action::Doctor { .. } => "doctor",
1378        Action::Tree => "tree",
1379        Action::SessionEnd => "session_end",
1380        Action::CursorPosition => "cursor_position",
1381        Action::Batch { .. } => "batch",
1382        Action::SessionPrune { .. } => "session_prune",
1383        Action::PolicyCheck => "policy_check",
1384        Action::Pause => "pause",
1385        Action::Resume => "resume",
1386        Action::MenuList { .. } => "menu_list",
1387        Action::MenuClick { .. } => "menu_click",
1388        Action::AppleEvent { .. } => "apple_event",
1389        Action::ShortcutsRun { .. } => "shortcuts_run",
1390        Action::DockList { .. } => "dock_list",
1391        Action::DockClick { .. } => "dock_click",
1392        Action::Spaces { .. } => "spaces",
1393        Action::BrowserTabs { .. } => "browser_tabs",
1394        Action::BrowserActivate { .. } => "browser_activate",
1395        Action::BrowserNavigate { .. } => "browser_navigate",
1396        Action::BrowserClick { .. } => "browser_click",
1397        Action::BrowserText { .. } => "browser_text",
1398        Action::BrowserType { .. } => "browser_type",
1399        Action::BrowserEval { .. } => "browser_eval",
1400        Action::Ocr { .. } => "ocr",
1401    }
1402}
1403
1404#[derive(Clone, Copy)]
1405enum PolicyClass {
1406    Always,
1407    Benign,
1408    Observe,
1409    Mutate,
1410    Quit,
1411    Key,
1412    ClipboardSet,
1413}
1414
1415fn policy_class(action: &Action) -> PolicyClass {
1416    match action {
1417        Action::Wait { .. }
1418        | Action::Doctor { .. }
1419        | Action::Displays
1420        | Action::Isolation
1421        | Action::ListApps { .. }
1422        | Action::ListWindows
1423        | Action::Pause
1424        | Action::Resume => PolicyClass::Always,
1425        Action::SessionEnd | Action::SessionPrune { .. } | Action::CursorPosition => {
1426            PolicyClass::Benign
1427        }
1428        Action::Screenshot { .. }
1429        | Action::Zoom { .. }
1430        | Action::Tree
1431        | Action::GetState
1432        | Action::PolicyCheck
1433        | Action::MenuList { .. }
1434        | Action::DockList { .. }
1435        | Action::Spaces { .. }
1436        | Action::BrowserTabs { .. }
1437        | Action::BrowserText { .. }
1438        | Action::BrowserActivate { .. }
1439        | Action::Ocr { .. } => PolicyClass::Observe,
1440        Action::Quit { .. } => PolicyClass::Quit,
1441        Action::Key { .. } | Action::HoldKey { .. } => PolicyClass::Key,
1442        Action::ClipboardGet => PolicyClass::Benign,
1443        Action::ClipboardSet { .. } => PolicyClass::ClipboardSet,
1444        _ => PolicyClass::Mutate,
1445    }
1446}
1447
1448struct Prepared {
1449    stop: Option<Response>,
1450    warning: Option<String>,
1451    subject: policy::Subject,
1452    decision: &'static str,
1453    source: &'static str,
1454}
1455
1456fn early_stop(warning: Option<String>, subject: policy::Subject, response: Response) -> Prepared {
1457    Prepared {
1458        stop: Some(response),
1459        warning,
1460        subject,
1461        decision: "allow",
1462        source: "default",
1463    }
1464}
1465
1466fn prepare_action(
1467    backend: &dyn DesktopBackend,
1468    action: &Action,
1469    context: &CallContext,
1470) -> Prepared {
1471    let class = policy_class(action);
1472    let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1473        Ok(policy) => policy,
1474        Err(error) => {
1475            if matches!(class, PolicyClass::Always) {
1476                policy::LoadedPolicy::default()
1477            } else {
1478                return early_stop(
1479                    None,
1480                    policy::Subject::default(),
1481                    Response::from_error(error),
1482                );
1483            }
1484        }
1485    };
1486    // The backends consult the built-in deny list without carrying the policy,
1487    // so the exemptions go into the process-wide registry here.
1488    policy::set_builtin_exemptions(loaded.builtin_exempt());
1489    let warning = if loaded.pause_on_user_input {
1490        backend.user_input_pause_warning().map(str::to_string)
1491    } else {
1492        None
1493    };
1494    if matches!(action, Action::Pause | Action::Resume | Action::PolicyCheck) {
1495        return Prepared {
1496            stop: None,
1497            warning,
1498            subject: policy::Subject::default(),
1499            decision: "allow",
1500            source: "default",
1501        };
1502    }
1503    let subject = match resolve_subject(backend, action, context) {
1504        Ok(subject) => subject,
1505        Err(error) => {
1506            return early_stop(
1507                warning,
1508                policy::Subject::default(),
1509                Response::from_error(error),
1510            )
1511        }
1512    };
1513    let policy_action = match action {
1514        Action::Quit { .. } => policy::PolicyAction::Quit,
1515        Action::Key { keys } | Action::HoldKey { keys, .. } => policy::PolicyAction::Key {
1516            combo: keys.clone(),
1517        },
1518        Action::Screenshot { .. } | Action::Zoom { .. } | Action::Tree | Action::GetState => {
1519            policy::PolicyAction::Observe
1520        }
1521        Action::Ocr { .. } => policy::PolicyAction::Observe,
1522        Action::ClipboardSet { .. } => policy::PolicyAction::ClipboardSet,
1523        _ => match class {
1524            PolicyClass::Always => policy::PolicyAction::Always,
1525            PolicyClass::Benign => policy::PolicyAction::Benign,
1526            PolicyClass::Mutate => policy::PolicyAction::Mutate,
1527            PolicyClass::Observe => policy::PolicyAction::Observe,
1528            PolicyClass::Quit => policy::PolicyAction::Quit,
1529            PolicyClass::Key => policy::PolicyAction::Key {
1530                combo: String::new(),
1531            },
1532            PolicyClass::ClipboardSet => policy::PolicyAction::ClipboardSet,
1533        },
1534    };
1535    let decision = policy::decide(&loaded, &subject, &policy_action);
1536    if let policy::Decision::Deny { message, source } = &decision {
1537        return Prepared {
1538            stop: Some(Response::from_error(ComputerError::PolicyDenied(
1539                message.clone(),
1540            ))),
1541            warning,
1542            subject,
1543            decision: "deny",
1544            source,
1545        };
1546    }
1547    if let Err(error) = refuse_blocked_url(backend, &loaded, &subject) {
1548        let (decision_name, source) = decision_record(&decision);
1549        return Prepared {
1550            stop: Some(Response::from_error(error)),
1551            warning,
1552            subject,
1553            decision: decision_name,
1554            source,
1555        };
1556    }
1557    let mutating = matches!(
1558        class,
1559        PolicyClass::Mutate | PolicyClass::Quit | PolicyClass::Key | PolicyClass::ClipboardSet
1560    );
1561    if mutating && policy::is_paused() {
1562        let (decision_name, source) = decision_record(&decision);
1563        return Prepared {
1564            stop: Some(Response::from_error(ComputerError::Paused(
1565                "local input paused mutating actions; run computer-use resume".into(),
1566            ))),
1567            warning,
1568            subject,
1569            decision: decision_name,
1570            source,
1571        };
1572    }
1573    if let Action::Key { .. } | Action::HoldKey { .. } = action {
1574        if let Some(index) = context.element {
1575            if let Ok(Some(peek)) = backend.peek_element(
1576                context.app.as_deref(),
1577                context.window.as_deref(),
1578                context.session.as_deref(),
1579                index,
1580            ) {
1581                if peek.secure {
1582                    return Prepared {
1583                        stop: Some(Response::from_error(ComputerError::PolicyDenied(
1584                            "refusing a key event aimed at a secure field".into(),
1585                        ))),
1586                        warning,
1587                        subject,
1588                        decision: "deny",
1589                        source: "builtin",
1590                    };
1591                }
1592            }
1593        }
1594    }
1595    if let policy::Decision::Confirm { reason } = &decision {
1596        if let Err(error) = confirm_action(context, &loaded, &subject, action_label(action), reason)
1597        {
1598            return Prepared {
1599                stop: Some(Response::from_error(error)),
1600                warning,
1601                subject,
1602                decision: "confirm",
1603                source: "builtin",
1604            };
1605        }
1606    }
1607    let (decision_name, source) = decision_record(&decision);
1608    Prepared {
1609        stop: None,
1610        warning,
1611        subject,
1612        decision: decision_name,
1613        source,
1614    }
1615}
1616
1617fn decision_record(decision: &policy::Decision) -> (&'static str, &'static str) {
1618    match decision {
1619        policy::Decision::Allow { source } => ("allow", *source),
1620        policy::Decision::Deny { source, .. } => ("deny", *source),
1621        policy::Decision::Confirm { .. } => ("confirm", "builtin"),
1622    }
1623}
1624
1625fn confirm_action(
1626    context: &CallContext,
1627    loaded: &policy::LoadedPolicy,
1628    subject: &policy::Subject,
1629    action: &str,
1630    reason: &str,
1631) -> Result<(), ComputerError> {
1632    let command = policy::confirm_program(loaded, context.confirm_cmd.as_deref());
1633    policy::run_confirm(
1634        command.as_deref(),
1635        loaded.timeout_secs(),
1636        &policy::ConfirmDocument {
1637            action,
1638            bundle_id: subject.bundle_id.as_deref(),
1639            path: subject.path.as_deref(),
1640            pid: subject.pid,
1641            window_id: subject.window_id,
1642            element: context.element,
1643            reason,
1644        },
1645    )
1646}
1647
1648fn refuse_blocked_url(
1649    backend: &dyn DesktopBackend,
1650    loaded: &policy::LoadedPolicy,
1651    subject: &policy::Subject,
1652) -> Result<(), ComputerError> {
1653    if loaded.blocked_urls().is_empty() {
1654        return Ok(());
1655    }
1656    if !policy::is_browser(
1657        subject.bundle_id.as_deref(),
1658        subject.path.as_deref().map(Path::new),
1659        subject.name.as_deref(),
1660    ) {
1661        return Ok(());
1662    }
1663    let from_title = subject
1664        .window_title
1665        .as_deref()
1666        .and_then(policy::url_in_text)
1667        .map(str::to_string);
1668    let url = if let Some(url) = from_title {
1669        Some(url)
1670    } else if let Some(pid) = subject.pid {
1671        backend.browser_url(pid)?
1672    } else {
1673        None
1674    };
1675    if let Some(url) = url.as_deref() {
1676        if let Some(pattern) = policy::url_blocked(url, loaded.blocked_urls()) {
1677            return Err(ComputerError::BlockedUrl(format!("matched {pattern}")));
1678        }
1679    }
1680    Ok(())
1681}
1682
1683fn resolve_subject(
1684    backend: &dyn DesktopBackend,
1685    action: &Action,
1686    context: &CallContext,
1687) -> Result<policy::Subject, ComputerError> {
1688    let class = policy_class(action);
1689    if matches!(class, PolicyClass::Always | PolicyClass::Benign) {
1690        return Ok(policy::Subject::default());
1691    }
1692    let screenshot = matches!(action, Action::Screenshot { .. } | Action::Zoom { .. });
1693    let frontmost_observe =
1694        matches!(action, Action::Tree | Action::GetState) && context.app.is_none();
1695    let app_text = context.app.clone().or_else(|| {
1696        if frontmost_observe {
1697            backend.frontmost().app
1698        } else {
1699            None
1700        }
1701    });
1702    if app_text.is_none() && context.window.is_none() {
1703        return Ok(policy::Subject::default());
1704    }
1705    if app_text.is_none() && screenshot {
1706        return subject_from_window(backend, context);
1707    }
1708    let Some(app_text) = app_text else {
1709        return Ok(policy::Subject::default());
1710    };
1711    let selector = parse_app_selector(&app_text)?;
1712    let fallback = subject_from_selector(&selector);
1713    let apps = match backend.list_apps(true) {
1714        Ok(apps) => apps,
1715        Err(error) => {
1716            if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1717                return Ok(fallback);
1718            }
1719            return Err(error);
1720        }
1721    };
1722    let info = match resolve_app(&apps, &selector) {
1723        Ok(info) => info.clone(),
1724        Err(error) => {
1725            if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1726                return Ok(fallback);
1727            }
1728            // An implicit frontmost name that is not running is not a target.
1729            // The backend resolves the key window itself.
1730            if context.app.is_none() {
1731                return Ok(policy::Subject::default());
1732            }
1733            return Err(error);
1734        }
1735    };
1736    let mut subject = policy::Subject {
1737        name: Some(info.name.clone()),
1738        bundle_id: info.bundle_id.clone(),
1739        path: info.path.clone(),
1740        pid: Some(info.pid),
1741        window_id: None,
1742        window_title: None,
1743        ax_identifier: None,
1744    };
1745    let want_window = context.window.is_some()
1746        || matches!(
1747            action,
1748            Action::Tree | Action::GetState | Action::Screenshot { .. } | Action::Zoom { .. }
1749        );
1750    if want_window {
1751        if let Ok(listed) = backend.list_windows() {
1752            let owned = filter_windows_for_pid(&listed.windows, info.pid);
1753            let chosen = match context.window.as_deref() {
1754                Some(window) => {
1755                    let selector = parse_window_selector(window)?;
1756                    Some(resolve_window(&owned, &selector)?.clone())
1757                }
1758                None => owned.first().cloned(),
1759            };
1760            if let Some(window) = chosen {
1761                subject.window_id = Some(window.id);
1762                if context.window.is_some()
1763                    || matches!(
1764                        action,
1765                        Action::Tree
1766                            | Action::GetState
1767                            | Action::Screenshot { .. }
1768                            | Action::Zoom { .. }
1769                    )
1770                {
1771                    subject.window_title = Some(window.title);
1772                }
1773            }
1774        }
1775    }
1776    Ok(subject)
1777}
1778
1779fn subject_from_window(
1780    backend: &dyn DesktopBackend,
1781    context: &CallContext,
1782) -> Result<policy::Subject, ComputerError> {
1783    let Some(window) = context.window.as_deref() else {
1784        return Ok(policy::Subject::default());
1785    };
1786    let listed = backend.list_windows()?;
1787    let selector = parse_window_selector(window)?;
1788    let window = resolve_window(&listed.windows, &selector)?.clone();
1789    let apps = backend.list_apps(true)?;
1790    let info = apps.into_iter().find(|app| app.pid == window.pid);
1791    Ok(policy::Subject {
1792        name: info
1793            .as_ref()
1794            .map(|app| app.name.clone())
1795            .or(Some(window.app.clone())),
1796        bundle_id: info.as_ref().and_then(|app| app.bundle_id.clone()),
1797        path: info.and_then(|app| app.path),
1798        pid: Some(window.pid),
1799        window_id: Some(window.id),
1800        window_title: Some(window.title),
1801        ax_identifier: None,
1802    })
1803}
1804
1805fn subject_from_selector(selector: &crate::target::AppSelector) -> policy::Subject {
1806    use crate::target::AppSelector;
1807    match selector {
1808        AppSelector::BundleId(bundle) => policy::Subject {
1809            bundle_id: Some(bundle.clone()),
1810            ..policy::Subject::default()
1811        },
1812        AppSelector::Name(name) => policy::Subject {
1813            name: Some(name.clone()),
1814            ..policy::Subject::default()
1815        },
1816        AppSelector::Path(path) => policy::Subject {
1817            path: Some(path.display().to_string()),
1818            ..policy::Subject::default()
1819        },
1820        AppSelector::Pid(pid) => policy::Subject {
1821            pid: Some(*pid),
1822            ..policy::Subject::default()
1823        },
1824    }
1825}
1826
1827fn policy_check(
1828    backend: &dyn DesktopBackend,
1829    context: &CallContext,
1830) -> (Response, policy::Subject) {
1831    let Some(app) = context
1832        .app
1833        .as_deref()
1834        .map(str::trim)
1835        .filter(|text| !text.is_empty())
1836    else {
1837        return (
1838            Response::from_error(ComputerError::InvalidArguments("app is required".into())),
1839            policy::Subject::default(),
1840        );
1841    };
1842    let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1843        Ok(policy) => policy,
1844        Err(error) => return (Response::from_error(error), policy::Subject::default()),
1845    };
1846    policy::set_builtin_exemptions(loaded.builtin_exempt());
1847    let selector = match parse_app_selector(app) {
1848        Ok(selector) => selector,
1849        Err(error) => return (Response::from_error(error), policy::Subject::default()),
1850    };
1851    let mut subject = subject_from_selector(&selector);
1852    if let Ok(apps) = backend.list_apps(true) {
1853        if let Ok(info) = resolve_app(&apps, &selector) {
1854            subject.name = Some(info.name.clone());
1855            subject.bundle_id = info.bundle_id.clone();
1856            subject.path = info.path.clone();
1857            subject.pid = Some(info.pid);
1858        }
1859    }
1860    let decision = policy::app_verdict(&loaded, &subject);
1861    let (name, source, code) = match decision {
1862        policy::Decision::Allow { source } => ("allow", source, "ok"),
1863        policy::Decision::Deny { source, .. } => ("deny", source, "policy_denied"),
1864        policy::Decision::Confirm { .. } => ("allow", "default", "ok"),
1865    };
1866    (
1867        Response::success(json!({
1868            "decision": name,
1869            "source": source,
1870            "code": code,
1871            "bundle_id": subject.bundle_id,
1872        })),
1873        subject,
1874    )
1875}
1876
1877fn pause_response(paused: bool) -> Response {
1878    match policy::set_paused(paused) {
1879        Ok(()) => Response::success(json!({
1880            "action": if paused { "pause" } else { "resume" },
1881            "paused": paused,
1882        })),
1883        Err(error) => Response::from_error(error),
1884    }
1885}
1886
1887fn result_response(result: Result<Value, ComputerError>) -> Response {
1888    match result {
1889        Ok(value) => Response::success(value),
1890        Err(error) => Response::from_error(error),
1891    }
1892}
1893
1894fn post_state_kind(action: &Action) -> PostKind {
1895    match action {
1896        Action::Screenshot { .. } | Action::Zoom { .. } => PostKind::Screenshot,
1897        Action::Click { .. }
1898        | Action::DoubleClick { .. }
1899        | Action::TripleClick { .. }
1900        | Action::Drag { .. }
1901        | Action::MouseDown { .. }
1902        | Action::MouseUp { .. }
1903        | Action::HoldKey { .. }
1904        | Action::ClipboardSet { .. }
1905        | Action::Press { .. }
1906        | Action::Focus
1907        | Action::SetValue { .. }
1908        | Action::SelectText { .. }
1909        | Action::PerformAction { .. }
1910        | Action::ScrollTo
1911        | Action::Move { .. }
1912        | Action::Type { .. }
1913        | Action::Key { .. }
1914        | Action::Scroll { .. }
1915        | Action::Launch
1916        | Action::Activate
1917        | Action::Quit { .. }
1918        | Action::Batch { .. }
1919        | Action::MenuClick { .. }
1920        | Action::AppleEvent { .. }
1921        | Action::ShortcutsRun { .. }
1922        | Action::DockClick { .. }
1923        | Action::BrowserNavigate { .. }
1924        | Action::BrowserClick { .. }
1925        | Action::BrowserText { .. }
1926        | Action::BrowserType { .. }
1927        | Action::BrowserEval { .. }
1928        | Action::BrowserActivate { .. }
1929        | Action::Ocr { .. } => PostKind::Mutating,
1930        Action::GetState
1931        | Action::Wait { .. }
1932        | Action::Displays
1933        | Action::Isolation
1934        | Action::ListApps { .. }
1935        | Action::ListWindows
1936        | Action::Doctor { .. }
1937        | Action::Tree
1938        | Action::SessionEnd
1939        | Action::SessionPrune { .. }
1940        | Action::CursorPosition
1941        | Action::ClipboardGet
1942        | Action::PolicyCheck
1943        | Action::Pause
1944        | Action::Resume
1945        | Action::MenuList { .. }
1946        | Action::DockList { .. }
1947        | Action::Spaces { .. }
1948        | Action::BrowserTabs { .. } => PostKind::ReadOnly,
1949    }
1950}
1951
1952fn remap_coordinates(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1953    match context.coord.as_deref() {
1954        None | Some("screen") => Ok(action),
1955        Some("image") => remap_image_action(context, action),
1956        Some(other) => Err(ComputerError::InvalidArguments(format!(
1957            "coord must be screen or image (got {other})"
1958        ))),
1959    }
1960}
1961
1962fn remap_image_action(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1963    match action {
1964        Action::Click {
1965            x,
1966            y,
1967            button,
1968            double,
1969            count,
1970        } => {
1971            let (x, y) = remap_optional_point(context, x, y)?;
1972            Ok(Action::Click {
1973                x,
1974                y,
1975                button,
1976                double,
1977                count,
1978            })
1979        }
1980        Action::DoubleClick { x, y, button } => {
1981            let (x, y) = remap_optional_point(context, x, y)?;
1982            Ok(Action::DoubleClick { x, y, button })
1983        }
1984        Action::TripleClick { x, y, button } => {
1985            let (x, y) = remap_optional_point(context, x, y)?;
1986            Ok(Action::TripleClick { x, y, button })
1987        }
1988        Action::Drag { path, button } => {
1989            let mut mapped = Vec::with_capacity(path.len());
1990            for point in path {
1991                let (x, y) = remap_point(context, point.x, point.y)?;
1992                mapped.push(Point { x, y });
1993            }
1994            Ok(Action::Drag {
1995                path: mapped,
1996                button,
1997            })
1998        }
1999        Action::MouseDown { x, y, button } => {
2000            let (x, y) = remap_point(context, x, y)?;
2001            Ok(Action::MouseDown { x, y, button })
2002        }
2003        Action::MouseUp { x, y, button } => {
2004            let (x, y) = remap_point(context, x, y)?;
2005            Ok(Action::MouseUp { x, y, button })
2006        }
2007        Action::Move { x, y } => {
2008            let (x, y) = remap_point(context, x, y)?;
2009            Ok(Action::Move { x, y })
2010        }
2011        Action::Scroll {
2012            x,
2013            y,
2014            dx,
2015            dy,
2016            direction,
2017            amount,
2018        } => {
2019            let (x, y) = remap_optional_point(context, x, y)?;
2020            Ok(Action::Scroll {
2021                x,
2022                y,
2023                dx,
2024                dy,
2025                direction,
2026                amount,
2027            })
2028        }
2029        Action::Screenshot {
2030            output,
2031            region,
2032            format,
2033            max_width,
2034            canvas,
2035            base64,
2036            marks,
2037        } => {
2038            let region = match region {
2039                Some(region) => Some(remap_region(context, region)?),
2040                None => None,
2041            };
2042            Ok(Action::Screenshot {
2043                output,
2044                region,
2045                format,
2046                max_width,
2047                canvas,
2048                base64,
2049                marks,
2050            })
2051        }
2052        Action::Zoom {
2053            output,
2054            region,
2055            format,
2056            max_width,
2057            canvas,
2058            base64,
2059        } => {
2060            let region = remap_region(context, region)?;
2061            Ok(Action::Zoom {
2062                output,
2063                region,
2064                format,
2065                max_width,
2066                canvas,
2067                base64,
2068            })
2069        }
2070        other => Ok(other),
2071    }
2072}
2073
2074/// Resolve `--display` once per action, then check every screen point it names.
2075///
2076/// Runs after [`remap_coordinates`], so `--coord image` has already become a
2077/// screen point and the filter sees the same space the click will use. Points
2078/// stay virtual-desktop points: `--display` narrows what may be addressed, it
2079/// does not rebase them.
2080fn gate_display(
2081    backend: &dyn DesktopBackend,
2082    context: &CallContext,
2083    action: &Action,
2084) -> Result<(), ComputerError> {
2085    let Some(spec) = context.display.as_deref() else {
2086        return Ok(());
2087    };
2088    // Syntax is checked for every action, so garbage never reaches the backend.
2089    // `virtual` is a CLI-only keyword: a JSON request cannot exec a prepared
2090    // X server, so it gets a structured `unsupported` instead of a silent
2091    // whole-desktop capture.
2092    crate::isolation::parse_selector(spec)?;
2093    // `displays` and `isolation` report on filtering, so they do not consume it.
2094    if matches!(action, Action::Isolation | Action::Displays) {
2095        return Ok(());
2096    }
2097    let displays = backend.list_displays()?;
2098    let target = crate::isolation::resolve(spec, &displays)?;
2099    match action {
2100        // A click without a point is element-targeted, so there is nothing to
2101        // filter: the element already resolved inside its window.
2102        Action::Click { x, y, .. }
2103        | Action::DoubleClick { x, y, .. }
2104        | Action::TripleClick { x, y, .. }
2105        | Action::Scroll { x, y, .. } => match (x, y) {
2106            (Some(x), Some(y)) => crate::isolation::ensure_point_inside(&target, *x, *y),
2107            _ => Ok(()),
2108        },
2109        Action::Move { x, y } | Action::MouseDown { x, y, .. } | Action::MouseUp { x, y, .. } => {
2110            crate::isolation::ensure_point_inside(&target, *x, *y)
2111        }
2112        Action::Drag { path, .. } => {
2113            for point in path {
2114                crate::isolation::ensure_point_inside(&target, point.x, point.y)?;
2115            }
2116            Ok(())
2117        }
2118        Action::Screenshot {
2119            region: Some(region),
2120            ..
2121        } => crate::isolation::ensure_region_inside(&target, *region),
2122        Action::Screenshot { .. } => Ok(()),
2123        Action::Zoom { region, .. } => crate::isolation::ensure_region_inside(&target, *region),
2124        _ => Ok(()),
2125    }
2126}
2127
2128fn remap_optional_point(
2129    context: &CallContext,
2130    x: Option<i32>,
2131    y: Option<i32>,
2132) -> Result<(Option<i32>, Option<i32>), ComputerError> {
2133    match (x, y) {
2134        (Some(x), Some(y)) => {
2135            let (x, y) = remap_point(context, x, y)?;
2136            Ok((Some(x), Some(y)))
2137        }
2138        other => Ok(other),
2139    }
2140}
2141
2142fn remap_point(context: &CallContext, x: i32, y: i32) -> Result<(i32, i32), ComputerError> {
2143    let map = require_screenshot_map(context)?;
2144    ensure_point_inside(&map, x, y)?;
2145    let (screen_x, screen_y) = image_util::image_to_screen(&map, f64::from(x), f64::from(y));
2146    Ok((round_screen(screen_x)?, round_screen(screen_y)?))
2147}
2148
2149fn remap_region(context: &CallContext, region: Region) -> Result<Region, ComputerError> {
2150    let map = require_screenshot_map(context)?;
2151    ensure_region_inside(&map, region)?;
2152    let x2 = i64::from(region.x) + i64::from(region.width);
2153    let y2 = i64::from(region.y) + i64::from(region.height);
2154    let (screen_x1, screen_y1) =
2155        image_util::image_to_screen(&map, f64::from(region.x), f64::from(region.y));
2156    let (screen_x2, screen_y2) = image_util::image_to_screen(&map, x2 as f64, y2 as f64);
2157    let x = round_screen(screen_x1)?;
2158    let y = round_screen(screen_y1)?;
2159    let far_x = round_screen(screen_x2)?;
2160    let far_y = round_screen(screen_y2)?;
2161    let width = u32::try_from(i64::from(far_x) - i64::from(x))
2162        .unwrap_or(0)
2163        .max(1);
2164    let height = u32::try_from(i64::from(far_y) - i64::from(y))
2165        .unwrap_or(0)
2166        .max(1);
2167    Region::new(x, y, width, height)
2168}
2169
2170fn require_screenshot_map(
2171    context: &CallContext,
2172) -> Result<image_util::ScreenshotMap, ComputerError> {
2173    match cache::load_screenshot_map(context.session.as_deref())? {
2174        Some(map) => Ok(map),
2175        None => Err(ComputerError::InvalidArguments(
2176            "take a screenshot first".into(),
2177        )),
2178    }
2179}
2180
2181fn content_box(map: &image_util::ScreenshotMap) -> (i64, i64, i64, i64) {
2182    let left = i64::from(map.pad.x);
2183    let top = i64::from(map.pad.y);
2184    let right = left + i64::from(map.content_width);
2185    let bottom = top + i64::from(map.content_height);
2186    (left, top, right, bottom)
2187}
2188
2189fn ensure_point_inside(
2190    map: &image_util::ScreenshotMap,
2191    x: i32,
2192    y: i32,
2193) -> Result<(), ComputerError> {
2194    let (left, top, right, bottom) = content_box(map);
2195    let x = i64::from(x);
2196    let y = i64::from(y);
2197    if x < left || y < top || x >= right || y >= bottom {
2198        return Err(ComputerError::InvalidArguments(format!(
2199            "coordinate ({x}, {y}) is outside the screenshot"
2200        )));
2201    }
2202    Ok(())
2203}
2204
2205fn ensure_region_inside(
2206    map: &image_util::ScreenshotMap,
2207    region: Region,
2208) -> Result<(), ComputerError> {
2209    let (left, top, right, bottom) = content_box(map);
2210    let x1 = i64::from(region.x);
2211    let y1 = i64::from(region.y);
2212    let x2 = x1 + i64::from(region.width);
2213    let y2 = y1 + i64::from(region.height);
2214    if x1 < left || y1 < top || x1 >= right || y1 >= bottom || x2 > right || y2 > bottom {
2215        return Err(ComputerError::InvalidArguments(format!(
2216            "region {},{} {}x{} is outside the screenshot",
2217            region.x, region.y, region.width, region.height
2218        )));
2219    }
2220    Ok(())
2221}
2222
2223fn round_screen(value: f64) -> Result<i32, ComputerError> {
2224    if !value.is_finite() {
2225        return Err(ComputerError::InvalidArguments(
2226            "coordinate remap was not finite".into(),
2227        ));
2228    }
2229    let rounded = value.round();
2230    if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) {
2231        return Err(ComputerError::InvalidArguments(
2232            "coordinate remap is out of range".into(),
2233        ));
2234    }
2235    Ok(rounded as i32)
2236}
2237
2238fn run_batch(
2239    backend: &dyn DesktopBackend,
2240    context: &CallContext,
2241    actions: Vec<Value>,
2242    stop_on_error: bool,
2243) -> Response {
2244    // Steps share one hold. The drop releases only when this is the outermost session.
2245    let _session = InputSession::open(backend);
2246    let mut results = Vec::with_capacity(actions.len());
2247    let mut first_error: Option<(usize, ErrorBody)> = None;
2248    for (index, step) in actions.into_iter().enumerate() {
2249        let response = dispatch_batch_step(backend, context, step);
2250        let failed = !response.ok;
2251        if failed && first_error.is_none() {
2252            if let Some(error) = response.error.clone() {
2253                first_error = Some((index, error));
2254            }
2255        }
2256        results.push(response.to_value());
2257        if failed && stop_on_error {
2258            return attach_post_state(
2259                backend,
2260                context,
2261                PostKind::Mutating,
2262                batch_response(results, Some(index), first_error),
2263            );
2264        }
2265    }
2266    attach_post_state(
2267        backend,
2268        context,
2269        PostKind::Mutating,
2270        batch_response(results, None, first_error),
2271    )
2272}
2273
2274fn dispatch_batch_step(
2275    backend: &dyn DesktopBackend,
2276    outer: &CallContext,
2277    value: Value,
2278) -> Response {
2279    match prepare_batch_step(outer, value) {
2280        Ok((action, context)) => dispatch_with_context(backend, action, &context),
2281        Err(error) => Response::from_error(error),
2282    }
2283}
2284
2285fn prepare_batch_step(
2286    outer: &CallContext,
2287    mut value: Value,
2288) -> Result<(Action, CallContext), ComputerError> {
2289    let Some(object) = value.as_object_mut() else {
2290        return Err(ComputerError::InvalidArguments(
2291            "batch step must be a JSON object".into(),
2292        ));
2293    };
2294    if object.get("action").and_then(Value::as_str) == Some("batch") {
2295        return Err(ComputerError::InvalidArguments(
2296            "nested batch is not allowed".into(),
2297        ));
2298    }
2299    let mut context = outer.clone();
2300    overlay_context(&mut context, object)?;
2301    context.return_state = false;
2302    context.state_diff = false;
2303    context.include_screenshot = false;
2304    let action = serde_json::from_value(value)
2305        .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
2306    Ok((action, context))
2307}
2308
2309fn overlay_context(
2310    context: &mut CallContext,
2311    object: &mut Map<String, Value>,
2312) -> Result<(), ComputerError> {
2313    if object.contains_key("app") {
2314        context.app = take_stringish(object, "app")?;
2315    }
2316    if object.contains_key("window") {
2317        context.window = take_stringish(object, "window")?;
2318    }
2319    if object.contains_key("element") {
2320        context.element = take_optional_u32(object, "element")?;
2321    }
2322    if object.contains_key("diff") {
2323        context.diff = take_bool(object, "diff")?;
2324    }
2325    if object.contains_key("full") {
2326        context.full = take_bool(object, "full")?;
2327    }
2328    let _ = object.remove("return_state");
2329    let _ = object.remove("state_diff");
2330    let _ = object.remove("include_screenshot");
2331    let _ = object.remove("id");
2332    let _ = object.remove("stop_on_error");
2333    if object.contains_key("coord") {
2334        context.coord = take_stringish(object, "coord")?;
2335    }
2336    if object.contains_key("session") {
2337        context.session = take_stringish(object, "session")?;
2338    }
2339    if object.contains_key("tier") {
2340        context.tier = take_stringish(object, "tier")?;
2341    }
2342    if object.contains_key("adapter") {
2343        context.adapter = take_stringish(object, "adapter")?;
2344    }
2345    if object.contains_key("display") {
2346        context.display = take_stringish(object, "display")?;
2347    }
2348    if object.contains_key("require_visible") {
2349        context.require_visible = take_bool(object, "require_visible")?;
2350    }
2351    if object.contains_key("allow_foreground") {
2352        context.allow_foreground = take_bool(object, "allow_foreground")?;
2353    }
2354    if object.contains_key("require_background") {
2355        context.require_background = take_bool(object, "require_background")?;
2356    }
2357    if object.contains_key("no_overlay") {
2358        context.no_overlay = take_bool(object, "no_overlay")?;
2359    }
2360    Ok(())
2361}
2362
2363fn batch_response(
2364    results: Vec<Value>,
2365    stopped_at: Option<usize>,
2366    first_error: Option<(usize, ErrorBody)>,
2367) -> Response {
2368    let mut fields = Map::new();
2369    fields.insert("action".into(), json!("batch"));
2370    fields.insert("results".into(), Value::Array(results));
2371    if let Some(index) = stopped_at {
2372        fields.insert("stopped_at".into(), json!(index));
2373    }
2374    match first_error {
2375        Some((index, error)) => {
2376            let mut response = Response::failure(
2377                error.code,
2378                format!("batch failed at action {index}: {}", error.message),
2379            );
2380            response.fields = fields;
2381            response
2382        }
2383        None => Response {
2384            ok: true,
2385            id: None,
2386            error: None,
2387            fields,
2388        },
2389    }
2390}
2391
2392fn attach_post_state(
2393    backend: &dyn DesktopBackend,
2394    context: &CallContext,
2395    kind: PostKind,
2396    mut response: Response,
2397) -> Response {
2398    // `state_diff` implies the post-state: it is the same attachment without
2399    // the full tree.
2400    if !(context.return_state || context.state_diff) || matches!(kind, PostKind::ReadOnly) {
2401        return response;
2402    }
2403    let mut warnings = Vec::new();
2404    let mut state = match backend.tree(&tree_query(context)) {
2405        Ok(report) => {
2406            // `diff` is `None` when this observation created the baseline.
2407            let baseline = if report.diff.is_some() {
2408                "existing"
2409            } else {
2410                "created"
2411            };
2412            let mut state = json!({
2413                "app": report.app,
2414                "pid": report.pid,
2415                "window_id": report.window_id,
2416                "revision": report.revision,
2417                "diff": report.diff,
2418                "truncated": report.truncated,
2419                "baseline": baseline,
2420            });
2421            // `--state-diff` keeps the delta and drops the full tree. The full
2422            // tree is the expensive half of this payload.
2423            if !context.state_diff {
2424                state["tree"] = json!(report.full_text);
2425            }
2426            state
2427        }
2428        Err(error) => {
2429            if !response.ok {
2430                return response;
2431            }
2432            warnings.push(format!("tree snapshot failed: {error}"));
2433            json!({
2434                "app": null,
2435                "pid": null,
2436                "window_id": null,
2437                "revision": null,
2438                "tree": null,
2439                "diff": null,
2440                "truncated": null,
2441                "baseline": null,
2442            })
2443        }
2444    };
2445    if context.include_screenshot {
2446        let shot = if response.ok && matches!(kind, PostKind::Screenshot) {
2447            Ok(Value::Object(response.fields.clone()))
2448        } else if matches!(kind, PostKind::Screenshot) {
2449            Err(ComputerError::ExecutionFailed(
2450                response
2451                    .error
2452                    .as_ref()
2453                    .map(|error| error.message.clone())
2454                    .unwrap_or_else(|| "screenshot failed".into()),
2455            ))
2456        } else {
2457            screenshot(
2458                backend,
2459                context,
2460                ScreenshotArgs {
2461                    output: None,
2462                    region: None,
2463                    format: None,
2464                    max_width: None,
2465                    canvas: None,
2466                    include_base64: false,
2467                    window_crop: false,
2468                    marks: false,
2469                },
2470            )
2471        };
2472        match shot {
2473            Ok(value) => state["screenshot"] = value,
2474            Err(error) => {
2475                state["screenshot"] = Value::Null;
2476                warnings.push(format!("screenshot failed: {error}"));
2477            }
2478        }
2479    }
2480    response.fields.insert("state".into(), state);
2481    push_warnings(&mut response, warnings);
2482    response
2483}
2484
2485fn push_warnings(response: &mut Response, extra: Vec<String>) {
2486    if extra.is_empty() {
2487        return;
2488    }
2489    let mut warnings = match response.fields.remove("warnings") {
2490        Some(Value::Array(items)) => items,
2491        Some(other) => vec![other],
2492        None => Vec::new(),
2493    };
2494    for warning in extra {
2495        warnings.push(json!(warning));
2496    }
2497    response
2498        .fields
2499        .insert("warnings".into(), Value::Array(warnings));
2500}
2501
2502fn cursor_position(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2503    let (x, y) = backend.cursor_position()?;
2504    Ok(json!({"action": "cursor_position", "x": x, "y": y}))
2505}
2506
2507fn invalid_arguments(message: &str) -> ComputerError {
2508    ComputerError::InvalidArguments(message.to_string())
2509}
2510
2511fn unsupported_error(operation: &str) -> ComputerError {
2512    ComputerError::Unsupported(format!(
2513        "{operation} is not available on {}",
2514        std::env::consts::OS
2515    ))
2516}
2517
2518fn menu_list(backend: &dyn DesktopBackend, app: &str, depth: u32) -> Result<Value, ComputerError> {
2519    let report = backend.menu_list(app, depth)?;
2520    Ok(json!({
2521        "action": "menu_list",
2522        "app": report.app,
2523        "sections": report.sections,
2524    }))
2525}
2526
2527fn menu_click(backend: &dyn DesktopBackend, app: &str, path: &str) -> Result<Value, ComputerError> {
2528    let parsed = crate::menu_path::parse(path)?;
2529    let result = backend.menu_click(app, &parsed)?;
2530    Ok(json!({
2531        "action": "menu_click",
2532        "app": app,
2533        "path": parsed.segments,
2534        "pressed": result.pressed,
2535    }))
2536}
2537
2538fn apple_event_action(
2539    backend: &dyn DesktopBackend,
2540    app: &str,
2541    script: &str,
2542    language: AppleEventLanguage,
2543) -> Result<Value, ComputerError> {
2544    let report = backend.apple_event(app, script, language)?;
2545    Ok(json!({
2546        "action": "apple_event",
2547        "app": app,
2548        "language": language.as_str(),
2549        "script_bytes": report.script_bytes,
2550        "script_hash": report.script_hash,
2551        "stdout": report.stdout,
2552        "stderr": report.stderr,
2553        "exit_status": report.exit_status,
2554    }))
2555}
2556
2557fn shortcuts_run_action(name: String) -> Result<Value, ComputerError> {
2558    let report = backend_shortcuts_run(&name)?;
2559    Ok(json!({
2560        "action": "shortcuts_run",
2561        "name": name,
2562        "exit_status": report.exit_status,
2563        "stderr": report.stderr,
2564    }))
2565}
2566
2567fn dock_list_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2568    let report = backend.dock_list()?;
2569    Ok(json!({
2570        "action": "dock_list",
2571        "items": report.items,
2572    }))
2573}
2574
2575fn dock_click_action(backend: &dyn DesktopBackend, title: &str) -> Result<Value, ComputerError> {
2576    let report = backend.dock_click(title)?;
2577    Ok(json!({
2578        "action": "dock_click",
2579        "title": title,
2580        "pressed": report.pressed,
2581    }))
2582}
2583
2584fn spaces_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2585    let report = backend.spaces()?;
2586    Ok(json!({
2587        "action": "spaces",
2588        "screens": report.screens,
2589        "note": report.note,
2590    }))
2591}
2592
2593// `shortcuts` does not need the desktop backend, but the helper signature
2594// mirrors the others and leaves room for backend-specific policy hooks.
2595fn backend_shortcuts_run(name: &str) -> Result<crate::shortcuts::ShortcutsReport, ComputerError> {
2596    crate::shortcuts::run(name)
2597}
2598
2599fn session_prune(dry_run: bool, ttl_secs: Option<u64>) -> Result<Value, ComputerError> {
2600    let ttl = match ttl_secs {
2601        Some(seconds) => Duration::from_secs(seconds),
2602        None => cache::DEFAULT_SESSION_TTL,
2603    };
2604    let report = cache::prune_sessions(
2605        &cache::sessions_root(),
2606        &cache::PruneOptions {
2607            dry_run,
2608            ttl,
2609            now: SystemTime::now(),
2610        },
2611        &cache::pid_alive,
2612    )?;
2613    Ok(json!({
2614        "action": "session_prune",
2615        "dry_run": dry_run,
2616        "ttl_secs": ttl.as_secs(),
2617        "removed": report.removed,
2618        "kept": report.kept,
2619    }))
2620}
2621
2622struct ScreenshotArgs {
2623    output: Option<String>,
2624    region: Option<Region>,
2625    format: Option<ImageFormat>,
2626    max_width: Option<u32>,
2627    canvas: Option<String>,
2628    include_base64: bool,
2629    /// When set, a window target crops `region` in window-local points.
2630    /// Screenshot still refuses a region together with a window.
2631    window_crop: bool,
2632    /// Stamp numbered element rectangles on the image before encoding.
2633    marks: bool,
2634}
2635
2636fn screenshot(
2637    backend: &dyn DesktopBackend,
2638    context: &CallContext,
2639    args: ScreenshotArgs,
2640) -> Result<Value, ComputerError> {
2641    let ScreenshotArgs {
2642        output,
2643        region,
2644        format,
2645        max_width,
2646        canvas,
2647        include_base64,
2648        window_crop,
2649        marks,
2650    } = args;
2651    let targeted = context.app.is_some() || context.window.is_some();
2652    if region.is_some() && targeted && !window_crop {
2653        return Err(ComputerError::InvalidArguments(
2654            "pass a region or a window target, not both".into(),
2655        ));
2656    }
2657    let format = format.unwrap_or_default();
2658    let max_width = max_width.unwrap_or(image_util::DEFAULT_SCREENSHOT_MAX_WIDTH);
2659    let canvas = canvas
2660        .as_deref()
2661        .map(image_util::parse_canvas)
2662        .transpose()?;
2663    let raw = if targeted {
2664        if let Some(error) = backend.display_block() {
2665            return Err(error);
2666        }
2667        let target = resolve_screenshot_window(backend, context)?;
2668        // A window capture cannot be redirected onto another monitor, so a
2669        // filtered request only works when the window already sits there.
2670        if let Some(filter) = display_filter(backend, context)? {
2671            if !crate::isolation::window_inside(&filter, target.bounds) {
2672                return Err(ComputerError::InvalidArguments(format!(
2673                    "window {}x{}+{}+{} is not inside display {} ({} covers {})",
2674                    target.bounds.width,
2675                    target.bounds.height,
2676                    target.bounds.x,
2677                    target.bounds.y,
2678                    filter.display.id,
2679                    filter.display.name,
2680                    filter.bounds()
2681                )));
2682            }
2683        }
2684        let captured = backend.capture_window(&target)?;
2685        match region {
2686            Some(region) if window_crop => crop_window_region(captured, region)?,
2687            _ => captured,
2688        }
2689    } else {
2690        // Capture the chosen monitor only. `capture_region` maps the exact
2691        // monitor rectangle onto one monitor capture, so the screenshot map
2692        // origin is that monitor's origin and the image is its pixels.
2693        let region = match display_filter(backend, context)? {
2694            Some(filter) => Some(filter.region),
2695            None => region,
2696        };
2697        backend.capture_screen(region)?
2698    };
2699    let prepared = image_util::prepare_shot(
2700        raw.frame.image(),
2701        image_util::ShotFit {
2702            scale: raw.frame.scale(),
2703            origin_x: raw.origin_x,
2704            origin_y: raw.origin_y,
2705            logical_width: raw.logical_width,
2706            logical_height: raw.logical_height,
2707            max_width,
2708            canvas,
2709        },
2710    )?;
2711    let mut image = prepared.image;
2712    let mut warnings = raw.warnings;
2713    if marks {
2714        let applied = apply_marks(&mut image, &prepared.map, context, backend)?;
2715        if applied == 0 {
2716            // A silent zero is indistinguishable from "the marks were drawn".
2717            // Say so instead of writing a clean capture that answers `ok`.
2718            warnings.push("marks: no accessibility element frame fell inside this capture".into());
2719        }
2720    }
2721    let encoded = image_util::encode_rgba(&image, format)?;
2722    let path = match output {
2723        Some(path) => PathBuf::from(path),
2724        None => default_screenshot_path(format),
2725    };
2726    if let Some(parent) = path.parent() {
2727        if !parent.as_os_str().is_empty() {
2728            fs::create_dir_all(parent)?;
2729        }
2730    }
2731    fs::write(&path, &encoded.bytes)?;
2732    if let Err(error) = cache::store_screenshot_map(context.session.as_deref(), &prepared.map) {
2733        warnings.push(format!("screenshot map was not cached: {error}"));
2734    }
2735    let mut body = serde_json::to_value(&prepared.map).map_err(|error| {
2736        ComputerError::ExecutionFailed(format!("screenshot map encode failed: {error}"))
2737    })?;
2738    body["path"] = json!(path_string(&path));
2739    body["format"] = json!(format.as_str());
2740    body["capture_source"] = json!(raw.capture_source);
2741    if raw.occluded {
2742        body["occluded"] = json!(true);
2743    }
2744    if !warnings.is_empty() {
2745        body["warnings"] = json!(warnings);
2746    }
2747    if include_base64 {
2748        body["data"] = json!(encode_base64(&encoded.bytes));
2749    }
2750    Ok(body)
2751}
2752
2753fn resolve_screenshot_window(
2754    backend: &dyn DesktopBackend,
2755    context: &CallContext,
2756) -> Result<WindowTarget, ComputerError> {
2757    let listed = backend.list_windows()?;
2758    if let Some(app) = context.app.as_deref() {
2759        let apps = backend.list_apps(true)?;
2760        let selector = parse_app_selector(app)?;
2761        let app = resolve_app(&apps, &selector)?;
2762        let owned = filter_windows_for_pid(&listed.windows, app.pid);
2763        let selector = match context.window.as_deref() {
2764            Some(window) => parse_window_selector(window)?,
2765            None => WindowSelector::Index(0),
2766        };
2767        let window = resolve_window(&owned, &selector)?;
2768        return Ok(window_target(window));
2769    }
2770    let selector = parse_window_selector(context.window.as_deref().unwrap_or(""))?;
2771    let window = resolve_window(&listed.windows, &selector)?;
2772    Ok(window_target(window))
2773}
2774
2775fn window_target(window: &crate::target::WindowInfo) -> WindowTarget {
2776    WindowTarget {
2777        window_id: window.id,
2778        pid: window.pid,
2779        bounds: window.bounds,
2780    }
2781}
2782
2783/// Crop `region` in window-local logical points. The far edge may equal the window size.
2784fn crop_window_region(
2785    raw: crate::types::RawCapture,
2786    region: Region,
2787) -> Result<crate::types::RawCapture, ComputerError> {
2788    region_inside(region, raw.logical_width, raw.logical_height)?;
2789    let scale = raw.frame.scale();
2790    let x = logical_to_device(i64::from(region.x), scale)?;
2791    let y = logical_to_device(i64::from(region.y), scale)?;
2792    let width = logical_len_to_device(region.width, scale)?;
2793    let height = logical_len_to_device(region.height, scale)?;
2794    let image = raw.frame.image();
2795    if x >= image.width() || y >= image.height() {
2796        return Err(ComputerError::InvalidArguments(
2797            "region is outside the capture".into(),
2798        ));
2799    }
2800    let width = width.min(image.width() - x).max(1);
2801    let height = height.min(image.height() - y).max(1);
2802    let cropped = image::imageops::crop_imm(image, x, y, width, height).to_image();
2803    Ok(crate::types::RawCapture {
2804        frame: Frame::from_image(cropped, scale),
2805        origin_x: raw.origin_x.saturating_add(region.x),
2806        origin_y: raw.origin_y.saturating_add(region.y),
2807        logical_width: region.width,
2808        logical_height: region.height,
2809        capture_source: raw.capture_source,
2810        occluded: raw.occluded,
2811        warnings: raw.warnings,
2812    })
2813}
2814
2815fn logical_to_device(value: i64, scale: f64) -> Result<u32, ComputerError> {
2816    if value < 0 {
2817        return Err(ComputerError::InvalidArguments(
2818            "region is outside the capture".into(),
2819        ));
2820    }
2821    let pixels = (value as f64) * scale;
2822    if !pixels.is_finite() || pixels < 0.0 || pixels > f64::from(u32::MAX) {
2823        return Err(ComputerError::ExecutionFailed(
2824            "region pixel coordinate is out of range".into(),
2825        ));
2826    }
2827    Ok(pixels.round() as u32)
2828}
2829
2830fn logical_len_to_device(value: u32, scale: f64) -> Result<u32, ComputerError> {
2831    let pixels = logical_to_device(i64::from(value), scale)?;
2832    Ok(pixels.max(1))
2833}
2834
2835fn default_screenshot_path(format: ImageFormat) -> PathBuf {
2836    static SEQUENCE: AtomicU64 = AtomicU64::new(0);
2837    let nanos = SystemTime::now()
2838        .duration_since(UNIX_EPOCH)
2839        .map(|duration| duration.as_nanos())
2840        .unwrap_or(0);
2841    let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed);
2842    let name = format!(
2843        "computer-use-{}-{nanos}-{sequence}.{}",
2844        std::process::id(),
2845        format.extension()
2846    );
2847    std::env::temp_dir().join(name)
2848}
2849
2850fn path_string(path: &Path) -> String {
2851    path.to_string_lossy().into_owned()
2852}
2853
2854fn click(
2855    backend: &dyn DesktopBackend,
2856    context: &CallContext,
2857    x: Option<i32>,
2858    y: Option<i32>,
2859    button: MouseButton,
2860    double: bool,
2861    count: u8,
2862) -> Response {
2863    let count = match effective_click_count(double, count) {
2864        Ok(count) => count,
2865        Err(error) => return Response::from_error(error),
2866    };
2867    match (context.element, x, y) {
2868        (Some(_), Some(_), _) | (Some(_), _, Some(_)) => {
2869            Response::failure("invalid_arguments", "pass element or x and y, not both")
2870        }
2871        (None, None, None) => Response::failure(
2872            "invalid_arguments",
2873            "click requires element, or both x and y",
2874        ),
2875        (None, Some(_), None) | (None, None, Some(_)) => {
2876            Response::failure("invalid_arguments", "x and y must be passed together")
2877        }
2878        (Some(_index), None, None) => {
2879            element_command(backend, context, ElementOp::Press { count, button })
2880        }
2881        (None, Some(x), Some(y)) => coordinate_click(backend, context, x, y, button, count),
2882    }
2883}
2884
2885fn effective_click_count(double: bool, count: u8) -> Result<u8, ComputerError> {
2886    if count == 0 || count > 3 {
2887        return Err(ComputerError::InvalidArguments(
2888            "click count must be 1, 2, or 3".into(),
2889        ));
2890    }
2891    if double && count != 1 && count != 2 {
2892        return Err(ComputerError::InvalidArguments(
2893            "double and count disagree".into(),
2894        ));
2895    }
2896    if double {
2897        Ok(2)
2898    } else {
2899        Ok(count)
2900    }
2901}
2902
2903fn coordinate_click(
2904    backend: &dyn DesktopBackend,
2905    context: &CallContext,
2906    x: i32,
2907    y: i32,
2908    button: MouseButton,
2909    count: u8,
2910) -> Response {
2911    let kind = InputKind::Click {
2912        x,
2913        y,
2914        button,
2915        count,
2916    };
2917    // Tier rejection, including synthetic without the feature, happens before hit-testing.
2918    if let Err(error) = planned_tiers(backend, context) {
2919        return Response::from_error(error);
2920    }
2921    let mut warnings = Vec::new();
2922    let hit = match backend.hit_test(x, y) {
2923        Ok(hit) => hit,
2924        Err(error) => {
2925            warnings.push(format!("hit test failed: {error}"));
2926            None
2927        }
2928    };
2929    if let Some(title) = hit.as_ref().and_then(|hit| hit.title.as_deref()) {
2930        if policy::sensitive_title(title) {
2931            if let Err(error) = confirm_title(context, "click", "sensitive control title") {
2932                return Response::from_error(error);
2933            }
2934        }
2935    }
2936    match run_attempts(backend, context, kind) {
2937        Ok(report) => {
2938            let action_name = match count {
2939                2 => "double_click",
2940                3 => "triple_click",
2941                _ => "click",
2942            };
2943            let body = json!({
2944                "action": action_name,
2945                "x": x,
2946                "y": y,
2947                "button": button.as_str(),
2948                "double": count == 2,
2949                "count": count,
2950                "hit": hit_value(hit),
2951            });
2952            Response::success(finish_input(body, report, warnings))
2953        }
2954        Err(error) => Response::from_error(error),
2955    }
2956}
2957
2958fn hit_value(hit: Option<crate::element::HitInfo>) -> Value {
2959    match hit {
2960        Some(hit) => json!({"role": hit.role, "title": hit.title}),
2961        None => Value::Null,
2962    }
2963}
2964
2965fn type_command(backend: &dyn DesktopBackend, context: &CallContext, text: String) -> Response {
2966    if context.element.is_some() {
2967        return element_command(backend, context, ElementOp::Type { text });
2968    }
2969    if text.is_empty() {
2970        return Response::from_error(ComputerError::InvalidArguments(
2971            "text must not be empty".into(),
2972        ));
2973    }
2974    if text.contains('\0') {
2975        return Response::from_error(ComputerError::InvalidArguments(
2976            "text must not contain a null byte".into(),
2977        ));
2978    }
2979    let chars = text.chars().count();
2980    match run_attempts(backend, context, InputKind::Type { text: text.clone() }) {
2981        Ok(report) => {
2982            let body = json!({
2983                "action": "type",
2984                "text": text,
2985                "chars": chars,
2986            });
2987            Response::success(finish_input(body, report, Vec::new()))
2988        }
2989        Err(error) => Response::from_error(error),
2990    }
2991}
2992
2993fn element_command(backend: &dyn DesktopBackend, context: &CallContext, op: ElementOp) -> Response {
2994    if let Err(error) = precheck(&op) {
2995        return Response::from_error(error);
2996    }
2997    if let Err(error) = confirm_element_title(backend, context, &op) {
2998        return Response::from_error(error);
2999    }
3000    let index = match context.element {
3001        Some(index) => index,
3002        None => {
3003            return Response::from_error(ComputerError::InvalidArguments(
3004                "element is required".into(),
3005            ))
3006        }
3007    };
3008    let request = ElementRequest {
3009        op,
3010        index,
3011        require_visible: context.require_visible,
3012        app: context.app.clone(),
3013        window: context.window.clone(),
3014        session: context.session.clone(),
3015        tier: context.tier.clone(),
3016        allow_foreground: context.allow_foreground,
3017        require_background: context.require_background,
3018        no_overlay: context.no_overlay,
3019    };
3020    match backend.element_action(&request) {
3021        Ok(outcome) => Response::success(outcome_value(&outcome)),
3022        Err(error) => element_error_response(error),
3023    }
3024}
3025
3026fn confirm_element_title(
3027    backend: &dyn DesktopBackend,
3028    context: &CallContext,
3029    op: &ElementOp,
3030) -> Result<(), ComputerError> {
3031    let press = matches!(op, ElementOp::Press { .. })
3032        || matches!(op, ElementOp::Perform { name } if name.eq_ignore_ascii_case("AXPress"));
3033    if !press {
3034        return Ok(());
3035    }
3036    let Some(index) = context.element else {
3037        return Ok(());
3038    };
3039    let Ok(Some(peek)) = backend.peek_element(
3040        context.app.as_deref(),
3041        context.window.as_deref(),
3042        context.session.as_deref(),
3043        index,
3044    ) else {
3045        return Ok(());
3046    };
3047    if peek.title.as_deref().is_some_and(policy::sensitive_title) {
3048        confirm_title(context, "press", "sensitive control title")?;
3049    }
3050    Ok(())
3051}
3052
3053fn confirm_title(context: &CallContext, action: &str, reason: &str) -> Result<(), ComputerError> {
3054    let loaded = policy::load_policy(context.policy.as_deref().map(Path::new))?;
3055    let command = policy::confirm_program(&loaded, context.confirm_cmd.as_deref());
3056    policy::run_confirm(
3057        command.as_deref(),
3058        loaded.timeout_secs(),
3059        &policy::ConfirmDocument {
3060            action,
3061            bundle_id: None,
3062            path: None,
3063            pid: None,
3064            window_id: None,
3065            element: context.element,
3066            reason,
3067        },
3068    )
3069}
3070
3071fn precheck(op: &ElementOp) -> Result<(), ComputerError> {
3072    match op {
3073        ElementOp::Press { count, .. } => element::validate_count(*count),
3074        ElementOp::SetValue { value } => element::validate_text(value, true),
3075        ElementOp::Type { text } => element::validate_text(text, false),
3076        ElementOp::SelectText { start, end } => element::validate_range(*start, *end, None),
3077        ElementOp::Perform { name } => element::validate_perform_name(name),
3078        ElementOp::Focus | ElementOp::ScrollTo => Ok(()),
3079    }
3080}
3081
3082fn outcome_value(outcome: &ElementOutcome) -> Value {
3083    let mut body = json!({
3084        "action": outcome.action,
3085        "element": outcome.element,
3086        "revision": outcome.revision,
3087        "tier": outcome.tier,
3088        "cursor_moved": outcome.cursor_moved,
3089    });
3090    if let Some(restored) = outcome.focus_restored {
3091        body["focus_restored"] = json!(restored);
3092    }
3093    if !outcome.warnings.is_empty() {
3094        body["warnings"] = json!(outcome.warnings);
3095    }
3096    if let Some(via) = &outcome.via {
3097        body["via"] = json!(via);
3098    }
3099    if let Some(name) = &outcome.name {
3100        body["name"] = json!(name);
3101    }
3102    if let Some(count) = outcome.count {
3103        body["count"] = json!(count);
3104    }
3105    body
3106}
3107
3108fn element_error_response(error: ElementError) -> Response {
3109    let mut response = Response::from_error(error.error);
3110    if let Some(tree) = error.tree {
3111        response.fields.insert("tree".into(), json!(tree));
3112    }
3113    response
3114}
3115
3116fn move_mouse(backend: &dyn DesktopBackend, context: &CallContext, x: i32, y: i32) -> Response {
3117    match run_attempts(backend, context, InputKind::Move { x, y }) {
3118        Ok(report) => Response::success(finish_input(
3119            json!({"action": "move", "x": x, "y": y}),
3120            report,
3121            Vec::new(),
3122        )),
3123        Err(error) => Response::from_error(error),
3124    }
3125}
3126
3127fn press_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str) -> Response {
3128    let strokes = match key_strokes(keys) {
3129        Ok(strokes) => strokes,
3130        Err(error) => return Response::from_error(error),
3131    };
3132    let mut body = json!({
3133        "action": "key",
3134        "keys": keys,
3135    });
3136    if strokes.len() == 1 {
3137        let modifiers: Vec<&str> = strokes[0]
3138            .modifiers
3139            .iter()
3140            .copied()
3141            .map(Modifier::as_str)
3142            .collect();
3143        body["modifiers"] = json!(modifiers);
3144        body["key"] = json!(strokes[0].key);
3145    } else {
3146        body["strokes"] = json!(strokes.len());
3147    }
3148    match run_attempts(backend, context, InputKind::Key { strokes }) {
3149        Ok(report) => Response::success(finish_input(body, report, Vec::new())),
3150        Err(error) => Response::from_error(error),
3151    }
3152}
3153
3154fn key_strokes(keys: &str) -> Result<Vec<KeyStroke>, ComputerError> {
3155    let combos = parse_key_sequence(keys)?;
3156    let mut strokes = Vec::with_capacity(combos.len());
3157    for combo in combos {
3158        validate_key(&combo.key)?;
3159        strokes.push(KeyStroke {
3160            modifiers: combo.modifiers,
3161            key: combo.key,
3162        });
3163    }
3164    Ok(strokes)
3165}
3166
3167fn drag(
3168    backend: &dyn DesktopBackend,
3169    context: &CallContext,
3170    path: Vec<Point>,
3171    button: MouseButton,
3172) -> Response {
3173    if path.len() < 2 {
3174        return Response::from_error(ComputerError::InvalidArguments(
3175            "drag needs at least 2 points".into(),
3176        ));
3177    }
3178    let points: Vec<(i32, i32)> = path.iter().map(|point| (point.x, point.y)).collect();
3179    let moves = points.len() - 1;
3180    match run_attempts(backend, context, InputKind::Drag { points, button }) {
3181        Ok(report) => Response::success(finish_input(
3182            json!({
3183                "action": "drag",
3184                "button": button.as_str(),
3185                "points": path.len(),
3186                "moves": moves,
3187            }),
3188            report,
3189            Vec::new(),
3190        )),
3191        Err(error) => Response::from_error(error),
3192    }
3193}
3194
3195fn pointer_half(
3196    backend: &dyn DesktopBackend,
3197    context: &CallContext,
3198    x: i32,
3199    y: i32,
3200    button: MouseButton,
3201    down: bool,
3202) -> Response {
3203    let kind = if down {
3204        InputKind::MouseDown { x, y, button }
3205    } else {
3206        InputKind::MouseUp { x, y, button }
3207    };
3208    let name = if down { "mouse_down" } else { "mouse_up" };
3209    match run_attempts(backend, context, kind) {
3210        Ok(report) => Response::success(finish_input(
3211            json!({
3212                "action": name,
3213                "x": x,
3214                "y": y,
3215                "button": button.as_str(),
3216            }),
3217            report,
3218            Vec::new(),
3219        )),
3220        Err(error) => Response::from_error(error),
3221    }
3222}
3223
3224fn hold_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str, ms: u64) -> Response {
3225    let stroke = match one_hold(keys, ms) {
3226        Ok(stroke) => stroke,
3227        Err(error) => return Response::from_error(error),
3228    };
3229    match run_attempts(
3230        backend,
3231        context,
3232        InputKind::HoldKey {
3233            modifiers: stroke.modifiers.clone(),
3234            key: stroke.key.clone(),
3235            ms,
3236        },
3237    ) {
3238        Ok(report) => Response::success(finish_input(
3239            json!({
3240                "action": "hold_key",
3241                "keys": keys,
3242                "key": stroke.key,
3243                "ms": ms,
3244            }),
3245            report,
3246            Vec::new(),
3247        )),
3248        Err(error) => Response::from_error(error),
3249    }
3250}
3251
3252fn one_hold(keys: &str, ms: u64) -> Result<KeyStroke, ComputerError> {
3253    if ms > tier::MAX_HOLD_MS {
3254        return Err(ComputerError::InvalidArguments(format!(
3255            "hold_key ms must be at most {}",
3256            tier::MAX_HOLD_MS
3257        )));
3258    }
3259    if keys.split_whitespace().count() != 1 {
3260        return Err(ComputerError::InvalidArguments(
3261            "hold_key accepts one key combination".into(),
3262        ));
3263    }
3264    let combo = parse_key_combo(keys)?;
3265    validate_key(&combo.key)?;
3266    Ok(KeyStroke {
3267        modifiers: combo.modifiers,
3268        key: combo.key,
3269    })
3270}
3271
3272fn clipboard_chars(action: &Action) -> Option<u64> {
3273    match action {
3274        Action::ClipboardSet {
3275            text: Some(text), ..
3276        } => Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX)),
3277        Action::BrowserType { text, .. } => {
3278            Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX))
3279        }
3280        _ => None,
3281    }
3282}
3283
3284fn preflight_action(action: &Action) -> Result<(), ComputerError> {
3285    match action {
3286        Action::Click { double, count, .. } => {
3287            effective_click_count(*double, *count)?;
3288        }
3289        Action::Press { count, .. } => element::validate_count(*count)?,
3290        Action::Drag { path, .. } if path.len() < 2 => {
3291            return Err(ComputerError::InvalidArguments(
3292                "drag needs at least 2 points".into(),
3293            ));
3294        }
3295        Action::HoldKey { keys, ms } => {
3296            one_hold(keys, *ms)?;
3297        }
3298        Action::Key { keys } => {
3299            key_strokes(keys)?;
3300        }
3301        Action::ClipboardSet { text, image } => match (text, image) {
3302            (Some(text), None) if text.is_empty() => {
3303                return Err(ComputerError::InvalidArguments(
3304                    "clipboard text must not be empty".into(),
3305                ));
3306            }
3307            (None, None) => {
3308                return Err(ComputerError::InvalidArguments(
3309                    "clipboard set requires text or image".into(),
3310                ));
3311            }
3312            (Some(_), Some(_)) => {
3313                return Err(ComputerError::InvalidArguments(
3314                    "pass clipboard text or image, not both".into(),
3315                ));
3316            }
3317            _ => {}
3318        },
3319        _ => {}
3320    }
3321    Ok(())
3322}
3323
3324fn clipboard_get(
3325    backend: &dyn DesktopBackend,
3326    context: &CallContext,
3327) -> Result<serde_json::Value, ComputerError> {
3328    match backend.clipboard_get()? {
3329        ClipboardContents::Empty => Ok(json!({
3330            "action": "clipboard_get",
3331            "kind": "empty",
3332        })),
3333        ClipboardContents::Text(text) => {
3334            let chars = text.chars().count();
3335            Ok(json!({
3336                "action": "clipboard_get",
3337                "kind": "text",
3338                "text": text,
3339                "chars": chars,
3340            }))
3341        }
3342        ClipboardContents::Image {
3343            width,
3344            height,
3345            rgba,
3346        } => {
3347            let frame = Frame::new(width, height, 1.0, rgba)?;
3348            let encoded = image_util::encode_rgba(frame.image(), ImageFormat::Png)?;
3349            let dir = cache::session_dir(context.session.as_deref())?;
3350            let path = dir.join("clipboard.png");
3351            cache::write_secret_bytes(&path, &encoded.bytes)?;
3352            Ok(json!({
3353                "action": "clipboard_get",
3354                "kind": "image",
3355                "path": path_string(&path),
3356                "width": width,
3357                "height": height,
3358            }))
3359        }
3360    }
3361}
3362
3363fn clipboard_set(
3364    backend: &dyn DesktopBackend,
3365    text: Option<String>,
3366    image: Option<String>,
3367) -> Result<serde_json::Value, ComputerError> {
3368    match (text, image) {
3369        (Some(text), None) => {
3370            let chars = text.chars().count();
3371            backend.clipboard_set_text(&text)?;
3372            Ok(json!({
3373                "action": "clipboard_set",
3374                "kind": "text",
3375                "chars": chars,
3376            }))
3377        }
3378        (None, Some(path)) => {
3379            let (width, height, rgba) = decode_clipboard_image(&path)?;
3380            backend.clipboard_set_image(width, height, &rgba)?;
3381            Ok(json!({
3382                "action": "clipboard_set",
3383                "kind": "image",
3384                "width": width,
3385                "height": height,
3386            }))
3387        }
3388        _ => Err(ComputerError::InvalidArguments(
3389            "clipboard set requires text or image".into(),
3390        )),
3391    }
3392}
3393
3394fn decode_clipboard_image(path: &str) -> Result<(u32, u32, Vec<u8>), ComputerError> {
3395    let image = image::ImageReader::open(path)
3396        .map_err(|error| {
3397            ComputerError::InvalidArguments(format!("clipboard image could not be opened: {error}"))
3398        })?
3399        .decode()
3400        .map_err(|error| {
3401            ComputerError::InvalidArguments(format!(
3402                "clipboard image could not be decoded: {error}"
3403            ))
3404        })?
3405        .to_rgba8();
3406    let width = image.width();
3407    let height = image.height();
3408    Ok((width, height, image.into_raw()))
3409}
3410
3411fn scroll(
3412    backend: &dyn DesktopBackend,
3413    context: &CallContext,
3414    point: (Option<i32>, Option<i32>),
3415    delta: (Option<i32>, Option<i32>),
3416    direction: Option<ScrollDirection>,
3417    amount: Option<i32>,
3418) -> Response {
3419    let (x, y) = point;
3420    let (dx, dy) = delta;
3421    if x.is_some() != y.is_some() {
3422        return Response::from_error(ComputerError::InvalidArguments(
3423            "scroll x and y must be set together".into(),
3424        ));
3425    }
3426    let (scroll_dx, scroll_dy) = match resolve_scroll(dx, dy, direction, amount) {
3427        Ok(delta) => delta,
3428        Err(error) => return Response::from_error(error),
3429    };
3430    match run_attempts(
3431        backend,
3432        context,
3433        InputKind::Scroll {
3434            x,
3435            y,
3436            dx: scroll_dx,
3437            dy: scroll_dy,
3438        },
3439    ) {
3440        Ok(report) => {
3441            let body = json!({
3442                "action": "scroll",
3443                "x": x,
3444                "y": y,
3445                "dx": scroll_dx,
3446                "dy": scroll_dy,
3447            });
3448            Response::success(finish_input(scroll_body(body, &report), report, Vec::new()))
3449        }
3450        Err(error) => Response::from_error(error),
3451    }
3452}
3453
3454/// Add the verified-scroll fields when the backend reported them.
3455///
3456/// `via` is `event` when the posted wheel event moved the scroll bar and
3457/// `ax_scrollbar` when `AXValue` had to be written instead. `step` is the
3458/// per-line fraction (0..=1) the AX scroll-bar path computed from the
3459/// content-over-visible sizes, surfaced for transparency even when the wheel
3460/// event tier did the scrolling. Windows and Linux report no outcome, so
3461/// their response keeps its existing shape.
3462fn scroll_body(mut body: Value, report: &crate::tier::InputReport) -> Value {
3463    let Some(outcome) = &report.scroll else {
3464        return body;
3465    };
3466    body["via"] = json!(outcome.via.as_str());
3467    body["scrolled"] = json!(outcome.scrolled);
3468    body["before"] = json!(outcome.before);
3469    body["after"] = json!(outcome.after);
3470    body["step"] = json!(outcome.step);
3471    body
3472}
3473
3474fn planned_tiers(
3475    backend: &dyn DesktopBackend,
3476    context: &CallContext,
3477) -> Result<Vec<TierName>, ComputerError> {
3478    let requested = crate::tier::parse_tier(context.tier.as_deref())?;
3479    crate::tier::plan(
3480        requested,
3481        crate::tier::InputClass::Background,
3482        backend.input_support(),
3483        crate::tier::skylight_compiled(),
3484    )
3485}
3486
3487fn run_attempts(
3488    backend: &dyn DesktopBackend,
3489    context: &CallContext,
3490    kind: InputKind,
3491) -> Result<crate::tier::InputReport, ComputerError> {
3492    attempt::run(backend, &attempt_flags(context), kind)
3493}
3494
3495fn attempt_flags(context: &CallContext) -> AttemptFlags {
3496    AttemptFlags {
3497        app: context.app.clone(),
3498        window: context.window.clone(),
3499        tier: context.tier.clone(),
3500        allow_foreground: context.allow_foreground,
3501        require_background: context.require_background,
3502        no_overlay: context.no_overlay,
3503        session: context.session.clone(),
3504    }
3505}
3506
3507fn finish_input(
3508    mut body: Value,
3509    report: crate::tier::InputReport,
3510    mut warnings: Vec<String>,
3511) -> Value {
3512    warnings.extend(report.warnings);
3513    body["tier"] = json!(report.tier);
3514    body["cursor_moved"] = json!(report.cursor_moved);
3515    if let Some(restored) = report.focus_restored {
3516        body["focus_restored"] = json!(restored);
3517    }
3518    if !warnings.is_empty() {
3519        body["warnings"] = json!(warnings);
3520    }
3521    body
3522}
3523
3524pub(crate) fn resolve_scroll(
3525    dx: Option<i32>,
3526    dy: Option<i32>,
3527    direction: Option<ScrollDirection>,
3528    amount: Option<i32>,
3529) -> Result<(i32, i32), ComputerError> {
3530    let has_delta = dx.is_some() || dy.is_some();
3531    let has_direction = direction.is_some() || amount.is_some();
3532    if has_delta && has_direction {
3533        return Err(ComputerError::InvalidArguments(
3534            "use either dx/dy or direction/amount, not both".into(),
3535        ));
3536    }
3537    if let Some(direction) = direction {
3538        let amount = amount.unwrap_or(1);
3539        return Ok(match direction {
3540            ScrollDirection::Up => (0, -amount),
3541            ScrollDirection::Down => (0, amount),
3542            ScrollDirection::Left => (-amount, 0),
3543            ScrollDirection::Right => (amount, 0),
3544        });
3545    }
3546    if amount.is_some() {
3547        return Err(ComputerError::InvalidArguments(
3548            "amount requires direction".into(),
3549        ));
3550    }
3551    if !has_delta {
3552        return Err(ComputerError::InvalidArguments(
3553            "scroll requires dx/dy or direction".into(),
3554        ));
3555    }
3556    Ok((dx.unwrap_or(0), dy.unwrap_or(0)))
3557}
3558
3559fn wait(ms: Option<u64>, seconds: Option<f64>) -> Result<Value, ComputerError> {
3560    let waited = resolve_wait(ms, seconds)?;
3561    thread::sleep(Duration::from_millis(waited));
3562    Ok(json!({"waited_ms": waited}))
3563}
3564
3565pub(crate) fn resolve_wait(ms: Option<u64>, seconds: Option<f64>) -> Result<u64, ComputerError> {
3566    if ms.is_none() && seconds.is_none() {
3567        return Err(ComputerError::InvalidArguments(
3568            "wait requires ms, seconds, or both".into(),
3569        ));
3570    }
3571    let mut total: u128 = u128::from(ms.unwrap_or(0));
3572    if let Some(seconds) = seconds {
3573        if !seconds.is_finite() || seconds < 0.0 {
3574            return Err(ComputerError::InvalidArguments(
3575                "seconds must be a finite number greater than or equal to 0".into(),
3576            ));
3577        }
3578        let extra = (seconds * 1000.0).round();
3579        if extra > u128::MAX as f64 {
3580            return Err(ComputerError::InvalidArguments("wait is too long".into()));
3581        }
3582        total = total.saturating_add(extra as u128);
3583    }
3584    if total > MAX_WAIT_MS {
3585        return Err(ComputerError::InvalidArguments(
3586            "wait must be at most 24 hours".into(),
3587        ));
3588    }
3589    u64::try_from(total).map_err(|_| ComputerError::InvalidArguments("wait is too long".into()))
3590}
3591
3592fn displays(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
3593    let displays = backend.list_displays()?;
3594    Ok(json!({"displays": displays}))
3595}
3596
3597/// Resolve `--display` for a capture. `None` means no filter.
3598fn display_filter(
3599    backend: &dyn DesktopBackend,
3600    context: &CallContext,
3601) -> Result<Option<crate::isolation::DisplayTarget>, ComputerError> {
3602    match context.display.as_deref() {
3603        None => Ok(None),
3604        Some(spec) => Ok(Some(crate::isolation::resolve(
3605            spec,
3606            &backend.list_displays()?,
3607        )?)),
3608    }
3609}
3610
3611/// Body of the `isolation` action: what this build can and cannot isolate.
3612fn isolation() -> Result<Value, ComputerError> {
3613    Ok(crate::isolation::matrix())
3614}
3615
3616fn tree_query(context: &CallContext) -> TreeQuery {
3617    TreeQuery {
3618        app: context.app.clone(),
3619        window: context.window.clone(),
3620        full: context.full,
3621        session: context.session.clone(),
3622    }
3623}
3624
3625/// OCR result for a window capture. The dispatcher returns one block list per
3626/// recognized line. macOS reuses `VNRecognizeTextRequest`, Windows uses
3627/// `Windows.Media.Ocr.OcrEngine`, and Linux is a structured `Unsupported`.
3628fn ocr_action(
3629    backend: &dyn DesktopBackend,
3630    context: &CallContext,
3631    _force: bool,
3632) -> Result<Value, ComputerError> {
3633    let targeted = context.app.is_some() || context.window.is_some();
3634    if !targeted {
3635        return Err(ComputerError::InvalidArguments(
3636            "ocr requires --app or --window".into(),
3637        ));
3638    }
3639    let prepared = latest_prepared_ocr_capture(backend, context)?;
3640    let OcrResult { blocks, note } = backend.recognize_text(&prepared.image, &prepared.map)?;
3641    let redacted = redact_secure_ocr_blocks(blocks, backend, context, &prepared.map)?;
3642    let blocks_json: Vec<Value> = redacted
3643        .iter()
3644        .map(|block| {
3645            let image_rect = crate::vision::ocr::ImageRect {
3646                x: block.x,
3647                y: block.y,
3648                width: block.width,
3649                height: block.height,
3650            };
3651            let screen = image_rect.to_screen(&prepared.map);
3652            let screen_json = match screen {
3653                Some((x, y)) => json!({
3654                    "x": x.round() as i64,
3655                    "y": y.round() as i64,
3656                }),
3657                None => Value::Null,
3658            };
3659            json!({
3660                "text": block.text,
3661                "x": block.x,
3662                "y": block.y,
3663                "width": block.width,
3664                "height": block.height,
3665                "confidence": block.confidence,
3666                "screen": screen_json,
3667            })
3668        })
3669        .collect();
3670    let mut body = json!({
3671        "action": "ocr",
3672        "blocks": blocks_json,
3673    });
3674    if let Some(note) = note {
3675        body["note"] = json!(note);
3676    }
3677    Ok(body)
3678}
3679
3680/// Drop blocks that fall inside any secure element's frame. Re-uses the
3681/// vision redaction helper so the rule lives in one place. The dispatcher
3682/// is conservative: when the secure frame list is empty (the typical
3683/// Linux/Windows case or a tree-unavailable snapshot), every block is kept.
3684fn redact_secure_ocr_blocks(
3685    blocks: Vec<OcrBlock>,
3686    backend: &dyn DesktopBackend,
3687    context: &CallContext,
3688    map: &crate::image_util::ScreenshotMap,
3689) -> Result<Vec<OcrBlock>, ComputerError> {
3690    // We do not walk the UiNode tree directly; the backend's secure-frame
3691    // extraction lives in a `secure_ocr_frames()` method. It receives the
3692    // screenshot map because platform frames are screen points and the OCR
3693    // blocks are image pixels. The default is an empty list.
3694    let secure_frames = backend.secure_ocr_frames(&tree_query(context), map)?;
3695    Ok(crate::vision::redact_blocks(blocks, secure_frames))
3696}
3697
3698/// One OCR-ready capture: the prepared shot pixels plus its screenshot map.
3699struct OcrPreparedCapture {
3700    image: image::RgbaImage,
3701    map: crate::image_util::ScreenshotMap,
3702}
3703
3704/// Re-run the same capture the screenshot command uses so OCR has the same
3705/// letterboxed source. Failures here are returned to the caller, never
3706/// swallowed.
3707fn latest_prepared_ocr_capture(
3708    backend: &dyn DesktopBackend,
3709    context: &CallContext,
3710) -> Result<OcrPreparedCapture, ComputerError> {
3711    let targeted = context.app.is_some() || context.window.is_some();
3712    if targeted {
3713        if let Some(error) = backend.display_block() {
3714            return Err(error);
3715        }
3716        let target = resolve_screenshot_window(backend, context)?;
3717        let captured = backend.capture_window(&target)?;
3718        let prepared = image_util::prepare_shot(
3719            captured.frame.image(),
3720            image_util::ShotFit {
3721                scale: captured.frame.scale(),
3722                origin_x: captured.origin_x,
3723                origin_y: captured.origin_y,
3724                logical_width: captured.logical_width,
3725                logical_height: captured.logical_height,
3726                max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3727                canvas: None,
3728            },
3729        )?;
3730        Ok(OcrPreparedCapture {
3731            image: prepared.image,
3732            map: prepared.map,
3733        })
3734    } else {
3735        // Full screen capture for OCR (rare, but allowed by the protocol).
3736        let raw = backend.capture_screen(None)?;
3737        let prepared = image_util::prepare_shot(
3738            raw.frame.image(),
3739            image_util::ShotFit {
3740                scale: raw.frame.scale(),
3741                origin_x: raw.origin_x,
3742                origin_y: raw.origin_y,
3743                logical_width: raw.logical_width,
3744                logical_height: raw.logical_height,
3745                max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3746                canvas: None,
3747            },
3748        )?;
3749        Ok(OcrPreparedCapture {
3750            image: prepared.image,
3751            map: prepared.map,
3752        })
3753    }
3754}
3755
3756/// Draw tree index marks on `image`. Returns the count of marks applied.
3757///
3758/// The overlay is painted onto a copy, so the returned image has to be written
3759/// back into `*image`. Dropping it leaves the caller's pixels untouched and
3760/// `screenshot --marks` writes a clean capture while still answering `ok`.
3761fn apply_marks(
3762    image: &mut image::RgbaImage,
3763    map: &crate::image_util::ScreenshotMap,
3764    context: &CallContext,
3765    backend: &dyn DesktopBackend,
3766) -> Result<usize, ComputerError> {
3767    let boxes = backend.marks_for_tree(&tree_query(context), map)?;
3768    if boxes.is_empty() {
3769        return Ok(0);
3770    }
3771    let marked = crate::vision::draw_marks(image, &boxes);
3772    *image = marked;
3773    Ok(boxes.len())
3774}
3775
3776fn tree_action(
3777    backend: &dyn DesktopBackend,
3778    context: &CallContext,
3779) -> Result<Value, ComputerError> {
3780    let report = backend.tree(&tree_query(context))?;
3781    let mut body = json!({
3782        "pid": report.pid,
3783        "window_id": report.window_id,
3784        "revision": report.revision,
3785        "source": report.source,
3786        "truncated": report.truncated,
3787        "tree": report.tree,
3788        // `diff` is `None` when this observation created the baseline.
3789        "baseline": if report.diff.is_some() { "existing" } else { "created" },
3790    });
3791    if let Some(note) = report.note {
3792        body["note"] = json!(note);
3793    }
3794    if let Some(legacy) = report.legacy.as_deref() {
3795        body["accessibility_legacy"] = serde_json::from_str(legacy).unwrap_or(Value::Null);
3796    }
3797    let tree_was_empty = report.full_text.trim().is_empty();
3798    if tree_was_empty {
3799        // P11: surface the OCR fallback when the tree produced nothing, and
3800        // a window target was selected. The agent decides whether to run it.
3801        if context.app.is_some() || context.window.is_some() {
3802            body["ocr_fallback_available"] = json!(true);
3803        }
3804    }
3805    Ok(body)
3806}
3807
3808fn session_end(
3809    backend: &dyn DesktopBackend,
3810    context: &CallContext,
3811) -> Result<Value, ComputerError> {
3812    let restored = backend.end_session(context.session.as_deref())?;
3813    let mut body = json!({
3814        "action": "session_end",
3815        "restored": restored,
3816    });
3817    if let Err(error) = cache::prune_empty_sessions() {
3818        body["warnings"] = json!([format!("session prune failed: {error}")]);
3819    }
3820    Ok(body)
3821}
3822
3823fn list_apps(backend: &dyn DesktopBackend, all: bool) -> Result<Value, ComputerError> {
3824    let apps = backend.list_apps(all)?;
3825    Ok(json!({"apps": apps}))
3826}
3827
3828fn list_windows(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3829    let listed = backend.list_windows()?;
3830    let windows = match app {
3831        None => listed.windows,
3832        Some(app) => {
3833            let selector = parse_app_selector(app)?;
3834            let apps = backend.list_apps(true)?;
3835            let info = resolve_app(&apps, &selector)?;
3836            filter_windows_for_pid(&listed.windows, info.pid)
3837        }
3838    };
3839    let mut body = json!({"windows": windows});
3840    if let Some(note) = listed.note {
3841        body["note"] = json!(note);
3842    }
3843    Ok(body)
3844}
3845
3846fn launch(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3847    let app = require_app(app)?;
3848    let result = backend.launch(app)?;
3849    Ok(json!({
3850        "action": "launch",
3851        "name": result.name,
3852        "bundle_id": result.bundle_id,
3853        "pid": result.pid,
3854        "path": result.path,
3855    }))
3856}
3857
3858fn activate(
3859    backend: &dyn DesktopBackend,
3860    app: Option<&str>,
3861    window: Option<&str>,
3862) -> Result<Value, ComputerError> {
3863    let app = require_app(app)?;
3864    let result = backend.activate(app, window)?;
3865    Ok(json!({
3866        "action": "activate",
3867        "name": result.name,
3868        "bundle_id": result.bundle_id,
3869        "pid": result.pid,
3870        "path": result.path,
3871        "window_id": result.window_id,
3872        "window_title": result.window_title,
3873    }))
3874}
3875
3876fn quit(
3877    backend: &dyn DesktopBackend,
3878    app: Option<&str>,
3879    force: bool,
3880) -> Result<Value, ComputerError> {
3881    let app = require_app(app)?;
3882    let result = backend.quit(app, force)?;
3883    Ok(json!({
3884        "action": "quit",
3885        "name": result.name,
3886        "bundle_id": result.bundle_id,
3887        "pid": result.pid,
3888        "accepted": result.accepted,
3889        "force": force,
3890    }))
3891}
3892
3893fn doctor(
3894    backend: &dyn DesktopBackend,
3895    options: crate::doctor::DoctorOptions,
3896) -> Result<Value, ComputerError> {
3897    let report = backend.doctor(options)?;
3898    serde_json::to_value(report).map_err(|error| {
3899        ComputerError::ExecutionFailed(format!("failed to encode doctor report: {error}"))
3900    })
3901}
3902
3903fn require_app(app: Option<&str>) -> Result<&str, ComputerError> {
3904    match app.map(str::trim).filter(|text| !text.is_empty()) {
3905        Some(app) => Ok(app),
3906        None => Err(ComputerError::InvalidArguments("app is required".into())),
3907    }
3908}
3909
3910/// JSON result printed by the CLI and returned by [`dispatch`].
3911#[derive(Debug, Clone, PartialEq)]
3912pub struct Response {
3913    /// Whether the action completed.
3914    pub ok: bool,
3915    /// Request id, when the caller sent one.
3916    pub id: Option<Value>,
3917    /// Present when `ok` is false.
3918    pub error: Option<ErrorBody>,
3919    /// Extra fields flattened into the JSON object.
3920    pub fields: Map<String, Value>,
3921}
3922
3923impl Response {
3924    /// Successful response. `data` must be a JSON object; other values are stored as `result`.
3925    pub fn success(data: Value) -> Self {
3926        let fields = match data {
3927            Value::Object(map) => map,
3928            other => {
3929                let mut map = Map::new();
3930                map.insert("result".into(), other);
3931                map
3932            }
3933        };
3934        Self {
3935            ok: true,
3936            id: None,
3937            error: None,
3938            fields,
3939        }
3940    }
3941
3942    /// Failed response with a stable `code` and `message`.
3943    pub fn failure(code: impl Into<String>, message: impl Into<String>) -> Self {
3944        Self {
3945            ok: false,
3946            id: None,
3947            error: Some(ErrorBody {
3948                code: code.into(),
3949                message: message.into(),
3950            }),
3951            fields: Map::new(),
3952        }
3953    }
3954
3955    /// Failed response from a [`ComputerError`].
3956    pub fn from_error(error: ComputerError) -> Self {
3957        Self::failure(error.code(), error.to_string())
3958    }
3959
3960    /// Attach a request id.
3961    pub fn with_id(mut self, id: Option<Value>) -> Self {
3962        self.id = id;
3963        self
3964    }
3965
3966    /// JSON object for this response.
3967    pub fn to_value(&self) -> Value {
3968        let mut map = Map::new();
3969        map.insert("ok".into(), Value::Bool(self.ok));
3970        if let Some(id) = &self.id {
3971            map.insert("id".into(), id.clone());
3972        }
3973        if let Some(error) = &self.error {
3974            map.insert(
3975                "error".into(),
3976                serde_json::to_value(error).unwrap_or_else(
3977                    |_| json!({"code": "execution_failed", "message": "failed to encode error"}),
3978                ),
3979            );
3980        }
3981        for (key, value) in &self.fields {
3982            if key != "ok" && key != "id" && key != "error" {
3983                map.insert(key.clone(), value.clone());
3984            }
3985        }
3986        Value::Object(map)
3987    }
3988
3989    /// Serialize as compact or pretty JSON, followed by a newline.
3990    pub fn to_json_line(&self, pretty: bool) -> String {
3991        let value = self.to_value();
3992        let body = if pretty {
3993            serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3994        } else {
3995            serde_json::to_string(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3996        };
3997        format!("{body}\n")
3998    }
3999}
4000
4001impl From<ComputerError> for Response {
4002    fn from(error: ComputerError) -> Self {
4003        Self::from_error(error)
4004    }
4005}
4006
4007impl Serialize for Response {
4008    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
4009        self.to_value().serialize(serializer)
4010    }
4011}
4012
4013impl<'de> Deserialize<'de> for Response {
4014    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
4015        let value = Value::deserialize(deserializer)?;
4016        let Value::Object(mut object) = value else {
4017            return Err(de::Error::custom("response must be a JSON object"));
4018        };
4019        let ok = object
4020            .remove("ok")
4021            .ok_or_else(|| de::Error::missing_field("ok"))?;
4022        let ok = ok
4023            .as_bool()
4024            .ok_or_else(|| de::Error::custom("ok must be a boolean"))?;
4025        let id = object.remove("id");
4026        let error = match object.remove("error") {
4027            Some(value) => Some(serde_json::from_value(value).map_err(de::Error::custom)?),
4028            None => None,
4029        };
4030        Ok(Self {
4031            ok,
4032            id,
4033            error,
4034            fields: object,
4035        })
4036    }
4037}
4038
4039#[cfg(test)]
4040mod tests {
4041    use super::*;
4042    use crate::test_backend::StubBackend;
4043    use std::sync::atomic::Ordering;
4044
4045    #[test]
4046    fn action_roundtrip_and_aliases() {
4047        let click: Action = serde_json::from_str(
4048            r#"{"action":"click","x":10,"y":20,"button":"right","double":true}"#,
4049        )
4050        .unwrap();
4051        match &click {
4052            Action::Click {
4053                x,
4054                y,
4055                button,
4056                double,
4057                count,
4058            } => {
4059                assert_eq!(
4060                    (*x, *y, *button, *double, *count),
4061                    (Some(10), Some(20), MouseButton::Right, true, 1)
4062                );
4063            }
4064            other => panic!("unexpected {other:?}"),
4065        }
4066        let encoded = serde_json::to_string(&click).unwrap();
4067        let decoded: Action = serde_json::from_str(&encoded).unwrap();
4068        assert_eq!(decoded, click);
4069
4070        let moved: Action = serde_json::from_str(r#"{"action":"move_mouse","x":1,"y":2}"#).unwrap();
4071        assert!(matches!(moved, Action::Move { x: 1, y: 2 }));
4072
4073        let state: Action = serde_json::from_str(r#"{"action":"get-state"}"#).unwrap();
4074        assert!(matches!(state, Action::GetState));
4075        let state: Action = serde_json::from_str(r#"{"action":"get_state"}"#).unwrap();
4076        assert!(matches!(state, Action::GetState));
4077
4078        let shot: Action = serde_json::from_str(
4079            r#"{"action":"screenshot","region":"0,0,8,9","format":"jpeg","max_width":640,"base64":true}"#,
4080        )
4081        .unwrap();
4082        match shot {
4083            Action::Screenshot {
4084                region,
4085                format,
4086                max_width,
4087                base64,
4088                ..
4089            } => {
4090                assert_eq!(region.unwrap().width, 8);
4091                assert_eq!(format, Some(ImageFormat::Jpeg));
4092                assert_eq!(max_width, Some(640));
4093                assert!(base64);
4094            }
4095            other => panic!("unexpected {other:?}"),
4096        }
4097        let shot: Action = serde_json::from_str(
4098            r#"{"action":"screenshot","region":{"x":0,"y":1,"width":2,"height":3}}"#,
4099        )
4100        .unwrap();
4101        match shot {
4102            Action::Screenshot { region, .. } => {
4103                assert_eq!(region.unwrap().height, 3);
4104            }
4105            other => panic!("unexpected {other:?}"),
4106        }
4107
4108        assert!(serde_json::from_str::<Action>(r#"{"action":"fly"}"#).is_err());
4109        let bare: Action = serde_json::from_str(r#"{"action":"click"}"#).unwrap();
4110        assert!(matches!(
4111            bare,
4112            Action::Click {
4113                x: None,
4114                y: None,
4115                ..
4116            }
4117        ));
4118        let press: Action = serde_json::from_str(r#"{"action":"press"}"#).unwrap();
4119        assert!(matches!(
4120            press,
4121            Action::Press {
4122                count: 1,
4123                button: MouseButton::Left
4124            }
4125        ));
4126        let renamed: Action =
4127            serde_json::from_str(r#"{"action":"set-value","value":"hi"}"#).unwrap();
4128        assert!(matches!(renamed, Action::SetValue { .. }));
4129        let scrolled: Action = serde_json::from_str(r#"{"action":"scroll-to-visible"}"#).unwrap();
4130        assert!(matches!(scrolled, Action::ScrollTo));
4131    }
4132
4133    #[test]
4134    fn response_shape_and_roundtrip() {
4135        let ok = Response::success(json!({"waited_ms": 1})).with_id(Some(json!(7)));
4136        let value = ok.to_value();
4137        assert_eq!(value["ok"], true);
4138        assert!(value.get("error").is_none());
4139        assert_eq!(value["waited_ms"], 1);
4140        assert_eq!(value["id"], 7);
4141        let text = serde_json::to_string(&ok).unwrap();
4142        let back: Response = serde_json::from_str(&text).unwrap();
4143        assert_eq!(back.to_value(), value);
4144
4145        let err = Response::failure("invalid_arguments", "bad region");
4146        let value = err.to_value();
4147        assert_eq!(value["ok"], false);
4148        assert_eq!(value["error"]["code"], "invalid_arguments");
4149        assert_eq!(value["error"]["message"], "bad region");
4150        assert!(value.get("waited_ms").is_none());
4151        let line = err.to_json_line(false);
4152        assert!(line.ends_with('\n'));
4153        assert!(!line[..line.len() - 1].contains('\n'));
4154    }
4155
4156    #[test]
4157    fn scroll_and_wait_resolution() {
4158        assert_eq!(
4159            resolve_scroll(Some(1), Some(-2), None, None).unwrap(),
4160            (1, -2)
4161        );
4162        assert_eq!(
4163            resolve_scroll(None, None, Some(ScrollDirection::Up), Some(3)).unwrap(),
4164            (0, -3)
4165        );
4166        assert_eq!(
4167            resolve_scroll(None, None, Some(ScrollDirection::Right), None).unwrap(),
4168            (1, 0)
4169        );
4170        assert_eq!(
4171            resolve_scroll(None, None, Some(ScrollDirection::Left), Some(4)).unwrap(),
4172            (-4, 0)
4173        );
4174        assert_eq!(
4175            resolve_scroll(None, None, Some(ScrollDirection::Down), Some(2)).unwrap(),
4176            (0, 2)
4177        );
4178        assert!(resolve_scroll(Some(1), None, Some(ScrollDirection::Up), None).is_err());
4179        assert!(resolve_scroll(None, None, None, Some(2)).is_err());
4180        assert!(resolve_scroll(None, None, None, None).is_err());
4181
4182        assert_eq!(resolve_wait(Some(10), None).unwrap(), 10);
4183        assert_eq!(resolve_wait(Some(10), Some(0.5)).unwrap(), 510);
4184        assert_eq!(resolve_wait(None, Some(1.0)).unwrap(), 1000);
4185        assert!(resolve_wait(None, None).is_err());
4186        assert!(resolve_wait(None, Some(-1.0)).is_err());
4187        assert!(resolve_wait(None, Some(f64::NAN)).is_err());
4188    }
4189
4190    #[test]
4191    fn json_line_dispatch_and_stub_actions() {
4192        let wait = handle_request_line(r#"{"id":1,"action":"wait","ms":0}"#);
4193        assert!(wait.ok);
4194        assert_eq!(wait.to_value()["id"], 1);
4195        assert_eq!(wait.to_value()["waited_ms"], 0);
4196
4197        let bad = handle_request_line("not-json");
4198        assert!(!bad.ok);
4199        assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
4200
4201        let missing = handle_request_line(r#"{"id":"x","action":"nope"}"#);
4202        assert!(!missing.ok);
4203        assert_eq!(missing.to_value()["id"], "x");
4204
4205        let request = parse_request(json!({"id": 3, "action": "displays"})).unwrap();
4206        assert_eq!(request.id, Some(json!(3)));
4207        assert!(matches!(request.action, Action::Displays));
4208
4209        let backend = StubBackend::default();
4210        let response = dispatch_with(
4211            &backend,
4212            Action::Click {
4213                x: Some(3),
4214                y: Some(4),
4215                button: MouseButton::Left,
4216                double: true,
4217                count: 1,
4218            },
4219        );
4220        assert!(response.ok);
4221        assert_eq!(response.to_value()["action"], "double_click");
4222        assert_eq!(backend.clicks.load(Ordering::SeqCst), 2);
4223
4224        let response = dispatch_with(&backend, Action::Move { x: 8, y: 9 });
4225        assert!(response.ok);
4226        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4227
4228        let response = dispatch_with(&backend, Action::Type { text: "hi".into() });
4229        assert_eq!(response.to_value()["chars"], 2);
4230        assert_eq!(backend.typed.lock().unwrap().as_slice(), ["hi"]);
4231
4232        let response = dispatch_with(
4233            &backend,
4234            Action::Key {
4235                keys: "cmd+shift+4".into(),
4236            },
4237        );
4238        assert!(response.ok);
4239        assert_eq!(response.to_value()["key"], "4");
4240        let keys = backend.keys.lock().unwrap();
4241        assert_eq!(keys[0].1, "4");
4242        assert_eq!(keys[0].0, vec![Modifier::Command, Modifier::Shift]);
4243        drop(keys);
4244
4245        let response = dispatch_with(
4246            &backend,
4247            Action::Key {
4248                keys: "not+a+key".into(),
4249            },
4250        );
4251        assert!(!response.ok);
4252
4253        let response = dispatch_with(
4254            &backend,
4255            Action::Scroll {
4256                x: Some(1),
4257                y: Some(2),
4258                dx: None,
4259                dy: None,
4260                direction: Some(ScrollDirection::Down),
4261                amount: Some(3),
4262            },
4263        );
4264        assert!(response.ok);
4265        assert_eq!(backend.scrolls.lock().unwrap().as_slice(), [(0, 3)]);
4266        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4267        assert_eq!(response.to_value()["tier"], "event");
4268        assert_eq!(response.to_value()["cursor_moved"], false);
4269
4270        let response = dispatch_with(&backend, Action::Displays);
4271        assert_eq!(response.to_value()["displays"][0]["name"], "Stub");
4272
4273        let response = dispatch_with(&backend, Action::GetState);
4274        assert_eq!(response.to_value()["cursor"]["x"], 4);
4275        assert!(response.ok);
4276
4277        let _cache = crate::cache::CacheEnv::acquire();
4278        let shot = dispatch_with(
4279            &backend,
4280            Action::Screenshot {
4281                output: None,
4282                region: None,
4283                format: Some(ImageFormat::Png),
4284                max_width: Some(1),
4285                canvas: None,
4286                base64: true,
4287                marks: false,
4288            },
4289        );
4290        assert!(shot.ok, "{shot:?}");
4291        let value = shot.to_value();
4292        assert_eq!(value["width"], 1);
4293        assert_eq!(value["height"], 1);
4294        assert_eq!(value["scale"], 2.0);
4295        assert_eq!(value["format"], "png");
4296        let path = value["path"].as_str().unwrap();
4297        let bytes = fs::read(path).unwrap();
4298        assert_eq!(
4299            &bytes[..8],
4300            &[0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n']
4301        );
4302        assert!(value["data"].as_str().unwrap().starts_with("iVBOR"));
4303        assert_eq!(value["capture_source"], "stub");
4304        assert_eq!(value["origin"]["x"], 0);
4305        assert_eq!(value["origin"]["y"], 0);
4306        assert_eq!(value["fit"], "width");
4307        assert!(value["map"]["image_to_screen"].is_string());
4308        let _ = fs::remove_file(path);
4309        if let Ok(dir) = crate::cache::session_dir(None) {
4310            let _ = fs::remove_file(dir.join("screenshot-map.json"));
4311        }
4312
4313        let failed = StubBackend {
4314            fail_click: true,
4315            ..StubBackend::default()
4316        };
4317        let response = dispatch_with(
4318            &failed,
4319            Action::Click {
4320                x: Some(0),
4321                y: Some(0),
4322                button: MouseButton::Left,
4323                double: false,
4324                count: 1,
4325            },
4326        );
4327        assert!(!response.ok);
4328        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4329    }
4330
4331    #[test]
4332    fn dispatch_wait_does_not_need_a_display() {
4333        let response = dispatch(Action::Wait {
4334            ms: Some(0),
4335            seconds: None,
4336        });
4337        assert!(response.ok);
4338        assert_eq!(response.to_value()["waited_ms"], 0);
4339    }
4340
4341    fn shot(max_width: Option<u32>, canvas: Option<&str>) -> Action {
4342        Action::Screenshot {
4343            output: None,
4344            region: None,
4345            format: Some(ImageFormat::Png),
4346            max_width,
4347            canvas: canvas.map(str::to_string),
4348            base64: false,
4349            marks: false,
4350        }
4351    }
4352
4353    #[test]
4354    fn dispatch_applies_default_max_width_and_keeps_explicit_zero() {
4355        let _cache = crate::cache::CacheEnv::acquire();
4356        let backend = StubBackend {
4357            capture_pixels: Some((1600, 1000)),
4358            ..StubBackend::default()
4359        };
4360        let response = dispatch_with_context(
4361            &backend,
4362            shot(None, None),
4363            &CallContext {
4364                session: Some("p4-default-width".into()),
4365                ..CallContext::default()
4366            },
4367        );
4368        assert!(response.ok, "{response:?}");
4369        let value = response.to_value();
4370        assert_eq!(value["width"], 1440);
4371        assert_eq!(value["height"], 900);
4372        assert_eq!(value["scale"], 2.0);
4373        assert_eq!(value["logical_width"], 800);
4374        assert_eq!(value["logical_height"], 500);
4375        assert_eq!(value["fit"], "width");
4376        assert_eq!(value["capture_source"], "stub");
4377        assert_eq!(value["content_width"], 1440);
4378        let path = value["path"].as_str().unwrap();
4379        let _ = fs::remove_file(path);
4380        let map = crate::cache::load_screenshot_map(Some("p4-default-width"))
4381            .unwrap()
4382            .expect("map");
4383        assert_eq!(map.width, 1440);
4384        let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-default-width")).unwrap());
4385
4386        let full = dispatch_with_context(
4387            &backend,
4388            shot(Some(0), None),
4389            &CallContext {
4390                session: Some("p4-zero-width".into()),
4391                ..CallContext::default()
4392            },
4393        );
4394        assert!(full.ok, "{full:?}");
4395        assert_eq!(full.to_value()["width"], 1600);
4396        assert_eq!(full.to_value()["height"], 1000);
4397        assert_eq!(full.to_value()["fit"], "none");
4398        let path = full.to_value()["path"].as_str().unwrap().to_string();
4399        let _ = fs::remove_file(path);
4400        let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-zero-width")).unwrap());
4401    }
4402
4403    #[test]
4404    fn dispatch_window_capture_records_origin_and_faults() {
4405        let _cache = crate::cache::CacheEnv::acquire();
4406        let backend = StubBackend::default();
4407        let response = dispatch_with_context(
4408            &backend,
4409            shot(Some(0), None),
4410            &CallContext {
4411                app: Some("Notes".into()),
4412                window: Some("0".into()),
4413                session: Some("p4-window".into()),
4414                ..CallContext::default()
4415            },
4416        );
4417        assert!(response.ok, "{response:?}");
4418        let value = response.to_value();
4419        assert_eq!(value["capture_source"], "stub");
4420        assert_eq!(value["origin"]["x"], 10);
4421        assert_eq!(value["origin"]["y"], 20);
4422        assert_eq!(value["logical_width"], 400);
4423        assert_eq!(value["logical_height"], 300);
4424        assert_eq!(value["width"], 800);
4425        assert_eq!(value["fit"], "none");
4426        let path = value["path"].as_str().unwrap();
4427        let _ = fs::remove_file(path);
4428        let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-window")).unwrap());
4429
4430        let denied = StubBackend {
4431            capture_fault: crate::test_backend::StubCaptureFault::Permission,
4432            ..StubBackend::default()
4433        };
4434        let response = dispatch_with_context(
4435            &denied,
4436            shot(Some(0), None),
4437            &CallContext {
4438                app: Some("Notes".into()),
4439                ..CallContext::default()
4440            },
4441        );
4442        assert!(!response.ok);
4443        assert_eq!(response.to_value()["error"]["code"], "permission_required");
4444        assert!(response.to_value()["error"]["message"]
4445            .as_str()
4446            .unwrap()
4447            .contains("Screen Recording"));
4448
4449        let missing = StubBackend {
4450            capture_fault: crate::test_backend::StubCaptureFault::MissingWindow,
4451            ..StubBackend::default()
4452        };
4453        let response = dispatch_with_context(
4454            &missing,
4455            shot(Some(0), None),
4456            &CallContext {
4457                window: Some("id:100".into()),
4458                ..CallContext::default()
4459            },
4460        );
4461        assert!(!response.ok);
4462        assert_eq!(response.to_value()["error"]["code"], "window_not_found");
4463
4464        let both = dispatch_with_context(
4465            &backend,
4466            Action::Screenshot {
4467                output: None,
4468                region: Some(Region::new(0, 0, 8, 9).unwrap()),
4469                format: None,
4470                max_width: Some(0),
4471                canvas: None,
4472                base64: false,
4473                marks: false,
4474            },
4475            &CallContext {
4476                app: Some("Notes".into()),
4477                ..CallContext::default()
4478            },
4479        );
4480        assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
4481
4482        let locked = StubBackend {
4483            block_display: true,
4484            windows: Vec::new(),
4485            ..StubBackend::default()
4486        };
4487        let response = dispatch_with_context(
4488            &locked,
4489            shot(Some(0), None),
4490            &CallContext {
4491                app: Some("Notes".into()),
4492                ..CallContext::default()
4493            },
4494        );
4495        assert_eq!(response.to_value()["error"]["code"], "display_unavailable");
4496        assert!(response.to_value()["error"]["message"]
4497            .as_str()
4498            .unwrap()
4499            .contains("locked"));
4500    }
4501
4502    #[test]
4503    fn shared_flags_are_stripped_before_the_action_decodes() {
4504        let request = parse_request(json!({
4505            "id": 4,
4506            "action": "list_windows",
4507            "app": "Foo \"Bar\"",
4508            "window": 0,
4509            "element": "3",
4510            "diff": true,
4511            "full": false,
4512            "return_state": true,
4513            "require_visible": true
4514        }))
4515        .unwrap();
4516        assert_eq!(request.id, Some(json!(4)));
4517        assert!(matches!(request.action, Action::ListWindows));
4518        assert_eq!(request.context.app.as_deref(), Some("Foo \"Bar\""));
4519        assert_eq!(request.context.window.as_deref(), Some("0"));
4520        assert_eq!(request.context.element, Some(3));
4521        assert!(request.context.diff);
4522        assert!(!request.context.full);
4523        assert!(request.context.return_state);
4524        assert!(request.context.require_visible);
4525    }
4526
4527    fn ctx(app: &str) -> CallContext {
4528        CallContext {
4529            app: Some(app.to_string()),
4530            ..CallContext::default()
4531        }
4532    }
4533
4534    #[test]
4535    fn stub_inventory_launch_and_doctor() {
4536        let backend = StubBackend::default();
4537
4538        let response = dispatch_with(&backend, Action::ListApps { all: false });
4539        assert!(response.ok);
4540        let value = response.to_value();
4541        let apps = value["apps"].as_array().unwrap();
4542        assert!(apps.iter().any(|app| app["name"] == "Notes"));
4543        assert!(apps.iter().all(|app| app["name"] != "Widget"));
4544
4545        let response = dispatch_with(&backend, Action::ListApps { all: true });
4546        let value = response.to_value();
4547        assert!(value["apps"]
4548            .as_array()
4549            .unwrap()
4550            .iter()
4551            .any(|app| app["pid"] == 77));
4552
4553        let response = dispatch_with_context(&backend, Action::ListWindows, &ctx("Notes"));
4554        assert!(response.ok, "{response:?}");
4555        let value = response.to_value();
4556        let windows = &value["windows"];
4557        assert_eq!(windows.as_array().unwrap().len(), 2);
4558        assert_eq!(windows[0]["z"], 0);
4559        assert_eq!(windows[0]["id"], 100);
4560        assert_eq!(windows[1]["z"], 1);
4561        assert_eq!(windows[1]["title"], "Welcome");
4562        assert_eq!(windows[0]["ax_matched"], true);
4563        assert!(value.get("note").is_none());
4564
4565        let response = dispatch_with_context(&backend, Action::Launch, &ctx("Notes"));
4566        assert!(response.ok, "{response:?}");
4567        assert_eq!(response.to_value()["pid"], 10);
4568        assert_eq!(response.to_value()["action"], "launch");
4569        assert_eq!(backend.launched.lock().unwrap().as_slice(), ["Notes"]);
4570
4571        let response = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
4572        assert!(!response.ok);
4573        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4574        assert_eq!(backend.launched.lock().unwrap().len(), 1);
4575
4576        let response = dispatch_with_context(&backend, Action::Activate, &ctx("10"));
4577        assert!(response.ok, "{response:?}");
4578        assert!(response.to_value()["window_id"].is_null());
4579        assert_eq!(backend.activated.lock().unwrap().len(), 1);
4580
4581        let mut context = ctx("Notes");
4582        context.window = Some("Welcome".into());
4583        let response = dispatch_with_context(&backend, Action::Activate, &context);
4584        assert_eq!(response.to_value()["window_id"], 101);
4585        assert_eq!(response.to_value()["window_title"], "Welcome");
4586
4587        let response =
4588            dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("Terminal"));
4589        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4590        assert!(backend.quit.lock().unwrap().is_empty());
4591
4592        let response = dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("412"));
4593        assert_eq!(response.to_value()["error"]["code"], "confirm_required");
4594        assert!(backend.quit.lock().unwrap().is_empty());
4595
4596        let response = dispatch_with(&backend, Action::Launch);
4597        assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4598
4599        let response = dispatch_with(
4600            &backend,
4601            Action::Doctor {
4602                prompt: false,
4603                open_settings: false,
4604                x11: false,
4605            },
4606        );
4607        let value = response.to_value();
4608        assert_eq!(value["ok"], true);
4609        assert_eq!(value["permissions_ok"], false);
4610        assert_eq!(value["accessibility"], true);
4611        assert_eq!(value["screen_capture"], false);
4612        assert_eq!(value["screen_recording"], false);
4613        assert!(value["input_monitor"].is_null());
4614        assert_eq!(value["missing"][0], "screen_recording");
4615        assert_eq!(value["responsible_app"]["name"], "Terminal");
4616        assert!(value["fixes"][0]
4617            .as_str()
4618            .unwrap()
4619            .contains("Screen Recording"));
4620        assert_eq!(value["fix"][0]["target"], "Terminal");
4621
4622        let prompted: Action =
4623            serde_json::from_str(r#"{"action":"doctor","request":true}"#).unwrap();
4624        assert!(matches!(
4625            prompted,
4626            Action::Doctor {
4627                prompt: true,
4628                open_settings: false,
4629                ..
4630            }
4631        ));
4632        let plain: Action = serde_json::from_str(r#"{"action":"doctor"}"#).unwrap();
4633        assert!(matches!(
4634            plain,
4635            Action::Doctor {
4636                prompt: false,
4637                open_settings: false,
4638                ..
4639            }
4640        ));
4641
4642        let response = dispatch_with(&backend, Action::GetState);
4643        assert!(response.ok);
4644        let value = response.to_value();
4645        assert_eq!(value["pid"], 10);
4646        assert_eq!(value["source"], "stub");
4647        assert!(value["tree"].as_str().unwrap().contains("AXWindow"));
4648        assert!(value["accessibility_legacy"].is_null());
4649
4650        let response = dispatch_with_context(&backend, Action::GetState, &ctx("412"));
4651        assert_eq!(response.to_value()["pid"], 412);
4652        assert_eq!(response.to_value()["source"], "stub");
4653        assert!(response.to_value()["tree"]
4654            .as_str()
4655            .unwrap()
4656            .contains("AXWindow"));
4657        let seen = backend.accessibility_apps.lock().unwrap().clone();
4658        assert_eq!(seen.last().and_then(|name| name.as_deref()), Some("412"));
4659
4660        // Get-state already stored a baseline. Clear it so this sequence
4661        // still covers "no baseline prints the full tree".
4662        backend.baselines.lock().unwrap().clear();
4663        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4664        assert!(first.ok, "{first:?}");
4665        let full = first.to_value()["tree"].as_str().unwrap().to_string();
4666        assert!(full.contains("value=\"a\""), "{full}");
4667        assert_eq!(first.to_value()["source"], "stub");
4668        *backend.stub_value.lock().unwrap() = "ab".into();
4669        let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4670        let diff_text = diff.to_value()["tree"].as_str().unwrap().to_string();
4671        assert!(
4672            diff_text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4673            "{diff_text}"
4674        );
4675        let mut full_context = ctx("Notes");
4676        full_context.full = true;
4677        let replaced = dispatch_with_context(&backend, Action::Tree, &full_context);
4678        let replaced_text = replaced.to_value()["tree"].as_str().unwrap().to_string();
4679        assert!(replaced_text.contains("value=\"ab\""), "{replaced_text}");
4680        assert!(!replaced_text.contains("~ [1]"));
4681        let after = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4682        assert_eq!(after.to_value()["tree"].as_str().unwrap(), "");
4683
4684        let _cache = crate::cache::CacheEnv::acquire();
4685        let ended = dispatch_with(&backend, Action::SessionEnd);
4686        assert!(ended.ok, "{ended:?}");
4687        assert_eq!(ended.to_value()["action"], "session_end");
4688        assert!(ended.to_value()["restored"].as_array().unwrap().is_empty());
4689    }
4690
4691    #[test]
4692    fn stub_refuses_privacy_window_and_self_quit() {
4693        let mut backend = StubBackend::default();
4694        backend.apps.push(crate::target::AppInfo {
4695            name: "System Settings".into(),
4696            bundle_id: Some("com.apple.systempreferences".into()),
4697            pid: 50,
4698            path: Some(
4699                "/System/Applications/System Settings.app/Contents/MacOS/System Settings".into(),
4700            ),
4701            active: false,
4702            hidden: false,
4703            activation_policy: crate::target::ActivationPolicy::Regular,
4704        });
4705        backend.windows.push(crate::target::WindowInfo {
4706            id: 9,
4707            pid: 50,
4708            app: "System Settings".into(),
4709            title: "Privacy & Security".into(),
4710            z: 4,
4711            bounds: crate::target::WindowBounds {
4712                x: 0,
4713                y: 0,
4714                width: 10,
4715                height: 10,
4716            },
4717            layer: 0,
4718            on_screen: true,
4719            ax_matched: false,
4720        });
4721        let mut context = ctx("com.apple.systempreferences");
4722        context.window = Some("Privacy".into());
4723        let response = dispatch_with_context(&backend, Action::Activate, &context);
4724        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4725        assert!(backend.activated.lock().unwrap().is_empty());
4726
4727        let response = dispatch_with_context(&backend, Action::Activate, &ctx("System Settings"));
4728        assert!(response.ok, "{response:?}");
4729
4730        let self_pid = std::process::id() as i32;
4731        backend.apps.retain(|app| app.pid != self_pid);
4732        backend.apps.push(crate::target::AppInfo {
4733            name: "computer-use".into(),
4734            bundle_id: None,
4735            pid: self_pid,
4736            path: None,
4737            active: false,
4738            hidden: false,
4739            activation_policy: crate::target::ActivationPolicy::Regular,
4740        });
4741        let response = dispatch_with_context(
4742            &backend,
4743            Action::Quit { force: false },
4744            &ctx(&self_pid.to_string()),
4745        );
4746        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4747    }
4748
4749    fn notes_button() -> crate::tree::UiNode {
4750        let mut button = crate::tree::UiNode::bare("AXButton");
4751        button.title = Some("Save".into());
4752        button.actions = vec!["AXPress".into(), "AXShowMenu".into()];
4753        button.frame = Some(crate::tree::NodeFrame {
4754            x: 12.0,
4755            y: 10.0,
4756            width: 80.0,
4757            height: 24.0,
4758        });
4759        let mut root = crate::tree::UiNode::bare("AXWindow");
4760        root.title = Some("Notes".into());
4761        root.frame = Some(crate::tree::NodeFrame {
4762            x: 0.0,
4763            y: 0.0,
4764            width: 800.0,
4765            height: 600.0,
4766        });
4767        root.children = vec![button];
4768        root
4769    }
4770
4771    fn element_context(action_element: u32) -> CallContext {
4772        CallContext {
4773            app: Some("Notes".into()),
4774            element: Some(action_element),
4775            ..CallContext::default()
4776        }
4777    }
4778
4779    #[test]
4780    fn element_json_shape_revalidation_and_hit_test() {
4781        let backend = StubBackend::default();
4782        backend.seed_element(notes_button());
4783        let request = parse_request(json!({
4784            "action": "press",
4785            "app": "Notes",
4786            "element": 1,
4787            "count": 1
4788        }))
4789        .unwrap();
4790        let response = dispatch_with_context(&backend, request.action, &request.context);
4791        assert!(response.ok, "{response:?}");
4792        let value = response.to_value();
4793        assert_eq!(value["action"], "press");
4794        assert_eq!(value["element"], 1);
4795        assert_eq!(value["tier"], "ax");
4796        assert_eq!(value["count"], 1);
4797        assert!(value["revision"].is_number());
4798        assert!(value.get("warnings").is_none());
4799
4800        let request = parse_request(json!({
4801            "action": "perform-action",
4802            "app": "Notes",
4803            "element": 1,
4804            "name": "AXShowMenu"
4805        }))
4806        .unwrap();
4807        let response = dispatch_with_context(&backend, request.action, &request.context);
4808        assert!(response.ok, "{response:?}");
4809        assert_eq!(response.to_value()["action"], "perform_action");
4810        assert_eq!(response.to_value()["name"], "AXShowMenu");
4811        assert_eq!(response.to_value()["tier"], "ax");
4812
4813        let before = backend.performs.lock().unwrap().len();
4814        let request = parse_request(json!({
4815            "action": "perform_action",
4816            "app": "Notes",
4817            "element": 1,
4818            "name": "AXDelete"
4819        }))
4820        .unwrap();
4821        let response = dispatch_with_context(&backend, request.action, &request.context);
4822        assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4823        assert_eq!(backend.performs.lock().unwrap().len(), before);
4824
4825        let request = parse_request(json!({
4826            "action": "set_value",
4827            "app": "Notes",
4828            "element": 1,
4829            "value": "hello"
4830        }))
4831        .unwrap();
4832        let response = dispatch_with_context(&backend, request.action, &request.context);
4833        assert!(response.ok, "{response:?}");
4834        assert_eq!(response.to_value()["action"], "set_value");
4835
4836        let request = parse_request(json!({
4837            "action": "select_text",
4838            "app": "Notes",
4839            "element": 1,
4840            "start": 0,
4841            "end": 5
4842        }))
4843        .unwrap();
4844        let response = dispatch_with_context(&backend, request.action, &request.context);
4845        assert!(response.ok, "{response:?}");
4846        assert_eq!(response.to_value()["action"], "select_text");
4847
4848        let request = parse_request(json!({
4849            "action": "scroll_to_visible",
4850            "app": "Notes",
4851            "element": 1
4852        }))
4853        .unwrap();
4854        assert!(matches!(request.action, Action::ScrollTo));
4855        let response = dispatch_with_context(&backend, request.action, &request.context);
4856        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4857        assert!(response.to_value()["error"]["message"]
4858            .as_str()
4859            .unwrap()
4860            .contains("AXScrollToVisible"));
4861
4862        let clicks = backend.clicks.load(Ordering::SeqCst);
4863        let request = parse_request(json!({
4864            "action": "click",
4865            "app": "Notes",
4866            "element": 1,
4867            "double": true
4868        }))
4869        .unwrap();
4870        let response = dispatch_with_context(&backend, request.action, &request.context);
4871        assert!(response.ok, "{response:?}");
4872        assert_eq!(response.to_value()["action"], "press");
4873        assert_eq!(response.to_value()["count"], 2);
4874        assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks);
4875
4876        *backend.hit.lock().unwrap() = Some(crate::element::HitInfo {
4877            role: "AXButton".into(),
4878            title: Some("Save".into()),
4879        });
4880        let response = dispatch_with(
4881            &backend,
4882            Action::Click {
4883                x: Some(12),
4884                y: Some(10),
4885                button: MouseButton::Left,
4886                double: false,
4887                count: 1,
4888            },
4889        );
4890        assert!(response.ok, "{response:?}");
4891        assert_eq!(response.to_value()["hit"]["role"], "AXButton");
4892        assert_eq!(response.to_value()["hit"]["title"], "Save");
4893        assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks + 1);
4894
4895        let mut changed = notes_button();
4896        changed.children[0].role = "AXTextArea".into();
4897        backend.set_live(changed);
4898        let response = dispatch_with_context(&backend, Action::Focus, &element_context(1));
4899        assert_eq!(response.to_value()["error"]["code"], "stale_element");
4900        assert!(response.to_value()["tree"]
4901            .as_str()
4902            .unwrap()
4903            .contains("AXTextArea"));
4904
4905        let empty = StubBackend::default();
4906        let response = dispatch_with_context(&empty, Action::Focus, &element_context(0));
4907        assert_eq!(response.to_value()["error"]["code"], "element_not_found");
4908        assert!(response.to_value()["error"]["message"]
4909            .as_str()
4910            .unwrap()
4911            .contains("refresh indices"));
4912
4913        let bare = dispatch_with(
4914            &empty,
4915            Action::Click {
4916                x: None,
4917                y: None,
4918                button: MouseButton::Left,
4919                double: false,
4920                count: 1,
4921            },
4922        );
4923        assert_eq!(bare.to_value()["error"]["code"], "invalid_arguments");
4924    }
4925
4926    #[test]
4927    fn element_action_does_not_replace_the_tree_diff_baseline() {
4928        let backend = StubBackend::default();
4929        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4930        assert!(first.ok, "{first:?}");
4931        let key = (String::new(), 10, 100);
4932        let baseline = backend.baselines.lock().unwrap().get(&key).cloned();
4933        let response = dispatch_with_context(&backend, Action::Focus, &element_context(0));
4934        assert!(response.ok, "{response:?}");
4935        assert_eq!(
4936            backend.baselines.lock().unwrap().get(&key),
4937            baseline.as_ref()
4938        );
4939        *backend.stub_value.lock().unwrap() = "ab".into();
4940        let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4941        let text = diff.to_value()["tree"].as_str().unwrap().to_string();
4942        assert!(
4943            text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4944            "{text}"
4945        );
4946    }
4947
4948    fn sample_map() -> crate::image_util::ScreenshotMap {
4949        crate::image_util::ScreenshotMap {
4950            width: 1440,
4951            height: 900,
4952            scale: 2.0,
4953            logical_width: 800,
4954            logical_height: 500,
4955            origin: crate::image_util::MapPoint { x: 120, y: 40 },
4956            fit: "width".into(),
4957            pad: crate::image_util::MapPoint { x: 0, y: 0 },
4958            content_width: 1440,
4959            content_height: 900,
4960            map: crate::image_util::MapFormulas {
4961                image_to_screen:
4962                    "screen = origin + (image - pad) * logical_size / image_content_size".into(),
4963                screen_to_image:
4964                    "image = pad + (screen - origin) * image_content_size / logical_size".into(),
4965            },
4966        }
4967    }
4968
4969    #[test]
4970    fn secure_ocr_frames_receives_the_map_of_the_ocr_capture() {
4971        // Platform frames are screen points; OCR blocks are image pixels. The
4972        // dispatcher must hand the backend the screenshot map of the capture
4973        // the OCR ran on, or redaction compares two different spaces.
4974        let cache = crate::cache::CacheEnv::acquire();
4975        assert!(cache.dir().is_dir());
4976        let backend = StubBackend::default();
4977        *backend.secure_frames.lock().unwrap() = Some(vec![crate::vision::ocr::ImageRect {
4978            x: 10,
4979            y: 20,
4980            width: 100,
4981            height: 40,
4982        }]);
4983        let context = CallContext {
4984            app: Some("Finder".into()),
4985            session: Some("p12b-redact".into()),
4986            ..CallContext::default()
4987        };
4988        let blocks = vec![OcrBlock {
4989            text: "hunter2".into(),
4990            x: 20,
4991            y: 30,
4992            width: 30,
4993            height: 10,
4994            confidence: 0.9,
4995        }];
4996        let map = sample_map();
4997        let kept = redact_secure_ocr_blocks(blocks.clone(), &backend, &context, &map).unwrap();
4998        assert!(kept.is_empty(), "the block sits inside the secure frame");
4999
5000        let seen = backend
5001            .secure_frames_map
5002            .lock()
5003            .unwrap()
5004            .clone()
5005            .expect("secure_ocr_frames was called");
5006        assert_eq!(seen.origin, map.origin);
5007        assert_eq!(seen.logical_width, map.logical_width);
5008        assert_eq!(seen.content_width, map.content_width);
5009
5010        let outside = vec![OcrBlock {
5011            text: "visible".into(),
5012            x: 900,
5013            y: 700,
5014            width: 30,
5015            height: 10,
5016            confidence: 0.9,
5017        }];
5018        let kept = redact_secure_ocr_blocks(outside, &backend, &context, &map).unwrap();
5019        assert_eq!(kept.len(), 1);
5020    }
5021
5022    /// Screenshot with `--marks` on a real window answered `ok` and wrote an
5023    /// unmarked PNG. `apply_marks` painted the overlay onto a copy of the image
5024    /// and threw the copy away, so the encoded bytes never changed.
5025    #[test]
5026    fn marks_change_the_pixels_the_screenshot_writes() {
5027        let cache = crate::cache::CacheEnv::acquire();
5028        let backend = StubBackend::default();
5029        let context = CallContext {
5030            app: Some("Notes".into()),
5031            session: Some("marks".into()),
5032            ..CallContext::default()
5033        };
5034        let plain = cache.dir().join("plain.png");
5035        let marked = cache.dir().join("marked.png");
5036
5037        let shot = |path: std::path::PathBuf, marks: bool| {
5038            screenshot(
5039                &backend,
5040                &context,
5041                ScreenshotArgs {
5042                    output: Some(path.to_string_lossy().into_owned()),
5043                    region: None,
5044                    format: Some(ImageFormat::Png),
5045                    max_width: None,
5046                    canvas: None,
5047                    include_base64: false,
5048                    window_crop: false,
5049                    marks,
5050                },
5051            )
5052        };
5053
5054        let clean = shot(plain.clone(), false).expect("plain screenshot");
5055        let marked_body = shot(marked.clone(), true).expect("marked screenshot");
5056
5057        let clean_bytes = std::fs::read(&plain).expect("plain png");
5058        let marked_bytes = std::fs::read(&marked).expect("marked png");
5059        assert_ne!(
5060            clean_bytes, marked_bytes,
5061            "--marks wrote the same image as a plain screenshot"
5062        );
5063
5064        // The map the backend converted frames with must be the map of this
5065        // capture, otherwise screen points are read as image pixels.
5066        let seen = backend
5067            .marks_map
5068            .lock()
5069            .unwrap()
5070            .clone()
5071            .expect("marks_for_tree was called");
5072        assert_eq!(
5073            i64::from(seen.origin.x),
5074            clean["origin"]["x"].as_i64().unwrap()
5075        );
5076        assert_eq!(
5077            i64::from(seen.origin.y),
5078            clean["origin"]["y"].as_i64().unwrap()
5079        );
5080        assert_eq!(
5081            seen.content_width,
5082            clean["content_width"].as_u64().unwrap() as u32
5083        );
5084        assert_eq!(
5085            seen.logical_width,
5086            clean["logical_width"].as_u64().unwrap() as u32
5087        );
5088
5089        // Marks were applied, so this is not the "nothing to draw" warning.
5090        let warnings = marked_body["warnings"].as_array();
5091        assert!(
5092            !warnings
5093                .map(|items| items
5094                    .iter()
5095                    .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))))
5096                .unwrap_or(false),
5097            "unexpected marks warning: {warnings:?}"
5098        );
5099    }
5100
5101    #[test]
5102    fn marks_warn_when_no_frame_is_inside_the_capture() {
5103        // A silent zero is indistinguishable from "the marks were drawn".
5104        let cache = crate::cache::CacheEnv::acquire();
5105        let backend = StubBackend::default();
5106        // A tree whose window sits far away from the captured screen: every
5107        // frame is off-canvas, so nothing can be drawn.
5108        backend.set_live({
5109            let mut window = crate::tree::UiNode::bare("AXWindow");
5110            window.frame = Some(crate::tree::NodeFrame {
5111                x: 9_000.0,
5112                y: 9_000.0,
5113                width: 100.0,
5114                height: 100.0,
5115            });
5116            window
5117        });
5118        let context = CallContext {
5119            session: Some("marks-empty".into()),
5120            ..CallContext::default()
5121        };
5122        let body = screenshot(
5123            &backend,
5124            &context,
5125            ScreenshotArgs {
5126                output: Some(
5127                    cache
5128                        .dir()
5129                        .join("marks-empty.png")
5130                        .to_string_lossy()
5131                        .into_owned(),
5132                ),
5133                region: None,
5134                format: Some(ImageFormat::Png),
5135                max_width: None,
5136                canvas: None,
5137                include_base64: false,
5138                window_crop: false,
5139                marks: true,
5140            },
5141        )
5142        .expect("screenshot");
5143        let warnings = body["warnings"].as_array().expect("warnings");
5144        assert!(
5145            warnings
5146                .iter()
5147                .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))),
5148            "expected a marks warning, got {warnings:?}"
5149        );
5150    }
5151
5152    #[test]
5153    fn no_secure_frames_keeps_every_ocr_block() {
5154        let cache = crate::cache::CacheEnv::acquire();
5155        assert!(cache.dir().is_dir());
5156        let backend = StubBackend::default();
5157        let context = CallContext {
5158            app: Some("Finder".into()),
5159            session: Some("p12b-redact-none".into()),
5160            ..CallContext::default()
5161        };
5162        let blocks = vec![OcrBlock {
5163            text: "hello".into(),
5164            x: 1,
5165            y: 2,
5166            width: 3,
5167            height: 4,
5168            confidence: 0.9,
5169        }];
5170        let kept = redact_secure_ocr_blocks(blocks, &backend, &context, &sample_map()).unwrap();
5171        assert_eq!(kept.len(), 1);
5172    }
5173
5174    #[test]
5175    fn image_coordinate_on_a_stored_map_becomes_the_screen_point() {
5176        let cache = crate::cache::CacheEnv::acquire();
5177        assert!(cache.dir().is_dir());
5178        let map = sample_map();
5179        cache::store_screenshot_map(Some("p5-map"), &map).unwrap();
5180        let backend = StubBackend::default();
5181        let context = CallContext {
5182            coord: Some("image".into()),
5183            session: Some("p5-map".into()),
5184            ..CallContext::default()
5185        };
5186        let response = dispatch_with_context(&backend, Action::Move { x: 100, y: 50 }, &context);
5187        assert!(response.ok, "{response:?}");
5188        let (screen_x, screen_y) = image_util::image_to_screen(&map, 100.0, 50.0);
5189        let value = response.to_value();
5190        assert_eq!(value["x"], json!(screen_x.round() as i64));
5191        assert_eq!(value["y"], json!(screen_y.round() as i64));
5192        assert_eq!(value["x"], 176);
5193        assert_eq!(value["y"], 68);
5194        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5195
5196        let outside = dispatch_with_context(&backend, Action::Move { x: 1440, y: 50 }, &context);
5197        assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
5198        assert!(outside.to_value()["error"]["message"]
5199            .as_str()
5200            .unwrap()
5201            .contains("outside"));
5202        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5203
5204        let missing = dispatch_with_context(
5205            &backend,
5206            Action::Move { x: 1, y: 2 },
5207            &CallContext {
5208                coord: Some("image".into()),
5209                session: Some("no-map".into()),
5210                ..CallContext::default()
5211            },
5212        );
5213        assert_eq!(missing.to_value()["error"]["code"], "invalid_arguments");
5214        assert!(missing.to_value()["error"]["message"]
5215            .as_str()
5216            .unwrap()
5217            .contains("take a screenshot first"));
5218
5219        let action =
5220            crate::adapters::translate("openai", &json!({"type": "move", "x": 100, "y": 50}))
5221                .unwrap();
5222        let adapted = dispatch_with_context(
5223            &backend,
5224            action,
5225            &CallContext {
5226                coord: Some("image".into()),
5227                session: Some("p5-map".into()),
5228                adapter: Some("openai".into()),
5229                ..CallContext::default()
5230            },
5231        );
5232        assert!(adapted.ok, "{adapted:?}");
5233        assert_eq!(adapted.to_value()["adapter"], "openai");
5234        assert_eq!(adapted.to_value()["x"], 176);
5235        assert_eq!(adapted.to_value()["y"], 68);
5236        assert!(adapted.to_value().get("state").is_none());
5237
5238        let missing_adapter = handle_request_line_with(
5239            r#"{"adapter":"anthropic","action":"mouse_move","coordinate":[1,2],"session":"no-map"}"#,
5240            None,
5241        );
5242        assert!(!missing_adapter.ok);
5243        assert_eq!(
5244            missing_adapter.to_value()["error"]["code"],
5245            "invalid_arguments"
5246        );
5247        assert!(missing_adapter.to_value()["error"]["message"]
5248            .as_str()
5249            .unwrap()
5250            .contains("take a screenshot first"));
5251        assert_eq!(missing_adapter.to_value()["adapter"], "anthropic");
5252
5253        let waited = handle_request_line_with(r#"{"adapter":"openai","type":"wait"}"#, None);
5254        assert!(waited.ok, "{waited:?}");
5255        assert_eq!(waited.to_value()["adapter"], "openai");
5256        assert_eq!(waited.to_value()["waited_ms"], 0);
5257    }
5258
5259    #[test]
5260    fn return_state_includes_the_numbered_tree_and_ignores_read_only() {
5261        let backend = StubBackend::default();
5262        let before = backend.accessibility_apps.lock().unwrap().len();
5263        let response = dispatch_with_context(
5264            &backend,
5265            Action::Click {
5266                x: Some(3),
5267                y: Some(4),
5268                button: MouseButton::Left,
5269                double: false,
5270                count: 1,
5271            },
5272            &CallContext {
5273                app: Some("Notes".into()),
5274                return_state: true,
5275                ..CallContext::default()
5276            },
5277        );
5278        assert!(response.ok, "{response:?}");
5279        let value = response.to_value();
5280        assert_eq!(value["action"], "click");
5281        let tree = value["state"]["tree"].as_str().unwrap();
5282        assert!(tree.contains("[0] AXWindow"), "{tree}");
5283        assert!(tree.contains("AXTextArea"), "{tree}");
5284        assert!(value["state"]["diff"].is_null());
5285        assert_eq!(value["state"]["app"], "com.apple.Notes");
5286        assert_eq!(value["state"]["pid"], 10);
5287        assert_eq!(value["state"]["window_id"], 100);
5288        assert!(value.get("warnings").is_none());
5289        assert!(backend.accessibility_apps.lock().unwrap().len() > before);
5290
5291        let displays = dispatch_with_context(
5292            &backend,
5293            Action::Displays,
5294            &CallContext {
5295                return_state: true,
5296                include_screenshot: true,
5297                ..CallContext::default()
5298            },
5299        );
5300        assert!(displays.ok, "{displays:?}");
5301        assert!(displays.to_value().get("state").is_none());
5302
5303        let cursor = dispatch_with(&backend, Action::CursorPosition);
5304        assert_eq!(cursor.to_value()["x"], 4);
5305        assert_eq!(cursor.to_value()["y"], 5);
5306        assert!(cursor.to_value().get("adapter").is_none());
5307    }
5308
5309    #[test]
5310    fn return_state_refreshes_the_tree_diff_baseline() {
5311        let backend = StubBackend::default();
5312        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5313        assert!(first.ok, "{first:?}");
5314        *backend.stub_value.lock().unwrap() = "ab".into();
5315        let mut context = element_context(0);
5316        context.return_state = true;
5317        let response = dispatch_with_context(&backend, Action::Focus, &context);
5318        assert!(response.ok, "{response:?}");
5319        let value = response.to_value();
5320        let diff = value["state"]["diff"].as_str().unwrap();
5321        assert!(
5322            diff.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
5323            "{diff}"
5324        );
5325        assert!(value["state"]["tree"]
5326            .as_str()
5327            .unwrap()
5328            .contains("value=\"ab\""));
5329        *backend.stub_value.lock().unwrap() = "abc".into();
5330        let next = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5331        let value = next.to_value();
5332        let text = value["tree"].as_str().unwrap();
5333        assert!(
5334            text.contains("~ [1] AXTextArea value: \"ab\" -> \"abc\""),
5335            "{text}"
5336        );
5337        assert!(!text.contains("\"a\" -> \"abc\""), "{text}");
5338    }
5339
5340    #[test]
5341    fn state_diff_omits_the_full_tree_and_keeps_the_delta() {
5342        let backend = StubBackend::default();
5343        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5344        assert!(first.ok, "{first:?}");
5345        *backend.stub_value.lock().unwrap() = "ab".into();
5346        let mut context = element_context(0);
5347        context.return_state = true;
5348        context.state_diff = true;
5349        let response = dispatch_with_context(&backend, Action::Focus, &context);
5350        assert!(response.ok, "{response:?}");
5351        let value = response.to_value();
5352        // The full tree is the expensive half of the payload. It is gone.
5353        assert!(value["state"].get("tree").is_none(), "{value}");
5354        let diff = value["state"]["diff"].as_str().unwrap();
5355        assert!(
5356            diff.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
5357            "{diff}"
5358        );
5359        assert_eq!(value["state"]["baseline"], "existing");
5360        assert_eq!(value["state"]["app"], "com.apple.Notes");
5361    }
5362
5363    #[test]
5364    fn state_diff_without_state_still_attaches_the_post_state() {
5365        let backend = StubBackend::default();
5366        let seeded = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5367        assert!(seeded.ok, "{seeded:?}");
5368        let mut context = element_context(0);
5369        context.state_diff = true;
5370        let response = dispatch_with_context(&backend, Action::Focus, &context);
5371        assert!(response.ok, "{response:?}");
5372        let value = response.to_value();
5373        assert!(value["state"].get("tree").is_none(), "{value}");
5374        assert!(value["state"].get("diff").is_some(), "{value}");
5375    }
5376
5377    #[test]
5378    fn the_tree_response_says_whether_a_baseline_existed() {
5379        let backend = StubBackend::default();
5380        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5381        assert!(first.ok, "{first:?}");
5382        assert_eq!(first.to_value()["baseline"], "created");
5383        *backend.stub_value.lock().unwrap() = "ab".into();
5384        let second = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5385        assert!(second.ok, "{second:?}");
5386        assert_eq!(second.to_value()["baseline"], "existing");
5387    }
5388
5389    #[test]
5390    fn policy_check_reports_a_code_next_to_the_decision() {
5391        let backend = StubBackend::default();
5392        let response = dispatch_with_context(
5393            &backend,
5394            Action::PolicyCheck,
5395            &CallContext {
5396                app: Some("Notes".into()),
5397                ..CallContext::default()
5398            },
5399        );
5400        assert!(response.ok, "{response:?}");
5401        let value = response.to_value();
5402        assert_eq!(value["decision"], "allow");
5403        assert_eq!(value["code"], "ok");
5404        assert!(value["source"].is_string(), "{value}");
5405    }
5406
5407    #[test]
5408    fn include_screenshot_failure_keeps_the_action_result() {
5409        let backend = StubBackend {
5410            capture_fault: crate::test_backend::StubCaptureFault::Permission,
5411            ..StubBackend::default()
5412        };
5413        let response = dispatch_with_context(
5414            &backend,
5415            Action::Click {
5416                x: Some(1),
5417                y: Some(2),
5418                button: MouseButton::Left,
5419                double: false,
5420                count: 1,
5421            },
5422            &CallContext {
5423                app: Some("Notes".into()),
5424                return_state: true,
5425                include_screenshot: true,
5426                ..CallContext::default()
5427            },
5428        );
5429        assert!(response.ok, "{response:?}");
5430        let value = response.to_value();
5431        assert_eq!(value["action"], "click");
5432        assert_eq!(value["x"], 1);
5433        assert_eq!(value["y"], 2);
5434        assert!(value["state"]["tree"]
5435            .as_str()
5436            .unwrap()
5437            .contains("AXWindow"));
5438        assert!(value["state"]["screenshot"].is_null());
5439        let warnings = value["warnings"].as_array().unwrap();
5440        assert!(warnings
5441            .iter()
5442            .any(|item| { item.as_str().unwrap().contains("screenshot failed") }));
5443        assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5444    }
5445
5446    fn batch_clicks(stop_on_error: bool) -> Value {
5447        json!({
5448            "action": "batch",
5449            "stop_on_error": stop_on_error,
5450            "actions": [
5451                {"action": "click", "x": 1, "y": 2},
5452                {"action": "click", "x": 3, "y": 4}
5453            ]
5454        })
5455    }
5456
5457    #[test]
5458    fn batch_stop_on_error_controls_the_second_attempt() {
5459        let stopped = StubBackend {
5460            fail_click: true,
5461            ..StubBackend::default()
5462        };
5463        let request = parse_request(batch_clicks(true)).unwrap();
5464        let response = dispatch_with_context(&stopped, request.action, &request.context);
5465        assert!(!response.ok);
5466        let value = response.to_value();
5467        assert_eq!(value["error"]["code"], "execution_failed");
5468        assert_eq!(value["stopped_at"], 0);
5469        assert_eq!(value["results"].as_array().unwrap().len(), 1);
5470        assert!(!value["results"][0]["ok"].as_bool().unwrap());
5471        assert_eq!(stopped.click_attempts.load(Ordering::SeqCst), 2);
5472        assert!(value["error"]["message"]
5473            .as_str()
5474            .unwrap()
5475            .contains("batch failed at action 0"));
5476
5477        let both = StubBackend {
5478            fail_click: true,
5479            ..StubBackend::default()
5480        };
5481        let request = parse_request(batch_clicks(false)).unwrap();
5482        let response = dispatch_with_context(&both, request.action, &request.context);
5483        assert!(!response.ok);
5484        let value = response.to_value();
5485        assert!(value.get("stopped_at").is_none(), "{value}");
5486        assert_eq!(value["results"].as_array().unwrap().len(), 2);
5487        assert!(!value["results"][1]["ok"].as_bool().unwrap());
5488        assert_eq!(both.click_attempts.load(Ordering::SeqCst), 4);
5489
5490        let nested = parse_request(json!({
5491            "action": "batch",
5492            "actions": [{"action": "batch", "actions": []}]
5493        }))
5494        .unwrap();
5495        let response = dispatch_with(&StubBackend::default(), nested.action);
5496        assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
5497        assert_eq!(response.to_value()["stopped_at"], 0);
5498        assert!(response.to_value()["error"]["message"]
5499            .as_str()
5500            .unwrap()
5501            .contains("nested batch"));
5502
5503        let backend = StubBackend::default();
5504        backend.seed_element(notes_button());
5505        let request = parse_request(json!({
5506            "action": "batch",
5507            "app": "Notes",
5508            "actions": [
5509                {"action": "press", "element": 1},
5510                {"action": "focus", "element": 1}
5511            ]
5512        }))
5513        .unwrap();
5514        assert!(matches!(
5515            request.action,
5516            Action::Batch {
5517                stop_on_error: true,
5518                ..
5519            }
5520        ));
5521        let response = dispatch_with_context(&backend, request.action, &request.context);
5522        assert!(response.ok, "{response:?}");
5523        let results = response.to_value()["results"].clone();
5524        assert_eq!(results[0]["action"], "press");
5525        assert_eq!(results[0]["element"], 1);
5526        assert_eq!(results[1]["action"], "focus");
5527        assert!(response.to_value().get("state").is_none());
5528    }
5529
5530    #[test]
5531    fn p9_menu_list_round_trip_records_no_args() {
5532        let backend = StubBackend::default();
5533        backend
5534            .menu_sections
5535            .lock()
5536            .unwrap()
5537            .push(crate::backend::MenuNode {
5538                title: "File".into(),
5539                enabled: true,
5540                children: vec![crate::backend::MenuNode {
5541                    title: "Save".into(),
5542                    enabled: true,
5543                    children: Vec::new(),
5544                }],
5545            });
5546        let response = dispatch_with_context(
5547            &backend,
5548            Action::MenuList {
5549                app: Some("Notes".into()),
5550                depth: Some(2),
5551            },
5552            &ctx("Notes"),
5553        );
5554        assert!(response.ok, "{:?}", response.to_value());
5555        let value = response.to_value();
5556        assert_eq!(value["action"], "menu_list");
5557        assert_eq!(value["app"], "Notes");
5558        assert_eq!(value["sections"][0]["title"], "File");
5559        assert_eq!(value["sections"][0]["children"][0]["title"], "Save");
5560    }
5561
5562    #[test]
5563    fn p9_menu_click_records_path_and_reports_disabled() {
5564        let mut backend = StubBackend::default();
5565        let response = dispatch_with_context(
5566            &backend,
5567            Action::MenuClick {
5568                app: Some("Notes".into()),
5569                path: "File > Save".into(),
5570            },
5571            &ctx("Notes"),
5572        );
5573        assert!(response.ok, "{:?}", response.to_value());
5574        let value = response.to_value();
5575        assert_eq!(value["path"], serde_json::json!(["File", "Save"]));
5576        assert_eq!(value["pressed"], true);
5577        let recorded = backend.menu_clicks.lock().unwrap().clone();
5578        assert_eq!(recorded.len(), 1);
5579        assert_eq!(
5580            recorded[0].segments,
5581            vec!["File".to_string(), "Save".to_string()]
5582        );
5583
5584        backend.menu_disabled = true;
5585        let response = dispatch_with_context(
5586            &backend,
5587            Action::MenuClick {
5588                app: Some("Notes".into()),
5589                path: "File > Save".into(),
5590            },
5591            &ctx("Notes"),
5592        );
5593        assert!(!response.ok, "disabled menu item must fail");
5594        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5595    }
5596
5597    #[test]
5598    fn p9_apple_event_records_only_length_and_language() {
5599        let backend = StubBackend::default();
5600        *backend.apple_event_stdout.lock().unwrap() = "Notes".into();
5601        let secret = "system prompt with token ABCDEFG";
5602        let response = dispatch_with_context(
5603            &backend,
5604            Action::AppleEvent {
5605                app: Some("Notes".into()),
5606                script: secret.into(),
5607                language: Some(AppleEventLanguage::AppleScript),
5608            },
5609            &ctx("Notes"),
5610        );
5611        assert!(response.ok, "{:?}", response.to_value());
5612        let value = response.to_value();
5613        assert_eq!(value["language"], "AppleScript");
5614        assert_eq!(value["script_bytes"], secret.len());
5615        assert_eq!(value["stdout"], "Notes");
5616        assert!(value["script_hash"].is_string());
5617        let recorded = backend.apple_event_calls.lock().unwrap().clone();
5618        assert_eq!(recorded.len(), 1);
5619        assert_eq!(recorded[0].1, AppleEventLanguage::AppleScript);
5620        // Audit-style storage must not capture the body verbatim.
5621        assert_eq!(recorded[0].2, secret.len());
5622    }
5623
5624    #[test]
5625    fn p9_apple_event_records_language_javascript() {
5626        let backend = StubBackend::default();
5627        let response = dispatch_with_context(
5628            &backend,
5629            Action::AppleEvent {
5630                app: Some("Notes".into()),
5631                script: "Application('Notes').name()".into(),
5632                language: Some(AppleEventLanguage::JavaScript),
5633            },
5634            &ctx("Notes"),
5635        );
5636        assert!(response.ok);
5637        assert_eq!(response.to_value()["language"], "JavaScript");
5638    }
5639
5640    #[test]
5641    fn p9_apple_event_failure_surfaces_execution_failed() {
5642        let backend = StubBackend {
5643            apple_event_fail: true,
5644            ..Default::default()
5645        };
5646        *backend.apple_event_stderr.lock().unwrap() = "execution error: 42".into();
5647        let response = dispatch_with_context(
5648            &backend,
5649            Action::AppleEvent {
5650                app: Some("Notes".into()),
5651                script: "1/0".into(),
5652                language: None,
5653            },
5654            &ctx("Notes"),
5655        );
5656        assert!(!response.ok);
5657        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5658    }
5659
5660    #[test]
5661    fn p9_dock_list_and_click_round_trip() {
5662        let backend = StubBackend::default();
5663        backend.dock_items.lock().unwrap().extend([
5664            "Finder".to_string(),
5665            "Terminal".to_string(),
5666            "Mail".to_string(),
5667        ]);
5668        let response = dispatch_with(&backend, Action::DockList {});
5669        assert!(response.ok);
5670        let value = response.to_value();
5671        let items = value["items"].as_array().unwrap();
5672        assert_eq!(items.len(), 3);
5673        assert_eq!(items[0]["title"], "Finder");
5674
5675        let response = dispatch_with(
5676            &backend,
5677            Action::DockClick {
5678                title: "terminal".into(),
5679            },
5680        );
5681        assert!(response.ok, "{:?}", response.to_value());
5682        assert_eq!(response.to_value()["pressed"], true);
5683        assert_eq!(
5684            backend.dock_clicks.lock().unwrap().clone(),
5685            vec!["terminal".to_string()]
5686        );
5687
5688        let response = dispatch_with(
5689            &backend,
5690            Action::DockClick {
5691                title: "missing".into(),
5692            },
5693        );
5694        assert!(!response.ok);
5695        assert_eq!(response.to_value()["error"]["code"], "element_not_found");
5696    }
5697
5698    #[test]
5699    fn p9_spaces_reports_screen_groups_from_fake_list() {
5700        use crate::target::{WindowBounds, WindowInfo};
5701        use crate::types::DisplayInfo;
5702        use crate::window_groups::SpacesReport;
5703
5704        let mut spaces_report = SpacesReport {
5705            screens: Vec::new(),
5706            note: String::new(),
5707        };
5708        spaces_report
5709            .screens
5710            .push(crate::window_groups::SpaceScreen {
5711                index: 0,
5712                name: "Display 0".into(),
5713                x: 0,
5714                y: 0,
5715                width: 1000,
5716                height: 1000,
5717                is_primary: true,
5718                windows: vec![WindowInfo {
5719                    id: 1,
5720                    pid: 100,
5721                    app: "Notes".into(),
5722                    title: "Note 1".into(),
5723                    bounds: WindowBounds {
5724                        x: 10,
5725                        y: 10,
5726                        width: 200,
5727                        height: 200,
5728                    },
5729                    z: 0,
5730                    layer: 0,
5731                    on_screen: true,
5732                    ax_matched: false,
5733                }],
5734            });
5735
5736        let backend = StubBackend::default();
5737        *backend.spaces_report.lock().unwrap() = spaces_report;
5738        let response = dispatch_with(&backend, Action::Spaces {});
5739        assert!(response.ok);
5740        let value = response.to_value();
5741        assert_eq!(value["action"], "spaces");
5742        assert_eq!(value["screens"][0]["name"], "Display 0");
5743        assert_eq!(value["screens"][0]["windows"][0]["title"], "Note 1");
5744        assert_eq!(value["screens"][0]["is_primary"], true);
5745        // `group_windows` itself is exercised in window_groups tests.
5746        let _ = DisplayInfo {
5747            id: 0,
5748            name: String::new(),
5749            x: 0,
5750            y: 0,
5751            width: 0,
5752            height: 0,
5753            scale: 1.0,
5754            is_primary: false,
5755        };
5756    }
5757
5758    #[test]
5759    fn p9_shortcuts_run_invokes_with_only_the_name() {
5760        // The test backend does not exercise the binary; this test only
5761        // confirms dispatch reaches the helper when the binary is missing.
5762        let backend = StubBackend::default();
5763        let response = dispatch_with(
5764            &backend,
5765            Action::ShortcutsRun {
5766                name: "Open URL".into(),
5767            },
5768        );
5769        // On macOS the dispatch path reaches the helper, so the response is
5770        // either ok (binary present) or `execution_failed` (binary absent).
5771        // Off macOS the dispatcher short-circuits to `unsupported`. Either
5772        // outcome is acceptable; the failure code must be the documented
5773        // one.
5774        if !response.ok {
5775            let value = response.to_value();
5776            let code = value["error"]["code"].as_str().unwrap_or_default();
5777            assert!(
5778                matches!(code, "execution_failed" | "unsupported"),
5779                "{value:?}"
5780            );
5781        }
5782    }
5783
5784    #[test]
5785    fn session_prune_action_removes_an_empty_temp_session() {
5786        let _cache = crate::cache::CacheEnv::acquire();
5787        let dir = cache::session_dir(Some("empty-session")).unwrap();
5788        assert!(dir.join("owner.json").is_file());
5789        let response = dispatch(Action::SessionPrune {
5790            dry_run: true,
5791            ttl_secs: Some(0),
5792        });
5793        assert!(response.ok, "{response:?}");
5794        assert_eq!(response.to_value()["dry_run"], true);
5795        assert!(response.to_value()["removed"]
5796            .as_array()
5797            .unwrap()
5798            .iter()
5799            .any(|name| name == "empty-session"));
5800        assert!(dir.exists());
5801        let response = dispatch(Action::SessionPrune {
5802            dry_run: false,
5803            ttl_secs: Some(0),
5804        });
5805        assert!(response.ok, "{response:?}");
5806        assert!(!dir.exists());
5807    }
5808
5809    #[test]
5810    fn pause_short_circuits_before_the_stub_clicks() {
5811        let backend = StubBackend::default();
5812        let _paused = policy::override_paused(true);
5813        let response = dispatch_with(
5814            &backend,
5815            Action::Click {
5816                x: Some(1),
5817                y: Some(2),
5818                button: MouseButton::Left,
5819                double: false,
5820                count: 1,
5821            },
5822        );
5823        assert_eq!(response.to_value()["error"]["code"], "paused");
5824        assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
5825        assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5826        let waited = dispatch_with(
5827            &backend,
5828            Action::Wait {
5829                ms: Some(0),
5830                seconds: None,
5831            },
5832        );
5833        assert!(waited.ok, "{waited:?}");
5834    }
5835
5836    #[test]
5837    fn confirm_hook_allows_quit_and_a_failure_does_not() {
5838        let backend = StubBackend::default();
5839        let dir = std::env::temp_dir().join(format!(
5840            "computer-use-hook-{}-{}",
5841            std::process::id(),
5842            std::time::SystemTime::now()
5843                .duration_since(std::time::UNIX_EPOCH)
5844                .map(|duration| duration.as_nanos())
5845                .unwrap_or(0)
5846        ));
5847        let _ = std::fs::remove_dir_all(&dir);
5848        std::fs::create_dir_all(&dir).unwrap();
5849        #[cfg(unix)]
5850        let (allow, deny) = {
5851            let allow = dir.join("allow.sh");
5852            let deny = dir.join("deny.sh");
5853            std::fs::write(&allow, "#!/bin/sh\ncat >/dev/null\nexit 0\n").unwrap();
5854            std::fs::write(&deny, "#!/bin/sh\ncat >/dev/null\nexit 1\n").unwrap();
5855            use std::os::unix::fs::PermissionsExt;
5856            std::fs::set_permissions(&allow, std::fs::Permissions::from_mode(0o700)).unwrap();
5857            std::fs::set_permissions(&deny, std::fs::Permissions::from_mode(0o700)).unwrap();
5858            (allow.display().to_string(), deny.display().to_string())
5859        };
5860        #[cfg(windows)]
5861        let (allow, deny) = ("cmd /C exit 0".to_string(), "cmd /C exit 1".to_string());
5862        let mut context = ctx("412");
5863        context.confirm_cmd = Some(deny);
5864        let denied = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5865        assert_eq!(denied.to_value()["error"]["code"], "confirm_denied");
5866        assert!(backend.quit.lock().unwrap().is_empty());
5867        context.confirm_cmd = Some(allow);
5868        let allowed = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5869        assert!(allowed.ok, "{allowed:?}");
5870        assert_eq!(backend.quit.lock().unwrap().len(), 1);
5871        let _ = std::fs::remove_dir_all(&dir);
5872    }
5873
5874    #[test]
5875    fn blocked_url_prevents_the_click() {
5876        let mut backend = StubBackend::default();
5877        backend.apps.push(crate::target::AppInfo {
5878            name: "Safari".into(),
5879            bundle_id: Some("com.apple.Safari".into()),
5880            pid: 80,
5881            path: Some("/Applications/Safari.app/Contents/MacOS/Safari".into()),
5882            active: false,
5883            hidden: false,
5884            activation_policy: crate::target::ActivationPolicy::Regular,
5885        });
5886        backend
5887            .browser_urls
5888            .lock()
5889            .unwrap()
5890            .insert(80, "https://accounts.example.com/login".into());
5891        let dir = std::env::temp_dir().join(format!(
5892            "computer-use-url-{}-{}",
5893            std::process::id(),
5894            std::time::SystemTime::now()
5895                .duration_since(std::time::UNIX_EPOCH)
5896                .map(|duration| duration.as_nanos())
5897                .unwrap_or(0)
5898        ));
5899        let _ = std::fs::remove_dir_all(&dir);
5900        std::fs::create_dir_all(&dir).unwrap();
5901        let policy_path = dir.join("policy.toml");
5902        std::fs::write(
5903            &policy_path,
5904            "version = 1\n[browser]\nblocked_urls = [\"https://accounts.example.com/*\"]\n",
5905        )
5906        .unwrap();
5907        let mut context = ctx("80");
5908        context.policy = Some(policy_path.display().to_string());
5909        let response = dispatch_with_context(
5910            &backend,
5911            Action::Click {
5912                x: Some(3),
5913                y: Some(4),
5914                button: MouseButton::Left,
5915                double: false,
5916                count: 1,
5917            },
5918            &context,
5919        );
5920        assert_eq!(response.to_value()["error"]["code"], "blocked_url");
5921        assert!(response.to_value()["error"]["message"]
5922            .as_str()
5923            .unwrap()
5924            .contains("https://accounts.example.com/*"));
5925        assert!(!response.to_value()["error"]["message"]
5926            .as_str()
5927            .unwrap()
5928            .contains("login"));
5929        assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5930        backend
5931            .browser_urls
5932            .lock()
5933            .unwrap()
5934            .insert(80, "https://example.com/".into());
5935        let allowed = dispatch_with_context(
5936            &backend,
5937            Action::Click {
5938                x: Some(3),
5939                y: Some(4),
5940                button: MouseButton::Left,
5941                double: false,
5942                count: 1,
5943            },
5944            &context,
5945        );
5946        assert!(allowed.ok, "{allowed:?}");
5947        assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5948        let _ = std::fs::remove_dir_all(&dir);
5949    }
5950
5951    fn temp_dir(label: &str) -> std::path::PathBuf {
5952        let dir = std::env::temp_dir().join(format!(
5953            "computer-use-{label}-{}-{}",
5954            std::process::id(),
5955            std::time::SystemTime::now()
5956                .duration_since(std::time::UNIX_EPOCH)
5957                .map(|duration| duration.as_nanos())
5958                .unwrap_or(0)
5959        ));
5960        let _ = std::fs::remove_dir_all(&dir);
5961        std::fs::create_dir_all(&dir).unwrap();
5962        dir
5963    }
5964
5965    fn click_at(x: i32, y: i32) -> Action {
5966        Action::Click {
5967            x: Some(x),
5968            y: Some(y),
5969            button: MouseButton::Left,
5970            double: false,
5971            count: 1,
5972        }
5973    }
5974
5975    fn last_audit(dir: &std::path::Path) -> Value {
5976        let text = std::fs::read_to_string(dir.join("audit.jsonl")).unwrap();
5977        serde_json::from_str(text.lines().last().unwrap()).unwrap()
5978    }
5979
5980    #[test]
5981    fn auto_event_falls_back_to_activate_unless_blocked() {
5982        let backend = StubBackend {
5983            fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5984            ..StubBackend::default()
5985        };
5986        let response = dispatch_with(&backend, click_at(3, 4));
5987        assert!(response.ok, "{response:?}");
5988        let value = response.to_value();
5989        assert_eq!(value["tier"], "activate");
5990        assert_eq!(value["cursor_moved"], true);
5991        assert_eq!(value["focus_restored"], true);
5992        assert_eq!(
5993            backend.tiers_run.lock().unwrap().as_slice(),
5994            ["event", "activate"]
5995        );
5996        assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5997
5998        let held = StubBackend {
5999            fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
6000            ..StubBackend::default()
6001        };
6002        let dir = temp_dir("lock-fallback");
6003        let path = dir.join("foreground.lock");
6004        let _slot = crate::foreground::push_lock_path(&path);
6005        let _lock = crate::foreground::try_lock(&path).unwrap();
6006        let response = dispatch_with(&held, click_at(3, 4));
6007        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
6008        assert!(response.to_value()["error"]["message"]
6009            .as_str()
6010            .unwrap()
6011            .contains("stub tier event failed"));
6012        assert_eq!(held.tiers_run.lock().unwrap().as_slice(), ["event"]);
6013        assert_eq!(held.clicks.load(Ordering::SeqCst), 0);
6014        drop(_lock);
6015        let _ = std::fs::remove_dir_all(&dir);
6016
6017        let background = StubBackend {
6018            fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
6019            ..StubBackend::default()
6020        };
6021        let response = dispatch_with_context(
6022            &background,
6023            click_at(1, 2),
6024            &CallContext {
6025                require_background: true,
6026                ..CallContext::default()
6027            },
6028        );
6029        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
6030        assert_eq!(background.tiers_run.lock().unwrap().as_slice(), ["event"]);
6031        assert_eq!(background.clicks.load(Ordering::SeqCst), 0);
6032
6033        let refused = StubBackend::default();
6034        let response = dispatch_with_context(
6035            &refused,
6036            click_at(1, 2),
6037            &CallContext {
6038                tier: Some("activate".into()),
6039                require_background: true,
6040                ..CallContext::default()
6041            },
6042        );
6043        assert!(response.to_value()["error"]["message"]
6044            .as_str()
6045            .unwrap()
6046            .contains("--require-background"));
6047        assert!(refused.tiers_run.lock().unwrap().is_empty());
6048        assert_eq!(refused.clicks.load(Ordering::SeqCst), 0);
6049    }
6050
6051    #[test]
6052    fn activate_names_the_lock_holder_before_it_clicks() {
6053        let backend = StubBackend::default();
6054        let dir = temp_dir("lock-held");
6055        let path = dir.join("foreground.lock");
6056        let _slot = crate::foreground::push_lock_path(&path);
6057        let _lock = crate::foreground::try_lock(&path).unwrap();
6058        let response = dispatch_with_context(
6059            &backend,
6060            click_at(9, 8),
6061            &CallContext {
6062                tier: Some("activate".into()),
6063                ..CallContext::default()
6064            },
6065        );
6066        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
6067        let message = response.to_value()["error"]["message"]
6068            .as_str()
6069            .unwrap()
6070            .to_string();
6071        assert!(
6072            message.contains(&format!(
6073                "foreground.lock is held by pid {}",
6074                std::process::id()
6075            )),
6076            "{message}"
6077        );
6078        assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
6079        assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 1);
6080        assert!(backend.tiers_run.lock().unwrap().is_empty());
6081        drop(_lock);
6082        let _ = std::fs::remove_dir_all(&dir);
6083    }
6084
6085    #[test]
6086    fn synthetic_without_the_feature_does_not_hit_test_or_click() {
6087        let backend = StubBackend::default();
6088        let response = dispatch_with_context(
6089            &backend,
6090            click_at(1, 1),
6091            &CallContext {
6092                tier: Some("synthetic".into()),
6093                ..CallContext::default()
6094            },
6095        );
6096        if crate::tier::skylight_compiled() {
6097            assert!(response.ok, "{response:?}");
6098            assert_eq!(response.to_value()["tier"], "synthetic");
6099            assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
6100            assert_eq!(backend.tiers_run.lock().unwrap().as_slice(), ["synthetic"]);
6101        } else {
6102            assert_eq!(response.to_value()["error"]["code"], "unsupported");
6103            assert!(response.to_value()["error"]["message"]
6104                .as_str()
6105                .unwrap()
6106                .contains("built without skylight feature"));
6107            assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 0);
6108            assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
6109            assert!(backend.tiers_run.lock().unwrap().is_empty());
6110        }
6111    }
6112
6113    #[test]
6114    fn missing_overlay_helper_warns_and_no_overlay_skips_it() {
6115        let dir = temp_dir("overlay-warn");
6116        let _slot = crate::overlay::push_socket_path(dir.join("missing.sock"));
6117        let backend = StubBackend::default();
6118        let response = dispatch_with_context(
6119            &backend,
6120            click_at(4, 5),
6121            &CallContext {
6122                session: Some("agent".into()),
6123                ..CallContext::default()
6124            },
6125        );
6126        assert!(response.ok, "{response:?}");
6127        assert_eq!(response.to_value()["tier"], "event");
6128        assert_eq!(response.to_value()["cursor_moved"], false);
6129        assert!(response.to_value().get("focus_restored").is_none());
6130        let warnings = response.to_value()["warnings"].as_array().unwrap().clone();
6131        assert!(warnings
6132            .iter()
6133            .any(|item| item == "overlay helper not running"));
6134
6135        let quiet = dispatch_with_context(
6136            &backend,
6137            click_at(4, 5),
6138            &CallContext {
6139                no_overlay: true,
6140                ..CallContext::default()
6141            },
6142        );
6143        assert!(quiet.ok, "{quiet:?}");
6144        assert!(quiet.to_value().get("warnings").is_none());
6145        let _ = std::fs::remove_dir_all(&dir);
6146    }
6147
6148    #[test]
6149    fn audit_records_decision_source_and_the_real_code() {
6150        let dir = temp_dir("audit");
6151        let _state = crate::audit::push_state_dir(&dir);
6152        let backend = StubBackend::default();
6153        let response = dispatch_with(&backend, click_at(2, 3));
6154        assert!(response.ok, "{response:?}");
6155        let line = last_audit(&dir);
6156        assert_eq!(line["action"], "click");
6157        assert_eq!(line["code"], "ok");
6158        assert_eq!(line["decision"], "allow");
6159        assert_eq!(line["source"], "default");
6160        assert_eq!(line["tier"], "event");
6161        assert!(line.get("text").is_none());
6162
6163        let denied = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
6164        assert_eq!(denied.to_value()["error"]["code"], "policy_denied");
6165        let line = last_audit(&dir);
6166        assert_eq!(line["action"], "launch");
6167        assert_eq!(line["code"], "policy_denied");
6168        assert_eq!(line["decision"], "deny");
6169        assert_eq!(line["source"], "builtin");
6170
6171        let checked =
6172            dispatch_with_context(&backend, Action::PolicyCheck, &ctx("com.apple.Terminal"));
6173        assert!(checked.ok, "{checked:?}");
6174        assert_eq!(checked.to_value()["decision"], "deny");
6175        let line = last_audit(&dir);
6176        assert_eq!(line["action"], "policy_check");
6177        assert_eq!(line["code"], "ok");
6178        assert_eq!(line["decision"], "deny");
6179        assert_eq!(line["source"], "builtin");
6180
6181        let bad = dispatch_with(
6182            &backend,
6183            Action::Click {
6184                x: None,
6185                y: None,
6186                button: MouseButton::Left,
6187                double: false,
6188                count: 1,
6189            },
6190        );
6191        assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
6192        let line = last_audit(&dir);
6193        assert_eq!(line["code"], "invalid_arguments");
6194        assert_eq!(line["decision"], "allow");
6195        assert_eq!(line["source"], "default");
6196        let _ = std::fs::remove_dir_all(&dir);
6197    }
6198
6199    #[test]
6200    fn drag_hold_click_and_keys_use_the_stub() {
6201        let backend = StubBackend::default();
6202        let short = dispatch_with(
6203            &backend,
6204            Action::Drag {
6205                path: vec![crate::types::Point { x: 1, y: 1 }],
6206                button: MouseButton::Left,
6207            },
6208        );
6209        assert_eq!(short.to_value()["error"]["code"], "invalid_arguments");
6210        assert_eq!(backend.moves.load(Ordering::SeqCst), 0);
6211
6212        let dragged = dispatch_with(
6213            &backend,
6214            Action::Drag {
6215                path: vec![
6216                    crate::types::Point { x: 1, y: 2 },
6217                    crate::types::Point { x: 3, y: 4 },
6218                    crate::types::Point { x: 5, y: 6 },
6219                ],
6220                button: MouseButton::Left,
6221            },
6222        );
6223        assert!(dragged.ok, "{dragged:?}");
6224        assert_eq!(dragged.to_value()["moves"], 2);
6225        assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6226        assert_eq!(dragged.to_value()["tier"], "event");
6227
6228        let ax = dispatch_with_context(
6229            &backend,
6230            Action::Drag {
6231                path: vec![
6232                    crate::types::Point { x: 1, y: 2 },
6233                    crate::types::Point { x: 3, y: 4 },
6234                ],
6235                button: MouseButton::Left,
6236            },
6237            &CallContext {
6238                tier: Some("ax".into()),
6239                ..CallContext::default()
6240            },
6241        );
6242        assert_eq!(ax.to_value()["error"]["code"], "unsupported");
6243        assert!(ax.to_value()["error"]["message"]
6244            .as_str()
6245            .unwrap()
6246            .contains("pass --element"));
6247        assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6248
6249        let triple = dispatch_with(
6250            &backend,
6251            Action::TripleClick {
6252                x: Some(8),
6253                y: Some(9),
6254                button: MouseButton::Left,
6255            },
6256        );
6257        assert!(triple.ok, "{triple:?}");
6258        assert_eq!(triple.to_value()["action"], "triple_click");
6259        assert_eq!(triple.to_value()["count"], 3);
6260        assert_eq!(backend.clicks.load(Ordering::SeqCst), 3);
6261
6262        let held = dispatch_with(
6263            &backend,
6264            Action::HoldKey {
6265                keys: "shift".into(),
6266                ms: 40_000,
6267            },
6268        );
6269        assert_eq!(held.to_value()["error"]["code"], "invalid_arguments");
6270        assert!(backend.holds_ms.lock().unwrap().is_empty());
6271
6272        let shift = dispatch_with(
6273            &backend,
6274            Action::HoldKey {
6275                keys: "shift".into(),
6276                ms: 400,
6277            },
6278        );
6279        assert!(shift.ok, "{shift:?}");
6280        assert_eq!(backend.holds_ms.lock().unwrap().as_slice(), [400]);
6281
6282        let sequence = dispatch_with(
6283            &backend,
6284            Action::Key {
6285                keys: "super+c Return".into(),
6286            },
6287        );
6288        assert!(sequence.ok, "{sequence:?}");
6289        assert_eq!(sequence.to_value()["strokes"], 2);
6290        let keys = backend.keys.lock().unwrap();
6291        let sequence = &keys[keys.len() - 2..];
6292        assert_eq!(sequence[0].1, "c");
6293        assert_eq!(sequence[0].0, vec![Modifier::Meta]);
6294        assert_eq!(sequence[1].1, "enter");
6295        drop(keys);
6296
6297        let unknown = dispatch_with(
6298            &backend,
6299            Action::Key {
6300                keys: "bogus".into(),
6301            },
6302        );
6303        assert_eq!(unknown.to_value()["error"]["code"], "invalid_arguments");
6304    }
6305
6306    #[test]
6307    fn mouse_down_releases_when_the_session_ends() {
6308        let backend = StubBackend::default();
6309        let down = Action::MouseDown {
6310            x: 4,
6311            y: 5,
6312            button: MouseButton::Left,
6313        };
6314        let alone = dispatch_with(&backend, down.clone());
6315        assert!(alone.ok, "{alone:?}");
6316        assert!(backend.held_buttons.lock().unwrap().is_empty());
6317        assert_eq!(
6318            backend.releases.lock().unwrap().as_slice(),
6319            [MouseButton::Left]
6320        );
6321
6322        backend.releases.lock().unwrap().clear();
6323        let _session = crate::InputSession::open(&backend);
6324        let kept = dispatch_with(&backend, down);
6325        assert!(kept.ok, "{kept:?}");
6326        assert_eq!(
6327            backend.held_buttons.lock().unwrap().as_slice(),
6328            [MouseButton::Left]
6329        );
6330        assert!(backend.releases.lock().unwrap().is_empty());
6331        drop(_session);
6332        assert!(backend.held_buttons.lock().unwrap().is_empty());
6333        assert_eq!(
6334            backend.releases.lock().unwrap().as_slice(),
6335            [MouseButton::Left]
6336        );
6337
6338        let batch = dispatch_with(
6339            &backend,
6340            Action::Batch {
6341                stop_on_error: true,
6342                actions: vec![
6343                    json!({"action": "mouse_down", "x": 1, "y": 2}),
6344                    json!({"action": "mouse_up", "x": 1, "y": 2}),
6345                ],
6346            },
6347        );
6348        assert!(batch.ok, "{batch:?}");
6349        assert!(backend.held_buttons.lock().unwrap().is_empty());
6350    }
6351
6352    #[test]
6353    fn clipboard_confirms_and_omits_the_text_from_the_audit() {
6354        let dir = temp_dir("clipboard-audit");
6355        let _state = crate::audit::push_state_dir(&dir);
6356        let _cache = crate::cache::CacheEnv::acquire();
6357        let backend = StubBackend::default();
6358        let secret = "do-not-log";
6359        let response = dispatch_with(
6360            &backend,
6361            Action::ClipboardSet {
6362                text: Some(secret.into()),
6363                image: None,
6364            },
6365        );
6366        assert_eq!(response.to_value()["error"]["code"], "confirm_required");
6367        assert!(backend.clipboard_text.lock().unwrap().is_none());
6368        let line = last_audit(&dir);
6369        assert_eq!(line["action"], "clipboard_set");
6370        assert_eq!(line["decision"], "confirm");
6371        assert_eq!(line["chars"], secret.chars().count() as u64);
6372        let encoded = line.to_string();
6373        assert!(!encoded.contains(secret));
6374        assert!(line.get("text").is_none());
6375
6376        *backend.clipboard_text.lock().unwrap() = Some("ready".into());
6377        let got = dispatch_with(&backend, Action::ClipboardGet);
6378        assert!(got.ok, "{got:?}");
6379        assert_eq!(got.to_value()["kind"], "text");
6380        assert_eq!(got.to_value()["text"], "ready");
6381
6382        let empty = dispatch_with(
6383            &backend,
6384            Action::ClipboardSet {
6385                text: Some(String::new()),
6386                image: None,
6387            },
6388        );
6389        assert_eq!(empty.to_value()["error"]["code"], "invalid_arguments");
6390        let _ = std::fs::remove_dir_all(&dir);
6391    }
6392
6393    #[test]
6394    fn zoom_crops_a_window_region_and_rejects_the_far_edge() {
6395        let _cache = crate::cache::CacheEnv::acquire();
6396        let backend = StubBackend::default();
6397        let response = dispatch_with_context(
6398            &backend,
6399            Action::Zoom {
6400                output: None,
6401                region: Region::new(0, 0, 10, 10).unwrap(),
6402                format: Some(ImageFormat::Png),
6403                max_width: Some(0),
6404                canvas: None,
6405                base64: false,
6406            },
6407            &CallContext {
6408                app: Some("Notes".into()),
6409                session: Some("p8-zoom".into()),
6410                ..CallContext::default()
6411            },
6412        );
6413        assert!(response.ok, "{response:?}");
6414        let value = response.to_value();
6415        assert_eq!(value["width"], 20);
6416        assert_eq!(value["height"], 20);
6417        assert_eq!(value["logical_width"], 10);
6418        assert_eq!(value["logical_height"], 10);
6419
6420        let outside = dispatch_with_context(
6421            &backend,
6422            Action::Zoom {
6423                output: None,
6424                region: Region::new(0, 0, 401, 300).unwrap(),
6425                format: None,
6426                max_width: Some(0),
6427                canvas: None,
6428                base64: false,
6429            },
6430            &CallContext {
6431                app: Some("Notes".into()),
6432                session: Some("p8-zoom-out".into()),
6433                ..CallContext::default()
6434            },
6435        );
6436        assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
6437
6438        let both = dispatch_with_context(
6439            &backend,
6440            Action::Screenshot {
6441                output: None,
6442                region: Some(Region::new(0, 0, 10, 10).unwrap()),
6443                format: None,
6444                max_width: None,
6445                canvas: None,
6446                base64: false,
6447                marks: false,
6448            },
6449            &CallContext {
6450                app: Some("Notes".into()),
6451                ..CallContext::default()
6452            },
6453        );
6454        assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
6455        assert!(both.to_value()["error"]["message"]
6456            .as_str()
6457            .unwrap()
6458            .contains("not both"));
6459    }
6460
6461    // ----- P10 gap fixes -----
6462
6463    #[test]
6464    fn browser_text_action_round_trips_with_only_selector() {
6465        let text: Action =
6466            serde_json::from_str(r#"{"action":"browser_text","selector":"h1"}"#).unwrap();
6467        match &text {
6468            Action::BrowserText {
6469                selector,
6470                target_id,
6471            } => {
6472                assert_eq!(selector, "h1");
6473                assert!(target_id.is_none());
6474            }
6475            other => panic!("unexpected {other:?}"),
6476        }
6477        let encoded = serde_json::to_string(&text).unwrap();
6478        let decoded: Action = serde_json::from_str(&encoded).unwrap();
6479        assert_eq!(decoded, text);
6480    }
6481
6482    #[test]
6483    fn browser_type_action_round_trips_with_text_and_optional_selector() {
6484        let action: Action =
6485            serde_json::from_str(r##"{"action":"browser_type","text":"hi","selector":"#name"}"##)
6486                .unwrap();
6487        match &action {
6488            Action::BrowserType {
6489                text,
6490                selector,
6491                target_id,
6492            } => {
6493                assert_eq!(text, "hi");
6494                assert_eq!(selector.as_deref(), Some("#name"));
6495                assert!(target_id.is_none());
6496            }
6497            other => panic!("unexpected {other:?}"),
6498        }
6499        let encoded = serde_json::to_string(&action).unwrap();
6500        let decoded: Action = serde_json::from_str(&encoded).unwrap();
6501        assert_eq!(decoded, action);
6502    }
6503
6504    #[test]
6505    fn browser_text_label_is_browser_text_and_class_is_observe() {
6506        // BrowserText reads innerText; classified Observe (read-only).
6507        let action = Action::BrowserText {
6508            selector: "h1".into(),
6509            target_id: None,
6510        };
6511        assert_eq!(action_label(&action), "browser_text");
6512        assert!(matches!(policy_class(&action), PolicyClass::Observe));
6513    }
6514
6515    #[test]
6516    fn browser_type_label_is_browser_type_and_class_is_mutate() {
6517        // BrowserType types text; classified Mutate (write). The text body
6518        // never leaves the audit row — only the char count does.
6519        let action = Action::BrowserType {
6520            text: "abc".into(),
6521            selector: Some("#name".into()),
6522            target_id: None,
6523        };
6524        assert_eq!(action_label(&action), "browser_type");
6525        assert!(matches!(policy_class(&action), PolicyClass::Mutate));
6526        assert_eq!(clipboard_chars(&action), Some(3));
6527    }
6528
6529    #[test]
6530    fn browser_text_does_not_record_chars_in_audit() {
6531        // Reading innerText must not push a chars count into the audit row.
6532        let action = Action::BrowserText {
6533            selector: "h1".into(),
6534            target_id: None,
6535        };
6536        assert_eq!(clipboard_chars(&action), None);
6537    }
6538
6539    #[test]
6540    #[cfg(feature = "cdp")]
6541    fn browser_navigate_blocks_url_before_connecting() {
6542        // Use a deliberately unreachable WebSocket URL. If the URL block ran
6543        // before the connect attempt, we get blocked_url. If the order were
6544        // wrong we would see cdp_unavailable from the connection failure.
6545        let backend = StubBackend::default();
6546        let context = CallContext {
6547            cdp: true,
6548            cdp_url: Some("ws://127.0.0.1:1".into()),
6549            browser_blocked_urls: vec!["https://blocked.example.com".into()],
6550            ..CallContext::default()
6551        };
6552        let response = dispatch_with_context(
6553            &backend,
6554            Action::BrowserNavigate {
6555                url: "https://blocked.example.com".into(),
6556                target_id: None,
6557            },
6558            &context,
6559        );
6560        assert!(!response.ok);
6561        let value = response.to_value();
6562        assert_eq!(value["error"]["code"], "blocked_url");
6563    }
6564
6565    #[test]
6566    #[cfg(feature = "cdp")]
6567    fn browser_navigate_unblocked_url_still_attempts_connect() {
6568        // Sanity: without a matching pattern, navigate goes through the
6569        // normal path and reports the connect failure (cdp_unavailable).
6570        let backend = StubBackend::default();
6571        let context = CallContext {
6572            cdp: true,
6573            cdp_url: Some("ws://127.0.0.1:1".into()),
6574            ..CallContext::default()
6575        };
6576        let response = dispatch_with_context(
6577            &backend,
6578            Action::BrowserNavigate {
6579                url: "https://example.com".into(),
6580                target_id: None,
6581            },
6582            &context,
6583        );
6584        assert!(!response.ok);
6585        let value = response.to_value();
6586        assert_eq!(value["error"]["code"], "cdp_unavailable");
6587    }
6588
6589    // -----------------------------------------------------------------------
6590    // P14: the `--display` filter.
6591    // -----------------------------------------------------------------------
6592
6593    fn filtered(spec: &str) -> CallContext {
6594        CallContext {
6595            display: Some(spec.into()),
6596            ..CallContext::default()
6597        }
6598    }
6599
6600    fn code_of(response: &Response) -> String {
6601        response.to_value()["error"]["code"]
6602            .as_str()
6603            .unwrap_or("")
6604            .to_string()
6605    }
6606
6607    #[test]
6608    fn display_filter_rejects_a_point_outside_the_chosen_monitor() {
6609        // The stub reports 800x600 at the origin and 640x480 at (800, 100).
6610        let backend = StubBackend::with_two_displays();
6611        let context = filtered("2");
6612
6613        // On the second monitor: allowed, and the click really happened.
6614        let response = dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &context);
6615        assert!(response.ok, "{response:?}");
6616
6617        // On the first monitor, so outside the filter: invalid_arguments, and
6618        // nothing moved.
6619        let before = backend.moves.load(Ordering::SeqCst);
6620        for (x, y) in [(0, 0), (799, 599), (1440, 580), (800, 99), (900, 90)] {
6621            let response = dispatch_with_context(&backend, Action::Move { x, y }, &context);
6622            assert_eq!(code_of(&response), "invalid_arguments", "{x},{y}");
6623            assert_eq!(backend.moves.load(Ordering::SeqCst), before);
6624        }
6625        let message = dispatch_with_context(&backend, Action::Move { x: 0, y: 0 }, &context)
6626            .to_value()["error"]["message"]
6627            .as_str()
6628            .expect("message")
6629            .to_string();
6630        assert!(message.contains("640x480+800+100"), "{message}");
6631    }
6632
6633    #[test]
6634    fn display_filter_covers_click_move_drag_scroll_and_mouse_halves() {
6635        let backend = StubBackend::with_two_displays();
6636        let context = filtered("2");
6637        let inside = Action::Click {
6638            x: Some(900),
6639            y: Some(300),
6640            button: MouseButton::Left,
6641            double: false,
6642            count: 1,
6643        };
6644        let outside = Action::Click {
6645            x: Some(10),
6646            y: Some(10),
6647            button: MouseButton::Left,
6648            double: false,
6649            count: 1,
6650        };
6651
6652        assert!(dispatch_with_context(&backend, inside.clone(), &context).ok);
6653        assert_eq!(
6654            code_of(&dispatch_with_context(&backend, outside.clone(), &context)),
6655            "invalid_arguments"
6656        );
6657        assert_eq!(
6658            code_of(&dispatch_with_context(
6659                &backend,
6660                Action::DoubleClick {
6661                    x: Some(10),
6662                    y: Some(10),
6663                    button: MouseButton::Left
6664                },
6665                &context
6666            )),
6667            "invalid_arguments"
6668        );
6669        assert_eq!(
6670            code_of(&dispatch_with_context(
6671                &backend,
6672                Action::MouseDown {
6673                    x: 10,
6674                    y: 10,
6675                    button: MouseButton::Left
6676                },
6677                &context
6678            )),
6679            "invalid_arguments"
6680        );
6681        assert_eq!(
6682            code_of(&dispatch_with_context(
6683                &backend,
6684                Action::Scroll {
6685                    x: Some(10),
6686                    y: Some(10),
6687                    dx: Some(0),
6688                    dy: Some(1),
6689                    direction: None,
6690                    amount: None,
6691                },
6692                &context
6693            )),
6694            "invalid_arguments"
6695        );
6696        let path = vec![Point { x: 900, y: 300 }, Point { x: 10, y: 10 }];
6697        assert_eq!(
6698            code_of(&dispatch_with_context(
6699                &backend,
6700                Action::Drag {
6701                    path,
6702                    button: MouseButton::Left
6703                },
6704                &context
6705            )),
6706            "invalid_arguments"
6707        );
6708    }
6709
6710    #[test]
6711    fn a_coordinate_action_without_a_point_passes_the_filter() {
6712        // Element-targeted and wheel-only actions name no point, so the filter
6713        // has nothing to reject. (A bare `click` with neither `element` nor
6714        // `x`/`y` is refused earlier by `preflight_action`, not by this gate.)
6715        let backend = StubBackend::with_two_displays();
6716        let response = dispatch_with_context(
6717            &backend,
6718            Action::Scroll {
6719                x: None,
6720                y: None,
6721                dx: Some(0),
6722                dy: Some(2),
6723                direction: None,
6724                amount: None,
6725            },
6726            &filtered("2"),
6727        );
6728        assert!(response.ok, "{response:?}");
6729        assert_eq!(backend.scrolls.lock().expect("scrolls").len(), 1);
6730    }
6731
6732    #[test]
6733    fn an_unknown_display_id_is_invalid_arguments() {
6734        let backend = StubBackend::with_two_displays();
6735        let response =
6736            dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &filtered("9"));
6737        assert_eq!(code_of(&response), "invalid_arguments");
6738
6739        // `displays` and `isolation` do not consume the filter, so a valid id
6740        // that no longer exists still lets them answer.
6741        let response = dispatch_with_context(&backend, Action::Displays, &filtered("99999"));
6742        assert!(response.ok, "{response:?}");
6743        let response = dispatch_with_context(&backend, Action::Isolation, &filtered("99999"));
6744        assert!(response.ok, "{response:?}");
6745        assert_eq!(backend.list_displays().unwrap().len(), 2);
6746
6747        // Bad syntax is refused everywhere, `displays` included.
6748        let response = dispatch_with_context(&backend, Action::Displays, &filtered("nope"));
6749        assert_eq!(code_of(&response), "invalid_arguments");
6750        // `virtual` is valid syntax, and `displays` does not consume the filter,
6751        // so it answers normally. A command that *would* capture under that
6752        // filter reports `unsupported` instead.
6753        let response = dispatch_with_context(&backend, Action::Displays, &filtered("virtual"));
6754        assert!(response.ok, "{response:?}");
6755    }
6756
6757    #[test]
6758    fn virtual_is_unsupported_over_the_json_path() {
6759        // The JSON path cannot exec a prepared X server, so it says so instead
6760        // of silently capturing the whole desktop.
6761        let backend = StubBackend::default();
6762        let response = dispatch_with_context(
6763            &backend,
6764            Action::Move { x: 10, y: 10 },
6765            &filtered("virtual"),
6766        );
6767        assert_eq!(code_of(&response), "unsupported");
6768
6769        let response = dispatch_with_context(
6770            &backend,
6771            Action::Screenshot {
6772                output: None,
6773                region: None,
6774                format: None,
6775                max_width: None,
6776                canvas: None,
6777                base64: false,
6778                marks: false,
6779            },
6780            &filtered("virtual"),
6781        );
6782        assert_eq!(code_of(&response), "unsupported");
6783    }
6784
6785    #[test]
6786    fn the_display_filter_never_leaks_into_displays_or_isolation() {
6787        let backend = StubBackend::with_two_displays();
6788        let context = filtered("9");
6789        assert!(dispatch_with_context(&backend, Action::Displays, &context).ok);
6790        assert!(dispatch_with_context(&backend, Action::Isolation, &context).ok);
6791    }
6792
6793    #[test]
6794    fn a_screenshot_region_must_fit_inside_the_filtered_monitor() {
6795        let _cache = crate::cache::CacheEnv::acquire();
6796        let backend = StubBackend::with_two_displays();
6797        let context = filtered("2");
6798        let inside = Action::Screenshot {
6799            output: None,
6800            region: Some(Region::new(800, 100, 640, 480).unwrap()),
6801            format: None,
6802            max_width: None,
6803            canvas: None,
6804            base64: false,
6805            marks: false,
6806        };
6807        let response = dispatch_with_context(&backend, inside, &context);
6808        assert!(response.ok, "{response:?}");
6809
6810        // The map origin is the monitor origin, not the desktop origin.
6811        let map = response.to_value();
6812        assert_eq!(map["origin"]["x"], 800);
6813        assert_eq!(map["origin"]["y"], 100);
6814        let _ = fs::remove_file(
6815            map["path"]
6816                .as_str()
6817                .expect("path")
6818                .parse::<std::path::PathBuf>()
6819                .expect("path"),
6820        );
6821
6822        let straddling = Action::Screenshot {
6823            output: None,
6824            region: Some(Region::new(700, 100, 200, 200).unwrap()),
6825            format: None,
6826            max_width: None,
6827            canvas: None,
6828            base64: false,
6829            marks: false,
6830        };
6831        assert_eq!(
6832            code_of(&dispatch_with_context(&backend, straddling, &context)),
6833            "invalid_arguments"
6834        );
6835    }
6836
6837    #[test]
6838    fn isolation_reports_the_matrix_and_the_infeasible_keys() {
6839        let response = dispatch_with(&StubBackend::default(), Action::Isolation);
6840        assert!(response.ok, "{response:?}");
6841        let value = response.to_value();
6842        assert_eq!(value["platform"], crate::isolation::platform());
6843        assert_eq!(value["display_filter"], true);
6844        assert_eq!(value["xvfb"], "linux only");
6845        assert_eq!(value["vm"], "documented, not shipped");
6846        assert_eq!(
6847            value["xvfb_recipe"]["server_command"],
6848            "Xvfb :99 -screen 0 1440x900x24"
6849        );
6850        for key in [
6851            "macos_vm_per_agent",
6852            "share_host_app_into_guest",
6853            "hide_virtual_display_from_user",
6854            "windows_virtual_display",
6855            "nested_wayland_input_without_portal",
6856        ] {
6857            assert!(
6858                value["infeasible"][key]
6859                    .as_str()
6860                    .is_some_and(|s| !s.is_empty()),
6861                "infeasible.{key} missing"
6862            );
6863        }
6864    }
6865
6866    #[test]
6867    fn isolation_is_read_only_and_ignores_the_state_flags() {
6868        assert_eq!(action_label(&Action::Isolation), "isolation");
6869        assert!(matches!(
6870            post_state_kind(&Action::Isolation),
6871            PostKind::ReadOnly
6872        ));
6873        let backend = StubBackend::default();
6874        let response = dispatch_with_context(
6875            &backend,
6876            Action::Isolation,
6877            &CallContext {
6878                return_state: true,
6879                include_screenshot: true,
6880                ..CallContext::default()
6881            },
6882        );
6883        assert!(response.ok);
6884        assert!(response.to_value().get("state").is_none());
6885    }
6886
6887    #[test]
6888    fn the_isolation_action_parses_from_json() {
6889        let request = parse_request(json!({"id": 7, "action": "isolation"})).unwrap();
6890        assert_eq!(request.id, Some(json!(7)));
6891        assert!(matches!(request.action, Action::Isolation));
6892        // The shared `display` key is stripped, never decoded as an action field.
6893        let request = parse_request(json!({"action": "isolation", "display": "1"})).unwrap();
6894        assert_eq!(request.context.display.as_deref(), Some("1"));
6895    }
6896
6897    fn scroll_action(dx: Option<i32>, dy: Option<i32>) -> Action {
6898        Action::Scroll {
6899            x: None,
6900            y: None,
6901            dx,
6902            dy,
6903            direction: None,
6904            amount: None,
6905        }
6906    }
6907
6908    /// Compare a JSON float with a tolerance, so `0.05 * 3` is `0.15`.
6909    fn near(value: f64, expected: f64) -> bool {
6910        (value - expected).abs() < 1e-9
6911    }
6912
6913    fn tier_context(tier: &str) -> CallContext {
6914        CallContext {
6915            tier: Some(tier.to_string()),
6916            ..CallContext::default()
6917        }
6918    }
6919
6920    #[test]
6921    fn a_scroll_reports_the_tier_that_actually_moved_the_scroll_bar() {
6922        let backend = StubBackend::default();
6923        let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6924        assert_eq!(value["ok"], json!(true));
6925        assert_eq!(value["via"], json!("event"));
6926        assert_eq!(value["scrolled"], json!(true));
6927        assert_eq!(value["before"], json!(0.0));
6928        assert!(near(value["after"].as_f64().unwrap_or_default(), 0.15));
6929        assert_eq!(value["tier"], json!("event"));
6930        // A delivered wheel event has nothing to warn about.
6931        assert!(value.get("warnings").is_none());
6932    }
6933
6934    #[test]
6935    fn a_dropped_wheel_event_falls_back_to_the_ax_scroll_bar_and_says_so() {
6936        let backend = StubBackend {
6937            scroll_event_delivered: false,
6938            ..StubBackend::default()
6939        };
6940        let value = dispatch_with(&backend, scroll_action(None, Some(2))).to_value();
6941        assert_eq!(value["ok"], json!(true));
6942        assert_eq!(value["via"], json!("ax_scrollbar"));
6943        assert_eq!(value["scrolled"], json!(true));
6944        assert_eq!(value["before"], json!(0.0));
6945        assert!(near(value["after"].as_f64().unwrap_or_default(), 0.1));
6946        assert_eq!(
6947            value["warnings"][0],
6948            json!("wheel event not delivered to background app; used AX scroll bar")
6949        );
6950    }
6951
6952    #[test]
6953    fn a_bar_already_at_its_limit_is_ok_and_never_claims_a_scroll() {
6954        let backend = StubBackend {
6955            scroll_event_delivered: false,
6956            scroll_value: std::sync::Mutex::new(1.0),
6957            ..StubBackend::default()
6958        };
6959        let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6960        assert_eq!(value["ok"], json!(true));
6961        assert_eq!(value["scrolled"], json!(false));
6962        assert_eq!(value["before"], json!(1.0));
6963        assert_eq!(value["after"], json!(1.0));
6964        let warning = value["warnings"][0].as_str().unwrap_or_default();
6965        assert!(warning.contains("bottom"), "{warning}");
6966    }
6967
6968    #[test]
6969    fn a_scroll_that_cannot_be_verified_is_never_reported_as_a_scroll() {
6970        let backend = StubBackend {
6971            scroll_area_missing: true,
6972            ..StubBackend::default()
6973        };
6974        let response = dispatch_with_context(
6975            &backend,
6976            scroll_action(None, Some(3)),
6977            &tier_context("event"),
6978        );
6979        assert!(!response.ok);
6980        assert_eq!(code_of(&response), "execution_failed");
6981        let message = response.to_value()["error"]["message"]
6982            .as_str()
6983            .unwrap_or_default()
6984            .to_string();
6985        assert!(message.contains("cannot be verified"), "{message}");
6986        assert!(response.to_value().get("scrolled").is_none());
6987    }
6988
6989    #[test]
6990    fn an_ax_refusal_is_a_failure_and_not_a_scroll() {
6991        let backend = StubBackend {
6992            scroll_event_delivered: false,
6993            scroll_ax_refused: true,
6994            ..StubBackend::default()
6995        };
6996        let response = dispatch_with_context(
6997            &backend,
6998            scroll_action(None, Some(3)),
6999            &tier_context("event"),
7000        );
7001        assert!(!response.ok);
7002        assert_eq!(code_of(&response), "execution_failed");
7003        assert!(
7004            response.to_value()["error"]["message"]
7005                .as_str()
7006                .unwrap_or_default()
7007                .contains("refused"),
7008            "{:?}",
7009            response.to_value()
7010        );
7011    }
7012
7013    #[test]
7014    fn a_zero_delta_scroll_reports_no_scroll_and_does_not_panic() {
7015        // `--dx 0 --dy 0` addresses no axis at all. There is no bar to report,
7016        // so the answer is `scrolled: false` with null values, not a failure.
7017        for tier in ["event", "ax"] {
7018            let backend = StubBackend::default();
7019            let value = dispatch_with_context(
7020                &backend,
7021                scroll_action(Some(0), Some(0)),
7022                &tier_context(tier),
7023            )
7024            .to_value();
7025            assert_eq!(value["ok"], json!(true), "tier {tier}");
7026            assert_eq!(value["scrolled"], json!(false), "tier {tier}");
7027            assert_eq!(value["before"], json!(null), "tier {tier}");
7028            assert_eq!(value["after"], json!(null), "tier {tier}");
7029            assert!(value.get("warnings").is_none(), "tier {tier}");
7030        }
7031    }
7032
7033    #[test]
7034    fn an_explicit_ax_scroll_runs_the_scroll_bar_path_without_posting_an_event() {
7035        let backend = StubBackend::default();
7036        let response =
7037            dispatch_with_context(&backend, scroll_action(None, Some(3)), &tier_context("ax"));
7038        let value = response.to_value();
7039        assert_eq!(value["ok"], json!(true));
7040        assert_eq!(value["tier"], json!("ax"));
7041        assert_eq!(value["via"], json!("ax_scrollbar"));
7042        assert_eq!(value["scrolled"], json!(true));
7043        // No wheel event was posted, so the "not delivered" note would be false.
7044        assert!(
7045            value.get("warnings").is_none(),
7046            "unexpected warnings: {:?}",
7047            value["warnings"]
7048        );
7049        assert_eq!(
7050            *backend.tiers_run.lock().unwrap(),
7051            vec!["ax".to_string()],
7052            "tier ax must not fall back to activate"
7053        );
7054    }
7055
7056    #[test]
7057    fn require_background_never_reaches_the_activate_scroll_tier() {
7058        // The AX scroll-bar path needs no activation, so it is the only honest
7059        // answer when the wheel event is not delivered to a background app.
7060        let backend = StubBackend {
7061            scroll_event_delivered: false,
7062            scroll_area_missing: true,
7063            ..StubBackend::default()
7064        };
7065        let context = CallContext {
7066            tier: Some("event".into()),
7067            require_background: true,
7068            ..CallContext::default()
7069        };
7070        let response = dispatch_with_context(&backend, scroll_action(None, Some(3)), &context);
7071        assert!(!response.ok);
7072        assert_eq!(code_of(&response), "execution_failed");
7073        let tiers = backend.tiers_run.lock().unwrap().clone();
7074        assert!(
7075            !tiers.iter().any(|tier| tier == "activate"),
7076            "activate must never run: {tiers:?}"
7077        );
7078    }
7079
7080    #[test]
7081    fn a_horizontal_scroll_reports_the_horizontal_bar() {
7082        let backend = StubBackend {
7083            scroll_event_delivered: false,
7084            scroll_value: std::sync::Mutex::new(0.5),
7085            ..StubBackend::default()
7086        };
7087        let value = dispatch_with(&backend, scroll_action(Some(-2), None)).to_value();
7088        assert_eq!(value["via"], json!("ax_scrollbar"));
7089        assert_eq!(value["before"], json!(0.5));
7090        assert!(near(value["after"].as_f64().unwrap_or_default(), 0.4));
7091        assert_eq!(value["dx"], json!(-2));
7092    }
7093}
7094
7095// -----------------------------------------------------------------------------
7096// P10: CDP-backed browser helpers. Always return a structured error when the
7097// context flag is off or when the runtime refuses the URL.
7098//
7099// `allow(items_after_test_module)`: these helpers live after the test module
7100// for convenience; they are only here because P10 grew on top of the existing
7101// module. Adding a new module for them would cost more than it saves.
7102#[allow(clippy::items_after_test_module)]
7103fn browser_action_options(
7104    context: &CallContext,
7105    subject: &policy::Subject,
7106    target_id: Option<String>,
7107) -> crate::browser::ActionOptions {
7108    // Merge per-request `browser_blocked_urls` with whatever the loaded policy
7109    // says. The CLI sets the per-request field to an empty vec by default, so
7110    // the policy file is the source. `--policy` (and `XDG_CONFIG_HOME`)
7111    // configure the policy file location.
7112    let mut policy_blocked_urls = context.browser_blocked_urls.clone();
7113    if policy_blocked_urls.is_empty() {
7114        if let Ok(loaded) = policy::load_policy(context.policy.as_deref().map(Path::new)) {
7115            policy_blocked_urls = loaded.blocked_urls().to_vec();
7116        }
7117    }
7118    crate::browser::ActionOptions {
7119        debug_url: context.cdp_url.clone(),
7120        app: context.app.clone(),
7121        app_pid: subject.pid,
7122        target_id,
7123        url: None,
7124        selector: None,
7125        text: None,
7126        script: None,
7127        command_timeout: Duration::from_secs(30),
7128        policy_blocked_urls,
7129        policy_subject: Some(subject.clone()),
7130    }
7131}
7132
7133fn browser_require_cdp(context: &CallContext) -> Result<(), ComputerError> {
7134    if !context.cdp {
7135        return Err(ComputerError::Unsupported(
7136            "browser commands require --cdp (or `cdp: true` in the JSON request)".into(),
7137        ));
7138    }
7139    Ok(())
7140}
7141
7142fn browser_tabs(context: &CallContext, subject: &policy::Subject) -> Result<Value, ComputerError> {
7143    browser_require_cdp(context)?;
7144    let opts = browser_action_options(context, subject, None);
7145    let tabs = crate::browser::cdp::list_tabs(&opts)?;
7146    serde_json::to_value(&tabs)
7147        .map_err(|error| ComputerError::ExecutionFailed(format!("encode tabs: {error}")))
7148}
7149
7150fn browser_activate(
7151    context: &CallContext,
7152    subject: &policy::Subject,
7153    target_id: Option<String>,
7154) -> Result<Value, ComputerError> {
7155    browser_require_cdp(context)?;
7156    let target = target_id.ok_or_else(|| {
7157        ComputerError::InvalidArguments("browser activate requires --target".into())
7158    })?;
7159    let opts = browser_action_options(context, subject, Some(target));
7160    let active = crate::browser::cdp::activate_tab(&opts)?;
7161    Ok(json!({ "url": active }))
7162}
7163
7164fn browser_navigate(
7165    context: &CallContext,
7166    subject: &policy::Subject,
7167    url: String,
7168    target_id: Option<String>,
7169) -> Result<Value, ComputerError> {
7170    browser_require_cdp(context)?;
7171    let opts = browser_action_options(context, subject, target_id);
7172    let landed = crate::browser::cdp::navigate(&crate::browser::ActionOptions {
7173        url: Some(url.clone()),
7174        ..opts
7175    })?;
7176    Ok(json!({ "url": landed }))
7177}
7178
7179fn browser_click(
7180    context: &CallContext,
7181    subject: &policy::Subject,
7182    selector: String,
7183    target_id: Option<String>,
7184) -> Result<Value, ComputerError> {
7185    browser_require_cdp(context)?;
7186    let opts = browser_action_options(context, subject, target_id);
7187    crate::browser::cdp::click(&crate::browser::ActionOptions {
7188        selector: Some(selector.clone()),
7189        ..opts
7190    })?;
7191    Ok(json!({ "selector": selector }))
7192}
7193
7194fn browser_text(
7195    context: &CallContext,
7196    subject: &policy::Subject,
7197    selector: String,
7198    target_id: Option<String>,
7199) -> Result<Value, ComputerError> {
7200    browser_require_cdp(context)?;
7201    let opts = browser_action_options(context, subject, target_id);
7202    let text = crate::browser::cdp::read_inner_text(&crate::browser::ActionOptions {
7203        selector: Some(selector.clone()),
7204        ..opts
7205    })?;
7206    Ok(json!({ "selector": selector, "text": text }))
7207}
7208
7209fn browser_type(
7210    context: &CallContext,
7211    subject: &policy::Subject,
7212    text: String,
7213    selector: Option<String>,
7214    target_id: Option<String>,
7215) -> Result<Value, ComputerError> {
7216    browser_require_cdp(context)?;
7217    let opts = browser_action_options(context, subject, target_id);
7218    crate::browser::cdp::type_text(&crate::browser::ActionOptions {
7219        selector,
7220        text: Some(text.clone()),
7221        ..opts
7222    })?;
7223    // Audit captures the character count via `clipboard_chars`; the text body
7224    // never leaves this function.
7225    Ok(json!({ "chars": text.chars().count() as u64 }))
7226}
7227
7228fn browser_eval(
7229    context: &CallContext,
7230    subject: &policy::Subject,
7231    script: String,
7232    target_id: Option<String>,
7233) -> Result<Value, ComputerError> {
7234    browser_require_cdp(context)?;
7235    let opts = browser_action_options(context, subject, target_id);
7236    let value = crate::browser::cdp::eval(&crate::browser::ActionOptions {
7237        script: Some(script.clone()),
7238        ..opts
7239    })?;
7240    // The script body never leaves this function. The audit row only sees
7241    // its length and a 16-byte blake3-ish hash so an auditor can spot
7242    // duplicates without seeing the contents.
7243    let length = script.chars().count() as u64;
7244    use std::hash::{Hash, Hasher};
7245    let mut hasher = std::collections::hash_map::DefaultHasher::new();
7246    script.hash(&mut hasher);
7247    let hash = hasher.finish();
7248    Ok(json!({
7249        "result": value,
7250        "script_chars": length,
7251        "script_hash": format!("{hash:016x}"),
7252    }))
7253}