Skip to main content

computer_use/
protocol.rs

1//! Actions, JSON requests, and the shared dispatcher.
2
3use std::fs;
4use std::panic::{catch_unwind, AssertUnwindSafe};
5use std::path::{Path, PathBuf};
6use std::sync::atomic::{AtomicU64, Ordering};
7use std::thread;
8use std::time::{Duration, SystemTime, UNIX_EPOCH};
9
10use serde::de::{self, Deserializer};
11use serde::ser::{Serialize, Serializer};
12use serde::{Deserialize, Serialize as SerializeDerive};
13use serde_json::{json, Map, Value};
14
15use crate::attempt::{self, AttemptFlags};
16use crate::backend::clipboard::ClipboardContents;
17use crate::backend::create_desktop_backend;
18use crate::backend::{validate_key, DesktopBackend, WindowTarget};
19use crate::cache;
20use crate::element::{self, ElementError, ElementOp, ElementOutcome, ElementRequest};
21use crate::error::{ComputerError, ErrorBody};
22use crate::image_util::{self, encode_base64};
23use crate::input_session::{self, InputSession};
24use crate::keys::{parse_key_combo, parse_key_sequence};
25use crate::policy;
26use crate::state::collect_state_for;
27use crate::target::{
28    filter_windows_for_pid, parse_app_selector, parse_window_selector, resolve_app, resolve_window,
29    WindowSelector,
30};
31use crate::tier::{self, InputKind, KeyStroke, TierName};
32use crate::tree::TreeQuery;
33use crate::types::{
34    parse_path, region_inside, Frame, ImageFormat, Modifier, MouseButton, Point, Region,
35    ScrollDirection,
36};
37use crate::vision::{OcrBlock, OcrResult};
38
39const MAX_WAIT_MS: u128 = 24 * 60 * 60 * 1000;
40
41/// Default recursion depth for `menu_list`. Matches the AXMenuBar chain
42/// (menu bar -> menu bar item -> menu -> submenu) plus one extra level.
43pub const MENU_DEFAULT_DEPTH: u32 = 4;
44
45/// Maximum recursion depth accepted by `menu_list`. AX nesting beyond this
46/// stops with a `*` placeholder so the report never explodes.
47pub const MENU_MAX_DEPTH: u32 = 8;
48
49/// Default timeout for a `shortcuts run` invocation, in seconds.
50pub const SHORTCUTS_TIMEOUT_SECS: u64 = 60;
51
52/// Default timeout for an `osascript` invocation, in seconds.
53pub const APPLE_EVENT_TIMEOUT_SECS: u64 = 30;
54
55/// One desktop action. Serialized with an `action` tag.
56///
57/// ```json
58/// {"action":"click","x":10,"y":20}
59/// ```
60#[derive(Debug, Clone, PartialEq, SerializeDerive, Deserialize)]
61#[serde(tag = "action", rename_all = "snake_case")]
62pub enum Action {
63    /// Capture the screen or a region.
64    Screenshot {
65        /// File path. A temp path is used when omitted.
66        #[serde(default, skip_serializing_if = "Option::is_none")]
67        output: Option<String>,
68        /// Logical region, as `x,y,width,height` or an object.
69        #[serde(default, skip_serializing_if = "Option::is_none")]
70        region: Option<Region>,
71        /// `png` (default) or `jpeg`.
72        #[serde(default, skip_serializing_if = "Option::is_none")]
73        format: Option<ImageFormat>,
74        /// Scale the image down so its width is at most this many pixels.
75        ///
76        /// Absent means 1440. `0` keeps the source width.
77        #[serde(default, skip_serializing_if = "Option::is_none")]
78        max_width: Option<u32>,
79        /// Letterbox into `WxH` pixels, for example `1440x900`.
80        #[serde(default, skip_serializing_if = "Option::is_none")]
81        canvas: Option<String>,
82        /// Include standard base64 bytes in `data`.
83        #[serde(default, skip_serializing_if = "is_false")]
84        base64: bool,
85        /// Stamp numbered element rectangles on the image. tree indices
86        /// (`n`) on a window target; OCR indices (`on`) when a tree cannot
87        /// be read. `--marks` from the CLI turns this on.
88        #[serde(default, skip_serializing_if = "is_false")]
89        marks: bool,
90    },
91    /// Move to a point and click, or press an element when `element` is set.
92    Click {
93        /// Logical x. Omit it when `element` is set.
94        #[serde(default, skip_serializing_if = "Option::is_none")]
95        x: Option<i32>,
96        /// Logical y. Omit it when `element` is set.
97        #[serde(default, skip_serializing_if = "Option::is_none")]
98        y: Option<i32>,
99        /// Button. Defaults to left.
100        #[serde(default)]
101        button: MouseButton,
102        /// Click twice when true. With `element`, this is `press` count 2.
103        #[serde(default, skip_serializing_if = "is_false")]
104        double: bool,
105        /// Repeat count. 1, 2, or 3. `double: true` means 2 when `count` is 1.
106        #[serde(default = "default_press_count", skip_serializing_if = "is_one")]
107        count: u8,
108    },
109    /// Click three times.
110    #[serde(rename = "triple_click", alias = "triple-click")]
111    TripleClick {
112        /// Logical x. Omit it when `element` is set.
113        #[serde(default, skip_serializing_if = "Option::is_none")]
114        x: Option<i32>,
115        /// Logical y. Omit it when `element` is set.
116        #[serde(default, skip_serializing_if = "Option::is_none")]
117        y: Option<i32>,
118        /// Button. Defaults to left.
119        #[serde(default)]
120        button: MouseButton,
121    },
122    /// Click twice. Equivalent to `click` with `double: true`.
123    #[serde(rename = "double_click")]
124    DoubleClick {
125        /// Logical x. Omit it when `element` is set.
126        #[serde(default, skip_serializing_if = "Option::is_none")]
127        x: Option<i32>,
128        /// Logical y. Omit it when `element` is set.
129        #[serde(default, skip_serializing_if = "Option::is_none")]
130        y: Option<i32>,
131        /// Button. Defaults to left.
132        #[serde(default)]
133        button: MouseButton,
134    },
135    /// Press an element. `count` is 1, 2, or 3.
136    Press {
137        /// Repeat count. Defaults to 1.
138        #[serde(default = "default_press_count")]
139        count: u8,
140        /// Mouse button. Left uses `AXPress`. Right uses `AXShowMenu` when advertised.
141        #[serde(default)]
142        button: MouseButton,
143    },
144    /// Button down at the first point, dragged events through the rest, up at the last.
145    Drag {
146        /// At least two points. A JSON array, or a flat `x,y,x,y` string.
147        #[serde(alias = "points", deserialize_with = "deserialize_path")]
148        path: Vec<Point>,
149        /// Mouse button. Defaults to left.
150        #[serde(default)]
151        button: MouseButton,
152    },
153    /// Button down. A later `mouse_up`, or the end of a one-shot command, releases it.
154    #[serde(alias = "mouse-down")]
155    MouseDown {
156        /// Logical x.
157        x: i32,
158        /// Logical y.
159        y: i32,
160        /// Mouse button. Defaults to left.
161        #[serde(default)]
162        button: MouseButton,
163    },
164    /// Button up.
165    #[serde(alias = "mouse-up")]
166    MouseUp {
167        /// Logical x.
168        x: i32,
169        /// Logical y.
170        y: i32,
171        /// Mouse button. Defaults to left.
172        #[serde(default)]
173        button: MouseButton,
174    },
175    /// Key down, sleep `ms`, key up. One combination. `ms` above 30000 is refused.
176    #[serde(alias = "hold-key")]
177    HoldKey {
178        /// One combo such as `shift` or `ctrl+alt+Delete`.
179        keys: String,
180        /// Milliseconds to hold.
181        ms: u64,
182    },
183    /// Read the clipboard. Text wins over an image. Empty is `kind: empty`.
184    ClipboardGet,
185    /// Write the clipboard. Confirmed. The audit stores the character count, not the text.
186    #[serde(alias = "clipboard-set")]
187    ClipboardSet {
188        /// Text to write. Empty is `invalid_arguments`.
189        #[serde(default, skip_serializing_if = "Option::is_none")]
190        text: Option<String>,
191        /// PNG path. Used when `text` is absent.
192        #[serde(default, skip_serializing_if = "Option::is_none")]
193        image: Option<String>,
194    },
195    /// Screenshot of a region. A window target crops that region in window-local points.
196    Zoom {
197        /// File path. A temp path is used when omitted.
198        #[serde(default, skip_serializing_if = "Option::is_none")]
199        output: Option<String>,
200        /// Logical region, as `x,y,width,height` or an object. Required.
201        region: Region,
202        /// `png` (default) or `jpeg`.
203        #[serde(default, skip_serializing_if = "Option::is_none")]
204        format: Option<ImageFormat>,
205        /// Scale the image down so its width is at most this many pixels.
206        #[serde(default, skip_serializing_if = "Option::is_none")]
207        max_width: Option<u32>,
208        /// Letterbox into `WxH` pixels, for example `1440x900`.
209        #[serde(default, skip_serializing_if = "Option::is_none")]
210        canvas: Option<String>,
211        /// Include standard base64 bytes in `data`.
212        #[serde(default, skip_serializing_if = "is_false")]
213        base64: bool,
214    },
215    /// Set `AXFocused` on an element.
216    Focus,
217    /// Set an element's value. Refused for secure fields.
218    #[serde(alias = "set-value")]
219    SetValue {
220        /// New value.
221        value: String,
222    },
223    /// Set the selected character range. Refused for secure fields.
224    #[serde(alias = "select-text")]
225    SelectText {
226        /// First selected character.
227        start: i64,
228        /// Character after the selection.
229        end: i64,
230    },
231    /// Perform an allowlisted accessibility action.
232    #[serde(alias = "perform-action")]
233    PerformAction {
234        /// Action name such as `AXShowMenu`.
235        name: String,
236    },
237    /// Scroll an element into view.
238    #[serde(
239        alias = "scroll-to",
240        alias = "scroll_to_visible",
241        alias = "scroll-to-visible"
242    )]
243    ScrollTo,
244    /// Move the pointer without clicking.
245    #[serde(alias = "move_mouse")]
246    Move {
247        /// Logical x.
248        x: i32,
249        /// Logical y.
250        y: i32,
251    },
252    /// Type text into the focused control.
253    Type {
254        /// Text to type.
255        text: String,
256    },
257    /// Press a key combination such as `cmd+shift+4`.
258    Key {
259        /// Combo string.
260        keys: String,
261    },
262    /// Scroll the wheel. Positive `dy` is down. Positive `dx` is right.
263    Scroll {
264        /// Optional pointer x. Must be paired with `y`.
265        #[serde(default, skip_serializing_if = "Option::is_none")]
266        x: Option<i32>,
267        /// Optional pointer y. Must be paired with `x`.
268        #[serde(default, skip_serializing_if = "Option::is_none")]
269        y: Option<i32>,
270        /// Horizontal wheel steps.
271        #[serde(default, alias = "scroll_x", skip_serializing_if = "Option::is_none")]
272        dx: Option<i32>,
273        /// Vertical wheel steps.
274        #[serde(default, alias = "scroll_y", skip_serializing_if = "Option::is_none")]
275        dy: Option<i32>,
276        /// Direction used with `amount`.
277        #[serde(default, skip_serializing_if = "Option::is_none")]
278        direction: Option<ScrollDirection>,
279        /// Wheel steps used with `direction`. Defaults to 1.
280        #[serde(default, skip_serializing_if = "Option::is_none")]
281        amount: Option<i32>,
282    },
283    /// Screen, cursor, frontmost window, and a best-effort accessibility tree.
284    #[serde(alias = "get-state")]
285    GetState,
286    /// Sleep, then return.
287    Wait {
288        /// Milliseconds. Added to `seconds` when both are set.
289        #[serde(default, skip_serializing_if = "Option::is_none")]
290        ms: Option<u64>,
291        /// Seconds, including fractions. Added to `ms`.
292        #[serde(default, skip_serializing_if = "Option::is_none")]
293        seconds: Option<f64>,
294    },
295    /// List displays without capturing pixels.
296    Displays,
297    /// Report what this build can isolate and what it cannot.
298    Isolation,
299    /// List running applications. `all` includes accessory and prohibited apps.
300    ListApps {
301        /// Include accessory and prohibited activation policies.
302        #[serde(default, rename = "all")]
303        all: bool,
304    },
305    /// List on-screen windows. `app` comes from the shared request context.
306    ListWindows,
307    /// Launch an application named by the shared `app` field.
308    Launch,
309    /// Activate an application named by the shared `app` field.
310    Activate,
311    /// Ask an application to quit. `force` requests a forced terminate.
312    Quit {
313        /// Force-quit when the platform supports it. Still requires confirmation.
314        #[serde(default, skip_serializing_if = "is_false")]
315        force: bool,
316    },
317    /// Report whether policy would allow `app`. Does not perform an action.
318    #[serde(rename = "policy_check")]
319    PolicyCheck,
320    /// Pause mutating actions until `resume`.
321    Pause,
322    /// Clear the pause flag.
323    Resume,
324    /// Report permissions and the TCC identity.
325    ///
326    /// The process exits 0 when the report is produced. `permissions_ok` in the
327    /// body is the permission aggregate. `prompt` asks macOS to show dialogs.
328    /// `request` is an alias of `prompt`.
329    Doctor {
330        /// Prompt for Accessibility, Screen Recording, Post Event, and Input Monitoring.
331        #[serde(default, alias = "request")]
332        prompt: bool,
333        /// Open the first missing System Settings pane. macOS only.
334        #[serde(default)]
335        open_settings: bool,
336        /// Linux only: force X11 detection even when WAYLAND_DISPLAY is set.
337        /// Other platforms ignore the field so the dispatcher can pass it
338        /// unconditionally.
339        #[serde(default)]
340        x11: bool,
341    },
342    /// Numbered accessibility tree for the key window.
343    Tree,
344    /// Restore Electron accessibility flags and stop observers.
345    SessionEnd,
346    /// Read the pointer. The Anthropic adapter maps `cursor_position` here.
347    CursorPosition,
348    /// Run several actions that share one target and one cache revision sequence.
349    Batch {
350        /// Stop after the first failed step. Defaults to true.
351        #[serde(default = "default_stop_on_error")]
352        stop_on_error: bool,
353        /// Native action objects. A nested `batch` is rejected at dispatch.
354        actions: Vec<Value>,
355    },
356    /// Remove empty session directories and stale ones.
357    #[serde(alias = "session-prune")]
358    SessionPrune {
359        /// Report matches without deleting them.
360        #[serde(default)]
361        dry_run: bool,
362        /// Age in seconds after which a dead owner's session is stale. Absent means 24 hours.
363        #[serde(default, skip_serializing_if = "Option::is_none")]
364        ttl_secs: Option<u64>,
365    },
366    /// Walk an application's menu bar. Reads AXMenuBar -> AXMenuBarItem -> AXMenu.
367    ///
368    /// `--app` selects the running app. The default depth is 4; `depth` caps it.
369    /// Matches are case-insensitive. Disabled items are still reported.
370    #[serde(alias = "menu_list", alias = "menu-list")]
371    MenuList {
372        /// Override the shared `--app` flag with one of these variants: name, bundle id, or pid.
373        #[serde(default, skip_serializing_if = "Option::is_none")]
374        app: Option<String>,
375        /// Maximum depth below the menu bar. Default: 4. Caps recursive submenus.
376        #[serde(default, skip_serializing_if = "Option::is_none")]
377        depth: Option<u32>,
378    },
379    /// Click a menu item by `">"`-separated titles.
380    ///
381    /// `--app` selects the running app. `--path` is a list of titles such as
382    /// `"File > Save As > Save"`. Matches are case-insensitive. Disabled items
383    /// are not pressed.
384    #[serde(alias = "menu_click", alias = "menu-click")]
385    MenuClick {
386        /// Override the shared `--app` flag.
387        #[serde(default, skip_serializing_if = "Option::is_none")]
388        app: Option<String>,
389        /// Path of menu titles separated by `>`.
390        path: String,
391    },
392    /// Send an Apple event via `osascript`. macOS only.
393    ///
394    /// `--app` is recorded and used by the deny policy; the script body decides
395    /// which application the event targets. `--language` is `applescript` (the
396    /// default) or `javascript`. The script body is never logged verbatim.
397    #[serde(alias = "apple_event", alias = "apple-event")]
398    AppleEvent {
399        /// Override the shared `--app` flag.
400        #[serde(default, skip_serializing_if = "Option::is_none")]
401        app: Option<String>,
402        /// Script body passed to `osascript -e`. Never logged verbatim.
403        script: String,
404        /// `applescript` (default) or `javascript`.
405        #[serde(default, skip_serializing_if = "Option::is_none")]
406        language: Option<AppleEventLanguage>,
407    },
408    /// Run a Shortcuts workflow by name. macOS only.
409    ///
410    /// The `shortcuts` binary is called with the workflow name as a single
411    /// argument. A missing binary is `execution_failed`.
412    #[serde(alias = "shortcuts_run", alias = "shortcuts-run")]
413    ShortcutsRun {
414        /// Workflow name.
415        name: String,
416    },
417    /// List the Dock items. macOS only.
418    #[serde(alias = "dock_list", alias = "dock-list")]
419    DockList {},
420    /// Click a Dock item by title. macOS only.
421    #[serde(alias = "dock_click", alias = "dock-click")]
422    DockClick {
423        /// Dock item title.
424        title: String,
425    },
426    /// Group the on-screen windows by the screen that contains each window's center.
427    ///
428    /// Mission Control moves are not attempted; the report only describes the
429    /// current layout. The skylight feature, when active, may move windows
430    /// between spaces; otherwise the action reports the visible state only.
431    #[serde(alias = "spaces")]
432    Spaces {},
433    /// List the tabs the running Chrome exposes. Requires `--features cdp`.
434    /// Without the feature this returns `unsupported`.
435    #[serde(rename = "browser_tabs", alias = "browser-tabs")]
436    BrowserTabs {},
437    /// Activate a tab by `target_id`. Requires `--features cdp`.
438    #[serde(rename = "browser_activate", alias = "browser-activate")]
439    BrowserActivate {
440        /// CDP `targetId` to bring to the front.
441        #[serde(default, skip_serializing_if = "Option::is_none")]
442        target_id: Option<String>,
443    },
444    /// Navigate the named tab (or first page) to `url`. The URL is checked
445    /// against `[browser].blocked_urls` before the websocket is touched.
446    #[serde(rename = "browser_navigate", alias = "browser-navigate")]
447    BrowserNavigate {
448        /// Destination URL.
449        url: String,
450        /// Restrict to one tab. Optional.
451        #[serde(default, skip_serializing_if = "Option::is_none")]
452        target_id: Option<String>,
453    },
454    /// Click an element matched by CSS `selector`. Requires `--features cdp`.
455    #[serde(rename = "browser_click", alias = "browser-click")]
456    BrowserClick {
457        /// CSS selector.
458        selector: String,
459        /// Restrict to one tab. Optional.
460        #[serde(default, skip_serializing_if = "Option::is_none")]
461        target_id: Option<String>,
462    },
463    /// Read the `innerText` of the first element matched by CSS `selector`.
464    /// Requires `--features cdp`. The current page URL is checked against
465    /// `[browser].blocked_urls` first; a selector that matches nothing returns
466    /// `element_not_found`.
467    #[serde(rename = "browser_text", alias = "browser-text")]
468    BrowserText {
469        /// CSS selector whose `innerText` should be returned.
470        selector: String,
471        /// Restrict to one tab. Optional.
472        #[serde(default, skip_serializing_if = "Option::is_none")]
473        target_id: Option<String>,
474    },
475    /// Type `text` into the element matched by `selector` (or the page body
476    /// when no selector is given). Requires `--features cdp`. Classified like
477    /// [`Action::Type`] for audit: the body is never stored, only its
478    /// character count. The current page URL is checked against
479    /// `[browser].blocked_urls` first.
480    #[serde(rename = "browser_type", alias = "browser-type")]
481    BrowserType {
482        /// Text to type.
483        text: String,
484        /// Optional CSS selector. When omitted the page body receives input.
485        #[serde(default, skip_serializing_if = "Option::is_none")]
486        selector: Option<String>,
487        /// Restrict to one tab. Optional.
488        #[serde(default, skip_serializing_if = "Option::is_none")]
489        target_id: Option<String>,
490    },
491    /// Evaluate `script` in the page context and return the JSON value.
492    /// Requires `--features cdp`. The script body is never logged verbatim;
493    /// only its length and a small hash appear in the audit row.
494    #[serde(rename = "browser_eval", alias = "browser-eval")]
495    BrowserEval {
496        /// JavaScript source. Quoted on the CLI via `--eval`; here it is just
497        /// a string field that the caller has already prepared.
498        script: String,
499        /// Restrict to one tab. Optional.
500        #[serde(default, skip_serializing_if = "Option::is_none")]
501        target_id: Option<String>,
502    },
503    /// Run OCR on the most recent capture for `--app`/window target. macOS
504    /// only. Linux returns `unsupported`; Windows is a stub until P12.
505    #[serde(rename = "ocr")]
506    Ocr {
507        /// Re-run OCR even when `--app` already produced it. Defaults to false.
508        #[serde(default, skip_serializing_if = "is_false")]
509        force: bool,
510    },
511}
512
513fn is_false(value: &bool) -> bool {
514    !*value
515}
516
517fn is_one(value: &u8) -> bool {
518    *value == 1
519}
520
521fn default_press_count() -> u8 {
522    1
523}
524
525/// Language of an Apple event script. macOS only.
526///
527/// `applescript` is the default. `javascript` selects the JavaScript for
528/// Automation (`JXA`) runtime in `osascript`.
529#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDerive, Deserialize)]
530#[serde(rename_all = "snake_case")]
531pub enum AppleEventLanguage {
532    /// AppleScript. Default.
533    #[default]
534    #[serde(alias = "applescript", alias = "osa")]
535    AppleScript,
536    /// JavaScript for Automation. `osascript -l JavaScript`.
537    #[serde(alias = "javascript", alias = "js", alias = "JavaScript")]
538    JavaScript,
539}
540
541impl AppleEventLanguage {
542    /// Short tag passed to `osascript -l`.
543    pub fn as_str(&self) -> &'static str {
544        match self {
545            Self::AppleScript => "AppleScript",
546            Self::JavaScript => "JavaScript",
547        }
548    }
549}
550
551fn deserialize_path<'de, D>(deserializer: D) -> Result<Vec<Point>, D::Error>
552where
553    D: Deserializer<'de>,
554{
555    let value = Value::deserialize(deserializer)?;
556    match value {
557        Value::String(text) => parse_path(&text).map_err(de::Error::custom),
558        Value::Array(_) => serde_json::from_value(value).map_err(de::Error::custom),
559        _ => Err(de::Error::custom(
560            "path must be an array of points or an x,y,x,y string",
561        )),
562    }
563}
564
565fn default_stop_on_error() -> bool {
566    true
567}
568
569/// Flags shared by every action. JSON keys are removed before the action is decoded.
570#[derive(Debug, Clone, PartialEq, Default)]
571pub struct CallContext {
572    /// Target application: name, bundle id, or pid.
573    pub app: Option<String>,
574    /// Target window: title, z-order index, or `id:<number>`.
575    pub window: Option<String>,
576    /// Element index in the current revision. Used from P3.
577    pub element: Option<u32>,
578    /// Emit a tree diff when a baseline exists.
579    pub diff: bool,
580    /// Replace the tree baseline with the full tree.
581    pub full: bool,
582    /// Attach post-action state. Used from P5.
583    pub return_state: bool,
584    /// Attach a screenshot with post-action state. Used from P5.
585    pub include_screenshot: bool,
586    /// Coordinate space name. Used from P5.
587    pub coord: Option<String>,
588    /// Cache namespace. Defaults to the process id in later phases.
589    pub session: Option<String>,
590    /// Input tier name. Used from P7.
591    pub tier: Option<String>,
592    /// Adapter name. Used from P5.
593    pub adapter: Option<String>,
594    /// Display filter. Used from P14.
595    pub display: Option<String>,
596    /// Fail when the element is outside the window and scroll-into-view cannot run.
597    pub require_visible: bool,
598    /// User policy file from `--policy`. Absent uses the default path.
599    pub policy: Option<String>,
600    /// Confirm hook from `--confirm-cmd`. Wins over the environment and the policy file.
601    pub confirm_cmd: Option<String>,
602    /// Skip `foreground.lock` for tier `activate`.
603    pub allow_foreground: bool,
604    /// Refuse tier `activate`, and treat cursor motion as failure.
605    pub require_background: bool,
606    /// Do not send the agent point to the overlay helper.
607    pub no_overlay: bool,
608    /// Route browser commands through CDP. The dispatcher refuses the request
609    /// when this flag is off (`invalid_arguments` for the CLI, `unsupported`
610    /// for the JSON path).
611    pub cdp: bool,
612    /// Override the discovered CDP websocket URL. Wins over `--app` argv
613    /// discovery. Accepts `ws://`, `wss://`, `http://`, `https://`.
614    pub cdp_url: Option<String>,
615    /// Cached `[browser].blocked_urls` from the loaded policy. The dispatcher
616    /// refuses the websocket connection (and the page activation) when the
617    /// destination URL is in here.
618    pub browser_blocked_urls: Vec<String>,
619}
620
621/// A JSON request: an optional `id`, shared flags, and an [`Action`].
622#[derive(Debug, Clone, PartialEq)]
623pub struct Request {
624    /// Caller-supplied id, echoed on the response when present.
625    pub id: Option<Value>,
626    /// Action to run.
627    pub action: Action,
628    /// Flags that were siblings of `action`.
629    pub context: CallContext,
630}
631
632/// Parse one JSON object. `id` and the shared flag keys are removed before the action is decoded.
633pub fn parse_request(mut value: Value) -> Result<Request, ComputerError> {
634    if !value.is_object() {
635        return Err(ComputerError::InvalidArguments(
636            "request must be a JSON object".into(),
637        ));
638    }
639    let object = value
640        .as_object_mut()
641        .expect("object check above guarantees a map");
642    let id = object.remove("id");
643    let context = take_context(object)?;
644    let action = serde_json::from_value(value)
645        .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
646    Ok(Request {
647        id,
648        action,
649        context,
650    })
651}
652
653fn take_context(object: &mut Map<String, Value>) -> Result<CallContext, ComputerError> {
654    Ok(CallContext {
655        app: take_stringish(object, "app")?,
656        window: take_stringish(object, "window")?,
657        element: take_optional_u32(object, "element")?,
658        diff: take_bool(object, "diff")?,
659        full: take_bool(object, "full")?,
660        return_state: take_bool(object, "return_state")?,
661        include_screenshot: take_bool(object, "include_screenshot")?,
662        coord: take_stringish(object, "coord")?,
663        session: take_stringish(object, "session")?,
664        tier: take_stringish(object, "tier")?,
665        adapter: take_stringish(object, "adapter")?,
666        display: take_stringish(object, "display")?,
667        require_visible: take_bool(object, "require_visible")?,
668        policy: None,
669        confirm_cmd: None,
670        allow_foreground: take_bool(object, "allow_foreground")?,
671        require_background: take_bool(object, "require_background")?,
672        no_overlay: take_bool(object, "no_overlay")?,
673        cdp: take_bool(object, "cdp")?,
674        cdp_url: take_stringish(object, "cdp_url")?,
675        browser_blocked_urls: take_string_list(object, "browser_blocked_urls")?,
676    })
677}
678
679fn take_stringish(
680    object: &mut Map<String, Value>,
681    key: &str,
682) -> Result<Option<String>, ComputerError> {
683    match object.remove(key) {
684        None | Some(Value::Null) => Ok(None),
685        Some(Value::String(text)) => Ok(Some(text)),
686        Some(Value::Number(number)) => Ok(Some(number.to_string())),
687        Some(_) => Err(ComputerError::InvalidArguments(format!(
688            "{key} must be a string or number"
689        ))),
690    }
691}
692
693fn take_bool(object: &mut Map<String, Value>, key: &str) -> Result<bool, ComputerError> {
694    match object.remove(key) {
695        None | Some(Value::Null) => Ok(false),
696        Some(Value::Bool(value)) => Ok(value),
697        Some(_) => Err(ComputerError::InvalidArguments(format!(
698            "{key} must be a boolean"
699        ))),
700    }
701}
702
703fn take_string_list(
704    object: &mut Map<String, Value>,
705    key: &str,
706) -> Result<Vec<String>, ComputerError> {
707    match object.remove(key) {
708        None | Some(Value::Null) => Ok(Vec::new()),
709        Some(Value::Array(values)) => values
710            .into_iter()
711            .map(|value| match value {
712                Value::String(text) => Ok(text),
713                _ => Err(ComputerError::InvalidArguments(format!(
714                    "{key} entries must be strings"
715                ))),
716            })
717            .collect(),
718        Some(_) => Err(ComputerError::InvalidArguments(format!(
719            "{key} must be a list of strings"
720        ))),
721    }
722}
723
724fn take_optional_u32(
725    object: &mut Map<String, Value>,
726    key: &str,
727) -> Result<Option<u32>, ComputerError> {
728    match object.remove(key) {
729        None | Some(Value::Null) => Ok(None),
730        Some(Value::Number(number)) => number
731            .as_u64()
732            .and_then(|value| u32::try_from(value).ok())
733            .map(Some)
734            .ok_or_else(|| ComputerError::InvalidArguments(format!("{key} is out of range"))),
735        Some(Value::String(text)) => text
736            .parse::<u32>()
737            .map(Some)
738            .map_err(|_| ComputerError::InvalidArguments(format!("{key} must be an integer"))),
739        Some(_) => Err(ComputerError::InvalidArguments(format!(
740            "{key} must be an integer"
741        ))),
742    }
743}
744
745/// Decode one JSON line and dispatch it.
746///
747/// Parse errors return `ok: false` and do not panic. The adapter is native
748/// unless the object sets `adapter`.
749pub fn handle_request_line(line: &str) -> Response {
750    handle_request_line_with(line, None)
751}
752
753/// Options from the CLI that JSON requests do not carry.
754#[derive(Debug, Clone, Copy, Default)]
755pub struct LineOptions<'a> {
756    /// Process `--adapter` flag. A request field wins.
757    pub adapter: Option<&'a str>,
758    /// Process `--policy` path.
759    pub policy: Option<&'a str>,
760    /// Process `--confirm-cmd` program.
761    pub confirm_cmd: Option<&'a str>,
762    /// Process `--tier`. A request field wins. `--synthetic-focus` arrives as `synthetic`.
763    pub tier: Option<&'a str>,
764    /// Process `--allow-foreground`. Combined with the request by or.
765    pub allow_foreground: bool,
766    /// Process `--require-background`. Combined with the request by or.
767    pub require_background: bool,
768    /// Process `--no-overlay`. Combined with the request by or.
769    pub no_overlay: bool,
770    /// Process `--cdp` flag. Combined with the request by or.
771    pub cdp: bool,
772    /// Process `--cdp-url` flag. A request field wins.
773    pub cdp_url: Option<&'a str>,
774}
775
776/// Decode one JSON line. `cli_adapter` is the process `--adapter` flag.
777///
778/// A request field `adapter` wins. `native` skips translation. There is no
779/// detection when both are absent.
780pub fn handle_request_line_with(line: &str, cli_adapter: Option<&str>) -> Response {
781    handle_request_line_opts(
782        line,
783        LineOptions {
784            adapter: cli_adapter,
785            ..LineOptions::default()
786        },
787    )
788}
789
790/// Decode one JSON line and apply CLI `--policy` and `--confirm-cmd`.
791pub fn handle_request_line_opts(line: &str, options: LineOptions<'_>) -> Response {
792    let value = match serde_json::from_str::<Value>(line) {
793        Ok(value) => value,
794        Err(error) => {
795            return Response::failure("invalid_arguments", format!("invalid JSON: {error}"))
796        }
797    };
798    let id = value.get("id").cloned();
799    let echo = match effective_adapter(&value, options.adapter) {
800        Ok(name) => name,
801        Err(error) => return Response::from_error(error).with_id(id),
802    };
803    match build_request(value, echo.as_deref()) {
804        Ok(mut request) => {
805            if let Some(policy) = options.policy {
806                request.context.policy = Some(policy.to_string());
807            }
808            if let Some(confirm) = options.confirm_cmd {
809                request.context.confirm_cmd = Some(confirm.to_string());
810            }
811            if request.context.tier.is_none() {
812                if let Some(tier) = options.tier {
813                    request.context.tier = Some(tier.to_string());
814                }
815            }
816            request.context.allow_foreground |= options.allow_foreground;
817            request.context.require_background |= options.require_background;
818            request.context.no_overlay |= options.no_overlay;
819            request.context.cdp |= options.cdp;
820            if request.context.cdp_url.is_none() {
821                if let Some(url) = options.cdp_url {
822                    request.context.cdp_url = Some(url.to_string());
823                }
824            }
825            dispatch_context(request.action, &request.context).with_id(request.id)
826        }
827        Err(error) => {
828            let mut response = Response::from_error(error).with_id(id);
829            stamp_adapter_name(&mut response, echo.as_deref());
830            response
831        }
832    }
833}
834
835fn effective_adapter(
836    value: &Value,
837    cli_adapter: Option<&str>,
838) -> Result<Option<String>, ComputerError> {
839    if !value.is_object() {
840        return Ok(None);
841    }
842    let from_request = match value.get("adapter") {
843        None | Some(Value::Null) => None,
844        Some(Value::String(text)) => Some(text.as_str()),
845        Some(_) => {
846            return Err(ComputerError::InvalidArguments(
847                "adapter must be a string".into(),
848            ))
849        }
850    };
851    let name = from_request.or(cli_adapter);
852    match name {
853        None | Some("native") => Ok(None),
854        Some("openai" | "anthropic") => Ok(name.map(str::to_string)),
855        Some(other) => Err(ComputerError::InvalidArguments(format!(
856            "unknown adapter {other:?}"
857        ))),
858    }
859}
860
861fn build_request(value: Value, adapter: Option<&str>) -> Result<Request, ComputerError> {
862    let Some(name) = adapter else {
863        return parse_request(value);
864    };
865    if !value.is_object() {
866        return Err(ComputerError::InvalidArguments(
867            "request must be a JSON object".into(),
868        ));
869    }
870    let mut value = value;
871    let object = value
872        .as_object_mut()
873        .expect("object check above guarantees a map");
874    let id = object.remove("id");
875    let mut context = take_context(object)?;
876    context.coord = Some("image".into());
877    context.adapter = Some(name.to_string());
878    let action = crate::adapters::translate(name, &value)?;
879    Ok(Request {
880        id,
881        action,
882        context,
883    })
884}
885
886fn stamp_adapter_name(response: &mut Response, adapter: Option<&str>) {
887    if let Some(name @ ("openai" | "anthropic")) = adapter {
888        response.fields.insert("adapter".into(), json!(name));
889    }
890}
891
892/// Run `action` on the platform backend.
893pub fn dispatch(action: Action) -> Response {
894    dispatch_context(action, &CallContext::default())
895}
896
897/// Run `action` with shared flags on the platform backend.
898pub fn dispatch_context(action: Action, context: &CallContext) -> Response {
899    let backend = create_desktop_backend();
900    dispatch_with_context(backend.as_ref(), action, context)
901}
902
903/// Run `action` on `backend`. Panics inside the backend become an error response.
904pub fn dispatch_with(backend: &dyn DesktopBackend, action: Action) -> Response {
905    dispatch_with_context(backend, action, &CallContext::default())
906}
907
908/// Run `action` on `backend` with shared flags.
909pub fn dispatch_with_context(
910    backend: &dyn DesktopBackend,
911    action: Action,
912    context: &CallContext,
913) -> Response {
914    let mut response = match catch_unwind(AssertUnwindSafe(|| {
915        dispatch_inner(backend, action, context)
916    })) {
917        Ok(response) => response,
918        Err(_) => Response::failure("execution_failed", "action panicked and was recovered"),
919    };
920    stamp_adapter_name(&mut response, context.adapter.as_deref());
921    // A serve or batch session stays open so a later mouse_up can pair with this down.
922    // A one-shot command has no session, so anything still held is released here.
923    if !input_session::session_open() {
924        let _ = backend.release_held();
925    }
926    response
927}
928
929enum PostKind {
930    /// Successful or failed mutating action. `return_state` snapshots afterward.
931    Mutating,
932    /// `return_state` and `include_screenshot` are ignored.
933    ReadOnly,
934    /// The action already captured. `include_screenshot` reuses that body.
935    Screenshot,
936}
937
938fn dispatch_inner(backend: &dyn DesktopBackend, action: Action, context: &CallContext) -> Response {
939    let label = action_label(&action);
940    let chars = clipboard_chars(&action);
941    let action = match remap_coordinates(context, action) {
942        Ok(action) => action,
943        Err(error) => {
944            return finish_action(
945                context,
946                label,
947                &policy::Subject::default(),
948                None,
949                "allow",
950                "default",
951                chars,
952                Response::from_error(error),
953            )
954        }
955    };
956    if let Err(error) = preflight_action(&action) {
957        return finish_action(
958            context,
959            label,
960            &policy::Subject::default(),
961            None,
962            "allow",
963            "default",
964            chars,
965            Response::from_error(error),
966        );
967    }
968    if let Err(error) = gate_display(backend, context, &action) {
969        return finish_action(
970            context,
971            label,
972            &policy::Subject::default(),
973            None,
974            "allow",
975            "default",
976            chars,
977            Response::from_error(error),
978        );
979    }
980    let prepared = prepare_action(backend, &action, context);
981    if let Some(response) = prepared.stop {
982        return finish_action(
983            context,
984            label,
985            &prepared.subject,
986            prepared.warning.as_deref(),
987            prepared.decision,
988            prepared.source,
989            chars,
990            response,
991        );
992    }
993    let kind = post_state_kind(&action);
994    let response = match action {
995        Action::Batch {
996            actions,
997            stop_on_error,
998        } => {
999            return finish_action(
1000                context,
1001                label,
1002                &prepared.subject,
1003                prepared.warning.as_deref(),
1004                prepared.decision,
1005                prepared.source,
1006                chars,
1007                run_batch(backend, context, actions, stop_on_error),
1008            )
1009        }
1010        Action::Screenshot {
1011            output,
1012            region,
1013            format,
1014            max_width,
1015            canvas,
1016            base64,
1017            marks,
1018        } => result_response(screenshot(
1019            backend,
1020            context,
1021            ScreenshotArgs {
1022                output,
1023                region,
1024                format,
1025                max_width,
1026                canvas,
1027                include_base64: base64,
1028                window_crop: false,
1029                marks,
1030            },
1031        )),
1032        Action::Zoom {
1033            output,
1034            region,
1035            format,
1036            max_width,
1037            canvas,
1038            base64,
1039        } => result_response(screenshot(
1040            backend,
1041            context,
1042            ScreenshotArgs {
1043                output,
1044                region: Some(region),
1045                format,
1046                max_width,
1047                canvas,
1048                include_base64: base64,
1049                window_crop: true,
1050                marks: false,
1051            },
1052        )),
1053        Action::Click {
1054            x,
1055            y,
1056            button,
1057            double,
1058            count,
1059        } => click(backend, context, x, y, button, double, count),
1060        Action::DoubleClick { x, y, button } => click(backend, context, x, y, button, true, 1),
1061        Action::TripleClick { x, y, button } => click(backend, context, x, y, button, false, 3),
1062        Action::Press { count, button } => {
1063            element_command(backend, context, ElementOp::Press { count, button })
1064        }
1065        Action::Drag { path, button } => drag(backend, context, path, button),
1066        Action::MouseDown { x, y, button } => pointer_half(backend, context, x, y, button, true),
1067        Action::MouseUp { x, y, button } => pointer_half(backend, context, x, y, button, false),
1068        Action::HoldKey { keys, ms } => hold_key(backend, context, &keys, ms),
1069        Action::ClipboardGet => result_response(clipboard_get(backend, context)),
1070        Action::ClipboardSet { text, image } => {
1071            result_response(clipboard_set(backend, text, image))
1072        }
1073        Action::Focus => element_command(backend, context, ElementOp::Focus),
1074        Action::SetValue { value } => {
1075            element_command(backend, context, ElementOp::SetValue { value })
1076        }
1077        Action::SelectText { start, end } => {
1078            element_command(backend, context, ElementOp::SelectText { start, end })
1079        }
1080        Action::PerformAction { name } => {
1081            element_command(backend, context, ElementOp::Perform { name })
1082        }
1083        Action::ScrollTo => element_command(backend, context, ElementOp::ScrollTo),
1084        Action::Move { x, y } => move_mouse(backend, context, x, y),
1085        Action::Type { text } => type_command(backend, context, text),
1086        Action::Key { keys } => press_key(backend, context, &keys),
1087        Action::Scroll {
1088            x,
1089            y,
1090            dx,
1091            dy,
1092            direction,
1093            amount,
1094        } => scroll(backend, context, (x, y), (dx, dy), direction, amount),
1095        Action::GetState => result_response(Ok(collect_state_for(backend, &tree_query(context)))),
1096        Action::Tree => result_response(tree_action(backend, context)),
1097        Action::Ocr { force } => result_response(ocr_action(backend, context, force)),
1098        Action::SessionEnd => result_response(session_end(backend, context)),
1099        Action::Wait { ms, seconds } => result_response(wait(ms, seconds)),
1100        Action::Displays => result_response(displays(backend)),
1101        Action::Isolation => result_response(isolation()),
1102        Action::ListApps { all } => result_response(list_apps(backend, all)),
1103        Action::ListWindows => result_response(list_windows(backend, context.app.as_deref())),
1104        Action::Launch => result_response(launch(backend, context.app.as_deref())),
1105        Action::Activate => result_response(activate(
1106            backend,
1107            context.app.as_deref(),
1108            context.window.as_deref(),
1109        )),
1110        Action::Quit { force } => result_response(quit(backend, context.app.as_deref(), force)),
1111        Action::Doctor {
1112            prompt,
1113            open_settings,
1114            x11,
1115        } => result_response(doctor(
1116            backend,
1117            crate::doctor::DoctorOptions {
1118                prompt,
1119                open_settings,
1120                x11,
1121            },
1122        )),
1123        Action::CursorPosition => result_response(cursor_position(backend)),
1124        Action::SessionPrune { dry_run, ttl_secs } => {
1125            result_response(session_prune(dry_run, ttl_secs))
1126        }
1127        Action::MenuList { app, depth } => {
1128            let app = match app.or_else(|| context.app.clone()) {
1129                Some(app) => app,
1130                None => return Response::from_error(invalid_arguments("menu list requires --app")),
1131            };
1132            let depth = depth.unwrap_or(MENU_DEFAULT_DEPTH);
1133            result_response(menu_list(backend, &app, depth))
1134        }
1135        Action::MenuClick { app, path } => {
1136            let app = match app.or_else(|| context.app.clone()) {
1137                Some(app) => app,
1138                None => {
1139                    return Response::from_error(invalid_arguments("menu click requires --app"))
1140                }
1141            };
1142            if path.trim().is_empty() {
1143                return Response::from_error(invalid_arguments("menu click requires --path"));
1144            }
1145            result_response(menu_click(backend, &app, &path))
1146        }
1147        Action::AppleEvent {
1148            app,
1149            script,
1150            language,
1151        } => {
1152            let app = match app.or_else(|| context.app.clone()) {
1153                Some(app) => app,
1154                None => {
1155                    return Response::from_error(invalid_arguments("apple-event requires --app"))
1156                }
1157            };
1158            if script.is_empty() {
1159                return Response::from_error(invalid_arguments("apple-event requires --script"));
1160            }
1161            let language = language.unwrap_or_default();
1162            result_response(apple_event_action(backend, &app, &script, language))
1163        }
1164        Action::ShortcutsRun { name } => {
1165            if !cfg!(target_os = "macos") {
1166                return Response::from_error(unsupported_error("shortcuts_run"));
1167            }
1168            if name.trim().is_empty() {
1169                return Response::from_error(invalid_arguments(
1170                    "shortcuts run requires a workflow name",
1171                ));
1172            }
1173            result_response(shortcuts_run_action(name))
1174        }
1175        Action::DockList {} => result_response(dock_list_action(backend)),
1176        Action::DockClick { title } => {
1177            if title.trim().is_empty() {
1178                return Response::from_error(invalid_arguments("dock click requires --title"));
1179            }
1180            result_response(dock_click_action(backend, &title))
1181        }
1182        Action::Spaces {} => result_response(spaces_action(backend)),
1183        Action::BrowserTabs {} => result_response(browser_tabs(context, &prepared.subject)),
1184        Action::BrowserActivate { target_id } => result_response(browser_activate(
1185            context,
1186            &prepared.subject,
1187            target_id.clone(),
1188        )),
1189        Action::BrowserNavigate { url, target_id } => result_response(browser_navigate(
1190            context,
1191            &prepared.subject,
1192            url.clone(),
1193            target_id.clone(),
1194        )),
1195        Action::BrowserClick {
1196            selector,
1197            target_id,
1198        } => result_response(browser_click(
1199            context,
1200            &prepared.subject,
1201            selector.clone(),
1202            target_id.clone(),
1203        )),
1204        Action::BrowserText {
1205            selector,
1206            target_id,
1207        } => result_response(browser_text(
1208            context,
1209            &prepared.subject,
1210            selector.clone(),
1211            target_id.clone(),
1212        )),
1213        Action::BrowserType {
1214            text,
1215            selector,
1216            target_id,
1217        } => result_response(browser_type(
1218            context,
1219            &prepared.subject,
1220            text.clone(),
1221            selector.clone(),
1222            target_id.clone(),
1223        )),
1224        Action::BrowserEval { script, target_id } => result_response(browser_eval(
1225            context,
1226            &prepared.subject,
1227            script.clone(),
1228            target_id.clone(),
1229        )),
1230        Action::PolicyCheck => {
1231            let (response, subject) = policy_check(backend, context);
1232            let response = attach_post_state(backend, context, kind, response);
1233            return finish_action(
1234                context,
1235                label,
1236                &subject,
1237                prepared.warning.as_deref(),
1238                prepared.decision,
1239                prepared.source,
1240                chars,
1241                response,
1242            );
1243        }
1244        Action::Pause => pause_response(true),
1245        Action::Resume => pause_response(false),
1246    };
1247    let response = attach_post_state(backend, context, kind, response);
1248    finish_action(
1249        context,
1250        label,
1251        &prepared.subject,
1252        prepared.warning.as_deref(),
1253        prepared.decision,
1254        prepared.source,
1255        chars,
1256        response,
1257    )
1258}
1259
1260#[allow(clippy::too_many_arguments)]
1261fn finish_action(
1262    context: &CallContext,
1263    label: &str,
1264    subject: &policy::Subject,
1265    warning: Option<&str>,
1266    decision: &str,
1267    source: &str,
1268    chars: Option<u64>,
1269    mut response: Response,
1270) -> Response {
1271    if let Some(warning) = warning {
1272        push_warning(&mut response, warning);
1273    }
1274    let code = if response.ok {
1275        "ok"
1276    } else {
1277        response
1278            .error
1279            .as_ref()
1280            .map(|error| error.code.as_str())
1281            .unwrap_or("execution_failed")
1282    };
1283    let mut decision = decision.to_string();
1284    let mut source = source.to_string();
1285    if label == "policy_check" {
1286        if let Some(value) = response
1287            .fields
1288            .get("decision")
1289            .and_then(|value| value.as_str())
1290        {
1291            decision = value.to_string();
1292        }
1293        if let Some(value) = response
1294            .fields
1295            .get("source")
1296            .and_then(|value| value.as_str())
1297        {
1298            source = value.to_string();
1299        }
1300    }
1301    let tier = response
1302        .fields
1303        .get("tier")
1304        .and_then(|value| value.as_str())
1305        .map(str::to_string);
1306    let session = context
1307        .session
1308        .clone()
1309        .unwrap_or_else(|| std::process::id().to_string());
1310    if let Err(error) = crate::audit::record(&crate::audit::AuditRecord {
1311        session: &session,
1312        action: label,
1313        bundle_id: subject.bundle_id.as_deref(),
1314        pid: subject.pid,
1315        window_id: subject.window_id,
1316        element: context.element,
1317        code,
1318        decision: &decision,
1319        source: &source,
1320        tier: tier.as_deref(),
1321        chars,
1322    }) {
1323        push_warning(
1324            &mut response,
1325            &format!("audit log was not written: {error}"),
1326        );
1327    }
1328    response
1329}
1330
1331fn push_warning(response: &mut Response, warning: &str) {
1332    let warnings = response
1333        .fields
1334        .entry("warnings")
1335        .or_insert_with(|| json!([]));
1336    if let Some(items) = warnings.as_array_mut() {
1337        items.push(json!(warning));
1338    }
1339}
1340
1341fn action_label(action: &Action) -> &'static str {
1342    match action {
1343        Action::Screenshot { .. } => "screenshot",
1344        Action::Zoom { .. } => "zoom",
1345        Action::Click { .. } => "click",
1346        Action::DoubleClick { .. } => "double_click",
1347        Action::TripleClick { .. } => "triple_click",
1348        Action::Press { .. } => "press",
1349        Action::Drag { .. } => "drag",
1350        Action::MouseDown { .. } => "mouse_down",
1351        Action::MouseUp { .. } => "mouse_up",
1352        Action::HoldKey { .. } => "hold_key",
1353        Action::ClipboardGet => "clipboard_get",
1354        Action::ClipboardSet { .. } => "clipboard_set",
1355        Action::Focus => "focus",
1356        Action::SetValue { .. } => "set_value",
1357        Action::SelectText { .. } => "select_text",
1358        Action::PerformAction { .. } => "perform_action",
1359        Action::ScrollTo => "scroll_to",
1360        Action::Move { .. } => "move",
1361        Action::Type { .. } => "type",
1362        Action::Key { .. } => "key",
1363        Action::Scroll { .. } => "scroll",
1364        Action::GetState => "get_state",
1365        Action::Wait { .. } => "wait",
1366        Action::Displays => "displays",
1367        Action::Isolation => "isolation",
1368        Action::ListApps { .. } => "list_apps",
1369        Action::ListWindows => "list_windows",
1370        Action::Launch => "launch",
1371        Action::Activate => "activate",
1372        Action::Quit { .. } => "quit",
1373        Action::Doctor { .. } => "doctor",
1374        Action::Tree => "tree",
1375        Action::SessionEnd => "session_end",
1376        Action::CursorPosition => "cursor_position",
1377        Action::Batch { .. } => "batch",
1378        Action::SessionPrune { .. } => "session_prune",
1379        Action::PolicyCheck => "policy_check",
1380        Action::Pause => "pause",
1381        Action::Resume => "resume",
1382        Action::MenuList { .. } => "menu_list",
1383        Action::MenuClick { .. } => "menu_click",
1384        Action::AppleEvent { .. } => "apple_event",
1385        Action::ShortcutsRun { .. } => "shortcuts_run",
1386        Action::DockList { .. } => "dock_list",
1387        Action::DockClick { .. } => "dock_click",
1388        Action::Spaces { .. } => "spaces",
1389        Action::BrowserTabs { .. } => "browser_tabs",
1390        Action::BrowserActivate { .. } => "browser_activate",
1391        Action::BrowserNavigate { .. } => "browser_navigate",
1392        Action::BrowserClick { .. } => "browser_click",
1393        Action::BrowserText { .. } => "browser_text",
1394        Action::BrowserType { .. } => "browser_type",
1395        Action::BrowserEval { .. } => "browser_eval",
1396        Action::Ocr { .. } => "ocr",
1397    }
1398}
1399
1400#[derive(Clone, Copy)]
1401enum PolicyClass {
1402    Always,
1403    Benign,
1404    Observe,
1405    Mutate,
1406    Quit,
1407    Key,
1408    ClipboardSet,
1409}
1410
1411fn policy_class(action: &Action) -> PolicyClass {
1412    match action {
1413        Action::Wait { .. }
1414        | Action::Doctor { .. }
1415        | Action::Displays
1416        | Action::Isolation
1417        | Action::ListApps { .. }
1418        | Action::ListWindows
1419        | Action::Pause
1420        | Action::Resume => PolicyClass::Always,
1421        Action::SessionEnd | Action::SessionPrune { .. } | Action::CursorPosition => {
1422            PolicyClass::Benign
1423        }
1424        Action::Screenshot { .. }
1425        | Action::Zoom { .. }
1426        | Action::Tree
1427        | Action::GetState
1428        | Action::PolicyCheck
1429        | Action::MenuList { .. }
1430        | Action::DockList { .. }
1431        | Action::Spaces { .. }
1432        | Action::BrowserTabs { .. }
1433        | Action::BrowserText { .. }
1434        | Action::BrowserActivate { .. }
1435        | Action::Ocr { .. } => PolicyClass::Observe,
1436        Action::Quit { .. } => PolicyClass::Quit,
1437        Action::Key { .. } | Action::HoldKey { .. } => PolicyClass::Key,
1438        Action::ClipboardGet => PolicyClass::Benign,
1439        Action::ClipboardSet { .. } => PolicyClass::ClipboardSet,
1440        _ => PolicyClass::Mutate,
1441    }
1442}
1443
1444struct Prepared {
1445    stop: Option<Response>,
1446    warning: Option<String>,
1447    subject: policy::Subject,
1448    decision: &'static str,
1449    source: &'static str,
1450}
1451
1452fn early_stop(warning: Option<String>, subject: policy::Subject, response: Response) -> Prepared {
1453    Prepared {
1454        stop: Some(response),
1455        warning,
1456        subject,
1457        decision: "allow",
1458        source: "default",
1459    }
1460}
1461
1462fn prepare_action(
1463    backend: &dyn DesktopBackend,
1464    action: &Action,
1465    context: &CallContext,
1466) -> Prepared {
1467    let class = policy_class(action);
1468    let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1469        Ok(policy) => policy,
1470        Err(error) => {
1471            if matches!(class, PolicyClass::Always) {
1472                policy::LoadedPolicy::default()
1473            } else {
1474                return early_stop(
1475                    None,
1476                    policy::Subject::default(),
1477                    Response::from_error(error),
1478                );
1479            }
1480        }
1481    };
1482    let warning = if loaded.pause_on_user_input {
1483        backend.user_input_pause_warning().map(str::to_string)
1484    } else {
1485        None
1486    };
1487    if matches!(action, Action::Pause | Action::Resume | Action::PolicyCheck) {
1488        return Prepared {
1489            stop: None,
1490            warning,
1491            subject: policy::Subject::default(),
1492            decision: "allow",
1493            source: "default",
1494        };
1495    }
1496    let subject = match resolve_subject(backend, action, context) {
1497        Ok(subject) => subject,
1498        Err(error) => {
1499            return early_stop(
1500                warning,
1501                policy::Subject::default(),
1502                Response::from_error(error),
1503            )
1504        }
1505    };
1506    let policy_action = match action {
1507        Action::Quit { .. } => policy::PolicyAction::Quit,
1508        Action::Key { keys } | Action::HoldKey { keys, .. } => policy::PolicyAction::Key {
1509            combo: keys.clone(),
1510        },
1511        Action::Screenshot { .. } | Action::Zoom { .. } | Action::Tree | Action::GetState => {
1512            policy::PolicyAction::Observe
1513        }
1514        Action::Ocr { .. } => policy::PolicyAction::Observe,
1515        Action::ClipboardSet { .. } => policy::PolicyAction::ClipboardSet,
1516        _ => match class {
1517            PolicyClass::Always => policy::PolicyAction::Always,
1518            PolicyClass::Benign => policy::PolicyAction::Benign,
1519            PolicyClass::Mutate => policy::PolicyAction::Mutate,
1520            PolicyClass::Observe => policy::PolicyAction::Observe,
1521            PolicyClass::Quit => policy::PolicyAction::Quit,
1522            PolicyClass::Key => policy::PolicyAction::Key {
1523                combo: String::new(),
1524            },
1525            PolicyClass::ClipboardSet => policy::PolicyAction::ClipboardSet,
1526        },
1527    };
1528    let decision = policy::decide(&loaded, &subject, &policy_action);
1529    if let policy::Decision::Deny { message, source } = &decision {
1530        return Prepared {
1531            stop: Some(Response::from_error(ComputerError::PolicyDenied(
1532                message.clone(),
1533            ))),
1534            warning,
1535            subject,
1536            decision: "deny",
1537            source,
1538        };
1539    }
1540    if let Err(error) = refuse_blocked_url(backend, &loaded, &subject) {
1541        let (decision_name, source) = decision_record(&decision);
1542        return Prepared {
1543            stop: Some(Response::from_error(error)),
1544            warning,
1545            subject,
1546            decision: decision_name,
1547            source,
1548        };
1549    }
1550    let mutating = matches!(
1551        class,
1552        PolicyClass::Mutate | PolicyClass::Quit | PolicyClass::Key | PolicyClass::ClipboardSet
1553    );
1554    if mutating && policy::is_paused() {
1555        let (decision_name, source) = decision_record(&decision);
1556        return Prepared {
1557            stop: Some(Response::from_error(ComputerError::Paused(
1558                "local input paused mutating actions; run computer-use resume".into(),
1559            ))),
1560            warning,
1561            subject,
1562            decision: decision_name,
1563            source,
1564        };
1565    }
1566    if let Action::Key { .. } | Action::HoldKey { .. } = action {
1567        if let Some(index) = context.element {
1568            if let Ok(Some(peek)) = backend.peek_element(
1569                context.app.as_deref(),
1570                context.window.as_deref(),
1571                context.session.as_deref(),
1572                index,
1573            ) {
1574                if peek.secure {
1575                    return Prepared {
1576                        stop: Some(Response::from_error(ComputerError::PolicyDenied(
1577                            "refusing a key event aimed at a secure field".into(),
1578                        ))),
1579                        warning,
1580                        subject,
1581                        decision: "deny",
1582                        source: "builtin",
1583                    };
1584                }
1585            }
1586        }
1587    }
1588    if let policy::Decision::Confirm { reason } = &decision {
1589        if let Err(error) = confirm_action(context, &loaded, &subject, action_label(action), reason)
1590        {
1591            return Prepared {
1592                stop: Some(Response::from_error(error)),
1593                warning,
1594                subject,
1595                decision: "confirm",
1596                source: "builtin",
1597            };
1598        }
1599    }
1600    let (decision_name, source) = decision_record(&decision);
1601    Prepared {
1602        stop: None,
1603        warning,
1604        subject,
1605        decision: decision_name,
1606        source,
1607    }
1608}
1609
1610fn decision_record(decision: &policy::Decision) -> (&'static str, &'static str) {
1611    match decision {
1612        policy::Decision::Allow { source } => ("allow", *source),
1613        policy::Decision::Deny { source, .. } => ("deny", *source),
1614        policy::Decision::Confirm { .. } => ("confirm", "builtin"),
1615    }
1616}
1617
1618fn confirm_action(
1619    context: &CallContext,
1620    loaded: &policy::LoadedPolicy,
1621    subject: &policy::Subject,
1622    action: &str,
1623    reason: &str,
1624) -> Result<(), ComputerError> {
1625    let command = policy::confirm_program(loaded, context.confirm_cmd.as_deref());
1626    policy::run_confirm(
1627        command.as_deref(),
1628        loaded.timeout_secs(),
1629        &policy::ConfirmDocument {
1630            action,
1631            bundle_id: subject.bundle_id.as_deref(),
1632            path: subject.path.as_deref(),
1633            pid: subject.pid,
1634            window_id: subject.window_id,
1635            element: context.element,
1636            reason,
1637        },
1638    )
1639}
1640
1641fn refuse_blocked_url(
1642    backend: &dyn DesktopBackend,
1643    loaded: &policy::LoadedPolicy,
1644    subject: &policy::Subject,
1645) -> Result<(), ComputerError> {
1646    if loaded.blocked_urls().is_empty() {
1647        return Ok(());
1648    }
1649    if !policy::is_browser(
1650        subject.bundle_id.as_deref(),
1651        subject.path.as_deref().map(Path::new),
1652        subject.name.as_deref(),
1653    ) {
1654        return Ok(());
1655    }
1656    let from_title = subject
1657        .window_title
1658        .as_deref()
1659        .and_then(policy::url_in_text)
1660        .map(str::to_string);
1661    let url = if let Some(url) = from_title {
1662        Some(url)
1663    } else if let Some(pid) = subject.pid {
1664        backend.browser_url(pid)?
1665    } else {
1666        None
1667    };
1668    if let Some(url) = url.as_deref() {
1669        if let Some(pattern) = policy::url_blocked(url, loaded.blocked_urls()) {
1670            return Err(ComputerError::BlockedUrl(format!("matched {pattern}")));
1671        }
1672    }
1673    Ok(())
1674}
1675
1676fn resolve_subject(
1677    backend: &dyn DesktopBackend,
1678    action: &Action,
1679    context: &CallContext,
1680) -> Result<policy::Subject, ComputerError> {
1681    let class = policy_class(action);
1682    if matches!(class, PolicyClass::Always | PolicyClass::Benign) {
1683        return Ok(policy::Subject::default());
1684    }
1685    let screenshot = matches!(action, Action::Screenshot { .. } | Action::Zoom { .. });
1686    let frontmost_observe =
1687        matches!(action, Action::Tree | Action::GetState) && context.app.is_none();
1688    let app_text = context.app.clone().or_else(|| {
1689        if frontmost_observe {
1690            backend.frontmost().app
1691        } else {
1692            None
1693        }
1694    });
1695    if app_text.is_none() && context.window.is_none() {
1696        return Ok(policy::Subject::default());
1697    }
1698    if app_text.is_none() && screenshot {
1699        return subject_from_window(backend, context);
1700    }
1701    let Some(app_text) = app_text else {
1702        return Ok(policy::Subject::default());
1703    };
1704    let selector = parse_app_selector(&app_text)?;
1705    let fallback = subject_from_selector(&selector);
1706    let apps = match backend.list_apps(true) {
1707        Ok(apps) => apps,
1708        Err(error) => {
1709            if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1710                return Ok(fallback);
1711            }
1712            return Err(error);
1713        }
1714    };
1715    let info = match resolve_app(&apps, &selector) {
1716        Ok(info) => info.clone(),
1717        Err(error) => {
1718            if policy::selector_denial(&selector).is_some() || matches!(action, Action::Launch) {
1719                return Ok(fallback);
1720            }
1721            // An implicit frontmost name that is not running is not a target.
1722            // The backend resolves the key window itself.
1723            if context.app.is_none() {
1724                return Ok(policy::Subject::default());
1725            }
1726            return Err(error);
1727        }
1728    };
1729    let mut subject = policy::Subject {
1730        name: Some(info.name.clone()),
1731        bundle_id: info.bundle_id.clone(),
1732        path: info.path.clone(),
1733        pid: Some(info.pid),
1734        window_id: None,
1735        window_title: None,
1736        ax_identifier: None,
1737    };
1738    let want_window = context.window.is_some()
1739        || matches!(
1740            action,
1741            Action::Tree | Action::GetState | Action::Screenshot { .. } | Action::Zoom { .. }
1742        );
1743    if want_window {
1744        if let Ok(listed) = backend.list_windows() {
1745            let owned = filter_windows_for_pid(&listed.windows, info.pid);
1746            let chosen = match context.window.as_deref() {
1747                Some(window) => {
1748                    let selector = parse_window_selector(window)?;
1749                    Some(resolve_window(&owned, &selector)?.clone())
1750                }
1751                None => owned.first().cloned(),
1752            };
1753            if let Some(window) = chosen {
1754                subject.window_id = Some(window.id);
1755                if context.window.is_some()
1756                    || matches!(
1757                        action,
1758                        Action::Tree
1759                            | Action::GetState
1760                            | Action::Screenshot { .. }
1761                            | Action::Zoom { .. }
1762                    )
1763                {
1764                    subject.window_title = Some(window.title);
1765                }
1766            }
1767        }
1768    }
1769    Ok(subject)
1770}
1771
1772fn subject_from_window(
1773    backend: &dyn DesktopBackend,
1774    context: &CallContext,
1775) -> Result<policy::Subject, ComputerError> {
1776    let Some(window) = context.window.as_deref() else {
1777        return Ok(policy::Subject::default());
1778    };
1779    let listed = backend.list_windows()?;
1780    let selector = parse_window_selector(window)?;
1781    let window = resolve_window(&listed.windows, &selector)?.clone();
1782    let apps = backend.list_apps(true)?;
1783    let info = apps.into_iter().find(|app| app.pid == window.pid);
1784    Ok(policy::Subject {
1785        name: info
1786            .as_ref()
1787            .map(|app| app.name.clone())
1788            .or(Some(window.app.clone())),
1789        bundle_id: info.as_ref().and_then(|app| app.bundle_id.clone()),
1790        path: info.and_then(|app| app.path),
1791        pid: Some(window.pid),
1792        window_id: Some(window.id),
1793        window_title: Some(window.title),
1794        ax_identifier: None,
1795    })
1796}
1797
1798fn subject_from_selector(selector: &crate::target::AppSelector) -> policy::Subject {
1799    use crate::target::AppSelector;
1800    match selector {
1801        AppSelector::BundleId(bundle) => policy::Subject {
1802            bundle_id: Some(bundle.clone()),
1803            ..policy::Subject::default()
1804        },
1805        AppSelector::Name(name) => policy::Subject {
1806            name: Some(name.clone()),
1807            ..policy::Subject::default()
1808        },
1809        AppSelector::Path(path) => policy::Subject {
1810            path: Some(path.display().to_string()),
1811            ..policy::Subject::default()
1812        },
1813        AppSelector::Pid(pid) => policy::Subject {
1814            pid: Some(*pid),
1815            ..policy::Subject::default()
1816        },
1817    }
1818}
1819
1820fn policy_check(
1821    backend: &dyn DesktopBackend,
1822    context: &CallContext,
1823) -> (Response, policy::Subject) {
1824    let Some(app) = context
1825        .app
1826        .as_deref()
1827        .map(str::trim)
1828        .filter(|text| !text.is_empty())
1829    else {
1830        return (
1831            Response::from_error(ComputerError::InvalidArguments("app is required".into())),
1832            policy::Subject::default(),
1833        );
1834    };
1835    let loaded = match policy::load_policy(context.policy.as_deref().map(Path::new)) {
1836        Ok(policy) => policy,
1837        Err(error) => return (Response::from_error(error), policy::Subject::default()),
1838    };
1839    let selector = match parse_app_selector(app) {
1840        Ok(selector) => selector,
1841        Err(error) => return (Response::from_error(error), policy::Subject::default()),
1842    };
1843    let mut subject = subject_from_selector(&selector);
1844    if let Ok(apps) = backend.list_apps(true) {
1845        if let Ok(info) = resolve_app(&apps, &selector) {
1846            subject.name = Some(info.name.clone());
1847            subject.bundle_id = info.bundle_id.clone();
1848            subject.path = info.path.clone();
1849            subject.pid = Some(info.pid);
1850        }
1851    }
1852    let decision = policy::app_verdict(&loaded, &subject);
1853    let (name, source) = match decision {
1854        policy::Decision::Allow { source } => ("allow", source),
1855        policy::Decision::Deny { source, .. } => ("deny", source),
1856        policy::Decision::Confirm { .. } => ("allow", "default"),
1857    };
1858    (
1859        Response::success(json!({
1860            "decision": name,
1861            "source": source,
1862            "bundle_id": subject.bundle_id,
1863        })),
1864        subject,
1865    )
1866}
1867
1868fn pause_response(paused: bool) -> Response {
1869    match policy::set_paused(paused) {
1870        Ok(()) => Response::success(json!({
1871            "action": if paused { "pause" } else { "resume" },
1872            "paused": paused,
1873        })),
1874        Err(error) => Response::from_error(error),
1875    }
1876}
1877
1878fn result_response(result: Result<Value, ComputerError>) -> Response {
1879    match result {
1880        Ok(value) => Response::success(value),
1881        Err(error) => Response::from_error(error),
1882    }
1883}
1884
1885fn post_state_kind(action: &Action) -> PostKind {
1886    match action {
1887        Action::Screenshot { .. } | Action::Zoom { .. } => PostKind::Screenshot,
1888        Action::Click { .. }
1889        | Action::DoubleClick { .. }
1890        | Action::TripleClick { .. }
1891        | Action::Drag { .. }
1892        | Action::MouseDown { .. }
1893        | Action::MouseUp { .. }
1894        | Action::HoldKey { .. }
1895        | Action::ClipboardSet { .. }
1896        | Action::Press { .. }
1897        | Action::Focus
1898        | Action::SetValue { .. }
1899        | Action::SelectText { .. }
1900        | Action::PerformAction { .. }
1901        | Action::ScrollTo
1902        | Action::Move { .. }
1903        | Action::Type { .. }
1904        | Action::Key { .. }
1905        | Action::Scroll { .. }
1906        | Action::Launch
1907        | Action::Activate
1908        | Action::Quit { .. }
1909        | Action::Batch { .. }
1910        | Action::MenuClick { .. }
1911        | Action::AppleEvent { .. }
1912        | Action::ShortcutsRun { .. }
1913        | Action::DockClick { .. }
1914        | Action::BrowserNavigate { .. }
1915        | Action::BrowserClick { .. }
1916        | Action::BrowserText { .. }
1917        | Action::BrowserType { .. }
1918        | Action::BrowserEval { .. }
1919        | Action::BrowserActivate { .. }
1920        | Action::Ocr { .. } => PostKind::Mutating,
1921        Action::GetState
1922        | Action::Wait { .. }
1923        | Action::Displays
1924        | Action::Isolation
1925        | Action::ListApps { .. }
1926        | Action::ListWindows
1927        | Action::Doctor { .. }
1928        | Action::Tree
1929        | Action::SessionEnd
1930        | Action::SessionPrune { .. }
1931        | Action::CursorPosition
1932        | Action::ClipboardGet
1933        | Action::PolicyCheck
1934        | Action::Pause
1935        | Action::Resume
1936        | Action::MenuList { .. }
1937        | Action::DockList { .. }
1938        | Action::Spaces { .. }
1939        | Action::BrowserTabs { .. } => PostKind::ReadOnly,
1940    }
1941}
1942
1943fn remap_coordinates(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1944    match context.coord.as_deref() {
1945        None | Some("screen") => Ok(action),
1946        Some("image") => remap_image_action(context, action),
1947        Some(other) => Err(ComputerError::InvalidArguments(format!(
1948            "coord must be screen or image (got {other})"
1949        ))),
1950    }
1951}
1952
1953fn remap_image_action(context: &CallContext, action: Action) -> Result<Action, ComputerError> {
1954    match action {
1955        Action::Click {
1956            x,
1957            y,
1958            button,
1959            double,
1960            count,
1961        } => {
1962            let (x, y) = remap_optional_point(context, x, y)?;
1963            Ok(Action::Click {
1964                x,
1965                y,
1966                button,
1967                double,
1968                count,
1969            })
1970        }
1971        Action::DoubleClick { x, y, button } => {
1972            let (x, y) = remap_optional_point(context, x, y)?;
1973            Ok(Action::DoubleClick { x, y, button })
1974        }
1975        Action::TripleClick { x, y, button } => {
1976            let (x, y) = remap_optional_point(context, x, y)?;
1977            Ok(Action::TripleClick { x, y, button })
1978        }
1979        Action::Drag { path, button } => {
1980            let mut mapped = Vec::with_capacity(path.len());
1981            for point in path {
1982                let (x, y) = remap_point(context, point.x, point.y)?;
1983                mapped.push(Point { x, y });
1984            }
1985            Ok(Action::Drag {
1986                path: mapped,
1987                button,
1988            })
1989        }
1990        Action::MouseDown { x, y, button } => {
1991            let (x, y) = remap_point(context, x, y)?;
1992            Ok(Action::MouseDown { x, y, button })
1993        }
1994        Action::MouseUp { x, y, button } => {
1995            let (x, y) = remap_point(context, x, y)?;
1996            Ok(Action::MouseUp { x, y, button })
1997        }
1998        Action::Move { x, y } => {
1999            let (x, y) = remap_point(context, x, y)?;
2000            Ok(Action::Move { x, y })
2001        }
2002        Action::Scroll {
2003            x,
2004            y,
2005            dx,
2006            dy,
2007            direction,
2008            amount,
2009        } => {
2010            let (x, y) = remap_optional_point(context, x, y)?;
2011            Ok(Action::Scroll {
2012                x,
2013                y,
2014                dx,
2015                dy,
2016                direction,
2017                amount,
2018            })
2019        }
2020        Action::Screenshot {
2021            output,
2022            region,
2023            format,
2024            max_width,
2025            canvas,
2026            base64,
2027            marks,
2028        } => {
2029            let region = match region {
2030                Some(region) => Some(remap_region(context, region)?),
2031                None => None,
2032            };
2033            Ok(Action::Screenshot {
2034                output,
2035                region,
2036                format,
2037                max_width,
2038                canvas,
2039                base64,
2040                marks,
2041            })
2042        }
2043        Action::Zoom {
2044            output,
2045            region,
2046            format,
2047            max_width,
2048            canvas,
2049            base64,
2050        } => {
2051            let region = remap_region(context, region)?;
2052            Ok(Action::Zoom {
2053                output,
2054                region,
2055                format,
2056                max_width,
2057                canvas,
2058                base64,
2059            })
2060        }
2061        other => Ok(other),
2062    }
2063}
2064
2065/// Resolve `--display` once per action, then check every screen point it names.
2066///
2067/// Runs after [`remap_coordinates`], so `--coord image` has already become a
2068/// screen point and the filter sees the same space the click will use. Points
2069/// stay virtual-desktop points: `--display` narrows what may be addressed, it
2070/// does not rebase them.
2071fn gate_display(
2072    backend: &dyn DesktopBackend,
2073    context: &CallContext,
2074    action: &Action,
2075) -> Result<(), ComputerError> {
2076    let Some(spec) = context.display.as_deref() else {
2077        return Ok(());
2078    };
2079    // Syntax is checked for every action, so garbage never reaches the backend.
2080    // `virtual` is a CLI-only keyword: a JSON request cannot exec a prepared
2081    // X server, so it gets a structured `unsupported` instead of a silent
2082    // whole-desktop capture.
2083    crate::isolation::parse_selector(spec)?;
2084    // `displays` and `isolation` report on filtering, so they do not consume it.
2085    if matches!(action, Action::Isolation | Action::Displays) {
2086        return Ok(());
2087    }
2088    let displays = backend.list_displays()?;
2089    let target = crate::isolation::resolve(spec, &displays)?;
2090    match action {
2091        // A click without a point is element-targeted, so there is nothing to
2092        // filter: the element already resolved inside its window.
2093        Action::Click { x, y, .. }
2094        | Action::DoubleClick { x, y, .. }
2095        | Action::TripleClick { x, y, .. }
2096        | Action::Scroll { x, y, .. } => match (x, y) {
2097            (Some(x), Some(y)) => crate::isolation::ensure_point_inside(&target, *x, *y),
2098            _ => Ok(()),
2099        },
2100        Action::Move { x, y } | Action::MouseDown { x, y, .. } | Action::MouseUp { x, y, .. } => {
2101            crate::isolation::ensure_point_inside(&target, *x, *y)
2102        }
2103        Action::Drag { path, .. } => {
2104            for point in path {
2105                crate::isolation::ensure_point_inside(&target, point.x, point.y)?;
2106            }
2107            Ok(())
2108        }
2109        Action::Screenshot {
2110            region: Some(region),
2111            ..
2112        } => crate::isolation::ensure_region_inside(&target, *region),
2113        Action::Screenshot { .. } => Ok(()),
2114        Action::Zoom { region, .. } => crate::isolation::ensure_region_inside(&target, *region),
2115        _ => Ok(()),
2116    }
2117}
2118
2119fn remap_optional_point(
2120    context: &CallContext,
2121    x: Option<i32>,
2122    y: Option<i32>,
2123) -> Result<(Option<i32>, Option<i32>), ComputerError> {
2124    match (x, y) {
2125        (Some(x), Some(y)) => {
2126            let (x, y) = remap_point(context, x, y)?;
2127            Ok((Some(x), Some(y)))
2128        }
2129        other => Ok(other),
2130    }
2131}
2132
2133fn remap_point(context: &CallContext, x: i32, y: i32) -> Result<(i32, i32), ComputerError> {
2134    let map = require_screenshot_map(context)?;
2135    ensure_point_inside(&map, x, y)?;
2136    let (screen_x, screen_y) = image_util::image_to_screen(&map, f64::from(x), f64::from(y));
2137    Ok((round_screen(screen_x)?, round_screen(screen_y)?))
2138}
2139
2140fn remap_region(context: &CallContext, region: Region) -> Result<Region, ComputerError> {
2141    let map = require_screenshot_map(context)?;
2142    ensure_region_inside(&map, region)?;
2143    let x2 = i64::from(region.x) + i64::from(region.width);
2144    let y2 = i64::from(region.y) + i64::from(region.height);
2145    let (screen_x1, screen_y1) =
2146        image_util::image_to_screen(&map, f64::from(region.x), f64::from(region.y));
2147    let (screen_x2, screen_y2) = image_util::image_to_screen(&map, x2 as f64, y2 as f64);
2148    let x = round_screen(screen_x1)?;
2149    let y = round_screen(screen_y1)?;
2150    let far_x = round_screen(screen_x2)?;
2151    let far_y = round_screen(screen_y2)?;
2152    let width = u32::try_from(i64::from(far_x) - i64::from(x))
2153        .unwrap_or(0)
2154        .max(1);
2155    let height = u32::try_from(i64::from(far_y) - i64::from(y))
2156        .unwrap_or(0)
2157        .max(1);
2158    Region::new(x, y, width, height)
2159}
2160
2161fn require_screenshot_map(
2162    context: &CallContext,
2163) -> Result<image_util::ScreenshotMap, ComputerError> {
2164    match cache::load_screenshot_map(context.session.as_deref())? {
2165        Some(map) => Ok(map),
2166        None => Err(ComputerError::InvalidArguments(
2167            "take a screenshot first".into(),
2168        )),
2169    }
2170}
2171
2172fn content_box(map: &image_util::ScreenshotMap) -> (i64, i64, i64, i64) {
2173    let left = i64::from(map.pad.x);
2174    let top = i64::from(map.pad.y);
2175    let right = left + i64::from(map.content_width);
2176    let bottom = top + i64::from(map.content_height);
2177    (left, top, right, bottom)
2178}
2179
2180fn ensure_point_inside(
2181    map: &image_util::ScreenshotMap,
2182    x: i32,
2183    y: i32,
2184) -> Result<(), ComputerError> {
2185    let (left, top, right, bottom) = content_box(map);
2186    let x = i64::from(x);
2187    let y = i64::from(y);
2188    if x < left || y < top || x >= right || y >= bottom {
2189        return Err(ComputerError::InvalidArguments(format!(
2190            "coordinate ({x}, {y}) is outside the screenshot"
2191        )));
2192    }
2193    Ok(())
2194}
2195
2196fn ensure_region_inside(
2197    map: &image_util::ScreenshotMap,
2198    region: Region,
2199) -> Result<(), ComputerError> {
2200    let (left, top, right, bottom) = content_box(map);
2201    let x1 = i64::from(region.x);
2202    let y1 = i64::from(region.y);
2203    let x2 = x1 + i64::from(region.width);
2204    let y2 = y1 + i64::from(region.height);
2205    if x1 < left || y1 < top || x1 >= right || y1 >= bottom || x2 > right || y2 > bottom {
2206        return Err(ComputerError::InvalidArguments(format!(
2207            "region {},{} {}x{} is outside the screenshot",
2208            region.x, region.y, region.width, region.height
2209        )));
2210    }
2211    Ok(())
2212}
2213
2214fn round_screen(value: f64) -> Result<i32, ComputerError> {
2215    if !value.is_finite() {
2216        return Err(ComputerError::InvalidArguments(
2217            "coordinate remap was not finite".into(),
2218        ));
2219    }
2220    let rounded = value.round();
2221    if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) {
2222        return Err(ComputerError::InvalidArguments(
2223            "coordinate remap is out of range".into(),
2224        ));
2225    }
2226    Ok(rounded as i32)
2227}
2228
2229fn run_batch(
2230    backend: &dyn DesktopBackend,
2231    context: &CallContext,
2232    actions: Vec<Value>,
2233    stop_on_error: bool,
2234) -> Response {
2235    // Steps share one hold. The drop releases only when this is the outermost session.
2236    let _session = InputSession::open(backend);
2237    let mut results = Vec::with_capacity(actions.len());
2238    let mut first_error: Option<(usize, ErrorBody)> = None;
2239    for (index, step) in actions.into_iter().enumerate() {
2240        let response = dispatch_batch_step(backend, context, step);
2241        let failed = !response.ok;
2242        if failed && first_error.is_none() {
2243            if let Some(error) = response.error.clone() {
2244                first_error = Some((index, error));
2245            }
2246        }
2247        results.push(response.to_value());
2248        if failed && stop_on_error {
2249            return attach_post_state(
2250                backend,
2251                context,
2252                PostKind::Mutating,
2253                batch_response(results, Some(index), first_error),
2254            );
2255        }
2256    }
2257    attach_post_state(
2258        backend,
2259        context,
2260        PostKind::Mutating,
2261        batch_response(results, None, first_error),
2262    )
2263}
2264
2265fn dispatch_batch_step(
2266    backend: &dyn DesktopBackend,
2267    outer: &CallContext,
2268    value: Value,
2269) -> Response {
2270    match prepare_batch_step(outer, value) {
2271        Ok((action, context)) => dispatch_with_context(backend, action, &context),
2272        Err(error) => Response::from_error(error),
2273    }
2274}
2275
2276fn prepare_batch_step(
2277    outer: &CallContext,
2278    mut value: Value,
2279) -> Result<(Action, CallContext), ComputerError> {
2280    let Some(object) = value.as_object_mut() else {
2281        return Err(ComputerError::InvalidArguments(
2282            "batch step must be a JSON object".into(),
2283        ));
2284    };
2285    if object.get("action").and_then(Value::as_str) == Some("batch") {
2286        return Err(ComputerError::InvalidArguments(
2287            "nested batch is not allowed".into(),
2288        ));
2289    }
2290    let mut context = outer.clone();
2291    overlay_context(&mut context, object)?;
2292    context.return_state = false;
2293    context.include_screenshot = false;
2294    let action = serde_json::from_value(value)
2295        .map_err(|error| ComputerError::InvalidArguments(error.to_string()))?;
2296    Ok((action, context))
2297}
2298
2299fn overlay_context(
2300    context: &mut CallContext,
2301    object: &mut Map<String, Value>,
2302) -> Result<(), ComputerError> {
2303    if object.contains_key("app") {
2304        context.app = take_stringish(object, "app")?;
2305    }
2306    if object.contains_key("window") {
2307        context.window = take_stringish(object, "window")?;
2308    }
2309    if object.contains_key("element") {
2310        context.element = take_optional_u32(object, "element")?;
2311    }
2312    if object.contains_key("diff") {
2313        context.diff = take_bool(object, "diff")?;
2314    }
2315    if object.contains_key("full") {
2316        context.full = take_bool(object, "full")?;
2317    }
2318    let _ = object.remove("return_state");
2319    let _ = object.remove("include_screenshot");
2320    let _ = object.remove("id");
2321    let _ = object.remove("stop_on_error");
2322    if object.contains_key("coord") {
2323        context.coord = take_stringish(object, "coord")?;
2324    }
2325    if object.contains_key("session") {
2326        context.session = take_stringish(object, "session")?;
2327    }
2328    if object.contains_key("tier") {
2329        context.tier = take_stringish(object, "tier")?;
2330    }
2331    if object.contains_key("adapter") {
2332        context.adapter = take_stringish(object, "adapter")?;
2333    }
2334    if object.contains_key("display") {
2335        context.display = take_stringish(object, "display")?;
2336    }
2337    if object.contains_key("require_visible") {
2338        context.require_visible = take_bool(object, "require_visible")?;
2339    }
2340    if object.contains_key("allow_foreground") {
2341        context.allow_foreground = take_bool(object, "allow_foreground")?;
2342    }
2343    if object.contains_key("require_background") {
2344        context.require_background = take_bool(object, "require_background")?;
2345    }
2346    if object.contains_key("no_overlay") {
2347        context.no_overlay = take_bool(object, "no_overlay")?;
2348    }
2349    Ok(())
2350}
2351
2352fn batch_response(
2353    results: Vec<Value>,
2354    stopped_at: Option<usize>,
2355    first_error: Option<(usize, ErrorBody)>,
2356) -> Response {
2357    let mut fields = Map::new();
2358    fields.insert("action".into(), json!("batch"));
2359    fields.insert("results".into(), Value::Array(results));
2360    if let Some(index) = stopped_at {
2361        fields.insert("stopped_at".into(), json!(index));
2362    }
2363    match first_error {
2364        Some((index, error)) => {
2365            let mut response = Response::failure(
2366                error.code,
2367                format!("batch failed at action {index}: {}", error.message),
2368            );
2369            response.fields = fields;
2370            response
2371        }
2372        None => Response {
2373            ok: true,
2374            id: None,
2375            error: None,
2376            fields,
2377        },
2378    }
2379}
2380
2381fn attach_post_state(
2382    backend: &dyn DesktopBackend,
2383    context: &CallContext,
2384    kind: PostKind,
2385    mut response: Response,
2386) -> Response {
2387    if !context.return_state || matches!(kind, PostKind::ReadOnly) {
2388        return response;
2389    }
2390    let mut warnings = Vec::new();
2391    let mut state = match backend.tree(&tree_query(context)) {
2392        Ok(report) => json!({
2393            "app": report.app,
2394            "pid": report.pid,
2395            "window_id": report.window_id,
2396            "revision": report.revision,
2397            "tree": report.full_text,
2398            "diff": report.diff,
2399            "truncated": report.truncated,
2400        }),
2401        Err(error) => {
2402            if !response.ok {
2403                return response;
2404            }
2405            warnings.push(format!("tree snapshot failed: {error}"));
2406            json!({
2407                "app": null,
2408                "pid": null,
2409                "window_id": null,
2410                "revision": null,
2411                "tree": null,
2412                "diff": null,
2413                "truncated": null,
2414            })
2415        }
2416    };
2417    if context.include_screenshot {
2418        let shot = if response.ok && matches!(kind, PostKind::Screenshot) {
2419            Ok(Value::Object(response.fields.clone()))
2420        } else if matches!(kind, PostKind::Screenshot) {
2421            Err(ComputerError::ExecutionFailed(
2422                response
2423                    .error
2424                    .as_ref()
2425                    .map(|error| error.message.clone())
2426                    .unwrap_or_else(|| "screenshot failed".into()),
2427            ))
2428        } else {
2429            screenshot(
2430                backend,
2431                context,
2432                ScreenshotArgs {
2433                    output: None,
2434                    region: None,
2435                    format: None,
2436                    max_width: None,
2437                    canvas: None,
2438                    include_base64: false,
2439                    window_crop: false,
2440                    marks: false,
2441                },
2442            )
2443        };
2444        match shot {
2445            Ok(value) => state["screenshot"] = value,
2446            Err(error) => {
2447                state["screenshot"] = Value::Null;
2448                warnings.push(format!("screenshot failed: {error}"));
2449            }
2450        }
2451    }
2452    response.fields.insert("state".into(), state);
2453    push_warnings(&mut response, warnings);
2454    response
2455}
2456
2457fn push_warnings(response: &mut Response, extra: Vec<String>) {
2458    if extra.is_empty() {
2459        return;
2460    }
2461    let mut warnings = match response.fields.remove("warnings") {
2462        Some(Value::Array(items)) => items,
2463        Some(other) => vec![other],
2464        None => Vec::new(),
2465    };
2466    for warning in extra {
2467        warnings.push(json!(warning));
2468    }
2469    response
2470        .fields
2471        .insert("warnings".into(), Value::Array(warnings));
2472}
2473
2474fn cursor_position(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2475    let (x, y) = backend.cursor_position()?;
2476    Ok(json!({"action": "cursor_position", "x": x, "y": y}))
2477}
2478
2479fn invalid_arguments(message: &str) -> ComputerError {
2480    ComputerError::InvalidArguments(message.to_string())
2481}
2482
2483fn unsupported_error(operation: &str) -> ComputerError {
2484    ComputerError::Unsupported(format!(
2485        "{operation} is not available on {}",
2486        std::env::consts::OS
2487    ))
2488}
2489
2490fn menu_list(backend: &dyn DesktopBackend, app: &str, depth: u32) -> Result<Value, ComputerError> {
2491    let report = backend.menu_list(app, depth)?;
2492    Ok(json!({
2493        "action": "menu_list",
2494        "app": report.app,
2495        "sections": report.sections,
2496    }))
2497}
2498
2499fn menu_click(backend: &dyn DesktopBackend, app: &str, path: &str) -> Result<Value, ComputerError> {
2500    let parsed = crate::menu_path::parse(path)?;
2501    let result = backend.menu_click(app, &parsed)?;
2502    Ok(json!({
2503        "action": "menu_click",
2504        "app": app,
2505        "path": parsed.segments,
2506        "pressed": result.pressed,
2507    }))
2508}
2509
2510fn apple_event_action(
2511    backend: &dyn DesktopBackend,
2512    app: &str,
2513    script: &str,
2514    language: AppleEventLanguage,
2515) -> Result<Value, ComputerError> {
2516    let report = backend.apple_event(app, script, language)?;
2517    Ok(json!({
2518        "action": "apple_event",
2519        "app": app,
2520        "language": language.as_str(),
2521        "script_bytes": report.script_bytes,
2522        "script_hash": report.script_hash,
2523        "stdout": report.stdout,
2524        "stderr": report.stderr,
2525        "exit_status": report.exit_status,
2526    }))
2527}
2528
2529fn shortcuts_run_action(name: String) -> Result<Value, ComputerError> {
2530    let report = backend_shortcuts_run(&name)?;
2531    Ok(json!({
2532        "action": "shortcuts_run",
2533        "name": name,
2534        "exit_status": report.exit_status,
2535        "stderr": report.stderr,
2536    }))
2537}
2538
2539fn dock_list_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2540    let report = backend.dock_list()?;
2541    Ok(json!({
2542        "action": "dock_list",
2543        "items": report.items,
2544    }))
2545}
2546
2547fn dock_click_action(backend: &dyn DesktopBackend, title: &str) -> Result<Value, ComputerError> {
2548    let report = backend.dock_click(title)?;
2549    Ok(json!({
2550        "action": "dock_click",
2551        "title": title,
2552        "pressed": report.pressed,
2553    }))
2554}
2555
2556fn spaces_action(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
2557    let report = backend.spaces()?;
2558    Ok(json!({
2559        "action": "spaces",
2560        "screens": report.screens,
2561        "note": report.note,
2562    }))
2563}
2564
2565// `shortcuts` does not need the desktop backend, but the helper signature
2566// mirrors the others and leaves room for backend-specific policy hooks.
2567fn backend_shortcuts_run(name: &str) -> Result<crate::shortcuts::ShortcutsReport, ComputerError> {
2568    crate::shortcuts::run(name)
2569}
2570
2571fn session_prune(dry_run: bool, ttl_secs: Option<u64>) -> Result<Value, ComputerError> {
2572    let ttl = match ttl_secs {
2573        Some(seconds) => Duration::from_secs(seconds),
2574        None => cache::DEFAULT_SESSION_TTL,
2575    };
2576    let report = cache::prune_sessions(
2577        &cache::sessions_root(),
2578        &cache::PruneOptions {
2579            dry_run,
2580            ttl,
2581            now: SystemTime::now(),
2582        },
2583        &cache::pid_alive,
2584    )?;
2585    Ok(json!({
2586        "action": "session_prune",
2587        "dry_run": dry_run,
2588        "ttl_secs": ttl.as_secs(),
2589        "removed": report.removed,
2590        "kept": report.kept,
2591    }))
2592}
2593
2594struct ScreenshotArgs {
2595    output: Option<String>,
2596    region: Option<Region>,
2597    format: Option<ImageFormat>,
2598    max_width: Option<u32>,
2599    canvas: Option<String>,
2600    include_base64: bool,
2601    /// When set, a window target crops `region` in window-local points.
2602    /// Screenshot still refuses a region together with a window.
2603    window_crop: bool,
2604    /// Stamp numbered element rectangles on the image before encoding.
2605    marks: bool,
2606}
2607
2608fn screenshot(
2609    backend: &dyn DesktopBackend,
2610    context: &CallContext,
2611    args: ScreenshotArgs,
2612) -> Result<Value, ComputerError> {
2613    let ScreenshotArgs {
2614        output,
2615        region,
2616        format,
2617        max_width,
2618        canvas,
2619        include_base64,
2620        window_crop,
2621        marks,
2622    } = args;
2623    let targeted = context.app.is_some() || context.window.is_some();
2624    if region.is_some() && targeted && !window_crop {
2625        return Err(ComputerError::InvalidArguments(
2626            "pass a region or a window target, not both".into(),
2627        ));
2628    }
2629    let format = format.unwrap_or_default();
2630    let max_width = max_width.unwrap_or(image_util::DEFAULT_SCREENSHOT_MAX_WIDTH);
2631    let canvas = canvas
2632        .as_deref()
2633        .map(image_util::parse_canvas)
2634        .transpose()?;
2635    let raw = if targeted {
2636        if let Some(error) = backend.display_block() {
2637            return Err(error);
2638        }
2639        let target = resolve_screenshot_window(backend, context)?;
2640        // A window capture cannot be redirected onto another monitor, so a
2641        // filtered request only works when the window already sits there.
2642        if let Some(filter) = display_filter(backend, context)? {
2643            if !crate::isolation::window_inside(&filter, target.bounds) {
2644                return Err(ComputerError::InvalidArguments(format!(
2645                    "window {}x{}+{}+{} is not inside display {} ({} covers {})",
2646                    target.bounds.width,
2647                    target.bounds.height,
2648                    target.bounds.x,
2649                    target.bounds.y,
2650                    filter.display.id,
2651                    filter.display.name,
2652                    filter.bounds()
2653                )));
2654            }
2655        }
2656        let captured = backend.capture_window(&target)?;
2657        match region {
2658            Some(region) if window_crop => crop_window_region(captured, region)?,
2659            _ => captured,
2660        }
2661    } else {
2662        // Capture the chosen monitor only. `capture_region` maps the exact
2663        // monitor rectangle onto one monitor capture, so the screenshot map
2664        // origin is that monitor's origin and the image is its pixels.
2665        let region = match display_filter(backend, context)? {
2666            Some(filter) => Some(filter.region),
2667            None => region,
2668        };
2669        backend.capture_screen(region)?
2670    };
2671    let prepared = image_util::prepare_shot(
2672        raw.frame.image(),
2673        image_util::ShotFit {
2674            scale: raw.frame.scale(),
2675            origin_x: raw.origin_x,
2676            origin_y: raw.origin_y,
2677            logical_width: raw.logical_width,
2678            logical_height: raw.logical_height,
2679            max_width,
2680            canvas,
2681        },
2682    )?;
2683    let mut image = prepared.image;
2684    let mut warnings = raw.warnings;
2685    if marks {
2686        let applied = apply_marks(&mut image, &prepared.map, context, backend)?;
2687        if applied == 0 {
2688            // A silent zero is indistinguishable from "the marks were drawn".
2689            // Say so instead of writing a clean capture that answers `ok`.
2690            warnings.push("marks: no accessibility element frame fell inside this capture".into());
2691        }
2692    }
2693    let encoded = image_util::encode_rgba(&image, format)?;
2694    let path = match output {
2695        Some(path) => PathBuf::from(path),
2696        None => default_screenshot_path(format),
2697    };
2698    if let Some(parent) = path.parent() {
2699        if !parent.as_os_str().is_empty() {
2700            fs::create_dir_all(parent)?;
2701        }
2702    }
2703    fs::write(&path, &encoded.bytes)?;
2704    if let Err(error) = cache::store_screenshot_map(context.session.as_deref(), &prepared.map) {
2705        warnings.push(format!("screenshot map was not cached: {error}"));
2706    }
2707    let mut body = serde_json::to_value(&prepared.map).map_err(|error| {
2708        ComputerError::ExecutionFailed(format!("screenshot map encode failed: {error}"))
2709    })?;
2710    body["path"] = json!(path_string(&path));
2711    body["format"] = json!(format.as_str());
2712    body["capture_source"] = json!(raw.capture_source);
2713    if raw.occluded {
2714        body["occluded"] = json!(true);
2715    }
2716    if !warnings.is_empty() {
2717        body["warnings"] = json!(warnings);
2718    }
2719    if include_base64 {
2720        body["data"] = json!(encode_base64(&encoded.bytes));
2721    }
2722    Ok(body)
2723}
2724
2725fn resolve_screenshot_window(
2726    backend: &dyn DesktopBackend,
2727    context: &CallContext,
2728) -> Result<WindowTarget, ComputerError> {
2729    let listed = backend.list_windows()?;
2730    if let Some(app) = context.app.as_deref() {
2731        let apps = backend.list_apps(true)?;
2732        let selector = parse_app_selector(app)?;
2733        let app = resolve_app(&apps, &selector)?;
2734        let owned = filter_windows_for_pid(&listed.windows, app.pid);
2735        let selector = match context.window.as_deref() {
2736            Some(window) => parse_window_selector(window)?,
2737            None => WindowSelector::Index(0),
2738        };
2739        let window = resolve_window(&owned, &selector)?;
2740        return Ok(window_target(window));
2741    }
2742    let selector = parse_window_selector(context.window.as_deref().unwrap_or(""))?;
2743    let window = resolve_window(&listed.windows, &selector)?;
2744    Ok(window_target(window))
2745}
2746
2747fn window_target(window: &crate::target::WindowInfo) -> WindowTarget {
2748    WindowTarget {
2749        window_id: window.id,
2750        pid: window.pid,
2751        bounds: window.bounds,
2752    }
2753}
2754
2755/// Crop `region` in window-local logical points. The far edge may equal the window size.
2756fn crop_window_region(
2757    raw: crate::types::RawCapture,
2758    region: Region,
2759) -> Result<crate::types::RawCapture, ComputerError> {
2760    region_inside(region, raw.logical_width, raw.logical_height)?;
2761    let scale = raw.frame.scale();
2762    let x = logical_to_device(i64::from(region.x), scale)?;
2763    let y = logical_to_device(i64::from(region.y), scale)?;
2764    let width = logical_len_to_device(region.width, scale)?;
2765    let height = logical_len_to_device(region.height, scale)?;
2766    let image = raw.frame.image();
2767    if x >= image.width() || y >= image.height() {
2768        return Err(ComputerError::InvalidArguments(
2769            "region is outside the capture".into(),
2770        ));
2771    }
2772    let width = width.min(image.width() - x).max(1);
2773    let height = height.min(image.height() - y).max(1);
2774    let cropped = image::imageops::crop_imm(image, x, y, width, height).to_image();
2775    Ok(crate::types::RawCapture {
2776        frame: Frame::from_image(cropped, scale),
2777        origin_x: raw.origin_x.saturating_add(region.x),
2778        origin_y: raw.origin_y.saturating_add(region.y),
2779        logical_width: region.width,
2780        logical_height: region.height,
2781        capture_source: raw.capture_source,
2782        occluded: raw.occluded,
2783        warnings: raw.warnings,
2784    })
2785}
2786
2787fn logical_to_device(value: i64, scale: f64) -> Result<u32, ComputerError> {
2788    if value < 0 {
2789        return Err(ComputerError::InvalidArguments(
2790            "region is outside the capture".into(),
2791        ));
2792    }
2793    let pixels = (value as f64) * scale;
2794    if !pixels.is_finite() || pixels < 0.0 || pixels > f64::from(u32::MAX) {
2795        return Err(ComputerError::ExecutionFailed(
2796            "region pixel coordinate is out of range".into(),
2797        ));
2798    }
2799    Ok(pixels.round() as u32)
2800}
2801
2802fn logical_len_to_device(value: u32, scale: f64) -> Result<u32, ComputerError> {
2803    let pixels = logical_to_device(i64::from(value), scale)?;
2804    Ok(pixels.max(1))
2805}
2806
2807fn default_screenshot_path(format: ImageFormat) -> PathBuf {
2808    static SEQUENCE: AtomicU64 = AtomicU64::new(0);
2809    let nanos = SystemTime::now()
2810        .duration_since(UNIX_EPOCH)
2811        .map(|duration| duration.as_nanos())
2812        .unwrap_or(0);
2813    let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed);
2814    let name = format!(
2815        "computer-use-{}-{nanos}-{sequence}.{}",
2816        std::process::id(),
2817        format.extension()
2818    );
2819    std::env::temp_dir().join(name)
2820}
2821
2822fn path_string(path: &Path) -> String {
2823    path.to_string_lossy().into_owned()
2824}
2825
2826fn click(
2827    backend: &dyn DesktopBackend,
2828    context: &CallContext,
2829    x: Option<i32>,
2830    y: Option<i32>,
2831    button: MouseButton,
2832    double: bool,
2833    count: u8,
2834) -> Response {
2835    let count = match effective_click_count(double, count) {
2836        Ok(count) => count,
2837        Err(error) => return Response::from_error(error),
2838    };
2839    match (context.element, x, y) {
2840        (Some(_), Some(_), _) | (Some(_), _, Some(_)) => {
2841            Response::failure("invalid_arguments", "pass element or x and y, not both")
2842        }
2843        (None, None, None) => Response::failure(
2844            "invalid_arguments",
2845            "click requires element, or both x and y",
2846        ),
2847        (None, Some(_), None) | (None, None, Some(_)) => {
2848            Response::failure("invalid_arguments", "x and y must be passed together")
2849        }
2850        (Some(_index), None, None) => {
2851            element_command(backend, context, ElementOp::Press { count, button })
2852        }
2853        (None, Some(x), Some(y)) => coordinate_click(backend, context, x, y, button, count),
2854    }
2855}
2856
2857fn effective_click_count(double: bool, count: u8) -> Result<u8, ComputerError> {
2858    if count == 0 || count > 3 {
2859        return Err(ComputerError::InvalidArguments(
2860            "click count must be 1, 2, or 3".into(),
2861        ));
2862    }
2863    if double && count != 1 && count != 2 {
2864        return Err(ComputerError::InvalidArguments(
2865            "double and count disagree".into(),
2866        ));
2867    }
2868    if double {
2869        Ok(2)
2870    } else {
2871        Ok(count)
2872    }
2873}
2874
2875fn coordinate_click(
2876    backend: &dyn DesktopBackend,
2877    context: &CallContext,
2878    x: i32,
2879    y: i32,
2880    button: MouseButton,
2881    count: u8,
2882) -> Response {
2883    let kind = InputKind::Click {
2884        x,
2885        y,
2886        button,
2887        count,
2888    };
2889    // Tier rejection, including synthetic without the feature, happens before hit-testing.
2890    if let Err(error) = planned_tiers(backend, context) {
2891        return Response::from_error(error);
2892    }
2893    let mut warnings = Vec::new();
2894    let hit = match backend.hit_test(x, y) {
2895        Ok(hit) => hit,
2896        Err(error) => {
2897            warnings.push(format!("hit test failed: {error}"));
2898            None
2899        }
2900    };
2901    if let Some(title) = hit.as_ref().and_then(|hit| hit.title.as_deref()) {
2902        if policy::sensitive_title(title) {
2903            if let Err(error) = confirm_title(context, "click", "sensitive control title") {
2904                return Response::from_error(error);
2905            }
2906        }
2907    }
2908    match run_attempts(backend, context, kind) {
2909        Ok(report) => {
2910            let action_name = match count {
2911                2 => "double_click",
2912                3 => "triple_click",
2913                _ => "click",
2914            };
2915            let body = json!({
2916                "action": action_name,
2917                "x": x,
2918                "y": y,
2919                "button": button.as_str(),
2920                "double": count == 2,
2921                "count": count,
2922                "hit": hit_value(hit),
2923            });
2924            Response::success(finish_input(body, report, warnings))
2925        }
2926        Err(error) => Response::from_error(error),
2927    }
2928}
2929
2930fn hit_value(hit: Option<crate::element::HitInfo>) -> Value {
2931    match hit {
2932        Some(hit) => json!({"role": hit.role, "title": hit.title}),
2933        None => Value::Null,
2934    }
2935}
2936
2937fn type_command(backend: &dyn DesktopBackend, context: &CallContext, text: String) -> Response {
2938    if context.element.is_some() {
2939        return element_command(backend, context, ElementOp::Type { text });
2940    }
2941    if text.is_empty() {
2942        return Response::from_error(ComputerError::InvalidArguments(
2943            "text must not be empty".into(),
2944        ));
2945    }
2946    if text.contains('\0') {
2947        return Response::from_error(ComputerError::InvalidArguments(
2948            "text must not contain a null byte".into(),
2949        ));
2950    }
2951    let chars = text.chars().count();
2952    match run_attempts(backend, context, InputKind::Type { text: text.clone() }) {
2953        Ok(report) => {
2954            let body = json!({
2955                "action": "type",
2956                "text": text,
2957                "chars": chars,
2958            });
2959            Response::success(finish_input(body, report, Vec::new()))
2960        }
2961        Err(error) => Response::from_error(error),
2962    }
2963}
2964
2965fn element_command(backend: &dyn DesktopBackend, context: &CallContext, op: ElementOp) -> Response {
2966    if let Err(error) = precheck(&op) {
2967        return Response::from_error(error);
2968    }
2969    if let Err(error) = confirm_element_title(backend, context, &op) {
2970        return Response::from_error(error);
2971    }
2972    let index = match context.element {
2973        Some(index) => index,
2974        None => {
2975            return Response::from_error(ComputerError::InvalidArguments(
2976                "element is required".into(),
2977            ))
2978        }
2979    };
2980    let request = ElementRequest {
2981        op,
2982        index,
2983        require_visible: context.require_visible,
2984        app: context.app.clone(),
2985        window: context.window.clone(),
2986        session: context.session.clone(),
2987        tier: context.tier.clone(),
2988        allow_foreground: context.allow_foreground,
2989        require_background: context.require_background,
2990        no_overlay: context.no_overlay,
2991    };
2992    match backend.element_action(&request) {
2993        Ok(outcome) => Response::success(outcome_value(&outcome)),
2994        Err(error) => element_error_response(error),
2995    }
2996}
2997
2998fn confirm_element_title(
2999    backend: &dyn DesktopBackend,
3000    context: &CallContext,
3001    op: &ElementOp,
3002) -> Result<(), ComputerError> {
3003    let press = matches!(op, ElementOp::Press { .. })
3004        || matches!(op, ElementOp::Perform { name } if name.eq_ignore_ascii_case("AXPress"));
3005    if !press {
3006        return Ok(());
3007    }
3008    let Some(index) = context.element else {
3009        return Ok(());
3010    };
3011    let Ok(Some(peek)) = backend.peek_element(
3012        context.app.as_deref(),
3013        context.window.as_deref(),
3014        context.session.as_deref(),
3015        index,
3016    ) else {
3017        return Ok(());
3018    };
3019    if peek.title.as_deref().is_some_and(policy::sensitive_title) {
3020        confirm_title(context, "press", "sensitive control title")?;
3021    }
3022    Ok(())
3023}
3024
3025fn confirm_title(context: &CallContext, action: &str, reason: &str) -> Result<(), ComputerError> {
3026    let loaded = policy::load_policy(context.policy.as_deref().map(Path::new))?;
3027    let command = policy::confirm_program(&loaded, context.confirm_cmd.as_deref());
3028    policy::run_confirm(
3029        command.as_deref(),
3030        loaded.timeout_secs(),
3031        &policy::ConfirmDocument {
3032            action,
3033            bundle_id: None,
3034            path: None,
3035            pid: None,
3036            window_id: None,
3037            element: context.element,
3038            reason,
3039        },
3040    )
3041}
3042
3043fn precheck(op: &ElementOp) -> Result<(), ComputerError> {
3044    match op {
3045        ElementOp::Press { count, .. } => element::validate_count(*count),
3046        ElementOp::SetValue { value } => element::validate_text(value, true),
3047        ElementOp::Type { text } => element::validate_text(text, false),
3048        ElementOp::SelectText { start, end } => element::validate_range(*start, *end, None),
3049        ElementOp::Perform { name } => element::validate_perform_name(name),
3050        ElementOp::Focus | ElementOp::ScrollTo => Ok(()),
3051    }
3052}
3053
3054fn outcome_value(outcome: &ElementOutcome) -> Value {
3055    let mut body = json!({
3056        "action": outcome.action,
3057        "element": outcome.element,
3058        "revision": outcome.revision,
3059        "tier": outcome.tier,
3060        "cursor_moved": outcome.cursor_moved,
3061    });
3062    if let Some(restored) = outcome.focus_restored {
3063        body["focus_restored"] = json!(restored);
3064    }
3065    if !outcome.warnings.is_empty() {
3066        body["warnings"] = json!(outcome.warnings);
3067    }
3068    if let Some(via) = &outcome.via {
3069        body["via"] = json!(via);
3070    }
3071    if let Some(name) = &outcome.name {
3072        body["name"] = json!(name);
3073    }
3074    if let Some(count) = outcome.count {
3075        body["count"] = json!(count);
3076    }
3077    body
3078}
3079
3080fn element_error_response(error: ElementError) -> Response {
3081    let mut response = Response::from_error(error.error);
3082    if let Some(tree) = error.tree {
3083        response.fields.insert("tree".into(), json!(tree));
3084    }
3085    response
3086}
3087
3088fn move_mouse(backend: &dyn DesktopBackend, context: &CallContext, x: i32, y: i32) -> Response {
3089    match run_attempts(backend, context, InputKind::Move { x, y }) {
3090        Ok(report) => Response::success(finish_input(
3091            json!({"action": "move", "x": x, "y": y}),
3092            report,
3093            Vec::new(),
3094        )),
3095        Err(error) => Response::from_error(error),
3096    }
3097}
3098
3099fn press_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str) -> Response {
3100    let strokes = match key_strokes(keys) {
3101        Ok(strokes) => strokes,
3102        Err(error) => return Response::from_error(error),
3103    };
3104    let mut body = json!({
3105        "action": "key",
3106        "keys": keys,
3107    });
3108    if strokes.len() == 1 {
3109        let modifiers: Vec<&str> = strokes[0]
3110            .modifiers
3111            .iter()
3112            .copied()
3113            .map(Modifier::as_str)
3114            .collect();
3115        body["modifiers"] = json!(modifiers);
3116        body["key"] = json!(strokes[0].key);
3117    } else {
3118        body["strokes"] = json!(strokes.len());
3119    }
3120    match run_attempts(backend, context, InputKind::Key { strokes }) {
3121        Ok(report) => Response::success(finish_input(body, report, Vec::new())),
3122        Err(error) => Response::from_error(error),
3123    }
3124}
3125
3126fn key_strokes(keys: &str) -> Result<Vec<KeyStroke>, ComputerError> {
3127    let combos = parse_key_sequence(keys)?;
3128    let mut strokes = Vec::with_capacity(combos.len());
3129    for combo in combos {
3130        validate_key(&combo.key)?;
3131        strokes.push(KeyStroke {
3132            modifiers: combo.modifiers,
3133            key: combo.key,
3134        });
3135    }
3136    Ok(strokes)
3137}
3138
3139fn drag(
3140    backend: &dyn DesktopBackend,
3141    context: &CallContext,
3142    path: Vec<Point>,
3143    button: MouseButton,
3144) -> Response {
3145    if path.len() < 2 {
3146        return Response::from_error(ComputerError::InvalidArguments(
3147            "drag needs at least 2 points".into(),
3148        ));
3149    }
3150    let points: Vec<(i32, i32)> = path.iter().map(|point| (point.x, point.y)).collect();
3151    let moves = points.len() - 1;
3152    match run_attempts(backend, context, InputKind::Drag { points, button }) {
3153        Ok(report) => Response::success(finish_input(
3154            json!({
3155                "action": "drag",
3156                "button": button.as_str(),
3157                "points": path.len(),
3158                "moves": moves,
3159            }),
3160            report,
3161            Vec::new(),
3162        )),
3163        Err(error) => Response::from_error(error),
3164    }
3165}
3166
3167fn pointer_half(
3168    backend: &dyn DesktopBackend,
3169    context: &CallContext,
3170    x: i32,
3171    y: i32,
3172    button: MouseButton,
3173    down: bool,
3174) -> Response {
3175    let kind = if down {
3176        InputKind::MouseDown { x, y, button }
3177    } else {
3178        InputKind::MouseUp { x, y, button }
3179    };
3180    let name = if down { "mouse_down" } else { "mouse_up" };
3181    match run_attempts(backend, context, kind) {
3182        Ok(report) => Response::success(finish_input(
3183            json!({
3184                "action": name,
3185                "x": x,
3186                "y": y,
3187                "button": button.as_str(),
3188            }),
3189            report,
3190            Vec::new(),
3191        )),
3192        Err(error) => Response::from_error(error),
3193    }
3194}
3195
3196fn hold_key(backend: &dyn DesktopBackend, context: &CallContext, keys: &str, ms: u64) -> Response {
3197    let stroke = match one_hold(keys, ms) {
3198        Ok(stroke) => stroke,
3199        Err(error) => return Response::from_error(error),
3200    };
3201    match run_attempts(
3202        backend,
3203        context,
3204        InputKind::HoldKey {
3205            modifiers: stroke.modifiers.clone(),
3206            key: stroke.key.clone(),
3207            ms,
3208        },
3209    ) {
3210        Ok(report) => Response::success(finish_input(
3211            json!({
3212                "action": "hold_key",
3213                "keys": keys,
3214                "key": stroke.key,
3215                "ms": ms,
3216            }),
3217            report,
3218            Vec::new(),
3219        )),
3220        Err(error) => Response::from_error(error),
3221    }
3222}
3223
3224fn one_hold(keys: &str, ms: u64) -> Result<KeyStroke, ComputerError> {
3225    if ms > tier::MAX_HOLD_MS {
3226        return Err(ComputerError::InvalidArguments(format!(
3227            "hold_key ms must be at most {}",
3228            tier::MAX_HOLD_MS
3229        )));
3230    }
3231    if keys.split_whitespace().count() != 1 {
3232        return Err(ComputerError::InvalidArguments(
3233            "hold_key accepts one key combination".into(),
3234        ));
3235    }
3236    let combo = parse_key_combo(keys)?;
3237    validate_key(&combo.key)?;
3238    Ok(KeyStroke {
3239        modifiers: combo.modifiers,
3240        key: combo.key,
3241    })
3242}
3243
3244fn clipboard_chars(action: &Action) -> Option<u64> {
3245    match action {
3246        Action::ClipboardSet {
3247            text: Some(text), ..
3248        } => Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX)),
3249        Action::BrowserType { text, .. } => {
3250            Some(u64::try_from(text.chars().count()).unwrap_or(u64::MAX))
3251        }
3252        _ => None,
3253    }
3254}
3255
3256fn preflight_action(action: &Action) -> Result<(), ComputerError> {
3257    match action {
3258        Action::Click { double, count, .. } => {
3259            effective_click_count(*double, *count)?;
3260        }
3261        Action::Press { count, .. } => element::validate_count(*count)?,
3262        Action::Drag { path, .. } if path.len() < 2 => {
3263            return Err(ComputerError::InvalidArguments(
3264                "drag needs at least 2 points".into(),
3265            ));
3266        }
3267        Action::HoldKey { keys, ms } => {
3268            one_hold(keys, *ms)?;
3269        }
3270        Action::Key { keys } => {
3271            key_strokes(keys)?;
3272        }
3273        Action::ClipboardSet { text, image } => match (text, image) {
3274            (Some(text), None) if text.is_empty() => {
3275                return Err(ComputerError::InvalidArguments(
3276                    "clipboard text must not be empty".into(),
3277                ));
3278            }
3279            (None, None) => {
3280                return Err(ComputerError::InvalidArguments(
3281                    "clipboard set requires text or image".into(),
3282                ));
3283            }
3284            (Some(_), Some(_)) => {
3285                return Err(ComputerError::InvalidArguments(
3286                    "pass clipboard text or image, not both".into(),
3287                ));
3288            }
3289            _ => {}
3290        },
3291        _ => {}
3292    }
3293    Ok(())
3294}
3295
3296fn clipboard_get(
3297    backend: &dyn DesktopBackend,
3298    context: &CallContext,
3299) -> Result<serde_json::Value, ComputerError> {
3300    match backend.clipboard_get()? {
3301        ClipboardContents::Empty => Ok(json!({
3302            "action": "clipboard_get",
3303            "kind": "empty",
3304        })),
3305        ClipboardContents::Text(text) => {
3306            let chars = text.chars().count();
3307            Ok(json!({
3308                "action": "clipboard_get",
3309                "kind": "text",
3310                "text": text,
3311                "chars": chars,
3312            }))
3313        }
3314        ClipboardContents::Image {
3315            width,
3316            height,
3317            rgba,
3318        } => {
3319            let frame = Frame::new(width, height, 1.0, rgba)?;
3320            let encoded = image_util::encode_rgba(frame.image(), ImageFormat::Png)?;
3321            let dir = cache::session_dir(context.session.as_deref())?;
3322            let path = dir.join("clipboard.png");
3323            cache::write_secret_bytes(&path, &encoded.bytes)?;
3324            Ok(json!({
3325                "action": "clipboard_get",
3326                "kind": "image",
3327                "path": path_string(&path),
3328                "width": width,
3329                "height": height,
3330            }))
3331        }
3332    }
3333}
3334
3335fn clipboard_set(
3336    backend: &dyn DesktopBackend,
3337    text: Option<String>,
3338    image: Option<String>,
3339) -> Result<serde_json::Value, ComputerError> {
3340    match (text, image) {
3341        (Some(text), None) => {
3342            let chars = text.chars().count();
3343            backend.clipboard_set_text(&text)?;
3344            Ok(json!({
3345                "action": "clipboard_set",
3346                "kind": "text",
3347                "chars": chars,
3348            }))
3349        }
3350        (None, Some(path)) => {
3351            let (width, height, rgba) = decode_clipboard_image(&path)?;
3352            backend.clipboard_set_image(width, height, &rgba)?;
3353            Ok(json!({
3354                "action": "clipboard_set",
3355                "kind": "image",
3356                "width": width,
3357                "height": height,
3358            }))
3359        }
3360        _ => Err(ComputerError::InvalidArguments(
3361            "clipboard set requires text or image".into(),
3362        )),
3363    }
3364}
3365
3366fn decode_clipboard_image(path: &str) -> Result<(u32, u32, Vec<u8>), ComputerError> {
3367    let image = image::ImageReader::open(path)
3368        .map_err(|error| {
3369            ComputerError::InvalidArguments(format!("clipboard image could not be opened: {error}"))
3370        })?
3371        .decode()
3372        .map_err(|error| {
3373            ComputerError::InvalidArguments(format!(
3374                "clipboard image could not be decoded: {error}"
3375            ))
3376        })?
3377        .to_rgba8();
3378    let width = image.width();
3379    let height = image.height();
3380    Ok((width, height, image.into_raw()))
3381}
3382
3383fn scroll(
3384    backend: &dyn DesktopBackend,
3385    context: &CallContext,
3386    point: (Option<i32>, Option<i32>),
3387    delta: (Option<i32>, Option<i32>),
3388    direction: Option<ScrollDirection>,
3389    amount: Option<i32>,
3390) -> Response {
3391    let (x, y) = point;
3392    let (dx, dy) = delta;
3393    if x.is_some() != y.is_some() {
3394        return Response::from_error(ComputerError::InvalidArguments(
3395            "scroll x and y must be set together".into(),
3396        ));
3397    }
3398    let (scroll_dx, scroll_dy) = match resolve_scroll(dx, dy, direction, amount) {
3399        Ok(delta) => delta,
3400        Err(error) => return Response::from_error(error),
3401    };
3402    match run_attempts(
3403        backend,
3404        context,
3405        InputKind::Scroll {
3406            x,
3407            y,
3408            dx: scroll_dx,
3409            dy: scroll_dy,
3410        },
3411    ) {
3412        Ok(report) => {
3413            let body = json!({
3414                "action": "scroll",
3415                "x": x,
3416                "y": y,
3417                "dx": scroll_dx,
3418                "dy": scroll_dy,
3419            });
3420            Response::success(finish_input(scroll_body(body, &report), report, Vec::new()))
3421        }
3422        Err(error) => Response::from_error(error),
3423    }
3424}
3425
3426/// Add the verified-scroll fields when the backend reported them.
3427///
3428/// `via` is `event` when the posted wheel event moved the scroll bar and
3429/// `ax_scrollbar` when `AXValue` had to be written instead. `step` is the
3430/// per-line fraction (0..=1) the AX scroll-bar path computed from the
3431/// content-over-visible sizes, surfaced for transparency even when the wheel
3432/// event tier did the scrolling. Windows and Linux report no outcome, so
3433/// their response keeps its existing shape.
3434fn scroll_body(mut body: Value, report: &crate::tier::InputReport) -> Value {
3435    let Some(outcome) = &report.scroll else {
3436        return body;
3437    };
3438    body["via"] = json!(outcome.via.as_str());
3439    body["scrolled"] = json!(outcome.scrolled);
3440    body["before"] = json!(outcome.before);
3441    body["after"] = json!(outcome.after);
3442    body["step"] = json!(outcome.step);
3443    body
3444}
3445
3446fn planned_tiers(
3447    backend: &dyn DesktopBackend,
3448    context: &CallContext,
3449) -> Result<Vec<TierName>, ComputerError> {
3450    let requested = crate::tier::parse_tier(context.tier.as_deref())?;
3451    crate::tier::plan(
3452        requested,
3453        crate::tier::InputClass::Background,
3454        backend.input_support(),
3455        crate::tier::skylight_compiled(),
3456    )
3457}
3458
3459fn run_attempts(
3460    backend: &dyn DesktopBackend,
3461    context: &CallContext,
3462    kind: InputKind,
3463) -> Result<crate::tier::InputReport, ComputerError> {
3464    attempt::run(backend, &attempt_flags(context), kind)
3465}
3466
3467fn attempt_flags(context: &CallContext) -> AttemptFlags {
3468    AttemptFlags {
3469        app: context.app.clone(),
3470        window: context.window.clone(),
3471        tier: context.tier.clone(),
3472        allow_foreground: context.allow_foreground,
3473        require_background: context.require_background,
3474        no_overlay: context.no_overlay,
3475        session: context.session.clone(),
3476    }
3477}
3478
3479fn finish_input(
3480    mut body: Value,
3481    report: crate::tier::InputReport,
3482    mut warnings: Vec<String>,
3483) -> Value {
3484    warnings.extend(report.warnings);
3485    body["tier"] = json!(report.tier);
3486    body["cursor_moved"] = json!(report.cursor_moved);
3487    if let Some(restored) = report.focus_restored {
3488        body["focus_restored"] = json!(restored);
3489    }
3490    if !warnings.is_empty() {
3491        body["warnings"] = json!(warnings);
3492    }
3493    body
3494}
3495
3496pub(crate) fn resolve_scroll(
3497    dx: Option<i32>,
3498    dy: Option<i32>,
3499    direction: Option<ScrollDirection>,
3500    amount: Option<i32>,
3501) -> Result<(i32, i32), ComputerError> {
3502    let has_delta = dx.is_some() || dy.is_some();
3503    let has_direction = direction.is_some() || amount.is_some();
3504    if has_delta && has_direction {
3505        return Err(ComputerError::InvalidArguments(
3506            "use either dx/dy or direction/amount, not both".into(),
3507        ));
3508    }
3509    if let Some(direction) = direction {
3510        let amount = amount.unwrap_or(1);
3511        return Ok(match direction {
3512            ScrollDirection::Up => (0, -amount),
3513            ScrollDirection::Down => (0, amount),
3514            ScrollDirection::Left => (-amount, 0),
3515            ScrollDirection::Right => (amount, 0),
3516        });
3517    }
3518    if amount.is_some() {
3519        return Err(ComputerError::InvalidArguments(
3520            "amount requires direction".into(),
3521        ));
3522    }
3523    if !has_delta {
3524        return Err(ComputerError::InvalidArguments(
3525            "scroll requires dx/dy or direction".into(),
3526        ));
3527    }
3528    Ok((dx.unwrap_or(0), dy.unwrap_or(0)))
3529}
3530
3531fn wait(ms: Option<u64>, seconds: Option<f64>) -> Result<Value, ComputerError> {
3532    let waited = resolve_wait(ms, seconds)?;
3533    thread::sleep(Duration::from_millis(waited));
3534    Ok(json!({"waited_ms": waited}))
3535}
3536
3537pub(crate) fn resolve_wait(ms: Option<u64>, seconds: Option<f64>) -> Result<u64, ComputerError> {
3538    if ms.is_none() && seconds.is_none() {
3539        return Err(ComputerError::InvalidArguments(
3540            "wait requires ms, seconds, or both".into(),
3541        ));
3542    }
3543    let mut total: u128 = u128::from(ms.unwrap_or(0));
3544    if let Some(seconds) = seconds {
3545        if !seconds.is_finite() || seconds < 0.0 {
3546            return Err(ComputerError::InvalidArguments(
3547                "seconds must be a finite number greater than or equal to 0".into(),
3548            ));
3549        }
3550        let extra = (seconds * 1000.0).round();
3551        if extra > u128::MAX as f64 {
3552            return Err(ComputerError::InvalidArguments("wait is too long".into()));
3553        }
3554        total = total.saturating_add(extra as u128);
3555    }
3556    if total > MAX_WAIT_MS {
3557        return Err(ComputerError::InvalidArguments(
3558            "wait must be at most 24 hours".into(),
3559        ));
3560    }
3561    u64::try_from(total).map_err(|_| ComputerError::InvalidArguments("wait is too long".into()))
3562}
3563
3564fn displays(backend: &dyn DesktopBackend) -> Result<Value, ComputerError> {
3565    let displays = backend.list_displays()?;
3566    Ok(json!({"displays": displays}))
3567}
3568
3569/// Resolve `--display` for a capture. `None` means no filter.
3570fn display_filter(
3571    backend: &dyn DesktopBackend,
3572    context: &CallContext,
3573) -> Result<Option<crate::isolation::DisplayTarget>, ComputerError> {
3574    match context.display.as_deref() {
3575        None => Ok(None),
3576        Some(spec) => Ok(Some(crate::isolation::resolve(
3577            spec,
3578            &backend.list_displays()?,
3579        )?)),
3580    }
3581}
3582
3583/// Body of the `isolation` action: what this build can and cannot isolate.
3584fn isolation() -> Result<Value, ComputerError> {
3585    Ok(crate::isolation::matrix())
3586}
3587
3588fn tree_query(context: &CallContext) -> TreeQuery {
3589    TreeQuery {
3590        app: context.app.clone(),
3591        window: context.window.clone(),
3592        full: context.full,
3593        session: context.session.clone(),
3594    }
3595}
3596
3597/// OCR result for a window capture. The dispatcher returns one block list per
3598/// recognized line. macOS reuses `VNRecognizeTextRequest`, Windows uses
3599/// `Windows.Media.Ocr.OcrEngine`, and Linux is a structured `Unsupported`.
3600fn ocr_action(
3601    backend: &dyn DesktopBackend,
3602    context: &CallContext,
3603    _force: bool,
3604) -> Result<Value, ComputerError> {
3605    let targeted = context.app.is_some() || context.window.is_some();
3606    if !targeted {
3607        return Err(ComputerError::InvalidArguments(
3608            "ocr requires --app or --window".into(),
3609        ));
3610    }
3611    let prepared = latest_prepared_ocr_capture(backend, context)?;
3612    let OcrResult { blocks, note } = backend.recognize_text(&prepared.image, &prepared.map)?;
3613    let redacted = redact_secure_ocr_blocks(blocks, backend, context, &prepared.map)?;
3614    let blocks_json: Vec<Value> = redacted
3615        .iter()
3616        .map(|block| {
3617            let image_rect = crate::vision::ocr::ImageRect {
3618                x: block.x,
3619                y: block.y,
3620                width: block.width,
3621                height: block.height,
3622            };
3623            let screen = image_rect.to_screen(&prepared.map);
3624            let screen_json = match screen {
3625                Some((x, y)) => json!({
3626                    "x": x.round() as i64,
3627                    "y": y.round() as i64,
3628                }),
3629                None => Value::Null,
3630            };
3631            json!({
3632                "text": block.text,
3633                "x": block.x,
3634                "y": block.y,
3635                "width": block.width,
3636                "height": block.height,
3637                "confidence": block.confidence,
3638                "screen": screen_json,
3639            })
3640        })
3641        .collect();
3642    let mut body = json!({
3643        "action": "ocr",
3644        "blocks": blocks_json,
3645    });
3646    if let Some(note) = note {
3647        body["note"] = json!(note);
3648    }
3649    Ok(body)
3650}
3651
3652/// Drop blocks that fall inside any secure element's frame. Re-uses the
3653/// vision redaction helper so the rule lives in one place. The dispatcher
3654/// is conservative: when the secure frame list is empty (the typical
3655/// Linux/Windows case or a tree-unavailable snapshot), every block is kept.
3656fn redact_secure_ocr_blocks(
3657    blocks: Vec<OcrBlock>,
3658    backend: &dyn DesktopBackend,
3659    context: &CallContext,
3660    map: &crate::image_util::ScreenshotMap,
3661) -> Result<Vec<OcrBlock>, ComputerError> {
3662    // We do not walk the UiNode tree directly; the backend's secure-frame
3663    // extraction lives in a `secure_ocr_frames()` method. It receives the
3664    // screenshot map because platform frames are screen points and the OCR
3665    // blocks are image pixels. The default is an empty list.
3666    let secure_frames = backend.secure_ocr_frames(&tree_query(context), map)?;
3667    Ok(crate::vision::redact_blocks(blocks, secure_frames))
3668}
3669
3670/// One OCR-ready capture: the prepared shot pixels plus its screenshot map.
3671struct OcrPreparedCapture {
3672    image: image::RgbaImage,
3673    map: crate::image_util::ScreenshotMap,
3674}
3675
3676/// Re-run the same capture the screenshot command uses so OCR has the same
3677/// letterboxed source. Failures here are returned to the caller, never
3678/// swallowed.
3679fn latest_prepared_ocr_capture(
3680    backend: &dyn DesktopBackend,
3681    context: &CallContext,
3682) -> Result<OcrPreparedCapture, ComputerError> {
3683    let targeted = context.app.is_some() || context.window.is_some();
3684    if targeted {
3685        if let Some(error) = backend.display_block() {
3686            return Err(error);
3687        }
3688        let target = resolve_screenshot_window(backend, context)?;
3689        let captured = backend.capture_window(&target)?;
3690        let prepared = image_util::prepare_shot(
3691            captured.frame.image(),
3692            image_util::ShotFit {
3693                scale: captured.frame.scale(),
3694                origin_x: captured.origin_x,
3695                origin_y: captured.origin_y,
3696                logical_width: captured.logical_width,
3697                logical_height: captured.logical_height,
3698                max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3699                canvas: None,
3700            },
3701        )?;
3702        Ok(OcrPreparedCapture {
3703            image: prepared.image,
3704            map: prepared.map,
3705        })
3706    } else {
3707        // Full screen capture for OCR (rare, but allowed by the protocol).
3708        let raw = backend.capture_screen(None)?;
3709        let prepared = image_util::prepare_shot(
3710            raw.frame.image(),
3711            image_util::ShotFit {
3712                scale: raw.frame.scale(),
3713                origin_x: raw.origin_x,
3714                origin_y: raw.origin_y,
3715                logical_width: raw.logical_width,
3716                logical_height: raw.logical_height,
3717                max_width: image_util::DEFAULT_SCREENSHOT_MAX_WIDTH,
3718                canvas: None,
3719            },
3720        )?;
3721        Ok(OcrPreparedCapture {
3722            image: prepared.image,
3723            map: prepared.map,
3724        })
3725    }
3726}
3727
3728/// Draw tree index marks on `image`. Returns the count of marks applied.
3729///
3730/// The overlay is painted onto a copy, so the returned image has to be written
3731/// back into `*image`. Dropping it leaves the caller's pixels untouched and
3732/// `screenshot --marks` writes a clean capture while still answering `ok`.
3733fn apply_marks(
3734    image: &mut image::RgbaImage,
3735    map: &crate::image_util::ScreenshotMap,
3736    context: &CallContext,
3737    backend: &dyn DesktopBackend,
3738) -> Result<usize, ComputerError> {
3739    let boxes = backend.marks_for_tree(&tree_query(context), map)?;
3740    if boxes.is_empty() {
3741        return Ok(0);
3742    }
3743    let marked = crate::vision::draw_marks(image, &boxes);
3744    *image = marked;
3745    Ok(boxes.len())
3746}
3747
3748fn tree_action(
3749    backend: &dyn DesktopBackend,
3750    context: &CallContext,
3751) -> Result<Value, ComputerError> {
3752    let report = backend.tree(&tree_query(context))?;
3753    let mut body = json!({
3754        "pid": report.pid,
3755        "window_id": report.window_id,
3756        "revision": report.revision,
3757        "source": report.source,
3758        "truncated": report.truncated,
3759        "tree": report.tree,
3760    });
3761    if let Some(note) = report.note {
3762        body["note"] = json!(note);
3763    }
3764    if let Some(legacy) = report.legacy.as_deref() {
3765        body["accessibility_legacy"] = serde_json::from_str(legacy).unwrap_or(Value::Null);
3766    }
3767    let tree_was_empty = report.full_text.trim().is_empty();
3768    if tree_was_empty {
3769        // P11: surface the OCR fallback when the tree produced nothing, and
3770        // a window target was selected. The agent decides whether to run it.
3771        if context.app.is_some() || context.window.is_some() {
3772            body["ocr_fallback_available"] = json!(true);
3773        }
3774    }
3775    Ok(body)
3776}
3777
3778fn session_end(
3779    backend: &dyn DesktopBackend,
3780    context: &CallContext,
3781) -> Result<Value, ComputerError> {
3782    let restored = backend.end_session(context.session.as_deref())?;
3783    let mut body = json!({
3784        "action": "session_end",
3785        "restored": restored,
3786    });
3787    if let Err(error) = cache::prune_empty_sessions() {
3788        body["warnings"] = json!([format!("session prune failed: {error}")]);
3789    }
3790    Ok(body)
3791}
3792
3793fn list_apps(backend: &dyn DesktopBackend, all: bool) -> Result<Value, ComputerError> {
3794    let apps = backend.list_apps(all)?;
3795    Ok(json!({"apps": apps}))
3796}
3797
3798fn list_windows(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3799    let listed = backend.list_windows()?;
3800    let windows = match app {
3801        None => listed.windows,
3802        Some(app) => {
3803            let selector = parse_app_selector(app)?;
3804            let apps = backend.list_apps(true)?;
3805            let info = resolve_app(&apps, &selector)?;
3806            filter_windows_for_pid(&listed.windows, info.pid)
3807        }
3808    };
3809    let mut body = json!({"windows": windows});
3810    if let Some(note) = listed.note {
3811        body["note"] = json!(note);
3812    }
3813    Ok(body)
3814}
3815
3816fn launch(backend: &dyn DesktopBackend, app: Option<&str>) -> Result<Value, ComputerError> {
3817    let app = require_app(app)?;
3818    let result = backend.launch(app)?;
3819    Ok(json!({
3820        "action": "launch",
3821        "name": result.name,
3822        "bundle_id": result.bundle_id,
3823        "pid": result.pid,
3824        "path": result.path,
3825    }))
3826}
3827
3828fn activate(
3829    backend: &dyn DesktopBackend,
3830    app: Option<&str>,
3831    window: Option<&str>,
3832) -> Result<Value, ComputerError> {
3833    let app = require_app(app)?;
3834    let result = backend.activate(app, window)?;
3835    Ok(json!({
3836        "action": "activate",
3837        "name": result.name,
3838        "bundle_id": result.bundle_id,
3839        "pid": result.pid,
3840        "path": result.path,
3841        "window_id": result.window_id,
3842        "window_title": result.window_title,
3843    }))
3844}
3845
3846fn quit(
3847    backend: &dyn DesktopBackend,
3848    app: Option<&str>,
3849    force: bool,
3850) -> Result<Value, ComputerError> {
3851    let app = require_app(app)?;
3852    let result = backend.quit(app, force)?;
3853    Ok(json!({
3854        "action": "quit",
3855        "name": result.name,
3856        "bundle_id": result.bundle_id,
3857        "pid": result.pid,
3858        "accepted": result.accepted,
3859        "force": force,
3860    }))
3861}
3862
3863fn doctor(
3864    backend: &dyn DesktopBackend,
3865    options: crate::doctor::DoctorOptions,
3866) -> Result<Value, ComputerError> {
3867    let report = backend.doctor(options)?;
3868    serde_json::to_value(report).map_err(|error| {
3869        ComputerError::ExecutionFailed(format!("failed to encode doctor report: {error}"))
3870    })
3871}
3872
3873fn require_app(app: Option<&str>) -> Result<&str, ComputerError> {
3874    match app.map(str::trim).filter(|text| !text.is_empty()) {
3875        Some(app) => Ok(app),
3876        None => Err(ComputerError::InvalidArguments("app is required".into())),
3877    }
3878}
3879
3880/// JSON result printed by the CLI and returned by [`dispatch`].
3881#[derive(Debug, Clone, PartialEq)]
3882pub struct Response {
3883    /// Whether the action completed.
3884    pub ok: bool,
3885    /// Request id, when the caller sent one.
3886    pub id: Option<Value>,
3887    /// Present when `ok` is false.
3888    pub error: Option<ErrorBody>,
3889    /// Extra fields flattened into the JSON object.
3890    pub fields: Map<String, Value>,
3891}
3892
3893impl Response {
3894    /// Successful response. `data` must be a JSON object; other values are stored as `result`.
3895    pub fn success(data: Value) -> Self {
3896        let fields = match data {
3897            Value::Object(map) => map,
3898            other => {
3899                let mut map = Map::new();
3900                map.insert("result".into(), other);
3901                map
3902            }
3903        };
3904        Self {
3905            ok: true,
3906            id: None,
3907            error: None,
3908            fields,
3909        }
3910    }
3911
3912    /// Failed response with a stable `code` and `message`.
3913    pub fn failure(code: impl Into<String>, message: impl Into<String>) -> Self {
3914        Self {
3915            ok: false,
3916            id: None,
3917            error: Some(ErrorBody {
3918                code: code.into(),
3919                message: message.into(),
3920            }),
3921            fields: Map::new(),
3922        }
3923    }
3924
3925    /// Failed response from a [`ComputerError`].
3926    pub fn from_error(error: ComputerError) -> Self {
3927        Self::failure(error.code(), error.to_string())
3928    }
3929
3930    /// Attach a request id.
3931    pub fn with_id(mut self, id: Option<Value>) -> Self {
3932        self.id = id;
3933        self
3934    }
3935
3936    /// JSON object for this response.
3937    pub fn to_value(&self) -> Value {
3938        let mut map = Map::new();
3939        map.insert("ok".into(), Value::Bool(self.ok));
3940        if let Some(id) = &self.id {
3941            map.insert("id".into(), id.clone());
3942        }
3943        if let Some(error) = &self.error {
3944            map.insert(
3945                "error".into(),
3946                serde_json::to_value(error).unwrap_or_else(
3947                    |_| json!({"code": "execution_failed", "message": "failed to encode error"}),
3948                ),
3949            );
3950        }
3951        for (key, value) in &self.fields {
3952            if key != "ok" && key != "id" && key != "error" {
3953                map.insert(key.clone(), value.clone());
3954            }
3955        }
3956        Value::Object(map)
3957    }
3958
3959    /// Serialize as compact or pretty JSON, followed by a newline.
3960    pub fn to_json_line(&self, pretty: bool) -> String {
3961        let value = self.to_value();
3962        let body = if pretty {
3963            serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3964        } else {
3965            serde_json::to_string(&value).unwrap_or_else(|_| "{\"ok\":false}".to_string())
3966        };
3967        format!("{body}\n")
3968    }
3969}
3970
3971impl From<ComputerError> for Response {
3972    fn from(error: ComputerError) -> Self {
3973        Self::from_error(error)
3974    }
3975}
3976
3977impl Serialize for Response {
3978    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
3979        self.to_value().serialize(serializer)
3980    }
3981}
3982
3983impl<'de> Deserialize<'de> for Response {
3984    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
3985        let value = Value::deserialize(deserializer)?;
3986        let Value::Object(mut object) = value else {
3987            return Err(de::Error::custom("response must be a JSON object"));
3988        };
3989        let ok = object
3990            .remove("ok")
3991            .ok_or_else(|| de::Error::missing_field("ok"))?;
3992        let ok = ok
3993            .as_bool()
3994            .ok_or_else(|| de::Error::custom("ok must be a boolean"))?;
3995        let id = object.remove("id");
3996        let error = match object.remove("error") {
3997            Some(value) => Some(serde_json::from_value(value).map_err(de::Error::custom)?),
3998            None => None,
3999        };
4000        Ok(Self {
4001            ok,
4002            id,
4003            error,
4004            fields: object,
4005        })
4006    }
4007}
4008
4009#[cfg(test)]
4010mod tests {
4011    use super::*;
4012    use crate::test_backend::StubBackend;
4013    use std::sync::atomic::Ordering;
4014
4015    #[test]
4016    fn action_roundtrip_and_aliases() {
4017        let click: Action = serde_json::from_str(
4018            r#"{"action":"click","x":10,"y":20,"button":"right","double":true}"#,
4019        )
4020        .unwrap();
4021        match &click {
4022            Action::Click {
4023                x,
4024                y,
4025                button,
4026                double,
4027                count,
4028            } => {
4029                assert_eq!(
4030                    (*x, *y, *button, *double, *count),
4031                    (Some(10), Some(20), MouseButton::Right, true, 1)
4032                );
4033            }
4034            other => panic!("unexpected {other:?}"),
4035        }
4036        let encoded = serde_json::to_string(&click).unwrap();
4037        let decoded: Action = serde_json::from_str(&encoded).unwrap();
4038        assert_eq!(decoded, click);
4039
4040        let moved: Action = serde_json::from_str(r#"{"action":"move_mouse","x":1,"y":2}"#).unwrap();
4041        assert!(matches!(moved, Action::Move { x: 1, y: 2 }));
4042
4043        let state: Action = serde_json::from_str(r#"{"action":"get-state"}"#).unwrap();
4044        assert!(matches!(state, Action::GetState));
4045        let state: Action = serde_json::from_str(r#"{"action":"get_state"}"#).unwrap();
4046        assert!(matches!(state, Action::GetState));
4047
4048        let shot: Action = serde_json::from_str(
4049            r#"{"action":"screenshot","region":"0,0,8,9","format":"jpeg","max_width":640,"base64":true}"#,
4050        )
4051        .unwrap();
4052        match shot {
4053            Action::Screenshot {
4054                region,
4055                format,
4056                max_width,
4057                base64,
4058                ..
4059            } => {
4060                assert_eq!(region.unwrap().width, 8);
4061                assert_eq!(format, Some(ImageFormat::Jpeg));
4062                assert_eq!(max_width, Some(640));
4063                assert!(base64);
4064            }
4065            other => panic!("unexpected {other:?}"),
4066        }
4067        let shot: Action = serde_json::from_str(
4068            r#"{"action":"screenshot","region":{"x":0,"y":1,"width":2,"height":3}}"#,
4069        )
4070        .unwrap();
4071        match shot {
4072            Action::Screenshot { region, .. } => {
4073                assert_eq!(region.unwrap().height, 3);
4074            }
4075            other => panic!("unexpected {other:?}"),
4076        }
4077
4078        assert!(serde_json::from_str::<Action>(r#"{"action":"fly"}"#).is_err());
4079        let bare: Action = serde_json::from_str(r#"{"action":"click"}"#).unwrap();
4080        assert!(matches!(
4081            bare,
4082            Action::Click {
4083                x: None,
4084                y: None,
4085                ..
4086            }
4087        ));
4088        let press: Action = serde_json::from_str(r#"{"action":"press"}"#).unwrap();
4089        assert!(matches!(
4090            press,
4091            Action::Press {
4092                count: 1,
4093                button: MouseButton::Left
4094            }
4095        ));
4096        let renamed: Action =
4097            serde_json::from_str(r#"{"action":"set-value","value":"hi"}"#).unwrap();
4098        assert!(matches!(renamed, Action::SetValue { .. }));
4099        let scrolled: Action = serde_json::from_str(r#"{"action":"scroll-to-visible"}"#).unwrap();
4100        assert!(matches!(scrolled, Action::ScrollTo));
4101    }
4102
4103    #[test]
4104    fn response_shape_and_roundtrip() {
4105        let ok = Response::success(json!({"waited_ms": 1})).with_id(Some(json!(7)));
4106        let value = ok.to_value();
4107        assert_eq!(value["ok"], true);
4108        assert!(value.get("error").is_none());
4109        assert_eq!(value["waited_ms"], 1);
4110        assert_eq!(value["id"], 7);
4111        let text = serde_json::to_string(&ok).unwrap();
4112        let back: Response = serde_json::from_str(&text).unwrap();
4113        assert_eq!(back.to_value(), value);
4114
4115        let err = Response::failure("invalid_arguments", "bad region");
4116        let value = err.to_value();
4117        assert_eq!(value["ok"], false);
4118        assert_eq!(value["error"]["code"], "invalid_arguments");
4119        assert_eq!(value["error"]["message"], "bad region");
4120        assert!(value.get("waited_ms").is_none());
4121        let line = err.to_json_line(false);
4122        assert!(line.ends_with('\n'));
4123        assert!(!line[..line.len() - 1].contains('\n'));
4124    }
4125
4126    #[test]
4127    fn scroll_and_wait_resolution() {
4128        assert_eq!(
4129            resolve_scroll(Some(1), Some(-2), None, None).unwrap(),
4130            (1, -2)
4131        );
4132        assert_eq!(
4133            resolve_scroll(None, None, Some(ScrollDirection::Up), Some(3)).unwrap(),
4134            (0, -3)
4135        );
4136        assert_eq!(
4137            resolve_scroll(None, None, Some(ScrollDirection::Right), None).unwrap(),
4138            (1, 0)
4139        );
4140        assert_eq!(
4141            resolve_scroll(None, None, Some(ScrollDirection::Left), Some(4)).unwrap(),
4142            (-4, 0)
4143        );
4144        assert_eq!(
4145            resolve_scroll(None, None, Some(ScrollDirection::Down), Some(2)).unwrap(),
4146            (0, 2)
4147        );
4148        assert!(resolve_scroll(Some(1), None, Some(ScrollDirection::Up), None).is_err());
4149        assert!(resolve_scroll(None, None, None, Some(2)).is_err());
4150        assert!(resolve_scroll(None, None, None, None).is_err());
4151
4152        assert_eq!(resolve_wait(Some(10), None).unwrap(), 10);
4153        assert_eq!(resolve_wait(Some(10), Some(0.5)).unwrap(), 510);
4154        assert_eq!(resolve_wait(None, Some(1.0)).unwrap(), 1000);
4155        assert!(resolve_wait(None, None).is_err());
4156        assert!(resolve_wait(None, Some(-1.0)).is_err());
4157        assert!(resolve_wait(None, Some(f64::NAN)).is_err());
4158    }
4159
4160    #[test]
4161    fn json_line_dispatch_and_stub_actions() {
4162        let wait = handle_request_line(r#"{"id":1,"action":"wait","ms":0}"#);
4163        assert!(wait.ok);
4164        assert_eq!(wait.to_value()["id"], 1);
4165        assert_eq!(wait.to_value()["waited_ms"], 0);
4166
4167        let bad = handle_request_line("not-json");
4168        assert!(!bad.ok);
4169        assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
4170
4171        let missing = handle_request_line(r#"{"id":"x","action":"nope"}"#);
4172        assert!(!missing.ok);
4173        assert_eq!(missing.to_value()["id"], "x");
4174
4175        let request = parse_request(json!({"id": 3, "action": "displays"})).unwrap();
4176        assert_eq!(request.id, Some(json!(3)));
4177        assert!(matches!(request.action, Action::Displays));
4178
4179        let backend = StubBackend::default();
4180        let response = dispatch_with(
4181            &backend,
4182            Action::Click {
4183                x: Some(3),
4184                y: Some(4),
4185                button: MouseButton::Left,
4186                double: true,
4187                count: 1,
4188            },
4189        );
4190        assert!(response.ok);
4191        assert_eq!(response.to_value()["action"], "double_click");
4192        assert_eq!(backend.clicks.load(Ordering::SeqCst), 2);
4193
4194        let response = dispatch_with(&backend, Action::Move { x: 8, y: 9 });
4195        assert!(response.ok);
4196        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4197
4198        let response = dispatch_with(&backend, Action::Type { text: "hi".into() });
4199        assert_eq!(response.to_value()["chars"], 2);
4200        assert_eq!(backend.typed.lock().unwrap().as_slice(), ["hi"]);
4201
4202        let response = dispatch_with(
4203            &backend,
4204            Action::Key {
4205                keys: "cmd+shift+4".into(),
4206            },
4207        );
4208        assert!(response.ok);
4209        assert_eq!(response.to_value()["key"], "4");
4210        let keys = backend.keys.lock().unwrap();
4211        assert_eq!(keys[0].1, "4");
4212        assert_eq!(keys[0].0, vec![Modifier::Command, Modifier::Shift]);
4213        drop(keys);
4214
4215        let response = dispatch_with(
4216            &backend,
4217            Action::Key {
4218                keys: "not+a+key".into(),
4219            },
4220        );
4221        assert!(!response.ok);
4222
4223        let response = dispatch_with(
4224            &backend,
4225            Action::Scroll {
4226                x: Some(1),
4227                y: Some(2),
4228                dx: None,
4229                dy: None,
4230                direction: Some(ScrollDirection::Down),
4231                amount: Some(3),
4232            },
4233        );
4234        assert!(response.ok);
4235        assert_eq!(backend.scrolls.lock().unwrap().as_slice(), [(0, 3)]);
4236        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
4237        assert_eq!(response.to_value()["tier"], "event");
4238        assert_eq!(response.to_value()["cursor_moved"], false);
4239
4240        let response = dispatch_with(&backend, Action::Displays);
4241        assert_eq!(response.to_value()["displays"][0]["name"], "Stub");
4242
4243        let response = dispatch_with(&backend, Action::GetState);
4244        assert_eq!(response.to_value()["cursor"]["x"], 4);
4245        assert!(response.ok);
4246
4247        let _cache = crate::cache::CacheEnv::acquire();
4248        let shot = dispatch_with(
4249            &backend,
4250            Action::Screenshot {
4251                output: None,
4252                region: None,
4253                format: Some(ImageFormat::Png),
4254                max_width: Some(1),
4255                canvas: None,
4256                base64: true,
4257                marks: false,
4258            },
4259        );
4260        assert!(shot.ok, "{shot:?}");
4261        let value = shot.to_value();
4262        assert_eq!(value["width"], 1);
4263        assert_eq!(value["height"], 1);
4264        assert_eq!(value["scale"], 2.0);
4265        assert_eq!(value["format"], "png");
4266        let path = value["path"].as_str().unwrap();
4267        let bytes = fs::read(path).unwrap();
4268        assert_eq!(
4269            &bytes[..8],
4270            &[0x89, b'P', b'N', b'G', b'\r', b'\n', 0x1a, b'\n']
4271        );
4272        assert!(value["data"].as_str().unwrap().starts_with("iVBOR"));
4273        assert_eq!(value["capture_source"], "stub");
4274        assert_eq!(value["origin"]["x"], 0);
4275        assert_eq!(value["origin"]["y"], 0);
4276        assert_eq!(value["fit"], "width");
4277        assert!(value["map"]["image_to_screen"].is_string());
4278        let _ = fs::remove_file(path);
4279        if let Ok(dir) = crate::cache::session_dir(None) {
4280            let _ = fs::remove_file(dir.join("screenshot-map.json"));
4281        }
4282
4283        let failed = StubBackend {
4284            fail_click: true,
4285            ..StubBackend::default()
4286        };
4287        let response = dispatch_with(
4288            &failed,
4289            Action::Click {
4290                x: Some(0),
4291                y: Some(0),
4292                button: MouseButton::Left,
4293                double: false,
4294                count: 1,
4295            },
4296        );
4297        assert!(!response.ok);
4298        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4299    }
4300
4301    #[test]
4302    fn dispatch_wait_does_not_need_a_display() {
4303        let response = dispatch(Action::Wait {
4304            ms: Some(0),
4305            seconds: None,
4306        });
4307        assert!(response.ok);
4308        assert_eq!(response.to_value()["waited_ms"], 0);
4309    }
4310
4311    fn shot(max_width: Option<u32>, canvas: Option<&str>) -> Action {
4312        Action::Screenshot {
4313            output: None,
4314            region: None,
4315            format: Some(ImageFormat::Png),
4316            max_width,
4317            canvas: canvas.map(str::to_string),
4318            base64: false,
4319            marks: false,
4320        }
4321    }
4322
4323    #[test]
4324    fn dispatch_applies_default_max_width_and_keeps_explicit_zero() {
4325        let _cache = crate::cache::CacheEnv::acquire();
4326        let backend = StubBackend {
4327            capture_pixels: Some((1600, 1000)),
4328            ..StubBackend::default()
4329        };
4330        let response = dispatch_with_context(
4331            &backend,
4332            shot(None, None),
4333            &CallContext {
4334                session: Some("p4-default-width".into()),
4335                ..CallContext::default()
4336            },
4337        );
4338        assert!(response.ok, "{response:?}");
4339        let value = response.to_value();
4340        assert_eq!(value["width"], 1440);
4341        assert_eq!(value["height"], 900);
4342        assert_eq!(value["scale"], 2.0);
4343        assert_eq!(value["logical_width"], 800);
4344        assert_eq!(value["logical_height"], 500);
4345        assert_eq!(value["fit"], "width");
4346        assert_eq!(value["capture_source"], "stub");
4347        assert_eq!(value["content_width"], 1440);
4348        let path = value["path"].as_str().unwrap();
4349        let _ = fs::remove_file(path);
4350        let map = crate::cache::load_screenshot_map(Some("p4-default-width"))
4351            .unwrap()
4352            .expect("map");
4353        assert_eq!(map.width, 1440);
4354        let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-default-width")).unwrap());
4355
4356        let full = dispatch_with_context(
4357            &backend,
4358            shot(Some(0), None),
4359            &CallContext {
4360                session: Some("p4-zero-width".into()),
4361                ..CallContext::default()
4362            },
4363        );
4364        assert!(full.ok, "{full:?}");
4365        assert_eq!(full.to_value()["width"], 1600);
4366        assert_eq!(full.to_value()["height"], 1000);
4367        assert_eq!(full.to_value()["fit"], "none");
4368        let path = full.to_value()["path"].as_str().unwrap().to_string();
4369        let _ = fs::remove_file(path);
4370        let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-zero-width")).unwrap());
4371    }
4372
4373    #[test]
4374    fn dispatch_window_capture_records_origin_and_faults() {
4375        let _cache = crate::cache::CacheEnv::acquire();
4376        let backend = StubBackend::default();
4377        let response = dispatch_with_context(
4378            &backend,
4379            shot(Some(0), None),
4380            &CallContext {
4381                app: Some("Notes".into()),
4382                window: Some("0".into()),
4383                session: Some("p4-window".into()),
4384                ..CallContext::default()
4385            },
4386        );
4387        assert!(response.ok, "{response:?}");
4388        let value = response.to_value();
4389        assert_eq!(value["capture_source"], "stub");
4390        assert_eq!(value["origin"]["x"], 10);
4391        assert_eq!(value["origin"]["y"], 20);
4392        assert_eq!(value["logical_width"], 400);
4393        assert_eq!(value["logical_height"], 300);
4394        assert_eq!(value["width"], 800);
4395        assert_eq!(value["fit"], "none");
4396        let path = value["path"].as_str().unwrap();
4397        let _ = fs::remove_file(path);
4398        let _ = fs::remove_dir_all(crate::cache::session_dir(Some("p4-window")).unwrap());
4399
4400        let denied = StubBackend {
4401            capture_fault: crate::test_backend::StubCaptureFault::Permission,
4402            ..StubBackend::default()
4403        };
4404        let response = dispatch_with_context(
4405            &denied,
4406            shot(Some(0), None),
4407            &CallContext {
4408                app: Some("Notes".into()),
4409                ..CallContext::default()
4410            },
4411        );
4412        assert!(!response.ok);
4413        assert_eq!(response.to_value()["error"]["code"], "permission_required");
4414        assert!(response.to_value()["error"]["message"]
4415            .as_str()
4416            .unwrap()
4417            .contains("Screen Recording"));
4418
4419        let missing = StubBackend {
4420            capture_fault: crate::test_backend::StubCaptureFault::MissingWindow,
4421            ..StubBackend::default()
4422        };
4423        let response = dispatch_with_context(
4424            &missing,
4425            shot(Some(0), None),
4426            &CallContext {
4427                window: Some("id:100".into()),
4428                ..CallContext::default()
4429            },
4430        );
4431        assert!(!response.ok);
4432        assert_eq!(response.to_value()["error"]["code"], "window_not_found");
4433
4434        let both = dispatch_with_context(
4435            &backend,
4436            Action::Screenshot {
4437                output: None,
4438                region: Some(Region::new(0, 0, 8, 9).unwrap()),
4439                format: None,
4440                max_width: Some(0),
4441                canvas: None,
4442                base64: false,
4443                marks: false,
4444            },
4445            &CallContext {
4446                app: Some("Notes".into()),
4447                ..CallContext::default()
4448            },
4449        );
4450        assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
4451
4452        let locked = StubBackend {
4453            block_display: true,
4454            windows: Vec::new(),
4455            ..StubBackend::default()
4456        };
4457        let response = dispatch_with_context(
4458            &locked,
4459            shot(Some(0), None),
4460            &CallContext {
4461                app: Some("Notes".into()),
4462                ..CallContext::default()
4463            },
4464        );
4465        assert_eq!(response.to_value()["error"]["code"], "display_unavailable");
4466        assert!(response.to_value()["error"]["message"]
4467            .as_str()
4468            .unwrap()
4469            .contains("locked"));
4470    }
4471
4472    #[test]
4473    fn shared_flags_are_stripped_before_the_action_decodes() {
4474        let request = parse_request(json!({
4475            "id": 4,
4476            "action": "list_windows",
4477            "app": "Foo \"Bar\"",
4478            "window": 0,
4479            "element": "3",
4480            "diff": true,
4481            "full": false,
4482            "return_state": true,
4483            "require_visible": true
4484        }))
4485        .unwrap();
4486        assert_eq!(request.id, Some(json!(4)));
4487        assert!(matches!(request.action, Action::ListWindows));
4488        assert_eq!(request.context.app.as_deref(), Some("Foo \"Bar\""));
4489        assert_eq!(request.context.window.as_deref(), Some("0"));
4490        assert_eq!(request.context.element, Some(3));
4491        assert!(request.context.diff);
4492        assert!(!request.context.full);
4493        assert!(request.context.return_state);
4494        assert!(request.context.require_visible);
4495    }
4496
4497    fn ctx(app: &str) -> CallContext {
4498        CallContext {
4499            app: Some(app.to_string()),
4500            ..CallContext::default()
4501        }
4502    }
4503
4504    #[test]
4505    fn stub_inventory_launch_and_doctor() {
4506        let backend = StubBackend::default();
4507
4508        let response = dispatch_with(&backend, Action::ListApps { all: false });
4509        assert!(response.ok);
4510        let value = response.to_value();
4511        let apps = value["apps"].as_array().unwrap();
4512        assert!(apps.iter().any(|app| app["name"] == "Notes"));
4513        assert!(apps.iter().all(|app| app["name"] != "Widget"));
4514
4515        let response = dispatch_with(&backend, Action::ListApps { all: true });
4516        let value = response.to_value();
4517        assert!(value["apps"]
4518            .as_array()
4519            .unwrap()
4520            .iter()
4521            .any(|app| app["pid"] == 77));
4522
4523        let response = dispatch_with_context(&backend, Action::ListWindows, &ctx("Notes"));
4524        assert!(response.ok, "{response:?}");
4525        let value = response.to_value();
4526        let windows = &value["windows"];
4527        assert_eq!(windows.as_array().unwrap().len(), 2);
4528        assert_eq!(windows[0]["z"], 0);
4529        assert_eq!(windows[0]["id"], 100);
4530        assert_eq!(windows[1]["z"], 1);
4531        assert_eq!(windows[1]["title"], "Welcome");
4532        assert_eq!(windows[0]["ax_matched"], true);
4533        assert!(value.get("note").is_none());
4534
4535        let response = dispatch_with_context(&backend, Action::Launch, &ctx("Notes"));
4536        assert!(response.ok, "{response:?}");
4537        assert_eq!(response.to_value()["pid"], 10);
4538        assert_eq!(response.to_value()["action"], "launch");
4539        assert_eq!(backend.launched.lock().unwrap().as_slice(), ["Notes"]);
4540
4541        let response = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
4542        assert!(!response.ok);
4543        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4544        assert_eq!(backend.launched.lock().unwrap().len(), 1);
4545
4546        let response = dispatch_with_context(&backend, Action::Activate, &ctx("10"));
4547        assert!(response.ok, "{response:?}");
4548        assert!(response.to_value()["window_id"].is_null());
4549        assert_eq!(backend.activated.lock().unwrap().len(), 1);
4550
4551        let mut context = ctx("Notes");
4552        context.window = Some("Welcome".into());
4553        let response = dispatch_with_context(&backend, Action::Activate, &context);
4554        assert_eq!(response.to_value()["window_id"], 101);
4555        assert_eq!(response.to_value()["window_title"], "Welcome");
4556
4557        let response =
4558            dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("Terminal"));
4559        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4560        assert!(backend.quit.lock().unwrap().is_empty());
4561
4562        let response = dispatch_with_context(&backend, Action::Quit { force: false }, &ctx("412"));
4563        assert_eq!(response.to_value()["error"]["code"], "confirm_required");
4564        assert!(backend.quit.lock().unwrap().is_empty());
4565
4566        let response = dispatch_with(&backend, Action::Launch);
4567        assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4568
4569        let response = dispatch_with(
4570            &backend,
4571            Action::Doctor {
4572                prompt: false,
4573                open_settings: false,
4574                x11: false,
4575            },
4576        );
4577        let value = response.to_value();
4578        assert_eq!(value["ok"], true);
4579        assert_eq!(value["permissions_ok"], false);
4580        assert_eq!(value["accessibility"], true);
4581        assert_eq!(value["screen_capture"], false);
4582        assert_eq!(value["screen_recording"], false);
4583        assert!(value["input_monitor"].is_null());
4584        assert_eq!(value["missing"][0], "screen_recording");
4585        assert_eq!(value["responsible_app"]["name"], "Terminal");
4586        assert!(value["fixes"][0]
4587            .as_str()
4588            .unwrap()
4589            .contains("Screen Recording"));
4590        assert_eq!(value["fix"][0]["target"], "Terminal");
4591
4592        let prompted: Action =
4593            serde_json::from_str(r#"{"action":"doctor","request":true}"#).unwrap();
4594        assert!(matches!(
4595            prompted,
4596            Action::Doctor {
4597                prompt: true,
4598                open_settings: false,
4599                ..
4600            }
4601        ));
4602        let plain: Action = serde_json::from_str(r#"{"action":"doctor"}"#).unwrap();
4603        assert!(matches!(
4604            plain,
4605            Action::Doctor {
4606                prompt: false,
4607                open_settings: false,
4608                ..
4609            }
4610        ));
4611
4612        let response = dispatch_with(&backend, Action::GetState);
4613        assert!(response.ok);
4614        let value = response.to_value();
4615        assert_eq!(value["pid"], 10);
4616        assert_eq!(value["source"], "stub");
4617        assert!(value["tree"].as_str().unwrap().contains("AXWindow"));
4618        assert!(value["accessibility_legacy"].is_null());
4619
4620        let response = dispatch_with_context(&backend, Action::GetState, &ctx("412"));
4621        assert_eq!(response.to_value()["pid"], 412);
4622        assert_eq!(response.to_value()["source"], "stub");
4623        assert!(response.to_value()["tree"]
4624            .as_str()
4625            .unwrap()
4626            .contains("AXWindow"));
4627        let seen = backend.accessibility_apps.lock().unwrap().clone();
4628        assert_eq!(seen.last().and_then(|name| name.as_deref()), Some("412"));
4629
4630        // Get-state already stored a baseline. Clear it so this sequence
4631        // still covers "no baseline prints the full tree".
4632        backend.baselines.lock().unwrap().clear();
4633        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4634        assert!(first.ok, "{first:?}");
4635        let full = first.to_value()["tree"].as_str().unwrap().to_string();
4636        assert!(full.contains("value=\"a\""), "{full}");
4637        assert_eq!(first.to_value()["source"], "stub");
4638        *backend.stub_value.lock().unwrap() = "ab".into();
4639        let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4640        let diff_text = diff.to_value()["tree"].as_str().unwrap().to_string();
4641        assert!(
4642            diff_text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4643            "{diff_text}"
4644        );
4645        let mut full_context = ctx("Notes");
4646        full_context.full = true;
4647        let replaced = dispatch_with_context(&backend, Action::Tree, &full_context);
4648        let replaced_text = replaced.to_value()["tree"].as_str().unwrap().to_string();
4649        assert!(replaced_text.contains("value=\"ab\""), "{replaced_text}");
4650        assert!(!replaced_text.contains("~ [1]"));
4651        let after = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4652        assert_eq!(after.to_value()["tree"].as_str().unwrap(), "");
4653
4654        let _cache = crate::cache::CacheEnv::acquire();
4655        let ended = dispatch_with(&backend, Action::SessionEnd);
4656        assert!(ended.ok, "{ended:?}");
4657        assert_eq!(ended.to_value()["action"], "session_end");
4658        assert!(ended.to_value()["restored"].as_array().unwrap().is_empty());
4659    }
4660
4661    #[test]
4662    fn stub_refuses_privacy_window_and_self_quit() {
4663        let mut backend = StubBackend::default();
4664        backend.apps.push(crate::target::AppInfo {
4665            name: "System Settings".into(),
4666            bundle_id: Some("com.apple.systempreferences".into()),
4667            pid: 50,
4668            path: Some(
4669                "/System/Applications/System Settings.app/Contents/MacOS/System Settings".into(),
4670            ),
4671            active: false,
4672            hidden: false,
4673            activation_policy: crate::target::ActivationPolicy::Regular,
4674        });
4675        backend.windows.push(crate::target::WindowInfo {
4676            id: 9,
4677            pid: 50,
4678            app: "System Settings".into(),
4679            title: "Privacy & Security".into(),
4680            z: 4,
4681            bounds: crate::target::WindowBounds {
4682                x: 0,
4683                y: 0,
4684                width: 10,
4685                height: 10,
4686            },
4687            layer: 0,
4688            on_screen: true,
4689            ax_matched: false,
4690        });
4691        let mut context = ctx("com.apple.systempreferences");
4692        context.window = Some("Privacy".into());
4693        let response = dispatch_with_context(&backend, Action::Activate, &context);
4694        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4695        assert!(backend.activated.lock().unwrap().is_empty());
4696
4697        let response = dispatch_with_context(&backend, Action::Activate, &ctx("System Settings"));
4698        assert!(response.ok, "{response:?}");
4699
4700        let self_pid = std::process::id() as i32;
4701        backend.apps.retain(|app| app.pid != self_pid);
4702        backend.apps.push(crate::target::AppInfo {
4703            name: "computer-use".into(),
4704            bundle_id: None,
4705            pid: self_pid,
4706            path: None,
4707            active: false,
4708            hidden: false,
4709            activation_policy: crate::target::ActivationPolicy::Regular,
4710        });
4711        let response = dispatch_with_context(
4712            &backend,
4713            Action::Quit { force: false },
4714            &ctx(&self_pid.to_string()),
4715        );
4716        assert_eq!(response.to_value()["error"]["code"], "policy_denied");
4717    }
4718
4719    fn notes_button() -> crate::tree::UiNode {
4720        let mut button = crate::tree::UiNode::bare("AXButton");
4721        button.title = Some("Save".into());
4722        button.actions = vec!["AXPress".into(), "AXShowMenu".into()];
4723        button.frame = Some(crate::tree::NodeFrame {
4724            x: 12.0,
4725            y: 10.0,
4726            width: 80.0,
4727            height: 24.0,
4728        });
4729        let mut root = crate::tree::UiNode::bare("AXWindow");
4730        root.title = Some("Notes".into());
4731        root.frame = Some(crate::tree::NodeFrame {
4732            x: 0.0,
4733            y: 0.0,
4734            width: 800.0,
4735            height: 600.0,
4736        });
4737        root.children = vec![button];
4738        root
4739    }
4740
4741    fn element_context(action_element: u32) -> CallContext {
4742        CallContext {
4743            app: Some("Notes".into()),
4744            element: Some(action_element),
4745            ..CallContext::default()
4746        }
4747    }
4748
4749    #[test]
4750    fn element_json_shape_revalidation_and_hit_test() {
4751        let backend = StubBackend::default();
4752        backend.seed_element(notes_button());
4753        let request = parse_request(json!({
4754            "action": "press",
4755            "app": "Notes",
4756            "element": 1,
4757            "count": 1
4758        }))
4759        .unwrap();
4760        let response = dispatch_with_context(&backend, request.action, &request.context);
4761        assert!(response.ok, "{response:?}");
4762        let value = response.to_value();
4763        assert_eq!(value["action"], "press");
4764        assert_eq!(value["element"], 1);
4765        assert_eq!(value["tier"], "ax");
4766        assert_eq!(value["count"], 1);
4767        assert!(value["revision"].is_number());
4768        assert!(value.get("warnings").is_none());
4769
4770        let request = parse_request(json!({
4771            "action": "perform-action",
4772            "app": "Notes",
4773            "element": 1,
4774            "name": "AXShowMenu"
4775        }))
4776        .unwrap();
4777        let response = dispatch_with_context(&backend, request.action, &request.context);
4778        assert!(response.ok, "{response:?}");
4779        assert_eq!(response.to_value()["action"], "perform_action");
4780        assert_eq!(response.to_value()["name"], "AXShowMenu");
4781        assert_eq!(response.to_value()["tier"], "ax");
4782
4783        let before = backend.performs.lock().unwrap().len();
4784        let request = parse_request(json!({
4785            "action": "perform_action",
4786            "app": "Notes",
4787            "element": 1,
4788            "name": "AXDelete"
4789        }))
4790        .unwrap();
4791        let response = dispatch_with_context(&backend, request.action, &request.context);
4792        assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
4793        assert_eq!(backend.performs.lock().unwrap().len(), before);
4794
4795        let request = parse_request(json!({
4796            "action": "set_value",
4797            "app": "Notes",
4798            "element": 1,
4799            "value": "hello"
4800        }))
4801        .unwrap();
4802        let response = dispatch_with_context(&backend, request.action, &request.context);
4803        assert!(response.ok, "{response:?}");
4804        assert_eq!(response.to_value()["action"], "set_value");
4805
4806        let request = parse_request(json!({
4807            "action": "select_text",
4808            "app": "Notes",
4809            "element": 1,
4810            "start": 0,
4811            "end": 5
4812        }))
4813        .unwrap();
4814        let response = dispatch_with_context(&backend, request.action, &request.context);
4815        assert!(response.ok, "{response:?}");
4816        assert_eq!(response.to_value()["action"], "select_text");
4817
4818        let request = parse_request(json!({
4819            "action": "scroll_to_visible",
4820            "app": "Notes",
4821            "element": 1
4822        }))
4823        .unwrap();
4824        assert!(matches!(request.action, Action::ScrollTo));
4825        let response = dispatch_with_context(&backend, request.action, &request.context);
4826        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
4827        assert!(response.to_value()["error"]["message"]
4828            .as_str()
4829            .unwrap()
4830            .contains("AXScrollToVisible"));
4831
4832        let clicks = backend.clicks.load(Ordering::SeqCst);
4833        let request = parse_request(json!({
4834            "action": "click",
4835            "app": "Notes",
4836            "element": 1,
4837            "double": true
4838        }))
4839        .unwrap();
4840        let response = dispatch_with_context(&backend, request.action, &request.context);
4841        assert!(response.ok, "{response:?}");
4842        assert_eq!(response.to_value()["action"], "press");
4843        assert_eq!(response.to_value()["count"], 2);
4844        assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks);
4845
4846        *backend.hit.lock().unwrap() = Some(crate::element::HitInfo {
4847            role: "AXButton".into(),
4848            title: Some("Save".into()),
4849        });
4850        let response = dispatch_with(
4851            &backend,
4852            Action::Click {
4853                x: Some(12),
4854                y: Some(10),
4855                button: MouseButton::Left,
4856                double: false,
4857                count: 1,
4858            },
4859        );
4860        assert!(response.ok, "{response:?}");
4861        assert_eq!(response.to_value()["hit"]["role"], "AXButton");
4862        assert_eq!(response.to_value()["hit"]["title"], "Save");
4863        assert_eq!(backend.clicks.load(Ordering::SeqCst), clicks + 1);
4864
4865        let mut changed = notes_button();
4866        changed.children[0].role = "AXTextArea".into();
4867        backend.set_live(changed);
4868        let response = dispatch_with_context(&backend, Action::Focus, &element_context(1));
4869        assert_eq!(response.to_value()["error"]["code"], "stale_element");
4870        assert!(response.to_value()["tree"]
4871            .as_str()
4872            .unwrap()
4873            .contains("AXTextArea"));
4874
4875        let empty = StubBackend::default();
4876        let response = dispatch_with_context(&empty, Action::Focus, &element_context(0));
4877        assert_eq!(response.to_value()["error"]["code"], "element_not_found");
4878        assert!(response.to_value()["error"]["message"]
4879            .as_str()
4880            .unwrap()
4881            .contains("refresh indices"));
4882
4883        let bare = dispatch_with(
4884            &empty,
4885            Action::Click {
4886                x: None,
4887                y: None,
4888                button: MouseButton::Left,
4889                double: false,
4890                count: 1,
4891            },
4892        );
4893        assert_eq!(bare.to_value()["error"]["code"], "invalid_arguments");
4894    }
4895
4896    #[test]
4897    fn element_action_does_not_replace_the_tree_diff_baseline() {
4898        let backend = StubBackend::default();
4899        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4900        assert!(first.ok, "{first:?}");
4901        let key = (String::new(), 10, 100);
4902        let baseline = backend.baselines.lock().unwrap().get(&key).cloned();
4903        let response = dispatch_with_context(&backend, Action::Focus, &element_context(0));
4904        assert!(response.ok, "{response:?}");
4905        assert_eq!(
4906            backend.baselines.lock().unwrap().get(&key),
4907            baseline.as_ref()
4908        );
4909        *backend.stub_value.lock().unwrap() = "ab".into();
4910        let diff = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
4911        let text = diff.to_value()["tree"].as_str().unwrap().to_string();
4912        assert!(
4913            text.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
4914            "{text}"
4915        );
4916    }
4917
4918    fn sample_map() -> crate::image_util::ScreenshotMap {
4919        crate::image_util::ScreenshotMap {
4920            width: 1440,
4921            height: 900,
4922            scale: 2.0,
4923            logical_width: 800,
4924            logical_height: 500,
4925            origin: crate::image_util::MapPoint { x: 120, y: 40 },
4926            fit: "width".into(),
4927            pad: crate::image_util::MapPoint { x: 0, y: 0 },
4928            content_width: 1440,
4929            content_height: 900,
4930            map: crate::image_util::MapFormulas {
4931                image_to_screen:
4932                    "screen = origin + (image - pad) * logical_size / image_content_size".into(),
4933                screen_to_image:
4934                    "image = pad + (screen - origin) * image_content_size / logical_size".into(),
4935            },
4936        }
4937    }
4938
4939    #[test]
4940    fn secure_ocr_frames_receives_the_map_of_the_ocr_capture() {
4941        // Platform frames are screen points; OCR blocks are image pixels. The
4942        // dispatcher must hand the backend the screenshot map of the capture
4943        // the OCR ran on, or redaction compares two different spaces.
4944        let cache = crate::cache::CacheEnv::acquire();
4945        assert!(cache.dir().is_dir());
4946        let backend = StubBackend::default();
4947        *backend.secure_frames.lock().unwrap() = Some(vec![crate::vision::ocr::ImageRect {
4948            x: 10,
4949            y: 20,
4950            width: 100,
4951            height: 40,
4952        }]);
4953        let context = CallContext {
4954            app: Some("Finder".into()),
4955            session: Some("p12b-redact".into()),
4956            ..CallContext::default()
4957        };
4958        let blocks = vec![OcrBlock {
4959            text: "hunter2".into(),
4960            x: 20,
4961            y: 30,
4962            width: 30,
4963            height: 10,
4964            confidence: 0.9,
4965        }];
4966        let map = sample_map();
4967        let kept = redact_secure_ocr_blocks(blocks.clone(), &backend, &context, &map).unwrap();
4968        assert!(kept.is_empty(), "the block sits inside the secure frame");
4969
4970        let seen = backend
4971            .secure_frames_map
4972            .lock()
4973            .unwrap()
4974            .clone()
4975            .expect("secure_ocr_frames was called");
4976        assert_eq!(seen.origin, map.origin);
4977        assert_eq!(seen.logical_width, map.logical_width);
4978        assert_eq!(seen.content_width, map.content_width);
4979
4980        let outside = vec![OcrBlock {
4981            text: "visible".into(),
4982            x: 900,
4983            y: 700,
4984            width: 30,
4985            height: 10,
4986            confidence: 0.9,
4987        }];
4988        let kept = redact_secure_ocr_blocks(outside, &backend, &context, &map).unwrap();
4989        assert_eq!(kept.len(), 1);
4990    }
4991
4992    /// Screenshot with `--marks` on a real window answered `ok` and wrote an
4993    /// unmarked PNG. `apply_marks` painted the overlay onto a copy of the image
4994    /// and threw the copy away, so the encoded bytes never changed.
4995    #[test]
4996    fn marks_change_the_pixels_the_screenshot_writes() {
4997        let cache = crate::cache::CacheEnv::acquire();
4998        let backend = StubBackend::default();
4999        let context = CallContext {
5000            app: Some("Notes".into()),
5001            session: Some("marks".into()),
5002            ..CallContext::default()
5003        };
5004        let plain = cache.dir().join("plain.png");
5005        let marked = cache.dir().join("marked.png");
5006
5007        let shot = |path: std::path::PathBuf, marks: bool| {
5008            screenshot(
5009                &backend,
5010                &context,
5011                ScreenshotArgs {
5012                    output: Some(path.to_string_lossy().into_owned()),
5013                    region: None,
5014                    format: Some(ImageFormat::Png),
5015                    max_width: None,
5016                    canvas: None,
5017                    include_base64: false,
5018                    window_crop: false,
5019                    marks,
5020                },
5021            )
5022        };
5023
5024        let clean = shot(plain.clone(), false).expect("plain screenshot");
5025        let marked_body = shot(marked.clone(), true).expect("marked screenshot");
5026
5027        let clean_bytes = std::fs::read(&plain).expect("plain png");
5028        let marked_bytes = std::fs::read(&marked).expect("marked png");
5029        assert_ne!(
5030            clean_bytes, marked_bytes,
5031            "--marks wrote the same image as a plain screenshot"
5032        );
5033
5034        // The map the backend converted frames with must be the map of this
5035        // capture, otherwise screen points are read as image pixels.
5036        let seen = backend
5037            .marks_map
5038            .lock()
5039            .unwrap()
5040            .clone()
5041            .expect("marks_for_tree was called");
5042        assert_eq!(
5043            i64::from(seen.origin.x),
5044            clean["origin"]["x"].as_i64().unwrap()
5045        );
5046        assert_eq!(
5047            i64::from(seen.origin.y),
5048            clean["origin"]["y"].as_i64().unwrap()
5049        );
5050        assert_eq!(
5051            seen.content_width,
5052            clean["content_width"].as_u64().unwrap() as u32
5053        );
5054        assert_eq!(
5055            seen.logical_width,
5056            clean["logical_width"].as_u64().unwrap() as u32
5057        );
5058
5059        // Marks were applied, so this is not the "nothing to draw" warning.
5060        let warnings = marked_body["warnings"].as_array();
5061        assert!(
5062            !warnings
5063                .map(|items| items
5064                    .iter()
5065                    .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))))
5066                .unwrap_or(false),
5067            "unexpected marks warning: {warnings:?}"
5068        );
5069    }
5070
5071    #[test]
5072    fn marks_warn_when_no_frame_is_inside_the_capture() {
5073        // A silent zero is indistinguishable from "the marks were drawn".
5074        let cache = crate::cache::CacheEnv::acquire();
5075        let backend = StubBackend::default();
5076        // A tree whose window sits far away from the captured screen: every
5077        // frame is off-canvas, so nothing can be drawn.
5078        backend.set_live({
5079            let mut window = crate::tree::UiNode::bare("AXWindow");
5080            window.frame = Some(crate::tree::NodeFrame {
5081                x: 9_000.0,
5082                y: 9_000.0,
5083                width: 100.0,
5084                height: 100.0,
5085            });
5086            window
5087        });
5088        let context = CallContext {
5089            session: Some("marks-empty".into()),
5090            ..CallContext::default()
5091        };
5092        let body = screenshot(
5093            &backend,
5094            &context,
5095            ScreenshotArgs {
5096                output: Some(
5097                    cache
5098                        .dir()
5099                        .join("marks-empty.png")
5100                        .to_string_lossy()
5101                        .into_owned(),
5102                ),
5103                region: None,
5104                format: Some(ImageFormat::Png),
5105                max_width: None,
5106                canvas: None,
5107                include_base64: false,
5108                window_crop: false,
5109                marks: true,
5110            },
5111        )
5112        .expect("screenshot");
5113        let warnings = body["warnings"].as_array().expect("warnings");
5114        assert!(
5115            warnings
5116                .iter()
5117                .any(|item| item.as_str().is_some_and(|text| text.starts_with("marks:"))),
5118            "expected a marks warning, got {warnings:?}"
5119        );
5120    }
5121
5122    #[test]
5123    fn no_secure_frames_keeps_every_ocr_block() {
5124        let cache = crate::cache::CacheEnv::acquire();
5125        assert!(cache.dir().is_dir());
5126        let backend = StubBackend::default();
5127        let context = CallContext {
5128            app: Some("Finder".into()),
5129            session: Some("p12b-redact-none".into()),
5130            ..CallContext::default()
5131        };
5132        let blocks = vec![OcrBlock {
5133            text: "hello".into(),
5134            x: 1,
5135            y: 2,
5136            width: 3,
5137            height: 4,
5138            confidence: 0.9,
5139        }];
5140        let kept = redact_secure_ocr_blocks(blocks, &backend, &context, &sample_map()).unwrap();
5141        assert_eq!(kept.len(), 1);
5142    }
5143
5144    #[test]
5145    fn image_coordinate_on_a_stored_map_becomes_the_screen_point() {
5146        let cache = crate::cache::CacheEnv::acquire();
5147        assert!(cache.dir().is_dir());
5148        let map = sample_map();
5149        cache::store_screenshot_map(Some("p5-map"), &map).unwrap();
5150        let backend = StubBackend::default();
5151        let context = CallContext {
5152            coord: Some("image".into()),
5153            session: Some("p5-map".into()),
5154            ..CallContext::default()
5155        };
5156        let response = dispatch_with_context(&backend, Action::Move { x: 100, y: 50 }, &context);
5157        assert!(response.ok, "{response:?}");
5158        let (screen_x, screen_y) = image_util::image_to_screen(&map, 100.0, 50.0);
5159        let value = response.to_value();
5160        assert_eq!(value["x"], json!(screen_x.round() as i64));
5161        assert_eq!(value["y"], json!(screen_y.round() as i64));
5162        assert_eq!(value["x"], 176);
5163        assert_eq!(value["y"], 68);
5164        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5165
5166        let outside = dispatch_with_context(&backend, Action::Move { x: 1440, y: 50 }, &context);
5167        assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
5168        assert!(outside.to_value()["error"]["message"]
5169            .as_str()
5170            .unwrap()
5171            .contains("outside"));
5172        assert_eq!(backend.moves.load(Ordering::SeqCst), 1);
5173
5174        let missing = dispatch_with_context(
5175            &backend,
5176            Action::Move { x: 1, y: 2 },
5177            &CallContext {
5178                coord: Some("image".into()),
5179                session: Some("no-map".into()),
5180                ..CallContext::default()
5181            },
5182        );
5183        assert_eq!(missing.to_value()["error"]["code"], "invalid_arguments");
5184        assert!(missing.to_value()["error"]["message"]
5185            .as_str()
5186            .unwrap()
5187            .contains("take a screenshot first"));
5188
5189        let action =
5190            crate::adapters::translate("openai", &json!({"type": "move", "x": 100, "y": 50}))
5191                .unwrap();
5192        let adapted = dispatch_with_context(
5193            &backend,
5194            action,
5195            &CallContext {
5196                coord: Some("image".into()),
5197                session: Some("p5-map".into()),
5198                adapter: Some("openai".into()),
5199                ..CallContext::default()
5200            },
5201        );
5202        assert!(adapted.ok, "{adapted:?}");
5203        assert_eq!(adapted.to_value()["adapter"], "openai");
5204        assert_eq!(adapted.to_value()["x"], 176);
5205        assert_eq!(adapted.to_value()["y"], 68);
5206        assert!(adapted.to_value().get("state").is_none());
5207
5208        let missing_adapter = handle_request_line_with(
5209            r#"{"adapter":"anthropic","action":"mouse_move","coordinate":[1,2],"session":"no-map"}"#,
5210            None,
5211        );
5212        assert!(!missing_adapter.ok);
5213        assert_eq!(
5214            missing_adapter.to_value()["error"]["code"],
5215            "invalid_arguments"
5216        );
5217        assert!(missing_adapter.to_value()["error"]["message"]
5218            .as_str()
5219            .unwrap()
5220            .contains("take a screenshot first"));
5221        assert_eq!(missing_adapter.to_value()["adapter"], "anthropic");
5222
5223        let waited = handle_request_line_with(r#"{"adapter":"openai","type":"wait"}"#, None);
5224        assert!(waited.ok, "{waited:?}");
5225        assert_eq!(waited.to_value()["adapter"], "openai");
5226        assert_eq!(waited.to_value()["waited_ms"], 0);
5227    }
5228
5229    #[test]
5230    fn return_state_includes_the_numbered_tree_and_ignores_read_only() {
5231        let backend = StubBackend::default();
5232        let before = backend.accessibility_apps.lock().unwrap().len();
5233        let response = dispatch_with_context(
5234            &backend,
5235            Action::Click {
5236                x: Some(3),
5237                y: Some(4),
5238                button: MouseButton::Left,
5239                double: false,
5240                count: 1,
5241            },
5242            &CallContext {
5243                app: Some("Notes".into()),
5244                return_state: true,
5245                ..CallContext::default()
5246            },
5247        );
5248        assert!(response.ok, "{response:?}");
5249        let value = response.to_value();
5250        assert_eq!(value["action"], "click");
5251        let tree = value["state"]["tree"].as_str().unwrap();
5252        assert!(tree.contains("[0] AXWindow"), "{tree}");
5253        assert!(tree.contains("AXTextArea"), "{tree}");
5254        assert!(value["state"]["diff"].is_null());
5255        assert_eq!(value["state"]["app"], "com.apple.Notes");
5256        assert_eq!(value["state"]["pid"], 10);
5257        assert_eq!(value["state"]["window_id"], 100);
5258        assert!(value.get("warnings").is_none());
5259        assert!(backend.accessibility_apps.lock().unwrap().len() > before);
5260
5261        let displays = dispatch_with_context(
5262            &backend,
5263            Action::Displays,
5264            &CallContext {
5265                return_state: true,
5266                include_screenshot: true,
5267                ..CallContext::default()
5268            },
5269        );
5270        assert!(displays.ok, "{displays:?}");
5271        assert!(displays.to_value().get("state").is_none());
5272
5273        let cursor = dispatch_with(&backend, Action::CursorPosition);
5274        assert_eq!(cursor.to_value()["x"], 4);
5275        assert_eq!(cursor.to_value()["y"], 5);
5276        assert!(cursor.to_value().get("adapter").is_none());
5277    }
5278
5279    #[test]
5280    fn return_state_refreshes_the_tree_diff_baseline() {
5281        let backend = StubBackend::default();
5282        let first = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5283        assert!(first.ok, "{first:?}");
5284        *backend.stub_value.lock().unwrap() = "ab".into();
5285        let mut context = element_context(0);
5286        context.return_state = true;
5287        let response = dispatch_with_context(&backend, Action::Focus, &context);
5288        assert!(response.ok, "{response:?}");
5289        let value = response.to_value();
5290        let diff = value["state"]["diff"].as_str().unwrap();
5291        assert!(
5292            diff.contains("~ [1] AXTextArea value: \"a\" -> \"ab\""),
5293            "{diff}"
5294        );
5295        assert!(value["state"]["tree"]
5296            .as_str()
5297            .unwrap()
5298            .contains("value=\"ab\""));
5299        *backend.stub_value.lock().unwrap() = "abc".into();
5300        let next = dispatch_with_context(&backend, Action::Tree, &ctx("Notes"));
5301        let value = next.to_value();
5302        let text = value["tree"].as_str().unwrap();
5303        assert!(
5304            text.contains("~ [1] AXTextArea value: \"ab\" -> \"abc\""),
5305            "{text}"
5306        );
5307        assert!(!text.contains("\"a\" -> \"abc\""), "{text}");
5308    }
5309
5310    #[test]
5311    fn include_screenshot_failure_keeps_the_action_result() {
5312        let backend = StubBackend {
5313            capture_fault: crate::test_backend::StubCaptureFault::Permission,
5314            ..StubBackend::default()
5315        };
5316        let response = dispatch_with_context(
5317            &backend,
5318            Action::Click {
5319                x: Some(1),
5320                y: Some(2),
5321                button: MouseButton::Left,
5322                double: false,
5323                count: 1,
5324            },
5325            &CallContext {
5326                app: Some("Notes".into()),
5327                return_state: true,
5328                include_screenshot: true,
5329                ..CallContext::default()
5330            },
5331        );
5332        assert!(response.ok, "{response:?}");
5333        let value = response.to_value();
5334        assert_eq!(value["action"], "click");
5335        assert_eq!(value["x"], 1);
5336        assert_eq!(value["y"], 2);
5337        assert!(value["state"]["tree"]
5338            .as_str()
5339            .unwrap()
5340            .contains("AXWindow"));
5341        assert!(value["state"]["screenshot"].is_null());
5342        let warnings = value["warnings"].as_array().unwrap();
5343        assert!(warnings
5344            .iter()
5345            .any(|item| { item.as_str().unwrap().contains("screenshot failed") }));
5346        assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5347    }
5348
5349    fn batch_clicks(stop_on_error: bool) -> Value {
5350        json!({
5351            "action": "batch",
5352            "stop_on_error": stop_on_error,
5353            "actions": [
5354                {"action": "click", "x": 1, "y": 2},
5355                {"action": "click", "x": 3, "y": 4}
5356            ]
5357        })
5358    }
5359
5360    #[test]
5361    fn batch_stop_on_error_controls_the_second_attempt() {
5362        let stopped = StubBackend {
5363            fail_click: true,
5364            ..StubBackend::default()
5365        };
5366        let request = parse_request(batch_clicks(true)).unwrap();
5367        let response = dispatch_with_context(&stopped, request.action, &request.context);
5368        assert!(!response.ok);
5369        let value = response.to_value();
5370        assert_eq!(value["error"]["code"], "execution_failed");
5371        assert_eq!(value["stopped_at"], 0);
5372        assert_eq!(value["results"].as_array().unwrap().len(), 1);
5373        assert!(!value["results"][0]["ok"].as_bool().unwrap());
5374        assert_eq!(stopped.click_attempts.load(Ordering::SeqCst), 2);
5375        assert!(value["error"]["message"]
5376            .as_str()
5377            .unwrap()
5378            .contains("batch failed at action 0"));
5379
5380        let both = StubBackend {
5381            fail_click: true,
5382            ..StubBackend::default()
5383        };
5384        let request = parse_request(batch_clicks(false)).unwrap();
5385        let response = dispatch_with_context(&both, request.action, &request.context);
5386        assert!(!response.ok);
5387        let value = response.to_value();
5388        assert!(value.get("stopped_at").is_none(), "{value}");
5389        assert_eq!(value["results"].as_array().unwrap().len(), 2);
5390        assert!(!value["results"][1]["ok"].as_bool().unwrap());
5391        assert_eq!(both.click_attempts.load(Ordering::SeqCst), 4);
5392
5393        let nested = parse_request(json!({
5394            "action": "batch",
5395            "actions": [{"action": "batch", "actions": []}]
5396        }))
5397        .unwrap();
5398        let response = dispatch_with(&StubBackend::default(), nested.action);
5399        assert_eq!(response.to_value()["error"]["code"], "invalid_arguments");
5400        assert_eq!(response.to_value()["stopped_at"], 0);
5401        assert!(response.to_value()["error"]["message"]
5402            .as_str()
5403            .unwrap()
5404            .contains("nested batch"));
5405
5406        let backend = StubBackend::default();
5407        backend.seed_element(notes_button());
5408        let request = parse_request(json!({
5409            "action": "batch",
5410            "app": "Notes",
5411            "actions": [
5412                {"action": "press", "element": 1},
5413                {"action": "focus", "element": 1}
5414            ]
5415        }))
5416        .unwrap();
5417        assert!(matches!(
5418            request.action,
5419            Action::Batch {
5420                stop_on_error: true,
5421                ..
5422            }
5423        ));
5424        let response = dispatch_with_context(&backend, request.action, &request.context);
5425        assert!(response.ok, "{response:?}");
5426        let results = response.to_value()["results"].clone();
5427        assert_eq!(results[0]["action"], "press");
5428        assert_eq!(results[0]["element"], 1);
5429        assert_eq!(results[1]["action"], "focus");
5430        assert!(response.to_value().get("state").is_none());
5431    }
5432
5433    #[test]
5434    fn p9_menu_list_round_trip_records_no_args() {
5435        let backend = StubBackend::default();
5436        backend
5437            .menu_sections
5438            .lock()
5439            .unwrap()
5440            .push(crate::backend::MenuNode {
5441                title: "File".into(),
5442                enabled: true,
5443                children: vec![crate::backend::MenuNode {
5444                    title: "Save".into(),
5445                    enabled: true,
5446                    children: Vec::new(),
5447                }],
5448            });
5449        let response = dispatch_with_context(
5450            &backend,
5451            Action::MenuList {
5452                app: Some("Notes".into()),
5453                depth: Some(2),
5454            },
5455            &ctx("Notes"),
5456        );
5457        assert!(response.ok, "{:?}", response.to_value());
5458        let value = response.to_value();
5459        assert_eq!(value["action"], "menu_list");
5460        assert_eq!(value["app"], "Notes");
5461        assert_eq!(value["sections"][0]["title"], "File");
5462        assert_eq!(value["sections"][0]["children"][0]["title"], "Save");
5463    }
5464
5465    #[test]
5466    fn p9_menu_click_records_path_and_reports_disabled() {
5467        let mut backend = StubBackend::default();
5468        let response = dispatch_with_context(
5469            &backend,
5470            Action::MenuClick {
5471                app: Some("Notes".into()),
5472                path: "File > Save".into(),
5473            },
5474            &ctx("Notes"),
5475        );
5476        assert!(response.ok, "{:?}", response.to_value());
5477        let value = response.to_value();
5478        assert_eq!(value["path"], serde_json::json!(["File", "Save"]));
5479        assert_eq!(value["pressed"], true);
5480        let recorded = backend.menu_clicks.lock().unwrap().clone();
5481        assert_eq!(recorded.len(), 1);
5482        assert_eq!(
5483            recorded[0].segments,
5484            vec!["File".to_string(), "Save".to_string()]
5485        );
5486
5487        backend.menu_disabled = true;
5488        let response = dispatch_with_context(
5489            &backend,
5490            Action::MenuClick {
5491                app: Some("Notes".into()),
5492                path: "File > Save".into(),
5493            },
5494            &ctx("Notes"),
5495        );
5496        assert!(!response.ok, "disabled menu item must fail");
5497        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5498    }
5499
5500    #[test]
5501    fn p9_apple_event_records_only_length_and_language() {
5502        let backend = StubBackend::default();
5503        *backend.apple_event_stdout.lock().unwrap() = "Notes".into();
5504        let secret = "system prompt with token ABCDEFG";
5505        let response = dispatch_with_context(
5506            &backend,
5507            Action::AppleEvent {
5508                app: Some("Notes".into()),
5509                script: secret.into(),
5510                language: Some(AppleEventLanguage::AppleScript),
5511            },
5512            &ctx("Notes"),
5513        );
5514        assert!(response.ok, "{:?}", response.to_value());
5515        let value = response.to_value();
5516        assert_eq!(value["language"], "AppleScript");
5517        assert_eq!(value["script_bytes"], secret.len());
5518        assert_eq!(value["stdout"], "Notes");
5519        assert!(value["script_hash"].is_string());
5520        let recorded = backend.apple_event_calls.lock().unwrap().clone();
5521        assert_eq!(recorded.len(), 1);
5522        assert_eq!(recorded[0].1, AppleEventLanguage::AppleScript);
5523        // Audit-style storage must not capture the body verbatim.
5524        assert_eq!(recorded[0].2, secret.len());
5525    }
5526
5527    #[test]
5528    fn p9_apple_event_records_language_javascript() {
5529        let backend = StubBackend::default();
5530        let response = dispatch_with_context(
5531            &backend,
5532            Action::AppleEvent {
5533                app: Some("Notes".into()),
5534                script: "Application('Notes').name()".into(),
5535                language: Some(AppleEventLanguage::JavaScript),
5536            },
5537            &ctx("Notes"),
5538        );
5539        assert!(response.ok);
5540        assert_eq!(response.to_value()["language"], "JavaScript");
5541    }
5542
5543    #[test]
5544    fn p9_apple_event_failure_surfaces_execution_failed() {
5545        let backend = StubBackend {
5546            apple_event_fail: true,
5547            ..Default::default()
5548        };
5549        *backend.apple_event_stderr.lock().unwrap() = "execution error: 42".into();
5550        let response = dispatch_with_context(
5551            &backend,
5552            Action::AppleEvent {
5553                app: Some("Notes".into()),
5554                script: "1/0".into(),
5555                language: None,
5556            },
5557            &ctx("Notes"),
5558        );
5559        assert!(!response.ok);
5560        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5561    }
5562
5563    #[test]
5564    fn p9_dock_list_and_click_round_trip() {
5565        let backend = StubBackend::default();
5566        backend.dock_items.lock().unwrap().extend([
5567            "Finder".to_string(),
5568            "Terminal".to_string(),
5569            "Mail".to_string(),
5570        ]);
5571        let response = dispatch_with(&backend, Action::DockList {});
5572        assert!(response.ok);
5573        let value = response.to_value();
5574        let items = value["items"].as_array().unwrap();
5575        assert_eq!(items.len(), 3);
5576        assert_eq!(items[0]["title"], "Finder");
5577
5578        let response = dispatch_with(
5579            &backend,
5580            Action::DockClick {
5581                title: "terminal".into(),
5582            },
5583        );
5584        assert!(response.ok, "{:?}", response.to_value());
5585        assert_eq!(response.to_value()["pressed"], true);
5586        assert_eq!(
5587            backend.dock_clicks.lock().unwrap().clone(),
5588            vec!["terminal".to_string()]
5589        );
5590
5591        let response = dispatch_with(
5592            &backend,
5593            Action::DockClick {
5594                title: "missing".into(),
5595            },
5596        );
5597        assert!(!response.ok);
5598        assert_eq!(response.to_value()["error"]["code"], "element_not_found");
5599    }
5600
5601    #[test]
5602    fn p9_spaces_reports_screen_groups_from_fake_list() {
5603        use crate::target::{WindowBounds, WindowInfo};
5604        use crate::types::DisplayInfo;
5605        use crate::window_groups::SpacesReport;
5606
5607        let mut spaces_report = SpacesReport {
5608            screens: Vec::new(),
5609            note: String::new(),
5610        };
5611        spaces_report
5612            .screens
5613            .push(crate::window_groups::SpaceScreen {
5614                index: 0,
5615                name: "Display 0".into(),
5616                x: 0,
5617                y: 0,
5618                width: 1000,
5619                height: 1000,
5620                is_primary: true,
5621                windows: vec![WindowInfo {
5622                    id: 1,
5623                    pid: 100,
5624                    app: "Notes".into(),
5625                    title: "Note 1".into(),
5626                    bounds: WindowBounds {
5627                        x: 10,
5628                        y: 10,
5629                        width: 200,
5630                        height: 200,
5631                    },
5632                    z: 0,
5633                    layer: 0,
5634                    on_screen: true,
5635                    ax_matched: false,
5636                }],
5637            });
5638
5639        let backend = StubBackend::default();
5640        *backend.spaces_report.lock().unwrap() = spaces_report;
5641        let response = dispatch_with(&backend, Action::Spaces {});
5642        assert!(response.ok);
5643        let value = response.to_value();
5644        assert_eq!(value["action"], "spaces");
5645        assert_eq!(value["screens"][0]["name"], "Display 0");
5646        assert_eq!(value["screens"][0]["windows"][0]["title"], "Note 1");
5647        assert_eq!(value["screens"][0]["is_primary"], true);
5648        // `group_windows` itself is exercised in window_groups tests.
5649        let _ = DisplayInfo {
5650            id: 0,
5651            name: String::new(),
5652            x: 0,
5653            y: 0,
5654            width: 0,
5655            height: 0,
5656            scale: 1.0,
5657            is_primary: false,
5658        };
5659    }
5660
5661    #[test]
5662    fn p9_shortcuts_run_invokes_with_only_the_name() {
5663        // The test backend does not exercise the binary; this test only
5664        // confirms dispatch reaches the helper when the binary is missing.
5665        let backend = StubBackend::default();
5666        let response = dispatch_with(
5667            &backend,
5668            Action::ShortcutsRun {
5669                name: "Open URL".into(),
5670            },
5671        );
5672        // On macOS the dispatch path reaches the helper, so the response is
5673        // either ok (binary present) or `execution_failed` (binary absent).
5674        // Off macOS the dispatcher short-circuits to `unsupported`. Either
5675        // outcome is acceptable; the failure code must be the documented
5676        // one.
5677        if !response.ok {
5678            let value = response.to_value();
5679            let code = value["error"]["code"].as_str().unwrap_or_default();
5680            assert!(
5681                matches!(code, "execution_failed" | "unsupported"),
5682                "{value:?}"
5683            );
5684        }
5685    }
5686
5687    #[test]
5688    fn session_prune_action_removes_an_empty_temp_session() {
5689        let _cache = crate::cache::CacheEnv::acquire();
5690        let dir = cache::session_dir(Some("empty-session")).unwrap();
5691        assert!(dir.join("owner.json").is_file());
5692        let response = dispatch(Action::SessionPrune {
5693            dry_run: true,
5694            ttl_secs: Some(0),
5695        });
5696        assert!(response.ok, "{response:?}");
5697        assert_eq!(response.to_value()["dry_run"], true);
5698        assert!(response.to_value()["removed"]
5699            .as_array()
5700            .unwrap()
5701            .iter()
5702            .any(|name| name == "empty-session"));
5703        assert!(dir.exists());
5704        let response = dispatch(Action::SessionPrune {
5705            dry_run: false,
5706            ttl_secs: Some(0),
5707        });
5708        assert!(response.ok, "{response:?}");
5709        assert!(!dir.exists());
5710    }
5711
5712    #[test]
5713    fn pause_short_circuits_before_the_stub_clicks() {
5714        let backend = StubBackend::default();
5715        let _paused = policy::override_paused(true);
5716        let response = dispatch_with(
5717            &backend,
5718            Action::Click {
5719                x: Some(1),
5720                y: Some(2),
5721                button: MouseButton::Left,
5722                double: false,
5723                count: 1,
5724            },
5725        );
5726        assert_eq!(response.to_value()["error"]["code"], "paused");
5727        assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
5728        assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5729        let waited = dispatch_with(
5730            &backend,
5731            Action::Wait {
5732                ms: Some(0),
5733                seconds: None,
5734            },
5735        );
5736        assert!(waited.ok, "{waited:?}");
5737    }
5738
5739    #[test]
5740    fn confirm_hook_allows_quit_and_a_failure_does_not() {
5741        let backend = StubBackend::default();
5742        let dir = std::env::temp_dir().join(format!(
5743            "computer-use-hook-{}-{}",
5744            std::process::id(),
5745            std::time::SystemTime::now()
5746                .duration_since(std::time::UNIX_EPOCH)
5747                .map(|duration| duration.as_nanos())
5748                .unwrap_or(0)
5749        ));
5750        let _ = std::fs::remove_dir_all(&dir);
5751        std::fs::create_dir_all(&dir).unwrap();
5752        #[cfg(unix)]
5753        let (allow, deny) = {
5754            let allow = dir.join("allow.sh");
5755            let deny = dir.join("deny.sh");
5756            std::fs::write(&allow, "#!/bin/sh\ncat >/dev/null\nexit 0\n").unwrap();
5757            std::fs::write(&deny, "#!/bin/sh\ncat >/dev/null\nexit 1\n").unwrap();
5758            use std::os::unix::fs::PermissionsExt;
5759            std::fs::set_permissions(&allow, std::fs::Permissions::from_mode(0o700)).unwrap();
5760            std::fs::set_permissions(&deny, std::fs::Permissions::from_mode(0o700)).unwrap();
5761            (allow.display().to_string(), deny.display().to_string())
5762        };
5763        #[cfg(windows)]
5764        let (allow, deny) = ("cmd /C exit 0".to_string(), "cmd /C exit 1".to_string());
5765        let mut context = ctx("412");
5766        context.confirm_cmd = Some(deny);
5767        let denied = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5768        assert_eq!(denied.to_value()["error"]["code"], "confirm_denied");
5769        assert!(backend.quit.lock().unwrap().is_empty());
5770        context.confirm_cmd = Some(allow);
5771        let allowed = dispatch_with_context(&backend, Action::Quit { force: false }, &context);
5772        assert!(allowed.ok, "{allowed:?}");
5773        assert_eq!(backend.quit.lock().unwrap().len(), 1);
5774        let _ = std::fs::remove_dir_all(&dir);
5775    }
5776
5777    #[test]
5778    fn blocked_url_prevents_the_click() {
5779        let mut backend = StubBackend::default();
5780        backend.apps.push(crate::target::AppInfo {
5781            name: "Safari".into(),
5782            bundle_id: Some("com.apple.Safari".into()),
5783            pid: 80,
5784            path: Some("/Applications/Safari.app/Contents/MacOS/Safari".into()),
5785            active: false,
5786            hidden: false,
5787            activation_policy: crate::target::ActivationPolicy::Regular,
5788        });
5789        backend
5790            .browser_urls
5791            .lock()
5792            .unwrap()
5793            .insert(80, "https://accounts.example.com/login".into());
5794        let dir = std::env::temp_dir().join(format!(
5795            "computer-use-url-{}-{}",
5796            std::process::id(),
5797            std::time::SystemTime::now()
5798                .duration_since(std::time::UNIX_EPOCH)
5799                .map(|duration| duration.as_nanos())
5800                .unwrap_or(0)
5801        ));
5802        let _ = std::fs::remove_dir_all(&dir);
5803        std::fs::create_dir_all(&dir).unwrap();
5804        let policy_path = dir.join("policy.toml");
5805        std::fs::write(
5806            &policy_path,
5807            "version = 1\n[browser]\nblocked_urls = [\"https://accounts.example.com/*\"]\n",
5808        )
5809        .unwrap();
5810        let mut context = ctx("80");
5811        context.policy = Some(policy_path.display().to_string());
5812        let response = dispatch_with_context(
5813            &backend,
5814            Action::Click {
5815                x: Some(3),
5816                y: Some(4),
5817                button: MouseButton::Left,
5818                double: false,
5819                count: 1,
5820            },
5821            &context,
5822        );
5823        assert_eq!(response.to_value()["error"]["code"], "blocked_url");
5824        assert!(response.to_value()["error"]["message"]
5825            .as_str()
5826            .unwrap()
5827            .contains("https://accounts.example.com/*"));
5828        assert!(!response.to_value()["error"]["message"]
5829            .as_str()
5830            .unwrap()
5831            .contains("login"));
5832        assert_eq!(backend.click_attempts.load(Ordering::SeqCst), 0);
5833        backend
5834            .browser_urls
5835            .lock()
5836            .unwrap()
5837            .insert(80, "https://example.com/".into());
5838        let allowed = dispatch_with_context(
5839            &backend,
5840            Action::Click {
5841                x: Some(3),
5842                y: Some(4),
5843                button: MouseButton::Left,
5844                double: false,
5845                count: 1,
5846            },
5847            &context,
5848        );
5849        assert!(allowed.ok, "{allowed:?}");
5850        assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5851        let _ = std::fs::remove_dir_all(&dir);
5852    }
5853
5854    fn temp_dir(label: &str) -> std::path::PathBuf {
5855        let dir = std::env::temp_dir().join(format!(
5856            "computer-use-{label}-{}-{}",
5857            std::process::id(),
5858            std::time::SystemTime::now()
5859                .duration_since(std::time::UNIX_EPOCH)
5860                .map(|duration| duration.as_nanos())
5861                .unwrap_or(0)
5862        ));
5863        let _ = std::fs::remove_dir_all(&dir);
5864        std::fs::create_dir_all(&dir).unwrap();
5865        dir
5866    }
5867
5868    fn click_at(x: i32, y: i32) -> Action {
5869        Action::Click {
5870            x: Some(x),
5871            y: Some(y),
5872            button: MouseButton::Left,
5873            double: false,
5874            count: 1,
5875        }
5876    }
5877
5878    fn last_audit(dir: &std::path::Path) -> Value {
5879        let text = std::fs::read_to_string(dir.join("audit.jsonl")).unwrap();
5880        serde_json::from_str(text.lines().last().unwrap()).unwrap()
5881    }
5882
5883    #[test]
5884    fn auto_event_falls_back_to_activate_unless_blocked() {
5885        let backend = StubBackend {
5886            fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5887            ..StubBackend::default()
5888        };
5889        let response = dispatch_with(&backend, click_at(3, 4));
5890        assert!(response.ok, "{response:?}");
5891        let value = response.to_value();
5892        assert_eq!(value["tier"], "activate");
5893        assert_eq!(value["cursor_moved"], true);
5894        assert_eq!(value["focus_restored"], true);
5895        assert_eq!(
5896            backend.tiers_run.lock().unwrap().as_slice(),
5897            ["event", "activate"]
5898        );
5899        assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
5900
5901        let held = StubBackend {
5902            fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5903            ..StubBackend::default()
5904        };
5905        let dir = temp_dir("lock-fallback");
5906        let path = dir.join("foreground.lock");
5907        let _slot = crate::foreground::push_lock_path(&path);
5908        let _lock = crate::foreground::try_lock(&path).unwrap();
5909        let response = dispatch_with(&held, click_at(3, 4));
5910        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5911        assert!(response.to_value()["error"]["message"]
5912            .as_str()
5913            .unwrap()
5914            .contains("stub tier event failed"));
5915        assert_eq!(held.tiers_run.lock().unwrap().as_slice(), ["event"]);
5916        assert_eq!(held.clicks.load(Ordering::SeqCst), 0);
5917        drop(_lock);
5918        let _ = std::fs::remove_dir_all(&dir);
5919
5920        let background = StubBackend {
5921            fail_tiers: std::sync::Mutex::new(vec!["event".into()]),
5922            ..StubBackend::default()
5923        };
5924        let response = dispatch_with_context(
5925            &background,
5926            click_at(1, 2),
5927            &CallContext {
5928                require_background: true,
5929                ..CallContext::default()
5930            },
5931        );
5932        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5933        assert_eq!(background.tiers_run.lock().unwrap().as_slice(), ["event"]);
5934        assert_eq!(background.clicks.load(Ordering::SeqCst), 0);
5935
5936        let refused = StubBackend::default();
5937        let response = dispatch_with_context(
5938            &refused,
5939            click_at(1, 2),
5940            &CallContext {
5941                tier: Some("activate".into()),
5942                require_background: true,
5943                ..CallContext::default()
5944            },
5945        );
5946        assert!(response.to_value()["error"]["message"]
5947            .as_str()
5948            .unwrap()
5949            .contains("--require-background"));
5950        assert!(refused.tiers_run.lock().unwrap().is_empty());
5951        assert_eq!(refused.clicks.load(Ordering::SeqCst), 0);
5952    }
5953
5954    #[test]
5955    fn activate_names_the_lock_holder_before_it_clicks() {
5956        let backend = StubBackend::default();
5957        let dir = temp_dir("lock-held");
5958        let path = dir.join("foreground.lock");
5959        let _slot = crate::foreground::push_lock_path(&path);
5960        let _lock = crate::foreground::try_lock(&path).unwrap();
5961        let response = dispatch_with_context(
5962            &backend,
5963            click_at(9, 8),
5964            &CallContext {
5965                tier: Some("activate".into()),
5966                ..CallContext::default()
5967            },
5968        );
5969        assert_eq!(response.to_value()["error"]["code"], "execution_failed");
5970        let message = response.to_value()["error"]["message"]
5971            .as_str()
5972            .unwrap()
5973            .to_string();
5974        assert!(
5975            message.contains(&format!(
5976                "foreground.lock is held by pid {}",
5977                std::process::id()
5978            )),
5979            "{message}"
5980        );
5981        assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
5982        assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 1);
5983        assert!(backend.tiers_run.lock().unwrap().is_empty());
5984        drop(_lock);
5985        let _ = std::fs::remove_dir_all(&dir);
5986    }
5987
5988    #[test]
5989    fn synthetic_without_the_feature_does_not_hit_test_or_click() {
5990        let backend = StubBackend::default();
5991        let response = dispatch_with_context(
5992            &backend,
5993            click_at(1, 1),
5994            &CallContext {
5995                tier: Some("synthetic".into()),
5996                ..CallContext::default()
5997            },
5998        );
5999        if crate::tier::skylight_compiled() {
6000            assert!(response.ok, "{response:?}");
6001            assert_eq!(response.to_value()["tier"], "synthetic");
6002            assert_eq!(backend.clicks.load(Ordering::SeqCst), 1);
6003            assert_eq!(backend.tiers_run.lock().unwrap().as_slice(), ["synthetic"]);
6004        } else {
6005            assert_eq!(response.to_value()["error"]["code"], "unsupported");
6006            assert!(response.to_value()["error"]["message"]
6007                .as_str()
6008                .unwrap()
6009                .contains("built without skylight feature"));
6010            assert_eq!(backend.hit_calls.load(Ordering::SeqCst), 0);
6011            assert_eq!(backend.clicks.load(Ordering::SeqCst), 0);
6012            assert!(backend.tiers_run.lock().unwrap().is_empty());
6013        }
6014    }
6015
6016    #[test]
6017    fn missing_overlay_helper_warns_and_no_overlay_skips_it() {
6018        let dir = temp_dir("overlay-warn");
6019        let _slot = crate::overlay::push_socket_path(dir.join("missing.sock"));
6020        let backend = StubBackend::default();
6021        let response = dispatch_with_context(
6022            &backend,
6023            click_at(4, 5),
6024            &CallContext {
6025                session: Some("agent".into()),
6026                ..CallContext::default()
6027            },
6028        );
6029        assert!(response.ok, "{response:?}");
6030        assert_eq!(response.to_value()["tier"], "event");
6031        assert_eq!(response.to_value()["cursor_moved"], false);
6032        assert!(response.to_value().get("focus_restored").is_none());
6033        let warnings = response.to_value()["warnings"].as_array().unwrap().clone();
6034        assert!(warnings
6035            .iter()
6036            .any(|item| item == "overlay helper not running"));
6037
6038        let quiet = dispatch_with_context(
6039            &backend,
6040            click_at(4, 5),
6041            &CallContext {
6042                no_overlay: true,
6043                ..CallContext::default()
6044            },
6045        );
6046        assert!(quiet.ok, "{quiet:?}");
6047        assert!(quiet.to_value().get("warnings").is_none());
6048        let _ = std::fs::remove_dir_all(&dir);
6049    }
6050
6051    #[test]
6052    fn audit_records_decision_source_and_the_real_code() {
6053        let dir = temp_dir("audit");
6054        let _state = crate::audit::push_state_dir(&dir);
6055        let backend = StubBackend::default();
6056        let response = dispatch_with(&backend, click_at(2, 3));
6057        assert!(response.ok, "{response:?}");
6058        let line = last_audit(&dir);
6059        assert_eq!(line["action"], "click");
6060        assert_eq!(line["code"], "ok");
6061        assert_eq!(line["decision"], "allow");
6062        assert_eq!(line["source"], "default");
6063        assert_eq!(line["tier"], "event");
6064        assert!(line.get("text").is_none());
6065
6066        let denied = dispatch_with_context(&backend, Action::Launch, &ctx("com.apple.Terminal"));
6067        assert_eq!(denied.to_value()["error"]["code"], "policy_denied");
6068        let line = last_audit(&dir);
6069        assert_eq!(line["action"], "launch");
6070        assert_eq!(line["code"], "policy_denied");
6071        assert_eq!(line["decision"], "deny");
6072        assert_eq!(line["source"], "builtin");
6073
6074        let checked =
6075            dispatch_with_context(&backend, Action::PolicyCheck, &ctx("com.apple.Terminal"));
6076        assert!(checked.ok, "{checked:?}");
6077        assert_eq!(checked.to_value()["decision"], "deny");
6078        let line = last_audit(&dir);
6079        assert_eq!(line["action"], "policy_check");
6080        assert_eq!(line["code"], "ok");
6081        assert_eq!(line["decision"], "deny");
6082        assert_eq!(line["source"], "builtin");
6083
6084        let bad = dispatch_with(
6085            &backend,
6086            Action::Click {
6087                x: None,
6088                y: None,
6089                button: MouseButton::Left,
6090                double: false,
6091                count: 1,
6092            },
6093        );
6094        assert_eq!(bad.to_value()["error"]["code"], "invalid_arguments");
6095        let line = last_audit(&dir);
6096        assert_eq!(line["code"], "invalid_arguments");
6097        assert_eq!(line["decision"], "allow");
6098        assert_eq!(line["source"], "default");
6099        let _ = std::fs::remove_dir_all(&dir);
6100    }
6101
6102    #[test]
6103    fn drag_hold_click_and_keys_use_the_stub() {
6104        let backend = StubBackend::default();
6105        let short = dispatch_with(
6106            &backend,
6107            Action::Drag {
6108                path: vec![crate::types::Point { x: 1, y: 1 }],
6109                button: MouseButton::Left,
6110            },
6111        );
6112        assert_eq!(short.to_value()["error"]["code"], "invalid_arguments");
6113        assert_eq!(backend.moves.load(Ordering::SeqCst), 0);
6114
6115        let dragged = dispatch_with(
6116            &backend,
6117            Action::Drag {
6118                path: vec![
6119                    crate::types::Point { x: 1, y: 2 },
6120                    crate::types::Point { x: 3, y: 4 },
6121                    crate::types::Point { x: 5, y: 6 },
6122                ],
6123                button: MouseButton::Left,
6124            },
6125        );
6126        assert!(dragged.ok, "{dragged:?}");
6127        assert_eq!(dragged.to_value()["moves"], 2);
6128        assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6129        assert_eq!(dragged.to_value()["tier"], "event");
6130
6131        let ax = dispatch_with_context(
6132            &backend,
6133            Action::Drag {
6134                path: vec![
6135                    crate::types::Point { x: 1, y: 2 },
6136                    crate::types::Point { x: 3, y: 4 },
6137                ],
6138                button: MouseButton::Left,
6139            },
6140            &CallContext {
6141                tier: Some("ax".into()),
6142                ..CallContext::default()
6143            },
6144        );
6145        assert_eq!(ax.to_value()["error"]["code"], "unsupported");
6146        assert!(ax.to_value()["error"]["message"]
6147            .as_str()
6148            .unwrap()
6149            .contains("pass --element"));
6150        assert_eq!(backend.moves.load(Ordering::SeqCst), 2);
6151
6152        let triple = dispatch_with(
6153            &backend,
6154            Action::TripleClick {
6155                x: Some(8),
6156                y: Some(9),
6157                button: MouseButton::Left,
6158            },
6159        );
6160        assert!(triple.ok, "{triple:?}");
6161        assert_eq!(triple.to_value()["action"], "triple_click");
6162        assert_eq!(triple.to_value()["count"], 3);
6163        assert_eq!(backend.clicks.load(Ordering::SeqCst), 3);
6164
6165        let held = dispatch_with(
6166            &backend,
6167            Action::HoldKey {
6168                keys: "shift".into(),
6169                ms: 40_000,
6170            },
6171        );
6172        assert_eq!(held.to_value()["error"]["code"], "invalid_arguments");
6173        assert!(backend.holds_ms.lock().unwrap().is_empty());
6174
6175        let shift = dispatch_with(
6176            &backend,
6177            Action::HoldKey {
6178                keys: "shift".into(),
6179                ms: 400,
6180            },
6181        );
6182        assert!(shift.ok, "{shift:?}");
6183        assert_eq!(backend.holds_ms.lock().unwrap().as_slice(), [400]);
6184
6185        let sequence = dispatch_with(
6186            &backend,
6187            Action::Key {
6188                keys: "super+c Return".into(),
6189            },
6190        );
6191        assert!(sequence.ok, "{sequence:?}");
6192        assert_eq!(sequence.to_value()["strokes"], 2);
6193        let keys = backend.keys.lock().unwrap();
6194        let sequence = &keys[keys.len() - 2..];
6195        assert_eq!(sequence[0].1, "c");
6196        assert_eq!(sequence[0].0, vec![Modifier::Meta]);
6197        assert_eq!(sequence[1].1, "enter");
6198        drop(keys);
6199
6200        let unknown = dispatch_with(
6201            &backend,
6202            Action::Key {
6203                keys: "bogus".into(),
6204            },
6205        );
6206        assert_eq!(unknown.to_value()["error"]["code"], "invalid_arguments");
6207    }
6208
6209    #[test]
6210    fn mouse_down_releases_when_the_session_ends() {
6211        let backend = StubBackend::default();
6212        let down = Action::MouseDown {
6213            x: 4,
6214            y: 5,
6215            button: MouseButton::Left,
6216        };
6217        let alone = dispatch_with(&backend, down.clone());
6218        assert!(alone.ok, "{alone:?}");
6219        assert!(backend.held_buttons.lock().unwrap().is_empty());
6220        assert_eq!(
6221            backend.releases.lock().unwrap().as_slice(),
6222            [MouseButton::Left]
6223        );
6224
6225        backend.releases.lock().unwrap().clear();
6226        let _session = crate::InputSession::open(&backend);
6227        let kept = dispatch_with(&backend, down);
6228        assert!(kept.ok, "{kept:?}");
6229        assert_eq!(
6230            backend.held_buttons.lock().unwrap().as_slice(),
6231            [MouseButton::Left]
6232        );
6233        assert!(backend.releases.lock().unwrap().is_empty());
6234        drop(_session);
6235        assert!(backend.held_buttons.lock().unwrap().is_empty());
6236        assert_eq!(
6237            backend.releases.lock().unwrap().as_slice(),
6238            [MouseButton::Left]
6239        );
6240
6241        let batch = dispatch_with(
6242            &backend,
6243            Action::Batch {
6244                stop_on_error: true,
6245                actions: vec![
6246                    json!({"action": "mouse_down", "x": 1, "y": 2}),
6247                    json!({"action": "mouse_up", "x": 1, "y": 2}),
6248                ],
6249            },
6250        );
6251        assert!(batch.ok, "{batch:?}");
6252        assert!(backend.held_buttons.lock().unwrap().is_empty());
6253    }
6254
6255    #[test]
6256    fn clipboard_confirms_and_omits_the_text_from_the_audit() {
6257        let dir = temp_dir("clipboard-audit");
6258        let _state = crate::audit::push_state_dir(&dir);
6259        let _cache = crate::cache::CacheEnv::acquire();
6260        let backend = StubBackend::default();
6261        let secret = "do-not-log";
6262        let response = dispatch_with(
6263            &backend,
6264            Action::ClipboardSet {
6265                text: Some(secret.into()),
6266                image: None,
6267            },
6268        );
6269        assert_eq!(response.to_value()["error"]["code"], "confirm_required");
6270        assert!(backend.clipboard_text.lock().unwrap().is_none());
6271        let line = last_audit(&dir);
6272        assert_eq!(line["action"], "clipboard_set");
6273        assert_eq!(line["decision"], "confirm");
6274        assert_eq!(line["chars"], secret.chars().count() as u64);
6275        let encoded = line.to_string();
6276        assert!(!encoded.contains(secret));
6277        assert!(line.get("text").is_none());
6278
6279        *backend.clipboard_text.lock().unwrap() = Some("ready".into());
6280        let got = dispatch_with(&backend, Action::ClipboardGet);
6281        assert!(got.ok, "{got:?}");
6282        assert_eq!(got.to_value()["kind"], "text");
6283        assert_eq!(got.to_value()["text"], "ready");
6284
6285        let empty = dispatch_with(
6286            &backend,
6287            Action::ClipboardSet {
6288                text: Some(String::new()),
6289                image: None,
6290            },
6291        );
6292        assert_eq!(empty.to_value()["error"]["code"], "invalid_arguments");
6293        let _ = std::fs::remove_dir_all(&dir);
6294    }
6295
6296    #[test]
6297    fn zoom_crops_a_window_region_and_rejects_the_far_edge() {
6298        let _cache = crate::cache::CacheEnv::acquire();
6299        let backend = StubBackend::default();
6300        let response = dispatch_with_context(
6301            &backend,
6302            Action::Zoom {
6303                output: None,
6304                region: Region::new(0, 0, 10, 10).unwrap(),
6305                format: Some(ImageFormat::Png),
6306                max_width: Some(0),
6307                canvas: None,
6308                base64: false,
6309            },
6310            &CallContext {
6311                app: Some("Notes".into()),
6312                session: Some("p8-zoom".into()),
6313                ..CallContext::default()
6314            },
6315        );
6316        assert!(response.ok, "{response:?}");
6317        let value = response.to_value();
6318        assert_eq!(value["width"], 20);
6319        assert_eq!(value["height"], 20);
6320        assert_eq!(value["logical_width"], 10);
6321        assert_eq!(value["logical_height"], 10);
6322
6323        let outside = dispatch_with_context(
6324            &backend,
6325            Action::Zoom {
6326                output: None,
6327                region: Region::new(0, 0, 401, 300).unwrap(),
6328                format: None,
6329                max_width: Some(0),
6330                canvas: None,
6331                base64: false,
6332            },
6333            &CallContext {
6334                app: Some("Notes".into()),
6335                session: Some("p8-zoom-out".into()),
6336                ..CallContext::default()
6337            },
6338        );
6339        assert_eq!(outside.to_value()["error"]["code"], "invalid_arguments");
6340
6341        let both = dispatch_with_context(
6342            &backend,
6343            Action::Screenshot {
6344                output: None,
6345                region: Some(Region::new(0, 0, 10, 10).unwrap()),
6346                format: None,
6347                max_width: None,
6348                canvas: None,
6349                base64: false,
6350                marks: false,
6351            },
6352            &CallContext {
6353                app: Some("Notes".into()),
6354                ..CallContext::default()
6355            },
6356        );
6357        assert_eq!(both.to_value()["error"]["code"], "invalid_arguments");
6358        assert!(both.to_value()["error"]["message"]
6359            .as_str()
6360            .unwrap()
6361            .contains("not both"));
6362    }
6363
6364    // ----- P10 gap fixes -----
6365
6366    #[test]
6367    fn browser_text_action_round_trips_with_only_selector() {
6368        let text: Action =
6369            serde_json::from_str(r#"{"action":"browser_text","selector":"h1"}"#).unwrap();
6370        match &text {
6371            Action::BrowserText {
6372                selector,
6373                target_id,
6374            } => {
6375                assert_eq!(selector, "h1");
6376                assert!(target_id.is_none());
6377            }
6378            other => panic!("unexpected {other:?}"),
6379        }
6380        let encoded = serde_json::to_string(&text).unwrap();
6381        let decoded: Action = serde_json::from_str(&encoded).unwrap();
6382        assert_eq!(decoded, text);
6383    }
6384
6385    #[test]
6386    fn browser_type_action_round_trips_with_text_and_optional_selector() {
6387        let action: Action =
6388            serde_json::from_str(r##"{"action":"browser_type","text":"hi","selector":"#name"}"##)
6389                .unwrap();
6390        match &action {
6391            Action::BrowserType {
6392                text,
6393                selector,
6394                target_id,
6395            } => {
6396                assert_eq!(text, "hi");
6397                assert_eq!(selector.as_deref(), Some("#name"));
6398                assert!(target_id.is_none());
6399            }
6400            other => panic!("unexpected {other:?}"),
6401        }
6402        let encoded = serde_json::to_string(&action).unwrap();
6403        let decoded: Action = serde_json::from_str(&encoded).unwrap();
6404        assert_eq!(decoded, action);
6405    }
6406
6407    #[test]
6408    fn browser_text_label_is_browser_text_and_class_is_observe() {
6409        // BrowserText reads innerText; classified Observe (read-only).
6410        let action = Action::BrowserText {
6411            selector: "h1".into(),
6412            target_id: None,
6413        };
6414        assert_eq!(action_label(&action), "browser_text");
6415        assert!(matches!(policy_class(&action), PolicyClass::Observe));
6416    }
6417
6418    #[test]
6419    fn browser_type_label_is_browser_type_and_class_is_mutate() {
6420        // BrowserType types text; classified Mutate (write). The text body
6421        // never leaves the audit row — only the char count does.
6422        let action = Action::BrowserType {
6423            text: "abc".into(),
6424            selector: Some("#name".into()),
6425            target_id: None,
6426        };
6427        assert_eq!(action_label(&action), "browser_type");
6428        assert!(matches!(policy_class(&action), PolicyClass::Mutate));
6429        assert_eq!(clipboard_chars(&action), Some(3));
6430    }
6431
6432    #[test]
6433    fn browser_text_does_not_record_chars_in_audit() {
6434        // Reading innerText must not push a chars count into the audit row.
6435        let action = Action::BrowserText {
6436            selector: "h1".into(),
6437            target_id: None,
6438        };
6439        assert_eq!(clipboard_chars(&action), None);
6440    }
6441
6442    #[test]
6443    #[cfg(feature = "cdp")]
6444    fn browser_navigate_blocks_url_before_connecting() {
6445        // Use a deliberately unreachable WebSocket URL. If the URL block ran
6446        // before the connect attempt, we get blocked_url. If the order were
6447        // wrong we would see cdp_unavailable from the connection failure.
6448        let backend = StubBackend::default();
6449        let context = CallContext {
6450            cdp: true,
6451            cdp_url: Some("ws://127.0.0.1:1".into()),
6452            browser_blocked_urls: vec!["https://blocked.example.com".into()],
6453            ..CallContext::default()
6454        };
6455        let response = dispatch_with_context(
6456            &backend,
6457            Action::BrowserNavigate {
6458                url: "https://blocked.example.com".into(),
6459                target_id: None,
6460            },
6461            &context,
6462        );
6463        assert!(!response.ok);
6464        let value = response.to_value();
6465        assert_eq!(value["error"]["code"], "blocked_url");
6466    }
6467
6468    #[test]
6469    #[cfg(feature = "cdp")]
6470    fn browser_navigate_unblocked_url_still_attempts_connect() {
6471        // Sanity: without a matching pattern, navigate goes through the
6472        // normal path and reports the connect failure (cdp_unavailable).
6473        let backend = StubBackend::default();
6474        let context = CallContext {
6475            cdp: true,
6476            cdp_url: Some("ws://127.0.0.1:1".into()),
6477            ..CallContext::default()
6478        };
6479        let response = dispatch_with_context(
6480            &backend,
6481            Action::BrowserNavigate {
6482                url: "https://example.com".into(),
6483                target_id: None,
6484            },
6485            &context,
6486        );
6487        assert!(!response.ok);
6488        let value = response.to_value();
6489        assert_eq!(value["error"]["code"], "cdp_unavailable");
6490    }
6491
6492    // -----------------------------------------------------------------------
6493    // P14: the `--display` filter.
6494    // -----------------------------------------------------------------------
6495
6496    fn filtered(spec: &str) -> CallContext {
6497        CallContext {
6498            display: Some(spec.into()),
6499            ..CallContext::default()
6500        }
6501    }
6502
6503    fn code_of(response: &Response) -> String {
6504        response.to_value()["error"]["code"]
6505            .as_str()
6506            .unwrap_or("")
6507            .to_string()
6508    }
6509
6510    #[test]
6511    fn display_filter_rejects_a_point_outside_the_chosen_monitor() {
6512        // The stub reports 800x600 at the origin and 640x480 at (800, 100).
6513        let backend = StubBackend::with_two_displays();
6514        let context = filtered("2");
6515
6516        // On the second monitor: allowed, and the click really happened.
6517        let response = dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &context);
6518        assert!(response.ok, "{response:?}");
6519
6520        // On the first monitor, so outside the filter: invalid_arguments, and
6521        // nothing moved.
6522        let before = backend.moves.load(Ordering::SeqCst);
6523        for (x, y) in [(0, 0), (799, 599), (1440, 580), (800, 99), (900, 90)] {
6524            let response = dispatch_with_context(&backend, Action::Move { x, y }, &context);
6525            assert_eq!(code_of(&response), "invalid_arguments", "{x},{y}");
6526            assert_eq!(backend.moves.load(Ordering::SeqCst), before);
6527        }
6528        let message = dispatch_with_context(&backend, Action::Move { x: 0, y: 0 }, &context)
6529            .to_value()["error"]["message"]
6530            .as_str()
6531            .expect("message")
6532            .to_string();
6533        assert!(message.contains("640x480+800+100"), "{message}");
6534    }
6535
6536    #[test]
6537    fn display_filter_covers_click_move_drag_scroll_and_mouse_halves() {
6538        let backend = StubBackend::with_two_displays();
6539        let context = filtered("2");
6540        let inside = Action::Click {
6541            x: Some(900),
6542            y: Some(300),
6543            button: MouseButton::Left,
6544            double: false,
6545            count: 1,
6546        };
6547        let outside = Action::Click {
6548            x: Some(10),
6549            y: Some(10),
6550            button: MouseButton::Left,
6551            double: false,
6552            count: 1,
6553        };
6554
6555        assert!(dispatch_with_context(&backend, inside.clone(), &context).ok);
6556        assert_eq!(
6557            code_of(&dispatch_with_context(&backend, outside.clone(), &context)),
6558            "invalid_arguments"
6559        );
6560        assert_eq!(
6561            code_of(&dispatch_with_context(
6562                &backend,
6563                Action::DoubleClick {
6564                    x: Some(10),
6565                    y: Some(10),
6566                    button: MouseButton::Left
6567                },
6568                &context
6569            )),
6570            "invalid_arguments"
6571        );
6572        assert_eq!(
6573            code_of(&dispatch_with_context(
6574                &backend,
6575                Action::MouseDown {
6576                    x: 10,
6577                    y: 10,
6578                    button: MouseButton::Left
6579                },
6580                &context
6581            )),
6582            "invalid_arguments"
6583        );
6584        assert_eq!(
6585            code_of(&dispatch_with_context(
6586                &backend,
6587                Action::Scroll {
6588                    x: Some(10),
6589                    y: Some(10),
6590                    dx: Some(0),
6591                    dy: Some(1),
6592                    direction: None,
6593                    amount: None,
6594                },
6595                &context
6596            )),
6597            "invalid_arguments"
6598        );
6599        let path = vec![Point { x: 900, y: 300 }, Point { x: 10, y: 10 }];
6600        assert_eq!(
6601            code_of(&dispatch_with_context(
6602                &backend,
6603                Action::Drag {
6604                    path,
6605                    button: MouseButton::Left
6606                },
6607                &context
6608            )),
6609            "invalid_arguments"
6610        );
6611    }
6612
6613    #[test]
6614    fn a_coordinate_action_without_a_point_passes_the_filter() {
6615        // Element-targeted and wheel-only actions name no point, so the filter
6616        // has nothing to reject. (A bare `click` with neither `element` nor
6617        // `x`/`y` is refused earlier by `preflight_action`, not by this gate.)
6618        let backend = StubBackend::with_two_displays();
6619        let response = dispatch_with_context(
6620            &backend,
6621            Action::Scroll {
6622                x: None,
6623                y: None,
6624                dx: Some(0),
6625                dy: Some(2),
6626                direction: None,
6627                amount: None,
6628            },
6629            &filtered("2"),
6630        );
6631        assert!(response.ok, "{response:?}");
6632        assert_eq!(backend.scrolls.lock().expect("scrolls").len(), 1);
6633    }
6634
6635    #[test]
6636    fn an_unknown_display_id_is_invalid_arguments() {
6637        let backend = StubBackend::with_two_displays();
6638        let response =
6639            dispatch_with_context(&backend, Action::Move { x: 900, y: 300 }, &filtered("9"));
6640        assert_eq!(code_of(&response), "invalid_arguments");
6641
6642        // `displays` and `isolation` do not consume the filter, so a valid id
6643        // that no longer exists still lets them answer.
6644        let response = dispatch_with_context(&backend, Action::Displays, &filtered("99999"));
6645        assert!(response.ok, "{response:?}");
6646        let response = dispatch_with_context(&backend, Action::Isolation, &filtered("99999"));
6647        assert!(response.ok, "{response:?}");
6648        assert_eq!(backend.list_displays().unwrap().len(), 2);
6649
6650        // Bad syntax is refused everywhere, `displays` included.
6651        let response = dispatch_with_context(&backend, Action::Displays, &filtered("nope"));
6652        assert_eq!(code_of(&response), "invalid_arguments");
6653        // `virtual` is valid syntax, and `displays` does not consume the filter,
6654        // so it answers normally. A command that *would* capture under that
6655        // filter reports `unsupported` instead.
6656        let response = dispatch_with_context(&backend, Action::Displays, &filtered("virtual"));
6657        assert!(response.ok, "{response:?}");
6658    }
6659
6660    #[test]
6661    fn virtual_is_unsupported_over_the_json_path() {
6662        // The JSON path cannot exec a prepared X server, so it says so instead
6663        // of silently capturing the whole desktop.
6664        let backend = StubBackend::default();
6665        let response = dispatch_with_context(
6666            &backend,
6667            Action::Move { x: 10, y: 10 },
6668            &filtered("virtual"),
6669        );
6670        assert_eq!(code_of(&response), "unsupported");
6671
6672        let response = dispatch_with_context(
6673            &backend,
6674            Action::Screenshot {
6675                output: None,
6676                region: None,
6677                format: None,
6678                max_width: None,
6679                canvas: None,
6680                base64: false,
6681                marks: false,
6682            },
6683            &filtered("virtual"),
6684        );
6685        assert_eq!(code_of(&response), "unsupported");
6686    }
6687
6688    #[test]
6689    fn the_display_filter_never_leaks_into_displays_or_isolation() {
6690        let backend = StubBackend::with_two_displays();
6691        let context = filtered("9");
6692        assert!(dispatch_with_context(&backend, Action::Displays, &context).ok);
6693        assert!(dispatch_with_context(&backend, Action::Isolation, &context).ok);
6694    }
6695
6696    #[test]
6697    fn a_screenshot_region_must_fit_inside_the_filtered_monitor() {
6698        let _cache = crate::cache::CacheEnv::acquire();
6699        let backend = StubBackend::with_two_displays();
6700        let context = filtered("2");
6701        let inside = Action::Screenshot {
6702            output: None,
6703            region: Some(Region::new(800, 100, 640, 480).unwrap()),
6704            format: None,
6705            max_width: None,
6706            canvas: None,
6707            base64: false,
6708            marks: false,
6709        };
6710        let response = dispatch_with_context(&backend, inside, &context);
6711        assert!(response.ok, "{response:?}");
6712
6713        // The map origin is the monitor origin, not the desktop origin.
6714        let map = response.to_value();
6715        assert_eq!(map["origin"]["x"], 800);
6716        assert_eq!(map["origin"]["y"], 100);
6717        let _ = fs::remove_file(
6718            map["path"]
6719                .as_str()
6720                .expect("path")
6721                .parse::<std::path::PathBuf>()
6722                .expect("path"),
6723        );
6724
6725        let straddling = Action::Screenshot {
6726            output: None,
6727            region: Some(Region::new(700, 100, 200, 200).unwrap()),
6728            format: None,
6729            max_width: None,
6730            canvas: None,
6731            base64: false,
6732            marks: false,
6733        };
6734        assert_eq!(
6735            code_of(&dispatch_with_context(&backend, straddling, &context)),
6736            "invalid_arguments"
6737        );
6738    }
6739
6740    #[test]
6741    fn isolation_reports_the_matrix_and_the_infeasible_keys() {
6742        let response = dispatch_with(&StubBackend::default(), Action::Isolation);
6743        assert!(response.ok, "{response:?}");
6744        let value = response.to_value();
6745        assert_eq!(value["platform"], crate::isolation::platform());
6746        assert_eq!(value["display_filter"], true);
6747        assert_eq!(value["xvfb"], "linux only");
6748        assert_eq!(value["vm"], "documented, not shipped");
6749        assert_eq!(
6750            value["xvfb_recipe"]["server_command"],
6751            "Xvfb :99 -screen 0 1440x900x24"
6752        );
6753        for key in [
6754            "macos_vm_per_agent",
6755            "share_host_app_into_guest",
6756            "hide_virtual_display_from_user",
6757            "windows_virtual_display",
6758            "nested_wayland_input_without_portal",
6759        ] {
6760            assert!(
6761                value["infeasible"][key]
6762                    .as_str()
6763                    .is_some_and(|s| !s.is_empty()),
6764                "infeasible.{key} missing"
6765            );
6766        }
6767    }
6768
6769    #[test]
6770    fn isolation_is_read_only_and_ignores_the_state_flags() {
6771        assert_eq!(action_label(&Action::Isolation), "isolation");
6772        assert!(matches!(
6773            post_state_kind(&Action::Isolation),
6774            PostKind::ReadOnly
6775        ));
6776        let backend = StubBackend::default();
6777        let response = dispatch_with_context(
6778            &backend,
6779            Action::Isolation,
6780            &CallContext {
6781                return_state: true,
6782                include_screenshot: true,
6783                ..CallContext::default()
6784            },
6785        );
6786        assert!(response.ok);
6787        assert!(response.to_value().get("state").is_none());
6788    }
6789
6790    #[test]
6791    fn the_isolation_action_parses_from_json() {
6792        let request = parse_request(json!({"id": 7, "action": "isolation"})).unwrap();
6793        assert_eq!(request.id, Some(json!(7)));
6794        assert!(matches!(request.action, Action::Isolation));
6795        // The shared `display` key is stripped, never decoded as an action field.
6796        let request = parse_request(json!({"action": "isolation", "display": "1"})).unwrap();
6797        assert_eq!(request.context.display.as_deref(), Some("1"));
6798    }
6799
6800    fn scroll_action(dx: Option<i32>, dy: Option<i32>) -> Action {
6801        Action::Scroll {
6802            x: None,
6803            y: None,
6804            dx,
6805            dy,
6806            direction: None,
6807            amount: None,
6808        }
6809    }
6810
6811    /// Compare a JSON float with a tolerance, so `0.05 * 3` is `0.15`.
6812    fn near(value: f64, expected: f64) -> bool {
6813        (value - expected).abs() < 1e-9
6814    }
6815
6816    fn tier_context(tier: &str) -> CallContext {
6817        CallContext {
6818            tier: Some(tier.to_string()),
6819            ..CallContext::default()
6820        }
6821    }
6822
6823    #[test]
6824    fn a_scroll_reports_the_tier_that_actually_moved_the_scroll_bar() {
6825        let backend = StubBackend::default();
6826        let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6827        assert_eq!(value["ok"], json!(true));
6828        assert_eq!(value["via"], json!("event"));
6829        assert_eq!(value["scrolled"], json!(true));
6830        assert_eq!(value["before"], json!(0.0));
6831        assert!(near(value["after"].as_f64().unwrap_or_default(), 0.15));
6832        assert_eq!(value["tier"], json!("event"));
6833        // A delivered wheel event has nothing to warn about.
6834        assert!(value.get("warnings").is_none());
6835    }
6836
6837    #[test]
6838    fn a_dropped_wheel_event_falls_back_to_the_ax_scroll_bar_and_says_so() {
6839        let backend = StubBackend {
6840            scroll_event_delivered: false,
6841            ..StubBackend::default()
6842        };
6843        let value = dispatch_with(&backend, scroll_action(None, Some(2))).to_value();
6844        assert_eq!(value["ok"], json!(true));
6845        assert_eq!(value["via"], json!("ax_scrollbar"));
6846        assert_eq!(value["scrolled"], json!(true));
6847        assert_eq!(value["before"], json!(0.0));
6848        assert!(near(value["after"].as_f64().unwrap_or_default(), 0.1));
6849        assert_eq!(
6850            value["warnings"][0],
6851            json!("wheel event not delivered to background app; used AX scroll bar")
6852        );
6853    }
6854
6855    #[test]
6856    fn a_bar_already_at_its_limit_is_ok_and_never_claims_a_scroll() {
6857        let backend = StubBackend {
6858            scroll_event_delivered: false,
6859            scroll_value: std::sync::Mutex::new(1.0),
6860            ..StubBackend::default()
6861        };
6862        let value = dispatch_with(&backend, scroll_action(None, Some(3))).to_value();
6863        assert_eq!(value["ok"], json!(true));
6864        assert_eq!(value["scrolled"], json!(false));
6865        assert_eq!(value["before"], json!(1.0));
6866        assert_eq!(value["after"], json!(1.0));
6867        let warning = value["warnings"][0].as_str().unwrap_or_default();
6868        assert!(warning.contains("bottom"), "{warning}");
6869    }
6870
6871    #[test]
6872    fn a_scroll_that_cannot_be_verified_is_never_reported_as_a_scroll() {
6873        let backend = StubBackend {
6874            scroll_area_missing: true,
6875            ..StubBackend::default()
6876        };
6877        let response = dispatch_with_context(
6878            &backend,
6879            scroll_action(None, Some(3)),
6880            &tier_context("event"),
6881        );
6882        assert!(!response.ok);
6883        assert_eq!(code_of(&response), "execution_failed");
6884        let message = response.to_value()["error"]["message"]
6885            .as_str()
6886            .unwrap_or_default()
6887            .to_string();
6888        assert!(message.contains("cannot be verified"), "{message}");
6889        assert!(response.to_value().get("scrolled").is_none());
6890    }
6891
6892    #[test]
6893    fn an_ax_refusal_is_a_failure_and_not_a_scroll() {
6894        let backend = StubBackend {
6895            scroll_event_delivered: false,
6896            scroll_ax_refused: true,
6897            ..StubBackend::default()
6898        };
6899        let response = dispatch_with_context(
6900            &backend,
6901            scroll_action(None, Some(3)),
6902            &tier_context("event"),
6903        );
6904        assert!(!response.ok);
6905        assert_eq!(code_of(&response), "execution_failed");
6906        assert!(
6907            response.to_value()["error"]["message"]
6908                .as_str()
6909                .unwrap_or_default()
6910                .contains("refused"),
6911            "{:?}",
6912            response.to_value()
6913        );
6914    }
6915
6916    #[test]
6917    fn a_zero_delta_scroll_reports_no_scroll_and_does_not_panic() {
6918        // `--dx 0 --dy 0` addresses no axis at all. There is no bar to report,
6919        // so the answer is `scrolled: false` with null values, not a failure.
6920        for tier in ["event", "ax"] {
6921            let backend = StubBackend::default();
6922            let value = dispatch_with_context(
6923                &backend,
6924                scroll_action(Some(0), Some(0)),
6925                &tier_context(tier),
6926            )
6927            .to_value();
6928            assert_eq!(value["ok"], json!(true), "tier {tier}");
6929            assert_eq!(value["scrolled"], json!(false), "tier {tier}");
6930            assert_eq!(value["before"], json!(null), "tier {tier}");
6931            assert_eq!(value["after"], json!(null), "tier {tier}");
6932            assert!(value.get("warnings").is_none(), "tier {tier}");
6933        }
6934    }
6935
6936    #[test]
6937    fn an_explicit_ax_scroll_runs_the_scroll_bar_path_without_posting_an_event() {
6938        let backend = StubBackend::default();
6939        let response =
6940            dispatch_with_context(&backend, scroll_action(None, Some(3)), &tier_context("ax"));
6941        let value = response.to_value();
6942        assert_eq!(value["ok"], json!(true));
6943        assert_eq!(value["tier"], json!("ax"));
6944        assert_eq!(value["via"], json!("ax_scrollbar"));
6945        assert_eq!(value["scrolled"], json!(true));
6946        // No wheel event was posted, so the "not delivered" note would be false.
6947        assert!(
6948            value.get("warnings").is_none(),
6949            "unexpected warnings: {:?}",
6950            value["warnings"]
6951        );
6952        assert_eq!(
6953            *backend.tiers_run.lock().unwrap(),
6954            vec!["ax".to_string()],
6955            "tier ax must not fall back to activate"
6956        );
6957    }
6958
6959    #[test]
6960    fn require_background_never_reaches_the_activate_scroll_tier() {
6961        // The AX scroll-bar path needs no activation, so it is the only honest
6962        // answer when the wheel event is not delivered to a background app.
6963        let backend = StubBackend {
6964            scroll_event_delivered: false,
6965            scroll_area_missing: true,
6966            ..StubBackend::default()
6967        };
6968        let context = CallContext {
6969            tier: Some("event".into()),
6970            require_background: true,
6971            ..CallContext::default()
6972        };
6973        let response = dispatch_with_context(&backend, scroll_action(None, Some(3)), &context);
6974        assert!(!response.ok);
6975        assert_eq!(code_of(&response), "execution_failed");
6976        let tiers = backend.tiers_run.lock().unwrap().clone();
6977        assert!(
6978            !tiers.iter().any(|tier| tier == "activate"),
6979            "activate must never run: {tiers:?}"
6980        );
6981    }
6982
6983    #[test]
6984    fn a_horizontal_scroll_reports_the_horizontal_bar() {
6985        let backend = StubBackend {
6986            scroll_event_delivered: false,
6987            scroll_value: std::sync::Mutex::new(0.5),
6988            ..StubBackend::default()
6989        };
6990        let value = dispatch_with(&backend, scroll_action(Some(-2), None)).to_value();
6991        assert_eq!(value["via"], json!("ax_scrollbar"));
6992        assert_eq!(value["before"], json!(0.5));
6993        assert!(near(value["after"].as_f64().unwrap_or_default(), 0.4));
6994        assert_eq!(value["dx"], json!(-2));
6995    }
6996}
6997
6998// -----------------------------------------------------------------------------
6999// P10: CDP-backed browser helpers. Always return a structured error when the
7000// context flag is off or when the runtime refuses the URL.
7001//
7002// `allow(items_after_test_module)`: these helpers live after the test module
7003// for convenience; they are only here because P10 grew on top of the existing
7004// module. Adding a new module for them would cost more than it saves.
7005#[allow(clippy::items_after_test_module)]
7006fn browser_action_options(
7007    context: &CallContext,
7008    subject: &policy::Subject,
7009    target_id: Option<String>,
7010) -> crate::browser::ActionOptions {
7011    // Merge per-request `browser_blocked_urls` with whatever the loaded policy
7012    // says. The CLI sets the per-request field to an empty vec by default, so
7013    // the policy file is the source. `--policy` (and `XDG_CONFIG_HOME`)
7014    // configure the policy file location.
7015    let mut policy_blocked_urls = context.browser_blocked_urls.clone();
7016    if policy_blocked_urls.is_empty() {
7017        if let Ok(loaded) = policy::load_policy(context.policy.as_deref().map(Path::new)) {
7018            policy_blocked_urls = loaded.blocked_urls().to_vec();
7019        }
7020    }
7021    crate::browser::ActionOptions {
7022        debug_url: context.cdp_url.clone(),
7023        app: context.app.clone(),
7024        app_pid: subject.pid,
7025        target_id,
7026        url: None,
7027        selector: None,
7028        text: None,
7029        script: None,
7030        command_timeout: Duration::from_secs(30),
7031        policy_blocked_urls,
7032        policy_subject: Some(subject.clone()),
7033    }
7034}
7035
7036fn browser_require_cdp(context: &CallContext) -> Result<(), ComputerError> {
7037    if !context.cdp {
7038        return Err(ComputerError::Unsupported(
7039            "browser commands require --cdp (or `cdp: true` in the JSON request)".into(),
7040        ));
7041    }
7042    Ok(())
7043}
7044
7045fn browser_tabs(context: &CallContext, subject: &policy::Subject) -> Result<Value, ComputerError> {
7046    browser_require_cdp(context)?;
7047    let opts = browser_action_options(context, subject, None);
7048    let tabs = crate::browser::cdp::list_tabs(&opts)?;
7049    serde_json::to_value(&tabs)
7050        .map_err(|error| ComputerError::ExecutionFailed(format!("encode tabs: {error}")))
7051}
7052
7053fn browser_activate(
7054    context: &CallContext,
7055    subject: &policy::Subject,
7056    target_id: Option<String>,
7057) -> Result<Value, ComputerError> {
7058    browser_require_cdp(context)?;
7059    let target = target_id.ok_or_else(|| {
7060        ComputerError::InvalidArguments("browser activate requires --target".into())
7061    })?;
7062    let opts = browser_action_options(context, subject, Some(target));
7063    let active = crate::browser::cdp::activate_tab(&opts)?;
7064    Ok(json!({ "url": active }))
7065}
7066
7067fn browser_navigate(
7068    context: &CallContext,
7069    subject: &policy::Subject,
7070    url: String,
7071    target_id: Option<String>,
7072) -> Result<Value, ComputerError> {
7073    browser_require_cdp(context)?;
7074    let opts = browser_action_options(context, subject, target_id);
7075    let landed = crate::browser::cdp::navigate(&crate::browser::ActionOptions {
7076        url: Some(url.clone()),
7077        ..opts
7078    })?;
7079    Ok(json!({ "url": landed }))
7080}
7081
7082fn browser_click(
7083    context: &CallContext,
7084    subject: &policy::Subject,
7085    selector: String,
7086    target_id: Option<String>,
7087) -> Result<Value, ComputerError> {
7088    browser_require_cdp(context)?;
7089    let opts = browser_action_options(context, subject, target_id);
7090    crate::browser::cdp::click(&crate::browser::ActionOptions {
7091        selector: Some(selector.clone()),
7092        ..opts
7093    })?;
7094    Ok(json!({ "selector": selector }))
7095}
7096
7097fn browser_text(
7098    context: &CallContext,
7099    subject: &policy::Subject,
7100    selector: String,
7101    target_id: Option<String>,
7102) -> Result<Value, ComputerError> {
7103    browser_require_cdp(context)?;
7104    let opts = browser_action_options(context, subject, target_id);
7105    let text = crate::browser::cdp::read_inner_text(&crate::browser::ActionOptions {
7106        selector: Some(selector.clone()),
7107        ..opts
7108    })?;
7109    Ok(json!({ "selector": selector, "text": text }))
7110}
7111
7112fn browser_type(
7113    context: &CallContext,
7114    subject: &policy::Subject,
7115    text: String,
7116    selector: Option<String>,
7117    target_id: Option<String>,
7118) -> Result<Value, ComputerError> {
7119    browser_require_cdp(context)?;
7120    let opts = browser_action_options(context, subject, target_id);
7121    crate::browser::cdp::type_text(&crate::browser::ActionOptions {
7122        selector,
7123        text: Some(text.clone()),
7124        ..opts
7125    })?;
7126    // Audit captures the character count via `clipboard_chars`; the text body
7127    // never leaves this function.
7128    Ok(json!({ "chars": text.chars().count() as u64 }))
7129}
7130
7131fn browser_eval(
7132    context: &CallContext,
7133    subject: &policy::Subject,
7134    script: String,
7135    target_id: Option<String>,
7136) -> Result<Value, ComputerError> {
7137    browser_require_cdp(context)?;
7138    let opts = browser_action_options(context, subject, target_id);
7139    let value = crate::browser::cdp::eval(&crate::browser::ActionOptions {
7140        script: Some(script.clone()),
7141        ..opts
7142    })?;
7143    // The script body never leaves this function. The audit row only sees
7144    // its length and a 16-byte blake3-ish hash so an auditor can spot
7145    // duplicates without seeing the contents.
7146    let length = script.chars().count() as u64;
7147    use std::hash::{Hash, Hasher};
7148    let mut hasher = std::collections::hash_map::DefaultHasher::new();
7149    script.hash(&mut hasher);
7150    let hash = hasher.finish();
7151    Ok(json!({
7152        "result": value,
7153        "script_chars": length,
7154        "script_hash": format!("{hash:016x}"),
7155    }))
7156}