1pub mod act;
27pub mod apps;
28pub mod control;
29pub mod errors;
30pub mod keyboard;
31pub mod mouse;
32pub mod screenshot;
33pub mod snapshot;
34pub mod surface;
35pub mod touch;
36pub mod types;
37pub mod wait;
38
39#[cfg(test)]
40mod test;
41
42pub use act::act;
43pub use apps::AppsResult;
44pub use apps::apps;
45pub use control::control;
46pub use screenshot::screenshot;
47pub use snapshot::snapshot;
48pub use types::{
49 ActAction, ActOutput, AppInfo, AppsOutput, AppsTarget, ComputerAct, ComputerApps,
50 ComputerControl, ComputerScreenshot, ComputerSnapshot, ComputerWait, ControlOutput,
51 ElementStates, FocusedElement, FocusedOutput, PointerAction, PointerAnchor, ScreenshotOutput,
52 SnapshotFormat, SnapshotOutput, StateNode, ToggleState, WaitObservation, WaitOutput, WaitState,
53};
54pub use wait::wait;
55
56use crate::ToolDescription;
57
58pub(crate) fn shared_input_sim() -> Result<xa11y::InputSim, String> {
65 static SIM: std::sync::OnceLock<Result<xa11y::InputSim, String>> = std::sync::OnceLock::new();
66 SIM.get_or_init(|| {
67 xa11y::input_sim().map_err(|e| errors::render("computer", "initialize input backend", &e))
68 })
69 .clone()
70}
71
72pub struct Computer {
77 pub description_apps: ToolDescription,
79 pub description_snapshot: ToolDescription,
81 pub description_wait: ToolDescription,
83 pub description_screenshot: ToolDescription,
85 pub description_act: ToolDescription,
88 pub description_control: ToolDescription,
91}
92
93impl Default for Computer {
94 fn default() -> Self {
95 Self::new()
96 }
97}
98
99impl Computer {
100 #[must_use]
102 pub fn new() -> Self {
103 Self {
104 description_apps: serde_json::json!({
105 "name": "computer_apps",
106 "description": concat!(
107 "List running desktop applications visible to the OS accessibility ",
108 "tree, with their PIDs and which one is in the foreground. Call ",
109 "this first to find the `name` or `pid` to pass to computer_snapshot ",
110 "and computer_act.",
111 "\n\nWith target \"focused\" (default \"all\") it instead reports the ",
112 "FOREGROUND application and the element inside it that currently ",
113 "holds keyboard focus (role, name, current value, role path) — the ",
114 "cheap 'where will my keystrokes go?' check before computer_keyboard, ",
115 "which types into the focused element."
116 ),
117 "inputSchema": {
118 "type": "object",
119 "properties": {
120 "target": {
121 "type": "string",
122 "enum": ["all", "focused"],
123 "description": "\"all\" (default) = every running app with PIDs and the foreground flag; \"focused\" = the foreground app plus the element holding keyboard focus inside it."
124 },
125 "timeout_ms": {
126 "type": "integer",
127 "minimum": 0,
128 "description": "How long to wait for a foreground application to exist, in milliseconds (default 3000). Applies only to target \"focused\"."
129 }
130 },
131 "required": []
132 }
133 }),
134 description_snapshot: serde_json::json!({
135 "name": "computer_snapshot",
136 "description": concat!(
137 "Capture the accessibility tree of a desktop application as a ",
138 "compact outline (role, name, value and state flags of every ",
139 "element). Prefer ",
140 "this over a screenshot when you need structure: names and roles ",
141 "are exact and can be used directly as selectors for computer_act. ",
142 "Each line carries the element's current state as plain tokens — ",
143 "disabled, hidden, focused, checked/unchecked/mixed, selected, ",
144 "expanded/collapsed, editable, busy — so a disabled or hidden ",
145 "control is visible BEFORE you try to act on it. Optionally ",
146 "snapshot only the subtree matching a CSS-like selector.",
147 "\n\nSelector patterns: `button`, `button[name='OK']`, ",
148 "`text_field[name^='Search']`, `text_field[name*='email']`, ",
149 "`group > button` (direct child), `window button` (descendant), ",
150 "`button:nth(2)`."
151 ),
152 "inputSchema": {
153 "type": "object",
154 "properties": {
155 "name": {
156 "type": "string",
157 "minLength": 1,
158 "description": "Application name (exact match, e.g. 'Safari'). Provide `name` or `pid` — exactly one."
159 },
160 "pid": {
161 "type": "integer",
162 "minimum": 1,
163 "description": "Application process ID. Provide `name` or `pid` — exactly one."
164 },
165 "selector": {
166 "type": "string",
167 "description": "Optional CSS-like selector to snapshot only the matching subtree, e.g. \"window[name='Main'] > group\". Resolution is fail-fast (no auto-wait); if the element may not exist yet, re-call after the app's interface settles."
168 },
169 "nth": {
170 "type": "integer",
171 "minimum": 1,
172 "description": "1-based match index when `selector` matches multiple elements (default 1). Requires `selector`."
173 },
174 "max_depth": {
175 "type": "integer",
176 "minimum": 1,
177 "maximum": 40,
178 "description": "Maximum snapshot depth (default 12, max 40)"
179 },
180 "format": {
181 "type": "string",
182 "enum": ["tree", "json"],
183 "description": "'tree' = compact indented outline (default); 'json' = structured JSON"
184 },
185 "timeout_ms": {
186 "type": "integer",
187 "minimum": 0,
188 "description": "How long to wait for the app to appear, in milliseconds (default 3000). Applies only to the app lookup."
189 },
190 "surface": {
191 "type": "string",
192 "enum": ["menu_bar", "status_items", "taskbar", "panel", "dock", "desktop", "flyout", "unknown"],
193 "description": "Target an OS SHELL SURFACE instead of an application — exactly one of `name`/`pid`/`surface`: menu_bar (macOS: File, Apple menu), status_items (tray/menu-bar extras), taskbar (Windows), panel (Linux), dock (macOS), desktop, flyout (Notification Center / Quick Settings / shell menus open RIGHT NOW — they exist only while open). Read-only: enumeration never opens, closes or focuses anything."
194 }
195 },
196 "anyOf": [
197 { "required": ["name"] },
198 { "required": ["pid"] },
199 { "required": ["surface"] }
200 ],
201 "allOf": [
202 {
203 "if": { "required": ["name"] },
204 "then": { "not": { "anyOf": [{ "required": ["pid"] }, { "required": ["surface"] }] } }
205 },
206 {
207 "if": { "required": ["pid"] },
208 "then": { "not": { "required": ["surface"] } }
209 },
210 {
211 "if": { "required": ["nth"] },
212 "then": { "required": ["selector"] }
213 }
214 ]
215 }
216 }),
217 description_wait: serde_json::json!({
218 "name": "computer_wait",
219 "description": concat!(
220 "Block until an element of a desktop application reaches a state — ",
221 "or time out with a diagnosis of the LAST OBSERVED state. Replaces ",
222 "poll-loops (computer_snapshot → check → computer_snapshot …, each a ",
223 "full round trip): 'wait for the export to finish, then snapshot' is ",
224 "ONE call before the snapshot instead of N.",
225 "\n\ncomputer_act already auto-waits an element to be visible+enabled ",
226 "before acting — use computer_wait for the OTHER directions: waiting ",
227 "for things to DISAPPEAR (state \"detached\" — a dialog closed, a ",
228 "spinner went away; tolerates the element never having existed), to ",
229 "ENABLE later (\"enabled\"), to CLOSE (\"hidden\"), or to gain/lose ",
230 "keyboard focus (\"focused\"/\"unfocused\").",
231 "\n\nRead-only: no input is sent and nothing moves, so this tool is ",
232 "allowed in every mode. On timeout the error names the condition and ",
233 "the last observed state — decide the next move from that instead of ",
234 "probing again."
235 ),
236 "inputSchema": {
237 "type": "object",
238 "properties": {
239 "name": {
240 "type": "string",
241 "minLength": 1,
242 "description": "Application name (exact match, e.g. 'Safari'). Provide `name` or `pid` — exactly one."
243 },
244 "pid": {
245 "type": "integer",
246 "minimum": 1,
247 "description": "Application process ID. Provide `name` or `pid` — exactly one."
248 },
249 "selector": {
250 "type": "string",
251 "description": "CSS-like selector for the element to watch, e.g. \"progress_bar[name='Exporting…']\"."
252 },
253 "nth": {
254 "type": "integer",
255 "minimum": 1,
256 "description": "1-based match index when `selector` matches multiple elements (default 1)."
257 },
258 "state": {
259 "type": "string",
260 "enum": ["attached", "detached", "visible", "hidden", "enabled", "disabled", "focused", "unfocused"],
261 "description": "The condition to wait for (default \"visible\"): \"attached\" = exists; \"detached\" = gone (or never existed); \"visible\" = exists and is visible; \"hidden\" = hidden or gone; \"enabled\"/\"disabled\"; \"focused\"/\"unfocused\"."
262 },
263 "timeout_ms": {
264 "type": "integer",
265 "minimum": 1,
266 "maximum": 60000,
267 "description": "How long to wait for the condition, in milliseconds (default 10000, max 60000). A timeout is an ERROR carrying the last observed state — not a quiet false."
268 }
269 },
270 "required": ["selector"],
271 "anyOf": [
272 { "required": ["name"] },
273 { "required": ["pid"] }
274 ],
275 "allOf": [
276 {
277 "if": { "required": ["name"] },
278 "then": { "not": { "required": ["pid"] } }
279 }
280 ]
281 }
282 }),
283 description_screenshot: serde_json::json!({
284 "name": "computer_screenshot",
285 "description": concat!(
286 "Capture the screen and return it as an image you can see. Use when visual ",
287 "layout matters; use computer_snapshot when you need exact element ",
288 "structure.",
289 "\n\nThree capture forms (exactly one): full display (omit every ",
290 "argument); `region` = [x, y, width, height] in desktop pixels; ",
291 "element capture = a ROOT together with a CSS-like `selector` ",
292 "(pixels under the element's current bounds; occluded elements ",
293 "are NOT raised first) — the root is `app`, `pid` or `surface` ",
294 "(an OS shell surface: menu_bar, status_items, taskbar, panel, ",
295 "dock, desktop, or a flyout open right now). Oversized captures ",
296 "are downscaled ",
297 "to `max_width` (default 1568); the output reports the DELIVERED ",
298 "image's dimensions, its desktop-space origin, and the desktop scale ",
299 "— map an image coordinate back to the screen as ",
300 "desktop = desktop_origin + image_coord * desktop_scale.",
301 "\n\nWith `annotate: true` the capture becomes SELECTOR-GROUNDED: every ",
302 "element of the app matching `selector` (default \"*\" = all) gets a ",
303 "labeled box drawn on the image, and the result carries a legend ",
304 "mapping each tag (`B7`) back to a round-trippable selector ",
305 "(`*:nth(7)`) you pass straight to computer_act — visual grounding ",
306 "without pixel coordinates. Requires `app` or `pid` (annotation groups ",
307 "must be application-scoped) and captures the full display; omissions ",
308 "and truncation (>100 matches) are reported — narrow the selector when ",
309 "truncated. Selectors must be single clauses: comma alternations ",
310 "(`\"button, link\"`) are rejected — pass one selector per intent."
311 ),
312 "inputSchema": {
313 "type": "object",
314 "properties": {
315 "annotate": {
316 "type": "boolean",
317 "description": "Draw a labeled box on every matching element and return a legend mapping each tag to a selector computer_act accepts. Requires `app` or `pid`; mutually exclusive with `region`. Default false."
318 },
319 "region": {
320 "type": "array",
321 "items": { "type": "integer" },
322 "minItems": 4,
323 "maxItems": 4,
324 "description": "Capture only this region: [x, y, width, height] in desktop pixels. Mutually exclusive with element capture and with `annotate`."
325 },
326 "app": {
327 "type": "string",
328 "description": "Application name (exact match). Element capture: requires `selector` (or pair with `annotate` to box all elements). Annotated capture: the annotation scope."
329 },
330 "pid": {
331 "type": "integer",
332 "minimum": 1,
333 "description": "Application process ID. Element capture: requires `selector` (or pair with `annotate` to box all elements). Annotated capture: the annotation scope."
334 },
335 "selector": {
336 "type": "string",
337 "description": "CSS-like selector, e.g. \"window[name='Main'] > group\". Element capture: which element to shoot. Annotated capture: which elements get boxes (default \"*\" = every element of the app). Single clause only — comma alternations are rejected."
338 },
339 "nth": {
340 "type": "integer",
341 "minimum": 1,
342 "description": "1-based match index when `selector` matches multiple elements (default 1). Element capture only — annotated captures cover every match."
343 },
344 "max_width": {
345 "type": "integer",
346 "minimum": 256,
347 "description": "Maximum delivered image width in pixels (default 1568). Coordinates refer to the delivered image; use desktop_origin/desktop_scale from the output to map back."
348 },
349 "timeout_ms": {
350 "type": "integer",
351 "minimum": 0,
352 "description": "How long to wait for the app to appear (element/annotated capture), in milliseconds (default 3000)."
353 },
354 "surface": {
355 "type": "string",
356 "enum": ["menu_bar", "status_items", "taskbar", "panel", "dock", "desktop", "flyout", "unknown"],
357 "description": "Target an OS SHELL SURFACE instead of an application (element capture: requires `selector`; with `annotate: true` the legend covers the surface's elements the same way it covers an app's). Exactly one of `app`/`pid`/`surface`: menu_bar (macOS: File, Apple menu), status_items (tray/menu-bar extras), taskbar (Windows), panel (Linux), dock (macOS), desktop, flyout (open RIGHT NOW)."
358 }
359 },
360 "allOf": [
361 {
362 "if": { "required": ["region"] },
363 "then": {
364 "not": { "anyOf": [{ "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }, { "required": ["selector"] }] }
365 }
366 },
367 {
368 "if": {
369 "allOf": [
370 { "required": ["annotate"] },
371 { "const": true, "properties": { "annotate": true } }
372 ]
373 },
374 "then": {
375 "allOf": [
376 { "anyOf": [{ "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }] },
377 { "not": { "required": ["region"] } },
378 { "not": { "required": ["nth"] } }
379 ]
380 }
381 },
382 {
383 "if": { "anyOf": [{ "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }] },
384 "then": {
385 "anyOf": [
386 { "required": ["selector"] },
387 {
388 "allOf": [
389 { "required": ["annotate"] },
390 { "const": true, "properties": { "annotate": true } }
391 ]
392 }
393 ]
394 }
395 },
396 {
397 "if": { "required": ["app"] },
398 "then": { "not": { "anyOf": [{ "required": ["pid"] }, { "required": ["surface"] }] } }
399 },
400 {
401 "if": { "required": ["pid"] },
402 "then": { "not": { "required": ["surface"] } }
403 },
404 {
405 "if": { "required": ["selector"] },
406 "then": { "anyOf": [{ "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }] }
407 },
408 {
409 "if": { "required": ["nth"] },
410 "then": { "required": ["selector"] }
411 }
412 ],
413 "required": []
414 }
415 }),
416 description_act: serde_json::json!({
417 "name": "computer_act",
418 "description": concat!(
419 "Act on a desktop element AND type as ONE pipeline. A call is a ",
420 "STEP — either a SEMANTIC action on the element matched by ",
421 "`selector` (press/toggle/select/set_value/..., auto-waiting for ",
422 "the element to become visible and enabled), a keyboard action ",
423 "(`key` or `text`), or a `wait` pause — and a step may chain the ",
424 "NEXT step via `then`: all steps run sequentially in this ONE call ",
425 "with shell `&&` semantics, each only if the previous succeeded, ",
426 "and the first failure aborts reporting the exact failing step ",
427 "plus everything that completed (max 8 steps).",
428 "\n\nThe pipeline skeleton (`then` → `then` → `wait`) is IDENTICAL ",
429 "to computer_control's — the model learns the shape once; only the ",
430 "step kinds differ (semantic actions here, pointer actions there). ",
431 "Typical flow: computer_apps → computer_snapshot (find role+name) → ",
432 "computer_act. Value actions need payloads: `set_value`/",
433 "`type_text`/`perform_action` need `value`, `set_numeric_value` ",
434 "needs `numeric_value`, `select_text` needs `range: [start, end]` ",
435 "(0-based). A `wait` step (standalone, millisecond cap 10 s) lets ",
436 "the app settle between steps. A semantic `press` focuses the ",
437 "target on many toolkits but NOT reliably on every one — the one ",
438 "mechanism that always moves OS keyboard focus is a real click, ",
439 "which is computer_control's job: if chained typing keeps missing ",
440 "the field, click the field element there instead. In Build mode ",
441 "chains containing an element step ask for approval; keyboard ",
442 "steps with no element step in the chain are denied (the approval ",
443 "dialog hands focus to the TUI)."
444 ),
445 "inputSchema": {
446 "type": "object",
447 "$defs": {
448 "step": {
449 "type": "object",
450 "properties": {
451 "name": {
452 "type": "string",
453 "minLength": 1,
454 "description": "Application name (exact match, e.g. 'Safari'). Provide `name` or `pid` — exactly one. Semantic-action steps only."
455 },
456 "pid": {
457 "type": "integer",
458 "minimum": 1,
459 "description": "Application process ID. Provide `name` or `pid` — exactly one. Semantic-action steps only."
460 },
461 "selector": {
462 "type": "string",
463 "minLength": 1,
464 "description": "CSS-like selector for the target element, e.g. \"button[name='OK']\". Required on a semantic-action step; absent on keyboard/wait steps."
465 },
466 "nth": {
467 "type": "integer",
468 "minimum": 1,
469 "description": "1-based match index when `selector` matches multiple elements (default 1)."
470 },
471 "action": {
472 "type": "string",
473 "enum": [
474 "press", "focus", "blur", "toggle", "select", "expand",
475 "collapse", "show_menu", "increment", "decrement",
476 "scroll_into_view", "set_value", "set_numeric_value",
477 "select_text", "type_text", "perform_action"
478 ],
479 "description": "Semantic action to perform (default `press`). Semantic-action steps only."
480 },
481 "value": {
482 "type": "string",
483 "description": "Text for `set_value`, `type_text` or `perform_action` (action name for the latter, e.g. \"raise\")."
484 },
485 "numeric_value": {
486 "type": "number",
487 "description": "Number for `set_numeric_value` (slider, spinner)."
488 },
489 "range": {
490 "type": "array",
491 "items": { "type": "integer", "minimum": 0 },
492 "minItems": 2,
493 "maxItems": 2,
494 "description": "[start, end] (0-based; `end` EXCLUSIVE — [0, 3] selects three characters) for `select_text`."
495 },
496 "timeout_ms": {
497 "type": "integer",
498 "minimum": 0,
499 "description": "How long to wait for the app AND a visible+enabled element match, in milliseconds (default 3000). Semantic-action steps only."
500 },
501 "surface": {
502 "type": "string",
503 "enum": ["menu_bar", "status_items", "taskbar", "panel", "dock", "desktop", "flyout", "unknown"],
504 "description": "Target an OS SHELL SURFACE instead of an application (semantic-action steps only, with `selector`): the menu bar, Dock/taskbar/panel, tray status items, the desktop, or a flyout open right now. Exactly one of `name`/`pid`/`surface` per step."
505 },
506 "key": {
507 "type": "string",
508 "description": "Key to tap: single lowercase character (`a`, `5`, `.`), `enter`, `escape`, `tab`, `space`, `backspace`, `delete`, `insert`, `up`, `down`, `left`, `right`, `home`, `end`, `pageup`, `pagedown`, `f1`..`f12`. Types into whatever holds keyboard focus NOW."
509 },
510 "text": {
511 "type": "string",
512 "description": "Literal text to type into the focused element (any characters, including uppercase). Mutually exclusive with `key`."
513 },
514 "held": {
515 "type": "array",
516 "items": { "type": "string", "enum": ["shift", "ctrl", "alt", "meta"] },
517 "description": "Modifier keys held while tapping `key` (chord). Rejected with `text`."
518 },
519 "wait": {
520 "type": "integer",
521 "minimum": 1,
522 "maximum": 10000,
523 "description": "Milliseconds to WAIT as a STANDALONE step (no other fields): gives the app time to settle before the next step acts."
524 },
525 "then": {
526 "$ref": "#/$defs/step",
527 "description": "Optional NEXT step of the pipeline, executed only if this step succeeded. Same shape as this step, recursively (max 8 steps)."
528 }
529 },
530 "allOf": [
531 {
532 "anyOf": [
533 { "required": ["selector"] },
534 { "required": ["key"] },
535 { "required": ["text"] },
536 { "required": ["wait"] }
537 ]
538 },
539 {
540 "if": { "anyOf": [{ "required": ["key"] }, { "required": ["text"] }] },
541 "then": {
542 "not": { "anyOf": [{ "required": ["name"] }, { "required": ["pid"] }, { "required": ["surface"] }, { "required": ["selector"] }, { "required": ["nth"] }, { "required": ["action"] }, { "required": ["value"] }, { "required": ["numeric_value"] }, { "required": ["range"] }, { "required": ["timeout_ms"] }] }
543 }
544 },
545 {
546 "if": { "required": ["held"] },
547 "then": { "required": ["key"] }
548 },
549 {
550 "if": { "required": ["text"] },
551 "then": { "properties": { "key": { "not": {} }, "held": { "not": {} } } }
552 },
553 {
554 "if": { "required": ["key"] },
555 "then": { "properties": { "text": { "not": {} } } }
556 },
557 {
558 "if": { "required": ["wait"] },
559 "then": { "not": { "anyOf": [{ "required": ["name"] }, { "required": ["pid"] }, { "required": ["surface"] }, { "required": ["selector"] }, { "required": ["nth"] }, { "required": ["action"] }, { "required": ["value"] }, { "required": ["numeric_value"] }, { "required": ["range"] }, { "required": ["timeout_ms"] }, { "required": ["key"] }, { "required": ["text"] }, { "required": ["held"] }] } }
560 },
561 {
562 "if": { "anyOf": [{ "required": ["action"] }, { "required": ["value"] }, { "required": ["numeric_value"] }, { "required": ["range"] }, { "required": ["timeout_ms"] }] },
563 "then": { "required": ["selector"] }
564 },
565 {
566 "if": { "required": ["selector"] },
567 "then": { "anyOf": [{ "required": ["name"] }, { "required": ["pid"] }, { "required": ["surface"] }] }
568 },
569 {
570 "if": { "required": ["name"] },
571 "then": { "not": { "anyOf": [{ "required": ["pid"] }, { "required": ["surface"] }] } }
572 },
573 {
574 "if": { "required": ["pid"] },
575 "then": { "not": { "required": ["surface"] } }
576 },
577 {
578 "if": { "required": ["nth"] },
579 "then": { "required": ["selector"] }
580 },
581 {
582 "if": { "properties": { "action": { "enum": ["set_value", "type_text", "perform_action"] } }, "required": ["action"] },
583 "then": { "required": ["value"] }
584 },
585 {
586 "if": { "properties": { "action": { "const": "set_numeric_value" } }, "required": ["action"] },
587 "then": { "required": ["numeric_value"] }
588 },
589 {
590 "if": { "properties": { "action": { "const": "select_text" } }, "required": ["action"] },
591 "then": { "required": ["range"] }
592 }
593 ]
594 }
595 },
596 "allOf": [ { "$ref": "#/$defs/step" } ]
597 }
598 }),
599 description_control: serde_json::json!({
600 "name": "computer_control",
601 "description": concat!(
602 "Control the desktop: pointer AND keyboard as ONE pipeline. A call ",
603 "is a STEP — either a pointer action (`action`, default `click`, with ",
604 "a target) or a keyboard action (`key` or `text`) — and a step may ",
605 "chain the NEXT step via `then`: all steps run sequentially in this ",
606 "ONE call with shell `&&` semantics, each only if the previous ",
607 "succeeded, and the first failure aborts reporting the exact failing ",
608 "step plus everything that completed (max 8 steps).",
609 "\n\nTargeting is PER STEP (every step may carry `app`/`pid`/`surface` + ",
610 "`selector`). Two pointer target forms: ELEMENT form (preferred) — ",
611 "the point resolves from the element's CURRENT bounds at dispatch ",
612 "time, never stale; COORDINATE form — x/y from a computer_screenshot ",
613 "mapped as desktop = desktop_origin + image_coord * desktop_scale, ",
614 "ONLY when the target has no accessibility node. A real click on an ",
615 "element MOVES OS KEYBOARD FOCUS to it — the one mechanism that ",
616 "works on every toolkit — so the canonical pattern for typing into a ",
617 "field is: click the field element, `then` `text`, `then` `key` ",
618 "enter. Keyboard steps (`key`/`text`) always type into whatever ",
619 "holds focus NOW and reject pointer fields.",
620 "\n\nClick actions accept `count` (2 = double-click) and `held` ",
621 "modifiers (ctrl+click = held [\"ctrl\"] + click); key + held = ",
622 "chord (key `a` + held [\"ctrl\"] = select all; keys are lowercase — ",
623 "for uppercase hold `shift`); `text` types any literal string and ",
624 "rejects `held`. A `wait` step (millisecond cap 10 s, standalone — no ",
625 "other fields) gives the app time to catch up: click the menu item ",
626 "that opens a rename box, then {\"wait\": 600}, then type into the ",
627 "field it created. `action: drag` moves the pressed button from the ",
628 "start to the end point over `duration_ms` (default 150, min 50) — ",
629 "endpoints must share the form: x/y + x2/y2, or selector + ",
630 "to_selector. `action: up` needs no target. If the target appears ",
631 "in computer_snapshot, prefer computer_act — it is semantic and ",
632 "auto-waits. In Build mode the element form asks for approval; the ",
633 "coordinate form is denied (coordinates go stale when the approval ",
634 "dialog moves the pointer)."
635 ),
636 "inputSchema": {
637 "type": "object",
638 "$defs": {
639 "step": {
640 "type": "object",
641 "properties": {
642 "x": {
643 "type": "integer",
644 "description": "X coordinate in desktop pixels (coordinate form; also the drag START)."
645 },
646 "y": {
647 "type": "integer",
648 "description": "Y coordinate in desktop pixels (coordinate form; also the drag START)."
649 },
650 "action": {
651 "type": "string",
652 "enum": ["click", "double_click", "right_click", "move", "down", "up", "scroll", "drag"],
653 "description": "Pointer action (default `click`). `down`/`up` allow custom drags; `drag` is the native interpolated drag. Ignored — and rejected with any other pointer field — on a keyboard step (`key`/`text`)."
654 },
655 "button": {
656 "type": "string",
657 "enum": ["left", "right", "middle"],
658 "description": "Button for `down`/`up` and the button held during `drag` (default `left`)."
659 },
660 "dx": {
661 "type": "integer",
662 "description": "Horizontal wheel steps for `scroll`."
663 },
664 "dy": {
665 "type": "integer",
666 "description": "Vertical wheel steps for `scroll`; positive scrolls down."
667 },
668 "app": {
669 "type": "string",
670 "description": "Application name (exact match) for the element form. Exactly one of `app`/`pid`, only together with `selector`. Per step."
671 },
672 "pid": {
673 "type": "integer",
674 "description": "Application process ID for the element form. Per step."
675 },
676 "surface": {
677 "type": "string",
678 "enum": ["menu_bar", "status_items", "taskbar", "panel", "dock", "desktop", "flyout", "unknown"],
679 "description": "Target an OS SHELL SURFACE instead of an application (element form: requires `selector`): the menu bar, Dock/taskbar/panel, tray status items, the desktop, or a flyout open right now. Exactly one of `app`/`pid`/`surface` per step."
680 },
681 "selector": {
682 "type": "string",
683 "description": "CSS-like selector for the element form, e.g. \"button[name='OK']\". A click on it moves OS keyboard focus to the element — how the next step's typing finds the field. For `drag`, the START element (end is `to_selector`)."
684 },
685 "nth": {
686 "type": "integer",
687 "minimum": 1,
688 "description": "1-based match index when `selector` matches multiple elements (default 1)."
689 },
690 "anchor": {
691 "type": "string",
692 "enum": ["center", "top_left", "top_right", "bottom_left", "bottom_right"],
693 "description": "Where inside the element's bounds the point lands (default `center`)."
694 },
695 "count": {
696 "type": "integer",
697 "minimum": 1,
698 "description": "Consecutive clicks for click actions (default 1; 2 = double-click)."
699 },
700 "held": {
701 "type": "array",
702 "items": { "type": "string", "enum": ["shift", "ctrl", "alt", "meta"] },
703 "description": "Modifier keys for THIS step: held during a click or drag, or held while tapping `key` (chord). Rejected with `text`."
704 },
705 "x2": {
706 "type": "integer",
707 "description": "End X for `drag` (with `y2`); mutually exclusive with `to_selector`."
708 },
709 "y2": {
710 "type": "integer",
711 "description": "End Y for `drag`."
712 },
713 "to_selector": {
714 "type": "string",
715 "description": "End element selector for `drag` (element form; same root as the start — app or surface)."
716 },
717 "duration_ms": {
718 "type": "integer",
719 "minimum": 50,
720 "description": "Total duration of a `drag` movement in milliseconds (default 150)."
721 },
722 "key": {
723 "type": "string",
724 "description": "Key to tap: single lowercase character (`a`, `5`, `.`), `enter`, `escape`, `tab`, `space`, `backspace`, `delete`, `insert`, `up`, `down`, `left`, `right`, `home`, `end`, `pageup`, `pagedown`, `f1`..`f12`. Types into the element an earlier step's click focused."
725 },
726 "text": {
727 "type": "string",
728 "description": "Literal text to type into the focused element (any characters, including uppercase). Mutually exclusive with `key`."
729 },
730 "wait": {
731 "type": "integer",
732 "minimum": 1,
733 "maximum": 10000,
734 "description": "Milliseconds to WAIT as a STANDALONE step (no other fields): gives the app time to open a dialog or create an ephemeral field before the next step acts — e.g. click the menu item, then {\"wait\": 600}, then type into the rename box."
735 },
736 "then": {
737 "$ref": "#/$defs/step",
738 "description": "Optional NEXT step of the pipeline, executed only if this step succeeded. Same shape as this step, recursively (max 8 steps)."
739 }
740 },
741 "allOf": [
742 {
743 "if": { "anyOf": [{ "required": ["key"] }, { "required": ["text"] }] },
744 "then": {
745 "not": { "anyOf": [{ "required": ["action"] }, { "required": ["x"] }, { "required": ["y"] }, { "required": ["button"] }, { "required": ["dx"] }, { "required": ["dy"] }, { "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }, { "required": ["selector"] }, { "required": ["nth"] }, { "required": ["anchor"] }, { "required": ["count"] }, { "required": ["x2"] }, { "required": ["y2"] }, { "required": ["to_selector"] }, { "required": ["duration_ms"] }] }
746 }
747 },
748 {
749 "if": { "required": ["text"] },
750 "then": { "properties": { "key": { "not": {} }, "held": { "not": {} } } }
751 },
752 {
753 "if": { "required": ["key"] },
754 "then": { "properties": { "text": { "not": {} } } }
755 },
756 {
757 "if": { "properties": { "action": { "const": "scroll" } }, "required": ["action"] },
758 "then": { "anyOf": [{ "required": ["dx"] }, { "required": ["dy"] }] }
759 },
760 {
761 "if": { "not": { "properties": { "action": { "const": "up" } }, "required": ["action"] } },
762 "then": { "anyOf": [{ "allOf": [{ "required": ["key"] }, { "not": { "required": ["text"] } }] }, { "required": ["text"] }, { "allOf": [{ "required": ["x"] }, { "required": ["y"] }] }, { "required": ["selector"] }, { "required": ["wait"] }] }
763 },
764 {
765 "if": { "required": ["wait"] },
766 "then": { "not": { "anyOf": [{ "required": ["action"] }, { "required": ["x"] }, { "required": ["y"] }, { "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }, { "required": ["selector"] }, { "required": ["nth"] }, { "required": ["anchor"] }, { "required": ["count"] }, { "required": ["held"] }, { "required": ["x2"] }, { "required": ["y2"] }, { "required": ["to_selector"] }, { "required": ["duration_ms"] }, { "required": ["key"] }, { "required": ["text"] }, { "required": ["dx"] }, { "required": ["dy"] }, { "required": ["button"] }] } }
767 },
768 {
769 "if": { "anyOf": [{ "required": ["x"] }, { "required": ["y"] }] },
770 "then": { "allOf": [
771 { "required": ["y"] },
772 { "required": ["x"] },
773 { "not": { "anyOf": [{ "required": ["selector"] }, { "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }] } }
774 ] }
775 },
776 {
777 "if": { "required": ["selector"] },
778 "then": { "anyOf": [{ "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }] }
779 },
780 {
781 "if": { "anyOf": [{ "required": ["app"] }, { "required": ["pid"] }, { "required": ["surface"] }] },
782 "then": { "required": ["selector"] }
783 },
784 {
785 "if": { "required": ["app"] },
786 "then": { "not": { "anyOf": [{ "required": ["pid"] }, { "required": ["surface"] }] } }
787 },
788 {
789 "if": { "required": ["pid"] },
790 "then": { "not": { "required": ["surface"] } }
791 },
792 {
793 "if": { "anyOf": [{ "required": ["nth"] }, { "required": ["anchor"] }] },
794 "then": { "required": ["selector"] }
795 },
796 {
797 "if": { "properties": { "action": { "const": "drag" } }, "required": ["action"] },
798 "then": {
799 "anyOf": [
800 { "allOf": [{ "required": ["x2", "y2"] }, { "not": { "required": ["to_selector"] } }] },
801 { "allOf": [{ "required": ["to_selector"] }, { "not": { "anyOf": [{ "required": ["x2"] }, { "required": ["y2"] }] } }] }
802 ]
803 }
804 },
805 {
806 "if": { "required": ["duration_ms"] },
807 "then": { "properties": { "action": { "const": "drag" } }, "required": ["action"] }
808 },
809 {
810 "if": {
811 "anyOf": [
812 { "required": ["x2"] },
813 { "required": ["y2"] },
814 { "required": ["to_selector"] }
815 ]
816 },
817 "then": { "properties": { "action": { "const": "drag" } }, "required": ["action"] }
818 },
819 {
820 "if": {
821 "properties": { "action": { "enum": ["move", "down", "up", "scroll"] } },
822 "required": ["action"]
823 },
824 "then": {
825 "not": { "anyOf": [{ "required": ["count"] }, { "required": ["held"] }] }
826 }
827 }
828 ]
829 }
830 },
831 "allOf": [ { "$ref": "#/$defs/step" } ]
832 }
833 }),
834 }
835 }
836
837 pub async fn apps(&self, input: &ComputerApps) -> Result<apps::AppsResult, String> {
847 apps::apps(input).await
848 }
849
850 pub async fn snapshot(&self, input: &ComputerSnapshot) -> Result<SnapshotOutput, String> {
859 snapshot::snapshot(input).await
860 }
861
862 pub async fn wait(&self, input: &ComputerWait) -> Result<WaitOutput, String> {
873 wait::wait(input).await
874 }
875
876 pub async fn screenshot(&self, input: &ComputerScreenshot) -> Result<ScreenshotOutput, String> {
887 screenshot::screenshot(input).await
888 }
889
890 pub async fn act(&self, input: &ComputerAct) -> Result<ActOutput, String> {
902 act::act(input).await
903 }
904
905 pub async fn control(&self, input: &ComputerControl) -> Result<ControlOutput, String> {
916 control::control(input).await
917 }
918}