computer_protocol/
act.rs

1//! The `computer_act` batch: what an agent sends, and the typed actions `computerd` runs.
2
3use std::time::Duration;
4
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7
8use crate::Observation;
9
10/// Most actions one batch may hold, counting each double click as two clicks.
11pub const MAX_ACTIONS: usize = 24;
12
13/// Wait length when an action gives none, in milliseconds.
14pub const DEFAULT_WAIT_MS: u16 = 350;
15
16/// Longest single wait, and longest settle time, in milliseconds.
17pub const MAX_WAIT_MS: u16 = 5000;
18
19/// Settle time before the closing screenshot when the batch gives none, in milliseconds.
20pub const DEFAULT_SETTLE_MS: u16 = 300;
21
22/// Wheel steps per scroll action when it gives none.
23pub const DEFAULT_SCROLL_AMOUNT: u8 = 3;
24
25/// Most wheel steps one scroll action may take.
26pub const MAX_SCROLL_AMOUNT: u8 = 20;
27
28/// Most characters all `type` actions of one batch may hold together.
29pub const MAX_TYPE_CHARS: usize = 1000;
30
31/// Time `computerd` may spend on top of the waits and typing of a batch before giving up.
32const WORK_ALLOWANCE: Duration = Duration::from_secs(30);
33
34/// Time a `focus` action may take when it has to start an application.
35const LAUNCH_ALLOWANCE: Duration = Duration::from_secs(40);
36
37/// Time applications get to reread the keyboard after a temporary key binding changes.
38pub const KEYMAP_SETTLE: Duration = Duration::from_millis(20);
39
40/// Allowance for the X round trips behind one typed character.
41const ROUND_TRIP_ALLOWANCE: Duration = Duration::from_millis(10);
42
43/// Time one typed character may take: it can need its own binding, which costs two settle pauses.
44const TYPE_CHAR_ALLOWANCE: Duration = KEYMAP_SETTLE
45    .saturating_mul(2)
46    .saturating_add(ROUND_TRIP_ALLOWANCE);
47
48/// Mouse button.
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
50#[serde(rename_all = "lowercase")]
51pub enum Button {
52    Left,
53    Right,
54    Middle,
55}
56
57/// Scroll direction.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
59#[serde(rename_all = "lowercase")]
60pub enum Direction {
61    Up,
62    Down,
63    Left,
64    Right,
65}
66
67/// A position on the screen, in pixels from the top left corner.
68#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
69pub struct Point {
70    pub x: u16,
71    pub y: u16,
72}
73
74/// One validated action. Double clicks are already split into two clicks.
75#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
76#[serde(tag = "kind", rename_all = "lowercase")]
77pub enum Action {
78    Click {
79        at: Point,
80        button: Button,
81    },
82    Move {
83        at: Point,
84    },
85    Down {
86        at: Option<Point>,
87        button: Button,
88    },
89    Up {
90        at: Option<Point>,
91        button: Button,
92    },
93    Type {
94        text: String,
95    },
96    Key {
97        key: String,
98        modifiers: Vec<String>,
99    },
100    Scroll {
101        at: Option<Point>,
102        direction: Direction,
103        amount: u8,
104    },
105    Wait {
106        ms: u16,
107    },
108    Focus {
109        application: String,
110        uri: Option<String>,
111    },
112}
113
114/// Body of `POST /sessions/{id}/act`.
115#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
116pub struct ActRequest {
117    pub actions: Vec<Action>,
118    /// Whether to end with a screenshot.
119    pub observe: bool,
120    /// Pause before the closing screenshot, in milliseconds.
121    pub settle_ms: u16,
122}
123
124impl ActRequest {
125    /// Longest the batch can take: its waits, the settle time, typing, launches, and a fixed allowance for the rest.
126    #[must_use]
127    pub fn time_budget(&self) -> Duration {
128        let mut budget = WORK_ALLOWANCE + Duration::from_millis(u64::from(self.settle_ms));
129        for action in &self.actions {
130            match action {
131                Action::Wait { ms } => budget += Duration::from_millis(u64::from(*ms)),
132                Action::Type { text } => {
133                    budget += TYPE_CHAR_ALLOWANCE
134                        * u32::try_from(text.chars().count()).unwrap_or(u32::MAX);
135                }
136                Action::Focus { .. } => budget += LAUNCH_ALLOWANCE,
137                _ => {}
138            }
139        }
140        budget
141    }
142}
143
144/// Reply to `POST /sessions/{id}/act`.
145#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
146pub struct ActReply {
147    pub actions_run: usize,
148    /// The closing screenshot, when the batch asked for one.
149    pub observation: Option<Observation>,
150    /// Screen number this batch opened for the session, when it was the session's first call.
151    #[serde(default)]
152    pub opened_screen: Option<u8>,
153}
154
155/// Kind of a [`RawAction`].
156#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
157#[serde(rename_all = "lowercase")]
158pub enum Kind {
159    Click,
160    Move,
161    Down,
162    Up,
163    Type,
164    Key,
165    Scroll,
166    Wait,
167    Focus,
168}
169
170impl Kind {
171    fn name(self) -> &'static str {
172        match self {
173            Self::Click => "click",
174            Self::Move => "move",
175            Self::Down => "down",
176            Self::Up => "up",
177            Self::Type => "type",
178            Self::Key => "key",
179            Self::Scroll => "scroll",
180            Self::Wait => "wait",
181            Self::Focus => "focus",
182        }
183    }
184}
185
186/// An action as the agent writes it. Which fields apply depends on `kind`.
187#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
188pub struct RawAction {
189    /// What to do.
190    #[serde(default)]
191    pub kind: Option<Kind>,
192    /// Pixel column on the screen. Needed by click and move, optional for down, up, and scroll.
193    pub x: Option<f64>,
194    /// Pixel row on the screen. Needed by click and move, optional for down, up, and scroll.
195    pub y: Option<f64>,
196    /// Mouse button for click, down, and up. Default left.
197    pub button: Option<Button>,
198    /// For click: click twice.
199    pub double: Option<bool>,
200    /// For type: the text to type. Any Unicode works. A newline presses Enter.
201    pub text: Option<String>,
202    /// For key: a key name such as enter, esc, tab, backspace, delete, space, left, right, up, down, home, end, pageup, pagedown, f1 to f12, or a single character.
203    pub key: Option<String>,
204    /// For key: modifiers held while the key is pressed, from ctrl, alt, shift, and super (also cmd, meta, win, option).
205    pub modifiers: Option<Vec<String>>,
206    /// For scroll: the direction.
207    pub direction: Option<Direction>,
208    /// For scroll: wheel steps, 1 to 20. Default 3.
209    pub amount: Option<f64>,
210    /// For wait: milliseconds, up to 5000. Default 350.
211    pub ms: Option<f64>,
212    /// For focus: name or title of an open window to raise, matched without regard to case. Also names an application to start when no window matches.
213    pub application: Option<String>,
214    /// For focus: a page or file to open in the application. The application is started or reused with it.
215    pub uri: Option<String>,
216}
217
218/// Why a batch was refused. The text tells the agent what to fix.
219#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
220#[error("{0}")]
221pub struct ActionError(String);
222
223impl ActionError {
224    /// A refusal with the given message.
225    #[must_use]
226    pub fn new(message: impl Into<String>) -> Self {
227        Self(message.into())
228    }
229}
230
231fn to_u16(value: f64) -> Option<u16> {
232    let rounded = value.round();
233    if (0.0..=f64::from(u16::MAX)).contains(&rounded) {
234        #[expect(
235            clippy::cast_possible_truncation,
236            clippy::cast_sign_loss,
237            reason = "the range check above keeps the value inside u16"
238        )]
239        Some(rounded as u16)
240    } else {
241        None
242    }
243}
244
245fn coordinate(number: usize, kind: Kind, axis: &str, value: f64) -> Result<u16, ActionError> {
246    to_u16(value).ok_or_else(|| {
247        ActionError::new(format!(
248            "action {number} ({}): {axis} must be a screen pixel from 0 to {}, got {value}",
249            kind.name(),
250            u16::MAX
251        ))
252    })
253}
254
255fn point(number: usize, kind: Kind, raw: &RawAction) -> Result<Option<Point>, ActionError> {
256    match (raw.x, raw.y) {
257        (Some(x), Some(y)) => Ok(Some(Point {
258            x: coordinate(number, kind, "x", x)?,
259            y: coordinate(number, kind, "y", y)?,
260        })),
261        (None, None) => Ok(None),
262        _ => Err(ActionError::new(format!(
263            "action {number} ({}): give both x and y, or neither",
264            kind.name()
265        ))),
266    }
267}
268
269fn required_point(number: usize, kind: Kind, raw: &RawAction) -> Result<Point, ActionError> {
270    point(number, kind, raw)?.ok_or_else(|| {
271        ActionError::new(format!(
272            "action {number} ({}) needs x and y, the pixel position on the screen",
273            kind.name()
274        ))
275    })
276}
277
278/// Rounds `value` into `min..=max`. `None` when it is not a number.
279fn clamped(value: f64, min: u16, max: u16) -> Option<u16> {
280    if value.is_nan() {
281        return None;
282    }
283    to_u16(value.clamp(f64::from(min), f64::from(max)))
284}
285
286fn parse_one(number: usize, raw: &RawAction, out: &mut Vec<Action>) -> Result<(), ActionError> {
287    let Some(kind) = raw.kind else {
288        return Err(ActionError::new(format!(
289            "action {number} has no kind, use one of click, move, down, up, type, key, scroll, wait, focus"
290        )));
291    };
292    let button = raw.button.unwrap_or(Button::Left);
293    match kind {
294        Kind::Click => {
295            let at = required_point(number, kind, raw)?;
296            let click = Action::Click { at, button };
297            out.push(click.clone());
298            if raw.double == Some(true) {
299                out.push(click);
300            }
301        }
302        Kind::Move => out.push(Action::Move {
303            at: required_point(number, kind, raw)?,
304        }),
305        Kind::Down => out.push(Action::Down {
306            at: point(number, kind, raw)?,
307            button,
308        }),
309        Kind::Up => out.push(Action::Up {
310            at: point(number, kind, raw)?,
311            button,
312        }),
313        Kind::Type => {
314            let text = raw.text.clone().ok_or_else(|| {
315                ActionError::new(format!("action {number} (type) needs text to type"))
316            })?;
317            out.push(Action::Type { text });
318        }
319        Kind::Key => {
320            let key = raw
321                .key
322                .as_deref()
323                .map(str::trim)
324                .filter(|key| !key.is_empty())
325                .ok_or_else(|| {
326                    ActionError::new(format!(
327                        "action {number} (key) needs a key such as enter, esc, tab, or a letter"
328                    ))
329                })?;
330            out.push(Action::Key {
331                key: key.to_owned(),
332                modifiers: raw.modifiers.clone().unwrap_or_default(),
333            });
334        }
335        Kind::Scroll => {
336            let direction = raw.direction.ok_or_else(|| {
337                ActionError::new(format!(
338                    "action {number} (scroll) needs a direction: up, down, left, or right"
339                ))
340            })?;
341            let amount = match raw.amount {
342                None => DEFAULT_SCROLL_AMOUNT,
343                Some(value) => clamped(value, 1, u16::from(MAX_SCROLL_AMOUNT))
344                    .and_then(|amount| u8::try_from(amount).ok())
345                    .ok_or_else(|| {
346                        ActionError::new(format!(
347                            "action {number} (scroll): amount is not a number"
348                        ))
349                    })?,
350            };
351            out.push(Action::Scroll {
352                at: point(number, kind, raw)?,
353                direction,
354                amount,
355            });
356        }
357        Kind::Wait => {
358            let ms = match raw.ms {
359                None => DEFAULT_WAIT_MS,
360                Some(value) => clamped(value, 0, MAX_WAIT_MS).ok_or_else(|| {
361                    ActionError::new(format!("action {number} (wait): ms is not a number"))
362                })?,
363            };
364            out.push(Action::Wait { ms });
365        }
366        Kind::Focus => {
367            let application = raw
368                .application
369                .as_deref()
370                .map(str::trim)
371                .filter(|name| !name.is_empty())
372                .ok_or_else(|| {
373                    ActionError::new(format!(
374                        "action {number} (focus) needs the application name or window title to raise"
375                    ))
376                })?;
377            out.push(Action::Focus {
378                application: application.to_owned(),
379                uri: raw.uri.clone(),
380            });
381        }
382    }
383    Ok(())
384}
385
386impl ActRequest {
387    /// Turns the agent's batch into typed actions and applies every limit.
388    ///
389    /// `observe` defaults to true and `settle_ms` to [`DEFAULT_SETTLE_MS`]. Waits and
390    /// the settle time are capped at [`MAX_WAIT_MS`].
391    ///
392    /// # Errors
393    ///
394    /// Fails when the batch is empty or too long, an action lacks a field it needs,
395    /// or a number is not usable. The message names the action.
396    pub fn parse(
397        raw: &[RawAction],
398        observe: Option<bool>,
399        settle_ms: Option<f64>,
400    ) -> Result<Self, ActionError> {
401        if raw.is_empty() {
402            return Err(ActionError::new("actions must hold at least one action"));
403        }
404        if raw.len() > MAX_ACTIONS {
405            return Err(ActionError::new(format!(
406                "{} actions given, the limit is {MAX_ACTIONS}. Split the batch",
407                raw.len()
408            )));
409        }
410        let mut actions = Vec::with_capacity(raw.len());
411        for (index, action) in raw.iter().enumerate() {
412            parse_one(index + 1, action, &mut actions)?;
413        }
414        if actions.len() > MAX_ACTIONS {
415            return Err(ActionError::new(format!(
416                "the batch runs {} actions because each double click counts as two, the limit is {MAX_ACTIONS}. Split the batch",
417                actions.len()
418            )));
419        }
420        let typed: usize = actions
421            .iter()
422            .map(|action| match action {
423                Action::Type { text } => text.chars().count(),
424                _ => 0,
425            })
426            .sum();
427        if typed > MAX_TYPE_CHARS {
428            return Err(ActionError::new(format!(
429                "the batch types {typed} characters, the limit is {MAX_TYPE_CHARS}. Split the text across batches"
430            )));
431        }
432        let settle_ms = match settle_ms {
433            None => DEFAULT_SETTLE_MS,
434            Some(value) => clamped(value, 0, MAX_WAIT_MS)
435                .ok_or_else(|| ActionError::new("settle_ms is not a number"))?,
436        };
437        Ok(Self {
438            actions,
439            observe: observe.unwrap_or(true),
440            settle_ms,
441        })
442    }
443}
444
445#[cfg(test)]
446mod tests {
447    use super::*;
448
449    fn raw(json: serde_json::Value) -> Vec<RawAction> {
450        serde_json::from_value(json).unwrap()
451    }
452
453    fn parse(json: serde_json::Value) -> Result<ActRequest, ActionError> {
454        ActRequest::parse(&raw(json), None, None)
455    }
456
457    fn repeated(action: serde_json::Value, times: usize) -> serde_json::Value {
458        serde_json::Value::Array(vec![action; times])
459    }
460
461    #[test]
462    fn double_click_becomes_two_clicks_and_counts_toward_the_limit() {
463        let request = parse(serde_json::json!([
464            { "kind": "click", "x": 10.4, "y": 20.6, "button": "right", "double": true }
465        ]))
466        .unwrap();
467        let click = Action::Click {
468            at: Point { x: 10, y: 21 },
469            button: Button::Right,
470        };
471        assert_eq!(request.actions, vec![click.clone(), click]);
472
473        let double = serde_json::json!({ "kind": "click", "x": 1, "y": 1, "double": true });
474        assert_eq!(
475            parse(repeated(double.clone(), 12)).unwrap().actions.len(),
476            MAX_ACTIONS
477        );
478        let error = parse(repeated(double, 13)).unwrap_err();
479        assert!(error.to_string().contains("runs 26 actions"), "{error}");
480
481        let wait = serde_json::json!({ "kind": "wait" });
482        assert_eq!(parse(repeated(wait.clone(), 24)).unwrap().actions.len(), 24);
483        assert!(parse(repeated(wait, 25)).is_err());
484    }
485
486    #[test]
487    fn waits_scrolls_and_settle_are_clamped_and_defaulted() {
488        let request = ActRequest::parse(
489            &raw(serde_json::json!([
490                { "kind": "wait" },
491                { "kind": "wait", "ms": 99999 },
492                { "kind": "wait", "ms": -5 },
493                { "kind": "scroll", "direction": "down" },
494                { "kind": "scroll", "direction": "left", "amount": 500 },
495                { "kind": "scroll", "direction": "up", "amount": 0, "x": 5, "y": 6 },
496            ])),
497            Some(false),
498            Some(60000.0),
499        )
500        .unwrap();
501        let scroll = |direction, amount, at| Action::Scroll {
502            at,
503            direction,
504            amount,
505        };
506        assert_eq!(
507            request.actions,
508            vec![
509                Action::Wait { ms: 350 },
510                Action::Wait { ms: 5000 },
511                Action::Wait { ms: 0 },
512                scroll(Direction::Down, 3, None),
513                scroll(Direction::Left, 20, None),
514                scroll(Direction::Up, 1, Some(Point { x: 5, y: 6 })),
515            ]
516        );
517        assert!(!request.observe);
518        assert_eq!(request.settle_ms, 5000);
519
520        let defaults =
521            ActRequest::parse(&raw(serde_json::json!([{ "kind": "wait" }])), None, None).unwrap();
522        assert!(defaults.observe);
523        assert_eq!(defaults.settle_ms, DEFAULT_SETTLE_MS);
524    }
525
526    #[test]
527    fn missing_fields_and_bad_numbers_name_the_action() {
528        for (action, expected) in [
529            (serde_json::json!({}), "action 2 has no kind"),
530            (
531                serde_json::json!({ "kind": "click" }),
532                "action 2 (click) needs x and y",
533            ),
534            (
535                serde_json::json!({ "kind": "click", "x": 1 }),
536                "action 2 (click): give both x and y",
537            ),
538            (
539                serde_json::json!({ "kind": "down", "x": 1 }),
540                "action 2 (down): give both x and y",
541            ),
542            (
543                serde_json::json!({ "kind": "move", "x": -1, "y": 2 }),
544                "action 2 (move): x must be",
545            ),
546            (
547                serde_json::json!({ "kind": "type" }),
548                "action 2 (type) needs text",
549            ),
550            (
551                serde_json::json!({ "kind": "key", "key": " " }),
552                "action 2 (key) needs a key",
553            ),
554            (
555                serde_json::json!({ "kind": "scroll" }),
556                "action 2 (scroll) needs a direction",
557            ),
558            (
559                serde_json::json!({ "kind": "focus", "application": " " }),
560                "action 2 (focus) needs",
561            ),
562        ] {
563            let error = parse(serde_json::json!([{ "kind": "wait" }, action])).unwrap_err();
564            assert!(error.to_string().starts_with(expected), "{error}");
565        }
566        assert!(parse(serde_json::json!([])).is_err());
567    }
568
569    #[test]
570    fn unknown_kinds_do_not_deserialize() {
571        let error = serde_json::from_value::<RawAction>(serde_json::json!({ "kind": "tap" }))
572            .unwrap_err()
573            .to_string();
574        assert!(error.contains("expected one of"), "{error}");
575    }
576
577    #[test]
578    fn typed_text_is_capped_across_the_batch() {
579        let half = "é".repeat(MAX_TYPE_CHARS / 2 + 1);
580        let error = parse(serde_json::json!([
581            { "kind": "type", "text": half },
582            { "kind": "type", "text": half },
583        ]))
584        .unwrap_err();
585        assert!(error.to_string().contains("types"), "{error}");
586    }
587
588    #[test]
589    fn time_budget_covers_waits_settle_and_typing() {
590        let request = ActRequest::parse(
591            &raw(serde_json::json!([
592                { "kind": "wait", "ms": 1000 },
593                { "kind": "type", "text": "abcde" },
594            ])),
595            None,
596            Some(500.0),
597        )
598        .unwrap();
599        assert_eq!(
600            request.time_budget(),
601            Duration::from_millis(30_000 + 500 + 1000 + 5 * 50)
602        );
603    }
604
605    #[test]
606    fn actions_survive_the_wire_round_trip() {
607        let request = parse(serde_json::json!([
608            { "kind": "down", "button": "middle" },
609            { "kind": "key", "key": "c", "modifiers": ["ctrl"] },
610            { "kind": "focus", "application": "xterm" },
611        ]))
612        .unwrap();
613        let json = serde_json::to_string(&request).unwrap();
614        assert_eq!(serde_json::from_str::<ActRequest>(&json).unwrap(), request);
615    }
616
617    #[test]
618    fn typing_budget_covers_a_binding_pause_for_every_character() {
619        let chars = MAX_TYPE_CHARS;
620        let request = ActRequest {
621            actions: vec![Action::Type {
622                text: "あ".repeat(chars),
623            }],
624            observe: false,
625            settle_ms: 0,
626        };
627        let per_char = KEYMAP_SETTLE * 2 + Duration::from_millis(10);
628        assert!(request.time_budget() >= WORK_ALLOWANCE + per_char * u32::try_from(chars).unwrap());
629    }
630}
computer_protocol/act.rs

computer_protocol/
act.rs
