computer_use/
types.rs

1//! Shared geometry, input, and image types.
2
3use serde::de::{self, Deserializer, IgnoredAny, MapAccess, Visitor};
4use serde::{Deserialize, Serialize};
5use std::fmt;
6
7use crate::error::ComputerError;
8
9const MAX_REGION_EDGE: u32 = 100_000;
10
11/// Rectangle in logical screen pixels. The origin is the top-left of the
12/// virtual desktop, matching click coordinates.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
14pub struct Region {
15    /// Left edge.
16    pub x: i32,
17    /// Top edge.
18    pub y: i32,
19    /// Width in logical pixels. Must be non-zero.
20    pub width: u32,
21    /// Height in logical pixels. Must be non-zero.
22    pub height: u32,
23}
24
25impl Region {
26    /// Build a region after checking size limits.
27    pub fn new(x: i32, y: i32, width: u32, height: u32) -> Result<Self, ComputerError> {
28        validate_region(x, y, width, height)?;
29        Ok(Self {
30            x,
31            y,
32            width,
33            height,
34        })
35    }
36}
37
38/// Parse `x,y,width,height`. Whitespace around commas is ignored.
39pub fn parse_region(text: &str) -> Result<Region, ComputerError> {
40    let parts: Vec<&str> = text.split(',').map(str::trim).collect();
41    if parts.len() != 4 {
42        return Err(ComputerError::InvalidArguments(format!(
43            "region must be x,y,width,height (got {text:?})"
44        )));
45    }
46    let x = parse_i32(parts[0], "x")?;
47    let y = parse_i32(parts[1], "y")?;
48    let width = parse_u32(parts[2], "width")?;
49    let height = parse_u32(parts[3], "height")?;
50    Region::new(x, y, width, height)
51}
52
53fn parse_i32(text: &str, name: &str) -> Result<i32, ComputerError> {
54    text.parse::<i32>().map_err(|_| {
55        ComputerError::InvalidArguments(format!("region {name} is not an integer: {text:?}"))
56    })
57}
58
59fn parse_u32(text: &str, name: &str) -> Result<u32, ComputerError> {
60    text.parse::<u32>().map_err(|_| {
61        ComputerError::InvalidArguments(format!(
62            "region {name} is not a non-negative integer: {text:?}"
63        ))
64    })
65}
66
67fn validate_region(x: i32, y: i32, width: u32, height: u32) -> Result<(), ComputerError> {
68    if width == 0 || height == 0 {
69        return Err(ComputerError::InvalidArguments(
70            "region width and height must be greater than 0".into(),
71        ));
72    }
73    if width > MAX_REGION_EDGE || height > MAX_REGION_EDGE {
74        return Err(ComputerError::InvalidArguments(format!(
75            "region width and height must be at most {MAX_REGION_EDGE}"
76        )));
77    }
78    let _ = (x, y);
79    Ok(())
80}
81
82impl<'de> Deserialize<'de> for Region {
83    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
84        struct RegionVisitor;
85
86        impl<'de> Visitor<'de> for RegionVisitor {
87            type Value = Region;
88
89            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
90                formatter.write_str("a \"x,y,width,height\" string or {x,y,width,height} object")
91            }
92
93            fn visit_str<E: de::Error>(self, value: &str) -> Result<Region, E> {
94                parse_region(value).map_err(E::custom)
95            }
96
97            fn visit_string<E: de::Error>(self, value: String) -> Result<Region, E> {
98                self.visit_str(&value)
99            }
100
101            fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Region, A::Error> {
102                let mut x = None;
103                let mut y = None;
104                let mut width = None;
105                let mut height = None;
106                while let Some(key) = map.next_key::<String>()? {
107                    match key.as_str() {
108                        "x" => x = Some(map.next_value()?),
109                        "y" => y = Some(map.next_value()?),
110                        "width" => width = Some(map.next_value()?),
111                        "height" => height = Some(map.next_value()?),
112                        _ => {
113                            let _ = map.next_value::<IgnoredAny>()?;
114                        }
115                    }
116                }
117                let x = x.ok_or_else(|| de::Error::missing_field("x"))?;
118                let y = y.ok_or_else(|| de::Error::missing_field("y"))?;
119                let width = width.ok_or_else(|| de::Error::missing_field("width"))?;
120                let height = height.ok_or_else(|| de::Error::missing_field("height"))?;
121                Region::new(x, y, width, height).map_err(de::Error::custom)
122            }
123        }
124
125        deserializer.deserialize_any(RegionVisitor)
126    }
127}
128
129/// Mouse button for click actions.
130#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
131#[serde(rename_all = "lowercase")]
132pub enum MouseButton {
133    /// Primary button.
134    #[default]
135    Left,
136    /// Secondary button.
137    Right,
138    /// Middle button or wheel click.
139    Middle,
140    /// Button 4. The event tier posts it as other-mouse button number 3.
141    Back,
142    /// Button 5. The event tier posts it as other-mouse button number 4.
143    Forward,
144}
145
146impl MouseButton {
147    /// JSON and CLI name.
148    pub fn as_str(self) -> &'static str {
149        match self {
150            Self::Left => "left",
151            Self::Right => "right",
152            Self::Middle => "middle",
153            Self::Back => "back",
154            Self::Forward => "forward",
155        }
156    }
157}
158
159/// One logical point. Drag paths are a list of these.
160#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
161pub struct Point {
162    /// Logical x.
163    pub x: i32,
164    /// Logical y.
165    pub y: i32,
166}
167
168/// Parse a flat `x,y,x,y,...` list. At least two points, and an even count.
169pub fn parse_path(text: &str) -> Result<Vec<Point>, ComputerError> {
170    let parts: Vec<&str> = text.split(',').map(str::trim).collect();
171    if parts.len() < 4 || !parts.len().is_multiple_of(2) || parts.iter().any(|part| part.is_empty())
172    {
173        return Err(ComputerError::InvalidArguments(format!(
174            "path must be at least two x,y points (got {text:?})"
175        )));
176    }
177    let mut points = Vec::with_capacity(parts.len() / 2);
178    for pair in parts.chunks(2) {
179        let x = pair[0].parse::<i32>().map_err(|_| {
180            ComputerError::InvalidArguments(format!("path x is not an integer: {:?}", pair[0]))
181        })?;
182        let y = pair[1].parse::<i32>().map_err(|_| {
183            ComputerError::InvalidArguments(format!("path y is not an integer: {:?}", pair[1]))
184        })?;
185        points.push(Point { x, y });
186    }
187    Ok(points)
188}
189
190/// True when the region sits inside `width` by `height`.
191///
192/// The far edge may equal the width or height. A point on that edge is outside.
193pub fn region_inside(region: Region, width: u32, height: u32) -> Result<(), ComputerError> {
194    if region.x < 0 || region.y < 0 {
195        return Err(ComputerError::InvalidArguments(
196            "region is outside the capture".into(),
197        ));
198    }
199    let right = i64::from(region.x) + i64::from(region.width);
200    let bottom = i64::from(region.y) + i64::from(region.height);
201    if right > i64::from(width) || bottom > i64::from(height) {
202        return Err(ComputerError::InvalidArguments(
203            "region is outside the capture".into(),
204        ));
205    }
206    Ok(())
207}
208
209/// Modifier held while another key is pressed.
210///
211/// `Command` and `Meta` both synthesize the platform meta key (Command on
212/// macOS, Super on Linux, Windows key on Windows).
213#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
214#[serde(rename_all = "lowercase")]
215pub enum Modifier {
216    /// Command key. Serialized as `command`.
217    #[serde(rename = "command")]
218    Command,
219    /// Control key.
220    Control,
221    /// Shift key.
222    Shift,
223    /// Alt or Option key.
224    Alt,
225    /// Meta or Super key.
226    Meta,
227}
228
229impl Modifier {
230    /// Name used in JSON responses.
231    pub fn as_str(self) -> &'static str {
232        match self {
233            Self::Command => "command",
234            Self::Control => "control",
235            Self::Shift => "shift",
236            Self::Alt => "alt",
237            Self::Meta => "meta",
238        }
239    }
240}
241
242/// Image file format for screenshots.
243#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
244#[serde(rename_all = "lowercase")]
245pub enum ImageFormat {
246    /// PNG, lossless.
247    #[default]
248    Png,
249    /// JPEG.
250    Jpeg,
251}
252
253impl ImageFormat {
254    /// Canonical format name.
255    pub fn as_str(self) -> &'static str {
256        match self {
257            Self::Png => "png",
258            Self::Jpeg => "jpeg",
259        }
260    }
261
262    /// File extension without a dot.
263    pub fn extension(self) -> &'static str {
264        match self {
265            Self::Png => "png",
266            Self::Jpeg => "jpg",
267        }
268    }
269}
270
271/// Scroll direction used with an amount.
272#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
273#[serde(rename_all = "lowercase")]
274pub enum ScrollDirection {
275    /// Toward the top of the screen.
276    Up,
277    /// Toward the bottom of the screen.
278    Down,
279    /// Toward the left.
280    Left,
281    /// Toward the right.
282    Right,
283}
284
285/// One connected display, in logical pixels.
286#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
287pub struct DisplayInfo {
288    /// Platform display id.
289    pub id: u32,
290    /// Human-readable name. May be empty when the platform has none.
291    pub name: String,
292    /// Left edge in the virtual desktop.
293    pub x: i32,
294    /// Top edge in the virtual desktop.
295    pub y: i32,
296    /// Logical width.
297    pub width: u32,
298    /// Logical height.
299    pub height: u32,
300    /// Device pixels per logical pixel. `1.0` when the platform does not report one.
301    pub scale: f64,
302    /// Whether this is the primary display.
303    pub is_primary: bool,
304}
305
306/// Best-effort frontmost application and window.
307#[derive(Debug, Clone, PartialEq, Eq, Default)]
308pub struct FrontmostInfo {
309    /// Application or process name.
310    pub app: Option<String>,
311    /// Front window title.
312    pub window: Option<String>,
313    /// Why the lookup was incomplete.
314    pub note: Option<String>,
315}
316
317/// Decoded screenshot pixels.
318#[derive(Clone)]
319pub struct Frame {
320    width: u32,
321    height: u32,
322    scale: f64,
323    image: image::RgbaImage,
324}
325
326impl Frame {
327    /// Build a frame from tightly packed RGBA8 pixels.
328    pub fn new(width: u32, height: u32, scale: f64, rgba: Vec<u8>) -> Result<Self, ComputerError> {
329        if width == 0 || height == 0 {
330            return Err(ComputerError::InvalidArguments(
331                "frame width and height must be greater than 0".into(),
332            ));
333        }
334        let image = image::RgbaImage::from_raw(width, height, rgba).ok_or_else(|| {
335            ComputerError::InvalidArguments(
336                "rgba buffer length does not match width*height*4".into(),
337            )
338        })?;
339        Ok(Self::from_image(image, scale))
340    }
341
342    pub(crate) fn from_image(image: image::RgbaImage, scale: f64) -> Self {
343        Self {
344            width: image.width(),
345            height: image.height(),
346            scale: normalize_scale(scale),
347            image,
348        }
349    }
350
351    /// Pixel width of the buffer.
352    pub fn width(&self) -> u32 {
353        self.width
354    }
355
356    /// Pixel height of the buffer.
357    pub fn height(&self) -> u32 {
358        self.height
359    }
360
361    /// Device scale captured with the frame.
362    pub fn scale(&self) -> f64 {
363        self.scale
364    }
365
366    pub(crate) fn image(&self) -> &image::RgbaImage {
367        &self.image
368    }
369}
370
371impl fmt::Debug for Frame {
372    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
373        formatter
374            .debug_struct("Frame")
375            .field("width", &self.width)
376            .field("height", &self.height)
377            .field("scale", &self.scale)
378            .finish()
379    }
380}
381
382/// Pixels from one capture, before max-width and canvas fitting.
383#[derive(Clone)]
384pub struct RawCapture {
385    /// Device-pixel buffer and its backing scale.
386    pub frame: Frame,
387    /// Left edge of the buffer in screen points.
388    pub origin_x: i32,
389    /// Top edge of the buffer in screen points.
390    pub origin_y: i32,
391    /// Width of the captured area in screen points.
392    pub logical_width: u32,
393    /// Height of the captured area in screen points.
394    pub logical_height: u32,
395    /// `sck`, `xcap`, or `stub`.
396    pub capture_source: String,
397    /// True when the image is a desktop crop and covered pixels may be wrong.
398    pub occluded: bool,
399    /// Non-fatal notes. Empty when the capture was clean.
400    pub warnings: Vec<String>,
401}
402
403/// Replace missing or invalid scale factors with 1.
404pub(crate) fn normalize_scale(scale: f64) -> f64 {
405    if scale.is_finite() && scale > 0.0 {
406        scale
407    } else {
408        1.0
409    }
410}
411
412#[cfg(test)]
413mod tests {
414    use super::*;
415
416    #[test]
417    fn parse_region_accepts_whitespace_and_rejects_bad_shapes() {
418        let region = parse_region(" 10, 20, 30, 40 ").unwrap();
419        assert_eq!(
420            region,
421            Region {
422                x: 10,
423                y: 20,
424                width: 30,
425                height: 40
426            }
427        );
428        assert_eq!(parse_region("-5,-8,2,3").unwrap().x, -5);
429
430        assert!(parse_region("1,2,3").is_err());
431        assert!(parse_region("1,2,3,4,5").is_err());
432        assert!(parse_region("a,2,3,4").is_err());
433        assert!(parse_region("1,2,-3,4").is_err());
434        assert!(parse_region("0,0,0,10").is_err());
435        assert!(parse_region("0,0,10,0").is_err());
436        assert!(parse_region("").is_err());
437    }
438
439    #[test]
440    fn region_deserializes_from_string_and_object() {
441        let from_string: Region = serde_json::from_str("\"1,2,3,4\"").unwrap();
442        let from_object: Region =
443            serde_json::from_str(r#"{"x":1,"y":2,"width":3,"height":4,"ignored":true}"#).unwrap();
444        assert_eq!(from_string, from_object);
445        let json = serde_json::to_value(from_object).unwrap();
446        assert_eq!(json["width"], 3);
447        assert!(serde_json::from_str::<Region>(r#"{"x":1,"y":2,"width":0,"height":4}"#).is_err());
448    }
449
450    #[test]
451    fn path_needs_two_points_and_region_may_touch_the_far_edge() {
452        let path = parse_path("10, 20, 200, 40").unwrap();
453        assert_eq!(path, vec![Point { x: 10, y: 20 }, Point { x: 200, y: 40 }]);
454        assert!(parse_path("1,1").is_err());
455        assert!(parse_path("1,2,3").is_err());
456        let inside = Region::new(0, 0, 400, 300).unwrap();
457        assert!(region_inside(inside, 400, 300).is_ok());
458        let past = Region::new(0, 0, 401, 300).unwrap();
459        assert!(region_inside(past, 400, 300).is_err());
460        assert_eq!(MouseButton::Back.as_str(), "back");
461        assert_eq!(MouseButton::Forward.as_str(), "forward");
462    }
463
464    #[test]
465    fn frame_checks_buffer_length_and_normalizes_scale() {
466        let frame = Frame::new(1, 1, 0.0, vec![1, 2, 3, 4]).unwrap();
467        assert_eq!(frame.width(), 1);
468        assert_eq!(frame.scale(), 1.0);
469        assert!(Frame::new(1, 1, 1.0, vec![0, 0, 0]).is_err());
470        assert!(Frame::new(0, 1, 1.0, vec![]).is_err());
471    }
472}
computer_use/types.rs

computer_use/
types.rs
