computer_use/adapters/
mod.rs

1//! Schema translators for `exec` and `serve`.
2//!
3//! Adapters turn a foreign action object into a native [`Action`]. They set
4//! `coord` to `image` and do not call the backend. Dispatch remaps image
5//! pixels with the stored screenshot map.
6
7mod anthropic;
8mod openai;
9
10use serde_json::{Map, Value};
11
12use crate::error::ComputerError;
13use crate::protocol::Action;
14
15pub use anthropic::translate as translate_anthropic;
16pub use openai::translate as translate_openai;
17
18pub(crate) fn as_object(value: &Value) -> Result<&Map<String, Value>, ComputerError> {
19    value.as_object().ok_or_else(|| {
20        ComputerError::InvalidArguments("adapter request must be a JSON object".into())
21    })
22}
23
24pub(crate) fn require_i32(object: &Map<String, Value>, key: &str) -> Result<i32, ComputerError> {
25    match object.get(key) {
26        Some(value) => number_i32(value, key),
27        None => Err(ComputerError::InvalidArguments(format!(
28            "{key} is required"
29        ))),
30    }
31}
32
33pub(crate) fn optional_i32(
34    object: &Map<String, Value>,
35    key: &str,
36) -> Result<Option<i32>, ComputerError> {
37    match object.get(key) {
38        None | Some(Value::Null) => Ok(None),
39        Some(value) => number_i32(value, key).map(Some),
40    }
41}
42
43pub(crate) fn require_string(
44    object: &Map<String, Value>,
45    key: &str,
46) -> Result<String, ComputerError> {
47    match object.get(key) {
48        Some(Value::String(text)) if !text.is_empty() => Ok(text.clone()),
49        Some(Value::String(_)) => Err(ComputerError::InvalidArguments(format!(
50            "{key} must not be empty"
51        ))),
52        Some(_) => Err(ComputerError::InvalidArguments(format!(
53            "{key} must be a string"
54        ))),
55        None => Err(ComputerError::InvalidArguments(format!(
56            "{key} is required"
57        ))),
58    }
59}
60
61pub(crate) fn coordinate_pair(value: &Value, key: &str) -> Result<(i32, i32), ComputerError> {
62    let Some(pair) = value.as_array() else {
63        return Err(ComputerError::InvalidArguments(format!(
64            "{key} must be [x, y]"
65        )));
66    };
67    if pair.len() != 2 {
68        return Err(ComputerError::InvalidArguments(format!(
69            "{key} must be [x, y]"
70        )));
71    }
72    Ok((number_i32(&pair[0], key)?, number_i32(&pair[1], key)?))
73}
74
75pub(crate) fn number_i32(value: &Value, key: &str) -> Result<i32, ComputerError> {
76    let Some(number) = value.as_f64() else {
77        return Err(ComputerError::InvalidArguments(format!(
78            "{key} must be a number"
79        )));
80    };
81    if !number.is_finite() || number < f64::from(i32::MIN) || number > f64::from(i32::MAX) {
82        return Err(ComputerError::InvalidArguments(format!(
83            "{key} is out of range"
84        )));
85    }
86    Ok(number.round() as i32)
87}
88
89/// Translate `value` with `adapter`. `native` is not handled here.
90pub fn translate(adapter: &str, value: &Value) -> Result<Action, ComputerError> {
91    match adapter {
92        "openai" => translate_openai(value),
93        "anthropic" => translate_anthropic(value),
94        other => Err(ComputerError::InvalidArguments(format!(
95            "unknown adapter {other:?}"
96        ))),
97    }
98}
99
100#[cfg(test)]
101mod tests {
102    use super::*;
103    use crate::error::ComputerError;
104    use serde_json::Value;
105    use std::fs;
106    use std::path::Path;
107
108    fn assert_fixtures(dir: &str, translate: fn(&Value) -> Result<Action, ComputerError>) {
109        let root = Path::new(env!("CARGO_MANIFEST_DIR"))
110            .join("tests/fixtures")
111            .join(dir);
112        let mut paths: Vec<_> = fs::read_dir(&root)
113            .unwrap_or_else(|error| panic!("read {}: {error}", root.display()))
114            .map(|entry| entry.unwrap().path())
115            .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("json"))
116            .collect();
117        paths.sort();
118        assert!(!paths.is_empty(), "no fixtures in {}", root.display());
119        for path in paths {
120            let fixture: Value = serde_json::from_str(
121                &fs::read_to_string(&path).unwrap_or_else(|error| panic!("{error}")),
122            )
123            .unwrap_or_else(|error| panic!("{}: {error}", path.display()));
124            let translated = translate(&fixture["request"]);
125            if let Some(code) = fixture.get("error_code").and_then(Value::as_str) {
126                let error = match translated {
127                    Err(error) => error,
128                    Ok(action) => panic!("{} translated to {action:?}", path.display()),
129                };
130                assert_eq!(error.code(), code, "{}", path.display());
131                if let Some(needle) = fixture.get("error_contains").and_then(Value::as_str) {
132                    assert!(
133                        error.to_string().contains(needle),
134                        "{}: {error}",
135                        path.display()
136                    );
137                }
138            } else {
139                let expected: Action = serde_json::from_value(fixture["expect"].clone())
140                    .unwrap_or_else(|error| panic!("{}: {error}", path.display()));
141                let action = match translated {
142                    Ok(action) => action,
143                    Err(error) => panic!("{}: {error}", path.display()),
144                };
145                assert_eq!(action, expected, "{}", path.display());
146            }
147        }
148    }
149
150    #[test]
151    fn openai_fixtures_match_the_expected_action() {
152        assert_fixtures("openai", translate_openai);
153    }
154
155    #[test]
156    fn anthropic_fixtures_match_the_expected_action() {
157        assert_fixtures("anthropic", translate_anthropic);
158    }
159
160    #[test]
161    fn openai_wheel_is_middle_and_back_and_forward_are_buttons() {
162        let wheel = translate_openai(&serde_json::json!({
163            "action": {"type": "click", "x": 1, "y": 2, "button": "wheel"}
164        }))
165        .unwrap();
166        assert_eq!(
167            wheel,
168            Action::Click {
169                x: Some(1),
170                y: Some(2),
171                button: crate::types::MouseButton::Middle,
172                double: false,
173                count: 1,
174            }
175        );
176        let back = translate_openai(&serde_json::json!({
177            "type": "click", "x": 1, "y": 2, "button": "back"
178        }))
179        .unwrap();
180        assert_eq!(
181            back,
182            Action::Click {
183                x: Some(1),
184                y: Some(2),
185                button: crate::types::MouseButton::Back,
186                double: false,
187                count: 1,
188            }
189        );
190        let forward = translate_openai(&serde_json::json!({
191            "type": "click", "x": 1, "y": 2, "button": "forward"
192        }))
193        .unwrap();
194        assert_eq!(
195            forward,
196            Action::Click {
197                x: Some(1),
198                y: Some(2),
199                button: crate::types::MouseButton::Forward,
200                double: false,
201                count: 1,
202            }
203        );
204    }
205
206    #[test]
207    fn anthropic_return_key_and_unknown_action() {
208        let key = translate_anthropic(&serde_json::json!({
209            "action": "key",
210            "text": "Return"
211        }))
212        .unwrap();
213        assert_eq!(
214            key,
215            Action::Key {
216                keys: "Return".into()
217            }
218        );
219        let unknown = translate_anthropic(&serde_json::json!({"action": "nope"})).unwrap_err();
220        assert_eq!(unknown.code(), "invalid_arguments");
221        assert!(unknown.to_string().contains("nope"));
222    }
223}
computer_use/adapters/mod.rs

computer_use/adapters/
mod.rs
