#[cfg(feature = "computer-local")]
pub mod local;
mod transport;
use std::sync::Arc;
use harn_vm::value::VmDictExt;
use harn_vm::VmValue;
use serde::{Deserialize, Serialize};
use crate::error::HostlibError;
use crate::registry::{BuiltinRegistry, HostlibCapability, RegisteredBuiltin, SyncHandler};
use crate::tools::args::{build_dict, dict_arg};
pub use transport::{handle_request_line, NullBackend, SocketBackend};
const MODULE: &str = "computer";
const SCREENSHOT_BUILTIN: &str = "hostlib_computer_screenshot";
const EXECUTE_BUILTIN: &str = "hostlib_computer_execute";
const UI_TREE_BUILTIN: &str = "hostlib_computer_ui_tree";
const PERMISSIONS_BUILTIN: &str = "hostlib_computer_permissions";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MouseButton {
Left,
Right,
Middle,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Modifier {
#[serde(alias = "control")]
Ctrl,
Shift,
#[serde(alias = "option")]
Alt,
#[serde(
alias = "cmd",
alias = "command",
alias = "meta",
alias = "win",
alias = "windows"
)]
Super,
}
impl Modifier {
pub fn as_key_name(self) -> &'static str {
match self {
Modifier::Ctrl => "ctrl",
Modifier::Shift => "shift",
Modifier::Alt => "alt",
Modifier::Super => "super",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ScrollDirection {
Up,
Down,
Left,
Right,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case")]
pub enum ComputerAction {
MouseMove {
x: i32,
y: i32,
},
Click {
#[serde(default = "default_left_button")]
button: MouseButton,
x: i32,
y: i32,
#[serde(default = "default_click_count")]
count: u32,
#[serde(default)]
modifiers: Vec<Modifier>,
},
MouseDown {
#[serde(default = "default_left_button")]
button: MouseButton,
x: i32,
y: i32,
},
MouseUp {
#[serde(default = "default_left_button")]
button: MouseButton,
x: i32,
y: i32,
},
Drag {
#[serde(default = "default_left_button")]
button: MouseButton,
from_x: i32,
from_y: i32,
to_x: i32,
to_y: i32,
#[serde(default)]
modifiers: Vec<Modifier>,
},
Scroll {
x: i32,
y: i32,
direction: ScrollDirection,
#[serde(default = "default_scroll_amount")]
amount: i32,
#[serde(default)]
modifiers: Vec<Modifier>,
},
Type {
text: String,
},
Key {
keys: String,
},
HoldKey {
keys: String,
duration_ms: u64,
},
Wait {
#[serde(default = "default_wait_ms")]
duration_ms: u64,
},
}
fn default_left_button() -> MouseButton {
MouseButton::Left
}
fn default_click_count() -> u32 {
1
}
fn default_scroll_amount() -> i32 {
3
}
fn default_wait_ms() -> u64 {
500
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ScreenImage {
pub base64: String,
pub media_type: String,
pub width: u32,
pub height: u32,
pub scale_factor: f64,
}
impl ScreenImage {
fn into_vm(self) -> VmValue {
let mut dict = harn_vm::value::DictMap::new();
dict.put_str("base64", &self.base64);
dict.put_str("media_type", &self.media_type);
dict.put_int("width", i64::from(self.width));
dict.put_int("height", i64::from(self.height));
dict.put("scale_factor", VmValue::Float(self.scale_factor));
VmValue::dict(dict)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct UiElement {
pub reference: String,
pub role: String,
pub name: String,
pub x: i32,
pub y: i32,
pub width: i32,
pub height: i32,
}
impl UiElement {
fn into_vm(self) -> VmValue {
let mut dict = harn_vm::value::DictMap::new();
dict.put_str("reference", &self.reference);
dict.put_str("role", &self.role);
dict.put_str("name", &self.name);
dict.put_int("x", i64::from(self.x));
dict.put_int("y", i64::from(self.y));
dict.put_int("width", i64::from(self.width));
dict.put_int("height", i64::from(self.height));
VmValue::dict(dict)
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct UiTree {
pub supported: bool,
pub elements: Vec<UiElement>,
}
impl UiTree {
fn into_vm(self) -> VmValue {
let elements: Vec<VmValue> = self.elements.into_iter().map(UiElement::into_vm).collect();
let mut dict = harn_vm::value::DictMap::new();
dict.put_bool("supported", self.supported);
dict.put("elements", VmValue::List(Arc::new(elements)));
VmValue::dict(dict)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum PermissionState {
Granted,
Denied,
Undetermined,
NotRequired,
Unknown,
}
impl PermissionState {
fn as_str(self) -> &'static str {
match self {
Self::Granted => "granted",
Self::Denied => "denied",
Self::Undetermined => "undetermined",
Self::NotRequired => "not_required",
Self::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PermissionStatus {
pub screen: PermissionState,
pub input: PermissionState,
pub accessibility: PermissionState,
pub os: String,
pub guidance: String,
}
impl PermissionStatus {
fn into_vm(self) -> VmValue {
let mut dict = harn_vm::value::DictMap::new();
dict.put_str("screen", self.screen.as_str());
dict.put_str("input", self.input.as_str());
dict.put_str("accessibility", self.accessibility.as_str());
dict.put_str("os", &self.os);
dict.put_str("guidance", &self.guidance);
dict.put_bool(
"ready",
matches!(
self.screen,
PermissionState::Granted | PermissionState::NotRequired
) && matches!(
self.input,
PermissionState::Granted | PermissionState::NotRequired
),
);
VmValue::dict(dict)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BackendCapabilities {
pub name: String,
pub screenshot: bool,
pub input: bool,
pub ui_tree: bool,
}
pub trait ComputerBackend: Send + Sync {
fn capabilities(&self) -> BackendCapabilities;
fn screenshot(&self) -> Result<ScreenImage, String>;
fn execute(&self, actions: &[ComputerAction]) -> Result<(), String>;
fn ui_tree(&self) -> Result<UiTree, String>;
fn permissions(&self) -> Result<PermissionStatus, String>;
}
pub struct ComputerUseCapability {
backend: Arc<dyn ComputerBackend>,
}
impl ComputerUseCapability {
pub fn new() -> Self {
Self {
backend: select_backend(),
}
}
pub fn with_backend(backend: Arc<dyn ComputerBackend>) -> Self {
Self { backend }
}
}
impl Default for ComputerUseCapability {
fn default() -> Self {
Self::new()
}
}
fn select_backend() -> Arc<dyn ComputerBackend> {
let transport = std::env::var("BURIN_COMPUTER_USE_TRANSPORT").unwrap_or_default();
match transport.as_str() {
"helper" | "remote" => match SocketBackend::from_env(&transport) {
Ok(backend) => Arc::new(backend),
Err(message) => Arc::new(NullBackend::new(message)),
},
"none" => Arc::new(NullBackend::new(
"computer use is disabled (BURIN_COMPUTER_USE_TRANSPORT=none)".to_string(),
)),
"local" => default_local_backend(),
"" => Arc::new(NullBackend::new(
"computer use is not armed (set BURIN_COMPUTER_USE_TRANSPORT=local for the local \
backend, or helper|remote for a socket backend)"
.to_string(),
)),
other => Arc::new(NullBackend::new(format!(
"unknown BURIN_COMPUTER_USE_TRANSPORT '{other}' (expected local|helper|remote|none)"
))),
}
}
#[cfg(feature = "computer-local")]
fn default_local_backend() -> Arc<dyn ComputerBackend> {
Arc::new(local::LocalBackend::new())
}
#[cfg(not(feature = "computer-local"))]
fn default_local_backend() -> Arc<dyn ComputerBackend> {
Arc::new(NullBackend::new(
"local computer-use backend is not compiled in (enable the `computer-local` feature)"
.to_string(),
))
}
impl HostlibCapability for ComputerUseCapability {
fn module_name(&self) -> &'static str {
MODULE
}
fn register_builtins(&self, registry: &mut BuiltinRegistry) {
let backend = self.backend.clone();
let handler: SyncHandler = {
let backend = backend.clone();
Arc::new(move |_args: &[VmValue]| screenshot_builtin(backend.as_ref()))
};
registry.register(RegisteredBuiltin {
name: SCREENSHOT_BUILTIN,
module: MODULE,
method: "screenshot",
handler,
});
let handler: SyncHandler = {
let backend = backend.clone();
Arc::new(move |args: &[VmValue]| execute_builtin(backend.as_ref(), args))
};
registry.register(RegisteredBuiltin {
name: EXECUTE_BUILTIN,
module: MODULE,
method: "execute",
handler,
});
let handler: SyncHandler = {
let backend = backend.clone();
Arc::new(move |_args: &[VmValue]| ui_tree_builtin(backend.as_ref()))
};
registry.register(RegisteredBuiltin {
name: UI_TREE_BUILTIN,
module: MODULE,
method: "ui_tree",
handler,
});
let handler: SyncHandler =
Arc::new(move |_args: &[VmValue]| permissions_builtin(backend.as_ref()));
registry.register(RegisteredBuiltin {
name: PERMISSIONS_BUILTIN,
module: MODULE,
method: "permissions",
handler,
});
}
}
fn backend_error(builtin: &'static str, message: String) -> HostlibError {
HostlibError::Backend { builtin, message }
}
fn screenshot_builtin(backend: &dyn ComputerBackend) -> Result<VmValue, HostlibError> {
let image = backend
.screenshot()
.map_err(|message| backend_error(SCREENSHOT_BUILTIN, message))?;
Ok(image.into_vm())
}
fn execute_builtin(
backend: &dyn ComputerBackend,
args: &[VmValue],
) -> Result<VmValue, HostlibError> {
let dict = dict_arg(EXECUTE_BUILTIN, args)?;
let actions = parse_actions(&dict)?;
let count = actions.len();
backend
.execute(&actions)
.map_err(|message| backend_error(EXECUTE_BUILTIN, message))?;
Ok(build_dict([("executed", VmValue::Int(count as i64))]))
}
fn ui_tree_builtin(backend: &dyn ComputerBackend) -> Result<VmValue, HostlibError> {
let tree = backend
.ui_tree()
.map_err(|message| backend_error(UI_TREE_BUILTIN, message))?;
Ok(tree.into_vm())
}
fn permissions_builtin(backend: &dyn ComputerBackend) -> Result<VmValue, HostlibError> {
let status = backend
.permissions()
.map_err(|message| backend_error(PERMISSIONS_BUILTIN, message))?;
Ok(status.into_vm())
}
fn parse_actions(dict: &harn_vm::value::DictMap) -> Result<Vec<ComputerAction>, HostlibError> {
let json = vm_dict_to_json(dict);
let obj = json
.as_object()
.ok_or_else(|| HostlibError::InvalidParameter {
builtin: EXECUTE_BUILTIN,
param: "params",
message: "expected a dict payload".to_string(),
})?;
let raw_actions: Vec<serde_json::Value> = if let Some(list) = obj.get("actions") {
list.as_array()
.ok_or_else(|| HostlibError::InvalidParameter {
builtin: EXECUTE_BUILTIN,
param: "actions",
message: "expected a list of actions".to_string(),
})?
.clone()
} else if obj.contains_key("action") {
vec![json.clone()]
} else {
return Err(HostlibError::MissingParameter {
builtin: EXECUTE_BUILTIN,
param: "actions",
});
};
raw_actions
.into_iter()
.enumerate()
.map(|(index, value)| {
serde_json::from_value::<ComputerAction>(value).map_err(|err| {
HostlibError::InvalidParameter {
builtin: EXECUTE_BUILTIN,
param: "actions",
message: format!("action at index {index} is invalid: {err}"),
}
})
})
.collect()
}
fn vm_value_to_json(value: &VmValue) -> serde_json::Value {
match value {
VmValue::Nil => serde_json::Value::Null,
VmValue::Bool(b) => serde_json::Value::Bool(*b),
VmValue::Int(n) => serde_json::Value::from(*n),
VmValue::Float(n) => serde_json::Number::from_f64(*n)
.map(serde_json::Value::Number)
.unwrap_or(serde_json::Value::Null),
VmValue::String(s) => serde_json::Value::String(s.to_string()),
VmValue::List(items) => {
serde_json::Value::Array(items.iter().map(vm_value_to_json).collect())
}
VmValue::Dict(dict) => vm_dict_to_json(dict),
_ => serde_json::Value::Null,
}
}
fn vm_dict_to_json(dict: &harn_vm::value::DictMap) -> serde_json::Value {
let mut map = serde_json::Map::new();
for (key, value) in dict.iter() {
map.insert(key.as_str().to_string(), vm_value_to_json(value));
}
serde_json::Value::Object(map)
}
pub fn split_chord(chord: &str) -> Vec<String> {
chord
.split('+')
.map(|part| part.trim().to_ascii_lowercase())
.filter(|part| !part.is_empty())
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn click_deserializes_with_defaults() {
let value = serde_json::json!({"action": "click", "x": 10, "y": 20});
let action: ComputerAction = serde_json::from_value(value).expect("parse");
assert_eq!(
action,
ComputerAction::Click {
button: MouseButton::Left,
x: 10,
y: 20,
count: 1,
modifiers: vec![],
}
);
}
#[test]
fn scroll_and_type_roundtrip() {
let value = serde_json::json!({
"action": "scroll", "x": 5, "y": 6,
"direction": "down", "amount": 4, "modifiers": ["shift"]
});
let action: ComputerAction = serde_json::from_value(value).expect("parse");
assert_eq!(
action,
ComputerAction::Scroll {
x: 5,
y: 6,
direction: ScrollDirection::Down,
amount: 4,
modifiers: vec![Modifier::Shift],
}
);
let typed: ComputerAction =
serde_json::from_value(serde_json::json!({"action": "type", "text": "hi"}))
.expect("parse");
assert_eq!(typed, ComputerAction::Type { text: "hi".into() });
}
#[test]
fn split_chord_normalizes() {
assert_eq!(split_chord("Ctrl+Shift+S"), vec!["ctrl", "shift", "s"]);
assert_eq!(split_chord(" cmd + a "), vec!["cmd", "a"]);
assert!(split_chord("").is_empty());
}
#[test]
fn null_backend_fails_cleanly() {
let backend = NullBackend::new("no backend".to_string());
assert!(backend.screenshot().is_err());
assert!(!backend.capabilities().screenshot);
}
}