usehid_core/
agent.rs

1//! Agent-friendly HID interface
2//!
3//! JSON-based interface for LLM agents to control HID devices.
4
5use crate::error::{Error, Result};
6use crate::{Device, Keyboard, Key, Modifiers, Mouse, MouseButton, Gamepad, GamepadButton};
7use serde::{Deserialize, Serialize};
8
9/// Action types for agent interface
10#[derive(Debug, Clone, Serialize, Deserialize)]
11#[serde(tag = "action", rename_all = "snake_case")]
12pub enum AgentAction {
13    // Mouse actions
14    MouseMove { x: i32, y: i32 },
15    MouseClick { button: Option<String> },
16    MouseDoubleClick { button: Option<String> },
17    MouseDown { button: Option<String> },
18    MouseUp { button: Option<String> },
19    MouseScroll { delta: i8 },
20    
21    // Keyboard actions
22    Type { text: String },
23    KeyPress { key: String },
24    KeyDown { key: String },
25    KeyUp { key: String },
26    KeyCombo { modifiers: Vec<String>, key: String },
27    
28    // Gamepad actions
29    GamepadPress { button: String },
30    GamepadRelease { button: String },
31    GamepadLeftStick { x: u8, y: u8 },
32    GamepadRightStick { x: u8, y: u8 },
33    GamepadTriggers { left: u8, right: u8 },
34}
35
36/// Result of an agent action
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct AgentResult {
39    pub success: bool,
40    pub error: Option<String>,
41}
42
43impl AgentResult {
44    fn ok() -> Self {
45        Self { success: true, error: None }
46    }
47    
48    fn err(msg: impl Into<String>) -> Self {
49        Self { success: false, error: Some(msg.into()) }
50    }
51}
52
53/// Agent HID controller
54pub struct AgentHID {
55    mouse: Mouse,
56    keyboard: Keyboard,
57    gamepad: Gamepad,
58    mouse_created: bool,
59    keyboard_created: bool,
60    gamepad_created: bool,
61}
62
63impl AgentHID {
64    /// Create a new agent HID controller
65    pub fn new() -> Self {
66        Self {
67            mouse: Mouse::new(),
68            keyboard: Keyboard::new(),
69            gamepad: Gamepad::new(),
70            mouse_created: false,
71            keyboard_created: false,
72            gamepad_created: false,
73        }
74    }
75    
76    /// Execute an action from JSON
77    pub fn execute_json(&mut self, json: &str) -> AgentResult {
78        match serde_json::from_str::<AgentAction>(json) {
79            Ok(action) => self.execute(action),
80            Err(e) => AgentResult::err(format!("Invalid JSON: {}", e)),
81        }
82    }
83    
84    /// Execute an action
85    pub fn execute(&mut self, action: AgentAction) -> AgentResult {
86        match action {
87            // Mouse
88            AgentAction::MouseMove { x, y } => self.mouse_move(x, y),
89            AgentAction::MouseClick { button } => self.mouse_click(button),
90            AgentAction::MouseDoubleClick { button } => self.mouse_double_click(button),
91            AgentAction::MouseDown { button } => self.mouse_down(button),
92            AgentAction::MouseUp { button } => self.mouse_up(button),
93            AgentAction::MouseScroll { delta } => self.mouse_scroll(delta),
94            
95            // Keyboard
96            AgentAction::Type { text } => self.type_text(&text),
97            AgentAction::KeyPress { key } => self.key_press(&key),
98            AgentAction::KeyDown { key } => self.key_down(&key),
99            AgentAction::KeyUp { key } => self.key_up(&key),
100            AgentAction::KeyCombo { modifiers, key } => self.key_combo(&modifiers, &key),
101            
102            // Gamepad
103            AgentAction::GamepadPress { button } => self.gamepad_press(&button),
104            AgentAction::GamepadRelease { button } => self.gamepad_release(&button),
105            AgentAction::GamepadLeftStick { x, y } => self.gamepad_left_stick(x, y),
106            AgentAction::GamepadRightStick { x, y } => self.gamepad_right_stick(x, y),
107            AgentAction::GamepadTriggers { left, right } => self.gamepad_triggers(left, right),
108        }
109    }
110    
111    fn ensure_mouse(&mut self) -> Result<()> {
112        if !self.mouse_created {
113            self.mouse.create()?;
114            self.mouse_created = true;
115        }
116        Ok(())
117    }
118    
119    fn ensure_keyboard(&mut self) -> Result<()> {
120        if !self.keyboard_created {
121            self.keyboard.create()?;
122            self.keyboard_created = true;
123        }
124        Ok(())
125    }
126    
127    fn ensure_gamepad(&mut self) -> Result<()> {
128        if !self.gamepad_created {
129            self.gamepad.create()?;
130            self.gamepad_created = true;
131        }
132        Ok(())
133    }
134    
135    fn parse_mouse_button(s: Option<String>) -> MouseButton {
136        match s.as_deref() {
137            Some("right") => MouseButton::RIGHT,
138            Some("middle") => MouseButton::MIDDLE,
139            _ => MouseButton::LEFT,
140        }
141    }
142    
143    fn parse_gamepad_button(s: &str) -> Result<GamepadButton> {
144        let btn = match s.to_lowercase().as_str() {
145            "a" => GamepadButton::A,
146            "b" => GamepadButton::B,
147            "x" => GamepadButton::X,
148            "y" => GamepadButton::Y,
149            "lb" | "left_bumper" => GamepadButton::LB,
150            "rb" | "right_bumper" => GamepadButton::RB,
151            "back" | "select" => GamepadButton::BACK,
152            "start" => GamepadButton::START,
153            "guide" | "home" => GamepadButton::GUIDE,
154            "left_stick" | "ls" => GamepadButton::LEFT_STICK,
155            "right_stick" | "rs" => GamepadButton::RIGHT_STICK,
156            "dpad_up" | "up" => GamepadButton::DPAD_UP,
157            "dpad_down" | "down" => GamepadButton::DPAD_DOWN,
158            "dpad_left" | "left" => GamepadButton::DPAD_LEFT,
159            "dpad_right" | "right" => GamepadButton::DPAD_RIGHT,
160            _ => return Err(Error::InvalidAction(format!("Unknown gamepad button: {}", s))),
161        };
162        Ok(btn)
163    }
164    
165    fn parse_modifiers(mods: &[String]) -> Modifiers {
166        let mut result = Modifiers::empty();
167        for m in mods {
168            match m.to_lowercase().as_str() {
169                "ctrl" | "control" => result |= Modifiers::CTRL,
170                "shift" => result |= Modifiers::SHIFT,
171                "alt" | "option" => result |= Modifiers::ALT,
172                "cmd" | "command" | "meta" | "gui" | "win" => result |= Modifiers::CMD,
173                _ => {}
174            }
175        }
176        result
177    }
178    
179    // Mouse actions
180    fn mouse_move(&mut self, x: i32, y: i32) -> AgentResult {
181        if let Err(e) = self.ensure_mouse() {
182            return AgentResult::err(e.to_string());
183        }
184        match self.mouse.move_by(x, y) {
185            Ok(_) => AgentResult::ok(),
186            Err(e) => AgentResult::err(e.to_string()),
187        }
188    }
189    
190    fn mouse_click(&mut self, button: Option<String>) -> AgentResult {
191        if let Err(e) = self.ensure_mouse() {
192            return AgentResult::err(e.to_string());
193        }
194        let btn = Self::parse_mouse_button(button);
195        match self.mouse.click(btn) {
196            Ok(_) => AgentResult::ok(),
197            Err(e) => AgentResult::err(e.to_string()),
198        }
199    }
200    
201    fn mouse_double_click(&mut self, button: Option<String>) -> AgentResult {
202        if let Err(e) = self.ensure_mouse() {
203            return AgentResult::err(e.to_string());
204        }
205        let btn = Self::parse_mouse_button(button);
206        match self.mouse.double_click(btn) {
207            Ok(_) => AgentResult::ok(),
208            Err(e) => AgentResult::err(e.to_string()),
209        }
210    }
211    
212    fn mouse_down(&mut self, button: Option<String>) -> AgentResult {
213        if let Err(e) = self.ensure_mouse() {
214            return AgentResult::err(e.to_string());
215        }
216        let btn = Self::parse_mouse_button(button);
217        match self.mouse.press(btn) {
218            Ok(_) => AgentResult::ok(),
219            Err(e) => AgentResult::err(e.to_string()),
220        }
221    }
222    
223    fn mouse_up(&mut self, button: Option<String>) -> AgentResult {
224        if let Err(e) = self.ensure_mouse() {
225            return AgentResult::err(e.to_string());
226        }
227        let btn = Self::parse_mouse_button(button);
228        match self.mouse.release(btn) {
229            Ok(_) => AgentResult::ok(),
230            Err(e) => AgentResult::err(e.to_string()),
231        }
232    }
233    
234    fn mouse_scroll(&mut self, delta: i8) -> AgentResult {
235        if let Err(e) = self.ensure_mouse() {
236            return AgentResult::err(e.to_string());
237        }
238        match self.mouse.scroll(delta) {
239            Ok(_) => AgentResult::ok(),
240            Err(e) => AgentResult::err(e.to_string()),
241        }
242    }
243    
244    // Keyboard actions
245    fn type_text(&mut self, text: &str) -> AgentResult {
246        if let Err(e) = self.ensure_keyboard() {
247            return AgentResult::err(e.to_string());
248        }
249        match self.keyboard.type_text(text) {
250            Ok(_) => AgentResult::ok(),
251            Err(e) => AgentResult::err(e.to_string()),
252        }
253    }
254    
255    fn key_press(&mut self, key: &str) -> AgentResult {
256        if let Err(e) = self.ensure_keyboard() {
257            return AgentResult::err(e.to_string());
258        }
259        match Key::from_str(key) {
260            Ok(k) => match self.keyboard.tap(k) {
261                Ok(_) => AgentResult::ok(),
262                Err(e) => AgentResult::err(e.to_string()),
263            },
264            Err(e) => AgentResult::err(e.to_string()),
265        }
266    }
267    
268    fn key_down(&mut self, key: &str) -> AgentResult {
269        if let Err(e) = self.ensure_keyboard() {
270            return AgentResult::err(e.to_string());
271        }
272        match Key::from_str(key) {
273            Ok(k) => match self.keyboard.press_key(k) {
274                Ok(_) => AgentResult::ok(),
275                Err(e) => AgentResult::err(e.to_string()),
276            },
277            Err(e) => AgentResult::err(e.to_string()),
278        }
279    }
280    
281    fn key_up(&mut self, key: &str) -> AgentResult {
282        if let Err(e) = self.ensure_keyboard() {
283            return AgentResult::err(e.to_string());
284        }
285        match Key::from_str(key) {
286            Ok(k) => match self.keyboard.release_key(k) {
287                Ok(_) => AgentResult::ok(),
288                Err(e) => AgentResult::err(e.to_string()),
289            },
290            Err(e) => AgentResult::err(e.to_string()),
291        }
292    }
293    
294    fn key_combo(&mut self, modifiers: &[String], key: &str) -> AgentResult {
295        if let Err(e) = self.ensure_keyboard() {
296            return AgentResult::err(e.to_string());
297        }
298        let mods = Self::parse_modifiers(modifiers);
299        match Key::from_str(key) {
300            Ok(k) => match self.keyboard.press_combo(mods, k) {
301                Ok(_) => AgentResult::ok(),
302                Err(e) => AgentResult::err(e.to_string()),
303            },
304            Err(e) => AgentResult::err(e.to_string()),
305        }
306    }
307    
308    // Gamepad actions
309    fn gamepad_press(&mut self, button: &str) -> AgentResult {
310        if let Err(e) = self.ensure_gamepad() {
311            return AgentResult::err(e.to_string());
312        }
313        match Self::parse_gamepad_button(button) {
314            Ok(btn) => match self.gamepad.press(btn) {
315                Ok(_) => AgentResult::ok(),
316                Err(e) => AgentResult::err(e.to_string()),
317            },
318            Err(e) => AgentResult::err(e.to_string()),
319        }
320    }
321    
322    fn gamepad_release(&mut self, button: &str) -> AgentResult {
323        if let Err(e) = self.ensure_gamepad() {
324            return AgentResult::err(e.to_string());
325        }
326        match Self::parse_gamepad_button(button) {
327            Ok(btn) => match self.gamepad.release(btn) {
328                Ok(_) => AgentResult::ok(),
329                Err(e) => AgentResult::err(e.to_string()),
330            },
331            Err(e) => AgentResult::err(e.to_string()),
332        }
333    }
334    
335    fn gamepad_left_stick(&mut self, x: u8, y: u8) -> AgentResult {
336        if let Err(e) = self.ensure_gamepad() {
337            return AgentResult::err(e.to_string());
338        }
339        match self.gamepad.set_left_stick(x, y) {
340            Ok(_) => AgentResult::ok(),
341            Err(e) => AgentResult::err(e.to_string()),
342        }
343    }
344    
345    fn gamepad_right_stick(&mut self, x: u8, y: u8) -> AgentResult {
346        if let Err(e) = self.ensure_gamepad() {
347            return AgentResult::err(e.to_string());
348        }
349        match self.gamepad.set_right_stick(x, y) {
350            Ok(_) => AgentResult::ok(),
351            Err(e) => AgentResult::err(e.to_string()),
352        }
353    }
354    
355    fn gamepad_triggers(&mut self, left: u8, right: u8) -> AgentResult {
356        if let Err(e) = self.ensure_gamepad() {
357            return AgentResult::err(e.to_string());
358        }
359        if let Err(e) = self.gamepad.set_left_trigger(left) {
360            return AgentResult::err(e.to_string());
361        }
362        match self.gamepad.set_right_trigger(right) {
363            Ok(_) => AgentResult::ok(),
364            Err(e) => AgentResult::err(e.to_string()),
365        }
366    }
367}
368
369impl Default for AgentHID {
370    fn default() -> Self {
371        Self::new()
372    }
373}
374
375impl Drop for AgentHID {
376    fn drop(&mut self) {
377        if self.mouse_created {
378            let _ = self.mouse.destroy();
379        }
380        if self.keyboard_created {
381            let _ = self.keyboard.destroy();
382        }
383        if self.gamepad_created {
384            let _ = self.gamepad.destroy();
385        }
386    }
387}
usehid_core/agent.rs

usehid_core/
agent.rs
