cosh_tools/computer/
keyboard.rs

1//! Shared keyboard engine for the pipeline computer tools.
2//!
3//! `computer_control` and `computer_act` both chain keyboard steps (tap a
4//! key, optionally with modifiers, or type literal text) through the same
5//! recursive `then`/`wait` skeleton, so the parsing, validation helpers and
6//! the dispatch live HERE, parameterized by the calling tool's error prefix.
7//! The keystrokes go into whatever element holds keyboard focus NOW —
8//! aiming them is the caller's job (a real click in `computer_control`, or
9//! a semantic action in `computer_act`).
10use xa11y::Key;
11
12/// Maximum duration of a `wait` step, in milliseconds — the pipeline stays
13/// one tool call, so a wait is a pause for the app to catch up, not a
14/// sleep primitive.
15pub const MAX_WAIT_MS: u64 = 10_000;
16
17/// The keyboard fields of ONE pipeline step, tool-agnostic: both
18/// `ComputerControl` and `ComputerAct` narrow into this before handing the
19/// step to [`run_keyboard_step`].
20#[derive(Debug, Clone, Default)]
21pub struct KeyboardStep<'a> {
22    /// Key to tap (single lowercase character or a named key).
23    pub key: Option<&'a str>,
24    /// Literal text to type.
25    pub text: Option<&'a str>,
26    /// Modifier keys held while tapping `key`.
27    pub held: Option<&'a [String]>,
28}
29
30/// True when the step's keyboard fields make it a KEYBOARD step. Matches
31/// the validators exactly (which trim `key` but take `text` as-is), so
32/// validation and execution can never disagree about the step's kind.
33pub fn is_keyboard_step(key: Option<&str>, text: Option<&str>) -> bool {
34    key.is_some_and(|k| !k.trim().is_empty()) || text.is_some_and(|t| !t.is_empty())
35}
36
37/// True when the step's `wait` field makes it a WAIT step.
38pub fn is_wait_step(wait_ms: Option<u64>) -> bool {
39    wait_ms.is_some()
40}
41
42/// Execute ONE validated keyboard step and return its human-facing report.
43///
44/// `tool` prefixes every error (e.g. `computer_control`) so a shared
45/// failure reads as coming from the tool the model actually called.
46pub fn run_keyboard_step(
47    sim: &xa11y::InputSim,
48    step: &KeyboardStep,
49    tool: &str,
50) -> Result<String, String> {
51    if let Some(text) = step.text.filter(|t| !t.is_empty()) {
52        sim.keyboard()
53            .type_text(text)
54            .map_err(|e| super::errors::render(tool, "type text", &e))?;
55        return Ok(format!("typed {text:?}"));
56    }
57
58    let key = step.key.unwrap_or_default().trim();
59    let parsed = parse_key(key, tool)?;
60    let held = parse_keys(step.held.unwrap_or_default(), tool)?;
61    if held.is_empty() {
62        sim.keyboard()
63            .press(parsed)
64            .map_err(|e| super::errors::render(tool, &format!("press {key}"), &e))?;
65        Ok(format!("pressed {key}"))
66    } else {
67        let names: Vec<String> = held.iter().map(key_name).collect();
68        sim.keyboard()
69            .chord(parsed, &held)
70            .map_err(|e| super::errors::render(tool, &format!("chord {key} + {names:?}"), &e))?;
71        Ok(format!("pressed {key} with {} held", names.join(",")))
72    }
73}
74
75/// Pause for a validated `wait` step and return its report fragment.
76pub fn run_wait_step(wait_ms: u64) -> String {
77    std::thread::sleep(std::time::Duration::from_millis(wait_ms));
78    format!("waited {wait_ms} ms")
79}
80
81/// Parse a key name into a [`Key`].
82///
83/// Named keys are matched case-insensitively; a single CHARACTER is taken
84/// literally — an uppercase letter is REJECTED (the caller must hold
85/// `shift` explicitly), matching the `Key::Char` contract in xa11y.
86pub fn parse_key(name: &str, tool: &str) -> Result<Key, String> {
87    let lower = name.to_ascii_lowercase();
88    let key = match lower.as_str() {
89        "enter" | "return" => Key::Enter,
90        "escape" | "esc" => Key::Escape,
91        "backspace" => Key::Backspace,
92        "tab" => Key::Tab,
93        "space" => Key::Space,
94        "delete" | "del" => Key::Delete,
95        "insert" => Key::Insert,
96        "up" | "arrowup" => Key::ArrowUp,
97        "down" | "arrowdown" => Key::ArrowDown,
98        "left" | "arrowleft" => Key::ArrowLeft,
99        "right" | "arrowright" => Key::ArrowRight,
100        "home" => Key::Home,
101        "end" => Key::End,
102        "pageup" => Key::PageUp,
103        "pagedown" => Key::PageDown,
104        _ if name.chars().count() == 1 => {
105            // Validate the ORIGINAL character, not the lowercased one — a
106            // silently-lowercased uppercase letter would send the wrong key
107            // and report success.
108            let ch = name.chars().next().unwrap_or_default();
109            if ch.is_uppercase() {
110                return Err(format!(
111                    "{tool}: uppercase `{name}` — pass the lowercase key with `held: [\"shift\"]`"
112                ));
113            }
114            Key::Char(ch)
115        }
116        _ if lower.starts_with('f')
117            && lower.len() <= 3
118            && lower[1..]
119                .parse::<u8>()
120                .is_ok_and(|n| (1..=12).contains(&n)) =>
121        {
122            Key::F(lower[1..].parse::<u8>().unwrap_or(1))
123        }
124        _ => {
125            return Err(format!("{tool}: unknown key `{name}`"));
126        }
127    };
128    Ok(key)
129}
130
131/// Parse modifier names into [`Key`]s (shared by click/drag `held` and
132/// keyboard chords).
133pub fn parse_keys(names: &[String], tool: &str) -> Result<Vec<Key>, String> {
134    names
135        .iter()
136        .map(|n| match n.to_ascii_lowercase().as_str() {
137            "shift" => Ok(Key::Shift),
138            "ctrl" | "control" => Ok(Key::Ctrl),
139            "alt" | "option" => Ok(Key::Alt),
140            "meta" | "cmd" | "command" | "super" | "win" => Ok(Key::Meta),
141            other => Err(format!(
142                "{tool}: unknown modifier `{other}` (expected shift, ctrl, alt or meta)"
143            )),
144        })
145        .collect()
146}
147
148/// Human-facing name of a modifier/`Key` (for the `sent` report).
149pub fn key_name(key: &Key) -> String {
150    match key {
151        Key::Shift => "shift".to_string(),
152        Key::Ctrl => "ctrl".to_string(),
153        Key::Alt => "alt".to_string(),
154        Key::Meta => "meta".to_string(),
155        Key::Char(c) => c.to_string(),
156        other => format!("{other:?}"),
157    }
158}
159
160/// Handle for the pipeline tools: both share the process-wide input
161/// simulator, so a down/up or drag never splits across devices.
162pub(crate) fn shared_sim() -> Result<xa11y::InputSim, String> {
163    super::shared_input_sim()
164}
cosh_tools/computer/keyboard.rs

cosh_tools/computer/
keyboard.rs
