agent_first_http/host/ops_panel/
input_relay.rs

1//! Pointer/keyboard event replay via Input.dispatch*. Preserves
2//! operator-supplied `performance.now()` timestamps as inter-event delays
3//! so trajectory/dwell-time entropy from real input is kept end-to-end
4//! (`architecture.md §9`).
5
6use std::time::{Duration, Instant};
7
8use axum::extract::ws::{Message, WebSocket};
9use futures::{SinkExt, StreamExt};
10use serde::{Deserialize, Serialize};
11use serde_json::Value;
12
13use crate::host::ops_panel::screencast::resolve_page_target;
14use crate::sdk::cdp::ws_client::Connection;
15
16#[derive(Debug, Clone, Serialize, Deserialize)]
17#[serde(tag = "type", rename_all = "snake_case")]
18pub enum OpsInputEvent {
19    PointerMove {
20        x: f32,
21        y: f32,
22        timestamp_ms: f64,
23    },
24    PointerDown {
25        x: f32,
26        y: f32,
27        button: String,
28        timestamp_ms: f64,
29    },
30    PointerUp {
31        x: f32,
32        y: f32,
33        button: String,
34        timestamp_ms: f64,
35    },
36    Wheel {
37        x: f32,
38        y: f32,
39        dx: f32,
40        dy: f32,
41        timestamp_ms: f64,
42    },
43    KeyDown {
44        key: String,
45        code: String,
46        modifiers: u32,
47        timestamp_ms: f64,
48    },
49    KeyUp {
50        key: String,
51        code: String,
52        modifiers: u32,
53        timestamp_ms: f64,
54    },
55    /// A whole string pasted by the operator (clipboard → Input.insertText).
56    /// Inserted at the focused element's caret in one shot — the operator's
57    /// clipboard never reaches the target browser, so we relay the text
58    /// itself rather than a Ctrl/⌘+V keystroke (which would paste the
59    /// target's own, unrelated clipboard).
60    InsertText {
61        text: String,
62        timestamp_ms: f64,
63    },
64}
65
66impl OpsInputEvent {
67    fn timestamp_ms(&self) -> f64 {
68        match self {
69            Self::PointerMove { timestamp_ms, .. }
70            | Self::PointerDown { timestamp_ms, .. }
71            | Self::PointerUp { timestamp_ms, .. }
72            | Self::Wheel { timestamp_ms, .. }
73            | Self::KeyDown { timestamp_ms, .. }
74            | Self::KeyUp { timestamp_ms, .. }
75            | Self::InsertText { timestamp_ms, .. } => *timestamp_ms,
76        }
77    }
78
79    fn to_cdp(&self) -> (&'static str, Value) {
80        match self {
81            Self::PointerMove { x, y, .. } => (
82                "Input.dispatchMouseEvent",
83                serde_json::json!({
84                    "type": "mouseMoved",
85                    "x": x,
86                    "y": y,
87                    "button": "none",
88                }),
89            ),
90            Self::PointerDown { x, y, button, .. } => (
91                "Input.dispatchMouseEvent",
92                serde_json::json!({
93                    "type": "mousePressed",
94                    "x": x,
95                    "y": y,
96                    "button": button,
97                    "clickCount": 1,
98                }),
99            ),
100            Self::PointerUp { x, y, button, .. } => (
101                "Input.dispatchMouseEvent",
102                serde_json::json!({
103                    "type": "mouseReleased",
104                    "x": x,
105                    "y": y,
106                    "button": button,
107                    "clickCount": 1,
108                }),
109            ),
110            Self::Wheel { x, y, dx, dy, .. } => (
111                "Input.dispatchMouseEvent",
112                serde_json::json!({
113                    "type": "mouseWheel",
114                    "x": x,
115                    "y": y,
116                    "deltaX": dx,
117                    "deltaY": dy,
118                }),
119            ),
120            Self::KeyDown {
121                key,
122                code,
123                modifiers,
124                ..
125            } => {
126                let mut params = serde_json::json!({
127                    "type": "keyDown",
128                    "key": key,
129                    "code": code,
130                    "modifiers": modifiers,
131                    "text": one_char_text(key, *modifiers),
132                });
133                if let Some(vk) = virtual_key_code(key) {
134                    params["windowsVirtualKeyCode"] = vk.into();
135                    params["nativeVirtualKeyCode"] = vk.into();
136                }
137                ("Input.dispatchKeyEvent", params)
138            }
139            Self::KeyUp {
140                key,
141                code,
142                modifiers,
143                ..
144            } => {
145                let mut params = serde_json::json!({
146                    "type": "keyUp",
147                    "key": key,
148                    "code": code,
149                    "modifiers": modifiers,
150                });
151                if let Some(vk) = virtual_key_code(key) {
152                    params["windowsVirtualKeyCode"] = vk.into();
153                    params["nativeVirtualKeyCode"] = vk.into();
154                }
155                ("Input.dispatchKeyEvent", params)
156            }
157            Self::InsertText { text, .. } => (
158                "Input.insertText",
159                serde_json::json!({
160                    "text": text,
161                }),
162            ),
163        }
164    }
165}
166
167/// If `key` is a single printable character, return it as the `text` field
168/// for Input.dispatchKeyEvent so the page sees a real keypress. Otherwise
169/// (Enter, Backspace, ArrowLeft, …) leave it out — CDP will synthesize the
170/// right behavior from key/code.
171///
172/// When Ctrl or ⌘ is held the keystroke is a shortcut (select-all, reload, …),
173/// not text, so we suppress `text` regardless of the key — otherwise chromium
174/// types the bare letter instead of running the shortcut. Shift/Alt don't
175/// count: Shift+a is still "A", and AltGr layouts produce real characters.
176fn one_char_text(key: &str, modifiers: u32) -> Value {
177    const CTRL: u32 = 2;
178    const META: u32 = 4;
179    if modifiers & (CTRL | META) != 0 {
180        return Value::Null;
181    }
182    let mut chars = key.chars();
183    let first = chars.next();
184    let second = chars.next();
185    match (first, second) {
186        (Some(c), None) if !c.is_control() => Value::String(c.to_string()),
187        _ => Value::Null,
188    }
189}
190
191/// Windows virtual-key code for a DOM `key` value, or `None` when there isn't a
192/// meaningful one. CDP needs this for any key whose effect is an *action*
193/// rather than inserted text: Backspace/Delete/Enter/Tab/arrows won't edit, and
194/// Ctrl/⌘+letter shortcuts won't fire, unless `windowsVirtualKeyCode` is set —
195/// the `text` field alone only covers character insertion. Letters and digits
196/// map to their ASCII-uppercase code (the VK code equals the uppercase ASCII
197/// value); named editing/navigation keys use the fixed table below.
198fn virtual_key_code(key: &str) -> Option<i64> {
199    let named = match key {
200        "Backspace" => 8,
201        "Tab" => 9,
202        "Enter" => 13,
203        "Escape" => 27,
204        " " | "Spacebar" => 32,
205        "PageUp" => 33,
206        "PageDown" => 34,
207        "End" => 35,
208        "Home" => 36,
209        "ArrowLeft" => 37,
210        "ArrowUp" => 38,
211        "ArrowRight" => 39,
212        "ArrowDown" => 40,
213        "Insert" => 45,
214        "Delete" => 46,
215        _ => 0,
216    };
217    if named != 0 {
218        return Some(named);
219    }
220    // Single ASCII letter/digit: VK code is the uppercase ASCII value.
221    let mut chars = key.chars();
222    match (chars.next(), chars.next()) {
223        (Some(c), None) if c.is_ascii_alphanumeric() => Some(c.to_ascii_uppercase() as i64),
224        _ => None,
225    }
226}
227
228/// Run the input-replay loop: accept JSON events, schedule them to
229/// preserve inter-event timing, and dispatch via CDP.
230pub async fn run(client_ws: WebSocket, browser_ws_url: &str) {
231    let conn = match Connection::connect(browser_ws_url, None).await {
232        Ok(c) => c,
233        Err(_) => {
234            let _ = close_with_error(client_ws, "browser connect failed").await;
235            return;
236        }
237    };
238    let Some(target_id) = resolve_page_target(&conn).await else {
239        let _ = close_with_error(client_ws, "no page target available").await;
240        return;
241    };
242    let attach = match conn
243        .send(
244            "Target.attachToTarget",
245            &serde_json::json!({"targetId": target_id, "flatten": true}),
246            None,
247        )
248        .await
249    {
250        Ok(v) => v,
251        Err(_) => {
252            let _ = close_with_error(client_ws, "attach failed").await;
253            return;
254        }
255    };
256    let Some(session_id) = attach["sessionId"].as_str().map(str::to_string) else {
257        let _ = close_with_error(client_ws, "no session id").await;
258        return;
259    };
260
261    replay_loop(client_ws, conn, session_id).await;
262}
263
264async fn replay_loop(client_ws: WebSocket, conn: Connection, session_id: String) {
265    let (_tx, mut rx) = client_ws.split();
266    let mut first_event_ts: Option<f64> = None;
267    let mut relay_start = Instant::now();
268
269    while let Some(Ok(msg)) = rx.next().await {
270        let payload = match msg {
271            Message::Text(t) => t.to_string(),
272            Message::Close(_) => break,
273            _ => continue,
274        };
275        let event: OpsInputEvent = match serde_json::from_str(&payload) {
276            Ok(e) => e,
277            Err(_) => continue,
278        };
279
280        if first_event_ts.is_none() {
281            first_event_ts = Some(event.timestamp_ms());
282            relay_start = Instant::now();
283        }
284        let target_offset_ms = event.timestamp_ms() - first_event_ts.unwrap_or(0.0);
285        let elapsed_ms = relay_start.elapsed().as_millis() as f64;
286        if target_offset_ms > elapsed_ms {
287            let sleep_ms = (target_offset_ms - elapsed_ms) as u64;
288            tokio::time::sleep(Duration::from_millis(sleep_ms)).await;
289        }
290
291        let (method, params) = event.to_cdp();
292        let _ = conn.send(method, &params, Some(&session_id)).await;
293    }
294
295    let _ = conn
296        .send(
297            "Target.detachFromTarget",
298            &serde_json::json!({"sessionId": session_id}),
299            None,
300        )
301        .await;
302    conn.close();
303}
304
305async fn close_with_error(mut client_ws: WebSocket, reason: &str) -> Result<(), axum::Error> {
306    let body = serde_json::json!({
307        "code": "ops_error",
308        "channel": "input",
309        "error": reason,
310    });
311    let _ = client_ws.send(Message::Text(body.to_string().into())).await;
312    client_ws.close().await
313}
314
315#[cfg(test)]
316mod tests {
317    use super::*;
318
319    #[test]
320    fn pointer_move_round_trips() {
321        let ev = OpsInputEvent::PointerMove {
322            x: 12.5,
323            y: 34.0,
324            timestamp_ms: 1_234.5,
325        };
326        let json = serde_json::to_string(&ev).unwrap_or_default();
327        let back: OpsInputEvent = serde_json::from_str(&json).unwrap();
328        match back {
329            OpsInputEvent::PointerMove { x, y, .. } => {
330                assert!((x - 12.5).abs() < 1e-3);
331                assert!((y - 34.0).abs() < 1e-3);
332            }
333            _ => panic!("wrong variant"),
334        }
335    }
336
337    #[test]
338    fn pointer_down_converts_to_mousepressed() {
339        let ev = OpsInputEvent::PointerDown {
340            x: 10.0,
341            y: 20.0,
342            button: "left".into(),
343            timestamp_ms: 0.0,
344        };
345        let (method, params) = ev.to_cdp();
346        assert_eq!(method, "Input.dispatchMouseEvent");
347        assert_eq!(params["type"], "mousePressed");
348        assert_eq!(params["x"], 10.0);
349        assert_eq!(params["button"], "left");
350    }
351
352    #[test]
353    fn keydown_with_printable_key_gets_text_field() {
354        let ev = OpsInputEvent::KeyDown {
355            key: "a".into(),
356            code: "KeyA".into(),
357            modifiers: 0,
358            timestamp_ms: 0.0,
359        };
360        let (_, params) = ev.to_cdp();
361        assert_eq!(params["text"], "a");
362    }
363
364    #[test]
365    fn keydown_with_special_key_has_no_text() {
366        let ev = OpsInputEvent::KeyDown {
367            key: "Enter".into(),
368            code: "Enter".into(),
369            modifiers: 0,
370            timestamp_ms: 0.0,
371        };
372        let (_, params) = ev.to_cdp();
373        assert!(params["text"].is_null(), "{params}");
374    }
375
376    #[test]
377    fn keydown_printable_with_ctrl_has_no_text() {
378        // Ctrl+A is select-all, not typing "a" — `text` must be suppressed so
379        // chromium runs the shortcut instead of inserting the letter.
380        let ev = OpsInputEvent::KeyDown {
381            key: "a".into(),
382            code: "KeyA".into(),
383            modifiers: 2, // Ctrl
384            timestamp_ms: 0.0,
385        };
386        let (_, params) = ev.to_cdp();
387        assert!(params["text"].is_null(), "{params}");
388    }
389
390    #[test]
391    fn keydown_printable_with_meta_has_no_text() {
392        let ev = OpsInputEvent::KeyDown {
393            key: "r".into(),
394            code: "KeyR".into(),
395            modifiers: 4, // Meta/⌘
396            timestamp_ms: 0.0,
397        };
398        let (_, params) = ev.to_cdp();
399        assert!(params["text"].is_null(), "{params}");
400    }
401
402    #[test]
403    fn keydown_printable_with_shift_keeps_text() {
404        // Shift is not a shortcut modifier — Shift+a still produces text.
405        let ev = OpsInputEvent::KeyDown {
406            key: "A".into(),
407            code: "KeyA".into(),
408            modifiers: 8, // Shift
409            timestamp_ms: 0.0,
410        };
411        let (_, params) = ev.to_cdp();
412        assert_eq!(params["text"], "A");
413    }
414
415    #[test]
416    fn backspace_carries_virtual_key_code() {
417        // Without windowsVirtualKeyCode chromium ignores Backspace, so deletes
418        // do nothing — the bug this guards against.
419        let ev = OpsInputEvent::KeyDown {
420            key: "Backspace".into(),
421            code: "Backspace".into(),
422            modifiers: 0,
423            timestamp_ms: 0.0,
424        };
425        let (_, params) = ev.to_cdp();
426        assert_eq!(params["windowsVirtualKeyCode"], 8);
427        assert_eq!(params["nativeVirtualKeyCode"], 8);
428        assert!(params["text"].is_null(), "{params}");
429    }
430
431    #[test]
432    fn enter_and_arrows_carry_virtual_key_code() {
433        for (key, vk) in [("Enter", 13), ("ArrowLeft", 37), ("Delete", 46)] {
434            let ev = OpsInputEvent::KeyDown {
435                key: key.into(),
436                code: key.into(),
437                modifiers: 0,
438                timestamp_ms: 0.0,
439            };
440            let (_, params) = ev.to_cdp();
441            assert_eq!(params["windowsVirtualKeyCode"], vk, "{key}");
442        }
443    }
444
445    #[test]
446    fn letter_carries_uppercase_virtual_key_code() {
447        // Ctrl+A needs VK 65 to trigger select-all even though text is dropped.
448        let ev = OpsInputEvent::KeyDown {
449            key: "a".into(),
450            code: "KeyA".into(),
451            modifiers: 2, // Ctrl
452            timestamp_ms: 0.0,
453        };
454        let (_, params) = ev.to_cdp();
455        assert_eq!(params["windowsVirtualKeyCode"], 65);
456        assert!(params["text"].is_null(), "{params}");
457    }
458
459    #[test]
460    fn keyup_also_carries_virtual_key_code() {
461        let ev = OpsInputEvent::KeyUp {
462            key: "Backspace".into(),
463            code: "Backspace".into(),
464            modifiers: 0,
465            timestamp_ms: 0.0,
466        };
467        let (_, params) = ev.to_cdp();
468        assert_eq!(params["type"], "keyUp");
469        assert_eq!(params["windowsVirtualKeyCode"], 8);
470    }
471
472    #[test]
473    fn insert_text_converts_to_input_inserttext() {
474        let ev = OpsInputEvent::InsertText {
475            text: "SuperSecretPassword!".into(),
476            timestamp_ms: 0.0,
477        };
478        let (method, params) = ev.to_cdp();
479        assert_eq!(method, "Input.insertText");
480        assert_eq!(params["text"], "SuperSecretPassword!");
481    }
482
483    #[test]
484    fn insert_text_round_trips_from_snake_case_tag() {
485        let json = r#"{"type":"insert_text","text":"héllo","timestamp_ms":5.0}"#;
486        let back: OpsInputEvent = serde_json::from_str(json).unwrap();
487        match back {
488            OpsInputEvent::InsertText { text, .. } => assert_eq!(text, "héllo"),
489            _ => panic!("wrong variant"),
490        }
491    }
492}
agent_first_http/host/ops_panel/input_relay.rs

agent_first_http/host/ops_panel/
input_relay.rs
