browser_control/session/
input.rs

1//! Native CDP input and geometry for ref-based interaction.
2//!
3//! Every function here works on an already-attached flat session (see
4//! [`crate::session::cdp_session::with_page_session`]) and a CDP
5//! `backendDOMNodeId`, which is what an element ref resolves to. Clicks go
6//! through `Input.dispatchMouseEvent` at the element's centre in viewport
7//! CSS pixels — the coordinate space `DOM.getContentQuads` already reports
8//! in, so no device-pixel-ratio maths is involved. Text goes through
9//! `Input.insertText`, which fires the same `beforeinput`/`input` events a
10//! paste does (Playwright's `fill` does the same).
11//!
12//! Deliberately not done in v1: occlusion checks (a click on a covered
13//! element lands on the overlay), auto-waiting, and `Page.bringToFront`
14//! (input works on background tabs, and the project keeps automated tabs
15//! in the background).
16
17use anyhow::{anyhow, Context, Result};
18use serde_json::{json, Value};
19
20use crate::cdp::CdpClient;
21use crate::dom::scripts::SELECT_ALL_JS;
22use crate::errors::{is_cdp_node_gone, SessionError};
23
24/// A point in viewport CSS pixels.
25#[derive(Debug, Clone, Copy, PartialEq)]
26pub struct Point {
27    pub x: f64,
28    pub y: f64,
29}
30
31/// Map a `DOM.*` failure for `backend_node_id` onto the typed
32/// [`SessionError::NodeGone`] when the message says the node left the
33/// document; otherwise attach context and pass through.
34fn node_err(backend_node_id: u64, op: &'static str) -> impl FnOnce(anyhow::Error) -> anyhow::Error {
35    move |e| {
36        let msg = format!("{e:#}");
37        if is_cdp_node_gone(&msg) {
38            SessionError::NodeGone {
39                backend_node_id,
40                details: msg,
41            }
42            .into()
43        } else {
44            e.context(format!("{op} on node {backend_node_id}"))
45        }
46    }
47}
48
49/// Viewport size from `Page.getLayoutMetrics` (`cssLayoutViewport`, with
50/// the pre-CSS field as fallback for older Chromium).
51fn viewport_size(metrics: &Value) -> (f64, f64) {
52    let vp = metrics
53        .get("cssLayoutViewport")
54        .or_else(|| metrics.get("layoutViewport"));
55    let w = vp
56        .and_then(|v| v.get("clientWidth"))
57        .and_then(Value::as_f64)
58        .unwrap_or(f64::MAX);
59    let h = vp
60        .and_then(|v| v.get("clientHeight"))
61        .and_then(Value::as_f64)
62        .unwrap_or(f64::MAX);
63    (w, h)
64}
65
66/// Scroll position from `Page.getLayoutMetrics`, in CSS pixels.
67fn viewport_offset(metrics: &Value) -> (f64, f64) {
68    let vp = metrics
69        .get("cssLayoutViewport")
70        .or_else(|| metrics.get("layoutViewport"));
71    let x = vp
72        .and_then(|v| v.get("pageX"))
73        .and_then(Value::as_f64)
74        .unwrap_or(0.0);
75    let y = vp
76        .and_then(|v| v.get("pageY"))
77        .and_then(Value::as_f64)
78        .unwrap_or(0.0);
79    (x, y)
80}
81
82/// Pick the centre of the first quad with a visible area after clipping
83/// to the viewport. Mirrors Puppeteer's `clickablePoint`.
84pub fn pick_point(quads: &Value, vw: f64, vh: f64) -> Option<Point> {
85    let quads = quads.as_array()?;
86    for q in quads {
87        let nums: Vec<f64> = q.as_array()?.iter().filter_map(Value::as_f64).collect();
88        if nums.len() != 8 {
89            continue;
90        }
91        let pts: Vec<(f64, f64)> = (0..4)
92            .map(|i| (nums[i * 2].clamp(0.0, vw), nums[i * 2 + 1].clamp(0.0, vh)))
93            .collect();
94        // Shoelace area of the clipped quad.
95        let mut area = 0.0;
96        for i in 0..4 {
97            let (x1, y1) = pts[i];
98            let (x2, y2) = pts[(i + 1) % 4];
99            area += x1 * y2 - x2 * y1;
100        }
101        if area.abs() / 2.0 <= 1.0 {
102            continue;
103        }
104        let x = pts.iter().map(|p| p.0).sum::<f64>() / 4.0;
105        let y = pts.iter().map(|p| p.1).sum::<f64>() / 4.0;
106        return Some(Point { x, y });
107    }
108    None
109}
110
111/// Scroll the node into view and return its clickable centre.
112pub async fn node_center(c: &CdpClient, sid: &str, backend_node_id: u64) -> Result<Point> {
113    let _ = c
114        .send_with_session("DOM.enable", json!({}), Some(sid))
115        .await;
116    c.send_with_session(
117        "DOM.scrollIntoViewIfNeeded",
118        json!({ "backendNodeId": backend_node_id }),
119        Some(sid),
120    )
121    .await
122    .map_err(node_err(backend_node_id, "scrollIntoViewIfNeeded"))?;
123    let quads = c
124        .send_with_session(
125            "DOM.getContentQuads",
126            json!({ "backendNodeId": backend_node_id }),
127            Some(sid),
128        )
129        .await
130        .map_err(node_err(backend_node_id, "getContentQuads"))?;
131    let metrics = c
132        .send_with_session("Page.getLayoutMetrics", json!({}), Some(sid))
133        .await?;
134    let (vw, vh) = viewport_size(&metrics);
135    pick_point(&quads["quads"], vw, vh)
136        .ok_or_else(|| anyhow!("element has no visible box (hidden, zero-size, or outside the viewport after scrolling)"))
137}
138
139async fn mouse(c: &CdpClient, sid: &str, kind: &str, p: Point, pressed: bool) -> Result<()> {
140    let mut params = json!({ "type": kind, "x": p.x, "y": p.y });
141    if pressed {
142        params["button"] = json!("left");
143        params["clickCount"] = json!(1);
144    }
145    c.send_with_session("Input.dispatchMouseEvent", params, Some(sid))
146        .await
147        .context("Input.dispatchMouseEvent")?;
148    Ok(())
149}
150
151/// Left-click the node's centre.
152pub async fn click(c: &CdpClient, sid: &str, backend_node_id: u64) -> Result<Point> {
153    let p = node_center(c, sid, backend_node_id).await?;
154    mouse(c, sid, "mouseMoved", p, false).await?;
155    mouse(c, sid, "mousePressed", p, true).await?;
156    mouse(c, sid, "mouseReleased", p, true).await?;
157    Ok(p)
158}
159
160/// Move the pointer over the node's centre.
161pub async fn hover(c: &CdpClient, sid: &str, backend_node_id: u64) -> Result<Point> {
162    let p = node_center(c, sid, backend_node_id).await?;
163    mouse(c, sid, "mouseMoved", p, false).await?;
164    Ok(p)
165}
166
167/// Focus the node, replace its current content with `text`, and
168/// optionally press Enter. `press_sequentially` inserts one character
169/// per event for inputs that react to each keystroke.
170pub async fn type_text(
171    c: &CdpClient,
172    sid: &str,
173    backend_node_id: u64,
174    text: &str,
175    press_sequentially: bool,
176    submit: bool,
177) -> Result<()> {
178    // Background tabs do not deliver focus events unless emulated.
179    let _ = c
180        .send_with_session(
181            "Emulation.setFocusEmulationEnabled",
182            json!({ "enabled": true }),
183            Some(sid),
184        )
185        .await;
186    c.send_with_session(
187        "DOM.focus",
188        json!({ "backendNodeId": backend_node_id }),
189        Some(sid),
190    )
191    .await
192    .map_err(node_err(backend_node_id, "focus"))?;
193    let resolved = c
194        .send_with_session(
195            "DOM.resolveNode",
196            json!({ "backendNodeId": backend_node_id }),
197            Some(sid),
198        )
199        .await
200        .map_err(node_err(backend_node_id, "resolveNode"))?;
201    let object_id = resolved["object"]["objectId"]
202        .as_str()
203        .ok_or_else(|| anyhow!("DOM.resolveNode returned no objectId"))?
204        .to_string();
205    let select = c
206        .send_with_session(
207            "Runtime.callFunctionOn",
208            json!({
209                "objectId": object_id,
210                "functionDeclaration": SELECT_ALL_JS,
211                "arguments": [{ "value": text.is_empty() }],
212                "returnByValue": true,
213            }),
214            Some(sid),
215        )
216        .await;
217    let _ = c
218        .send_with_session(
219            "Runtime.releaseObject",
220            json!({ "objectId": object_id }),
221            Some(sid),
222        )
223        .await;
224    select.context("selecting existing content")?;
225    if !text.is_empty() {
226        if press_sequentially {
227            for ch in text.chars() {
228                insert_text(c, sid, &ch.to_string()).await?;
229            }
230        } else {
231            insert_text(c, sid, text).await?;
232        }
233    }
234    if submit {
235        press_enter(c, sid).await?;
236    }
237    Ok(())
238}
239
240async fn insert_text(c: &CdpClient, sid: &str, text: &str) -> Result<()> {
241    c.send_with_session("Input.insertText", json!({ "text": text }), Some(sid))
242        .await
243        .context("Input.insertText")?;
244    Ok(())
245}
246
247/// Press and release Enter on the focused element. `text: "\r"` on the
248/// keyDown is what makes Chromium synthesise the `keypress` and submit
249/// forms, matching Puppeteer.
250pub async fn press_enter(c: &CdpClient, sid: &str) -> Result<()> {
251    c.send_with_session(
252        "Input.dispatchKeyEvent",
253        json!({
254            "type": "keyDown",
255            "key": "Enter",
256            "code": "Enter",
257            "windowsVirtualKeyCode": 13,
258            "nativeVirtualKeyCode": 13,
259            "text": "\r",
260            "unmodifiedText": "\r",
261        }),
262        Some(sid),
263    )
264    .await
265    .context("Input.dispatchKeyEvent keyDown")?;
266    c.send_with_session(
267        "Input.dispatchKeyEvent",
268        json!({
269            "type": "keyUp",
270            "key": "Enter",
271            "code": "Enter",
272            "windowsVirtualKeyCode": 13,
273            "nativeVirtualKeyCode": 13,
274        }),
275        Some(sid),
276    )
277    .await
278    .context("Input.dispatchKeyEvent keyUp")?;
279    Ok(())
280}
281
282/// Pointer-event drag from one node's centre to another's. HTML5 native
283/// drag-and-drop (`draggable`) needs `Input.setInterceptDrags`; not in v1.
284pub async fn drag(c: &CdpClient, sid: &str, from: u64, to: u64) -> Result<()> {
285    let a = node_center(c, sid, from).await?;
286    let b = node_center(c, sid, to).await?;
287    mouse(c, sid, "mouseMoved", a, false).await?;
288    mouse(c, sid, "mousePressed", a, true).await?;
289    const STEPS: usize = 5;
290    for i in 1..=STEPS {
291        let t = i as f64 / STEPS as f64;
292        let p = Point {
293            x: a.x + (b.x - a.x) * t,
294            y: a.y + (b.y - a.y) * t,
295        };
296        mouse(c, sid, "mouseMoved", p, true).await?;
297    }
298    mouse(c, sid, "mouseReleased", b, true).await?;
299    Ok(())
300}
301
302/// The Document node's `backendNodeId`: changes on every navigation, so
303/// it identifies "the document the refs were taken from".
304pub async fn document_token(c: &CdpClient, sid: &str) -> Result<u64> {
305    let doc = c
306        .send_with_session("DOM.getDocument", json!({ "depth": 0 }), Some(sid))
307        .await
308        .context("DOM.getDocument")?;
309    doc["root"]["backendNodeId"]
310        .as_u64()
311        .ok_or_else(|| anyhow!("DOM.getDocument returned no root backendNodeId"))
312}
313
314/// Border box of the node in *document* coordinates, the same contract as
315/// [`crate::dom::scripts::GET_CLIP_RECT_JS`], for element screenshots.
316pub async fn node_clip_rect(c: &CdpClient, sid: &str, backend_node_id: u64) -> Result<Value> {
317    let _ = c
318        .send_with_session("DOM.enable", json!({}), Some(sid))
319        .await;
320    c.send_with_session(
321        "DOM.scrollIntoViewIfNeeded",
322        json!({ "backendNodeId": backend_node_id }),
323        Some(sid),
324    )
325    .await
326    .map_err(node_err(backend_node_id, "scrollIntoViewIfNeeded"))?;
327    let model = c
328        .send_with_session(
329            "DOM.getBoxModel",
330            json!({ "backendNodeId": backend_node_id }),
331            Some(sid),
332        )
333        .await
334        .map_err(node_err(backend_node_id, "getBoxModel"))?;
335    let border: Vec<f64> = model["model"]["border"]
336        .as_array()
337        .map(|a| a.iter().filter_map(Value::as_f64).collect())
338        .unwrap_or_default();
339    if border.len() != 8 {
340        return Err(anyhow!("element has no box model (hidden or detached)"));
341    }
342    let xs = [border[0], border[2], border[4], border[6]];
343    let ys = [border[1], border[3], border[5], border[7]];
344    let min_x = xs.iter().cloned().fold(f64::MAX, f64::min);
345    let max_x = xs.iter().cloned().fold(f64::MIN, f64::max);
346    let min_y = ys.iter().cloned().fold(f64::MAX, f64::min);
347    let max_y = ys.iter().cloned().fold(f64::MIN, f64::max);
348    if max_x - min_x <= 0.0 || max_y - min_y <= 0.0 {
349        return Err(anyhow!("element has zero area"));
350    }
351    let metrics = c
352        .send_with_session("Page.getLayoutMetrics", json!({}), Some(sid))
353        .await?;
354    let (sx, sy) = viewport_offset(&metrics);
355    Ok(json!({
356        "x": min_x + sx,
357        "y": min_y + sy,
358        "width": max_x - min_x,
359        "height": max_y - min_y,
360    }))
361}
362
363#[cfg(test)]
364mod tests {
365    use super::*;
366    use futures_util::{SinkExt, StreamExt};
367    use std::sync::Arc;
368    use tokio::sync::Mutex;
369    use tokio_tungstenite::tungstenite::Message;
370
371    /// Records every request and answers geometry methods with canned
372    /// values; everything else gets `{}`.
373    async fn spawn_mock(node_gone: bool) -> (String, Arc<Mutex<Vec<Value>>>) {
374        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
375        let addr = listener.local_addr().unwrap();
376        let seen = Arc::new(Mutex::new(Vec::<Value>::new()));
377        tokio::spawn({
378            let seen = seen.clone();
379            async move {
380                let (stream, _) = listener.accept().await.unwrap();
381                let mut ws = tokio_tungstenite::accept_async(stream).await.unwrap();
382                while let Some(Ok(Message::Text(t))) = ws.next().await {
383                    let req: Value = serde_json::from_str(&t).unwrap();
384                    seen.lock().await.push(req.clone());
385                    let id = req["id"].as_u64().unwrap();
386                    let method = req["method"].as_str().unwrap_or("");
387                    if node_gone && method.starts_with("DOM.") && method != "DOM.enable" {
388                        let resp = json!({"id": id, "error": {"code": -32000, "message": "No node with given id found"}});
389                        ws.send(Message::Text(resp.to_string())).await.unwrap();
390                        continue;
391                    }
392                    let result = match method {
393                        "DOM.getContentQuads" => json!({"quads": [
394                            // Off-screen quad (negative), then a real 100x20 box at (10,30).
395                            [-50, -50, -10, -50, -10, -40, -50, -40],
396                            [10, 30, 110, 30, 110, 50, 10, 50],
397                        ]}),
398                        "Page.getLayoutMetrics" => json!({"cssLayoutViewport": {
399                            "pageX": 0, "pageY": 400, "clientWidth": 800, "clientHeight": 600
400                        }}),
401                        "DOM.resolveNode" => json!({"object": {"objectId": "obj-1"}}),
402                        "DOM.getDocument" => json!({"root": {"backendNodeId": 4242}}),
403                        "DOM.getBoxModel" => {
404                            json!({"model": {"border": [10, 30, 110, 30, 110, 50, 10, 50]}})
405                        }
406                        _ => json!({}),
407                    };
408                    let resp = json!({"id": id, "result": result});
409                    ws.send(Message::Text(resp.to_string())).await.unwrap();
410                }
411            }
412        });
413        (format!("ws://{addr}"), seen)
414    }
415
416    fn methods(seen: &[Value]) -> Vec<String> {
417        seen.iter()
418            .filter_map(|v| v["method"].as_str().map(String::from))
419            .collect()
420    }
421
422    #[test]
423    fn pick_point_skips_offscreen_and_clips() {
424        let quads = json!([
425            [-50, -50, -10, -50, -10, -40, -50, -40],
426            [700, 10, 900, 10, 900, 30, 700, 30],
427        ]);
428        let p = pick_point(&quads, 800.0, 600.0).unwrap();
429        assert_eq!(p, Point { x: 750.0, y: 20.0 });
430        assert!(pick_point(&json!([[0, 0, 0, 0, 0, 0, 0, 0]]), 800.0, 600.0).is_none());
431        assert!(pick_point(&json!(null), 800.0, 600.0).is_none());
432    }
433
434    #[tokio::test]
435    async fn click_dispatches_move_press_release_at_centre() {
436        let (url, seen) = spawn_mock(false).await;
437        let c = CdpClient::connect(&url).await.unwrap();
438        let p = click(&c, "S1", 77).await.unwrap();
439        assert_eq!(p, Point { x: 60.0, y: 40.0 });
440        let calls = seen.lock().await;
441        assert_eq!(
442            methods(&calls),
443            vec![
444                "DOM.enable",
445                "DOM.scrollIntoViewIfNeeded",
446                "DOM.getContentQuads",
447                "Page.getLayoutMetrics",
448                "Input.dispatchMouseEvent",
449                "Input.dispatchMouseEvent",
450                "Input.dispatchMouseEvent",
451            ]
452        );
453        let press = &calls[5];
454        assert_eq!(press["sessionId"], "S1");
455        assert_eq!(press["params"]["type"], "mousePressed");
456        assert_eq!(press["params"]["button"], "left");
457        assert_eq!(press["params"]["x"], 60.0);
458        assert_eq!(press["params"]["y"], 40.0);
459        assert_eq!(calls[1]["params"]["backendNodeId"], 77);
460    }
461
462    #[tokio::test]
463    async fn type_text_focuses_selects_inserts_and_submits() {
464        let (url, seen) = spawn_mock(false).await;
465        let c = CdpClient::connect(&url).await.unwrap();
466        type_text(&c, "S1", 5, "hi", false, true).await.unwrap();
467        let calls = seen.lock().await;
468        assert_eq!(
469            methods(&calls),
470            vec![
471                "Emulation.setFocusEmulationEnabled",
472                "DOM.focus",
473                "DOM.resolveNode",
474                "Runtime.callFunctionOn",
475                "Runtime.releaseObject",
476                "Input.insertText",
477                "Input.dispatchKeyEvent",
478                "Input.dispatchKeyEvent",
479            ]
480        );
481        assert_eq!(calls[3]["params"]["arguments"][0]["value"], false);
482        assert_eq!(calls[5]["params"]["text"], "hi");
483        assert_eq!(calls[6]["params"]["text"], "\r");
484        assert_eq!(calls[7]["params"]["type"], "keyUp");
485    }
486
487    #[tokio::test]
488    async fn type_text_sequential_and_clear() {
489        let (url, seen) = spawn_mock(false).await;
490        let c = CdpClient::connect(&url).await.unwrap();
491        type_text(&c, "S1", 5, "ab", true, false).await.unwrap();
492        type_text(&c, "S1", 5, "", false, false).await.unwrap();
493        let calls = seen.lock().await;
494        let inserts: Vec<&Value> = calls
495            .iter()
496            .filter(|v| v["method"] == "Input.insertText")
497            .collect();
498        assert_eq!(inserts.len(), 2);
499        assert_eq!(inserts[0]["params"]["text"], "a");
500        assert_eq!(inserts[1]["params"]["text"], "b");
501        let clears: Vec<&Value> = calls
502            .iter()
503            .filter(|v| v["method"] == "Runtime.callFunctionOn")
504            .collect();
505        assert_eq!(clears[1]["params"]["arguments"][0]["value"], true);
506    }
507
508    #[tokio::test]
509    async fn drag_presses_moves_and_releases() {
510        let (url, seen) = spawn_mock(false).await;
511        let c = CdpClient::connect(&url).await.unwrap();
512        drag(&c, "S1", 1, 2).await.unwrap();
513        let calls = seen.lock().await;
514        let types: Vec<&str> = calls
515            .iter()
516            .filter(|v| v["method"] == "Input.dispatchMouseEvent")
517            .map(|v| v["params"]["type"].as_str().unwrap())
518            .collect();
519        assert_eq!(types[0], "mouseMoved");
520        assert_eq!(types[1], "mousePressed");
521        assert_eq!(*types.last().unwrap(), "mouseReleased");
522        assert_eq!(types.len(), 2 + 5 + 1);
523    }
524
525    #[tokio::test]
526    async fn node_gone_maps_to_typed_error() {
527        let (url, _seen) = spawn_mock(true).await;
528        let c = CdpClient::connect(&url).await.unwrap();
529        let err = click(&c, "S1", 9).await.unwrap_err();
530        match err.downcast_ref::<SessionError>() {
531            Some(SessionError::NodeGone {
532                backend_node_id, ..
533            }) => assert_eq!(*backend_node_id, 9),
534            other => panic!("expected NodeGone, got {other:?}"),
535        }
536    }
537
538    #[tokio::test]
539    async fn document_token_and_clip_rect() {
540        let (url, _seen) = spawn_mock(false).await;
541        let c = CdpClient::connect(&url).await.unwrap();
542        assert_eq!(document_token(&c, "S1").await.unwrap(), 4242);
543        let rect = node_clip_rect(&c, "S1", 3).await.unwrap();
544        // Viewport is scrolled 400px down, so document y = 30 + 400.
545        assert_eq!(
546            rect,
547            json!({"x": 10.0, "y": 430.0, "width": 100.0, "height": 20.0})
548        );
549    }
550}
browser_control/session/input.rs

browser_control/session/
input.rs
