browser_control/session/
input_bidi.rs

1//! WebDriver BiDi (Firefox) counterpart of [`crate::session::input`].
2//!
3//! Element refs on BiDi are integer ids in a page-side registry written by
4//! the snapshot walker (`crate::dom::scripts::SNAPSHOT_TREE_JS`). Geometry
5//! is computed in the page (`REF_CENTER_JS`, `REF_CLIP_RECT_JS`) and input
6//! is dispatched with `input.performActions` at viewport-origin CSS pixel
7//! coordinates, so no `locateNodes` / `sharedId` bookkeeping is needed.
8//! Typing goes through in-page `execCommand('insertText')` (with a
9//! value-setter fallback) because BiDi has no `insertText`; per-character
10//! key actions cover `press_sequentially`, and Enter is a key action.
11//!
12//! Every helper returns `SessionError::NodeGone` when the registry no longer
13//! resolves the id, which the tool layer maps to `StaleRef`.
14
15use anyhow::{anyhow, Context, Result};
16use serde_json::{json, Value};
17
18use crate::bidi::BidiClient;
19use crate::dom::scripts::{
20    DOC_SIZE_JS, DOC_TOKEN_JS, REF_CENTER_JS, REF_CLIP_RECT_JS, REF_TYPE_JS, SNAPSHOT_TREE_JS,
21};
22use crate::errors::SessionError;
23use crate::session::input::Point;
24
25/// Node budget handed to the walker.
26const SNAPSHOT_MAX_NODES: u64 = 20_000;
27/// WebDriver key code point for Enter.
28const ENTER: &str = "\u{e007}";
29/// Interpolated pointer moves during a drag.
30const DRAG_STEPS: usize = 5;
31
32/// The `value` of a string `RemoteValue`.
33fn remote_string(v: &Value) -> Result<String> {
34    v["value"]
35        .as_str()
36        .map(String::from)
37        .ok_or_else(|| anyhow!("script returned no string result: {v}"))
38}
39
40/// Decode a ref helper's JSON string: `{"gone":true}` → `NodeGone`,
41/// `{"error":…}` → error, otherwise the payload.
42fn decode_ref_result(id: u64, op: &'static str, s: &str) -> Result<Value> {
43    let v: Value = serde_json::from_str(s)
44        .with_context(|| format!("{op} on node {id}: invalid helper result"))?;
45    if v["gone"].as_bool().unwrap_or(false) {
46        return Err(SessionError::NodeGone {
47            backend_node_id: id,
48            details: format!("{op}: ref id {id} is not in the page registry or was detached"),
49        }
50        .into());
51    }
52    if let Some(e) = v["error"].as_str() {
53        return Err(anyhow!("{op} on node {id}: {e}"));
54    }
55    Ok(v)
56}
57
58async fn call_ref(
59    c: &BidiClient,
60    ctx: &str,
61    id: u64,
62    op: &'static str,
63    script: &str,
64    extra: Vec<Value>,
65) -> Result<Value> {
66    let mut args = vec![json!(id)];
67    args.extend(extra);
68    let v = c.script_call_function(ctx, script, args).await?;
69    decode_ref_result(id, op, &remote_string(&v)?)
70}
71
72fn pointer_move(p: Point) -> Value {
73    json!({
74        "type": "pointerMove",
75        "x": p.x.round() as i64,
76        "y": p.y.round() as i64,
77        "origin": "viewport",
78    })
79}
80
81fn pointer_source(actions: Vec<Value>) -> Value {
82    json!({
83        "type": "pointer",
84        "id": "mouse",
85        "parameters": { "pointerType": "mouse" },
86        "actions": actions,
87    })
88}
89
90fn key_source(actions: Vec<Value>) -> Value {
91    json!({ "type": "key", "id": "kb", "actions": actions })
92}
93
94fn key_press(value: &str) -> [Value; 2] {
95    [
96        json!({ "type": "keyDown", "value": value }),
97        json!({ "type": "keyUp", "value": value }),
98    ]
99}
100
101/// Run the walker and parse its JSON.
102pub async fn accessibility_tree(c: &BidiClient, ctx: &str) -> Result<Value> {
103    let v = c
104        .script_call_function(ctx, SNAPSHOT_TREE_JS, vec![json!(SNAPSHOT_MAX_NODES)])
105        .await?;
106    let tree: Value = serde_json::from_str(&remote_string(&v)?)
107        .context("accessibility walker returned invalid JSON")?;
108    if tree["truncated"].as_bool().unwrap_or(false) {
109        tracing::warn!(
110            target = %ctx,
111            "accessibility snapshot truncated at {SNAPSHOT_MAX_NODES} nodes"
112        );
113    }
114    Ok(tree)
115}
116
117/// Per-document token (see `DOC_TOKEN_JS`).
118pub async fn document_token(c: &BidiClient, ctx: &str) -> Result<u64> {
119    let v = c.script_evaluate(ctx, DOC_TOKEN_JS).await?;
120    let v = crate::bidi::unwrap_script_result(v)?;
121    remote_string(&v)?
122        .parse::<u64>()
123        .context("document token is not an integer")
124}
125
126/// Scroll the node into view and return its clickable centre.
127pub async fn node_center(c: &BidiClient, ctx: &str, id: u64) -> Result<Point> {
128    let v = call_ref(c, ctx, id, "center", REF_CENTER_JS, vec![]).await?;
129    Ok(Point {
130        x: v["x"].as_f64().unwrap_or(0.0),
131        y: v["y"].as_f64().unwrap_or(0.0),
132    })
133}
134
135/// Left-click the node's centre.
136pub async fn click(c: &BidiClient, ctx: &str, id: u64) -> Result<Point> {
137    let p = node_center(c, ctx, id).await?;
138    c.input_perform_actions(
139        ctx,
140        json!([pointer_source(vec![
141            pointer_move(p),
142            json!({ "type": "pointerDown", "button": 0 }),
143            json!({ "type": "pointerUp", "button": 0 }),
144        ])]),
145    )
146    .await?;
147    let _ = c.input_release_actions(ctx).await;
148    Ok(p)
149}
150
151/// Move the pointer over the node's centre.
152pub async fn hover(c: &BidiClient, ctx: &str, id: u64) -> Result<Point> {
153    let p = node_center(c, ctx, id).await?;
154    c.input_perform_actions(ctx, json!([pointer_source(vec![pointer_move(p)])]))
155        .await?;
156    Ok(p)
157}
158
159/// Focus the node, replace its content with `text`, optionally one key at a
160/// time, and optionally press Enter.
161pub async fn type_text(
162    c: &BidiClient,
163    ctx: &str,
164    id: u64,
165    text: &str,
166    press_sequentially: bool,
167    submit: bool,
168) -> Result<()> {
169    let mode = if text.is_empty() {
170        "clear"
171    } else if press_sequentially {
172        "select"
173    } else {
174        "fill"
175    };
176    call_ref(
177        c,
178        ctx,
179        id,
180        "type",
181        REF_TYPE_JS,
182        vec![json!(text), json!(mode)],
183    )
184    .await?;
185    if press_sequentially && !text.is_empty() {
186        let mut keys = Vec::new();
187        for ch in text.chars() {
188            let v = match ch {
189                '\n' | '\r' => ENTER.to_string(),
190                other => other.to_string(),
191            };
192            keys.extend(key_press(&v));
193        }
194        c.input_perform_actions(ctx, json!([key_source(keys)]))
195            .await?;
196    }
197    if submit {
198        c.input_perform_actions(ctx, json!([key_source(key_press(ENTER).to_vec())]))
199            .await?;
200    }
201    let _ = c.input_release_actions(ctx).await;
202    Ok(())
203}
204
205/// Pointer drag from one node's centre to another's.
206pub async fn drag(c: &BidiClient, ctx: &str, from: u64, to: u64) -> Result<()> {
207    let a = node_center(c, ctx, from).await?;
208    let b = node_center(c, ctx, to).await?;
209    let mut actions = vec![
210        pointer_move(a),
211        json!({ "type": "pointerDown", "button": 0 }),
212    ];
213    for i in 1..=DRAG_STEPS {
214        let t = i as f64 / DRAG_STEPS as f64;
215        actions.push(pointer_move(Point {
216            x: a.x + (b.x - a.x) * t,
217            y: a.y + (b.y - a.y) * t,
218        }));
219    }
220    actions.push(pointer_move(b));
221    actions.push(json!({ "type": "pointerUp", "button": 0 }));
222    c.input_perform_actions(ctx, json!([pointer_source(actions)]))
223        .await?;
224    let _ = c.input_release_actions(ctx).await;
225    Ok(())
226}
227
228/// Border box of the node in document coordinates.
229pub async fn node_clip_rect(c: &BidiClient, ctx: &str, id: u64) -> Result<Value> {
230    call_ref(c, ctx, id, "clip", REF_CLIP_RECT_JS, vec![]).await
231}
232
233/// Document scroll size, for full-page screenshots.
234pub async fn document_size(c: &BidiClient, ctx: &str) -> Result<(f64, f64)> {
235    let v = c.script_evaluate(ctx, DOC_SIZE_JS).await?;
236    let v = crate::bidi::unwrap_script_result(v)?;
237    let parsed: Value = serde_json::from_str(&remote_string(&v)?)
238        .context("document size helper returned invalid JSON")?;
239    Ok((
240        parsed["width"].as_f64().unwrap_or(0.0),
241        parsed["height"].as_f64().unwrap_or(0.0),
242    ))
243}
244
245#[cfg(test)]
246mod tests {
247    use super::*;
248    use futures_util::{SinkExt, StreamExt};
249    use std::sync::{Arc, Mutex};
250    use tokio_tungstenite::tungstenite::Message;
251
252    /// BiDi-framed recording mock that dispatches `script.callFunction` on
253    /// the helper markers and records input commands.
254    async fn spawn_mock(gone: bool) -> (String, Arc<Mutex<Vec<Value>>>) {
255        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
256        let addr = listener.local_addr().unwrap();
257        let seen = Arc::new(Mutex::new(Vec::<Value>::new()));
258        tokio::spawn({
259            let seen = seen.clone();
260            async move {
261                let (stream, _) = listener.accept().await.unwrap();
262                let mut ws = tokio_tungstenite::accept_async(stream).await.unwrap();
263                while let Some(Ok(Message::Text(t))) = ws.next().await {
264                    let req: Value = serde_json::from_str(&t).unwrap();
265                    seen.lock().unwrap().push(req.clone());
266                    let id = req["id"].as_u64().unwrap();
267                    let method = req["method"].as_str().unwrap_or("");
268                    let decl = req["params"]["functionDeclaration"].as_str().unwrap_or("");
269                    let expr = req["params"]["expression"].as_str().unwrap_or("");
270                    let string = |s: String| json!({"type": "success", "result": {"type": "string", "value": s}, "realm": "R1"});
271                    let result = match method {
272                        "script.callFunction" if gone => string("{\"gone\":true}".into()),
273                        "script.callFunction" if decl.contains("bc:center") => {
274                            string("{\"x\":30.4,\"y\":20}".into())
275                        }
276                        "script.callFunction" if decl.contains("bc:clip") => {
277                            string("{\"x\":10,\"y\":430,\"width\":100,\"height\":20}".into())
278                        }
279                        "script.callFunction" if decl.contains("bc:type") => {
280                            string("{\"kind\":\"field\",\"method\":\"execCommand\"}".into())
281                        }
282                        "script.callFunction" if decl.contains("bc:snapshot") => string(
283                            json!({"nodes": [
284                                {"nodeId": "root", "backendDOMNodeId": 4294967296u64,
285                                 "role": {"value": "RootWebArea"}, "name": {"value": "T"}, "childIds": ["n1"]},
286                                {"nodeId": "n1", "parentId": "root", "backendDOMNodeId": 1,
287                                 "role": {"value": "button"}, "name": {"value": "Go"}, "childIds": [],
288                                 "properties": [{"name": "focusable", "value": {"value": true}}]}
289                            ], "truncated": false})
290                            .to_string(),
291                        ),
292                        "script.evaluate" if expr.contains("__bcDocToken") => {
293                            string("4294967296".into())
294                        }
295                        "script.evaluate" if expr.contains("scrollWidth") => {
296                            string("{\"width\":1000,\"height\":3000}".into())
297                        }
298                        _ => json!({}),
299                    };
300                    ws.send(Message::Text(
301                        json!({"type": "success", "id": id, "result": result}).to_string(),
302                    ))
303                    .await
304                    .unwrap();
305                }
306            }
307        });
308        (format!("ws://{addr}"), seen)
309    }
310
311    fn input_calls(seen: &[Value]) -> Vec<Value> {
312        seen.iter()
313            .filter(|r| r["method"] == "input.performActions")
314            .map(|r| r["params"]["actions"].clone())
315            .collect()
316    }
317
318    #[tokio::test]
319    async fn click_moves_presses_releases_then_releases_actions() {
320        let (url, seen) = spawn_mock(false).await;
321        let c = BidiClient::connect(&url).await.unwrap();
322        let p = click(&c, "C1", 7).await.unwrap();
323        assert_eq!(p, Point { x: 30.4, y: 20.0 });
324        let seen = seen.lock().unwrap();
325        let first = &seen[0];
326        assert_eq!(first["method"], "script.callFunction");
327        assert_eq!(
328            first["params"]["arguments"][0],
329            json!({"type": "number", "value": 7})
330        );
331        let acts = input_calls(&seen);
332        assert_eq!(acts.len(), 1);
333        let pointer = &acts[0][0];
334        assert_eq!(pointer["type"], "pointer");
335        assert_eq!(pointer["parameters"]["pointerType"], "mouse");
336        assert_eq!(
337            pointer["actions"],
338            json!([
339                {"type": "pointerMove", "x": 30, "y": 20, "origin": "viewport"},
340                {"type": "pointerDown", "button": 0},
341                {"type": "pointerUp", "button": 0},
342            ])
343        );
344        assert_eq!(seen.last().unwrap()["method"], "input.releaseActions");
345    }
346
347    #[tokio::test]
348    async fn type_fill_then_enter() {
349        let (url, seen) = spawn_mock(false).await;
350        let c = BidiClient::connect(&url).await.unwrap();
351        type_text(&c, "C1", 3, "hi", false, true).await.unwrap();
352        let seen = seen.lock().unwrap();
353        assert_eq!(
354            seen[0]["params"]["arguments"],
355            json!([
356                {"type": "number", "value": 3},
357                {"type": "string", "value": "hi"},
358                {"type": "string", "value": "fill"}
359            ])
360        );
361        let acts = input_calls(&seen);
362        assert_eq!(acts.len(), 1, "fill sends no key actions; only Enter");
363        assert_eq!(acts[0][0]["type"], "key");
364        assert_eq!(acts[0][0]["actions"][0]["value"], ENTER);
365        assert_eq!(acts[0][0]["actions"][1]["type"], "keyUp");
366    }
367
368    #[tokio::test]
369    async fn type_sequentially_emits_per_char_keys_and_clear_sends_none() {
370        let (url, seen) = spawn_mock(false).await;
371        let c = BidiClient::connect(&url).await.unwrap();
372        type_text(&c, "C1", 3, "a\n", true, false).await.unwrap();
373        type_text(&c, "C1", 3, "", false, false).await.unwrap();
374        let seen = seen.lock().unwrap();
375        assert_eq!(seen[0]["params"]["arguments"][2]["value"], "select");
376        let acts = input_calls(&seen);
377        assert_eq!(acts.len(), 1);
378        let keys = &acts[0][0]["actions"];
379        assert_eq!(keys.as_array().unwrap().len(), 4);
380        assert_eq!(keys[0]["value"], "a");
381        assert_eq!(keys[2]["value"], ENTER);
382        let clear = seen
383            .iter()
384            .filter(|r| r["method"] == "script.callFunction")
385            .nth(1)
386            .unwrap();
387        assert_eq!(clear["params"]["arguments"][2]["value"], "clear");
388    }
389
390    #[tokio::test]
391    async fn drag_sequence() {
392        let (url, seen) = spawn_mock(false).await;
393        let c = BidiClient::connect(&url).await.unwrap();
394        drag(&c, "C1", 1, 2).await.unwrap();
395        let seen = seen.lock().unwrap();
396        let acts = input_calls(&seen);
397        let types: Vec<&str> = acts[0][0]["actions"]
398            .as_array()
399            .unwrap()
400            .iter()
401            .map(|a| a["type"].as_str().unwrap())
402            .collect();
403        assert_eq!(types[0], "pointerMove");
404        assert_eq!(types[1], "pointerDown");
405        assert_eq!(*types.last().unwrap(), "pointerUp");
406        assert_eq!(types.len(), 2 + DRAG_STEPS + 1 + 1);
407    }
408
409    #[tokio::test]
410    async fn gone_maps_to_node_gone() {
411        let (url, _seen) = spawn_mock(true).await;
412        let c = BidiClient::connect(&url).await.unwrap();
413        let err = hover(&c, "C1", 9).await.unwrap_err();
414        match err.downcast_ref::<SessionError>() {
415            Some(SessionError::NodeGone {
416                backend_node_id, ..
417            }) => {
418                assert_eq!(*backend_node_id, 9)
419            }
420            other => panic!("expected NodeGone, got {other:?}"),
421        }
422    }
423
424    #[tokio::test]
425    async fn tree_token_clip_and_document_size() {
426        let (url, _seen) = spawn_mock(false).await;
427        let c = BidiClient::connect(&url).await.unwrap();
428        let tree = accessibility_tree(&c, "C1").await.unwrap();
429        let parsed = crate::a11y::parse_full_ax_tree(&tree).unwrap();
430        assert_eq!(crate::a11y::document_token(&parsed), Some(4294967296));
431        assert_eq!(document_token(&c, "C1").await.unwrap(), 4294967296);
432        assert_eq!(
433            node_clip_rect(&c, "C1", 1).await.unwrap(),
434            json!({"x": 10, "y": 430, "width": 100, "height": 20})
435        );
436        assert_eq!(document_size(&c, "C1").await.unwrap(), (1000.0, 3000.0));
437    }
438}
browser_control/session/input_bidi.rs

browser_control/session/
input_bidi.rs
