browser_control/session/
input_bidi.rs

1//! WebDriver BiDi (Firefox) counterpart of [`crate::session::input`].
2//!
3//! Element refs on BiDi are integer ids in a page-side registry written by
4//! the snapshot walker (`crate::dom::scripts::SNAPSHOT_TREE_JS`). Geometry
5//! is computed in the page (`REF_CENTER_JS`, `REF_CLIP_RECT_JS`) and input
6//! is dispatched with `input.performActions` at viewport-origin CSS pixel
7//! coordinates, so no `locateNodes` / `sharedId` bookkeeping is needed.
8//! Typing goes through in-page `execCommand('insertText')` (with a
9//! value-setter fallback) because BiDi has no `insertText`; per-character
10//! key actions cover `press_sequentially`, and Enter is a key action.
11//!
12//! Every helper returns `SessionError::NodeGone` when the registry no longer
13//! resolves the id, which the tool layer maps to `StaleRef`.
14
15use anyhow::{anyhow, Context, Result};
16use serde_json::{json, Value};
17
18use crate::bidi::BidiClient;
19use crate::dom::scripts::{
20    DOC_SIZE_JS, DOC_TOKEN_JS, REF_CENTER_JS, REF_CLIP_RECT_JS, REF_TYPE_JS, SNAPSHOT_TREE_JS,
21};
22use crate::errors::SessionError;
23use crate::session::input::Point;
24use crate::session::keys::Chord;
25
26/// Node budget handed to the walker.
27const SNAPSHOT_MAX_NODES: u64 = 20_000;
28/// WebDriver key code point for Enter.
29const ENTER: &str = "\u{e007}";
30/// Interpolated pointer moves during a drag.
31const DRAG_STEPS: usize = 5;
32
33/// The `value` of a string `RemoteValue`.
34fn remote_string(v: &Value) -> Result<String> {
35    v["value"]
36        .as_str()
37        .map(String::from)
38        .ok_or_else(|| anyhow!("script returned no string result: {v}"))
39}
40
41/// Decode a ref helper's JSON string: `{"gone":true}` → `NodeGone`,
42/// `{"error":…}` → error, otherwise the payload.
43fn decode_ref_result(id: u64, op: &'static str, s: &str) -> Result<Value> {
44    let v: Value = serde_json::from_str(s)
45        .with_context(|| format!("{op} on node {id}: invalid helper result"))?;
46    if v["gone"].as_bool().unwrap_or(false) {
47        return Err(SessionError::NodeGone {
48            backend_node_id: id,
49            details: format!("{op}: ref id {id} is not in the page registry or was detached"),
50        }
51        .into());
52    }
53    if let Some(e) = v["error"].as_str() {
54        return Err(anyhow!("{op} on node {id}: {e}"));
55    }
56    Ok(v)
57}
58
59async fn call_ref(
60    c: &BidiClient,
61    ctx: &str,
62    id: u64,
63    op: &'static str,
64    script: &str,
65    extra: Vec<Value>,
66) -> Result<Value> {
67    let mut args = vec![json!(id)];
68    args.extend(extra);
69    let v = c.script_call_function(ctx, script, args).await?;
70    decode_ref_result(id, op, &remote_string(&v)?)
71}
72
73fn pointer_move(p: Point) -> Value {
74    json!({
75        "type": "pointerMove",
76        "x": p.x.round() as i64,
77        "y": p.y.round() as i64,
78        "origin": "viewport",
79    })
80}
81
82fn pointer_source(actions: Vec<Value>) -> Value {
83    json!({
84        "type": "pointer",
85        "id": "mouse",
86        "parameters": { "pointerType": "mouse" },
87        "actions": actions,
88    })
89}
90
91fn key_source(actions: Vec<Value>) -> Value {
92    json!({ "type": "key", "id": "kb", "actions": actions })
93}
94
95fn key_press(value: &str) -> [Value; 2] {
96    [
97        json!({ "type": "keyDown", "value": value }),
98        json!({ "type": "keyUp", "value": value }),
99    ]
100}
101
102/// Run the walker and parse its JSON.
103pub async fn accessibility_tree(c: &BidiClient, ctx: &str) -> Result<Value> {
104    let v = c
105        .script_call_function(ctx, SNAPSHOT_TREE_JS, vec![json!(SNAPSHOT_MAX_NODES)])
106        .await?;
107    let tree: Value = serde_json::from_str(&remote_string(&v)?)
108        .context("accessibility walker returned invalid JSON")?;
109    if tree["truncated"].as_bool().unwrap_or(false) {
110        tracing::warn!(
111            target = %ctx,
112            "accessibility snapshot truncated at {SNAPSHOT_MAX_NODES} nodes"
113        );
114    }
115    Ok(tree)
116}
117
118/// Per-document token (see `DOC_TOKEN_JS`).
119pub async fn document_token(c: &BidiClient, ctx: &str) -> Result<u64> {
120    let v = c.script_evaluate(ctx, DOC_TOKEN_JS).await?;
121    let v = crate::bidi::unwrap_script_result(v)?;
122    remote_string(&v)?
123        .parse::<u64>()
124        .context("document token is not an integer")
125}
126
127/// Scroll the node into view and return its clickable centre.
128pub async fn node_center(c: &BidiClient, ctx: &str, id: u64) -> Result<Point> {
129    let v = call_ref(c, ctx, id, "center", REF_CENTER_JS, vec![]).await?;
130    Ok(Point {
131        x: v["x"].as_f64().unwrap_or(0.0),
132        y: v["y"].as_f64().unwrap_or(0.0),
133    })
134}
135
136/// Left-click the node's centre.
137pub async fn click(c: &BidiClient, ctx: &str, id: u64) -> Result<Point> {
138    let p = node_center(c, ctx, id).await?;
139    c.input_perform_actions(
140        ctx,
141        json!([pointer_source(vec![
142            pointer_move(p),
143            json!({ "type": "pointerDown", "button": 0 }),
144            json!({ "type": "pointerUp", "button": 0 }),
145        ])]),
146    )
147    .await?;
148    let _ = c.input_release_actions(ctx).await;
149    Ok(p)
150}
151
152/// Move the pointer over the node's centre.
153pub async fn hover(c: &BidiClient, ctx: &str, id: u64) -> Result<Point> {
154    let p = node_center(c, ctx, id).await?;
155    c.input_perform_actions(ctx, json!([pointer_source(vec![pointer_move(p)])]))
156        .await?;
157    Ok(p)
158}
159
160/// Focus the node, replace its content with `text`, optionally one key at a
161/// time, and optionally press Enter.
162pub async fn type_text(
163    c: &BidiClient,
164    ctx: &str,
165    id: u64,
166    text: &str,
167    press_sequentially: bool,
168    submit: bool,
169) -> Result<()> {
170    let mode = if text.is_empty() {
171        "clear"
172    } else if press_sequentially {
173        "select"
174    } else {
175        "fill"
176    };
177    call_ref(
178        c,
179        ctx,
180        id,
181        "type",
182        REF_TYPE_JS,
183        vec![json!(text), json!(mode)],
184    )
185    .await?;
186    if press_sequentially && !text.is_empty() {
187        let mut keys = Vec::new();
188        for ch in text.chars() {
189            let v = match ch {
190                '\n' | '\r' => ENTER.to_string(),
191                other => other.to_string(),
192            };
193            keys.extend(key_press(&v));
194        }
195        c.input_perform_actions(ctx, json!([key_source(keys)]))
196            .await?;
197    }
198    if submit {
199        c.input_perform_actions(ctx, json!([key_source(key_press(ENTER).to_vec())]))
200            .await?;
201    }
202    let _ = c.input_release_actions(ctx).await;
203    Ok(())
204}
205
206/// Press and release a key, with any modifiers held around it.
207///
208/// BiDi tracks modifier state from the keyDown/keyUp pairs itself, so unlike
209/// CDP there is no bitmask — the nesting order *is* the state. Modifiers are
210/// released in reverse for the same reason as CDP: a key left down leaks into
211/// whatever the page does next.
212///
213/// `input.releaseActions` runs afterwards regardless, so a failed action
214/// sequence cannot strand the browser with a modifier held.
215pub async fn press_key(c: &BidiClient, ctx: &str, chord: &Chord) -> Result<()> {
216    let mut actions = Vec::new();
217    for m in &chord.modifiers {
218        actions.push(json!({ "type": "keyDown", "value": m.def().bidi }));
219    }
220    actions.extend(key_press(chord.key.bidi));
221    for m in chord.modifiers.iter().rev() {
222        actions.push(json!({ "type": "keyUp", "value": m.def().bidi }));
223    }
224    let result = c
225        .input_perform_actions(ctx, json!([key_source(actions)]))
226        .await;
227    let _ = c.input_release_actions(ctx).await;
228    result
229}
230
231/// Type into whatever currently has focus, with no node id.
232///
233/// BiDi has no "insert text at the caret" primitive, so this sends the value
234/// as key actions — which is what typing into focus means on this engine.
235/// Unlike the CDP path there is no select-all first: without a node handle
236/// there is nothing to select, so the caller should clear the field before
237/// piping into it if replacement is wanted.
238pub async fn type_focused(c: &BidiClient, ctx: &str, text: &str, submit: bool) -> Result<()> {
239    let mut keys = Vec::new();
240    for ch in text.chars() {
241        let v = match ch {
242            '\n' | '\r' => ENTER.to_string(),
243            other => other.to_string(),
244        };
245        keys.extend(key_press(&v));
246    }
247    if !keys.is_empty() {
248        c.input_perform_actions(ctx, json!([key_source(keys)]))
249            .await?;
250    }
251    if submit {
252        c.input_perform_actions(ctx, json!([key_source(key_press(ENTER).to_vec())]))
253            .await?;
254    }
255    let _ = c.input_release_actions(ctx).await;
256    Ok(())
257}
258
259/// Pointer drag from one node's centre to another's.
260pub async fn drag(c: &BidiClient, ctx: &str, from: u64, to: u64) -> Result<()> {
261    let a = node_center(c, ctx, from).await?;
262    let b = node_center(c, ctx, to).await?;
263    let mut actions = vec![
264        pointer_move(a),
265        json!({ "type": "pointerDown", "button": 0 }),
266    ];
267    for i in 1..=DRAG_STEPS {
268        let t = i as f64 / DRAG_STEPS as f64;
269        actions.push(pointer_move(Point {
270            x: a.x + (b.x - a.x) * t,
271            y: a.y + (b.y - a.y) * t,
272        }));
273    }
274    actions.push(pointer_move(b));
275    actions.push(json!({ "type": "pointerUp", "button": 0 }));
276    c.input_perform_actions(ctx, json!([pointer_source(actions)]))
277        .await?;
278    let _ = c.input_release_actions(ctx).await;
279    Ok(())
280}
281
282/// Border box of the node in document coordinates.
283pub async fn node_clip_rect(c: &BidiClient, ctx: &str, id: u64) -> Result<Value> {
284    call_ref(c, ctx, id, "clip", REF_CLIP_RECT_JS, vec![]).await
285}
286
287/// Document scroll size, for full-page screenshots.
288pub async fn document_size(c: &BidiClient, ctx: &str) -> Result<(f64, f64)> {
289    let v = c.script_evaluate(ctx, DOC_SIZE_JS).await?;
290    let v = crate::bidi::unwrap_script_result(v)?;
291    let parsed: Value = serde_json::from_str(&remote_string(&v)?)
292        .context("document size helper returned invalid JSON")?;
293    Ok((
294        parsed["width"].as_f64().unwrap_or(0.0),
295        parsed["height"].as_f64().unwrap_or(0.0),
296    ))
297}
298
299#[cfg(test)]
300mod tests {
301    use super::*;
302    use futures_util::{SinkExt, StreamExt};
303    use std::sync::{Arc, Mutex};
304    use tokio_tungstenite::tungstenite::Message;
305
306    /// BiDi-framed recording mock that dispatches `script.callFunction` on
307    /// the helper markers and records input commands.
308    async fn spawn_mock(gone: bool) -> (String, Arc<Mutex<Vec<Value>>>) {
309        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
310        let addr = listener.local_addr().unwrap();
311        let seen = Arc::new(Mutex::new(Vec::<Value>::new()));
312        tokio::spawn({
313            let seen = seen.clone();
314            async move {
315                let (stream, _) = listener.accept().await.unwrap();
316                let mut ws = tokio_tungstenite::accept_async(stream).await.unwrap();
317                while let Some(Ok(Message::Text(t))) = ws.next().await {
318                    let req: Value = serde_json::from_str(&t).unwrap();
319                    seen.lock().unwrap().push(req.clone());
320                    let id = req["id"].as_u64().unwrap();
321                    let method = req["method"].as_str().unwrap_or("");
322                    let decl = req["params"]["functionDeclaration"].as_str().unwrap_or("");
323                    let expr = req["params"]["expression"].as_str().unwrap_or("");
324                    let string = |s: String| json!({"type": "success", "result": {"type": "string", "value": s}, "realm": "R1"});
325                    let result = match method {
326                        "script.callFunction" if gone => string("{\"gone\":true}".into()),
327                        "script.callFunction" if decl.contains("bc:center") => {
328                            string("{\"x\":30.4,\"y\":20}".into())
329                        }
330                        "script.callFunction" if decl.contains("bc:clip") => {
331                            string("{\"x\":10,\"y\":430,\"width\":100,\"height\":20}".into())
332                        }
333                        "script.callFunction" if decl.contains("bc:type") => {
334                            string("{\"kind\":\"field\",\"method\":\"execCommand\"}".into())
335                        }
336                        "script.callFunction" if decl.contains("bc:snapshot") => string(
337                            json!({"nodes": [
338                                {"nodeId": "root", "backendDOMNodeId": 4294967296u64,
339                                 "role": {"value": "RootWebArea"}, "name": {"value": "T"}, "childIds": ["n1"]},
340                                {"nodeId": "n1", "parentId": "root", "backendDOMNodeId": 1,
341                                 "role": {"value": "button"}, "name": {"value": "Go"}, "childIds": [],
342                                 "properties": [{"name": "focusable", "value": {"value": true}}]}
343                            ], "truncated": false})
344                            .to_string(),
345                        ),
346                        "script.evaluate" if expr.contains("__bcDocToken") => {
347                            string("4294967296".into())
348                        }
349                        "script.evaluate" if expr.contains("scrollWidth") => {
350                            string("{\"width\":1000,\"height\":3000}".into())
351                        }
352                        _ => json!({}),
353                    };
354                    ws.send(Message::Text(
355                        json!({"type": "success", "id": id, "result": result}).to_string(),
356                    ))
357                    .await
358                    .unwrap();
359                }
360            }
361        });
362        (format!("ws://{addr}"), seen)
363    }
364
365    fn input_calls(seen: &[Value]) -> Vec<Value> {
366        seen.iter()
367            .filter(|r| r["method"] == "input.performActions")
368            .map(|r| r["params"]["actions"].clone())
369            .collect()
370    }
371
372    #[tokio::test]
373    async fn click_moves_presses_releases_then_releases_actions() {
374        let (url, seen) = spawn_mock(false).await;
375        let c = BidiClient::connect(&url).await.unwrap();
376        let p = click(&c, "C1", 7).await.unwrap();
377        assert_eq!(p, Point { x: 30.4, y: 20.0 });
378        let seen = seen.lock().unwrap();
379        let first = &seen[0];
380        assert_eq!(first["method"], "script.callFunction");
381        assert_eq!(
382            first["params"]["arguments"][0],
383            json!({"type": "number", "value": 7})
384        );
385        let acts = input_calls(&seen);
386        assert_eq!(acts.len(), 1);
387        let pointer = &acts[0][0];
388        assert_eq!(pointer["type"], "pointer");
389        assert_eq!(pointer["parameters"]["pointerType"], "mouse");
390        assert_eq!(
391            pointer["actions"],
392            json!([
393                {"type": "pointerMove", "x": 30, "y": 20, "origin": "viewport"},
394                {"type": "pointerDown", "button": 0},
395                {"type": "pointerUp", "button": 0},
396            ])
397        );
398        assert_eq!(seen.last().unwrap()["method"], "input.releaseActions");
399    }
400
401    #[tokio::test]
402    async fn type_fill_then_enter() {
403        let (url, seen) = spawn_mock(false).await;
404        let c = BidiClient::connect(&url).await.unwrap();
405        type_text(&c, "C1", 3, "hi", false, true).await.unwrap();
406        let seen = seen.lock().unwrap();
407        assert_eq!(
408            seen[0]["params"]["arguments"],
409            json!([
410                {"type": "number", "value": 3},
411                {"type": "string", "value": "hi"},
412                {"type": "string", "value": "fill"}
413            ])
414        );
415        let acts = input_calls(&seen);
416        assert_eq!(acts.len(), 1, "fill sends no key actions; only Enter");
417        assert_eq!(acts[0][0]["type"], "key");
418        assert_eq!(acts[0][0]["actions"][0]["value"], ENTER);
419        assert_eq!(acts[0][0]["actions"][1]["type"], "keyUp");
420    }
421
422    #[tokio::test]
423    async fn type_sequentially_emits_per_char_keys_and_clear_sends_none() {
424        let (url, seen) = spawn_mock(false).await;
425        let c = BidiClient::connect(&url).await.unwrap();
426        type_text(&c, "C1", 3, "a\n", true, false).await.unwrap();
427        type_text(&c, "C1", 3, "", false, false).await.unwrap();
428        let seen = seen.lock().unwrap();
429        assert_eq!(seen[0]["params"]["arguments"][2]["value"], "select");
430        let acts = input_calls(&seen);
431        assert_eq!(acts.len(), 1);
432        let keys = &acts[0][0]["actions"];
433        assert_eq!(keys.as_array().unwrap().len(), 4);
434        assert_eq!(keys[0]["value"], "a");
435        assert_eq!(keys[2]["value"], ENTER);
436        let clear = seen
437            .iter()
438            .filter(|r| r["method"] == "script.callFunction")
439            .nth(1)
440            .unwrap();
441        assert_eq!(clear["params"]["arguments"][2]["value"], "clear");
442    }
443
444    #[tokio::test]
445    async fn drag_sequence() {
446        let (url, seen) = spawn_mock(false).await;
447        let c = BidiClient::connect(&url).await.unwrap();
448        drag(&c, "C1", 1, 2).await.unwrap();
449        let seen = seen.lock().unwrap();
450        let acts = input_calls(&seen);
451        let types: Vec<&str> = acts[0][0]["actions"]
452            .as_array()
453            .unwrap()
454            .iter()
455            .map(|a| a["type"].as_str().unwrap())
456            .collect();
457        assert_eq!(types[0], "pointerMove");
458        assert_eq!(types[1], "pointerDown");
459        assert_eq!(*types.last().unwrap(), "pointerUp");
460        assert_eq!(types.len(), 2 + DRAG_STEPS + 1 + 1);
461    }
462
463    #[tokio::test]
464    async fn gone_maps_to_node_gone() {
465        let (url, _seen) = spawn_mock(true).await;
466        let c = BidiClient::connect(&url).await.unwrap();
467        let err = hover(&c, "C1", 9).await.unwrap_err();
468        match err.downcast_ref::<SessionError>() {
469            Some(SessionError::NodeGone {
470                backend_node_id, ..
471            }) => {
472                assert_eq!(*backend_node_id, 9)
473            }
474            other => panic!("expected NodeGone, got {other:?}"),
475        }
476    }
477
478    #[tokio::test]
479    async fn tree_token_clip_and_document_size() {
480        let (url, _seen) = spawn_mock(false).await;
481        let c = BidiClient::connect(&url).await.unwrap();
482        let tree = accessibility_tree(&c, "C1").await.unwrap();
483        let parsed = crate::a11y::parse_full_ax_tree(&tree).unwrap();
484        assert_eq!(crate::a11y::document_token(&parsed), Some(4294967296));
485        assert_eq!(document_token(&c, "C1").await.unwrap(), 4294967296);
486        assert_eq!(
487            node_clip_rect(&c, "C1", 1).await.unwrap(),
488            json!({"x": 10, "y": 430, "width": 100, "height": 20})
489        );
490        assert_eq!(document_size(&c, "C1").await.unwrap(), (1000.0, 3000.0));
491    }
492}
browser_control/session/input_bidi.rs

browser_control/session/
input_bidi.rs
