glass/browser/session/
action.rs

1//! Action primitives: clicks, typing, keyboard, scroll, drag.
2//!
3//! Implementation of individual browser interaction actions on
4//! [`BrowserSession`]: click, double-click, hover, drag, key press,
5//! scroll, clear, check, uncheck, select, and file upload.
6
7use super::*;
8
9impl BrowserSession {
10    /// Click exact frame-local viewport coordinates. This is an explicit,
11    /// policy-gated escape hatch for canvas and map surfaces where no DOM
12    /// control can be published. Coordinates are validated against the live
13    /// viewport and are never adjusted to a nearby element.
14    pub async fn click_at(&self, x: f64, y: f64) -> BrowserResult<CoordinateClickOutcome> {
15        self.policy
16            .require(crate::browser::policy::PolicyCapability::CoordinateClick)?;
17        if !x.is_finite() || !y.is_finite() || x < 0.0 || y < 0.0 {
18            return Err("click-at coordinates must be finite and non-negative".into());
19        }
20
21        self.cdp
22            .with_current_route(async {
23                let hit = self
24                    .evaluate_value(&format!(
25                        "(() => {{ if ({x} >= innerWidth || {y} >= innerHeight) return null; const e = document.elementFromPoint({x}, {y}); if (!e) return null; return {{tag:e.tagName.toLowerCase(), role:e.getAttribute('role'), name:e.getAttribute('aria-label') || e.textContent?.trim().slice(0, 160) || null}}; }})()"
26                    ))
27                    .await?;
28                let hit = if hit.is_null() {
29                    None
30                } else {
31                    Some(CoordinateHit {
32                        tag: hit["tag"].as_str().unwrap_or("unknown").to_string(),
33                        role: hit["role"].as_str().map(str::to_string),
34                        name: hit["name"].as_str().map(str::to_string),
35                    })
36                };
37                if hit.is_none() {
38                    return Err("click-at coordinates are outside the viewport or hit no element".into());
39                }
40                self.cdp
41                    .dispatch_mouse_event("mouseMoved", x, y, None, None)
42                    .await?;
43                self.cdp
44                    .dispatch_mouse_event("mousePressed", x, y, Some("left"), Some(1))
45                    .await?;
46                self.cdp
47                    .dispatch_mouse_event("mouseReleased", x, y, Some("left"), Some(1))
48                    .await?;
49                let (target_id, frame_id) = self.ensured_route_identity().await?;
50                Ok(CoordinateClickOutcome {
51                    x,
52                    y,
53                    hit,
54                    revision: self.invalidate_observation(),
55                    target_id,
56                    frame_id,
57                })
58            })
59            .await
60    }
61
62    /// Scroll the viewport by the given pixel offsets.
63    ///
64    /// Positive `dy` scrolls down; positive `dx` scrolls right.
65    pub async fn scroll(&self, dx: f64, dy: f64) -> BrowserResult<ActionOutcome> {
66        self.cdp
67            .with_current_route(async {
68                self.cdp.scroll_by(dx, dy).await?;
69                let (target_id, frame_id) = self.ensured_route_identity().await?;
70                Ok(ActionOutcome {
71                    action: ActionKind::Scroll,
72                    target: None,
73                    revision: self.invalidate_observation(),
74                    target_id,
75                    frame_id,
76                    evidence: None,
77                })
78            })
79            .await
80    }
81
82    /// Capture the full accessibility tree snapshot for the current page.
83    ///
84    /// Returns the page info, accessibility roots, and all interactive elements.
85    /// Prefer [`observe`](BrowserSession::observe) for compact observations in
86    /// agent workflows.
87    pub async fn snapshot(&self) -> BrowserResult<AccessibilitySnapshot> {
88        self.cdp
89            .with_current_route(async {
90                let revision = self.page_revision.load(Ordering::Relaxed);
91                let raw = self.cdp.get_accessibility_tree().await?;
92                let roots = parse_accessibility_tree(&raw);
93                let interactive = interactive_elements(&roots, revision);
94                Ok(AccessibilitySnapshot {
95                    page: self.page_info().await?,
96                    roots,
97                    interactive,
98                })
99            })
100            .await
101    }
102
103    /// Click an element and return its structured action outcome.
104    pub async fn click(&self, target: &str) -> BrowserResult<ActionOutcome> {
105        self.pointer_click(target, false).await
106    }
107
108    /// Double-click an element with the same target, scroll, and pointer
109    /// contract as a single click.
110    pub async fn double_click(&self, target: &str) -> BrowserResult<ActionOutcome> {
111        self.pointer_click(target, true).await
112    }
113
114    /// Hover the pointer over an element without clicking.
115    ///
116    /// Resolves the target, moves the pointer to the element's center using the
117    /// configured interaction mode, then returns an [`ActionOutcome`].
118    pub async fn hover(&self, target: &str) -> BrowserResult<ActionOutcome> {
119        self.cdp
120            .with_current_route(async {
121                let element = self.resolve_element(target).await?;
122                let object_id = self
123                    .cdp
124                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
125                    .await?;
126                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
127                let local = self.verified_action_point(&remote.object_id).await?;
128                let point = self.target_viewport_point(local).await?;
129                self.move_pointer(point).await?;
130                self.action_outcome(ActionKind::Hover, Some(element), None)
131                    .await
132            })
133            .await
134    }
135
136    /// Drag an element from `source` to `destination`.
137    ///
138    /// Performs a mouse-press on the source element, moves the pointer to the
139    /// destination element, then releases.
140    pub async fn drag(&self, source: &str, destination: &str) -> BrowserResult<ActionOutcome> {
141        self.cdp
142            .with_current_route(async {
143                let source = self.resolve_element(source).await?;
144                let source_object = self
145                    .cdp
146                    .resolve_node_object(source.node_id, source.backend_dom_node_id)
147                    .await?;
148                let source_guard = RemoteObjectGuard::new(self.cdp.clone(), source_object);
149                let destination = self.resolve_element(destination).await?;
150                let destination_object = self
151                    .cdp
152                    .resolve_node_object(destination.node_id, destination.backend_dom_node_id)
153                    .await?;
154                let destination_guard =
155                    RemoteObjectGuard::new(self.cdp.clone(), destination_object);
156                let source_local = self.verified_action_point(&source_guard.object_id).await?;
157                let destination_local = self
158                    .verified_action_point(&destination_guard.object_id)
159                    .await?;
160                let source_point = self.target_viewport_point(source_local).await?;
161                let destination_point = self.target_viewport_point(destination_local).await?;
162                self.move_pointer(source_point).await?;
163                let verified_source = self.verified_action_point(&source_guard.object_id).await?;
164                if (verified_source.x - source_local.x).abs() > 1.0
165                    || (verified_source.y - source_local.y).abs() > 1.0
166                {
167                    return Err(TargetError {
168                        kind: TargetErrorKind::NotActionable,
169                        reason: Some(TargetActionabilityReason::GeometryChanged),
170                        candidates: Vec::new(),
171                        recovery: None,
172                    }
173                    .into());
174                }
175                self.cdp
176                    .dispatch_mouse_event(
177                        "mousePressed",
178                        source_point.x,
179                        source_point.y,
180                        Some("left"),
181                        Some(1),
182                    )
183                    .await?;
184                let mut pressed = PressedButtonGuard {
185                    cdp: self.cdp.clone(),
186                    point: source_point,
187                    click_count: 1,
188                    armed: true,
189                };
190                let drag_path = interaction_path(
191                    self.interaction_mode,
192                    &self.mouse,
193                    source_point,
194                    destination_point,
195                );
196                for window in drag_path.windows(2) {
197                    let point = window[1];
198                    if self.interaction_mode == InteractionMode::Human {
199                        tokio::time::sleep(self.mouse.move_delay(window[0], point)).await;
200                    }
201                    self.cdp
202                        .dispatch_mouse_event("mouseMoved", point.x, point.y, Some("left"), Some(1))
203                        .await?;
204                }
205                let verified_destination = self
206                    .verified_action_point(&destination_guard.object_id)
207                    .await?;
208                if (verified_destination.x - destination_local.x).abs() > 1.0
209                    || (verified_destination.y - destination_local.y).abs() > 1.0
210                {
211                    return Err(TargetError {
212                        kind: TargetErrorKind::NotActionable,
213                        reason: Some(TargetActionabilityReason::GeometryChanged),
214                        candidates: Vec::new(),
215                        recovery: None,
216                    }
217                    .into());
218                }
219                self.cdp
220                    .dispatch_mouse_event(
221                        "mouseReleased",
222                        destination_point.x,
223                        destination_point.y,
224                        Some("left"),
225                        Some(1),
226                    )
227                    .await?;
228                pressed.armed = false;
229                *self.pointer.lock().await = Some(destination_point);
230                self.action_outcome(ActionKind::Drag, Some(source), None)
231                    .await
232            })
233            .await
234    }
235
236    /// Press and hold a keyboard key.
237    ///
238    /// Dispatches a `rawKeyDown` CDP event for the given key.
239    pub async fn key_down(&self, key: &str) -> BrowserResult<ActionOutcome> {
240        self.keyboard_action(ActionKind::KeyDown, key, "rawKeyDown", 0)
241            .await
242    }
243
244    /// Release a keyboard key.
245    ///
246    /// Dispatches a `keyUp` CDP event for the given key.
247    pub async fn key_up(&self, key: &str) -> BrowserResult<ActionOutcome> {
248        self.keyboard_action(ActionKind::KeyUp, key, "keyUp", 0)
249            .await
250    }
251
252    /// Press and release a keyboard key.
253    ///
254    /// Dispatches `rawKeyDown`, `char` (for single-character keys), and `keyUp`
255    /// CDP events.
256    pub async fn key_press(&self, key: &str) -> BrowserResult<ActionOutcome> {
257        validate_key(key)?;
258        self.cdp
259            .with_current_route(async {
260                let code = key_code(key);
261                self.cdp
262                    .dispatch_key_event_with_modifiers("rawKeyDown", key, &code, "", 0)
263                    .await?;
264                if key.chars().count() == 1 {
265                    self.cdp
266                        .dispatch_key_event_with_modifiers("char", key, &code, key, 0)
267                        .await?;
268                }
269                self.cdp
270                    .dispatch_key_event_with_modifiers("keyUp", key, &code, "", 0)
271                    .await?;
272                self.action_outcome(ActionKind::KeyPress, None, None).await
273            })
274            .await
275    }
276
277    /// Execute a keyboard shortcut with modifier keys.
278    ///
279    /// Parses shortcuts like `"Ctrl+C"` or `"Meta+V"` and dispatches
280    /// the corresponding key events with the specified modifiers.
281    pub async fn shortcut(&self, shortcut: &str) -> BrowserResult<ActionOutcome> {
282        let (modifiers, key) = parse_shortcut(shortcut)?;
283        self.cdp
284            .with_current_route(async {
285                let code = key_code(&key);
286                self.cdp
287                    .dispatch_key_event_with_modifiers("rawKeyDown", &key, &code, "", modifiers)
288                    .await?;
289                self.cdp
290                    .dispatch_key_event_with_modifiers("keyUp", &key, &code, "", modifiers)
291                    .await?;
292                self.action_outcome(ActionKind::Shortcut, None, None).await
293            })
294            .await
295    }
296
297    /// Clear the contents of an editable element.
298    ///
299    /// Clicks the target, selects all content, then presses Backspace.
300    /// Verifies the element is empty afterward.
301    pub async fn clear(&self, target: &str) -> BrowserResult<ActionOutcome> {
302        self.cdp
303            .with_current_route(async {
304                let element = self.resolve_element(target).await?;
305                let object_id = self.cdp.resolve_node_object(element.node_id, element.backend_dom_node_id).await?;
306                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
307                let editable = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement || this.isContentEditable}").await?)?;
308                if editable.as_bool() != Some(true) { return Err("clear target is not editable".into()); }
309                let clicked = self.click(target).await?;
310                self.cdp.dispatch_select_all().await?;
311                self.key_press("Backspace").await?;
312                let empty = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement ? this.value === '' : this.textContent === ''}").await?)?;
313                if empty.as_bool() != Some(true) { return Err("clear target did not become empty".into()); }
314                self.action_outcome_from_target(ActionKind::Clear, clicked.target)
315                    .await
316            })
317            .await
318    }
319
320    /// Check a checkbox or radio button.
321    ///
322    /// Ensures the target element's `checked` property is set to `true`.
323    pub async fn check(&self, target: &str) -> BrowserResult<ActionOutcome> {
324        self.set_checked(target, true).await
325    }
326
327    /// Uncheck a checkbox.
328    ///
329    /// Ensures the target element's `checked` property is set to `false`.
330    pub async fn uncheck(&self, target: &str) -> BrowserResult<ActionOutcome> {
331        self.set_checked(target, false).await
332    }
333
334    /// Select an option from a `<select>` element by value.
335    ///
336    /// `value` must be 1–4096 bytes. Fires `input` and `change` events.
337    pub async fn select_option(&self, target: &str, value: &str) -> BrowserResult<ActionOutcome> {
338        if value.is_empty() || value.len() > 4096 {
339            return Err("select value must be 1..=4096 bytes".into());
340        }
341        let value_json = serde_json::to_string(value)?;
342        self.form_object_action(target, ActionKind::Select, &format!(r#"function() {{ if (!(this instanceof HTMLSelectElement)) return {{ok:false,reason:'not_select'}}; const option = Array.from(this.options).find(option => option.value === {value_json}); if (!option) return {{ok:false,reason:'option_not_found'}}; this.value = option.value; this.dispatchEvent(new Event('input',{{bubbles:true}})); this.dispatchEvent(new Event('change',{{bubbles:true}})); return {{ok:this.value === option.value}}; }}"#)).await
343    }
344
345    pub async fn upload_files(
346        &self,
347        target: &str,
348        paths: &[PathBuf],
349    ) -> BrowserResult<ActionOutcome> {
350        self.policy.require(PolicyCapability::Upload)?;
351        self.cdp.with_current_route(async {
352            if paths.is_empty() || paths.len() > 16 { return Err("upload requires 1..=16 files".into()); }
353            let mut files = Vec::with_capacity(paths.len());
354            for path in paths {
355                let canonical = self.policy.require_existing_path(path)?;
356                if !canonical.is_file() { return Err("upload path must be a regular file".into()); }
357                if !canonical.starts_with(&self.upload_root) { return Err("upload path is outside the allowed workspace root".into()); }
358                files.push(canonical.to_string_lossy().into_owned());
359            }
360            let element = self.resolve_element(target).await?;
361            let object_id = self.cdp.resolve_node_object(element.node_id, element.backend_dom_node_id).await?;
362            let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
363            self.verified_action_point(&remote.object_id).await?;
364            let input = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return {ok:this instanceof HTMLInputElement && this.type === 'file'}}").await?)?;
365            if input["ok"].as_bool() != Some(true) { return Err("upload target is not a file input".into()); }
366            if element.node_id.is_none() && element.backend_dom_node_id.is_none() { return Err("file input target has no DOM node ID".into()); }
367            self.cdp.set_file_input_files(element.node_id, element.backend_dom_node_id, &files).await?;
368            let verified = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this.files.length}").await?)?;
369            if verified.as_u64() != Some(files.len() as u64) { return Err("file input did not retain the requested file count".into()); }
370            let outcome = self.action_outcome(ActionKind::Upload, Some(element), Some(serde_json::json!({"file_count": files.len()}))).await?;
371            self.record_audit("upload", format!("{} files", files.len()));
372            Ok(outcome)
373        }).await
374    }
375
376    async fn resolve_click_target(
377        &self,
378        target: &str,
379    ) -> BrowserResult<(ResolvedElement, String, Point)> {
380        const MAX_NODE_RESOLUTION_ATTEMPTS: usize = 3;
381        let mut last_error: Option<Box<dyn Error>> = None;
382
383        for attempt in 0..MAX_NODE_RESOLUTION_ATTEMPTS {
384            let element = self.resolve_element(target).await?;
385            let object_id = match self
386                .cdp
387                .resolve_node_object(element.node_id, element.backend_dom_node_id)
388                .await
389            {
390                Ok(object_id) => object_id,
391                Err(error) => {
392                    tracing::debug!(%error, attempt, "target node could not be resolved");
393                    last_error = Some(Box::new(TargetError {
394                        kind: TargetErrorKind::NotActionable,
395                        reason: Some(TargetActionabilityReason::NodeUnavailable),
396                        candidates: Vec::new(),
397                        recovery: None,
398                    }));
399                    if attempt + 1 < MAX_NODE_RESOLUTION_ATTEMPTS {
400                        tokio::time::sleep(Duration::from_millis(50)).await;
401                        continue;
402                    }
403                    break;
404                }
405            };
406
407            let local_point = match self.verified_action_point(&object_id).await {
408                Ok(point) => point,
409                Err(error)
410                    if error
411                        .downcast_ref::<TargetError>()
412                        .and_then(|target_error| target_error.reason)
413                        == Some(TargetActionabilityReason::NodeUnavailable) =>
414                {
415                    last_error = Some(error);
416                    if attempt + 1 < MAX_NODE_RESOLUTION_ATTEMPTS {
417                        tokio::time::sleep(Duration::from_millis(50)).await;
418                        continue;
419                    }
420                    break;
421                }
422                Err(error) => return Err(error),
423            };
424
425            return Ok((element, object_id, local_point));
426        }
427
428        Err(last_error.expect("node resolution retry loop must retain its last error"))
429    }
430
431    async fn pointer_click(
432        &self,
433        target: &str,
434        double_click: bool,
435    ) -> BrowserResult<ActionOutcome> {
436        self.cdp
437            .with_current_route(async {
438                let (element, object_id, local_point) = self.resolve_click_target(target).await?;
439                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
440                let point = self.target_viewport_point(local_point).await?;
441                let events = if double_click {
442                    self.mouse.generate_double_click_events(point)
443                } else {
444                    self.mouse.generate_click_events(point)
445                };
446                self.dispatch_pointer_events(&remote.object_id, local_point, point, events)
447                    .await?;
448                let (target_id, frame_id) = self.route_identity().await?;
449                Ok(ActionOutcome {
450                    action: if double_click {
451                        ActionKind::DoubleClick
452                    } else {
453                        ActionKind::Click
454                    },
455                    target: Some(ActionTarget {
456                        label: element.label,
457                        reference: element.reference,
458                    }),
459                    revision: self.invalidate_observation(),
460                    target_id,
461                    frame_id,
462                    evidence: None,
463                })
464            })
465            .await
466    }
467
468    async fn dispatch_pointer_events(
469        &self,
470        object_id: &str,
471        local_point: Point,
472        point: Point,
473        events: Vec<crate::browser::mouse::MouseEvent>,
474    ) -> BrowserResult<()> {
475        let mut pointer = self.pointer.lock().await;
476        let start = match (self.interaction_mode, *pointer) {
477            (_, Some(point)) => point,
478            (InteractionMode::Human, None) => self
479                .viewport_center()
480                .await
481                .unwrap_or(Point { x: 640.0, y: 360.0 }),
482            (InteractionMode::Fast, None) => point,
483        };
484        let path = interaction_path(self.interaction_mode, &self.mouse, start, point);
485        if self.interaction_mode == InteractionMode::Human && pointer.is_none() {
486            self.cdp
487                .dispatch_mouse_event("mouseMoved", start.x, start.y, None, None)
488                .await?;
489        }
490        for window in path.windows(2) {
491            let next = window[1];
492            if self.interaction_mode == InteractionMode::Human {
493                tokio::time::sleep(self.mouse.move_delay(window[0], next)).await;
494            }
495            self.cdp
496                .dispatch_mouse_event("mouseMoved", next.x, next.y, None, None)
497                .await?;
498        }
499        let press_point = self.verified_action_point(object_id).await?;
500        if (press_point.x - local_point.x).abs() > 1.0
501            || (press_point.y - local_point.y).abs() > 1.0
502        {
503            return Err(TargetError {
504                kind: TargetErrorKind::NotActionable,
505                reason: Some(TargetActionabilityReason::GeometryChanged),
506                candidates: Vec::new(),
507                recovery: None,
508            }
509            .into());
510        }
511        let mut pressed = None;
512        for event in events {
513            if event.event_type == "mousePressed" {
514                pressed = Some(PressedButtonGuard {
515                    cdp: self.cdp.clone(),
516                    point,
517                    click_count: event.click_count,
518                    armed: true,
519                });
520            }
521            self.cdp
522                .dispatch_mouse_event(
523                    &event.event_type,
524                    event.x,
525                    event.y,
526                    Some(&event.button),
527                    Some(event.click_count),
528                )
529                .await?;
530            if event.event_type == "mouseReleased"
531                && let Some(mut guard) = pressed.take()
532            {
533                guard.armed = false;
534            }
535            if self.interaction_mode == InteractionMode::Human && event.event_type == "mousePressed"
536            {
537                tokio::time::sleep(self.mouse.click_delay()).await;
538            }
539        }
540        *pointer = Some(point);
541        Ok(())
542    }
543
544    /// Type text into the page.
545    ///
546    /// If `target` is provided, clicks the target element first to focus it,
547    /// then inserts the text via CDP `Input.insertText`. Otherwise types at
548    /// the current focus.
549    pub async fn type_text(
550        &self,
551        text: &str,
552        target: Option<&str>,
553    ) -> BrowserResult<ActionOutcome> {
554        self.cdp
555            .with_current_route(async {
556                let target = match target {
557                    Some(target) => self.click(target).await?.target,
558                    None => None,
559                };
560                self.cdp.insert_text(text).await?;
561                let (target_id, frame_id) = self.route_identity().await?;
562                Ok(ActionOutcome {
563                    action: ActionKind::Type,
564                    target,
565                    revision: self.invalidate_observation(),
566                    target_id,
567                    frame_id,
568                    evidence: None,
569                })
570            })
571            .await
572    }
573
574    async fn move_pointer(&self, destination: Point) -> BrowserResult<()> {
575        let mut pointer = self.pointer.lock().await;
576        let start = pointer.unwrap_or(destination);
577        for window in
578            interaction_path(self.interaction_mode, &self.mouse, start, destination).windows(2)
579        {
580            if self.interaction_mode == InteractionMode::Human {
581                tokio::time::sleep(self.mouse.move_delay(window[0], window[1])).await;
582            }
583            self.cdp
584                .dispatch_mouse_event("mouseMoved", window[1].x, window[1].y, None, None)
585                .await?;
586        }
587        if start == destination {
588            self.cdp
589                .dispatch_mouse_event("mouseMoved", destination.x, destination.y, None, None)
590                .await?;
591        }
592        *pointer = Some(destination);
593        Ok(())
594    }
595
596    async fn keyboard_action(
597        &self,
598        action: ActionKind,
599        key: &str,
600        event_type: &str,
601        modifiers: i64,
602    ) -> BrowserResult<ActionOutcome> {
603        validate_key(key)?;
604        self.cdp
605            .with_current_route(async {
606                self.cdp
607                    .dispatch_key_event_with_modifiers(
608                        event_type,
609                        key,
610                        &key_code(key),
611                        "",
612                        modifiers,
613                    )
614                    .await?;
615                self.action_outcome(action, None, None).await
616            })
617            .await
618    }
619
620    async fn set_checked(&self, target: &str, checked: bool) -> BrowserResult<ActionOutcome> {
621        let action = if checked {
622            ActionKind::Check
623        } else {
624            ActionKind::Uncheck
625        };
626        let script = format!(
627            r#"function() {{ if (!(this instanceof HTMLInputElement) || !['checkbox','radio'].includes(this.type)) return {{ok:false,reason:'not_checkable'}}; if (this.checked !== {checked}) this.click(); return {{ok:this.checked === {checked}}}; }}"#
628        );
629        self.form_object_action(target, action, &script).await
630    }
631
632    async fn form_object_action(
633        &self,
634        target: &str,
635        action: ActionKind,
636        function: &str,
637    ) -> BrowserResult<ActionOutcome> {
638        self.cdp
639            .with_current_route(async {
640                let element = self.resolve_element(target).await?;
641                let object_id = self
642                    .cdp
643                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
644                    .await?;
645                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
646                self.verified_action_point(&remote.object_id).await?;
647                let result = self.cdp.call_on_object(&remote.object_id, function).await?;
648                let value = runtime_value(&result)?;
649                if value["ok"].as_bool() != Some(true) {
650                    return Err(format!(
651                        "form action failed: {}",
652                        value["reason"].as_str().unwrap_or("verification_failed")
653                    )
654                    .into());
655                }
656                self.action_outcome(action, Some(element), None).await
657            })
658            .await
659    }
660
661    async fn action_outcome(
662        &self,
663        action: ActionKind,
664        element: Option<ResolvedElement>,
665        evidence: Option<Value>,
666    ) -> BrowserResult<ActionOutcome> {
667        let target = element.map(|element| ActionTarget {
668            label: element.label,
669            reference: element.reference,
670        });
671        let mut outcome = self.action_outcome_from_target(action, target).await?;
672        outcome.evidence = evidence;
673        Ok(outcome)
674    }
675
676    pub(crate) async fn action_outcome_from_target(
677        &self,
678        action: ActionKind,
679        target: Option<ActionTarget>,
680    ) -> BrowserResult<ActionOutcome> {
681        if let Some(interception) = &self.policy_interception {
682            // A same-route command is an ordering barrier for synchronous
683            // click/form navigation. The interception itself remains active
684            // for delayed page-authored navigation after this action returns.
685            let _ = self.cdp.evaluate("0").await;
686            tokio::task::yield_now().await;
687            if let Some(error) = interception.take_denial().await {
688                return Err(error.into());
689            }
690        }
691        let (target_id, frame_id) = self.route_identity().await?;
692        Ok(ActionOutcome {
693            action,
694            target,
695            revision: self.invalidate_observation(),
696            target_id,
697            frame_id,
698            evidence: None,
699        })
700    }
701
702    pub(crate) async fn viewport_center(&self) -> BrowserResult<Point> {
703        let value = self
704            .evaluate_value("[window.innerWidth / 2, window.innerHeight / 2]")
705            .await?;
706        let coordinates = value
707            .as_array()
708            .filter(|coordinates| coordinates.len() == 2)
709            .ok_or("viewport evaluation returned invalid coordinates")?;
710        let x = coordinates[0]
711            .as_f64()
712            .ok_or("viewport width was not numeric")?;
713        let y = coordinates[1]
714            .as_f64()
715            .ok_or("viewport height was not numeric")?;
716        Ok(Point { x, y })
717    }
718
719    pub(crate) async fn target_viewport_point(&self, point: Point) -> BrowserResult<Point> {
720        let Some(frame_id) = self.cdp.active_frame() else {
721            return Ok(point);
722        };
723        let frame = {
724            let topology = self.topology.lock().await;
725            topology
726                .frames
727                .iter()
728                .find(|frame| frame.id == frame_id)
729                .cloned()
730        };
731        let frame = match frame {
732            Some(frame) => frame,
733            None => self
734                .list_frames()
735                .await?
736                .into_iter()
737                .find(|frame| frame.id == frame_id)
738                .ok_or("selected frame is no longer attached")?,
739        };
740        if frame.parent_id.is_none() {
741            return Ok(point);
742        }
743        let (x, y) = self.cdp.frame_viewport_offset(&frame_id).await?;
744        Ok(Point {
745            x: point.x + x,
746            y: point.y + y,
747        })
748    }
749
750    pub(crate) async fn evaluate_value(&self, expression: &str) -> BrowserResult<Value> {
751        let raw = self.cdp.evaluate(expression).await?;
752        runtime_value(&raw)
753    }
754
755    pub(crate) fn invalidate_observation(&self) -> u64 {
756        self.page_revision.fetch_add(1, Ordering::Relaxed) + 1
757    }
758
759    pub(crate) async fn verified_action_point(&self, object_id: &str) -> BrowserResult<Point> {
760        let raw = match self.cdp.call_on_object(object_id, HIT_TEST_FUNCTION).await {
761            Ok(raw) => raw,
762            Err(error) => {
763                tracing::debug!(%error, "target node could not be verified");
764                return Err(TargetError {
765                    kind: TargetErrorKind::NotActionable,
766                    reason: Some(TargetActionabilityReason::NodeUnavailable),
767                    candidates: Vec::new(),
768                    recovery: None,
769                }
770                .into());
771            }
772        };
773        let value = runtime_value(&raw)?;
774        if value["ok"].as_bool() != Some(true) {
775            let reason = value["reason"].as_str().unwrap_or("verification_failed");
776            tracing::debug!(reason, "target actionability check failed");
777            return Err(TargetError {
778                kind: TargetErrorKind::NotActionable,
779                reason: Some(actionability_reason(reason)),
780                candidates: Vec::new(),
781                recovery: None,
782            }
783            .into());
784        }
785        let x = value["x"]
786            .as_f64()
787            .ok_or("verified target x was not numeric")?;
788        let y = value["y"]
789            .as_f64()
790            .ok_or("verified target y was not numeric")?;
791        Ok(Point { x, y })
792    }
793}
glass/browser/session/action.rs

glass/browser/session/
action.rs
