glass/browser/session/
action.rs

1//! Action primitives: clicks, typing, keyboard, scroll, drag.
2//!
3//! Implementation of individual browser interaction actions on
4//! [`BrowserSession`]: click, double-click, hover, drag, key press,
5//! scroll, clear, check, uncheck, select, and file upload.
6
7use super::*;
8
9impl BrowserSession {
10    /// Click exact frame-local viewport coordinates. This is an explicit,
11    /// policy-gated escape hatch for canvas and map surfaces where no DOM
12    /// control can be published. Coordinates are validated against the live
13    /// viewport and are never adjusted to a nearby element.
14    pub async fn click_at(&self, x: f64, y: f64) -> BrowserResult<CoordinateClickOutcome> {
15        self.policy
16            .require(crate::browser::policy::PolicyCapability::CoordinateClick)?;
17        if !x.is_finite() || !y.is_finite() || x < 0.0 || y < 0.0 {
18            return Err("click-at coordinates must be finite and non-negative".into());
19        }
20
21        self.cdp
22            .with_current_route(async {
23                let hit = self
24                    .evaluate_value(&format!(
25                        "(() => {{ if ({x} >= innerWidth || {y} >= innerHeight) return null; const e = document.elementFromPoint({x}, {y}); if (!e) return null; return {{tag:e.tagName.toLowerCase(), role:e.getAttribute('role'), name:e.getAttribute('aria-label') || e.textContent?.trim().slice(0, 160) || null}}; }})()"
26                    ))
27                    .await?;
28                let hit = if hit.is_null() {
29                    None
30                } else {
31                    Some(CoordinateHit {
32                        tag: hit["tag"].as_str().unwrap_or("unknown").to_string(),
33                        role: hit["role"].as_str().map(str::to_string),
34                        name: hit["name"].as_str().map(str::to_string),
35                    })
36                };
37                if hit.is_none() {
38                    return Err("click-at coordinates are outside the viewport or hit no element".into());
39                }
40                self.cdp
41                    .dispatch_mouse_event("mouseMoved", x, y, None, None)
42                    .await?;
43                self.cdp
44                    .dispatch_mouse_event("mousePressed", x, y, Some("left"), Some(1))
45                    .await?;
46                self.cdp
47                    .dispatch_mouse_event("mouseReleased", x, y, Some("left"), Some(1))
48                    .await?;
49                let (target_id, frame_id) = self.ensured_route_identity().await?;
50                Ok(CoordinateClickOutcome {
51                    x,
52                    y,
53                    hit,
54                    revision: self.invalidate_observation(),
55                    target_id,
56                    frame_id,
57                })
58            })
59            .await
60    }
61
62    /// Scroll the viewport by the given pixel offsets.
63    ///
64    /// Positive `dy` scrolls down; positive `dx` scrolls right.
65    pub async fn scroll(&self, dx: f64, dy: f64) -> BrowserResult<ActionOutcome> {
66        self.cdp
67            .with_current_route(async {
68                self.cdp.scroll_by(dx, dy).await?;
69                let (target_id, frame_id) = self.ensured_route_identity().await?;
70                Ok(ActionOutcome {
71                    action: ActionKind::Scroll,
72                    target: None,
73                    revision: self.invalidate_observation(),
74                    target_id,
75                    frame_id,
76                    evidence: None,
77                })
78            })
79            .await
80    }
81
82    /// Capture the full accessibility tree snapshot for the current page.
83    ///
84    /// Returns the page info, accessibility roots, and all interactive elements.
85    /// Prefer [`observe`](BrowserSession::observe) for compact observations in
86    /// agent workflows.
87    pub async fn snapshot(&self) -> BrowserResult<AccessibilitySnapshot> {
88        self.cdp
89            .with_current_route(async {
90                let revision = self.page_revision.load(Ordering::Relaxed);
91                let raw = self.cdp.get_accessibility_tree().await?;
92                let roots = parse_accessibility_tree(&raw);
93                let interactive = interactive_elements(&roots, revision);
94                Ok(AccessibilitySnapshot {
95                    page: self.page_info().await?,
96                    roots,
97                    interactive,
98                })
99            })
100            .await
101    }
102
103    /// Click an element and return its structured action outcome.
104    pub async fn click(&self, target: &str) -> BrowserResult<ActionOutcome> {
105        self.pointer_click(target, false).await
106    }
107
108    /// Double-click an element with the same target, scroll, and pointer
109    /// contract as a single click.
110    pub async fn double_click(&self, target: &str) -> BrowserResult<ActionOutcome> {
111        self.pointer_click(target, true).await
112    }
113
114    /// Hover the pointer over an element without clicking.
115    ///
116    /// Resolves the target, moves the pointer to the element's center using the
117    /// configured interaction mode, then returns an [`ActionOutcome`].
118    pub async fn hover(&self, target: &str) -> BrowserResult<ActionOutcome> {
119        self.cdp
120            .with_current_route(async {
121                let element = self.resolve_element(target).await?;
122                let object_id = self
123                    .cdp
124                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
125                    .await?;
126                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
127                let local = self.verified_action_point(&remote.object_id).await?;
128                let point = self.target_viewport_point(local).await?;
129                self.move_pointer(point).await?;
130                self.action_outcome(ActionKind::Hover, Some(element), None)
131                    .await
132            })
133            .await
134    }
135
136    /// Drag an element from `source` to `destination`.
137    ///
138    /// Performs a mouse-press on the source element, moves the pointer to the
139    /// destination element, then releases.
140    pub async fn drag(&self, source: &str, destination: &str) -> BrowserResult<ActionOutcome> {
141        self.cdp
142            .with_current_route(async {
143                let source = self.resolve_element(source).await?;
144                let source_object = self
145                    .cdp
146                    .resolve_node_object(source.node_id, source.backend_dom_node_id)
147                    .await?;
148                let source_guard = RemoteObjectGuard::new(self.cdp.clone(), source_object);
149                let destination = self.resolve_element(destination).await?;
150                let destination_object = self
151                    .cdp
152                    .resolve_node_object(destination.node_id, destination.backend_dom_node_id)
153                    .await?;
154                let destination_guard =
155                    RemoteObjectGuard::new(self.cdp.clone(), destination_object);
156                let source_local = self.verified_action_point(&source_guard.object_id).await?;
157                let destination_local = self
158                    .verified_action_point(&destination_guard.object_id)
159                    .await?;
160                let source_point = self.target_viewport_point(source_local).await?;
161                let destination_point = self.target_viewport_point(destination_local).await?;
162                self.move_pointer(source_point).await?;
163                let verified_source = self.verified_action_point(&source_guard.object_id).await?;
164                if (verified_source.x - source_local.x).abs() > 1.0
165                    || (verified_source.y - source_local.y).abs() > 1.0
166                {
167                    return Err(TargetError {
168                        kind: TargetErrorKind::NotActionable,
169                        reason: Some(TargetActionabilityReason::GeometryChanged),
170                        candidates: Vec::new(),
171                        recovery: None,
172                    }
173                    .into());
174                }
175                self.cdp
176                    .dispatch_mouse_event(
177                        "mousePressed",
178                        source_point.x,
179                        source_point.y,
180                        Some("left"),
181                        Some(1),
182                    )
183                    .await?;
184                let mut pressed = PressedButtonGuard {
185                    cdp: self.cdp.clone(),
186                    point: source_point,
187                    click_count: 1,
188                    armed: true,
189                };
190                let drag_path = interaction_path(
191                    self.interaction_mode,
192                    &self.mouse,
193                    source_point,
194                    destination_point,
195                );
196                for window in drag_path.windows(2) {
197                    let point = window[1];
198                    if self.interaction_mode == InteractionMode::Human {
199                        tokio::time::sleep(self.mouse.move_delay(window[0], point)).await;
200                    }
201                    self.cdp
202                        .dispatch_mouse_event("mouseMoved", point.x, point.y, Some("left"), Some(1))
203                        .await?;
204                }
205                let verified_destination = self
206                    .verified_action_point(&destination_guard.object_id)
207                    .await?;
208                if (verified_destination.x - destination_local.x).abs() > 1.0
209                    || (verified_destination.y - destination_local.y).abs() > 1.0
210                {
211                    return Err(TargetError {
212                        kind: TargetErrorKind::NotActionable,
213                        reason: Some(TargetActionabilityReason::GeometryChanged),
214                        candidates: Vec::new(),
215                        recovery: None,
216                    }
217                    .into());
218                }
219                self.cdp
220                    .dispatch_mouse_event(
221                        "mouseReleased",
222                        destination_point.x,
223                        destination_point.y,
224                        Some("left"),
225                        Some(1),
226                    )
227                    .await?;
228                pressed.armed = false;
229                *self.pointer.lock().await = Some(destination_point);
230                self.action_outcome(ActionKind::Drag, Some(source), None)
231                    .await
232            })
233            .await
234    }
235
236    /// Press and hold a keyboard key.
237    ///
238    /// Dispatches a `rawKeyDown` CDP event for the given key.
239    pub async fn key_down(&self, key: &str) -> BrowserResult<ActionOutcome> {
240        self.keyboard_action(ActionKind::KeyDown, key, "rawKeyDown", 0)
241            .await
242    }
243
244    /// Release a keyboard key.
245    ///
246    /// Dispatches a `keyUp` CDP event for the given key.
247    pub async fn key_up(&self, key: &str) -> BrowserResult<ActionOutcome> {
248        self.keyboard_action(ActionKind::KeyUp, key, "keyUp", 0)
249            .await
250    }
251
252    /// Press and release a keyboard key.
253    ///
254    /// Dispatches `rawKeyDown`, `char` (for single-character keys), and `keyUp`
255    /// CDP events.
256    pub async fn key_press(&self, key: &str) -> BrowserResult<ActionOutcome> {
257        validate_key(key)?;
258        self.cdp
259            .with_current_route(async {
260                let code = key_code(key);
261                self.cdp
262                    .dispatch_key_event_with_modifiers("rawKeyDown", key, &code, "", 0)
263                    .await?;
264                if key.chars().count() == 1 {
265                    self.cdp
266                        .dispatch_key_event_with_modifiers("char", key, &code, key, 0)
267                        .await?;
268                }
269                self.cdp
270                    .dispatch_key_event_with_modifiers("keyUp", key, &code, "", 0)
271                    .await?;
272                self.action_outcome(ActionKind::KeyPress, None, None).await
273            })
274            .await
275    }
276
277    /// Execute a keyboard shortcut with modifier keys.
278    ///
279    /// Parses shortcuts like `"Ctrl+C"` or `"Meta+V"` and dispatches
280    /// the corresponding key events with the specified modifiers.
281    pub async fn shortcut(&self, shortcut: &str) -> BrowserResult<ActionOutcome> {
282        let (modifiers, key) = parse_shortcut(shortcut)?;
283        self.cdp
284            .with_current_route(async {
285                let code = key_code(&key);
286                self.cdp
287                    .dispatch_key_event_with_modifiers("rawKeyDown", &key, &code, "", modifiers)
288                    .await?;
289                self.cdp
290                    .dispatch_key_event_with_modifiers("keyUp", &key, &code, "", modifiers)
291                    .await?;
292                self.action_outcome(ActionKind::Shortcut, None, None).await
293            })
294            .await
295    }
296
297    /// Clear the contents of an editable element.
298    ///
299    /// Clicks the target, selects all content, then presses Backspace.
300    /// Verifies the element is empty afterward.
301    pub async fn clear(&self, target: &str) -> BrowserResult<ActionOutcome> {
302        self.cdp
303            .with_current_route(async {
304                let element = self.resolve_element(target).await?;
305                let object_id = self.cdp.resolve_node_object(element.node_id, element.backend_dom_node_id).await?;
306                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
307                let editable = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement || this.isContentEditable}").await?)?;
308                if editable.as_bool() != Some(true) { return Err("clear target is not editable".into()); }
309                let clicked = self.click(target).await?;
310                self.cdp.dispatch_select_all().await?;
311                self.key_press("Backspace").await?;
312                let empty = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement ? this.value === '' : this.textContent === ''}").await?)?;
313                if empty.as_bool() != Some(true) { return Err("clear target did not become empty".into()); }
314                self.action_outcome_from_target(ActionKind::Clear, clicked.target)
315                    .await
316            })
317            .await
318    }
319
320    /// Check a checkbox or radio button.
321    ///
322    /// Ensures the target element's `checked` property is set to `true`.
323    pub async fn check(&self, target: &str) -> BrowserResult<ActionOutcome> {
324        self.set_checked(target, true).await
325    }
326
327    /// Uncheck a checkbox.
328    ///
329    /// Ensures the target element's `checked` property is set to `false`.
330    pub async fn uncheck(&self, target: &str) -> BrowserResult<ActionOutcome> {
331        self.set_checked(target, false).await
332    }
333
334    /// Select an option from a `<select>` element by value.
335    ///
336    /// `value` must be 1–4096 bytes. Fires `input` and `change` events.
337    pub async fn select_option(&self, target: &str, value: &str) -> BrowserResult<ActionOutcome> {
338        if value.is_empty() || value.len() > 4096 {
339            return Err("select value must be 1..=4096 bytes".into());
340        }
341        let value_json = serde_json::to_string(value)?;
342        self.form_object_action(target, ActionKind::Select, &format!(r#"function() {{ if (!(this instanceof HTMLSelectElement)) return {{ok:false,reason:'not_select'}}; const option = Array.from(this.options).find(option => option.value === {value_json}); if (!option) return {{ok:false,reason:'option_not_found'}}; this.value = option.value; this.dispatchEvent(new Event('input',{{bubbles:true}})); this.dispatchEvent(new Event('change',{{bubbles:true}})); return {{ok:this.value === option.value}}; }}"#)).await
343    }
344
345    pub async fn upload_files(
346        &self,
347        target: &str,
348        paths: &[PathBuf],
349    ) -> BrowserResult<ActionOutcome> {
350        self.policy.require(PolicyCapability::Upload)?;
351        self.cdp.with_current_route(async {
352            if paths.is_empty() || paths.len() > 16 { return Err("upload requires 1..=16 files".into()); }
353            let mut files = Vec::with_capacity(paths.len());
354            for path in paths {
355                let canonical = self.policy.require_existing_path(path)?;
356                if !canonical.is_file() { return Err("upload path must be a regular file".into()); }
357                if !canonical.starts_with(&self.upload_root) { return Err("upload path is outside the allowed workspace root".into()); }
358                files.push(canonical.to_string_lossy().into_owned());
359            }
360            let element = self.resolve_element(target).await?;
361            let object_id = self.cdp.resolve_node_object(element.node_id, element.backend_dom_node_id).await?;
362            let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
363            self.verified_action_point(&remote.object_id).await?;
364            let input = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return {ok:this instanceof HTMLInputElement && this.type === 'file'}}").await?)?;
365            if input["ok"].as_bool() != Some(true) { return Err("upload target is not a file input".into()); }
366            if element.node_id.is_none() && element.backend_dom_node_id.is_none() { return Err("file input target has no DOM node ID".into()); }
367            self.cdp.set_file_input_files(element.node_id, element.backend_dom_node_id, &files).await?;
368            let verified = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this.files.length}").await?)?;
369            if verified.as_u64() != Some(files.len() as u64) { return Err("file input did not retain the requested file count".into()); }
370            let outcome = self.action_outcome(ActionKind::Upload, Some(element), Some(serde_json::json!({"file_count": files.len()}))).await?;
371            self.record_audit("upload", format!("{} files", files.len()));
372            Ok(outcome)
373        }).await
374    }
375
376    async fn pointer_click(
377        &self,
378        target: &str,
379        double_click: bool,
380    ) -> BrowserResult<ActionOutcome> {
381        self.cdp
382            .with_current_route(async {
383                let element = self.resolve_element(target).await?;
384                let object_id = self
385                    .cdp
386                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
387                    .await
388                    .map_err(|error| {
389                        tracing::debug!(%error, "target node could not be resolved");
390                        TargetError {
391                            kind: TargetErrorKind::NotActionable,
392                            reason: Some(TargetActionabilityReason::NodeUnavailable),
393                            candidates: Vec::new(),
394                            recovery: None,
395                        }
396                    })?;
397                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
398                let local_point = self.verified_action_point(&remote.object_id).await?;
399                let point = self.target_viewport_point(local_point).await?;
400                let events = if double_click {
401                    self.mouse.generate_double_click_events(point)
402                } else {
403                    self.mouse.generate_click_events(point)
404                };
405                self.dispatch_pointer_events(&remote.object_id, local_point, point, events)
406                    .await?;
407                let (target_id, frame_id) = self.route_identity().await?;
408                Ok(ActionOutcome {
409                    action: if double_click {
410                        ActionKind::DoubleClick
411                    } else {
412                        ActionKind::Click
413                    },
414                    target: Some(ActionTarget {
415                        label: element.label,
416                        reference: element.reference,
417                    }),
418                    revision: self.invalidate_observation(),
419                    target_id,
420                    frame_id,
421                    evidence: None,
422                })
423            })
424            .await
425    }
426
427    async fn dispatch_pointer_events(
428        &self,
429        object_id: &str,
430        local_point: Point,
431        point: Point,
432        events: Vec<crate::browser::mouse::MouseEvent>,
433    ) -> BrowserResult<()> {
434        let mut pointer = self.pointer.lock().await;
435        let start = match (self.interaction_mode, *pointer) {
436            (_, Some(point)) => point,
437            (InteractionMode::Human, None) => self
438                .viewport_center()
439                .await
440                .unwrap_or(Point { x: 640.0, y: 360.0 }),
441            (InteractionMode::Fast, None) => point,
442        };
443        let path = interaction_path(self.interaction_mode, &self.mouse, start, point);
444        if self.interaction_mode == InteractionMode::Human && pointer.is_none() {
445            self.cdp
446                .dispatch_mouse_event("mouseMoved", start.x, start.y, None, None)
447                .await?;
448        }
449        for window in path.windows(2) {
450            let next = window[1];
451            if self.interaction_mode == InteractionMode::Human {
452                tokio::time::sleep(self.mouse.move_delay(window[0], next)).await;
453            }
454            self.cdp
455                .dispatch_mouse_event("mouseMoved", next.x, next.y, None, None)
456                .await?;
457        }
458        let press_point = self.verified_action_point(object_id).await?;
459        if (press_point.x - local_point.x).abs() > 1.0
460            || (press_point.y - local_point.y).abs() > 1.0
461        {
462            return Err(TargetError {
463                kind: TargetErrorKind::NotActionable,
464                reason: Some(TargetActionabilityReason::GeometryChanged),
465                candidates: Vec::new(),
466                recovery: None,
467            }
468            .into());
469        }
470        let mut pressed = None;
471        for event in events {
472            if event.event_type == "mousePressed" {
473                pressed = Some(PressedButtonGuard {
474                    cdp: self.cdp.clone(),
475                    point,
476                    click_count: event.click_count,
477                    armed: true,
478                });
479            }
480            self.cdp
481                .dispatch_mouse_event(
482                    &event.event_type,
483                    event.x,
484                    event.y,
485                    Some(&event.button),
486                    Some(event.click_count),
487                )
488                .await?;
489            if event.event_type == "mouseReleased"
490                && let Some(mut guard) = pressed.take()
491            {
492                guard.armed = false;
493            }
494            if self.interaction_mode == InteractionMode::Human && event.event_type == "mousePressed"
495            {
496                tokio::time::sleep(self.mouse.click_delay()).await;
497            }
498        }
499        *pointer = Some(point);
500        Ok(())
501    }
502
503    /// Type text into the page.
504    ///
505    /// If `target` is provided, clicks the target element first to focus it,
506    /// then inserts the text via CDP `Input.insertText`. Otherwise types at
507    /// the current focus.
508    pub async fn type_text(
509        &self,
510        text: &str,
511        target: Option<&str>,
512    ) -> BrowserResult<ActionOutcome> {
513        self.cdp
514            .with_current_route(async {
515                let target = match target {
516                    Some(target) => self.click(target).await?.target,
517                    None => None,
518                };
519                self.cdp.insert_text(text).await?;
520                let (target_id, frame_id) = self.route_identity().await?;
521                Ok(ActionOutcome {
522                    action: ActionKind::Type,
523                    target,
524                    revision: self.invalidate_observation(),
525                    target_id,
526                    frame_id,
527                    evidence: None,
528                })
529            })
530            .await
531    }
532
533    async fn move_pointer(&self, destination: Point) -> BrowserResult<()> {
534        let mut pointer = self.pointer.lock().await;
535        let start = pointer.unwrap_or(destination);
536        for window in
537            interaction_path(self.interaction_mode, &self.mouse, start, destination).windows(2)
538        {
539            if self.interaction_mode == InteractionMode::Human {
540                tokio::time::sleep(self.mouse.move_delay(window[0], window[1])).await;
541            }
542            self.cdp
543                .dispatch_mouse_event("mouseMoved", window[1].x, window[1].y, None, None)
544                .await?;
545        }
546        if start == destination {
547            self.cdp
548                .dispatch_mouse_event("mouseMoved", destination.x, destination.y, None, None)
549                .await?;
550        }
551        *pointer = Some(destination);
552        Ok(())
553    }
554
555    async fn keyboard_action(
556        &self,
557        action: ActionKind,
558        key: &str,
559        event_type: &str,
560        modifiers: i64,
561    ) -> BrowserResult<ActionOutcome> {
562        validate_key(key)?;
563        self.cdp
564            .with_current_route(async {
565                self.cdp
566                    .dispatch_key_event_with_modifiers(
567                        event_type,
568                        key,
569                        &key_code(key),
570                        "",
571                        modifiers,
572                    )
573                    .await?;
574                self.action_outcome(action, None, None).await
575            })
576            .await
577    }
578
579    async fn set_checked(&self, target: &str, checked: bool) -> BrowserResult<ActionOutcome> {
580        let action = if checked {
581            ActionKind::Check
582        } else {
583            ActionKind::Uncheck
584        };
585        let script = format!(
586            r#"function() {{ if (!(this instanceof HTMLInputElement) || !['checkbox','radio'].includes(this.type)) return {{ok:false,reason:'not_checkable'}}; if (this.checked !== {checked}) this.click(); return {{ok:this.checked === {checked}}}; }}"#
587        );
588        self.form_object_action(target, action, &script).await
589    }
590
591    async fn form_object_action(
592        &self,
593        target: &str,
594        action: ActionKind,
595        function: &str,
596    ) -> BrowserResult<ActionOutcome> {
597        self.cdp
598            .with_current_route(async {
599                let element = self.resolve_element(target).await?;
600                let object_id = self
601                    .cdp
602                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
603                    .await?;
604                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
605                self.verified_action_point(&remote.object_id).await?;
606                let result = self.cdp.call_on_object(&remote.object_id, function).await?;
607                let value = runtime_value(&result)?;
608                if value["ok"].as_bool() != Some(true) {
609                    return Err(format!(
610                        "form action failed: {}",
611                        value["reason"].as_str().unwrap_or("verification_failed")
612                    )
613                    .into());
614                }
615                self.action_outcome(action, Some(element), None).await
616            })
617            .await
618    }
619
620    async fn action_outcome(
621        &self,
622        action: ActionKind,
623        element: Option<ResolvedElement>,
624        evidence: Option<Value>,
625    ) -> BrowserResult<ActionOutcome> {
626        let target = element.map(|element| ActionTarget {
627            label: element.label,
628            reference: element.reference,
629        });
630        let mut outcome = self.action_outcome_from_target(action, target).await?;
631        outcome.evidence = evidence;
632        Ok(outcome)
633    }
634
635    pub(crate) async fn action_outcome_from_target(
636        &self,
637        action: ActionKind,
638        target: Option<ActionTarget>,
639    ) -> BrowserResult<ActionOutcome> {
640        if let Some(interception) = &self.policy_interception {
641            // A same-route command is an ordering barrier for synchronous
642            // click/form navigation. The interception itself remains active
643            // for delayed page-authored navigation after this action returns.
644            let _ = self.cdp.evaluate("0").await;
645            tokio::task::yield_now().await;
646            if let Some(error) = interception.take_denial().await {
647                return Err(error.into());
648            }
649        }
650        let (target_id, frame_id) = self.route_identity().await?;
651        Ok(ActionOutcome {
652            action,
653            target,
654            revision: self.invalidate_observation(),
655            target_id,
656            frame_id,
657            evidence: None,
658        })
659    }
660
661    pub(crate) async fn viewport_center(&self) -> BrowserResult<Point> {
662        let value = self
663            .evaluate_value("[window.innerWidth / 2, window.innerHeight / 2]")
664            .await?;
665        let coordinates = value
666            .as_array()
667            .filter(|coordinates| coordinates.len() == 2)
668            .ok_or("viewport evaluation returned invalid coordinates")?;
669        let x = coordinates[0]
670            .as_f64()
671            .ok_or("viewport width was not numeric")?;
672        let y = coordinates[1]
673            .as_f64()
674            .ok_or("viewport height was not numeric")?;
675        Ok(Point { x, y })
676    }
677
678    pub(crate) async fn target_viewport_point(&self, point: Point) -> BrowserResult<Point> {
679        let Some(frame_id) = self.cdp.active_frame() else {
680            return Ok(point);
681        };
682        let frame = {
683            let topology = self.topology.lock().await;
684            topology
685                .frames
686                .iter()
687                .find(|frame| frame.id == frame_id)
688                .cloned()
689        };
690        let frame = match frame {
691            Some(frame) => frame,
692            None => self
693                .list_frames()
694                .await?
695                .into_iter()
696                .find(|frame| frame.id == frame_id)
697                .ok_or("selected frame is no longer attached")?,
698        };
699        if frame.parent_id.is_none() {
700            return Ok(point);
701        }
702        let (x, y) = self.cdp.frame_viewport_offset(&frame_id).await?;
703        Ok(Point {
704            x: point.x + x,
705            y: point.y + y,
706        })
707    }
708
709    pub(crate) async fn evaluate_value(&self, expression: &str) -> BrowserResult<Value> {
710        let raw = self.cdp.evaluate(expression).await?;
711        runtime_value(&raw)
712    }
713
714    pub(crate) fn invalidate_observation(&self) -> u64 {
715        self.page_revision.fetch_add(1, Ordering::Relaxed) + 1
716    }
717
718    pub(crate) async fn verified_action_point(&self, object_id: &str) -> BrowserResult<Point> {
719        let raw = match self.cdp.call_on_object(object_id, HIT_TEST_FUNCTION).await {
720            Ok(raw) => raw,
721            Err(error) => {
722                tracing::debug!(%error, "target node could not be verified");
723                return Err(TargetError {
724                    kind: TargetErrorKind::NotActionable,
725                    reason: Some(TargetActionabilityReason::NodeUnavailable),
726                    candidates: Vec::new(),
727                    recovery: None,
728                }
729                .into());
730            }
731        };
732        let value = runtime_value(&raw)?;
733        if value["ok"].as_bool() != Some(true) {
734            let reason = value["reason"].as_str().unwrap_or("verification_failed");
735            tracing::debug!(reason, "target actionability check failed");
736            return Err(TargetError {
737                kind: TargetErrorKind::NotActionable,
738                reason: Some(actionability_reason(reason)),
739                candidates: Vec::new(),
740                recovery: None,
741            }
742            .into());
743        }
744        let x = value["x"]
745            .as_f64()
746            .ok_or("verified target x was not numeric")?;
747        let y = value["y"]
748            .as_f64()
749            .ok_or("verified target y was not numeric")?;
750        Ok(Point { x, y })
751    }
752}
glass/browser/session/action.rs

glass/browser/session/
action.rs
