glass/browser/session/
action.rs

1//! Action primitives: clicks, typing, keyboard, scroll, drag.
2//!
3//! Implementation of individual browser interaction actions on
4//! [`BrowserSession`]: click, double-click, hover, drag, key press,
5//! scroll, clear, check, uncheck, select, and file upload.
6
7use super::*;
8
9impl BrowserSession {
10    /// Click exact frame-local viewport coordinates. This is an explicit,
11    /// policy-gated escape hatch for canvas and map surfaces where no DOM
12    /// control can be published. Coordinates are validated against the live
13    /// viewport and are never adjusted to a nearby element.
14    pub async fn click_at(&self, x: f64, y: f64) -> BrowserResult<CoordinateClickOutcome> {
15        self.policy
16            .require(crate::browser::policy::PolicyCapability::CoordinateClick)?;
17        if !x.is_finite() || !y.is_finite() || x < 0.0 || y < 0.0 {
18            return Err("click-at coordinates must be finite and non-negative".into());
19        }
20
21        self.cdp
22            .with_current_route(async {
23                let hit = self
24                    .evaluate_value(&format!(
25                        "(() => {{ if ({x} >= innerWidth || {y} >= innerHeight) return null; const e = document.elementFromPoint({x}, {y}); if (!e) return null; return {{tag:e.tagName.toLowerCase(), role:e.getAttribute('role'), name:e.getAttribute('aria-label') || e.textContent?.trim().slice(0, 160) || null}}; }})()"
26                    ))
27                    .await?;
28                let hit = if hit.is_null() {
29                    None
30                } else {
31                    Some(CoordinateHit {
32                        tag: hit["tag"].as_str().unwrap_or("unknown").to_string(),
33                        role: hit["role"].as_str().map(str::to_string),
34                        name: hit["name"].as_str().map(str::to_string),
35                    })
36                };
37                if hit.is_none() {
38                    return Err("click-at coordinates are outside the viewport or hit no element".into());
39                }
40                self.cdp
41                    .dispatch_mouse_event("mouseMoved", x, y, None, None)
42                    .await?;
43                self.cdp
44                    .dispatch_mouse_event("mousePressed", x, y, Some("left"), Some(1))
45                    .await?;
46                self.cdp
47                    .dispatch_mouse_event("mouseReleased", x, y, Some("left"), Some(1))
48                    .await?;
49                let (target_id, frame_id) = self.ensured_route_identity().await?;
50                Ok(CoordinateClickOutcome {
51                    x,
52                    y,
53                    hit,
54                    execution_id: self.next_execution_id(),
55                    revision: self.invalidate_observation(),
56                    target_id,
57                    frame_id,
58                })
59            })
60            .await
61    }
62
63    /// Scroll the viewport by the given pixel offsets.
64    ///
65    /// Positive `dy` scrolls down; positive `dx` scrolls right.
66    pub async fn scroll(&self, dx: f64, dy: f64) -> BrowserResult<ActionOutcome> {
67        self.scroll_with_revision(dx, dy, None).await
68    }
69
70    /// Scroll while enforcing an optional observation revision.
71    pub async fn scroll_with_revision(
72        &self,
73        dx: f64,
74        dy: f64,
75        expected_revision: Option<u64>,
76    ) -> BrowserResult<ActionOutcome> {
77        self.cdp
78            .with_current_route(async {
79                self.require_expected_revision(expected_revision)?;
80                let previous_revision = self.page_revision.load(Ordering::Relaxed);
81                self.cdp.scroll_by(dx, dy).await?;
82                let (target_id, frame_id) = self.ensured_route_identity().await?;
83                let current_revision = self.invalidate_observation();
84                Ok(ActionOutcome {
85                    status: ActionStatus::Succeeded,
86                    action: ActionKind::Scroll,
87                    execution_id: self.next_execution_id(),
88                    target: None,
89                    revision: current_revision,
90                    previous_revision,
91                    current_revision,
92                    target_id,
93                    frame_id,
94                    verification: ActionVerificationEvidence {
95                        revision_delta: current_revision.saturating_sub(previous_revision),
96                        ..ActionVerificationEvidence::default()
97                    },
98                    evidence: None,
99                })
100            })
101            .await
102    }
103
104    /// Capture the full accessibility tree snapshot for the current page.
105    ///
106    /// Returns the page info, accessibility roots, and all interactive elements.
107    /// Prefer [`observe`](BrowserSession::observe) for compact observations in
108    /// agent workflows.
109    pub async fn snapshot(&self) -> BrowserResult<AccessibilitySnapshot> {
110        self.cdp
111            .with_current_route(async {
112                let revision = self.page_revision.load(Ordering::Relaxed);
113                let raw = serde_json::to_value(self.cdp.get_accessibility_tree().await?)?;
114                let roots = parse_accessibility_tree(&raw);
115                let interactive = interactive_elements(&roots, revision);
116                Ok(AccessibilitySnapshot {
117                    page: self.page_info().await?,
118                    roots,
119                    interactive,
120                })
121            })
122            .await
123    }
124
125    /// Click an element and return its structured action outcome.
126    pub async fn click(&self, target: &str) -> BrowserResult<ActionOutcome> {
127        self.pointer_click(ActionRequest::new(ActionKind::Click, target, None))
128            .await
129    }
130
131    /// Click an element only when the caller's observation revision is current.
132    pub async fn click_with_revision(
133        &self,
134        target: &str,
135        expected_revision: u64,
136    ) -> BrowserResult<ActionOutcome> {
137        self.pointer_click(ActionRequest::new(
138            ActionKind::Click,
139            target,
140            Some(expected_revision),
141        ))
142        .await
143    }
144
145    /// Double-click an element with the same target, scroll, and pointer
146    /// contract as a single click.
147    pub async fn double_click(&self, target: &str) -> BrowserResult<ActionOutcome> {
148        self.pointer_click(ActionRequest::new(ActionKind::DoubleClick, target, None))
149            .await
150    }
151
152    /// Double-click while enforcing an optional observation revision.
153    pub async fn double_click_with_revision(
154        &self,
155        target: &str,
156        expected_revision: Option<u64>,
157    ) -> BrowserResult<ActionOutcome> {
158        self.pointer_click(ActionRequest::new(
159            ActionKind::DoubleClick,
160            target,
161            expected_revision,
162        ))
163        .await
164    }
165
166    /// Hover the pointer over an element without clicking.
167    ///
168    /// Resolves the target, moves the pointer to the element's center using the
169    /// configured interaction mode, then returns an [`ActionOutcome`].
170    pub async fn hover(&self, target: &str) -> BrowserResult<ActionOutcome> {
171        self.cdp
172            .with_current_route(async {
173                let element = self.resolve_element(target).await?;
174                let object_id = self
175                    .cdp
176                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
177                    .await?;
178                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
179                let local = self.verified_action_point(&remote.object_id).await?;
180                let point = self.target_viewport_point(local).await?;
181                self.move_pointer(point).await?;
182                self.action_outcome(ActionKind::Hover, Some(element), None)
183                    .await
184            })
185            .await
186    }
187
188    /// Drag an element from `source` to `destination`.
189    ///
190    /// Performs a mouse-press on the source element, moves the pointer to the
191    /// destination element, then releases.
192    pub async fn drag(&self, source: &str, destination: &str) -> BrowserResult<ActionOutcome> {
193        self.drag_with_revision(source, destination, None).await
194    }
195
196    /// Drag while enforcing an optional observation revision.
197    pub async fn drag_with_revision(
198        &self,
199        source: &str,
200        destination: &str,
201        expected_revision: Option<u64>,
202    ) -> BrowserResult<ActionOutcome> {
203        self.cdp
204            .with_current_route(async {
205                self.require_expected_revision(expected_revision)?;
206                let source = self.resolve_element(source).await?;
207                let source_object = self
208                    .cdp
209                    .resolve_node_object(source.node_id, source.backend_dom_node_id)
210                    .await?;
211                let source_guard = RemoteObjectGuard::new(self.cdp.clone(), source_object);
212                let destination = self.resolve_element(destination).await?;
213                let destination_object = self
214                    .cdp
215                    .resolve_node_object(destination.node_id, destination.backend_dom_node_id)
216                    .await?;
217                let destination_guard =
218                    RemoteObjectGuard::new(self.cdp.clone(), destination_object);
219                let source_local = self.verified_action_point(&source_guard.object_id).await?;
220                let destination_local = self
221                    .verified_action_point(&destination_guard.object_id)
222                    .await?;
223                let source_point = self.target_viewport_point(source_local).await?;
224                let destination_point = self.target_viewport_point(destination_local).await?;
225                self.move_pointer(source_point).await?;
226                let verified_source = self.verified_action_point(&source_guard.object_id).await?;
227                if (verified_source.x - source_local.x).abs() > 1.0
228                    || (verified_source.y - source_local.y).abs() > 1.0
229                {
230                    return Err(TargetError {
231                        kind: TargetErrorKind::NotActionable,
232                        reason: Some(TargetActionabilityReason::GeometryChanged),
233                        candidates: Vec::new(),
234                        recovery: None,
235                    }
236                    .into());
237                }
238                self.cdp
239                    .dispatch_mouse_event(
240                        "mousePressed",
241                        source_point.x,
242                        source_point.y,
243                        Some("left"),
244                        Some(1),
245                    )
246                    .await?;
247                let mut pressed = PressedButtonGuard {
248                    cdp: self.cdp.clone(),
249                    point: source_point,
250                    click_count: 1,
251                    armed: true,
252                };
253                let drag_path = interaction_path(
254                    self.interaction_mode,
255                    &self.mouse,
256                    source_point,
257                    destination_point,
258                );
259                for window in drag_path.windows(2) {
260                    let point = window[1];
261                    if self.interaction_mode == InteractionMode::Human {
262                        tokio::time::sleep(self.mouse.move_delay(window[0], point)).await;
263                    }
264                    self.cdp
265                        .dispatch_mouse_event("mouseMoved", point.x, point.y, Some("left"), Some(1))
266                        .await?;
267                }
268                let verified_destination = self
269                    .verified_action_point(&destination_guard.object_id)
270                    .await?;
271                if (verified_destination.x - destination_local.x).abs() > 1.0
272                    || (verified_destination.y - destination_local.y).abs() > 1.0
273                {
274                    return Err(TargetError {
275                        kind: TargetErrorKind::NotActionable,
276                        reason: Some(TargetActionabilityReason::GeometryChanged),
277                        candidates: Vec::new(),
278                        recovery: None,
279                    }
280                    .into());
281                }
282                self.cdp
283                    .dispatch_mouse_event(
284                        "mouseReleased",
285                        destination_point.x,
286                        destination_point.y,
287                        Some("left"),
288                        Some(1),
289                    )
290                    .await?;
291                pressed.armed = false;
292                *self.pointer.lock().await = Some(destination_point);
293                self.action_outcome(ActionKind::Drag, Some(source), None)
294                    .await
295            })
296            .await
297    }
298
299    /// Press and hold a keyboard key.
300    ///
301    /// Dispatches a `rawKeyDown` CDP event for the given key.
302    pub async fn key_down(&self, key: &str) -> BrowserResult<ActionOutcome> {
303        self.key_down_with_revision(key, None).await
304    }
305
306    /// Press and hold a keyboard key while enforcing an optional revision.
307    pub async fn key_down_with_revision(
308        &self,
309        key: &str,
310        expected_revision: Option<u64>,
311    ) -> BrowserResult<ActionOutcome> {
312        self.keyboard_action(ActionKind::KeyDown, key, "rawKeyDown", 0, expected_revision)
313            .await
314    }
315
316    /// Release a keyboard key.
317    ///
318    /// Dispatches a `keyUp` CDP event for the given key.
319    pub async fn key_up(&self, key: &str) -> BrowserResult<ActionOutcome> {
320        self.key_up_with_revision(key, None).await
321    }
322
323    /// Release a keyboard key while enforcing an optional revision.
324    pub async fn key_up_with_revision(
325        &self,
326        key: &str,
327        expected_revision: Option<u64>,
328    ) -> BrowserResult<ActionOutcome> {
329        self.keyboard_action(ActionKind::KeyUp, key, "keyUp", 0, expected_revision)
330            .await
331    }
332
333    /// Press and release a keyboard key.
334    ///
335    /// Dispatches `rawKeyDown`, `char` (for single-character keys), and `keyUp`
336    /// CDP events.
337    pub async fn key_press(&self, key: &str) -> BrowserResult<ActionOutcome> {
338        self.key_press_with_revision(key, None).await
339    }
340
341    /// Press and release a keyboard key while enforcing an optional revision.
342    pub async fn key_press_with_revision(
343        &self,
344        key: &str,
345        expected_revision: Option<u64>,
346    ) -> BrowserResult<ActionOutcome> {
347        validate_key(key)?;
348        self.cdp
349            .with_current_route(async {
350                self.require_expected_revision(expected_revision)?;
351                let code = key_code(key);
352                self.cdp
353                    .dispatch_key_event_with_modifiers("rawKeyDown", key, &code, "", 0)
354                    .await?;
355                if key.chars().count() == 1 {
356                    self.cdp
357                        .dispatch_key_event_with_modifiers("char", key, &code, key, 0)
358                        .await?;
359                }
360                self.cdp
361                    .dispatch_key_event_with_modifiers("keyUp", key, &code, "", 0)
362                    .await?;
363                self.action_outcome(ActionKind::KeyPress, None, None).await
364            })
365            .await
366    }
367
368    /// Execute a keyboard shortcut with modifier keys.
369    ///
370    /// Parses shortcuts like `"Ctrl+C"` or `"Meta+V"` and dispatches
371    /// the corresponding key events with the specified modifiers.
372    pub async fn shortcut(&self, shortcut: &str) -> BrowserResult<ActionOutcome> {
373        self.shortcut_with_revision(shortcut, None).await
374    }
375
376    /// Execute a keyboard shortcut while enforcing an optional revision.
377    pub async fn shortcut_with_revision(
378        &self,
379        shortcut: &str,
380        expected_revision: Option<u64>,
381    ) -> BrowserResult<ActionOutcome> {
382        let (modifiers, key) = parse_shortcut(shortcut)?;
383        self.cdp
384            .with_current_route(async {
385                self.require_expected_revision(expected_revision)?;
386                let code = key_code(&key);
387                self.cdp
388                    .dispatch_key_event_with_modifiers("rawKeyDown", &key, &code, "", modifiers)
389                    .await?;
390                self.cdp
391                    .dispatch_key_event_with_modifiers("keyUp", &key, &code, "", modifiers)
392                    .await?;
393                self.action_outcome(ActionKind::Shortcut, None, None).await
394            })
395            .await
396    }
397
398    /// Clear the contents of an editable element.
399    ///
400    /// Clicks the target, selects all content, then presses Backspace.
401    /// Verifies the element is empty afterward.
402    pub async fn clear(&self, target: &str) -> BrowserResult<ActionOutcome> {
403        self.clear_with_revision(target, None).await
404    }
405
406    /// Clear an editable element while enforcing an optional revision.
407    pub async fn clear_with_revision(
408        &self,
409        target: &str,
410        expected_revision: Option<u64>,
411    ) -> BrowserResult<ActionOutcome> {
412        self.cdp
413            .with_current_route(async {
414                self.require_expected_revision(expected_revision)?;
415                let element = self.resolve_element(target).await?;
416                let object_id = self.cdp.resolve_node_object(element.node_id, element.backend_dom_node_id).await?;
417                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
418                let editable = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement || this.isContentEditable}").await?)?;
419                if editable.as_bool() != Some(true) { return Err("clear target is not editable".into()); }
420                let clicked = self
421                    .pointer_click(ActionRequest::new(
422                        ActionKind::Click,
423                        target,
424                        expected_revision,
425                    ))
426                    .await?;
427                self.cdp.dispatch_select_all().await?;
428                self.key_press("Backspace").await?;
429                let empty = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this instanceof HTMLInputElement || this instanceof HTMLTextAreaElement ? this.value === '' : this.textContent === ''}").await?)?;
430                if empty.as_bool() != Some(true) { return Err("clear target did not become empty".into()); }
431                self.action_outcome_from_target(ActionKind::Clear, clicked.target)
432                    .await
433            })
434            .await
435    }
436
437    /// Check a checkbox or radio button.
438    ///
439    /// Ensures the target element's `checked` property is set to `true`.
440    pub async fn check(&self, target: &str) -> BrowserResult<ActionOutcome> {
441        self.set_checked(target, true).await
442    }
443
444    /// Check a control while enforcing an optional observation revision.
445    pub async fn check_with_revision(
446        &self,
447        target: &str,
448        expected_revision: Option<u64>,
449    ) -> BrowserResult<ActionOutcome> {
450        self.set_checked_with_revision(target, true, expected_revision)
451            .await
452    }
453
454    /// Uncheck a checkbox.
455    ///
456    /// Ensures the target element's `checked` property is set to `false`.
457    pub async fn uncheck(&self, target: &str) -> BrowserResult<ActionOutcome> {
458        self.set_checked(target, false).await
459    }
460
461    /// Uncheck a control while enforcing an optional observation revision.
462    pub async fn uncheck_with_revision(
463        &self,
464        target: &str,
465        expected_revision: Option<u64>,
466    ) -> BrowserResult<ActionOutcome> {
467        self.set_checked_with_revision(target, false, expected_revision)
468            .await
469    }
470
471    /// Select an option from a `<select>` element by value.
472    ///
473    /// `value` must be 1–4096 bytes. Fires `input` and `change` events.
474    pub async fn select_option(&self, target: &str, value: &str) -> BrowserResult<ActionOutcome> {
475        self.select_option_with_revision(target, value, None).await
476    }
477
478    /// Select an option while enforcing an optional observation revision.
479    pub async fn select_option_with_revision(
480        &self,
481        target: &str,
482        value: &str,
483        expected_revision: Option<u64>,
484    ) -> BrowserResult<ActionOutcome> {
485        if value.is_empty() || value.len() > 4096 {
486            return Err("select value must be 1..=4096 bytes".into());
487        }
488        let value_json = serde_json::to_string(value)?;
489        self.form_object_action(
490            target,
491            ActionKind::Select,
492            &format!(r#"function() {{ if (!(this instanceof HTMLSelectElement)) return {{ok:false,reason:'not_select'}}; const option = Array.from(this.options).find(option => option.value === {value_json}); if (!option) return {{ok:false,reason:'option_not_found'}}; this.value = option.value; this.dispatchEvent(new Event('input',{{bubbles:true}})); this.dispatchEvent(new Event('change',{{bubbles:true}})); return {{ok:this.value === option.value}}; }}"#),
493            expected_revision,
494        )
495        .await
496    }
497
498    pub async fn upload_files(
499        &self,
500        target: &str,
501        paths: &[PathBuf],
502    ) -> BrowserResult<ActionOutcome> {
503        self.upload_files_with_revision(target, paths, None).await
504    }
505
506    /// Upload files while enforcing an optional observation revision.
507    pub async fn upload_files_with_revision(
508        &self,
509        target: &str,
510        paths: &[PathBuf],
511        expected_revision: Option<u64>,
512    ) -> BrowserResult<ActionOutcome> {
513        self.policy.require(PolicyCapability::Upload)?;
514        self.cdp.with_current_route(async {
515            self.require_expected_revision(expected_revision)?;
516            if paths.is_empty() || paths.len() > 16 { return Err("upload requires 1..=16 files".into()); }
517            let mut files = Vec::with_capacity(paths.len());
518            for path in paths {
519                let canonical = self.policy.require_existing_path(path)?;
520                if !canonical.is_file() { return Err("upload path must be a regular file".into()); }
521                if !canonical.starts_with(&self.upload_root) { return Err("upload path is outside the allowed workspace root".into()); }
522                files.push(canonical.to_string_lossy().into_owned());
523            }
524            let element = self.resolve_element(target).await?;
525            let object_id = self.cdp.resolve_node_object(element.node_id, element.backend_dom_node_id).await?;
526            let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
527            self.verified_action_point(&remote.object_id).await?;
528            let input = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return {ok:this instanceof HTMLInputElement && this.type === 'file'}}").await?)?;
529            if input["ok"].as_bool() != Some(true) { return Err("upload target is not a file input".into()); }
530            if element.node_id.is_none() && element.backend_dom_node_id.is_none() { return Err("file input target has no DOM node ID".into()); }
531            self.cdp.set_file_input_files(element.node_id, element.backend_dom_node_id, &files).await?;
532            let verified = runtime_value(&self.cdp.call_on_object(&remote.object_id, "function(){return this.files.length}").await?)?;
533            if verified.as_u64() != Some(files.len() as u64) { return Err("file input did not retain the requested file count".into()); }
534            let outcome = self.action_outcome(ActionKind::Upload, Some(element), Some(serde_json::json!({"file_count": files.len()}))).await?;
535            self.record_audit("upload", format!("{} files", files.len()));
536            Ok(outcome)
537        }).await
538    }
539
540    async fn resolve_click_target(
541        &self,
542        target: &str,
543    ) -> BrowserResult<(ResolvedElement, String, Point)> {
544        const MAX_NODE_RESOLUTION_ATTEMPTS: usize = 3;
545        let mut last_error: Option<Box<dyn Error>> = None;
546
547        for attempt in 0..MAX_NODE_RESOLUTION_ATTEMPTS {
548            let element = self.resolve_element(target).await?;
549            let object_id = match self
550                .cdp
551                .resolve_node_object(element.node_id, element.backend_dom_node_id)
552                .await
553            {
554                Ok(object_id) => object_id,
555                Err(error) => {
556                    tracing::debug!(%error, attempt, "target node could not be resolved");
557                    last_error = Some(Box::new(TargetError {
558                        kind: TargetErrorKind::NotActionable,
559                        reason: Some(TargetActionabilityReason::NodeUnavailable),
560                        candidates: Vec::new(),
561                        recovery: None,
562                    }));
563                    if attempt + 1 < MAX_NODE_RESOLUTION_ATTEMPTS {
564                        tokio::time::sleep(Duration::from_millis(50)).await;
565                        continue;
566                    }
567                    break;
568                }
569            };
570
571            let local_point = match self.verified_action_point(&object_id).await {
572                Ok(point) => point,
573                Err(error)
574                    if error
575                        .downcast_ref::<TargetError>()
576                        .and_then(|target_error| target_error.reason)
577                        == Some(TargetActionabilityReason::NodeUnavailable) =>
578                {
579                    last_error = Some(error);
580                    if attempt + 1 < MAX_NODE_RESOLUTION_ATTEMPTS {
581                        tokio::time::sleep(Duration::from_millis(50)).await;
582                        continue;
583                    }
584                    break;
585                }
586                Err(error) => return Err(error),
587            };
588
589            return Ok((element, object_id, local_point));
590        }
591
592        Err(last_error.expect("node resolution retry loop must retain its last error"))
593    }
594
595    async fn pointer_click(&self, request: ActionRequest<'_>) -> BrowserResult<ActionOutcome> {
596        self.cdp
597            .with_current_route(async {
598                self.require_expected_revision(request.expected_revision)?;
599                let previous_revision = self.page_revision.load(Ordering::Relaxed);
600                let before = self.page_info().await.ok();
601                let (element, object_id, local_point) =
602                    self.resolve_click_target(request.target).await?;
603                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
604                let point = self.target_viewport_point(local_point).await?;
605                let events = if request.action == ActionKind::DoubleClick {
606                    self.mouse.generate_double_click_events(point)
607                } else {
608                    self.mouse.generate_click_events(point)
609                };
610                self.dispatch_pointer_events(&remote.object_id, local_point, point, events)
611                    .await?;
612                let (target_id, frame_id) = self.route_identity().await?;
613                let current_revision = self.invalidate_observation();
614                let after = self.page_info().await.ok();
615                Ok(ActionOutcome {
616                    status: ActionStatus::Succeeded,
617                    action: request.action,
618                    execution_id: self.next_execution_id(),
619                    target: Some(ActionTarget {
620                        label: element.label,
621                        reference: element.reference,
622                    }),
623                    revision: current_revision,
624                    previous_revision,
625                    current_revision,
626                    target_id,
627                    frame_id,
628                    verification: ActionVerificationEvidence {
629                        revision_delta: current_revision.saturating_sub(previous_revision),
630                        url_changed: before
631                            .as_ref()
632                            .zip(after.as_ref())
633                            .is_some_and(|(before, after)| before.url != after.url),
634                        title_changed: before
635                            .as_ref()
636                            .zip(after.as_ref())
637                            .is_some_and(|(before, after)| before.title != after.title),
638                        target_changed: before
639                            .as_ref()
640                            .zip(after.as_ref())
641                            .is_some_and(|(before, after)| before.target_id != after.target_id),
642                        frame_changed: before
643                            .as_ref()
644                            .zip(after.as_ref())
645                            .is_some_and(|(before, after)| before.frame_id != after.frame_id),
646                        ..ActionVerificationEvidence::default()
647                    },
648                    evidence: None,
649                })
650            })
651            .await
652    }
653
654    async fn dispatch_pointer_events(
655        &self,
656        object_id: &str,
657        local_point: Point,
658        point: Point,
659        events: Vec<crate::browser::mouse::MouseEvent>,
660    ) -> BrowserResult<()> {
661        let mut pointer = self.pointer.lock().await;
662        let start = match (self.interaction_mode, *pointer) {
663            (_, Some(point)) => point,
664            (InteractionMode::Human, None) => self
665                .viewport_center()
666                .await
667                .unwrap_or(Point { x: 640.0, y: 360.0 }),
668            (InteractionMode::Fast, None) => point,
669        };
670        let path = interaction_path(self.interaction_mode, &self.mouse, start, point);
671        if self.interaction_mode == InteractionMode::Human && pointer.is_none() {
672            self.cdp
673                .dispatch_mouse_event("mouseMoved", start.x, start.y, None, None)
674                .await?;
675        }
676        for window in path.windows(2) {
677            let next = window[1];
678            if self.interaction_mode == InteractionMode::Human {
679                tokio::time::sleep(self.mouse.move_delay(window[0], next)).await;
680            }
681            self.cdp
682                .dispatch_mouse_event("mouseMoved", next.x, next.y, None, None)
683                .await?;
684        }
685        let press_point = self.verified_action_point(object_id).await?;
686        if (press_point.x - local_point.x).abs() > 1.0
687            || (press_point.y - local_point.y).abs() > 1.0
688        {
689            return Err(TargetError {
690                kind: TargetErrorKind::NotActionable,
691                reason: Some(TargetActionabilityReason::GeometryChanged),
692                candidates: Vec::new(),
693                recovery: None,
694            }
695            .into());
696        }
697        let mut pressed = None;
698        for event in events {
699            if event.event_type == "mousePressed" {
700                pressed = Some(PressedButtonGuard {
701                    cdp: self.cdp.clone(),
702                    point,
703                    click_count: event.click_count,
704                    armed: true,
705                });
706            }
707            self.cdp
708                .dispatch_mouse_event(
709                    &event.event_type,
710                    event.x,
711                    event.y,
712                    Some(&event.button),
713                    Some(event.click_count),
714                )
715                .await?;
716            if event.event_type == "mouseReleased"
717                && let Some(mut guard) = pressed.take()
718            {
719                guard.armed = false;
720            }
721            if self.interaction_mode == InteractionMode::Human && event.event_type == "mousePressed"
722            {
723                tokio::time::sleep(self.mouse.click_delay()).await;
724            }
725        }
726        *pointer = Some(point);
727        Ok(())
728    }
729
730    /// Type text into the page.
731    ///
732    /// If `target` is provided, clicks the target element first to focus it,
733    /// then inserts the text via CDP `Input.insertText`. Otherwise types at
734    /// the current focus.
735    pub async fn type_text(
736        &self,
737        text: &str,
738        target: Option<&str>,
739    ) -> BrowserResult<ActionOutcome> {
740        self.type_text_with_expected_revision(text, target, None)
741            .await
742    }
743
744    /// Type text while enforcing an optional observation revision.
745    pub async fn type_text_with_expected_revision(
746        &self,
747        text: &str,
748        target: Option<&str>,
749        expected_revision: Option<u64>,
750    ) -> BrowserResult<ActionOutcome> {
751        self.cdp
752            .with_current_route(async {
753                self.require_expected_revision(expected_revision)?;
754                let previous_revision = self.page_revision.load(Ordering::Relaxed);
755                let target = match target {
756                    Some(target) => {
757                        self.pointer_click(ActionRequest::new(
758                            ActionKind::Click,
759                            target,
760                            expected_revision,
761                        ))
762                        .await?
763                        .target
764                    }
765                    None => None,
766                };
767                self.cdp.insert_text(text).await?;
768                let (target_id, frame_id) = self.route_identity().await?;
769                let current_revision = self.invalidate_observation();
770                Ok(ActionOutcome {
771                    status: ActionStatus::Succeeded,
772                    action: ActionKind::Type,
773                    execution_id: self.next_execution_id(),
774                    target,
775                    revision: current_revision,
776                    previous_revision,
777                    current_revision,
778                    target_id,
779                    frame_id,
780                    verification: ActionVerificationEvidence {
781                        revision_delta: current_revision.saturating_sub(previous_revision),
782                        ..ActionVerificationEvidence::default()
783                    },
784                    evidence: None,
785                })
786            })
787            .await
788    }
789
790    async fn move_pointer(&self, destination: Point) -> BrowserResult<()> {
791        let mut pointer = self.pointer.lock().await;
792        let start = pointer.unwrap_or(destination);
793        for window in
794            interaction_path(self.interaction_mode, &self.mouse, start, destination).windows(2)
795        {
796            if self.interaction_mode == InteractionMode::Human {
797                tokio::time::sleep(self.mouse.move_delay(window[0], window[1])).await;
798            }
799            self.cdp
800                .dispatch_mouse_event("mouseMoved", window[1].x, window[1].y, None, None)
801                .await?;
802        }
803        if start == destination {
804            self.cdp
805                .dispatch_mouse_event("mouseMoved", destination.x, destination.y, None, None)
806                .await?;
807        }
808        *pointer = Some(destination);
809        Ok(())
810    }
811
812    async fn keyboard_action(
813        &self,
814        action: ActionKind,
815        key: &str,
816        event_type: &str,
817        modifiers: i64,
818        expected_revision: Option<u64>,
819    ) -> BrowserResult<ActionOutcome> {
820        validate_key(key)?;
821        self.cdp
822            .with_current_route(async {
823                self.require_expected_revision(expected_revision)?;
824                self.cdp
825                    .dispatch_key_event_with_modifiers(
826                        event_type,
827                        key,
828                        &key_code(key),
829                        "",
830                        modifiers,
831                    )
832                    .await?;
833                self.action_outcome(action, None, None).await
834            })
835            .await
836    }
837
838    async fn set_checked(&self, target: &str, checked: bool) -> BrowserResult<ActionOutcome> {
839        self.set_checked_with_revision(target, checked, None).await
840    }
841
842    async fn set_checked_with_revision(
843        &self,
844        target: &str,
845        checked: bool,
846        expected_revision: Option<u64>,
847    ) -> BrowserResult<ActionOutcome> {
848        let action = if checked {
849            ActionKind::Check
850        } else {
851            ActionKind::Uncheck
852        };
853        let script = format!(
854            r#"function() {{ if (!(this instanceof HTMLInputElement) || !['checkbox','radio'].includes(this.type)) return {{ok:false,reason:'not_checkable'}}; if (this.checked !== {checked}) this.click(); return {{ok:this.checked === {checked}}}; }}"#
855        );
856        self.form_object_action(target, action, &script, expected_revision)
857            .await
858    }
859
860    async fn form_object_action(
861        &self,
862        target: &str,
863        action: ActionKind,
864        function: &str,
865        expected_revision: Option<u64>,
866    ) -> BrowserResult<ActionOutcome> {
867        self.cdp
868            .with_current_route(async {
869                self.require_expected_revision(expected_revision)?;
870                let element = self.resolve_element(target).await?;
871                let object_id = self
872                    .cdp
873                    .resolve_node_object(element.node_id, element.backend_dom_node_id)
874                    .await?;
875                let remote = RemoteObjectGuard::new(self.cdp.clone(), object_id);
876                self.verified_action_point(&remote.object_id).await?;
877                let result = self.cdp.call_on_object(&remote.object_id, function).await?;
878                let value = runtime_value(&result)?;
879                if value["ok"].as_bool() != Some(true) {
880                    let current_revision = self.page_revision.load(Ordering::Relaxed);
881                    return Err(ActionVerificationError {
882                        kind: ActionFailureKind::VerificationFailed,
883                        action,
884                        phase: ActionFailurePhase::Verification,
885                        recovery_strategy: RecoveryStrategy::Report,
886                        execution_id: Some(self.next_execution_id()),
887                        target: Some(ActionTarget {
888                            label: element.label,
889                            reference: element.reference,
890                        }),
891                        revision: current_revision,
892                        reason: value["reason"]
893                            .as_str()
894                            .unwrap_or("verification_failed")
895                            .to_string(),
896                    }
897                    .into());
898                }
899                self.action_outcome(action, Some(element), None).await
900            })
901            .await
902    }
903
904    async fn action_outcome(
905        &self,
906        action: ActionKind,
907        element: Option<ResolvedElement>,
908        evidence: Option<Value>,
909    ) -> BrowserResult<ActionOutcome> {
910        let target = element.map(|element| ActionTarget {
911            label: element.label,
912            reference: element.reference,
913        });
914        let mut outcome = self.action_outcome_from_target(action, target).await?;
915        outcome.evidence = evidence;
916        Ok(outcome)
917    }
918
919    pub(crate) async fn action_outcome_from_target(
920        &self,
921        action: ActionKind,
922        target: Option<ActionTarget>,
923    ) -> BrowserResult<ActionOutcome> {
924        let previous_revision = self.page_revision.load(Ordering::Relaxed);
925        let before_page = self
926            .observation_cache
927            .lock()
928            .await
929            .as_ref()
930            .map(|cached| cached.context.page.clone());
931        let (before_popup_count, before_dialog_open, before_download_sequence) = {
932            let topology = self.topology.lock().await;
933            (
934                topology.targets.len(),
935                topology.pending_dialog.is_some(),
936                self.download_sequence.load(Ordering::Relaxed),
937            )
938        };
939        if let Some(interception) = &self.policy_interception {
940            // A same-route command is an ordering barrier for synchronous
941            // click/form navigation. The interception itself remains active
942            // for delayed page-authored navigation after this action returns.
943            let _ = self.cdp.evaluate("0").await;
944            tokio::task::yield_now().await;
945            if let Some(error) = interception.take_denial().await {
946                return Err(error.into());
947            }
948        }
949        let (target_id, frame_id) = self.route_identity().await?;
950        let after_page = self.page_info().await.ok();
951        let (popup_opened, dialog_open) = {
952            let topology = self.topology.lock().await;
953            (
954                before_popup_count > 0 && topology.targets.len() > before_popup_count,
955                topology.pending_dialog.is_some() && !before_dialog_open,
956            )
957        };
958        let download_started =
959            self.download_sequence.load(Ordering::Relaxed) > before_download_sequence;
960        let current_revision = self.invalidate_observation();
961        Ok(ActionOutcome {
962            status: ActionStatus::Succeeded,
963            action,
964            execution_id: self.next_execution_id(),
965            target,
966            revision: current_revision,
967            previous_revision,
968            current_revision,
969            target_id,
970            frame_id,
971            verification: ActionVerificationEvidence {
972                revision_delta: current_revision.saturating_sub(previous_revision),
973                url_changed: before_page
974                    .as_ref()
975                    .zip(after_page.as_ref())
976                    .is_some_and(|(before, after)| before.url != after.url),
977                title_changed: before_page
978                    .as_ref()
979                    .zip(after_page.as_ref())
980                    .is_some_and(|(before, after)| before.title != after.title),
981                target_changed: before_page
982                    .as_ref()
983                    .zip(after_page.as_ref())
984                    .is_some_and(|(before, after)| before.target_id != after.target_id),
985                frame_changed: before_page
986                    .as_ref()
987                    .zip(after_page.as_ref())
988                    .is_some_and(|(before, after)| before.frame_id != after.frame_id),
989                popup_opened,
990                dialog_open,
991                download_started,
992                ..ActionVerificationEvidence::default()
993            },
994            evidence: None,
995        })
996    }
997
998    pub(crate) async fn viewport_center(&self) -> BrowserResult<Point> {
999        let value = self
1000            .evaluate_value("[window.innerWidth / 2, window.innerHeight / 2]")
1001            .await?;
1002        let coordinates = value
1003            .as_array()
1004            .filter(|coordinates| coordinates.len() == 2)
1005            .ok_or("viewport evaluation returned invalid coordinates")?;
1006        let x = coordinates[0]
1007            .as_f64()
1008            .ok_or("viewport width was not numeric")?;
1009        let y = coordinates[1]
1010            .as_f64()
1011            .ok_or("viewport height was not numeric")?;
1012        Ok(Point { x, y })
1013    }
1014
1015    pub(crate) async fn target_viewport_point(&self, point: Point) -> BrowserResult<Point> {
1016        let Some(frame_id) = self.cdp.active_frame() else {
1017            return Ok(point);
1018        };
1019        let frame = {
1020            let topology = self.topology.lock().await;
1021            topology
1022                .frames
1023                .iter()
1024                .find(|frame| frame.id == frame_id)
1025                .cloned()
1026        };
1027        let frame = match frame {
1028            Some(frame) => frame,
1029            None => self
1030                .list_frames()
1031                .await?
1032                .into_iter()
1033                .find(|frame| frame.id == frame_id)
1034                .ok_or("selected frame is no longer attached")?,
1035        };
1036        if frame.parent_id.is_none() {
1037            return Ok(point);
1038        }
1039        let (x, y) = self.cdp.frame_viewport_offset(&frame_id).await?;
1040        Ok(Point {
1041            x: point.x + x,
1042            y: point.y + y,
1043        })
1044    }
1045
1046    pub(crate) async fn evaluate_value(&self, expression: &str) -> BrowserResult<Value> {
1047        let raw = self.cdp.evaluate(expression).await?;
1048        runtime_value(&raw)
1049    }
1050
1051    pub(crate) fn invalidate_observation(&self) -> u64 {
1052        self.page_revision.fetch_add(1, Ordering::Relaxed) + 1
1053    }
1054
1055    pub(crate) fn require_expected_revision(
1056        &self,
1057        expected_revision: Option<u64>,
1058    ) -> BrowserResult<()> {
1059        if let Some(expected_revision) = expected_revision {
1060            let current_revision = self.page_revision.load(Ordering::Relaxed);
1061            if expected_revision != current_revision {
1062                return Err(ActionContractError::stale_revision_with_execution(
1063                    expected_revision,
1064                    current_revision,
1065                    self.next_execution_id(),
1066                )
1067                .into());
1068            }
1069        }
1070        Ok(())
1071    }
1072
1073    pub(crate) async fn verified_action_point(&self, object_id: &str) -> BrowserResult<Point> {
1074        let raw = match self.cdp.call_on_object(object_id, HIT_TEST_FUNCTION).await {
1075            Ok(raw) => raw,
1076            Err(error) => {
1077                tracing::debug!(%error, "target node could not be verified");
1078                return Err(TargetError {
1079                    kind: TargetErrorKind::NotActionable,
1080                    reason: Some(TargetActionabilityReason::NodeUnavailable),
1081                    candidates: Vec::new(),
1082                    recovery: None,
1083                }
1084                .into());
1085            }
1086        };
1087        let value = runtime_value(&raw)?;
1088        if value["ok"].as_bool() != Some(true) {
1089            let reason = value["reason"].as_str().unwrap_or("verification_failed");
1090            tracing::debug!(reason, "target actionability check failed");
1091            return Err(TargetError {
1092                kind: TargetErrorKind::NotActionable,
1093                reason: Some(actionability_reason(reason)),
1094                candidates: Vec::new(),
1095                recovery: None,
1096            }
1097            .into());
1098        }
1099        let x = value["x"]
1100            .as_f64()
1101            .ok_or("verified target x was not numeric")?;
1102        let y = value["y"]
1103            .as_f64()
1104            .ok_or("verified target y was not numeric")?;
1105        Ok(Point { x, y })
1106    }
1107}
glass/browser/session/action.rs

glass/browser/session/
action.rs
