eoka_agent/
lib.rs

1//! # eoka-agent
2//!
3//! AI agent interaction layer for browser automation. Use directly or via MCP.
4//!
5//! ## Quick Start
6//!
7//! ```rust,no_run
8//! use eoka_agent::Session;
9//!
10//! # #[tokio::main]
11//! # async fn main() -> eoka::Result<()> {
12//! let mut session = Session::launch().await?;
13//! session.goto("https://example.com").await?;
14//!
15//! // Observe → get compact element list → act by index
16//! session.observe().await?;
17//! println!("{}", session.element_list());
18//! session.click(0).await?;
19//!
20//! session.close().await?;
21//! # Ok(())
22//! # }
23//! ```
24
25pub mod annotate;
26pub mod observe;
27pub mod spa;
28pub mod target;
29
30pub use spa::{RouterType, SpaRouterInfo};
31pub use target::{BBox, LivePattern, Resolved, Target};
32
33use std::collections::HashSet;
34use std::fmt;
35
36use eoka::{BoundingBox, Page, Result};
37
38// Re-export eoka types that users need
39pub use eoka::{Browser, Error, StealthConfig};
40
41/// An interactive element on the page, identified by index.
42#[derive(Debug, Clone)]
43pub struct InteractiveElement {
44    /// Zero-based index (stable until next `observe()`)
45    pub index: usize,
46    /// HTML tag name (e.g. "button", "input", "a")
47    pub tag: String,
48    /// ARIA role if set
49    pub role: Option<String>,
50    /// Visible text content, truncated to 60 chars
51    pub text: String,
52    /// Placeholder attribute for inputs
53    pub placeholder: Option<String>,
54    /// Input type (only for `<input>` and `<select>` elements)
55    pub input_type: Option<String>,
56    /// Unique CSS selector for this element
57    pub selector: String,
58    /// Whether the element is checked (radio/checkbox)
59    pub checked: bool,
60    /// Current value of form element (None if empty or non-form)
61    pub value: Option<String>,
62    /// Bounding box in viewport coordinates
63    pub bbox: BoundingBox,
64    /// Fingerprint for stale element detection (hash of tag+text+attributes)
65    pub fingerprint: u64,
66}
67
68impl InteractiveElement {
69    /// Create a fingerprint from element properties for stale detection.
70    /// Includes enough fields to distinguish similar elements.
71    pub fn compute_fingerprint(
72        tag: &str,
73        text: &str,
74        role: Option<&str>,
75        input_type: Option<&str>,
76        placeholder: Option<&str>,
77        selector: &str,
78    ) -> u64 {
79        use std::collections::hash_map::DefaultHasher;
80        use std::hash::{Hash, Hasher};
81        let mut hasher = DefaultHasher::new();
82        tag.hash(&mut hasher);
83        text.hash(&mut hasher);
84        role.hash(&mut hasher);
85        input_type.hash(&mut hasher);
86        placeholder.hash(&mut hasher);
87        // Include selector prefix (first 50 chars) for positional uniqueness
88        selector[..selector.len().min(50)].hash(&mut hasher);
89        hasher.finish()
90    }
91}
92
93impl fmt::Display for InteractiveElement {
94    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
95        write!(f, "[{}] <{}", self.index, self.tag)?;
96        if let Some(ref t) = self.input_type {
97            if t != "text" {
98                write!(f, " type=\"{}\"", t)?;
99            }
100        }
101        f.write_str(">")?;
102        if self.checked {
103            f.write_str(" [checked]")?;
104        }
105        if !self.text.is_empty() {
106            write!(f, " \"{}\"", self.text)?;
107        }
108        if let Some(ref v) = self.value {
109            write!(f, " value=\"{}\"", v)?;
110        }
111        if let Some(ref p) = self.placeholder {
112            write!(f, " placeholder=\"{}\"", p)?;
113        }
114        if let Some(ref r) = self.role {
115            let redundant = (r == "button" && self.tag == "button")
116                || (r == "link" && self.tag == "a")
117                || (r == "menuitem" && self.tag == "a");
118            if !redundant {
119                write!(f, " role=\"{}\"", r)?;
120            }
121        }
122        Ok(())
123    }
124}
125
126/// Configuration for observation behavior.
127#[derive(Debug, Clone)]
128pub struct ObserveConfig {
129    /// Only include elements visible in the current viewport.
130    /// Dramatically reduces token count on long pages. Default: true.
131    pub viewport_only: bool,
132}
133
134impl Default for ObserveConfig {
135    fn default() -> Self {
136        Self {
137            viewport_only: true,
138        }
139    }
140}
141
142/// Result of a diff-based observation.
143#[derive(Debug)]
144pub struct ObserveDiff {
145    /// Indices of elements that appeared since last observe.
146    pub added: Vec<usize>,
147    /// Count of elements that disappeared since last observe.
148    pub removed: usize,
149    /// Total element count after this observe.
150    pub total: usize,
151}
152
153impl fmt::Display for ObserveDiff {
154    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
155        if self.added.is_empty() && self.removed == 0 {
156            write!(f, "no changes ({} elements)", self.total)
157        } else {
158            let mut need_sep = false;
159            if !self.added.is_empty() {
160                write!(f, "+{} added", self.added.len())?;
161                need_sep = true;
162            }
163            if self.removed > 0 {
164                if need_sep {
165                    write!(f, ", ")?;
166                }
167                write!(f, "-{} removed", self.removed)?;
168            }
169            write!(f, " ({} total)", self.total)
170        }
171    }
172}
173
174/// Wraps a `Page` with agent-friendly observation and interaction methods.
175///
176/// The core loop is: `observe()` → read `element_list()` → `click(i)` / `fill(i, text)`.
177pub struct AgentPage<'a> {
178    page: &'a Page,
179    elements: Vec<InteractiveElement>,
180    config: ObserveConfig,
181}
182
183impl<'a> AgentPage<'a> {
184    /// Create an AgentPage wrapping an existing eoka Page.
185    pub fn new(page: &'a Page) -> Self {
186        Self {
187            page,
188            elements: Vec::new(),
189            config: ObserveConfig::default(),
190        }
191    }
192
193    /// Create with custom observation config.
194    pub fn with_config(page: &'a Page, config: ObserveConfig) -> Self {
195        Self {
196            page,
197            elements: Vec::new(),
198            config,
199        }
200    }
201
202    /// Get a reference to the underlying Page.
203    pub fn page(&self) -> &Page {
204        self.page
205    }
206
207    // =========================================================================
208    // Observation
209    // =========================================================================
210
211    /// Snapshot the page: enumerate all interactive elements.
212    pub async fn observe(&mut self) -> Result<&[InteractiveElement]> {
213        self.elements = observe::observe(self.page, self.config.viewport_only).await?;
214        Ok(&self.elements)
215    }
216
217    /// Observe and return a diff against the previous observation.
218    /// Use this in multi-step sessions to minimize tokens — only send
219    /// `added_element_list()` to the LLM instead of the full list.
220    pub async fn observe_diff(&mut self) -> Result<ObserveDiff> {
221        let old_selectors: HashSet<String> =
222            self.elements.iter().map(|e| e.selector.clone()).collect();
223
224        self.elements = observe::observe(self.page, self.config.viewport_only).await?;
225
226        let new_selectors: HashSet<&str> =
227            self.elements.iter().map(|e| e.selector.as_str()).collect();
228
229        let added: Vec<usize> = self
230            .elements
231            .iter()
232            .filter(|e| !old_selectors.contains(&e.selector))
233            .map(|e| e.index)
234            .collect();
235
236        let removed = old_selectors
237            .iter()
238            .filter(|s| !new_selectors.contains(s.as_str()))
239            .count();
240
241        Ok(ObserveDiff {
242            added,
243            removed,
244            total: self.elements.len(),
245        })
246    }
247
248    /// Compact text list of only the added elements from the last `observe_diff()`.
249    pub fn added_element_list(&self, diff: &ObserveDiff) -> String {
250        let mut out = String::new();
251        for &idx in &diff.added {
252            if let Some(el) = self.elements.get(idx) {
253                out.push_str(&el.to_string());
254                out.push('\n');
255            }
256        }
257        out
258    }
259
260    /// Take an annotated screenshot with numbered boxes on each element.
261    /// Calls `observe()` first if no elements have been enumerated yet.
262    pub async fn screenshot(&mut self) -> Result<Vec<u8>> {
263        if self.elements.is_empty() {
264            self.observe().await?;
265        }
266        annotate::annotated_screenshot(self.page, &self.elements).await
267    }
268
269    /// Take a plain screenshot without annotations.
270    pub async fn screenshot_plain(&self) -> Result<Vec<u8>> {
271        self.page.screenshot().await
272    }
273
274    /// Compact text list for LLM consumption.
275    /// Each line: `[index] <tag type="x"> "text" placeholder="y"`
276    pub fn element_list(&self) -> String {
277        let mut out = String::with_capacity(self.elements.len() * 40);
278        for el in &self.elements {
279            out.push_str(&el.to_string());
280            out.push('\n');
281        }
282        out
283    }
284
285    /// Get element info by index.
286    pub fn get(&self, index: usize) -> Option<&InteractiveElement> {
287        self.elements.get(index)
288    }
289
290    /// Get all observed elements.
291    pub fn elements(&self) -> &[InteractiveElement] {
292        &self.elements
293    }
294
295    /// Number of observed elements.
296    pub fn len(&self) -> usize {
297        self.elements.len()
298    }
299
300    /// Whether the element list is empty.
301    pub fn is_empty(&self) -> bool {
302        self.elements.is_empty()
303    }
304
305    /// Find first element whose text contains the given substring (case-insensitive).
306    /// Returns the element index, or None.
307    pub fn find_by_text(&self, needle: &str) -> Option<usize> {
308        let needle_lower = needle.to_lowercase();
309        self.elements
310            .iter()
311            .find(|e| e.text.to_lowercase().contains(&needle_lower))
312            .map(|e| e.index)
313    }
314
315    /// Find all elements whose text contains the given substring (case-insensitive).
316    pub fn find_all_by_text(&self, needle: &str) -> Vec<usize> {
317        let needle_lower = needle.to_lowercase();
318        self.elements
319            .iter()
320            .filter(|e| e.text.to_lowercase().contains(&needle_lower))
321            .map(|e| e.index)
322            .collect()
323    }
324
325    // =========================================================================
326    // Actions (index-based)
327    // =========================================================================
328
329    /// Click an element by its index.
330    pub async fn click(&self, index: usize) -> Result<()> {
331        let el = self.require(index)?;
332        self.page.click(&el.selector).await
333    }
334
335    /// Try to click — returns `Ok(false)` if element is missing or not visible.
336    pub async fn try_click(&self, index: usize) -> Result<bool> {
337        let el = self.require(index)?;
338        self.page.try_click(&el.selector).await
339    }
340
341    /// Human-like click by index.
342    pub async fn human_click(&self, index: usize) -> Result<()> {
343        let el = self.require(index)?;
344        self.page.human_click(&el.selector).await
345    }
346
347    /// Clear and type into an element by index.
348    pub async fn fill(&self, index: usize, text: &str) -> Result<()> {
349        let el = self.require(index)?;
350        self.page.fill(&el.selector, text).await
351    }
352
353    /// Human-like fill by index.
354    pub async fn human_fill(&self, index: usize, text: &str) -> Result<()> {
355        let el = self.require(index)?;
356        self.page.human_fill(&el.selector, text).await
357    }
358
359    /// Focus an element by index.
360    pub async fn focus(&self, index: usize) -> Result<()> {
361        let el = self.require(index)?;
362        self.page
363            .execute(&format!(
364                "document.querySelector({})?.focus()",
365                serde_json::to_string(&el.selector).unwrap()
366            ))
367            .await
368    }
369
370    /// Select a dropdown option by index. `value` matches the option's value or visible text.
371    pub async fn select(&self, index: usize, value: &str) -> Result<()> {
372        let el = self.require(index)?;
373        let arg = serde_json::json!({ "sel": el.selector, "val": value });
374        let js = format!(
375            r#"(() => {{
376                const arg = {arg};
377                const sel = document.querySelector(arg.sel);
378                if (!sel) return false;
379                const opt = Array.from(sel.options).find(o => o.value === arg.val || o.text === arg.val);
380                if (!opt) return false;
381                sel.value = opt.value;
382                sel.dispatchEvent(new Event('change', {{ bubbles: true }}));
383                return true;
384            }})()"#,
385            arg = serde_json::to_string(&arg).unwrap()
386        );
387        let selected: bool = self.page.evaluate(&js).await?;
388        if !selected {
389            return Err(eoka::Error::ElementNotFound(format!(
390                "option \"{}\" in element [{}]",
391                value, index
392            )));
393        }
394        Ok(())
395    }
396
397    /// Get dropdown options for a select element. Returns vec of (value, text) pairs.
398    pub async fn options(&self, index: usize) -> Result<Vec<(String, String)>> {
399        let el = self.require(index)?;
400        let js = format!(
401            r#"(() => {{
402                const sel = document.querySelector({});
403                if (!sel || !sel.options) return '[]';
404                return JSON.stringify(Array.from(sel.options).map(o => [o.value, o.text]));
405            }})()"#,
406            serde_json::to_string(&el.selector).unwrap()
407        );
408        let json_str: String = self.page.evaluate(&js).await?;
409        let pairs: Vec<(String, String)> = serde_json::from_str(&json_str)
410            .map_err(|e| eoka::Error::CdpSimple(format!("options parse error: {}", e)))?;
411        Ok(pairs)
412    }
413
414    /// Scroll element at index into view.
415    pub async fn scroll_to(&self, index: usize) -> Result<()> {
416        let el = self.require(index)?;
417        let js = format!(
418            "document.querySelector({})?.scrollIntoView({{behavior:'smooth',block:'center'}})",
419            serde_json::to_string(&el.selector).unwrap()
420        );
421        self.page.execute(&js).await
422    }
423
424    // =========================================================================
425    // Navigation
426    // =========================================================================
427
428    /// Navigate to a URL. Clears element list (call `observe()` after navigation).
429    pub async fn goto(&mut self, url: &str) -> Result<()> {
430        self.elements.clear();
431        self.page.goto(url).await
432    }
433
434    /// Go back in history. Clears element list (call `observe()` after navigation).
435    pub async fn back(&mut self) -> Result<()> {
436        self.elements.clear();
437        self.page.back().await
438    }
439
440    /// Go forward in history. Clears element list (call `observe()` after navigation).
441    pub async fn forward(&mut self) -> Result<()> {
442        self.elements.clear();
443        self.page.forward().await
444    }
445
446    /// Reload the page. Clears element list (call `observe()` after navigation).
447    pub async fn reload(&mut self) -> Result<()> {
448        self.elements.clear();
449        self.page.reload().await
450    }
451
452    // =========================================================================
453    // Page state
454    // =========================================================================
455
456    /// Get the current URL.
457    pub async fn url(&self) -> Result<String> {
458        self.page.url().await
459    }
460
461    /// Get the page title.
462    pub async fn title(&self) -> Result<String> {
463        self.page.title().await
464    }
465
466    /// Get visible text content of the page.
467    pub async fn text(&self) -> Result<String> {
468        self.page.text().await
469    }
470
471    // =========================================================================
472    // Scrolling
473    // =========================================================================
474
475    /// Scroll down by approximately one viewport height.
476    pub async fn scroll_down(&self) -> Result<()> {
477        self.page
478            .execute("window.scrollBy(0, window.innerHeight * 0.8)")
479            .await
480    }
481
482    /// Scroll up by approximately one viewport height.
483    pub async fn scroll_up(&self) -> Result<()> {
484        self.page
485            .execute("window.scrollBy(0, -window.innerHeight * 0.8)")
486            .await
487    }
488
489    /// Scroll to the top of the page.
490    pub async fn scroll_to_top(&self) -> Result<()> {
491        self.page.execute("window.scrollTo(0, 0)").await
492    }
493
494    /// Scroll to the bottom of the page.
495    pub async fn scroll_to_bottom(&self) -> Result<()> {
496        self.page
497            .execute("window.scrollTo(0, document.body.scrollHeight)")
498            .await
499    }
500
501    // =========================================================================
502    // Waiting
503    // =========================================================================
504
505    /// Wait for text to appear on the page.
506    pub async fn wait_for_text(&self, text: &str, timeout_ms: u64) -> Result<()> {
507        self.page.wait_for_text(text, timeout_ms).await?;
508        Ok(())
509    }
510
511    /// Wait for a URL pattern (substring match).
512    pub async fn wait_for_url(&self, pattern: &str, timeout_ms: u64) -> Result<()> {
513        self.page.wait_for_url_contains(pattern, timeout_ms).await
514    }
515
516    /// Wait for network activity to settle.
517    pub async fn wait_for_idle(&self, timeout_ms: u64) -> Result<()> {
518        self.page.wait_for_network_idle(500, timeout_ms).await
519    }
520
521    /// Fixed delay in milliseconds.
522    pub async fn wait(&self, ms: u64) {
523        self.page.wait(ms).await;
524    }
525
526    // =========================================================================
527    // JavaScript
528    // =========================================================================
529
530    /// Evaluate JavaScript and return the result.
531    pub async fn eval<T: serde::de::DeserializeOwned>(&self, js: &str) -> Result<T> {
532        self.page.evaluate(js).await
533    }
534
535    /// Execute JavaScript (no return value).
536    pub async fn exec(&self, js: &str) -> Result<()> {
537        self.page.execute(js).await
538    }
539
540    // =========================================================================
541    // Keyboard
542    // =========================================================================
543
544    /// Press a key (e.g. "Enter", "Tab", "Escape", "ArrowDown", "Backspace").
545    pub async fn press_key(&self, key: &str) -> Result<()> {
546        self.page.human().press_key(key).await
547    }
548
549    /// Focus element by index and press Enter (common for form submission).
550    pub async fn submit(&self, index: usize) -> Result<()> {
551        self.focus(index).await?;
552        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
553        self.page.human().press_key("Enter").await
554    }
555
556    // =========================================================================
557    // Hover
558    // =========================================================================
559
560    /// Hover over element by index (triggers hover states, tooltips, menus).
561    pub async fn hover(&self, index: usize) -> Result<()> {
562        let el = self.require(index)?;
563        let cx = el.bbox.x + el.bbox.width / 2.0;
564        let cy = el.bbox.y + el.bbox.height / 2.0;
565        self.page
566            .session()
567            .dispatch_mouse_event(eoka::cdp::MouseEventType::MouseMoved, cx, cy, None, None)
568            .await
569    }
570
571    // =========================================================================
572    // Extraction
573    // =========================================================================
574
575    /// Extract structured data from the page using a JS expression that returns JSON.
576    ///
577    /// Example:
578    /// ```rust,no_run
579    /// # use eoka_agent::AgentPage;
580    /// # async fn example(agent: &AgentPage<'_>) -> eoka::Result<()> {
581    /// let titles: Vec<String> = agent.extract(
582    ///     "Array.from(document.querySelectorAll('h2')).map(h => h.textContent.trim())"
583    /// ).await?;
584    /// # Ok(())
585    /// # }
586    /// ```
587    pub async fn extract<T: serde::de::DeserializeOwned>(&self, js_expression: &str) -> Result<T> {
588        // Use eval() to handle multi-statement code - returns value of last expression
589        // Safely escape the JS code to prevent injection
590        let escaped_js = serde_json::to_string(js_expression)
591            .map_err(|e| eoka::Error::CdpSimple(format!("Failed to escape JS: {}", e)))?;
592        let js = format!("JSON.stringify(eval({}))", escaped_js);
593        let json_str: String = self.page.evaluate(&js).await?;
594        if json_str == "null" || json_str == "undefined" || json_str.is_empty() {
595            return Err(eoka::Error::CdpSimple(format!(
596                "extract returned null/undefined for: {}",
597                if js_expression.len() > 60 {
598                    &js_expression[..60]
599                } else {
600                    js_expression
601                }
602            )));
603        }
604        serde_json::from_str(&json_str).map_err(|e| {
605            eoka::Error::CdpSimple(format!(
606                "extract parse error: {} (got: {})",
607                e,
608                if json_str.len() > 80 {
609                    &json_str[..80]
610                } else {
611                    &json_str
612                }
613            ))
614        })
615    }
616
617    // =========================================================================
618    // Smart Waiting
619    // =========================================================================
620
621    /// Wait for the page to stabilize after an action.
622    /// Waits up to 2s for network idle, then 50ms for DOM settle.
623    /// Intentionally succeeds even if network doesn't fully idle (some sites never stop polling).
624    pub async fn wait_for_stable(&self) -> Result<()> {
625        // Best-effort network wait - ignore timeout (some sites have constant polling)
626        let _ = self.page.wait_for_network_idle(200, 2000).await;
627        // Brief DOM settle time
628        self.page.wait(50).await;
629        Ok(())
630    }
631
632    /// Click an element and wait for page to stabilize.
633    pub async fn click_and_wait(&mut self, index: usize) -> Result<()> {
634        self.click(index).await?;
635        self.wait_for_stable().await?;
636        // Invalidate elements since page likely changed
637        self.elements.clear();
638        Ok(())
639    }
640
641    /// Fill an element and wait for page to stabilize.
642    pub async fn fill_and_wait(&mut self, index: usize, text: &str) -> Result<()> {
643        self.fill(index, text).await?;
644        self.wait_for_stable().await?;
645        Ok(())
646    }
647
648    /// Select an option and wait for page to stabilize.
649    pub async fn select_and_wait(&mut self, index: usize, value: &str) -> Result<()> {
650        self.select(index, value).await?;
651        self.wait_for_stable().await?;
652        Ok(())
653    }
654
655    // =========================================================================
656    // SPA Navigation
657    // =========================================================================
658
659    /// Detect the SPA router type and current route state.
660    pub async fn spa_info(&self) -> Result<SpaRouterInfo> {
661        spa::detect_router(self.page).await
662    }
663
664    /// Navigate the SPA to a new path without page reload.
665    /// Automatically detects the router type and uses the appropriate navigation method.
666    /// Clears element list since the DOM will change.
667    pub async fn spa_navigate(&mut self, path: &str) -> Result<String> {
668        let info = spa::detect_router(self.page).await?;
669        let result = spa::spa_navigate(self.page, &info.router_type, path).await?;
670        self.elements.clear();
671        Ok(result)
672    }
673
674    /// Navigate browser history by delta steps.
675    /// delta = -1 goes back, delta = 1 goes forward.
676    /// Clears element list since the DOM will change.
677    pub async fn history_go(&mut self, delta: i32) -> Result<()> {
678        spa::history_go(self.page, delta).await?;
679        self.elements.clear();
680        Ok(())
681    }
682
683    // =========================================================================
684    // Internal
685    // =========================================================================
686
687    fn require(&self, index: usize) -> Result<&InteractiveElement> {
688        self.elements.get(index).ok_or_else(|| {
689            eoka::Error::ElementNotFound(format!(
690                "element [{}] (observed {} elements — call observe() to refresh)",
691                index,
692                self.elements.len()
693            ))
694        })
695    }
696}
697
698// =============================================================================
699// Session - owns Browser and Page, no lifetime gymnastics
700// =============================================================================
701
702/// A browser session that owns its browser and page.
703/// This is the primary API for most use cases.
704pub struct Session {
705    browser: Browser,
706    page: Page,
707    elements: Vec<InteractiveElement>,
708    config: ObserveConfig,
709}
710
711impl Session {
712    /// Launch a new browser and create an owned agent page.
713    pub async fn launch() -> Result<Self> {
714        let browser = Browser::launch().await?;
715        let page = browser.new_page("about:blank").await?;
716        Ok(Self {
717            browser,
718            page,
719            elements: Vec::new(),
720            config: ObserveConfig::default(),
721        })
722    }
723
724    /// Launch with custom stealth config.
725    pub async fn launch_with_config(stealth: StealthConfig) -> Result<Self> {
726        let browser = Browser::launch_with_config(stealth).await?;
727        let page = browser.new_page("about:blank").await?;
728        Ok(Self {
729            browser,
730            page,
731            elements: Vec::new(),
732            config: ObserveConfig::default(),
733        })
734    }
735
736    /// Set observation config.
737    pub fn set_observe_config(&mut self, config: ObserveConfig) {
738        self.config = config;
739    }
740
741    /// Get reference to underlying page.
742    pub fn page(&self) -> &Page {
743        &self.page
744    }
745
746    /// Get reference to browser.
747    pub fn browser(&self) -> &Browser {
748        &self.browser
749    }
750
751    // =========================================================================
752    // Observation
753    // =========================================================================
754
755    /// Snapshot the page: enumerate all interactive elements.
756    pub async fn observe(&mut self) -> Result<&[InteractiveElement]> {
757        self.elements = observe::observe(&self.page, self.config.viewport_only).await?;
758        Ok(&self.elements)
759    }
760
761    /// Take an annotated screenshot with numbered boxes on each element.
762    pub async fn screenshot(&mut self) -> Result<Vec<u8>> {
763        if self.elements.is_empty() {
764            self.observe().await?;
765        }
766        annotate::annotated_screenshot(&self.page, &self.elements).await
767    }
768
769    /// Compact text list for LLM consumption.
770    pub fn element_list(&self) -> String {
771        let mut out = String::with_capacity(self.elements.len() * 40);
772        for el in &self.elements {
773            out.push_str(&el.to_string());
774            out.push('\n');
775        }
776        out
777    }
778
779    /// Get element info by index.
780    pub fn get(&self, index: usize) -> Option<&InteractiveElement> {
781        self.elements.get(index)
782    }
783
784    /// Get all observed elements.
785    pub fn elements(&self) -> &[InteractiveElement] {
786        &self.elements
787    }
788
789    /// Number of observed elements.
790    pub fn len(&self) -> usize {
791        self.elements.len()
792    }
793
794    /// Whether the element list is empty.
795    pub fn is_empty(&self) -> bool {
796        self.elements.is_empty()
797    }
798
799    /// Find first element whose text contains the given substring (case-insensitive).
800    pub fn find_by_text(&self, needle: &str) -> Option<usize> {
801        let needle_lower = needle.to_lowercase();
802        self.elements
803            .iter()
804            .find(|e| e.text.to_lowercase().contains(&needle_lower))
805            .map(|e| e.index)
806    }
807
808    // =========================================================================
809    // Actions with auto-recovery
810    // =========================================================================
811
812    /// Get an element, verifying it still exists in DOM.
813    /// If element moved, returns error with hint about new location.
814    async fn require_fresh(&mut self, index: usize) -> Result<&InteractiveElement> {
815        // First check if element exists at index
816        let stored = self.elements.get(index).cloned();
817
818        if let Some(ref el) = stored {
819            // Verify the element still exists in DOM
820            let js = format!(
821                "!!document.querySelector({})",
822                serde_json::to_string(&el.selector).unwrap()
823            );
824            let exists: bool = self.page.evaluate(&js).await.unwrap_or(false);
825
826            if exists {
827                return self.elements.get(index).ok_or_else(|| {
828                    eoka::Error::ElementNotFound(format!("element [{}] disappeared", index))
829                });
830            }
831
832            // Element gone from DOM - re-observe and look for it
833            self.observe().await?;
834
835            // Try to find element with matching fingerprint
836            if let Some(new_idx) = self
837                .elements
838                .iter()
839                .position(|e| e.fingerprint == el.fingerprint)
840            {
841                // Found at different index - error with helpful message
842                return Err(eoka::Error::ElementNotFound(format!(
843                    "element [{}] \"{}\" moved to [{}] - call observe() to refresh",
844                    index, el.text, new_idx
845                )));
846            }
847
848            return Err(eoka::Error::ElementNotFound(format!(
849                "element [{}] \"{}\" no longer exists on page",
850                index, el.text
851            )));
852        }
853
854        Err(eoka::Error::ElementNotFound(format!(
855            "element [{}] not found (observed {} elements)",
856            index,
857            self.elements.len()
858        )))
859    }
860
861    /// Click an element, auto-recovering if stale.
862    /// Clears element cache since clicks often trigger navigation/DOM changes.
863    pub async fn click(&mut self, index: usize) -> Result<()> {
864        let el = self.require_fresh(index).await?;
865        let selector = el.selector.clone();
866        self.page.click(&selector).await?;
867        self.wait_for_stable().await?;
868        self.elements.clear(); // Clicks often change the page
869        Ok(())
870    }
871
872    /// Fill an element, auto-recovering if stale.
873    /// Does NOT clear element cache (typing rarely changes DOM structure).
874    pub async fn fill(&mut self, index: usize, text: &str) -> Result<()> {
875        let el = self.require_fresh(index).await?;
876        let selector = el.selector.clone();
877        self.page.fill(&selector, text).await?;
878        self.wait_for_stable().await?;
879        Ok(())
880    }
881
882    /// Select a dropdown option, auto-recovering if stale.
883    /// Clears element cache since onChange handlers may modify DOM.
884    pub async fn select(&mut self, index: usize, value: &str) -> Result<()> {
885        let el = self.require_fresh(index).await?;
886        let selector = el.selector.clone();
887        let arg = serde_json::json!({ "sel": selector, "val": value });
888        let js = format!(
889            r#"(() => {{
890                const arg = {arg};
891                const sel = document.querySelector(arg.sel);
892                if (!sel) return false;
893                const opt = Array.from(sel.options).find(o => o.value === arg.val || o.text === arg.val);
894                if (!opt) return false;
895                sel.value = opt.value;
896                sel.dispatchEvent(new Event('change', {{ bubbles: true }}));
897                return true;
898            }})()"#,
899            arg = serde_json::to_string(&arg).unwrap()
900        );
901        let selected: bool = self.page.evaluate(&js).await?;
902        if !selected {
903            return Err(eoka::Error::ElementNotFound(format!(
904                "option \"{}\" in element [{}]",
905                value, index
906            )));
907        }
908        self.wait_for_stable().await?;
909        self.elements.clear(); // onChange handlers may modify DOM
910        Ok(())
911    }
912
913    /// Hover over element.
914    pub async fn hover(&mut self, index: usize) -> Result<()> {
915        let el = self.require_fresh(index).await?;
916        let cx = el.bbox.x + el.bbox.width / 2.0;
917        let cy = el.bbox.y + el.bbox.height / 2.0;
918        self.page
919            .session()
920            .dispatch_mouse_event(eoka::cdp::MouseEventType::MouseMoved, cx, cy, None, None)
921            .await
922    }
923
924    /// Scroll element into view.
925    pub async fn scroll_to(&mut self, index: usize) -> Result<()> {
926        let el = self.require_fresh(index).await?;
927        let selector = el.selector.clone();
928        let js = format!(
929            "document.querySelector({})?.scrollIntoView({{behavior:'smooth',block:'center'}})",
930            serde_json::to_string(&selector).unwrap()
931        );
932        self.page.execute(&js).await
933    }
934
935    // =========================================================================
936    // Navigation
937    // =========================================================================
938
939    /// Navigate to a URL.
940    pub async fn goto(&mut self, url: &str) -> Result<()> {
941        self.elements.clear();
942        self.page.goto(url).await?;
943        self.wait_for_stable().await
944    }
945
946    /// Go back in history.
947    pub async fn back(&mut self) -> Result<()> {
948        self.elements.clear();
949        self.page.back().await?;
950        self.wait_for_stable().await
951    }
952
953    /// Go forward in history.
954    pub async fn forward(&mut self) -> Result<()> {
955        self.elements.clear();
956        self.page.forward().await?;
957        self.wait_for_stable().await
958    }
959
960    // =========================================================================
961    // Page state
962    // =========================================================================
963
964    /// Get the current URL.
965    pub async fn url(&self) -> Result<String> {
966        self.page.url().await
967    }
968
969    /// Get the page title.
970    pub async fn title(&self) -> Result<String> {
971        self.page.title().await
972    }
973
974    /// Get visible text content of the page.
975    pub async fn text(&self) -> Result<String> {
976        self.page.text().await
977    }
978
979    // =========================================================================
980    // Scrolling
981    // =========================================================================
982
983    /// Scroll down by approximately one viewport height.
984    pub async fn scroll_down(&self) -> Result<()> {
985        self.page
986            .execute("window.scrollBy(0, window.innerHeight * 0.8)")
987            .await
988    }
989
990    /// Scroll up by approximately one viewport height.
991    pub async fn scroll_up(&self) -> Result<()> {
992        self.page
993            .execute("window.scrollBy(0, -window.innerHeight * 0.8)")
994            .await
995    }
996
997    /// Scroll to top.
998    pub async fn scroll_to_top(&self) -> Result<()> {
999        self.page.execute("window.scrollTo(0, 0)").await
1000    }
1001
1002    /// Scroll to bottom.
1003    pub async fn scroll_to_bottom(&self) -> Result<()> {
1004        self.page
1005            .execute("window.scrollTo(0, document.body.scrollHeight)")
1006            .await
1007    }
1008
1009    // =========================================================================
1010    // Smart Waiting
1011    // =========================================================================
1012
1013    /// Wait for the page to stabilize after an action.
1014    /// Waits up to 2s for network idle, then 50ms for DOM settle.
1015    /// Intentionally succeeds even if network doesn't fully idle (some sites never stop polling).
1016    pub async fn wait_for_stable(&self) -> Result<()> {
1017        // Best-effort network wait - ignore timeout (some sites have constant polling)
1018        let _ = self.page.wait_for_network_idle(200, 2000).await;
1019        // Brief DOM settle time
1020        self.page.wait(50).await;
1021        Ok(())
1022    }
1023
1024    /// Fixed delay in milliseconds.
1025    pub async fn wait(&self, ms: u64) {
1026        self.page.wait(ms).await;
1027    }
1028
1029    // =========================================================================
1030    // Keyboard
1031    // =========================================================================
1032
1033    /// Press a key.
1034    pub async fn press_key(&self, key: &str) -> Result<()> {
1035        self.page.human().press_key(key).await
1036    }
1037
1038    // =========================================================================
1039    // JavaScript
1040    // =========================================================================
1041
1042    /// Evaluate JavaScript and return the result.
1043    pub async fn eval<T: serde::de::DeserializeOwned>(&self, js: &str) -> Result<T> {
1044        self.page.evaluate(js).await
1045    }
1046
1047    /// Execute JavaScript (no return value).
1048    pub async fn exec(&self, js: &str) -> Result<()> {
1049        self.page.execute(js).await
1050    }
1051
1052    // =========================================================================
1053    // SPA Navigation
1054    // =========================================================================
1055
1056    /// Detect the SPA router type and current route state.
1057    pub async fn spa_info(&self) -> Result<SpaRouterInfo> {
1058        spa::detect_router(&self.page).await
1059    }
1060
1061    /// Navigate the SPA to a new path without page reload.
1062    /// Automatically detects the router type and uses the appropriate navigation method.
1063    /// Clears element cache since the DOM will change.
1064    pub async fn spa_navigate(&mut self, path: &str) -> Result<String> {
1065        let info = spa::detect_router(&self.page).await?;
1066        let result = spa::spa_navigate(&self.page, &info.router_type, path).await?;
1067        self.elements.clear();
1068        Ok(result)
1069    }
1070
1071    /// Navigate browser history by delta steps.
1072    /// delta = -1 goes back, delta = 1 goes forward.
1073    /// Clears element cache since the DOM will change.
1074    pub async fn history_go(&mut self, delta: i32) -> Result<()> {
1075        spa::history_go(&self.page, delta).await?;
1076        self.elements.clear();
1077        Ok(())
1078    }
1079
1080    // =========================================================================
1081    // Cleanup
1082    // =========================================================================
1083
1084    /// Close the browser.
1085    pub async fn close(self) -> Result<()> {
1086        self.browser.close().await
1087    }
1088}
1089
1090#[cfg(test)]
1091mod tests {
1092    use super::*;
1093
1094    fn make_element(
1095        index: usize,
1096        tag: &str,
1097        text: &str,
1098        role: Option<&str>,
1099        input_type: Option<&str>,
1100        placeholder: Option<&str>,
1101        value: Option<&str>,
1102        checked: bool,
1103    ) -> InteractiveElement {
1104        let selector = format!("[data-idx=\"{}\"]", index);
1105        let fingerprint = InteractiveElement::compute_fingerprint(
1106            tag,
1107            text,
1108            role,
1109            input_type,
1110            placeholder,
1111            &selector,
1112        );
1113        InteractiveElement {
1114            index,
1115            tag: tag.to_string(),
1116            text: text.to_string(),
1117            role: role.map(|s| s.to_string()),
1118            input_type: input_type.map(|s| s.to_string()),
1119            placeholder: placeholder.map(|s| s.to_string()),
1120            value: value.map(|s| s.to_string()),
1121            checked,
1122            selector,
1123            bbox: BoundingBox {
1124                x: 0.0,
1125                y: 0.0,
1126                width: 100.0,
1127                height: 30.0,
1128            },
1129            fingerprint,
1130        }
1131    }
1132
1133    #[test]
1134    fn test_element_display_basic() {
1135        let el = make_element(0, "button", "Submit", None, None, None, None, false);
1136        assert_eq!(el.to_string(), "[0] <button> \"Submit\"");
1137    }
1138
1139    #[test]
1140    fn test_element_display_with_input_type() {
1141        // text type is suppressed
1142        let el = make_element(0, "input", "", None, Some("text"), None, None, false);
1143        assert_eq!(el.to_string(), "[0] <input>");
1144
1145        // other types are shown
1146        let el = make_element(0, "input", "", None, Some("password"), None, None, false);
1147        assert_eq!(el.to_string(), "[0] <input type=\"password\">");
1148    }
1149
1150    #[test]
1151    fn test_element_display_with_placeholder() {
1152        let el = make_element(
1153            0,
1154            "input",
1155            "",
1156            None,
1157            Some("text"),
1158            Some("Enter email"),
1159            None,
1160            false,
1161        );
1162        assert_eq!(el.to_string(), "[0] <input> placeholder=\"Enter email\"");
1163    }
1164
1165    #[test]
1166    fn test_element_display_with_value() {
1167        let el = make_element(
1168            0,
1169            "input",
1170            "",
1171            None,
1172            Some("text"),
1173            None,
1174            Some("hello"),
1175            false,
1176        );
1177        assert_eq!(el.to_string(), "[0] <input> value=\"hello\"");
1178    }
1179
1180    #[test]
1181    fn test_element_display_checked() {
1182        let el = make_element(0, "input", "", None, Some("checkbox"), None, None, true);
1183        assert_eq!(el.to_string(), "[0] <input type=\"checkbox\"> [checked]");
1184    }
1185
1186    #[test]
1187    fn test_element_display_redundant_role_suppressed() {
1188        // button role on button tag is redundant
1189        let el = make_element(
1190            0,
1191            "button",
1192            "Click",
1193            Some("button"),
1194            None,
1195            None,
1196            None,
1197            false,
1198        );
1199        assert_eq!(el.to_string(), "[0] <button> \"Click\"");
1200
1201        // link role on a tag is redundant
1202        let el = make_element(0, "a", "Link", Some("link"), None, None, None, false);
1203        assert_eq!(el.to_string(), "[0] <a> \"Link\"");
1204
1205        // menuitem role on a tag is redundant
1206        let el = make_element(0, "a", "Menu", Some("menuitem"), None, None, None, false);
1207        assert_eq!(el.to_string(), "[0] <a> \"Menu\"");
1208    }
1209
1210    #[test]
1211    fn test_element_display_non_redundant_role_shown() {
1212        // tab role on button is meaningful
1213        let el = make_element(0, "button", "Tab 1", Some("tab"), None, None, None, false);
1214        assert_eq!(el.to_string(), "[0] <button> \"Tab 1\" role=\"tab\"");
1215
1216        // button role on div is meaningful
1217        let el = make_element(0, "div", "Click", Some("button"), None, None, None, false);
1218        assert_eq!(el.to_string(), "[0] <div> \"Click\" role=\"button\"");
1219    }
1220
1221    #[test]
1222    fn test_observe_diff_display_no_changes() {
1223        let diff = ObserveDiff {
1224            added: vec![],
1225            removed: 0,
1226            total: 5,
1227        };
1228        assert_eq!(diff.to_string(), "no changes (5 elements)");
1229    }
1230
1231    #[test]
1232    fn test_observe_diff_display_added_only() {
1233        let diff = ObserveDiff {
1234            added: vec![5, 6],
1235            removed: 0,
1236            total: 7,
1237        };
1238        assert_eq!(diff.to_string(), "+2 added (7 total)");
1239    }
1240
1241    #[test]
1242    fn test_observe_diff_display_removed_only() {
1243        let diff = ObserveDiff {
1244            added: vec![],
1245            removed: 3,
1246            total: 2,
1247        };
1248        assert_eq!(diff.to_string(), "-3 removed (2 total)");
1249    }
1250
1251    #[test]
1252    fn test_observe_diff_display_both() {
1253        let diff = ObserveDiff {
1254            added: vec![3, 4],
1255            removed: 1,
1256            total: 5,
1257        };
1258        assert_eq!(diff.to_string(), "+2 added, -1 removed (5 total)");
1259    }
1260
1261    #[test]
1262    fn test_observe_config_default() {
1263        let config = ObserveConfig::default();
1264        assert!(config.viewport_only);
1265    }
1266}
eoka_agent/lib.rs

eoka_agent/
lib.rs
