browsing 0.1.6

Browser automation: navigate, click, extract, screenshot. Standalone browser control via CDP.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
//! Fallback element resolution chain
//!
//! When an element index becomes stale (DOM changed, page navigated, AJAX update),
//! this resolver tries multiple strategies to find the intended element:
//!
//! 1. **Index** — Direct backend_node_id lookup from selector_map
//! 2. **Selector** — CSS selector / XPath stored in the selector_map entry
//! 3. **Text match** — Find element with same text content
//! 4. **Semantic role** — Find element with same inferred semantic role + similar text
//! 5. **JavaScript** — Evaluate custom JS to locate the element

use crate::actor::Page;
use crate::dom::views::{DOMInteractedElement, SemanticRole};
use crate::error::{BrowsingError, Result};
use std::collections::HashMap;

/// Strategy that successfully resolved the element
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResolutionStrategy {
    /// Direct index/backend_node_id lookup
    Index,
    /// Matched by stored CSS selector or XPath
    Selector,
    /// Matched by text content
    TextMatch,
    /// Matched by semantic role + text similarity
    SemanticRole,
    /// Found via JavaScript evaluation
    JavaScript,
}

/// Result of resolving an element through the fallback chain
#[derive(Debug, Clone)]
pub struct ResolutionResult {
    /// Backend node ID of the resolved element
    pub backend_node_id: u32,
    /// Which strategy succeeded
    pub strategy: ResolutionStrategy,
    /// Confidence score (0.0–1.0)
    pub confidence: f32,
}

/// Configuration for the fallback resolution chain
#[derive(Debug, Clone)]
pub struct ResolverConfig {
    /// Whether to enable selector fallback
    pub enable_selector: bool,
    /// Whether to enable text-match fallback
    pub enable_text_match: bool,
    /// Whether to enable semantic-role fallback
    pub enable_semantic_role: bool,
    /// Whether to enable JS-evaluation fallback
    pub enable_javascript: bool,
    /// Minimum text similarity (0.0–1.0) for text-match to be considered a hit
    pub text_match_threshold: f32,
}

impl Default for ResolverConfig {
    fn default() -> Self {
        Self {
            enable_selector: true,
            enable_text_match: true,
            enable_semantic_role: true,
            enable_javascript: true,
            text_match_threshold: 0.7,
        }
    }
}

/// Resolves elements through a multi-strategy fallback chain.
pub struct ElementResolver {
    config: ResolverConfig,
}

impl ElementResolver {
    /// Create a resolver with default config
    pub fn new() -> Self {
        Self {
            config: ResolverConfig::default(),
        }
    }

    /// Create a resolver with custom config
    pub fn with_config(config: ResolverConfig) -> Self {
        Self { config }
    }

    /// Resolve an element by its original index, falling back through strategies.
    ///
    /// # Arguments
    /// * `index` — The interactive index assigned during DOM serialization
    /// * `selector_map` — Snapshot of element metadata from the last serialization
    /// * `page` — Live page handle for querying the current DOM
    pub async fn resolve(
        &self,
        index: u32,
        selector_map: Option<&HashMap<u32, DOMInteractedElement>>,
        page: &Page,
    ) -> Result<ResolutionResult> {
        let original = selector_map.and_then(|m| m.get(&index)).cloned();

        // ── Strategy 1: Index / backend_node_id ──────────────────────────────
        if let Some(ref elem) = original
            && let Some(bnid) = elem.backend_node_id
                && Self::_verify_backend_node(page, bnid).await {
                    return Ok(ResolutionResult {
                        backend_node_id: bnid,
                        strategy: ResolutionStrategy::Index,
                        confidence: 1.0,
                    });
                }

        // We need the original metadata to try fallbacks
        let original = original.ok_or_else(|| {
            BrowsingError::Tool(format!(
                "Element [{index}] not found in selector map and no fallback metadata available"
            ))
        })?;

        // ── Strategy 2: Stored selector ────────────────────────────────────
        if self.config.enable_selector
            && let Some(ref selector) = original.selector
                && let Ok(elements) = page.get_elements_by_css_selector(selector).await
                    && let Some(element) = elements.first() {
                        let bnid = element.backend_node_id();
                        if Self::_verify_backend_node(page, bnid).await {
                            return Ok(ResolutionResult {
                                backend_node_id: bnid,
                                strategy: ResolutionStrategy::Selector,
                                confidence: 0.85,
                            });
                        }
                    }

        // ── Strategy 3: Text content match ───────────────────────────────────
        if self.config.enable_text_match
            && let Some(ref text) = original.text
                && let Ok(bnid) = Self::_find_by_text(page, text, &original.tag).await
                    && Self::_verify_backend_node(page, bnid).await {
                        return Ok(ResolutionResult {
                            backend_node_id: bnid,
                            strategy: ResolutionStrategy::TextMatch,
                            confidence: 0.75,
                        });
                    }

        // ── Strategy 4: Semantic role + text similarity ──────────────────────
        if self.config.enable_semantic_role
            && let Some(ref role) = original.semantic_role
                && let Ok(bnid) = Self::_find_by_semantic_role(
                    page,
                    role,
                    original.text.as_deref(),
                    &original.tag,
                )
                .await
                    && Self::_verify_backend_node(page, bnid).await {
                        return Ok(ResolutionResult {
                            backend_node_id: bnid,
                            strategy: ResolutionStrategy::SemanticRole,
                            confidence: 0.65,
                        });
                    }

        // ── Strategy 5: JavaScript evaluation ──────────────────────────────
        if self.config.enable_javascript
            && let Ok(bnid) = Self::_find_by_javascript(page, &original).await
                && Self::_verify_backend_node(page, bnid).await {
                    return Ok(ResolutionResult {
                        backend_node_id: bnid,
                        strategy: ResolutionStrategy::JavaScript,
                        confidence: 0.5,
                    });
                }

        Err(BrowsingError::Tool(format!(
            "Could not resolve element [{index}] through any fallback strategy. \
             Original tag: {}, text: {:?}, selector: {:?}",
            original.tag, original.text, original.selector
        )))
    }

    /// Verify that a backend node ID is still valid in the current DOM.
    async fn _verify_backend_node(page: &Page, backend_node_id: u32) -> bool {
        page.verify_backend_node(backend_node_id).await
    }

    /// Find an element by its text content using JavaScript.
    async fn _find_by_text(page: &Page, text: &str, tag: &str) -> Result<u32> {
        let escaped = text.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'");
        let js = format!(
            "(() => {{
                const xpath = \"//{tag}[contains(text(), '{escaped}')]\";
                const result = document.evaluate(
                    xpath, document, null,
                    XPathResult.FIRST_ORDERED_NODE_TYPE, null);
                const node = result.singleNodeValue;
                if (!node) return null;
                // Return a unique descriptor since we can't get backendNodeId from JS
                return JSON.stringify({{
                    tag: node.tagName.toLowerCase(),
                    text: node.textContent.trim().substring(0, 100),
                    id: node.id,
                    class: node.className
                }});
            }})()",
            tag = tag,
            escaped = escaped
        );

        let result = page.evaluate(&js).await?;
        let parsed: serde_json::Value = serde_json::from_str(&result)
            .unwrap_or(serde_json::Value::Null);

        if parsed.is_null() {
            return Err(BrowsingError::Dom("Text match found no element".to_string()));
        }

        // Since we can't get backendNodeId from JS, we use the descriptor to query
        // via CDP DOM.querySelector using id or class
        let id = parsed.get("id").and_then(|v| v.as_str()).unwrap_or("");
        let class = parsed.get("class").and_then(|v| v.as_str()).unwrap_or("");

        let selector = if !id.is_empty() {
            format!("#{}", id)
        } else if !class.is_empty() {
            let first_class = class.split_whitespace().next().unwrap_or("");
            format!(".{}", first_class)
        } else {
            return Err(BrowsingError::Dom(
                "Text-matched element has no id or class for re-query".to_string(),
            ));
        };

        let elements = page.get_elements_by_css_selector(&selector).await?;
        elements
            .first()
            .map(|e| e.backend_node_id())
            .ok_or_else(|| BrowsingError::Dom("Could not get backend node ID".to_string()))
    }

    /// Find an element by semantic role and optional text.
    async fn _find_by_semantic_role(
        page: &Page,
        role: &SemanticRole,
        text: Option<&str>,
        _tag: &str,
    ) -> Result<u32> {
        // Map semantic role to CSS selector / attribute hints
        let selector = match role {
            SemanticRole::SearchForm => {
                "input[type='search'], form[id*='search'], form[class*='search']"
            }
            SemanticRole::LoginForm => {
                "input[type='password'], form[id*='login'], form[class*='login']"
            }
            SemanticRole::Navigation => "nav, [role='navigation']",
            SemanticRole::Pagination => {
                "[class*='pagination'], [class*='pager'], a[rel='next'], a[rel='prev']"
            }
            SemanticRole::ProductCard => {
                "[class*='product'], [class*='item'], [data-product-id]"
            }
            SemanticRole::Article => "article, [role='article']",
            SemanticRole::FilterPanel => "[class*='filter'], [class*='sort'], [class*='facet']",
            SemanticRole::PrimaryAction | SemanticRole::SubmitButton => {
                "button[type='submit'], input[type='submit'], button[class*='primary']"
            }
            SemanticRole::TextInput => "input[type='text'], input:not([type])",
            SemanticRole::Dropdown => "select",
            SemanticRole::ToggleGroup => "input[type='checkbox'], input[type='radio']",
            SemanticRole::DatePicker => "input[type='date'], input[type='datetime-local']",
            SemanticRole::FileUpload => "input[type='file']",
            SemanticRole::Captcha => {
                "[class*='captcha'], [id*='captcha'], iframe[src*='recaptcha']"
            }
            SemanticRole::CookieConsent => {
                "[class*='cookie'], [id*='cookie'], [class*='consent']"
            }
            SemanticRole::Advertisement => "iframe[id*='ad'], [class*='ad']",
            SemanticRole::Header => "header, [role='banner']",
            SemanticRole::Footer => "footer, [role='contentinfo']",
            SemanticRole::Sidebar => "aside, [role='complementary']",
            SemanticRole::MainContent => "main, [role='main']",
            _ => return Err(BrowsingError::Dom("Unknown semantic role".to_string())),
        };

        let elements = page.get_elements_by_css_selector(selector).await?;

        // If text hint is provided, try to pick the closest match
        if let Some(target_text) = text {
            let _target_lower = target_text.to_lowercase();
            let mut best_match: Option<(u32, usize)> = None; // (backend_node_id, similarity)

            for element in &elements {
                let bnid = element.backend_node_id();
                // Use JS to get text content for comparison
                let js = "(() => {
                        const el = document.querySelector('*');
                        // We need a better way to identify the element by backendNodeId
                        // For now, just return empty and we'll pick the first match
                        return '';
                    })()".to_string();
                let _ = page.evaluate(&js).await;
                // Simplified: just pick first match for now
                if best_match.is_none() {
                    best_match = Some((bnid, 0));
                }
            }

            best_match
                .map(|(bnid, _)| bnid)
                .ok_or_else(|| BrowsingError::Dom("No semantic role match found".to_string()))
        } else {
            elements
                .first()
                .map(|e| e.backend_node_id())
                .ok_or_else(|| BrowsingError::Dom("No semantic role match found".to_string()))
        }
    }

    /// Last-resort JavaScript strategy: build a robust query from stored metadata.
    async fn _find_by_javascript(
        page: &Page,
        original: &DOMInteractedElement,
    ) -> Result<u32> {
        let tag = &original.tag;
        let text = original.text.as_deref().unwrap_or("");
        let id = original.attributes.get("id").map(|s| s.as_str()).unwrap_or("");
        let class = original
            .attributes
            .get("class")
            .map(|s| s.as_str())
            .unwrap_or("");
        let name = original
            .attributes
            .get("name")
            .map(|s| s.as_str())
            .unwrap_or("");
        let aria_label = original
            .attributes
            .get("aria-label")
            .map(|s| s.as_str())
            .unwrap_or("");
        let placeholder = original
            .attributes
            .get("placeholder")
            .map(|s| s.as_str())
            .unwrap_or("");
        let href = original.attributes.get("href").map(|s| s.as_str()).unwrap_or("");
        let src = original.attributes.get("src").map(|s| s.as_str()).unwrap_or("");
        let type_attr = original.attributes.get("type").map(|s| s.as_str()).unwrap_or("");

        let js = format!(
            "(() => {{
                function score(el) {{
                    let s = 0;
                    if ('{id}' && el.id === '{id}') s += 100;
                    if ('{name}' && el.name === '{name}') s += 80;
                    if ('{class}' && el.className && el.className.includes('{class}')) s += 60;
                    if ('{aria_label}' && el.getAttribute('aria-label') === '{aria_label}') s += 70;
                    if ('{placeholder}' && el.getAttribute('placeholder') === '{placeholder}') s += 70;
                    if ('{href}' && el.href && el.href.includes('{href}')) s += 50;
                    if ('{src}' && el.src && el.src.includes('{src}')) s += 50;
                    if ('{type_attr}' && el.type === '{type_attr}') s += 40;
                    if ('{text}' && el.textContent && el.textContent.trim().includes('{text}')) s += 30;
                    return s;
                }}
                const candidates = Array.from(document.querySelectorAll('{tag}'));
                if (candidates.length === 0) return null;
                let best = candidates[0];
                let bestScore = score(best);
                for (let i = 1; i < candidates.length; i++) {{
                    const sc = score(candidates[i]);
                    if (sc > bestScore) {{ best = candidates[i]; bestScore = sc; }}
                }}
                if (bestScore === 0) return null;
                return JSON.stringify({{
                    tag: best.tagName.toLowerCase(),
                    text: best.textContent ? best.textContent.trim().substring(0,100) : '',
                    id: best.id,
                    class: best.className
                }});
            }})()",
            id = id.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            name = name.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            class = class.split_whitespace().next().unwrap_or("").replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            aria_label = aria_label.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            placeholder = placeholder.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            href = href.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            src = src.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            type_attr = type_attr.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            text = text.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
            tag = tag
        );

        let result = page.evaluate(&js).await?;
        let parsed: serde_json::Value = serde_json::from_str(&result)
            .unwrap_or(serde_json::Value::Null);

        if parsed.is_null() {
            return Err(BrowsingError::Dom("JavaScript fallback found no element".to_string()));
        }

        // Re-query via CDP using id or class
        let found_id = parsed.get("id").and_then(|v| v.as_str()).unwrap_or("");
        let found_class = parsed.get("class").and_then(|v| v.as_str()).unwrap_or("");

        let selector = if !found_id.is_empty() {
            format!("#{}", found_id)
        } else if !found_class.is_empty() {
            let first_class = found_class.split_whitespace().next().unwrap_or("");
            format!(".{}", first_class)
        } else {
            return Err(BrowsingError::Dom(
                "JS-found element has no id or class for re-query".to_string(),
            ));
        };

        let elements = page.get_elements_by_css_selector(&selector).await?;
        elements
            .first()
            .map(|e| e.backend_node_id())
            .ok_or_else(|| BrowsingError::Dom("Could not get backend node ID from JS result".to_string()))
    }
}

impl Default for ElementResolver {
    fn default() -> Self {
        Self::new()
    }
}