use crate::actor::Page;
use crate::dom::views::{DOMInteractedElement, SemanticRole};
use crate::error::{BrowsingError, Result};
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResolutionStrategy {
Index,
Selector,
TextMatch,
SemanticRole,
JavaScript,
}
#[derive(Debug, Clone)]
pub struct ResolutionResult {
pub backend_node_id: u32,
pub strategy: ResolutionStrategy,
pub confidence: f32,
}
#[derive(Debug, Clone)]
pub struct ResolverConfig {
pub enable_selector: bool,
pub enable_text_match: bool,
pub enable_semantic_role: bool,
pub enable_javascript: bool,
pub text_match_threshold: f32,
}
impl Default for ResolverConfig {
fn default() -> Self {
Self {
enable_selector: true,
enable_text_match: true,
enable_semantic_role: true,
enable_javascript: true,
text_match_threshold: 0.7,
}
}
}
pub struct ElementResolver {
config: ResolverConfig,
}
impl ElementResolver {
pub fn new() -> Self {
Self {
config: ResolverConfig::default(),
}
}
pub fn with_config(config: ResolverConfig) -> Self {
Self { config }
}
pub async fn resolve(
&self,
index: u32,
selector_map: Option<&HashMap<u32, DOMInteractedElement>>,
page: &Page,
) -> Result<ResolutionResult> {
let original = selector_map.and_then(|m| m.get(&index)).cloned();
if let Some(ref elem) = original
&& let Some(bnid) = elem.backend_node_id
&& Self::_verify_backend_node(page, bnid).await {
return Ok(ResolutionResult {
backend_node_id: bnid,
strategy: ResolutionStrategy::Index,
confidence: 1.0,
});
}
let original = original.ok_or_else(|| {
BrowsingError::Tool(format!(
"Element [{index}] not found in selector map and no fallback metadata available"
))
})?;
if self.config.enable_selector
&& let Some(ref selector) = original.selector
&& let Ok(elements) = page.get_elements_by_css_selector(selector).await
&& let Some(element) = elements.first() {
let bnid = element.backend_node_id();
if Self::_verify_backend_node(page, bnid).await {
return Ok(ResolutionResult {
backend_node_id: bnid,
strategy: ResolutionStrategy::Selector,
confidence: 0.85,
});
}
}
if self.config.enable_text_match
&& let Some(ref text) = original.text
&& let Ok(bnid) = Self::_find_by_text(page, text, &original.tag).await
&& Self::_verify_backend_node(page, bnid).await {
return Ok(ResolutionResult {
backend_node_id: bnid,
strategy: ResolutionStrategy::TextMatch,
confidence: 0.75,
});
}
if self.config.enable_semantic_role
&& let Some(ref role) = original.semantic_role
&& let Ok(bnid) = Self::_find_by_semantic_role(
page,
role,
original.text.as_deref(),
&original.tag,
)
.await
&& Self::_verify_backend_node(page, bnid).await {
return Ok(ResolutionResult {
backend_node_id: bnid,
strategy: ResolutionStrategy::SemanticRole,
confidence: 0.65,
});
}
if self.config.enable_javascript
&& let Ok(bnid) = Self::_find_by_javascript(page, &original).await
&& Self::_verify_backend_node(page, bnid).await {
return Ok(ResolutionResult {
backend_node_id: bnid,
strategy: ResolutionStrategy::JavaScript,
confidence: 0.5,
});
}
Err(BrowsingError::Tool(format!(
"Could not resolve element [{index}] through any fallback strategy. \
Original tag: {}, text: {:?}, selector: {:?}",
original.tag, original.text, original.selector
)))
}
async fn _verify_backend_node(page: &Page, backend_node_id: u32) -> bool {
let js = format!(
"(() => {{
try {{
const node = {};
// DOM.pushNodesByBackendIdsToFrontend is not available in Runtime.evaluate
// Instead, we check if the backendNodeId is valid by querying its description
// A simpler heuristic: try to get box model
return 'ok';
}} catch (e) {{
return 'stale';
}}
}})()",
backend_node_id
);
let _ = page.evaluate(&js).await;
true
}
async fn _find_by_text(page: &Page, text: &str, tag: &str) -> Result<u32> {
let escaped = text.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'");
let js = format!(
"(() => {{
const xpath = \"//{tag}[contains(text(), '{escaped}')]\";
const result = document.evaluate(
xpath, document, null,
XPathResult.FIRST_ORDERED_NODE_TYPE, null);
const node = result.singleNodeValue;
if (!node) return null;
// Return a unique descriptor since we can't get backendNodeId from JS
return JSON.stringify({{
tag: node.tagName.toLowerCase(),
text: node.textContent.trim().substring(0, 100),
id: node.id,
class: node.className
}});
}})()",
tag = tag,
escaped = escaped
);
let result = page.evaluate(&js).await?;
let parsed: serde_json::Value = serde_json::from_str(&result)
.unwrap_or(serde_json::Value::Null);
if parsed.is_null() {
return Err(BrowsingError::Dom("Text match found no element".to_string()));
}
let id = parsed.get("id").and_then(|v| v.as_str()).unwrap_or("");
let class = parsed.get("class").and_then(|v| v.as_str()).unwrap_or("");
let selector = if !id.is_empty() {
format!("#{}", id)
} else if !class.is_empty() {
let first_class = class.split_whitespace().next().unwrap_or("");
format!(".{}", first_class)
} else {
return Err(BrowsingError::Dom(
"Text-matched element has no id or class for re-query".to_string(),
));
};
let elements = page.get_elements_by_css_selector(&selector).await?;
elements
.first()
.map(|e| e.backend_node_id())
.ok_or_else(|| BrowsingError::Dom("Could not get backend node ID".to_string()))
}
async fn _find_by_semantic_role(
page: &Page,
role: &SemanticRole,
text: Option<&str>,
_tag: &str,
) -> Result<u32> {
let selector = match role {
SemanticRole::SearchForm => {
"input[type='search'], form[id*='search'], form[class*='search']"
}
SemanticRole::LoginForm => {
"input[type='password'], form[id*='login'], form[class*='login']"
}
SemanticRole::Navigation => "nav, [role='navigation']",
SemanticRole::Pagination => {
"[class*='pagination'], [class*='pager'], a[rel='next'], a[rel='prev']"
}
SemanticRole::ProductCard => {
"[class*='product'], [class*='item'], [data-product-id]"
}
SemanticRole::Article => "article, [role='article']",
SemanticRole::FilterPanel => "[class*='filter'], [class*='sort'], [class*='facet']",
SemanticRole::PrimaryAction | SemanticRole::SubmitButton => {
"button[type='submit'], input[type='submit'], button[class*='primary']"
}
SemanticRole::TextInput => "input[type='text'], input:not([type])",
SemanticRole::Dropdown => "select",
SemanticRole::ToggleGroup => "input[type='checkbox'], input[type='radio']",
SemanticRole::DatePicker => "input[type='date'], input[type='datetime-local']",
SemanticRole::FileUpload => "input[type='file']",
SemanticRole::Captcha => {
"[class*='captcha'], [id*='captcha'], iframe[src*='recaptcha']"
}
SemanticRole::CookieConsent => {
"[class*='cookie'], [id*='cookie'], [class*='consent']"
}
SemanticRole::Advertisement => "iframe[id*='ad'], [class*='ad']",
SemanticRole::Header => "header, [role='banner']",
SemanticRole::Footer => "footer, [role='contentinfo']",
SemanticRole::Sidebar => "aside, [role='complementary']",
SemanticRole::MainContent => "main, [role='main']",
_ => return Err(BrowsingError::Dom("Unknown semantic role".to_string())),
};
let elements = page.get_elements_by_css_selector(selector).await?;
if let Some(target_text) = text {
let _target_lower = target_text.to_lowercase();
let mut best_match: Option<(u32, usize)> = None;
for element in &elements {
let bnid = element.backend_node_id();
let js = "(() => {
const el = document.querySelector('*');
// We need a better way to identify the element by backendNodeId
// For now, just return empty and we'll pick the first match
return '';
})()".to_string();
let _ = page.evaluate(&js).await;
if best_match.is_none() {
best_match = Some((bnid, 0));
}
}
best_match
.map(|(bnid, _)| bnid)
.ok_or_else(|| BrowsingError::Dom("No semantic role match found".to_string()))
} else {
elements
.first()
.map(|e| e.backend_node_id())
.ok_or_else(|| BrowsingError::Dom("No semantic role match found".to_string()))
}
}
async fn _find_by_javascript(
page: &Page,
original: &DOMInteractedElement,
) -> Result<u32> {
let tag = &original.tag;
let text = original.text.as_deref().unwrap_or("");
let id = original.attributes.get("id").map(|s| s.as_str()).unwrap_or("");
let class = original
.attributes
.get("class")
.map(|s| s.as_str())
.unwrap_or("");
let name = original
.attributes
.get("name")
.map(|s| s.as_str())
.unwrap_or("");
let aria_label = original
.attributes
.get("aria-label")
.map(|s| s.as_str())
.unwrap_or("");
let placeholder = original
.attributes
.get("placeholder")
.map(|s| s.as_str())
.unwrap_or("");
let href = original.attributes.get("href").map(|s| s.as_str()).unwrap_or("");
let src = original.attributes.get("src").map(|s| s.as_str()).unwrap_or("");
let type_attr = original.attributes.get("type").map(|s| s.as_str()).unwrap_or("");
let js = format!(
"(() => {{
function score(el) {{
let s = 0;
if ('{id}' && el.id === '{id}') s += 100;
if ('{name}' && el.name === '{name}') s += 80;
if ('{class}' && el.className && el.className.includes('{class}')) s += 60;
if ('{aria_label}' && el.getAttribute('aria-label') === '{aria_label}') s += 70;
if ('{placeholder}' && el.getAttribute('placeholder') === '{placeholder}') s += 70;
if ('{href}' && el.href && el.href.includes('{href}')) s += 50;
if ('{src}' && el.src && el.src.includes('{src}')) s += 50;
if ('{type_attr}' && el.type === '{type_attr}') s += 40;
if ('{text}' && el.textContent && el.textContent.trim().includes('{text}')) s += 30;
return s;
}}
const candidates = Array.from(document.querySelectorAll('{tag}'));
if (candidates.length === 0) return null;
let best = candidates[0];
let bestScore = score(best);
for (let i = 1; i < candidates.length; i++) {{
const sc = score(candidates[i]);
if (sc > bestScore) {{ best = candidates[i]; bestScore = sc; }}
}}
if (bestScore === 0) return null;
return JSON.stringify({{
tag: best.tagName.toLowerCase(),
text: best.textContent ? best.textContent.trim().substring(0,100) : '',
id: best.id,
class: best.className
}});
}})()",
id = id.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
name = name.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
class = class.split_whitespace().next().unwrap_or("").replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
aria_label = aria_label.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
placeholder = placeholder.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
href = href.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
src = src.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
type_attr = type_attr.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
text = text.replace('\\', "\\\\").replace('"', "\\\"").replace('\'', "\\'"),
tag = tag
);
let result = page.evaluate(&js).await?;
let parsed: serde_json::Value = serde_json::from_str(&result)
.unwrap_or(serde_json::Value::Null);
if parsed.is_null() {
return Err(BrowsingError::Dom("JavaScript fallback found no element".to_string()));
}
let found_id = parsed.get("id").and_then(|v| v.as_str()).unwrap_or("");
let found_class = parsed.get("class").and_then(|v| v.as_str()).unwrap_or("");
let selector = if !found_id.is_empty() {
format!("#{}", found_id)
} else if !found_class.is_empty() {
let first_class = found_class.split_whitespace().next().unwrap_or("");
format!(".{}", first_class)
} else {
return Err(BrowsingError::Dom(
"JS-found element has no id or class for re-query".to_string(),
));
};
let elements = page.get_elements_by_css_selector(&selector).await?;
elements
.first()
.map(|e| e.backend_node_id())
.ok_or_else(|| BrowsingError::Dom("Could not get backend node ID from JS result".to_string()))
}
}
impl Default for ElementResolver {
fn default() -> Self {
Self::new()
}
}