headless_engine/dom/
interactive.rs

1use scraper::{Html, Selector};
2use serde::{Deserialize, Serialize};
3
4#[derive(Debug, Clone, Serialize, Deserialize)]
5pub struct InteractiveElement {
6    pub index: usize,
7    pub tag: String,
8    pub role: String,
9    pub text: String,
10    pub name: String,
11    pub input_type: String,
12    pub placeholder: String,
13    pub value: String,
14    pub href: String,
15    pub selector: String,
16    pub is_clickable: bool,
17    pub is_input: bool,
18}
19
20impl InteractiveElement {
21    pub fn to_agent_string(&self) -> String {
22        if self.is_input {
23            let label = if !self.placeholder.is_empty() {
24                format!("placeholder=\"{}\"", self.placeholder)
25            } else if !self.name.is_empty() {
26                format!("name=\"{}\"", self.name)
27            } else {
28                format!("type=\"{}\"", self.input_type)
29            };
30            format!(
31                "[{}] <input {}> (value=\"{}\")",
32                self.index, label, self.value
33            )
34        } else if self.tag == "button" || self.role == "button" {
35            format!("[{}] <button \"{}\">", self.index, self.text)
36        } else if self.tag == "a" {
37            format!("[{}] <a \"{}\"> -> {}", self.index, self.text, self.href)
38        } else {
39            format!("[{}] <{} \"{}\">", self.index, self.tag, self.text)
40        }
41    }
42}
43
44#[derive(Debug, Clone, Serialize, Deserialize)]
45pub struct PageObservation {
46    pub url: String,
47    pub title: String,
48    pub is_captcha_detected: bool,
49    pub interactive_elements: Vec<InteractiveElement>,
50    pub agent_tree_text: String,
51    pub content_summary_markdown: String,
52}
53
54pub struct InteractiveParser;
55
56impl InteractiveParser {
57    pub fn parse(html_str: &str, base_url: Option<&str>) -> Vec<InteractiveElement> {
58        let document = Html::parse_document(html_str);
59        let mut elements = Vec::new();
60
61        let query = "a[href], button, input, textarea, select, [role='button'], [role='link'], [role='checkbox'], [role='tab'], [onclick]";
62        let selector = match Selector::parse(query) {
63            Ok(s) => s,
64            Err(_) => return elements,
65        };
66
67        let mut index = 1;
68        for el in document.select(&selector) {
69            let tag = el.value().name().to_string();
70            let role = el.value().attr("role").unwrap_or("").to_string();
71            let name = el.value().attr("name").unwrap_or("").to_string();
72            let input_type = el.value().attr("type").unwrap_or("text").to_string();
73            let placeholder = el.value().attr("placeholder").unwrap_or("").to_string();
74            let value = el.value().attr("value").unwrap_or("").to_string();
75            let id = el.value().attr("id").unwrap_or("").to_string();
76            let class = el.value().attr("class").unwrap_or("").to_string();
77
78            // Skip hidden inputs and disabled elements
79            if input_type == "hidden"
80                || el.value().attr("disabled").is_some()
81                || el.value().attr("aria-hidden") == Some("true")
82            {
83                continue;
84            }
85
86            let mut text = el.text().collect::<Vec<_>>().join(" ").trim().to_string();
87            if text.is_empty() {
88                if let Some(aria_label) = el
89                    .value()
90                    .attr("aria-label")
91                    .or_else(|| el.value().attr("title"))
92                {
93                    text = aria_label.trim().to_string();
94                }
95            }
96
97            // Resolve href for links
98            let raw_href = el.value().attr("href").unwrap_or("");
99            let href = if !raw_href.is_empty() && !raw_href.starts_with("javascript:") {
100                Self::resolve_url(raw_href, base_url)
101            } else {
102                String::new()
103            };
104
105            // Skip empty non-input elements
106            if text.is_empty()
107                && href.is_empty()
108                && placeholder.is_empty()
109                && name.is_empty()
110                && id.is_empty()
111            {
112                continue;
113            }
114
115            let is_input = tag == "input" || tag == "textarea" || tag == "select";
116            let is_clickable = tag == "a"
117                || tag == "button"
118                || role == "button"
119                || role == "link"
120                || el.value().attr("onclick").is_some();
121
122            // Build unique CSS selector
123            let css_selector = if !id.is_empty() {
124                format!("#{}", id)
125            } else if !name.is_empty() {
126                format!("{}[name='{}']", tag, name)
127            } else if !class.is_empty() {
128                let first_class = class.split_whitespace().next().unwrap_or("");
129                format!("{}.{}", tag, first_class)
130            } else {
131                tag.clone()
132            };
133
134            elements.push(InteractiveElement {
135                index,
136                tag,
137                role,
138                text,
139                name,
140                input_type,
141                placeholder,
142                value,
143                href,
144                selector: css_selector,
145                is_clickable,
146                is_input,
147            });
148
149            index += 1;
150        }
151
152        elements
153    }
154
155    fn resolve_url(href: &str, base_url: Option<&str>) -> String {
156        if href.starts_with("http://") || href.starts_with("https://") {
157            return href.to_string();
158        }
159        if let Some(base) = base_url {
160            if href.starts_with("//") {
161                return format!("https:{}", href);
162            }
163            if href.starts_with('/') {
164                if let Some(idx) = base.find("://") {
165                    let after = &base[idx + 3..];
166                    let host = after.split('/').next().unwrap_or(after);
167                    let scheme = &base[..idx + 3];
168                    return format!("{}{}{}", scheme, host, href);
169                }
170            }
171            let trimmed_base = base.split('?').next().unwrap_or(base);
172            let parent = if trimmed_base.ends_with('/') {
173                trimmed_base
174            } else if let Some(last_slash) = trimmed_base.rfind('/') {
175                &trimmed_base[..last_slash + 1]
176            } else {
177                trimmed_base
178            };
179            return format!("{}{}", parent, href);
180        }
181        href.to_string()
182    }
183}
headless_engine/dom/interactive.rs

headless_engine/dom/
interactive.rs
