headless_engine/dom/
interactive.rs

1use scraper::{Html, Selector};
2use serde::{Deserialize, Serialize};
3
4#[derive(Debug, Clone, Serialize, Deserialize)]
5pub struct InteractiveElement {
6    pub index: usize,
7    pub tag: String,
8    pub role: String,
9    pub text: String,
10    pub name: String,
11    pub input_type: String,
12    pub placeholder: String,
13    pub value: String,
14    pub href: String,
15    pub selector: String,
16    pub is_clickable: bool,
17    pub is_input: bool,
18}
19
20impl InteractiveElement {
21    pub fn to_agent_string(&self) -> String {
22        if self.is_input {
23            let label = if !self.placeholder.is_empty() {
24                format!("placeholder=\"{}\"", self.placeholder)
25            } else if !self.name.is_empty() {
26                format!("name=\"{}\"", self.name)
27            } else {
28                format!("type=\"{}\"", self.input_type)
29            };
30            format!(
31                "[{}] <input {}> (value=\"{}\")",
32                self.index, label, self.value
33            )
34        } else if self.tag == "button" || self.role == "button" {
35            format!("[{}] <button \"{}\">", self.index, self.text)
36        } else if self.tag == "a" {
37            format!("[{}] <a \"{}\"> -> {}", self.index, self.text, self.href)
38        } else {
39            format!("[{}] <{} \"{}\">", self.index, self.tag, self.text)
40        }
41    }
42}
43
44#[derive(Debug, Clone, Serialize, Deserialize)]
45pub struct PageObservation {
46    pub url: String,
47    pub title: String,
48    pub is_captcha_detected: bool,
49    pub interactive_elements: Vec<InteractiveElement>,
50    pub agent_tree_text: String,
51    pub content_summary_markdown: String,
52}
53
54pub struct InteractiveParser;
55
56impl InteractiveParser {
57    pub fn parse(html_str: &str, base_url: Option<&str>) -> Vec<InteractiveElement> {
58        let document = Html::parse_document(html_str);
59        let mut elements = Vec::new();
60
61        let query = "a[href], button, input, textarea, select, [contenteditable='true'], [role='button'], [role='link'], [role='checkbox'], [role='tab'], [onclick], [data-testid]";
62        let selector = match Selector::parse(query) {
63            Ok(s) => s,
64            Err(_) => return elements,
65        };
66
67        let mut index = 1;
68        for el in document.select(&selector) {
69            let tag = el.value().name().to_string();
70            let role = el.value().attr("role").unwrap_or("").to_string();
71            let name = el.value().attr("name").unwrap_or("").to_string();
72            let input_type = el.value().attr("type").unwrap_or("text").to_string();
73            let placeholder = el.value().attr("placeholder").unwrap_or("").to_string();
74            let value = el.value().attr("value").unwrap_or("").to_string();
75            let id = el.value().attr("id").unwrap_or("").to_string();
76            let class = el.value().attr("class").unwrap_or("").to_string();
77            let is_contenteditable = el.value().attr("contenteditable") == Some("true");
78
79            // Skip hidden inputs and disabled elements
80            if input_type == "hidden"
81                || el.value().attr("disabled").is_some()
82                || el.value().attr("aria-hidden") == Some("true")
83            {
84                continue;
85            }
86
87            let mut text = el.text().collect::<Vec<_>>().join(" ").trim().to_string();
88            if text.is_empty() {
89                if let Some(aria_label) = el
90                    .value()
91                    .attr("aria-label")
92                    .or_else(|| el.value().attr("title"))
93                {
94                    text = aria_label.trim().to_string();
95                }
96            }
97
98            // Resolve href for links
99            let raw_href = el.value().attr("href").unwrap_or("");
100            let href = if !raw_href.is_empty() && !raw_href.starts_with("javascript:") {
101                Self::resolve_url(raw_href, base_url)
102            } else {
103                String::new()
104            };
105
106            // Skip empty non-input elements
107            if text.is_empty()
108                && href.is_empty()
109                && placeholder.is_empty()
110                && name.is_empty()
111                && id.is_empty()
112                && !is_contenteditable
113            {
114                continue;
115            }
116
117            let is_input = tag == "input"
118                || tag == "textarea"
119                || tag == "select"
120                || is_contenteditable
121                || id == "prompt-textarea";
122            let is_clickable = tag == "a"
123                || tag == "button"
124                || role == "button"
125                || role == "link"
126                || el.value().attr("onclick").is_some();
127
128            // Build unique CSS selector
129            let css_selector = if !id.is_empty() {
130                format!("#{}", id)
131            } else if !name.is_empty() {
132                format!("{}[name='{}']", tag, name)
133            } else if !class.is_empty() {
134                let first_class = class.split_whitespace().next().unwrap_or("");
135                format!("{}.{}", tag, first_class)
136            } else {
137                tag.clone()
138            };
139
140            elements.push(InteractiveElement {
141                index,
142                tag,
143                role,
144                text,
145                name,
146                input_type,
147                placeholder,
148                value,
149                href,
150                selector: css_selector,
151                is_clickable,
152                is_input,
153            });
154
155            index += 1;
156        }
157
158        elements
159    }
160
161    fn resolve_url(href: &str, base_url: Option<&str>) -> String {
162        if href.starts_with("http://") || href.starts_with("https://") {
163            return href.to_string();
164        }
165        if let Some(base) = base_url {
166            if href.starts_with("//") {
167                return format!("https:{}", href);
168            }
169            if href.starts_with('/') {
170                if let Some(idx) = base.find("://") {
171                    let after = &base[idx + 3..];
172                    let host = after.split('/').next().unwrap_or(after);
173                    let scheme = &base[..idx + 3];
174                    return format!("{}{}{}", scheme, host, href);
175                }
176            }
177            let trimmed_base = base.split('?').next().unwrap_or(base);
178            let parent = if trimmed_base.ends_with('/') {
179                trimmed_base
180            } else if let Some(last_slash) = trimmed_base.rfind('/') {
181                &trimmed_base[..last_slash + 1]
182            } else {
183                trimmed_base
184            };
185            return format!("{}{}", parent, href);
186        }
187        href.to_string()
188    }
189}
headless_engine/dom/interactive.rs

headless_engine/dom/
interactive.rs
