ras_agent/application/
clickable_map.rs

1use ras_dom::{BrowserStateSummary, ClickableElement};
2use ras_llm::ChatMessage;
3
4const CLICKABLE_LIMIT: usize = 200;
5const NAME_BUDGET: usize = 80;
6
7/// A user message carrying the CURRENT page's clickable map (the live snapshot
8/// for THIS step). The model must choose indices from here, and the executor
9/// validates the chosen index against the SAME snapshot — so a picked index
10/// always exists (no stale 97-vs-87 mismatch). `None` when there are no
11/// clickables.
12pub(crate) fn build_current_page_message(summary: &BrowserStateSummary) -> Option<ChatMessage> {
13    let map = render_clickable_map(summary);
14    if map.is_empty() {
15        return None;
16    }
17    Some(ChatMessage::user_text(format!(
18        "CURRENT PAGE — choose actions using ONLY these live element indices:\nurl: {}\n{map}",
19        summary.url
20    )))
21}
22
23pub(crate) fn render_clickable_map(summary: &BrowserStateSummary) -> String {
24    if summary.clickables.is_empty() {
25        return String::new();
26    }
27    let mut ordered: Vec<&_> = summary.clickables.iter().collect();
28    let visible = |c: &&ClickableElement| c.bbox.width > 0.0 && c.bbox.height > 0.0;
29    ordered.sort_by_key(|c| !visible(c));
30
31    let mut buf = String::from("clickable_elements:\n");
32    for c in ordered.iter().take(CLICKABLE_LIMIT) {
33        buf.push_str(&format!("  [{}] {}", c.index, c.tag));
34        if let Some(name) = &c.ax_name {
35            buf.push_str(&format!(" \"{}\"", truncate(name, NAME_BUDGET)));
36        } else if let Some(label) = &c.label {
37            buf.push_str(&format!(" \"{}\"", truncate(label, NAME_BUDGET)));
38        }
39        buf.push('\n');
40    }
41    if summary.clickables.len() > CLICKABLE_LIMIT {
42        buf.push_str(&format!(
43            "  …and {} more (truncated)\n",
44            summary.clickables.len() - CLICKABLE_LIMIT
45        ));
46    }
47    buf
48}
49
50fn truncate(s: &str, max: usize) -> String {
51    if s.chars().count() <= max {
52        s.to_string()
53    } else {
54        let mut out: String = s.chars().take(max).collect();
55        out.push('…');
56        out
57    }
58}
59
60#[cfg(test)]
61mod tests {
62    use super::*;
63    use ras_dom::{BoundingBox, ClickableElement, PageStatistics};
64    use ras_types::{BackendNodeId, TargetId};
65
66    fn summary_with(clickables: Vec<ClickableElement>) -> BrowserStateSummary {
67        BrowserStateSummary {
68            target: TargetId("mock".into()),
69            url: "https://example.com/".parse().expect("url"),
70            title: "T".into(),
71            tree: None,
72            clickables,
73            screenshot_b64: None,
74            tabs: vec![],
75            page_stats: PageStatistics::default(),
76        }
77    }
78
79    fn click(idx: u32, tag: &str, ax_name: Option<&str>, label: Option<&str>) -> ClickableElement {
80        ClickableElement {
81            index: idx,
82            backend_node_id: BackendNodeId(idx as i64),
83            bbox: BoundingBox {
84                x: 0.0,
85                y: 0.0,
86                width: 1.0,
87                height: 1.0,
88            },
89            xpath: String::new(),
90            stable_hash: String::new(),
91            ax_name: ax_name.map(String::from),
92            tag: tag.into(),
93            label: label.map(String::from),
94        }
95    }
96
97    #[test]
98    fn empty_clickables_yields_empty_string() {
99        assert_eq!(render_clickable_map(&summary_with(vec![])), "");
100    }
101
102    #[test]
103    fn ax_name_takes_precedence_over_label() {
104        let s = summary_with(vec![click(0, "button", Some("Sign in"), Some("submit"))]);
105        let out = render_clickable_map(&s);
106        assert!(out.contains("[0] button \"Sign in\""));
107        assert!(!out.contains("submit"));
108    }
109
110    #[test]
111    fn label_used_when_no_ax_name() {
112        let s = summary_with(vec![click(0, "input", None, Some("user@example.com"))]);
113        let out = render_clickable_map(&s);
114        assert!(out.contains("[0] input \"user@example.com\""));
115    }
116
117    #[test]
118    fn no_quotes_when_neither_ax_nor_label() {
119        let s = summary_with(vec![click(0, "a", None, None)]);
120        let out = render_clickable_map(&s);
121        assert!(out.contains("[0] a\n"));
122        assert!(!out.contains('"'));
123    }
124
125    #[test]
126    fn visible_elements_survive_truncation_over_hidden() {
127        let mut many: Vec<ClickableElement> = (0..(CLICKABLE_LIMIT as u32))
128            .map(|i| click(i, "div", None, None))
129            .collect();
130        for c in &mut many {
131            c.bbox.width = 0.0;
132            c.bbox.height = 0.0;
133        }
134        let mut visible = click(999, "button", Some("Save"), None);
135        visible.bbox.width = 10.0;
136        visible.bbox.height = 10.0;
137        many.push(visible);
138        let out = render_clickable_map(&summary_with(many));
139        assert!(out.contains("[999] button \"Save\""));
140        assert!(out.contains("…and 1 more (truncated)"));
141    }
142
143    #[test]
144    fn truncates_beyond_limit_with_more_marker() {
145        let many: Vec<ClickableElement> = (0..(CLICKABLE_LIMIT as u32 + 5))
146            .map(|i| click(i, "div", None, None))
147            .collect();
148        let s = summary_with(many);
149        let out = render_clickable_map(&s);
150        assert!(out.contains(&format!("[{}]", CLICKABLE_LIMIT - 1)));
151        assert!(!out.contains(&format!("[{}]", CLICKABLE_LIMIT)));
152        assert!(out.contains("…and 5 more (truncated)"));
153    }
154
155    #[test]
156    fn current_page_message_carries_the_live_indices() {
157        let s = summary_with(vec![
158            click(0, "button", Some("Sign in"), None),
159            click(1, "input", Some("Email"), None),
160        ]);
161        let msg = build_current_page_message(&s).expect("message");
162        let ras_llm::ChatMessage::User(u) = msg else {
163            panic!("expected user message");
164        };
165        let ras_llm::ContentPart::Text { text } = &u.content[0] else {
166            panic!("expected text part");
167        };
168        assert!(text.contains("CURRENT PAGE"));
169        assert!(text.contains("[0] button \"Sign in\""));
170        assert!(text.contains("[1] input \"Email\""));
171    }
172
173    #[test]
174    fn current_page_message_is_none_without_clickables() {
175        assert!(build_current_page_message(&summary_with(vec![])).is_none());
176    }
177}
ras_agent/application/clickable_map.rs

ras_agent/application/
clickable_map.rs
