1use super::config::BrowseConfig;
26use super::engine::{BrowserEngine, BrowserError, Observation, ObservedElement};
27use crate::tools::{AgentTool, AgentToolResult, ToolContext, ToolError};
28use async_trait::async_trait;
29use futures::StreamExt;
30use parking_lot::Mutex;
31use serde_json::{Value, json};
32use std::sync::Arc;
33use tokio::sync::oneshot;
34
35const STOP_WORDS: &[&str] = &[
36 "the", "a", "an", "on", "in", "of", "to", "and", "or", "for", "with", "click", "tap", "press",
37];
38
39const ROLE_PRIORITY: &[&str] = &[
40 "button", "link", "textbox", "checkbox", "combobox", "menuitem", "tab", "generic",
41];
42
43const DEFAULT_CANDIDATE_TOP_N: usize = 20;
44
45fn tokenize(text: &str) -> Vec<String> {
48 text.split_whitespace()
49 .map(|w| {
50 w.trim_matches(|c: char| !c.is_alphanumeric())
51 .to_lowercase()
52 })
53 .filter(|w| !w.is_empty() && !STOP_WORDS.contains(&w.as_str()))
54 .collect()
55}
56
57fn role_rank(role: &str) -> usize {
58 ROLE_PRIORITY
59 .iter()
60 .position(|r| *r == role)
61 .unwrap_or(ROLE_PRIORITY.len())
62}
63
64pub fn candidate_tier(goal: &str, obs: &Observation, top_n: usize) -> Vec<ObservedElement> {
69 let goal_tokens = tokenize(goal);
70 if goal_tokens.is_empty() {
71 return Vec::new();
72 }
73 let dom_index: std::collections::HashMap<&str, usize> = obs
74 .elements
75 .iter()
76 .enumerate()
77 .map(|(i, e)| (e.ref_id.as_str(), i))
78 .collect();
79 let mut scored: Vec<(f64, &ObservedElement)> = obs
80 .elements
81 .iter()
82 .filter(|e| e.visible && e.interactive)
83 .filter_map(|e| {
84 let hay = tokenize(&format!("{} {}", e.name, e.role));
85 if hay.is_empty() {
86 return None;
87 }
88 let matched = goal_tokens.iter().filter(|t| hay.contains(t)).count();
89 if matched == 0 {
90 return None;
91 }
92 Some((matched as f64 / goal_tokens.len() as f64, e))
93 })
94 .collect();
95 scored.sort_by(|a, b| {
96 b.0.partial_cmp(&a.0)
97 .unwrap_or(std::cmp::Ordering::Equal)
98 .then(role_rank(&a.1.role).cmp(&role_rank(&b.1.role)))
99 .then(b.1.name.len().cmp(&a.1.name.len()))
100 .then(
101 dom_index
102 .get(a.1.ref_id.as_str())
103 .unwrap_or(&0)
104 .cmp(dom_index.get(b.1.ref_id.as_str()).unwrap_or(&0)),
105 )
106 });
107 scored
108 .into_iter()
109 .take(top_n)
110 .map(|(_, e)| e.clone())
111 .collect()
112}
113
114#[derive(Debug, Clone, Copy, PartialEq, Eq)]
119pub enum BrowserActAction {
120 Click,
122 Type,
124 Fill,
126 SelectOption,
128 Check,
130 Uncheck,
132 Press,
134 Hover,
136}
137
138impl BrowserActAction {
139 pub fn parse_str(s: &str) -> Option<Self> {
141 match s {
142 "click" => Some(Self::Click),
143 "type" => Some(Self::Type),
144 "fill" => Some(Self::Fill),
145 "select_option" => Some(Self::SelectOption),
146 "check" => Some(Self::Check),
147 "uncheck" => Some(Self::Uncheck),
148 "press" => Some(Self::Press),
149 "hover" => Some(Self::Hover),
150 _ => None,
151 }
152 }
153
154 pub fn requires_value(self) -> bool {
156 matches!(
157 self,
158 Self::Type | Self::Fill | Self::SelectOption | Self::Press
159 )
160 }
161
162 pub fn as_str(self) -> &'static str {
164 match self {
165 Self::Click => "click",
166 Self::Type => "type",
167 Self::Fill => "fill",
168 Self::SelectOption => "select_option",
169 Self::Check => "check",
170 Self::Uncheck => "uncheck",
171 Self::Press => "press",
172 Self::Hover => "hover",
173 }
174 }
175}
176
177#[derive(Debug, Clone)]
179pub struct LLMGroundResult {
180 pub ref_id: Option<String>,
182 pub action: BrowserActAction,
184 pub value: Option<String>,
186 pub reason: Option<String>,
188}
189
190pub async fn llm_ground(
196 provider: &dyn oxicode_ai::Provider,
197 model: &oxicode_ai::Model,
198 goal: &str,
199 candidates: &[ObservedElement],
200 observation: &Observation,
201) -> Result<LLMGroundResult, BrowserError> {
202 let prompt = format!(
203 "GOAL: {goal}\nURL: {url}\nTITLE: {title}\n\n\
204 CANDIDATE ELEMENTS:\n{cands}\n\n\
205 TASK: pick the single element that best matches the goal.\n\
206 Return JSON: {{\"ref_id\": \"eN\", \"action\": \"click|type|fill|select_option|check|uncheck|press|hover\", \"value\": \"...\", \"reason\": \"...\"}}\n\
207 If none match, return: {{\"ref_id\": null, \"reason\": \"...\"}}",
208 goal = goal,
209 url = observation.url,
210 title = observation.title,
211 cands = serde_json::to_string(candidates).unwrap_or_default(),
212 );
213
214 let mut ctx = oxicode_ai::Context::new();
215 ctx.add_message(oxicode_ai::Message::user(prompt));
216
217 let stream = provider
218 .stream(model, &ctx, None)
219 .await
220 .map_err(|e| BrowserError::Backend(e.to_string()))?;
221
222 let mut pinned: std::pin::Pin<
223 Box<dyn futures::Stream<Item = oxicode_ai::ProviderEvent> + Send>,
224 > = stream;
225 let mut text = String::new();
226 while let Some(ev) = pinned.next().await {
227 match ev {
228 oxicode_ai::ProviderEvent::TextDelta { delta, .. } => text.push_str(&delta),
229 oxicode_ai::ProviderEvent::Done { .. } => break,
230 oxicode_ai::ProviderEvent::Error { error, .. } => {
231 let msg = error
232 .error_message
233 .clone()
234 .unwrap_or_else(|| "LLM stream returned error event".into());
235 return Err(BrowserError::Backend(msg));
236 }
237 _ => {}
238 }
239 }
240
241 let trimmed = text
243 .trim()
244 .trim_start_matches("```json")
245 .trim_start_matches("```")
246 .trim_end_matches("```")
247 .trim();
248 let v: Value = serde_json::from_str(trimmed)
249 .map_err(|e| BrowserError::GroundingParse(format!("{e}: {trimmed}")))?;
250 let ref_id = v.get("ref_id").and_then(|x| x.as_str()).map(String::from);
251 let action_str = v.get("action").and_then(|x| x.as_str()).unwrap_or("click");
252 let action = BrowserActAction::parse_str(action_str)
253 .ok_or_else(|| BrowserError::GroundingParse(format!("unknown action: {action_str}")))?;
254 let value = v.get("value").and_then(|x| x.as_str()).map(String::from);
255 let reason = v.get("reason").and_then(|x| x.as_str()).map(String::from);
256 Ok(LLMGroundResult {
257 ref_id,
258 action,
259 value,
260 reason,
261 })
262}
263
264pub struct BrowseActTool {
272 engine: Arc<dyn BrowserEngine>,
274 config: BrowseConfig,
276 provider: Option<Arc<dyn oxicode_ai::Provider>>,
279 model: Option<oxicode_ai::Model>,
281 callbacks: super::callback_mixin::BrowseCallbacks,
283 tab_id_slot: Mutex<Arc<parking_lot::Mutex<Option<uuid::Uuid>>>>,
285}
286
287impl BrowseActTool {
288 pub fn new(
290 provider: Arc<dyn oxicode_ai::Provider>,
291 model: oxicode_ai::Model,
292 engine: Arc<dyn BrowserEngine>,
293 ) -> Self {
294 Self::with_config(provider, model, engine, BrowseConfig::default())
295 }
296
297 pub fn with_config(
299 provider: Arc<dyn oxicode_ai::Provider>,
300 model: oxicode_ai::Model,
301 engine: Arc<dyn BrowserEngine>,
302 config: BrowseConfig,
303 ) -> Self {
304 Self {
305 engine,
306 config,
307 provider: Some(provider),
308 model: Some(model),
309 callbacks: super::callback_mixin::BrowseCallbacks::new(),
310 tab_id_slot: Mutex::new(Arc::new(parking_lot::Mutex::new(None))),
311 }
312 }
313
314 pub fn new_deterministic(engine: Arc<dyn BrowserEngine>) -> Self {
320 Self::with_config_deterministic(engine, BrowseConfig::default())
321 }
322
323 pub fn with_config_deterministic(engine: Arc<dyn BrowserEngine>, config: BrowseConfig) -> Self {
325 Self {
326 engine,
327 config,
328 provider: None,
329 model: None,
330 callbacks: super::callback_mixin::BrowseCallbacks::new(),
331 tab_id_slot: Mutex::new(Arc::new(parking_lot::Mutex::new(None))),
332 }
333 }
334}
335
336#[async_trait]
337impl AgentTool for BrowseActTool {
338 fn name(&self) -> &str {
339 "browse_act"
340 }
341
342 fn label(&self) -> &str {
343 "Browse Act"
344 }
345
346 fn description(&self) -> &str {
347 "Act on a web page using a natural-language goal. The tool observes the page's \
348 interactive surface, calls a model to pick the element matching your goal, and \
349 dispatches the right click/type/fill/select_option/check/uncheck/press/hover. \
350 No CSS selectors required from the caller. Use when you know what you want to do \
351 but not which element to target."
352 }
353
354 fn parameters_schema(&self) -> Value {
355 json!({
356 "type": "object",
357 "properties": {
358 "url": {
359 "type": "string",
360 "description": "URL of the page to act on"
361 },
362 "goal": {
363 "type": "string",
364 "description": "Natural-language action description"
365 },
366 "value": {
367 "type": "string",
368 "description": "Value for type/fill/select_option/press"
369 },
370 "action_hint": {
371 "type": "string",
372 "enum": ["click", "type", "fill", "select_option", "check", "uncheck", "press", "hover"],
373 "description": "Optional action to bias the model's choice"
374 },
375 "timeout": {
376 "type": "integer",
377 "default": 30,
378 "description": "Seconds"
379 }
380 },
381 "required": ["url", "goal"]
382 })
383 }
384
385 fn on_progress(&self, callback: crate::tools::ProgressCallback) {
386 self.callbacks.store_progress(callback);
387 }
388
389 fn on_browse_progress(&self, callback: Arc<dyn Fn(super::BrowseProgress) + Send + Sync>) {
390 self.callbacks.store_browse(callback);
391 }
392
393 fn set_tab_id_slot(&self, slot: Arc<parking_lot::Mutex<Option<uuid::Uuid>>>) {
394 *self.tab_id_slot.lock() = slot;
395 }
396
397 fn current_tab_id(&self) -> Option<uuid::Uuid> {
398 *self.tab_id_slot.lock().lock()
399 }
400
401 async fn execute(
402 &self,
403 _tool_call_id: &str,
404 params: Value,
405 _signal: Option<oneshot::Receiver<()>>,
406 _ctx: &ToolContext,
407 ) -> Result<AgentToolResult, ToolError> {
408 let url = params["url"]
409 .as_str()
410 .ok_or_else(|| "Missing required parameter: url".to_string())?;
411 let goal = params["goal"]
412 .as_str()
413 .ok_or_else(|| "Missing required parameter: goal".to_string())?;
414 let value = params["value"].as_str();
415 let action_hint = params["action_hint"]
416 .as_str()
417 .and_then(BrowserActAction::parse_str);
418 let timeout_secs = params["timeout"]
419 .as_u64()
420 .unwrap_or(self.config.page_timeout_secs);
421
422 let raw_tab = self
424 .engine
425 .new_tab()
426 .await
427 .map_err(|e| format!("Failed to open browser tab: {e}"))?;
428 let tab_id = raw_tab.tab_id();
429 *self.tab_id_slot.lock().lock() = Some(tab_id);
430 self.callbacks
431 .register_on_registry(tab_id, self.engine.callback_registry().as_ref());
432 let guard = super::tab_guard::TabGuard::new(raw_tab);
433
434 let page = tokio::time::timeout(
436 std::time::Duration::from_secs(timeout_secs),
437 guard.tab().goto(url),
438 )
439 .await
440 .map_err(|_| format!("Navigation timed out after {timeout_secs}s"))?
441 .map_err(|e| format!("Navigation failed: {e}"))?;
442
443 let observation = guard
445 .tab()
446 .observe()
447 .await
448 .map_err(|e| format!("Observation failed: {e}"))?;
449
450 let candidates = candidate_tier(goal, &observation, DEFAULT_CANDIDATE_TOP_N);
452
453 let mut mode = "llm";
455 let mut ground_result = match (self.provider.as_ref(), self.model.as_ref()) {
456 (Some(p), Some(m)) => llm_ground(p.as_ref(), m, goal, &candidates, &observation).await,
457 _ => {
458 mode = "deterministic_only";
460 let action = action_hint.unwrap_or(BrowserActAction::Click);
461 match candidates.first() {
462 Some(top) => Ok(LLMGroundResult {
463 ref_id: Some(top.ref_id.clone()),
464 action,
465 value: value.map(String::from),
466 reason: Some("deterministic-only mode (no LLM wired)".into()),
467 }),
468 None => Err(BrowserError::NoMatch(
469 "no candidates on page (deterministic-only mode)".into(),
470 )),
471 }
472 }
473 };
474
475 if matches!(ground_result, Err(BrowserError::Backend(_))) {
476 if let Some(top) = candidates.first() {
478 let action = action_hint.unwrap_or(BrowserActAction::Click);
479 let fallback_reason = match &ground_result {
480 Err(BrowserError::Backend(s)) => format!("provider error: {s}"),
481 _ => "provider unavailable".into(),
482 };
483 ground_result = Ok(LLMGroundResult {
484 ref_id: Some(top.ref_id.clone()),
485 action,
486 value: value.map(String::from),
487 reason: Some(format!("deterministic fallback ({fallback_reason})")),
488 });
489 mode = "deterministic_fallback";
490 }
491 }
492 let ground = ground_result.map_err(|e| e.to_string())?;
493
494 if ground.ref_id.is_none() {
496 let body = json!({
497 "matched_ref": Value::Null,
498 "matched_name": Value::Null,
499 "matched_role": Value::Null,
500 "action": Value::Null,
501 "selector": Value::Null,
502 "score": 0.0,
503 "result": "no_match",
504 "mode": mode,
505 "reason": ground.reason,
506 "candidates_considered": candidates.len(),
507 });
508 guard.close().await;
509 *self.tab_id_slot.lock().lock() = None;
510 return Ok(AgentToolResult::success(body.to_string()));
511 }
512
513 let ref_id_str = ground.ref_id.as_deref().ok_or_else(|| {
515 "LLM produced null ref_id but did not return no-match error".to_string()
516 })?;
517 let el = candidates
518 .iter()
519 .find(|e| e.ref_id == ref_id_str)
520 .cloned()
521 .or_else(|| {
522 observation
523 .elements
524 .iter()
525 .find(|e| e.ref_id == ref_id_str)
526 .cloned()
527 })
528 .ok_or_else(|| {
529 format!("LLM picked ref_id={ref_id_str} but it is not in the observation")
530 })?;
531 let selector = el.selector.clone();
532 let matched_name = el.name.clone();
533 let matched_role = el.role.clone();
534
535 let action = action_hint.unwrap_or(ground.action);
537 let final_value = ground.value.as_deref().or(value);
538
539 if action.requires_value() && final_value.map(str::is_empty).unwrap_or(true) {
541 return Err(BrowserError::MissingValue {
542 action: action.as_str(),
543 }
544 .into());
545 }
546 let v = final_value.unwrap_or("");
547 match action {
548 BrowserActAction::Click => guard.tab().click(&selector).await,
549 BrowserActAction::Hover => guard.tab().hover(&selector).await,
550 BrowserActAction::Check => guard.tab().check(&selector).await,
551 BrowserActAction::Uncheck => guard.tab().uncheck(&selector).await,
552 BrowserActAction::Type => guard.tab().type_(&selector, v).await,
553 BrowserActAction::Fill => guard.tab().fill(&selector, v).await,
554 BrowserActAction::SelectOption => guard.tab().select_option(&selector, v).await,
555 BrowserActAction::Press => guard.tab().press(v).await,
556 }
557 .map_err(|e| e.to_string())?;
558
559 guard.close().await;
560 *self.tab_id_slot.lock().lock() = None;
561
562 let body = json!({
563 "matched_ref": ref_id_str,
564 "matched_name": matched_name,
565 "matched_role": matched_role,
566 "action": action.as_str(),
567 "selector": selector,
568 "score": 0.0,
569 "result": "ok",
570 "mode": mode,
571 "reason": ground.reason,
572 "candidates_considered": candidates.len(),
573 });
574 Ok(AgentToolResult::success(body.to_string())
575 .with_metadata(json!({ "url": page.url, "title": page.title })))
576 }
577}
578
579#[cfg(test)]
580mod tests {
581 use super::*;
582 use crate::tools::browse::engine::{Observation, ObservedElement};
583
584 fn el(
585 ref_id: &str,
586 role: &str,
587 name: &str,
588 visible: bool,
589 interactive: bool,
590 ) -> ObservedElement {
591 ObservedElement {
592 ref_id: ref_id.into(),
593 role: role.into(),
594 name: name.into(),
595 tag: "button".into(),
596 selector: format!("[data-oxicode-ref=\"{ref_id}\"]"),
597 visible,
598 interactive,
599 }
600 }
601
602 fn obs(elements: Vec<ObservedElement>) -> Observation {
603 Observation {
604 url: "https://example.com".into(),
605 title: "ex".into(),
606 elements,
607 }
608 }
609
610 #[test]
613 fn candidate_tier_picks_highest_token_overlap() {
614 let o = obs(vec![
615 el("e1", "link", "Documentation", true, true),
616 el("e2", "button", "Sign Up", true, true),
617 el("e3", "link", "About", true, true),
618 ]);
619 let c = candidate_tier("click the Sign Up button", &o, 5);
620 assert_eq!(c[0].ref_id, "e2");
621 }
622
623 #[test]
624 fn candidate_tier_filters_hidden_elements() {
625 let o = obs(vec![
626 el("e1", "button", "Sign Up", false, true),
627 el("e2", "button", "Cancel", true, true),
628 ]);
629 let c = candidate_tier("Sign Up", &o, 5);
630 assert!(c.iter().all(|x| x.ref_id != "e1"));
631 }
632
633 #[test]
634 fn candidate_tier_filters_non_interactive() {
635 let o = obs(vec![el("e1", "text", "Sign Up Here", true, false)]);
636 let c = candidate_tier("Sign Up", &o, 5);
637 assert!(c.is_empty());
638 }
639
640 #[test]
641 fn candidate_tier_prefers_button_over_link_on_ties() {
642 let o = obs(vec![
643 el("e1", "link", "Submit", true, true),
644 el("e2", "button", "Submit", true, true),
645 ]);
646 let c = candidate_tier("Submit", &o, 5);
647 assert_eq!(c[0].role, "button");
648 }
649
650 #[test]
651 fn candidate_tier_prefers_longer_name_on_ties() {
652 let o = obs(vec![
653 el("e1", "button", "Add", true, true),
654 el("e2", "button", "Add to Cart", true, true),
655 ]);
656 let c = candidate_tier("Add", &o, 5);
657 assert_eq!(c[0].name, "Add to Cart");
658 }
659
660 #[test]
661 fn candidate_tier_respects_top_n() {
662 let elements: Vec<_> = (0..30)
663 .map(|i| el(&format!("e{i}"), "button", &format!("Item {i}"), true, true))
664 .collect();
665 let o = obs(elements);
666 let c = candidate_tier("Item", &o, 5);
667 assert_eq!(c.len(), 5);
668 }
669
670 #[test]
671 fn candidate_tier_empty_goal_yields_empty() {
672 let o = obs(vec![el("e1", "button", "OK", true, true)]);
673 assert!(candidate_tier("", &o, 5).is_empty());
674 assert!(candidate_tier(" ", &o, 5).is_empty());
675 }
676
677 #[test]
678 fn candidate_tier_drop_stop_words() {
679 let o = obs(vec![el("e1", "button", "OK", true, true)]);
680 let c = candidate_tier("click the OK button", &o, 5);
681 assert_eq!(c.len(), 1);
682 assert_eq!(c[0].ref_id, "e1");
683 }
684
685 #[test]
688 fn action_from_str_maps_known_values() {
689 assert_eq!(
690 BrowserActAction::parse_str("click"),
691 Some(BrowserActAction::Click)
692 );
693 assert_eq!(
694 BrowserActAction::parse_str("type"),
695 Some(BrowserActAction::Type)
696 );
697 assert_eq!(
698 BrowserActAction::parse_str("fill"),
699 Some(BrowserActAction::Fill)
700 );
701 assert_eq!(
702 BrowserActAction::parse_str("select_option"),
703 Some(BrowserActAction::SelectOption)
704 );
705 assert_eq!(
706 BrowserActAction::parse_str("check"),
707 Some(BrowserActAction::Check)
708 );
709 assert_eq!(
710 BrowserActAction::parse_str("uncheck"),
711 Some(BrowserActAction::Uncheck)
712 );
713 assert_eq!(
714 BrowserActAction::parse_str("press"),
715 Some(BrowserActAction::Press)
716 );
717 assert_eq!(
718 BrowserActAction::parse_str("hover"),
719 Some(BrowserActAction::Hover)
720 );
721 assert_eq!(BrowserActAction::parse_str("nonsense"), None);
722 }
723
724 #[test]
725 fn action_requires_value_mapping() {
726 assert!(!BrowserActAction::Click.requires_value());
727 assert!(BrowserActAction::Type.requires_value());
728 assert!(BrowserActAction::Fill.requires_value());
729 assert!(BrowserActAction::SelectOption.requires_value());
730 assert!(BrowserActAction::Press.requires_value());
731 assert!(!BrowserActAction::Check.requires_value());
732 assert!(!BrowserActAction::Hover.requires_value());
733 }
734
735 use async_trait::async_trait;
738 use oxicode_ai::{
739 Api, Context, Model, Provider, ProviderError, ProviderEvent, StreamOptions, StreamResult,
740 };
741 use parking_lot::Mutex;
742 use std::pin::Pin;
743
744 struct ScriptedProvider {
746 payload: Mutex<String>,
747 }
748 impl ScriptedProvider {
749 fn new(payload: &str) -> Arc<Self> {
750 Arc::new(Self {
751 payload: Mutex::new(payload.into()),
752 })
753 }
754 }
755 #[async_trait]
756 impl Provider for ScriptedProvider {
757 fn stream<'a>(
758 &'a self,
759 _model: &'a Model,
760 _context: &'a Context,
761 _options: Option<StreamOptions>,
762 ) -> Pin<Box<dyn Future<Output = StreamResult> + Send + 'a>> {
763 let payload = self.payload.lock().clone();
764 Box::pin(async move {
765 use futures::stream;
766 let s = stream::iter(vec![
767 ProviderEvent::TextDelta {
768 content_index: 0,
769 delta: payload,
770 partial: Arc::new(oxicode_ai::AssistantMessage::new(
771 Api::OpenAiCompletions,
772 "test-provider",
773 "test-model",
774 )),
775 },
776 ProviderEvent::Done {
777 reason: oxicode_ai::StopReason::Stop,
778 message: oxicode_ai::AssistantMessage::new(
779 Api::OpenAiCompletions,
780 "test-provider",
781 "test-model",
782 ),
783 },
784 ]);
785 let stream: Pin<Box<dyn futures::Stream<Item = ProviderEvent> + Send>> =
786 Box::pin(s);
787 Ok(stream)
788 })
789 }
790 }
791
792 struct FailingProvider;
793 #[async_trait]
794 impl Provider for FailingProvider {
795 fn stream<'a>(
796 &'a self,
797 _model: &'a Model,
798 _context: &'a Context,
799 _options: Option<StreamOptions>,
800 ) -> Pin<Box<dyn Future<Output = StreamResult> + Send + 'a>> {
801 Box::pin(async move { Err(ProviderError::NetworkError("provider down".into())) })
802 }
803 }
804
805 fn test_model() -> Model {
806 Model::new(
807 "test-model",
808 "test-model",
809 Api::OpenAiCompletions,
810 "openai",
811 "",
812 )
813 }
814
815 #[tokio::test]
816 async fn llm_ground_parses_ref_id_and_action() {
817 let provider: Arc<dyn Provider> =
818 ScriptedProvider::new(r#"{"ref_id":"e2","action":"click","reason":"matches Sign Up"}"#);
819 let candidates = vec![
820 el("e1", "link", "Documentation", true, true),
821 el("e2", "button", "Sign Up", true, true),
822 el("e3", "link", "About", true, true),
823 ];
824 let o = obs(candidates.clone());
825 let r = llm_ground(provider.as_ref(), &test_model(), "Sign Up", &candidates, &o)
826 .await
827 .expect("ok");
828 assert_eq!(r.ref_id.as_deref(), Some("e2"));
829 assert_eq!(r.action, BrowserActAction::Click);
830 assert_eq!(r.reason.as_deref(), Some("matches Sign Up"));
831 }
832
833 #[tokio::test]
834 async fn llm_ground_handles_null_ref_id() {
835 let provider: Arc<dyn Provider> =
836 ScriptedProvider::new(r#"{"ref_id":null,"reason":"no matching element"}"#);
837 let candidates = vec![el("e1", "button", "Cancel", true, true)];
838 let o = obs(candidates.clone());
839 let r = llm_ground(
840 provider.as_ref(),
841 &test_model(),
842 "Submit Order",
843 &candidates,
844 &o,
845 )
846 .await
847 .expect("ok");
848 assert!(r.ref_id.is_none());
849 assert_eq!(r.reason.as_deref(), Some("no matching element"));
850 }
851
852 #[tokio::test]
853 async fn llm_ground_returns_grounding_parse_on_invalid_json() {
854 let provider: Arc<dyn Provider> = ScriptedProvider::new("not json at all");
855 let candidates = vec![el("e1", "button", "OK", true, true)];
856 let o = obs(candidates.clone());
857 let err = llm_ground(provider.as_ref(), &test_model(), "OK", &candidates, &o)
858 .await
859 .expect_err("should fail to parse");
860 assert!(matches!(err, BrowserError::GroundingParse(_)));
861 }
862
863 #[tokio::test]
864 async fn llm_ground_propagates_provider_error() {
865 let provider: Arc<dyn Provider> = Arc::new(FailingProvider);
866 let candidates = vec![el("e1", "button", "OK", true, true)];
867 let o = obs(candidates.clone());
868 let err = llm_ground(provider.as_ref(), &test_model(), "OK", &candidates, &o)
869 .await
870 .expect_err("provider error");
871 assert!(matches!(err, BrowserError::Backend(_)));
872 }
873
874 #[tokio::test]
875 async fn llm_ground_strips_json_fences() {
876 let provider: Arc<dyn Provider> =
877 ScriptedProvider::new("```json\n{\"ref_id\":\"e1\",\"action\":\"click\"}\n```");
878 let candidates = vec![el("e1", "button", "OK", true, true)];
879 let o = obs(candidates.clone());
880 let r = llm_ground(provider.as_ref(), &test_model(), "OK", &candidates, &o)
881 .await
882 .expect("ok");
883 assert_eq!(r.ref_id.as_deref(), Some("e1"));
884 assert_eq!(r.action, BrowserActAction::Click);
885 }
886}