use super::manager::{BrowserManager, split_frame_selector};
use crate::brain::tools::error::Result;
use crate::brain::tools::r#trait::{
Tool, ToolCapability, ToolExecutionContext, ToolHints, ToolResult,
};
use async_trait::async_trait;
use serde_json::Value;
use std::sync::Arc;
pub(crate) const MAX_ACTIONS: usize = 10;
pub(crate) const MAX_WAIT_MS: u64 = 10_000;
pub(crate) const BATCH_TIMEOUT_SECS: u64 = 120;
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Act {
Click { selector: String },
Fill { selector: String, text: String },
Press { key: String },
Select { selector: String, value: String },
Wait { ms: u64 },
}
impl Act {
pub(crate) fn selector(&self) -> Option<&str> {
match self {
Act::Click { selector } | Act::Fill { selector, .. } | Act::Select { selector, .. } => {
Some(selector)
}
Act::Press { .. } | Act::Wait { .. } => None,
}
}
pub(crate) fn label(&self) -> String {
match self {
Act::Click { selector } => format!("click {selector}"),
Act::Fill { selector, .. } => format!("fill {selector}"),
Act::Press { key } => format!("press {key}"),
Act::Select { selector, .. } => format!("select {selector}"),
Act::Wait { ms } => format!("wait {ms}ms"),
}
}
}
pub(crate) fn parse_actions(input: &Value) -> std::result::Result<Vec<Act>, String> {
let raw = match input["actions"].as_array() {
Some(a) if !a.is_empty() => a,
_ => return Err("'actions' must be a non-empty array".into()),
};
if raw.len() > MAX_ACTIONS {
return Err(format!(
"Too many actions: {} (cap is {MAX_ACTIONS}). Split into multiple browser_act calls.",
raw.len()
));
}
let mut out = Vec::with_capacity(raw.len());
for (i, a) in raw.iter().enumerate() {
let kind = a["type"].as_str().unwrap_or("");
let act = match kind {
"click" => Act::Click {
selector: req_str(a, "selector", i, "click")?,
},
"fill" => Act::Fill {
selector: req_str(a, "selector", i, "fill")?,
text: req_str(a, "text", i, "fill")?,
},
"press" => Act::Press {
key: req_str(a, "key", i, "press")?,
},
"select" => Act::Select {
selector: req_str(a, "selector", i, "select")?,
value: match a["value"].as_str() {
Some(v) => v.to_string(),
None => String::new(),
},
},
"wait" => {
let ms = a["ms"].as_u64().unwrap_or(0);
if ms == 0 {
return Err(format!(
"action {}: 'wait' requires a positive \"ms\" (1-{MAX_WAIT_MS})",
i + 1
));
}
if ms > MAX_WAIT_MS {
return Err(format!(
"action {}: wait of {ms}ms exceeds the per-action cap of {MAX_WAIT_MS}ms — \
split long waits into multiple actions",
i + 1
));
}
Act::Wait { ms }
}
other => {
return Err(format!(
"action {}: unknown type '{other}' (expected click/fill/press/select/wait)",
i + 1
));
}
};
out.push(act);
}
Ok(out)
}
fn req_str(a: &Value, field: &str, i: usize, kind: &str) -> std::result::Result<String, String> {
match a[field].as_str() {
Some(s) if !s.is_empty() => Ok(s.to_string()),
_ => Err(format!(
"action {}: '{kind}' requires a non-empty \"{field}\"",
i + 1
)),
}
}
pub(crate) fn format_prefix_report(done: usize, failed: &Act, reason: &str) -> String {
let label = failed.label();
if done == 0 {
format!("action 1 ({label}) failed: {reason}. No actions executed.")
} else {
format!(
"actions 1-{done} OK; action {} ({label}) failed: {reason}. Later actions were NOT run — \
the page state reflects the completed prefix only; re-inventory with browser_find \
before retrying the remainder.",
done + 1
)
}
}
pub(crate) fn build_pre_flight_js(selectors: &[String]) -> String {
let arr = serde_json::to_string(selectors).unwrap_or_else(|_| "[]".into());
format!(
r#"(function(){{
var sels = {arr};
var results = [];
for (var i = 0; i < sels.length; i++) {{
var s = sels[i];
var el = null;
try {{
if (s.indexOf("text=") === 0) {{
var needle = s.slice(5).toLowerCase();
var walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
var node;
while ((node = walker.nextNode())) {{
var t = (node.innerText || node.textContent || "").toLowerCase();
if (t.indexOf(needle) !== -1) {{
var r = node.getBoundingClientRect();
if (r.width > 0 && r.height > 0) {{ el = node; break; }}
}}
}}
}} else if (s.indexOf("xpath=") === 0) {{
var it = document.evaluate(s.slice(6), document, null,
XPathResult.FIRST_ORDERED_NODE_TYPE, null);
var n = it.singleNodeValue;
if (n) {{
var r2 = n.getBoundingClientRect();
if (r2.width > 0 && r2.height > 0) el = n;
}}
}} else {{
var q = document.querySelector(s);
if (q) {{
var r3 = q.getBoundingClientRect();
if (r3.width > 0 && r3.height > 0) el = q;
}}
}}
}} catch (e) {{
results.push({{i: i, ok: false, reason: "invalid selector: " + e.message}});
continue;
}}
results.push({{i: i, ok: !!el, reason: el ? "" : "not found or not visible"}});
}}
return results;
}})()"#
)
}
pub struct BrowserActTool {
manager: Arc<BrowserManager>,
}
impl BrowserActTool {
pub fn new(manager: Arc<BrowserManager>) -> Self {
Self { manager }
}
}
#[async_trait]
impl Tool for BrowserActTool {
fn name(&self) -> &str {
"browser_act"
}
fn description(&self) -> &str {
"Execute a batch of browser actions in ONE call against the current page. \
Actions run in order: click/fill/press/select/wait (max 10). Every selector is \
resolved BEFORE anything executes — one stale reference rejects the whole call \
with zero side effects. On mid-sequence failure the completed prefix is reported \
and the rest is skipped. Takes ONE screenshot at the end. Use this instead of \
chained browser_click/browser_type calls whenever the sequence is known: one \
round-trip, one stable view, one screenshot."
}
fn input_schema(&self) -> Value {
serde_json::json!({
"type": "object",
"properties": {
"actions": {
"type": "array",
"maxItems": 10,
"description": "Ordered actions. Each object has a \"type\" plus its fields: \
{\"type\":\"click\",\"selector\":\"…\"} (CSS, text=Label, xpath=//…, or f2:N frame index), \
{\"type\":\"fill\",\"selector\":\"…\",\"text\":\"…\"} (replaces the value, framework-safe), \
{\"type\":\"press\",\"key\":\"Enter\"}, \
{\"type\":\"select\",\"selector\":\"…\",\"value\":\"option value\"}, \
{\"type\":\"wait\",\"ms\":500} (max 10000).",
}
},
"required": ["actions"]
})
}
fn capabilities(&self) -> Vec<ToolCapability> {
vec![ToolCapability::Network]
}
fn hints(&self) -> ToolHints {
ToolHints {
read_only: false,
destructive: true,
idempotent: false,
open_world: true,
}
}
fn requires_approval(&self) -> bool {
true
}
async fn execute(&self, input: Value, context: &ToolExecutionContext) -> Result<ToolResult> {
let main_page = match self
.manager
.get_or_create_session_page(context.session_id)
.await
{
Ok(p) => p,
Err(e) => return Ok(ToolResult::error(format!("Browser error: {e}"))),
};
tokio::time::timeout(
std::time::Duration::from_secs(BATCH_TIMEOUT_SECS),
self.run_batch(&input, context, &main_page),
)
.await
.unwrap_or_else(|_| {
Ok(ToolResult::error(format!(
"Batch exceeded the overall {BATCH_TIMEOUT_SECS}s budget (pre-flight + execution). \
Reduce waits or split into multiple browser_act calls."
)))
})
}
}
impl BrowserActTool {
async fn run_batch(
&self,
input: &Value,
context: &ToolExecutionContext,
main_page: &chromiumoxide::Page,
) -> Result<ToolResult> {
let acts = match parse_actions(input) {
Ok(a) => a,
Err(e) => return Ok(ToolResult::error(e)),
};
use std::collections::HashMap;
let mut frame_pages: HashMap<String, chromiumoxide::Page> = HashMap::new();
for act in &acts {
let Some(sel) = act.selector() else { continue };
let Some((label, _)) = split_frame_selector(sel) else {
continue;
};
if let std::collections::hash_map::Entry::Vacant(slot) =
frame_pages.entry(label.clone())
{
match self
.manager
.oopif_page_by_label(context.session_id, &label)
.await
{
Ok(Some(p)) => {
slot.insert(p);
}
Ok(None) => {
return Ok(ToolResult::error(format!(
"Frame '{label}' not found. The frame may have navigated away — \
re-run `browser_find` to get a fresh inventory."
)));
}
Err(e) => {
return Ok(ToolResult::error(format!("Browser error: {e}")));
}
}
}
}
let mut page_selectors: Vec<(Option<String>, Vec<usize>)> = Vec::new(); for (idx, act) in acts.iter().enumerate() {
if let Some(sel) = act.selector() {
let label = split_frame_selector(sel).map(|(l, _)| l);
match page_selectors.iter_mut().find(|(l, _)| *l == label) {
Some((_, v)) => v.push(idx),
None => page_selectors.push((label, vec![idx])),
}
}
}
for (label, idxs) in &page_selectors {
let page = match label {
Some(l) => &frame_pages[l],
None => main_page,
};
let selectors: Vec<String> = idxs
.iter()
.map(|&i| acts[i].selector().unwrap().to_string())
.collect();
let js = build_pre_flight_js(&selectors);
let raw = match page.evaluate(js.as_str()).await {
Ok(r) => r.value().cloned().unwrap_or(Value::Null),
Err(e) => return Ok(ToolResult::error(format!("Pre-flight check failed: {e}"))),
};
if let Some(arr) = raw.as_array() {
for entry in arr {
let local_i = entry["i"].as_u64().unwrap_or(0) as usize;
let ok = entry["ok"].as_bool().unwrap_or(false);
if !ok && local_i < idxs.len() {
let act_idx = idxs[local_i];
let reason = entry["reason"].as_str().unwrap_or("unresolved");
let sel = acts[act_idx].selector().unwrap_or("?");
return Ok(ToolResult::error(format!(
"Pre-flight rejected: action {} targets '{sel}' — {reason}. \
NOTHING executed. Re-run `browser_find` for a fresh inventory.",
act_idx + 1
)));
}
}
}
}
let mut done = 0usize;
for act in &acts {
let page_ref = match act.selector().and_then(split_frame_selector) {
Some((label, _)) => &frame_pages[&label],
None => main_page,
};
let outcome: std::result::Result<(), String> = match act {
Act::Click { selector } => self.exec_click(page_ref, selector).await,
Act::Fill { selector, text } => self.exec_fill(page_ref, selector, text).await,
Act::Press { key } => page_ref
.press_key(key.as_str())
.await
.map(|_| ())
.map_err(|e| e.to_string()),
Act::Select { selector, value } => {
self.exec_select(page_ref, selector, value).await
}
Act::Wait { ms } => {
tokio::time::sleep(std::time::Duration::from_millis(*ms)).await;
Ok(())
}
};
match outcome {
Ok(()) => done += 1,
Err(e) => {
let report = format_prefix_report(done, act, &e);
self.manager
.reset_identical_screenshot_count(context.session_id)
.await;
let mut tr = ToolResult::error(super::events::append_line(
report,
self.manager.drain_recent_events(),
));
self.manager
.attach_screenshot(context.session_id, &mut tr)
.await;
return Ok(tr);
}
}
}
self.manager
.reset_identical_screenshot_count(context.session_id)
.await;
let summary = acts
.iter()
.map(|a| a.label())
.collect::<Vec<_>>()
.join(", ");
let mut result = ToolResult::success(super::events::append_line(
format!(
"Executed {} action{} (one call): {summary}",
acts.len(),
if acts.len() == 1 { "" } else { "s" }
),
self.manager.drain_recent_events(),
));
self.manager
.attach_screenshot(context.session_id, &mut result)
.await;
Ok(result)
}
}
impl BrowserActTool {
async fn exec_click(
&self,
page: &chromiumoxide::Page,
selector: &str,
) -> std::result::Result<(), String> {
let selector = selector.to_string(); if let Some(text) = selector.strip_prefix("text=") {
let escaped = text.replace('\\', "\\\\").replace('"', "\\\"");
let js = format!(
r#"
(() => {{
const needle = "{escaped}".toLowerCase();
const walker = document.createTreeWalker(
document.body, NodeFilter.SHOW_ELEMENT);
let node;
while ((node = walker.nextNode())) {{
const t = (node.innerText || node.textContent || "").toLowerCase();
if (!t.includes(needle)) continue;
const r = node.getBoundingClientRect();
if (r.width === 0 || r.height === 0) continue;
node.scrollIntoView({{block: "center"}});
node.click();
return "ok";
}}
return "not_found";
}})()
"#
);
let r = page
.evaluate(js.as_str())
.await
.map_err(|e| format!("text-click eval failed: {e}"))?;
let v = r.value().and_then(|v| v.as_str().map(String::from));
if v.as_deref() == Some("ok") {
let _ = page
.wait_for_network_almost_idle_with_timeout(std::time::Duration::from_secs(3))
.await;
Ok(())
} else {
Err(format!(
"no visible element matched text '{text}' (present during pre-flight; \
the view changed)"
))
}
} else if let Some(xpath) = selector.strip_prefix("xpath=") {
let escaped = xpath.replace('\\', "\\\\").replace('"', "\\\"");
let js = format!(
r#"
(() => {{
const it = document.evaluate("{escaped}", document, null,
XPathResult.FIRST_ORDERED_NODE_TYPE, null);
const node = it.singleNodeValue;
if (!node) return "not_found";
const r = node.getBoundingClientRect();
if (r.width === 0 || r.height === 0) return "not_visible";
node.scrollIntoView({{block: "center"}});
node.click();
return "ok";
}})()
"#
);
let r = page
.evaluate(js.as_str())
.await
.map_err(|e| format!("xpath-click eval failed: {e}"))?;
let v = r.value().and_then(|v| v.as_str().map(String::from));
if v.as_deref() == Some("ok") {
let _ = page
.wait_for_network_almost_idle_with_timeout(std::time::Duration::from_secs(3))
.await;
Ok(())
} else {
Err(format!("xpath '{xpath}' no longer resolves"))
}
} else {
let element = page
.find_element(&selector)
.await
.map_err(|e| format!("element '{selector}' not found: {e}"))?;
element
.click()
.await
.map_err(|e| format!("click failed: {e}"))?;
let _ = page
.wait_for_network_almost_idle_with_timeout(std::time::Duration::from_secs(3))
.await;
Ok(())
}
}
async fn exec_fill(
&self,
page: &chromiumoxide::Page,
selector: &str,
text: &str,
) -> std::result::Result<(), String> {
let sel_js = serde_json::to_string(selector).unwrap_or_else(|_| "null".into());
let val_js = serde_json::to_string(text).unwrap_or_else(|_| "\"\"".into());
let js = format!(
r#"(function(){{
var sel = {sel_js};
var val = {val_js};
var el = sel ? document.querySelector(sel) : document.activeElement;
if (!el) return "no_element";
var editable = el.isContentEditable;
if (el.value === undefined && !editable) return "not_input";
try {{ el.focus(); }} catch (e) {{}}
if (editable) {{
el.textContent = val;
el.dispatchEvent(new InputEvent("input", {{bubbles:true}}));
el.dispatchEvent(new Event("change", {{bubbles:true}}));
return "ok";
}}
var proto = el.tagName === "TEXTAREA"
? window.HTMLTextAreaElement.prototype
: window.HTMLInputElement.prototype;
var desc = Object.getOwnPropertyDescriptor(proto, "value");
if (desc && desc.set) {{ desc.set.call(el, val); }} else {{ el.value = val; }}
el.dispatchEvent(new Event("input", {{bubbles:true}}));
el.dispatchEvent(new Event("change", {{bubbles:true}}));
return "ok";
}})()"#
);
let r = page
.evaluate(js.as_str())
.await
.map_err(|e| format!("fill eval failed: {e}"))?;
let v = r.value().and_then(|v| v.as_str().map(String::from));
match v.as_deref() {
Some("ok") => Ok(()),
Some("no_element") => Err(format!(
"fill target '{selector}' disappeared (was present during pre-flight)"
)),
Some("not_input") => Err(format!("fill target '{selector}' is not an input")),
_ => Err(format!("fill failed on '{selector}'")),
}
}
async fn exec_select(
&self,
page: &chromiumoxide::Page,
selector: &str,
value: &str,
) -> std::result::Result<(), String> {
let sel_js = serde_json::to_string(selector).unwrap_or_else(|_| "null".into());
let val_js = serde_json::to_string(value).unwrap_or_else(|_| "\"\"".into());
let js = format!(
r#"(function(){{
var el = document.querySelector({sel_js});
if (!el) return "no_element";
if (el.tagName !== "SELECT") return "not_select";
var val = {val_js};
var opt = Array.prototype.find.call(el.options, function(o) {{
return o.value === val || o.text === val;
}});
if (!opt) return "no_option";
el.value = opt.value;
el.dispatchEvent(new Event("input", {{bubbles:true}}));
el.dispatchEvent(new Event("change", {{bubbles:true}}));
return "ok";
}})()"#
);
let r = page
.evaluate(js.as_str())
.await
.map_err(|e| format!("select eval failed: {e}"))?;
let v = r.value().and_then(|v| v.as_str().map(String::from));
match v.as_deref() {
Some("ok") => Ok(()),
Some("no_element") => Err(format!(
"select target '{selector}' disappeared (was present during pre-flight)"
)),
Some("not_select") => Err(format!("select target '{selector}' is not a <select>")),
Some("no_option") => Err(format!("no option matching '{value}' in '{selector}'")),
_ => Err(format!("select failed on '{selector}'")),
}
}
}