use std::collections::{HashMap, HashSet};
use std::ops::Deref;
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::Duration;
use futures_util::future::BoxFuture;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use tokio::sync::oneshot;
use super::web::{BootstrapMeta, HtmlPointer, WebRoom, WebRoomHooks};
use crate::authentification::CredentialsOrApiKey;
use crate::error::{Error, Result};
use crate::groups::Group;
use crate::parrot::generative::{Function, Tool};
use crate::shared::schema::Schema;
use crate::shared::server::{Authorization, Hooks, Presence, Room, connect as connect_server};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MouseButton {
Left,
Right,
Middle,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
pub struct Point {
pub x: f64,
pub y: f64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(
tag = "kind",
rename_all = "camelCase",
rename_all_fields = "camelCase"
)]
pub enum BrowserTool {
Move {
x: f64,
y: f64,
},
Click {
#[serde(default, skip_serializing_if = "Option::is_none")]
x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
y: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
down_ms: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
button: Option<MouseButton>,
#[serde(default, skip_serializing_if = "Option::is_none")]
count: Option<f64>,
},
Drag {
to: Point,
#[serde(default, skip_serializing_if = "Option::is_none")]
from: Option<Point>,
#[serde(default, skip_serializing_if = "Option::is_none")]
down_ms: Option<f64>,
},
Scroll {
#[serde(default, skip_serializing_if = "Option::is_none")]
delta_x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
delta_y: Option<f64>,
},
Type {
text: String,
},
ExecuteJavascript {
code: String,
},
Search {
query: String,
},
Exit {
#[serde(default, skip_serializing_if = "Option::is_none")]
answer: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
reason: Option<String>,
},
}
impl BrowserTool {
pub(crate) fn kind(&self) -> &'static str {
match self {
BrowserTool::Move { .. } => "move",
BrowserTool::Click { .. } => "click",
BrowserTool::Drag { .. } => "drag",
BrowserTool::Scroll { .. } => "scroll",
BrowserTool::Type { .. } => "type",
BrowserTool::ExecuteJavascript { .. } => "executeJavascript",
BrowserTool::Search { .. } => "search",
BrowserTool::Exit { .. } => "exit",
}
}
}
const ALL_TOOLS: [&str; 8] = [
"move",
"click",
"drag",
"scroll",
"type",
"executeJavascript",
"search",
"exit",
];
pub trait BrowserRoomHooks: Send + Sync {
fn on_cursor(&self, _room: &PageRoom, _cursor: &HtmlPointer) {}
fn on_tool(&self, _room: &PageRoom, _tool: &BrowserTool) {}
fn on_closed(&self, _room: &PageRoom) {}
fn on_presence(&self, _room: &PageRoom) {}
fn on_control(&self, _room: &PageRoom) {}
fn web(&self) -> Option<Arc<dyn WebRoomHooks>> {
None
}
}
impl BrowserRoomHooks for () {}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RoomAgent {
pub id: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub icon: Option<String>,
}
impl RoomAgent {
pub fn from_json(json: &Value) -> RoomAgent {
RoomAgent {
id: match &json["id"] {
Value::String(id) => id.clone(),
id => id.to_string(),
},
name: match &json["name"] {
Value::Null => "Agent".to_string(),
Value::String(name) => name.clone(),
name => name.to_string(),
},
icon: json["icon"]
.as_str()
.filter(|icon| !icon.is_empty())
.map(str::to_string),
}
}
}
pub type ToolInvoker = Arc<dyn Fn(BrowserTool) -> BoxFuture<'static, Result<String>> + Send + Sync>;
pub(crate) fn js_number(value: Option<&Value>) -> f64 {
match value {
None => f64::NAN,
Some(Value::Null) => 0.0,
Some(Value::Bool(value)) => f64::from(u8::from(*value)),
Some(Value::Number(number)) => number.as_f64().unwrap_or(f64::NAN),
Some(Value::String(text)) if text.trim().is_empty() => 0.0,
Some(Value::String(text)) => text.trim().parse().unwrap_or(f64::NAN),
Some(_) => f64::NAN,
}
}
pub(crate) fn js_string(value: Option<&Value>, fallback: &str) -> String {
match value {
None | Some(Value::Null) => fallback.to_string(),
Some(Value::String(text)) => text.clone(),
Some(value) => value.to_string(),
}
}
pub(crate) fn present(value: Option<&Value>) -> Option<&Value> {
value.filter(|value| !value.is_null())
}
pub(crate) fn tool(
name: &str,
description: &str,
parameters: Value,
evaluate: impl Fn(Value) -> BoxFuture<'static, Result<String>> + Send + Sync + 'static,
) -> Tool {
Tool {
function: Function {
name: name.to_string(),
parameters: serde_json::from_value::<Schema>(parameters)
.expect("tool schemas are valid"),
description: Some(description.to_string()),
default: None,
example: None,
behavior: None,
},
evaluate: Arc::new(evaluate),
instruction: None,
}
}
pub fn parrot_tools(enabled: &HashSet<String>, invoke: ToolInvoker) -> Vec<Tool> {
let mut tools = Vec::new();
let mut add = |tool: Tool| {
if enabled.contains(&tool.function.name) {
tools.push(tool);
}
};
let run = move |build: fn(&Value) -> BrowserTool| {
let invoke = invoke.clone();
move |params: Value| invoke(build(¶ms))
};
add(tool(
"move",
"Move the pointer to viewport coordinates. Does not click.",
json!({
"type": "OBJECT",
"properties": {
"x": { "type": "NUMBER", "description": "X pixel" },
"y": { "type": "NUMBER", "description": "Y pixel" },
},
"required": ["x", "y"],
}),
run(|p| BrowserTool::Move {
x: js_number(p.get("x")),
y: js_number(p.get("y")),
}),
));
add(tool(
"click",
"Click. Optional coordinates move there first. downMs is how long the button stays down. button right opens context menus; count 2 double clicks.",
json!({
"type": "OBJECT",
"properties": {
"x": { "type": "NUMBER", "description": "X pixel" },
"y": { "type": "NUMBER", "description": "Y pixel" },
"downMs": { "type": "NUMBER", "description": "Hold duration in ms" },
"button": { "type": "STRING", "enum": ["left", "right", "middle"], "description": "Mouse button, default left" },
"count": { "type": "NUMBER", "description": "Clicks in a row: 2 double clicks, 3 triple clicks. Default 1" },
},
}),
run(|p| BrowserTool::Click {
x: present(p.get("x")).map(|x| js_number(Some(x))),
y: present(p.get("y")).map(|y| js_number(Some(y))),
down_ms: present(p.get("downMs")).map(|ms| js_number(Some(ms))),
button: match p.get("button").and_then(Value::as_str) {
Some("right") => Some(MouseButton::Right),
Some("middle") => Some(MouseButton::Middle),
_ => None,
},
count: present(p.get("count")).map(|count| js_number(Some(count))),
}),
));
add(tool(
"drag",
"Drag from an optional start point to a destination.",
json!({
"type": "OBJECT",
"properties": {
"to": { "type": "OBJECT", "description": "Destination {x,y}" },
"from": { "type": "OBJECT", "description": "Start {x,y}" },
"downMs": { "type": "NUMBER", "description": "Hold at the end in ms" },
},
"required": ["to"],
}),
run(|p| BrowserTool::Drag {
to: Point {
x: js_number(p.get("to").and_then(|to| to.get("x"))),
y: js_number(p.get("to").and_then(|to| to.get("y"))),
},
from: p
.get("from")
.filter(|from| !from.is_null() && *from != &Value::Bool(false) && *from != "")
.map(|from| Point {
x: js_number(from.get("x")),
y: js_number(from.get("y")),
}),
down_ms: present(p.get("downMs")).map(|ms| js_number(Some(ms))),
}),
));
add(tool(
"scroll",
"Scroll the page with the wheel at the current pointer.",
json!({
"type": "OBJECT",
"properties": {
"deltaX": { "type": "NUMBER", "description": "Horizontal wheel delta" },
"deltaY": { "type": "NUMBER", "description": "Vertical wheel delta" },
},
}),
run(|p| BrowserTool::Scroll {
delta_x: present(p.get("deltaX")).map(|x| js_number(Some(x))),
delta_y: present(p.get("deltaY")).map(|y| js_number(Some(y))),
}),
));
add(tool(
"type",
"Type text at the current focus, one key at a time.",
json!({
"type": "OBJECT",
"properties": { "text": { "type": "STRING", "description": "Text to type" } },
"required": ["text"],
}),
run(|p| BrowserTool::Type {
text: js_string(p.get("text"), ""),
}),
));
add(tool(
"executeJavascript",
"Evaluate JavaScript in the page and return the stringified result.",
json!({
"type": "OBJECT",
"properties": { "code": { "type": "STRING", "description": "JavaScript to evaluate" } },
"required": ["code"],
}),
run(|p| BrowserTool::ExecuteJavascript {
code: js_string(p.get("code"), ""),
}),
));
add(tool(
"search",
"Search the web. Returns matching websites.",
json!({
"type": "OBJECT",
"properties": { "query": { "type": "STRING", "description": "Search query" } },
"required": ["query"],
}),
run(|p| BrowserTool::Search {
query: js_string(p.get("query"), ""),
}),
));
add(tool(
"exit",
"Leave the page. Pass answer when the goal is done, or reason when giving up.",
json!({
"type": "OBJECT",
"properties": {
"answer": { "type": "STRING", "description": "What was found" },
"reason": { "type": "STRING", "description": "Why the goal cannot be met" },
},
}),
run(|p| BrowserTool::Exit {
answer: present(p.get("answer")).map(|answer| js_string(Some(answer), "")),
reason: present(p.get("reason")).map(|reason| js_string(Some(reason), "")),
}),
));
tools
}
struct Page {
cursor: HtmlPointer,
controller: Option<String>,
agent: Option<RoomAgent>,
}
struct PageState {
web: WebRoom,
enabled_tools: HashSet<String>,
page: Mutex<Page>,
hooks: Mutex<Arc<dyn BrowserRoomHooks>>,
pending: Mutex<HashMap<String, oneshot::Sender<Result<String>>>>,
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().unwrap_or_else(PoisonError::into_inner)
}
#[derive(Clone)]
pub struct PageRoom {
room: Room,
inner: Arc<PageState>,
}
impl Deref for PageRoom {
type Target = Room;
fn deref(&self) -> &Room {
&self.room
}
}
impl PageRoom {
pub(crate) fn create(
room: &Room,
hooks: Arc<dyn BrowserRoomHooks>,
state: &Value,
enabled_tools: Option<Vec<String>>,
) -> PageRoom {
let enabled_tools: HashSet<String> = match enabled_tools.filter(|tools| !tools.is_empty()) {
Some(tools) => tools.into_iter().collect(),
None => ALL_TOOLS.iter().map(|kind| kind.to_string()).collect(),
};
let web = WebRoom::from_parent(room, hooks.web(), state);
if let Ok(bootstrap) = serde_json::from_value::<BootstrapMeta>(state["bootstrap"].clone()) {
web.ingest_bootstrap(bootstrap);
}
let page = PageRoom {
room: room.clone(),
inner: Arc::new(PageState {
web,
enabled_tools,
page: Mutex::new(Page {
cursor: serde_json::from_value(state["cursor"].clone()).unwrap_or_default(),
controller: None,
agent: None,
}),
hooks: Mutex::new(hooks),
pending: Mutex::new(HashMap::new()),
}),
};
page.apply_control(&state["control"]);
page
}
fn page(&self) -> MutexGuard<'_, Page> {
lock(&self.inner.page)
}
fn hooks(&self) -> Arc<dyn BrowserRoomHooks> {
lock(&self.inner.hooks).clone()
}
pub fn web(&self) -> &WebRoom {
&self.inner.web
}
pub fn cursor(&self) -> HtmlPointer {
self.page().cursor.clone()
}
pub(crate) fn set_cursor(&self, cursor: HtmlPointer) {
self.page().cursor = cursor;
}
pub fn enabled_tools(&self) -> &HashSet<String> {
&self.inner.enabled_tools
}
pub fn controller(&self) -> Option<String> {
self.page().controller.clone()
}
pub fn agent(&self) -> Option<RoomAgent> {
self.page().agent.clone()
}
pub fn watch(&self, hooks: Arc<dyn BrowserRoomHooks>) {
if let Some(web) = hooks.web() {
self.inner.web.set_hooks(web);
}
*lock(&self.inner.hooks) = hooks;
}
pub fn controlling(&self, me: &Presence) -> bool {
let page = self.page();
page.agent.is_none() && page.controller.as_deref() == Some(me.id())
}
async fn call(
&self,
packet: impl FnOnce(&str) -> Value,
timeout: Duration,
timed_out: String,
) -> Result<String> {
let id = uuid::Uuid::new_v4().to_string();
let (resolve, result) = oneshot::channel();
lock(&self.inner.pending).insert(id.clone(), resolve);
self.post(&packet(&id));
match tokio::time::timeout(timeout, result).await {
Ok(Ok(result)) => result,
_ => {
lock(&self.inner.pending).remove(&id);
Err(Error::Message(timed_out))
}
}
}
pub async fn set_controller(&self, seat: Option<&str>) -> Result<()> {
self.call(
|id| match seat.filter(|seat| !seat.is_empty()) {
Some(seat) => {
json!({ "event": "control-call", "data": { "id": id, "seat": seat } })
}
None => json!({ "event": "control-call", "data": { "id": id } }),
},
Duration::from_secs(30),
"Control timed out.".to_string(),
)
.await
.map(|_| ())
}
pub fn emit_presence(&self) {
self.hooks().on_presence(self);
}
pub fn emit_closed(&self) {
self.hooks().on_closed(self);
}
pub(crate) fn apply_control(&self, control: &Value) {
if !control.is_object() {
return;
}
let mut page = self.page();
page.controller = control["controller"].as_str().map(str::to_string);
page.agent = if control["agent"].is_object() {
Some(RoomAgent::from_json(&control["agent"]))
} else {
None
};
}
pub async fn invoke_tool(&self, tool: &BrowserTool) -> Result<String> {
if !self.inner.enabled_tools.contains(tool.kind()) {
return Err(Error::Message(format!(
"Tool is not enabled: {}",
tool.kind()
)));
}
self.call(
|id| json!({ "event": "browser-tool-call", "data": { "id": id, "tool": tool } }),
Duration::from_secs(30),
format!("Tool timed out: {}", tool.kind()),
)
.await
}
pub fn parrot_tools(&self) -> Vec<Tool> {
let room = self.clone();
parrot_tools(
&self.inner.enabled_tools,
Arc::new(move |tool| {
let room = room.clone();
Box::pin(async move { room.invoke_tool(&tool).await })
}),
)
}
fn settle(&self, data: &Value, ok: impl FnOnce() -> String) -> bool {
let Some(id) = data["id"].as_str().filter(|id| !id.is_empty()) else {
return true;
};
let Some(pending) = lock(&self.inner.pending).remove(id) else {
return true;
};
let failed = !(data["error"].is_null() || data["error"] == false || data["error"] == "");
let _ = pending.send(if failed {
Err(Error::Message(match &data["error"] {
Value::String(error) => error.clone(),
error => error.to_string(),
}))
} else {
Ok(ok())
});
true
}
pub fn handle_event(&self, packet: &Value) -> bool {
let event = packet["event"].as_str().unwrap_or_default();
if event == "browser-closed" || event == "server-closed" {
self.inner.web.handle_event(packet);
self.hooks().on_closed(self);
return true;
}
if self.inner.web.handle_event(packet) {
return true;
}
let data = &packet["data"];
match event {
"room-control" => {
let empty = json!({});
self.apply_control(if data.is_object() { data } else { &empty });
self.hooks().on_control(self);
true
}
"control-result" => self.settle(data, || "ok".to_string()),
"browser-cursor" => {
if let Ok(cursor) = serde_json::from_value::<HtmlPointer>(data["cursor"].clone()) {
self.set_cursor(cursor.clone());
self.hooks().on_cursor(self, &cursor);
}
true
}
"browser-tool-call" => {
if let Ok(tool) = serde_json::from_value::<BrowserTool>(data["tool"].clone()) {
self.hooks().on_tool(self, &tool);
}
true
}
"browser-tool-result" => self.settle(data, || match &data["result"] {
Value::Null => String::new(),
Value::String(result) => result.clone(),
result => result.to_string(),
}),
_ => false,
}
}
}
#[derive(Default)]
pub struct BrowserOptions {
pub tools: Option<Vec<String>>,
pub hooks: Option<Arc<dyn BrowserRoomHooks>>,
pub group: Option<Group>,
pub endpoint: Option<String>,
}
#[derive(Clone)]
pub struct BrowserRoom(PageRoom);
impl Deref for BrowserRoom {
type Target = PageRoom;
fn deref(&self) -> &PageRoom {
&self.0
}
}
impl BrowserRoom {
pub async fn connect(
authorization: &CredentialsOrApiKey,
reference: &str,
opts: BrowserOptions,
) -> Result<(BrowserRoom, Presence)> {
let group = opts.group.as_ref().map(|group| group.id.clone());
open_browser(
Authorization::Server {
authorization: authorization.clone(),
server: reference.to_string(),
group,
},
opts,
)
.await
}
pub async fn join(
authorization: &CredentialsOrApiKey,
access_code: &str,
opts: BrowserOptions,
) -> Result<(BrowserRoom, Presence)> {
open_browser(
Authorization::AccessCode {
authorization: authorization.clone(),
access_code: access_code.to_string(),
},
opts,
)
.await
}
}
struct BrowserHooks {
room: Mutex<Option<PageRoom>>,
hooks: Arc<dyn BrowserRoomHooks>,
tools: Option<Vec<String>>,
}
impl BrowserHooks {
fn room(&self) -> Option<PageRoom> {
lock(&self.room).clone()
}
}
impl Hooks for BrowserHooks {
fn create_room(&self, room: &Room, state: &Value) {
let tools = self.tools.clone().or_else(|| {
state["enabledTools"].as_array().map(|tools| {
tools
.iter()
.filter_map(|tool| tool.as_str().map(str::to_string))
.collect()
})
});
*lock(&self.room) = Some(PageRoom::create(room, self.hooks.clone(), state, tools));
}
fn on_connection(&self, _room: &Room, state: &Value) {
let Some(room) = self.room() else {
return;
};
if let Ok(bootstrap) = serde_json::from_value::<BootstrapMeta>(state["bootstrap"].clone()) {
room.web().ingest_bootstrap(bootstrap);
}
if let Ok(cursor) = serde_json::from_value::<HtmlPointer>(state["cursor"].clone()) {
room.set_cursor(cursor.clone());
room.hooks().on_cursor(&room, &cursor);
}
}
fn on_event(&self, _room: &Room, packet: &Value) -> bool {
self.room().is_some_and(|room| room.handle_event(packet))
}
fn on_presence(&self, _room: &Room) {
if let Some(room) = self.room() {
room.emit_presence();
}
}
fn on_closed(&self, _room: &Room, _reason: Option<&str>) {
if let Some(room) = self.room() {
room.emit_closed();
}
}
}
async fn open_browser(
auth: Authorization,
opts: BrowserOptions,
) -> Result<(BrowserRoom, Presence)> {
let endpoint = opts
.endpoint
.unwrap_or_else(|| format!("{}/browser/live", super::get_url()));
let hooks = Arc::new(BrowserHooks {
room: Mutex::new(None),
hooks: opts.hooks.unwrap_or_else(|| Arc::new(())),
tools: opts.tools,
});
let me = connect_server(auth, &endpoint, hooks.clone()).await?;
let room = hooks
.room()
.ok_or_else(|| Error::Message("Room not found".to_string()))?;
room.emit_presence();
Ok((BrowserRoom(room), me))
}