1use std::collections::{HashMap, HashSet};
5use std::ops::Deref;
6use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
7use std::time::Duration;
8
9use futures_util::future::BoxFuture;
10use serde::{Deserialize, Serialize};
11use serde_json::{Value, json};
12use tokio::sync::oneshot;
13
14use super::web::{BootstrapMeta, HtmlPointer, WebRoom, WebRoomHooks};
15use crate::authentification::CredentialsOrApiKey;
16use crate::error::{Error, Result};
17use crate::groups::Group;
18use crate::parrot::generative::{Function, Tool};
19use crate::shared::schema::Schema;
20use crate::shared::server::{Authorization, Hooks, Presence, Room, connect as connect_server};
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
23#[serde(rename_all = "lowercase")]
24pub enum MouseButton {
25 Left,
26 Right,
27 Middle,
28}
29
30#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
32pub struct Point {
33 pub x: f64,
34 pub y: f64,
35}
36
37#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
38#[serde(
39 tag = "kind",
40 rename_all = "camelCase",
41 rename_all_fields = "camelCase"
42)]
43pub enum BrowserTool {
44 Move {
45 x: f64,
46 y: f64,
47 },
48 Click {
51 #[serde(default, skip_serializing_if = "Option::is_none")]
52 x: Option<f64>,
53 #[serde(default, skip_serializing_if = "Option::is_none")]
54 y: Option<f64>,
55 #[serde(default, skip_serializing_if = "Option::is_none")]
56 down_ms: Option<f64>,
57 #[serde(default, skip_serializing_if = "Option::is_none")]
58 button: Option<MouseButton>,
59 #[serde(default, skip_serializing_if = "Option::is_none")]
60 count: Option<f64>,
61 },
62 Drag {
63 to: Point,
64 #[serde(default, skip_serializing_if = "Option::is_none")]
65 from: Option<Point>,
66 #[serde(default, skip_serializing_if = "Option::is_none")]
67 down_ms: Option<f64>,
68 },
69 Scroll {
70 #[serde(default, skip_serializing_if = "Option::is_none")]
71 delta_x: Option<f64>,
72 #[serde(default, skip_serializing_if = "Option::is_none")]
73 delta_y: Option<f64>,
74 },
75 Type {
76 text: String,
77 },
78 ExecuteJavascript {
79 code: String,
80 },
81 Search {
82 query: String,
83 },
84 Exit {
85 #[serde(default, skip_serializing_if = "Option::is_none")]
86 answer: Option<String>,
87 #[serde(default, skip_serializing_if = "Option::is_none")]
88 reason: Option<String>,
89 },
90}
91
92impl BrowserTool {
93 pub(crate) fn kind(&self) -> &'static str {
94 match self {
95 BrowserTool::Move { .. } => "move",
96 BrowserTool::Click { .. } => "click",
97 BrowserTool::Drag { .. } => "drag",
98 BrowserTool::Scroll { .. } => "scroll",
99 BrowserTool::Type { .. } => "type",
100 BrowserTool::ExecuteJavascript { .. } => "executeJavascript",
101 BrowserTool::Search { .. } => "search",
102 BrowserTool::Exit { .. } => "exit",
103 }
104 }
105}
106
107const ALL_TOOLS: [&str; 8] = [
108 "move",
109 "click",
110 "drag",
111 "scroll",
112 "type",
113 "executeJavascript",
114 "search",
115 "exit",
116];
117
118pub trait BrowserRoomHooks: Send + Sync {
121 fn on_cursor(&self, _room: &PageRoom, _cursor: &HtmlPointer) {}
122 fn on_tool(&self, _room: &PageRoom, _tool: &BrowserTool) {}
123 fn on_closed(&self, _room: &PageRoom) {}
124 fn on_presence(&self, _room: &PageRoom) {}
125 fn on_control(&self, _room: &PageRoom) {}
127 fn web(&self) -> Option<Arc<dyn WebRoomHooks>> {
128 None
129 }
130}
131
132impl BrowserRoomHooks for () {}
133
134#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
136pub struct RoomAgent {
137 pub id: String,
138 pub name: String,
139 #[serde(default, skip_serializing_if = "Option::is_none")]
140 pub icon: Option<String>,
141}
142
143impl RoomAgent {
144 pub fn from_json(json: &Value) -> RoomAgent {
145 RoomAgent {
146 id: match &json["id"] {
147 Value::String(id) => id.clone(),
148 id => id.to_string(),
149 },
150 name: match &json["name"] {
151 Value::Null => "Agent".to_string(),
152 Value::String(name) => name.clone(),
153 name => name.to_string(),
154 },
155 icon: json["icon"]
156 .as_str()
157 .filter(|icon| !icon.is_empty())
158 .map(str::to_string),
159 }
160 }
161}
162
163pub type ToolInvoker = Arc<dyn Fn(BrowserTool) -> BoxFuture<'static, Result<String>> + Send + Sync>;
165
166pub(crate) fn js_number(value: Option<&Value>) -> f64 {
168 match value {
169 None => f64::NAN,
170 Some(Value::Null) => 0.0,
171 Some(Value::Bool(value)) => f64::from(u8::from(*value)),
172 Some(Value::Number(number)) => number.as_f64().unwrap_or(f64::NAN),
173 Some(Value::String(text)) if text.trim().is_empty() => 0.0,
174 Some(Value::String(text)) => text.trim().parse().unwrap_or(f64::NAN),
175 Some(_) => f64::NAN,
176 }
177}
178
179pub(crate) fn js_string(value: Option<&Value>, fallback: &str) -> String {
181 match value {
182 None | Some(Value::Null) => fallback.to_string(),
183 Some(Value::String(text)) => text.clone(),
184 Some(value) => value.to_string(),
185 }
186}
187
188pub(crate) fn present(value: Option<&Value>) -> Option<&Value> {
189 value.filter(|value| !value.is_null())
190}
191
192pub(crate) fn tool(
193 name: &str,
194 description: &str,
195 parameters: Value,
196 evaluate: impl Fn(Value) -> BoxFuture<'static, Result<String>> + Send + Sync + 'static,
197) -> Tool {
198 Tool {
199 function: Function {
200 name: name.to_string(),
201 parameters: serde_json::from_value::<Schema>(parameters)
202 .expect("tool schemas are valid"),
203 description: Some(description.to_string()),
204 default: None,
205 example: None,
206 behavior: None,
207 },
208 evaluate: Arc::new(evaluate),
209 instruction: None,
210 }
211}
212
213pub fn parrot_tools(enabled: &HashSet<String>, invoke: ToolInvoker) -> Vec<Tool> {
216 let mut tools = Vec::new();
217 let mut add = |tool: Tool| {
218 if enabled.contains(&tool.function.name) {
219 tools.push(tool);
220 }
221 };
222 let run = move |build: fn(&Value) -> BrowserTool| {
223 let invoke = invoke.clone();
224 move |params: Value| invoke(build(¶ms))
225 };
226
227 add(tool(
228 "move",
229 "Move the pointer to viewport coordinates. Does not click.",
230 json!({
231 "type": "OBJECT",
232 "properties": {
233 "x": { "type": "NUMBER", "description": "X pixel" },
234 "y": { "type": "NUMBER", "description": "Y pixel" },
235 },
236 "required": ["x", "y"],
237 }),
238 run(|p| BrowserTool::Move {
239 x: js_number(p.get("x")),
240 y: js_number(p.get("y")),
241 }),
242 ));
243 add(tool(
244 "click",
245 "Click. Optional coordinates move there first. downMs is how long the button stays down. button right opens context menus; count 2 double clicks.",
246 json!({
247 "type": "OBJECT",
248 "properties": {
249 "x": { "type": "NUMBER", "description": "X pixel" },
250 "y": { "type": "NUMBER", "description": "Y pixel" },
251 "downMs": { "type": "NUMBER", "description": "Hold duration in ms" },
252 "button": { "type": "STRING", "enum": ["left", "right", "middle"], "description": "Mouse button, default left" },
253 "count": { "type": "NUMBER", "description": "Clicks in a row: 2 double clicks, 3 triple clicks. Default 1" },
254 },
255 }),
256 run(|p| BrowserTool::Click {
257 x: present(p.get("x")).map(|x| js_number(Some(x))),
258 y: present(p.get("y")).map(|y| js_number(Some(y))),
259 down_ms: present(p.get("downMs")).map(|ms| js_number(Some(ms))),
260 button: match p.get("button").and_then(Value::as_str) {
261 Some("right") => Some(MouseButton::Right),
262 Some("middle") => Some(MouseButton::Middle),
263 _ => None,
264 },
265 count: present(p.get("count")).map(|count| js_number(Some(count))),
266 }),
267 ));
268 add(tool(
269 "drag",
270 "Drag from an optional start point to a destination.",
271 json!({
272 "type": "OBJECT",
273 "properties": {
274 "to": { "type": "OBJECT", "description": "Destination {x,y}" },
275 "from": { "type": "OBJECT", "description": "Start {x,y}" },
276 "downMs": { "type": "NUMBER", "description": "Hold at the end in ms" },
277 },
278 "required": ["to"],
279 }),
280 run(|p| BrowserTool::Drag {
281 to: Point {
282 x: js_number(p.get("to").and_then(|to| to.get("x"))),
283 y: js_number(p.get("to").and_then(|to| to.get("y"))),
284 },
285 from: p
286 .get("from")
287 .filter(|from| !from.is_null() && *from != &Value::Bool(false) && *from != "")
288 .map(|from| Point {
289 x: js_number(from.get("x")),
290 y: js_number(from.get("y")),
291 }),
292 down_ms: present(p.get("downMs")).map(|ms| js_number(Some(ms))),
293 }),
294 ));
295 add(tool(
296 "scroll",
297 "Scroll the page with the wheel at the current pointer.",
298 json!({
299 "type": "OBJECT",
300 "properties": {
301 "deltaX": { "type": "NUMBER", "description": "Horizontal wheel delta" },
302 "deltaY": { "type": "NUMBER", "description": "Vertical wheel delta" },
303 },
304 }),
305 run(|p| BrowserTool::Scroll {
306 delta_x: present(p.get("deltaX")).map(|x| js_number(Some(x))),
307 delta_y: present(p.get("deltaY")).map(|y| js_number(Some(y))),
308 }),
309 ));
310 add(tool(
311 "type",
312 "Type text at the current focus, one key at a time.",
313 json!({
314 "type": "OBJECT",
315 "properties": { "text": { "type": "STRING", "description": "Text to type" } },
316 "required": ["text"],
317 }),
318 run(|p| BrowserTool::Type {
319 text: js_string(p.get("text"), ""),
320 }),
321 ));
322 add(tool(
323 "executeJavascript",
324 "Evaluate JavaScript in the page and return the stringified result.",
325 json!({
326 "type": "OBJECT",
327 "properties": { "code": { "type": "STRING", "description": "JavaScript to evaluate" } },
328 "required": ["code"],
329 }),
330 run(|p| BrowserTool::ExecuteJavascript {
331 code: js_string(p.get("code"), ""),
332 }),
333 ));
334 add(tool(
335 "search",
336 "Search the web. Returns matching websites.",
337 json!({
338 "type": "OBJECT",
339 "properties": { "query": { "type": "STRING", "description": "Search query" } },
340 "required": ["query"],
341 }),
342 run(|p| BrowserTool::Search {
343 query: js_string(p.get("query"), ""),
344 }),
345 ));
346 add(tool(
347 "exit",
348 "Leave the page. Pass answer when the goal is done, or reason when giving up.",
349 json!({
350 "type": "OBJECT",
351 "properties": {
352 "answer": { "type": "STRING", "description": "What was found" },
353 "reason": { "type": "STRING", "description": "Why the goal cannot be met" },
354 },
355 }),
356 run(|p| BrowserTool::Exit {
357 answer: present(p.get("answer")).map(|answer| js_string(Some(answer), "")),
358 reason: present(p.get("reason")).map(|reason| js_string(Some(reason), "")),
359 }),
360 ));
361 tools
362}
363
364struct Page {
365 cursor: HtmlPointer,
366 controller: Option<String>,
367 agent: Option<RoomAgent>,
368}
369
370struct PageState {
371 web: WebRoom,
372 enabled_tools: HashSet<String>,
373 page: Mutex<Page>,
374 hooks: Mutex<Arc<dyn BrowserRoomHooks>>,
375 pending: Mutex<HashMap<String, oneshot::Sender<Result<String>>>>,
376}
377
378fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
379 mutex.lock().unwrap_or_else(PoisonError::into_inner)
380}
381
382#[derive(Clone)]
387pub struct PageRoom {
388 room: Room,
389 inner: Arc<PageState>,
390}
391
392impl Deref for PageRoom {
393 type Target = Room;
394 fn deref(&self) -> &Room {
395 &self.room
396 }
397}
398
399impl PageRoom {
400 pub(crate) fn create(
403 room: &Room,
404 hooks: Arc<dyn BrowserRoomHooks>,
405 state: &Value,
406 enabled_tools: Option<Vec<String>>,
407 ) -> PageRoom {
408 let enabled_tools: HashSet<String> = match enabled_tools.filter(|tools| !tools.is_empty()) {
409 Some(tools) => tools.into_iter().collect(),
410 None => ALL_TOOLS.iter().map(|kind| kind.to_string()).collect(),
411 };
412 let web = WebRoom::from_parent(room, hooks.web(), state);
413 if let Ok(bootstrap) = serde_json::from_value::<BootstrapMeta>(state["bootstrap"].clone()) {
414 web.ingest_bootstrap(bootstrap);
415 }
416 let page = PageRoom {
417 room: room.clone(),
418 inner: Arc::new(PageState {
419 web,
420 enabled_tools,
421 page: Mutex::new(Page {
422 cursor: serde_json::from_value(state["cursor"].clone()).unwrap_or_default(),
423 controller: None,
424 agent: None,
425 }),
426 hooks: Mutex::new(hooks),
427 pending: Mutex::new(HashMap::new()),
428 }),
429 };
430 page.apply_control(&state["control"]);
431 page
432 }
433
434 fn page(&self) -> MutexGuard<'_, Page> {
435 lock(&self.inner.page)
436 }
437
438 fn hooks(&self) -> Arc<dyn BrowserRoomHooks> {
439 lock(&self.inner.hooks).clone()
440 }
441
442 pub fn web(&self) -> &WebRoom {
443 &self.inner.web
444 }
445
446 pub fn cursor(&self) -> HtmlPointer {
447 self.page().cursor.clone()
448 }
449
450 pub(crate) fn set_cursor(&self, cursor: HtmlPointer) {
451 self.page().cursor = cursor;
452 }
453
454 pub fn enabled_tools(&self) -> &HashSet<String> {
455 &self.inner.enabled_tools
456 }
457
458 pub fn controller(&self) -> Option<String> {
460 self.page().controller.clone()
461 }
462
463 pub fn agent(&self) -> Option<RoomAgent> {
465 self.page().agent.clone()
466 }
467
468 pub fn watch(&self, hooks: Arc<dyn BrowserRoomHooks>) {
469 if let Some(web) = hooks.web() {
470 self.inner.web.set_hooks(web);
471 }
472 *lock(&self.inner.hooks) = hooks;
473 }
474
475 pub fn controlling(&self, me: &Presence) -> bool {
477 let page = self.page();
478 page.agent.is_none() && page.controller.as_deref() == Some(me.id())
479 }
480
481 async fn call(
482 &self,
483 packet: impl FnOnce(&str) -> Value,
484 timeout: Duration,
485 timed_out: String,
486 ) -> Result<String> {
487 let id = uuid::Uuid::new_v4().to_string();
488 let (resolve, result) = oneshot::channel();
489 lock(&self.inner.pending).insert(id.clone(), resolve);
490 self.post(&packet(&id));
491 match tokio::time::timeout(timeout, result).await {
492 Ok(Ok(result)) => result,
493 _ => {
494 lock(&self.inner.pending).remove(&id);
495 Err(Error::Message(timed_out))
496 }
497 }
498 }
499
500 pub async fn set_controller(&self, seat: Option<&str>) -> Result<()> {
502 self.call(
503 |id| match seat.filter(|seat| !seat.is_empty()) {
504 Some(seat) => {
505 json!({ "event": "control-call", "data": { "id": id, "seat": seat } })
506 }
507 None => json!({ "event": "control-call", "data": { "id": id } }),
508 },
509 Duration::from_secs(30),
510 "Control timed out.".to_string(),
511 )
512 .await
513 .map(|_| ())
514 }
515
516 pub fn emit_presence(&self) {
518 self.hooks().on_presence(self);
519 }
520
521 pub fn emit_closed(&self) {
522 self.hooks().on_closed(self);
523 }
524
525 pub(crate) fn apply_control(&self, control: &Value) {
526 if !control.is_object() {
527 return;
528 }
529 let mut page = self.page();
530 page.controller = control["controller"].as_str().map(str::to_string);
531 page.agent = if control["agent"].is_object() {
532 Some(RoomAgent::from_json(&control["agent"]))
533 } else {
534 None
535 };
536 }
537
538 pub async fn invoke_tool(&self, tool: &BrowserTool) -> Result<String> {
539 if !self.inner.enabled_tools.contains(tool.kind()) {
540 return Err(Error::Message(format!(
541 "Tool is not enabled: {}",
542 tool.kind()
543 )));
544 }
545 self.call(
546 |id| json!({ "event": "browser-tool-call", "data": { "id": id, "tool": tool } }),
547 Duration::from_secs(30),
548 format!("Tool timed out: {}", tool.kind()),
549 )
550 .await
551 }
552
553 pub fn parrot_tools(&self) -> Vec<Tool> {
554 let room = self.clone();
555 parrot_tools(
556 &self.inner.enabled_tools,
557 Arc::new(move |tool| {
558 let room = room.clone();
559 Box::pin(async move { room.invoke_tool(&tool).await })
560 }),
561 )
562 }
563
564 fn settle(&self, data: &Value, ok: impl FnOnce() -> String) -> bool {
565 let Some(id) = data["id"].as_str().filter(|id| !id.is_empty()) else {
566 return true;
567 };
568 let Some(pending) = lock(&self.inner.pending).remove(id) else {
569 return true;
570 };
571 let failed = !(data["error"].is_null() || data["error"] == false || data["error"] == "");
572 let _ = pending.send(if failed {
573 Err(Error::Message(match &data["error"] {
574 Value::String(error) => error.clone(),
575 error => error.to_string(),
576 }))
577 } else {
578 Ok(ok())
579 });
580 true
581 }
582
583 pub fn handle_event(&self, packet: &Value) -> bool {
584 let event = packet["event"].as_str().unwrap_or_default();
585 if event == "browser-closed" || event == "server-closed" {
586 self.inner.web.handle_event(packet);
587 self.hooks().on_closed(self);
588 return true;
589 }
590 if self.inner.web.handle_event(packet) {
591 return true;
592 }
593 let data = &packet["data"];
594 match event {
595 "room-control" => {
596 let empty = json!({});
597 self.apply_control(if data.is_object() { data } else { &empty });
598 self.hooks().on_control(self);
599 true
600 }
601 "control-result" => self.settle(data, || "ok".to_string()),
602 "browser-cursor" => {
603 if let Ok(cursor) = serde_json::from_value::<HtmlPointer>(data["cursor"].clone()) {
604 self.set_cursor(cursor.clone());
605 self.hooks().on_cursor(self, &cursor);
606 }
607 true
608 }
609 "browser-tool-call" => {
610 if let Ok(tool) = serde_json::from_value::<BrowserTool>(data["tool"].clone()) {
611 self.hooks().on_tool(self, &tool);
612 }
613 true
614 }
615 "browser-tool-result" => self.settle(data, || match &data["result"] {
616 Value::Null => String::new(),
617 Value::String(result) => result.clone(),
618 result => result.to_string(),
619 }),
620 _ => false,
621 }
622 }
623}
624
625#[derive(Default)]
628pub struct BrowserOptions {
629 pub tools: Option<Vec<String>>,
630 pub hooks: Option<Arc<dyn BrowserRoomHooks>>,
631 pub group: Option<Group>,
632}
633
634#[derive(Clone)]
636pub struct BrowserRoom(PageRoom);
637
638impl Deref for BrowserRoom {
639 type Target = PageRoom;
640 fn deref(&self) -> &PageRoom {
641 &self.0
642 }
643}
644
645impl BrowserRoom {
646 pub async fn connect(
647 authorization: &CredentialsOrApiKey,
648 reference: &str,
649 opts: BrowserOptions,
650 ) -> Result<(BrowserRoom, Presence)> {
651 let group = opts.group.as_ref().map(|group| group.id.clone());
652 open_browser(
653 Authorization::Server {
654 authorization: authorization.clone(),
655 server: reference.to_string(),
656 group,
657 },
658 opts,
659 )
660 .await
661 }
662
663 pub async fn join(
664 authorization: &CredentialsOrApiKey,
665 access_code: &str,
666 opts: BrowserOptions,
667 ) -> Result<(BrowserRoom, Presence)> {
668 open_browser(
669 Authorization::AccessCode {
670 authorization: authorization.clone(),
671 access_code: access_code.to_string(),
672 },
673 opts,
674 )
675 .await
676 }
677}
678
679struct BrowserHooks {
680 room: Mutex<Option<PageRoom>>,
681 hooks: Arc<dyn BrowserRoomHooks>,
682 tools: Option<Vec<String>>,
683}
684
685impl BrowserHooks {
686 fn room(&self) -> Option<PageRoom> {
687 lock(&self.room).clone()
688 }
689}
690
691impl Hooks for BrowserHooks {
692 fn create_room(&self, room: &Room, state: &Value) {
693 let tools = self.tools.clone().or_else(|| {
694 state["enabledTools"].as_array().map(|tools| {
695 tools
696 .iter()
697 .filter_map(|tool| tool.as_str().map(str::to_string))
698 .collect()
699 })
700 });
701 *lock(&self.room) = Some(PageRoom::create(room, self.hooks.clone(), state, tools));
702 }
703
704 fn on_connection(&self, _room: &Room, state: &Value) {
705 let Some(room) = self.room() else {
706 return;
707 };
708 if let Ok(bootstrap) = serde_json::from_value::<BootstrapMeta>(state["bootstrap"].clone()) {
709 room.web().ingest_bootstrap(bootstrap);
710 }
711 if let Ok(cursor) = serde_json::from_value::<HtmlPointer>(state["cursor"].clone()) {
712 room.set_cursor(cursor.clone());
713 room.hooks().on_cursor(&room, &cursor);
714 }
715 }
716
717 fn on_event(&self, _room: &Room, packet: &Value) -> bool {
718 self.room().is_some_and(|room| room.handle_event(packet))
719 }
720
721 fn on_presence(&self, _room: &Room) {
722 if let Some(room) = self.room() {
723 room.emit_presence();
724 }
725 }
726
727 fn on_closed(&self, _room: &Room, _reason: Option<&str>) {
728 if let Some(room) = self.room() {
729 room.emit_closed();
730 }
731 }
732}
733
734async fn open_browser(
735 auth: Authorization,
736 opts: BrowserOptions,
737) -> Result<(BrowserRoom, Presence)> {
738 let endpoint = format!("{}/browser/live", super::get_url());
739 let hooks = Arc::new(BrowserHooks {
740 room: Mutex::new(None),
741 hooks: opts.hooks.unwrap_or_else(|| Arc::new(())),
742 tools: opts.tools,
743 });
744 let me = connect_server(auth, &endpoint, hooks.clone()).await?;
745 let room = hooks
746 .room()
747 .ok_or_else(|| Error::Message("Room not found".to_string()))?;
748 room.emit_presence();
749 Ok((BrowserRoom(room), me))
750}