1use std::collections::HashMap;
2use std::net::{IpAddr, Ipv6Addr};
3use std::path::{Path, PathBuf};
4use std::process::Stdio;
5use std::sync::atomic::{AtomicBool, Ordering};
6use std::sync::Arc;
7
8use anyhow::{anyhow, Context};
9use async_trait::async_trait;
10use base64::Engine;
11use flate2::read::GzDecoder;
12use serde::{Deserialize, Serialize};
13use serde_json::{json, Value};
14use tandem_browser::{
15 detect_sidecar_binary_path, run_doctor, BrowserActionResult, BrowserArtifactRef,
16 BrowserBlockingIssue, BrowserCloseParams, BrowserCloseResult, BrowserDoctorOptions,
17 BrowserExtractParams, BrowserExtractResult, BrowserNavigateParams, BrowserNavigateResult,
18 BrowserOpenRequest, BrowserOpenResult, BrowserPressParams, BrowserRpcRequest,
19 BrowserRpcResponse, BrowserScreenshotParams, BrowserScreenshotResult, BrowserSnapshotParams,
20 BrowserSnapshotResult, BrowserStatus, BrowserTypeParams, BrowserViewport, BrowserWaitParams,
21 BROWSER_PROTOCOL_VERSION,
22};
23use tandem_core::{resolve_shared_paths, BrowserConfig};
24use tandem_tools::{Tool, ToolRegistry};
25use tandem_types::{EngineEvent, ToolResult, ToolSchema};
26use tokio::fs;
27use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
28use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command};
29use tokio::sync::{Mutex, RwLock};
30use uuid::Uuid;
31
32use crate::{now_ms, AppState, RoutineRunArtifact, RuntimeState};
33
34const STATUS_CACHE_MAX_AGE_MS: u64 = 30_000;
35const INLINE_EXTRACT_LIMIT_BYTES: usize = 24_000;
36const SNAPSHOT_SCREENSHOT_LABEL: &str = "browser snapshot";
37const RELEASE_REPO: &str = "frumu-ai/tandem";
38const RELEASES_URL_ENV: &str = "TANDEM_BROWSER_RELEASES_URL";
39const BROWSER_INSTALL_USER_AGENT: &str = "tandem-browser-installer";
40
41#[derive(Debug)]
42struct BrowserSidecarClient {
43 _child: Child,
44 stdin: ChildStdin,
45 stdout: BufReader<ChildStdout>,
46 stderr: BufReader<ChildStderr>,
47 next_id: u64,
48}
49
50#[derive(Debug, Clone)]
51struct ManagedBrowserSession {
52 owner_session_id: Option<String>,
53 current_url: String,
54 _created_at_ms: u64,
55 updated_at_ms: u64,
56}
57
58#[derive(Debug, Clone, Serialize, Deserialize, Default)]
59pub struct BrowserHealthSummary {
60 pub enabled: bool,
61 pub runnable: bool,
62 pub tools_registered: bool,
63 pub sidecar_found: bool,
64 pub browser_found: bool,
65 #[serde(default, skip_serializing_if = "Option::is_none")]
66 pub browser_version: Option<String>,
67 #[serde(default, skip_serializing_if = "Option::is_none")]
68 pub last_checked_at_ms: Option<u64>,
69 #[serde(default, skip_serializing_if = "Option::is_none")]
70 pub last_error: Option<String>,
71}
72
73#[derive(Debug, Clone, Serialize, Deserialize)]
74pub struct BrowserSidecarInstallResult {
75 pub version: String,
76 pub asset_name: String,
77 pub installed_path: String,
78 pub downloaded_bytes: u64,
79 pub status: BrowserStatus,
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
83pub struct BrowserSmokeTestResult {
84 pub ok: bool,
85 pub status: BrowserStatus,
86 pub url: String,
87 pub final_url: String,
88 pub title: String,
89 pub load_state: String,
90 pub element_count: usize,
91 #[serde(default, skip_serializing_if = "Option::is_none")]
92 pub excerpt: Option<String>,
93 pub closed: bool,
94}
95
96#[derive(Debug, Clone, Deserialize)]
97struct GitHubRelease {
98 tag_name: String,
99 assets: Vec<GitHubAsset>,
100}
101
102#[derive(Debug, Clone, Deserialize)]
103struct GitHubAsset {
104 name: String,
105 browser_download_url: String,
106 size: u64,
107}
108
109#[derive(Clone)]
110pub struct BrowserSubsystem {
111 config: BrowserConfig,
112 status: Arc<RwLock<BrowserStatus>>,
113 tools_registered: Arc<AtomicBool>,
114 client: Arc<Mutex<Option<BrowserSidecarClient>>>,
115 sessions: Arc<RwLock<HashMap<String, ManagedBrowserSession>>>,
116 artifact_root: PathBuf,
117}
118
119#[derive(Clone, Copy)]
120enum BrowserToolKind {
121 Status,
122 Open,
123 Navigate,
124 Snapshot,
125 Click,
126 Type,
127 Press,
128 Wait,
129 Extract,
130 Screenshot,
131 Close,
132}
133
134#[derive(Clone)]
135pub struct BrowserTool {
136 kind: BrowserToolKind,
137 browser: BrowserSubsystem,
138 state: Option<AppState>,
139}
140
141#[derive(Debug, Deserialize)]
142struct BrowserTypeToolArgs {
143 session_id: String,
144 #[serde(default)]
145 element_id: Option<String>,
146 #[serde(default)]
147 selector: Option<String>,
148 #[serde(default)]
149 text: Option<String>,
150 #[serde(default)]
151 secret_ref: Option<String>,
152 #[serde(default)]
153 replace: bool,
154 #[serde(default)]
155 submit: bool,
156 #[serde(default)]
157 timeout_ms: Option<u64>,
158}
159
160#[derive(Debug, Deserialize)]
161struct BrowserToolContext {
162 #[serde(default, rename = "__session_id")]
163 model_session_id: Option<String>,
164}
165
166impl BrowserSidecarClient {
167 async fn spawn(config: &BrowserConfig) -> anyhow::Result<Self> {
168 let sidecar_path = detect_sidecar_binary_path(config.sidecar_path.as_deref())
169 .ok_or_else(|| anyhow!("browser_sidecar_not_found"))?;
170 let mut cmd = Command::new(&sidecar_path);
171 cmd.arg("serve")
172 .arg("--transport")
173 .arg("stdio")
174 .stdin(Stdio::piped())
175 .stdout(Stdio::piped())
176 .stderr(Stdio::piped());
177 if let Some(path) = config
178 .executable_path
179 .as_deref()
180 .filter(|v| !v.trim().is_empty())
181 {
182 cmd.env("TANDEM_BROWSER_EXECUTABLE", path);
183 }
184 if let Some(path) = config
185 .user_data_root
186 .as_deref()
187 .filter(|v| !v.trim().is_empty())
188 {
189 cmd.env("TANDEM_BROWSER_USER_DATA_ROOT", path);
190 }
191 cmd.env(
192 "TANDEM_BROWSER_ALLOW_NO_SANDBOX",
193 if config.allow_no_sandbox { "1" } else { "0" },
194 );
195 cmd.env(
196 "TANDEM_BROWSER_HEADLESS",
197 if config.headless_default { "1" } else { "0" },
198 );
199
200 let mut child = cmd.spawn().with_context(|| {
201 format!(
202 "failed to spawn tandem-browser sidecar at `{}`",
203 sidecar_path.display()
204 )
205 })?;
206 let stdin = child
207 .stdin
208 .take()
209 .ok_or_else(|| anyhow!("browser sidecar stdin unavailable"))?;
210 let stdout = child
211 .stdout
212 .take()
213 .ok_or_else(|| anyhow!("browser sidecar stdout unavailable"))?;
214 let stderr = child
215 .stderr
216 .take()
217 .ok_or_else(|| anyhow!("browser sidecar stderr unavailable"))?;
218 let mut client = Self {
219 _child: child,
220 stdin,
221 stdout: BufReader::new(stdout),
222 stderr: BufReader::new(stderr),
223 next_id: 1,
224 };
225 let version: Value = client.call_raw("browser.version", json!({})).await?;
226 let protocol = version
227 .get("protocol_version")
228 .and_then(Value::as_str)
229 .unwrap_or("");
230 if protocol != BROWSER_PROTOCOL_VERSION {
231 anyhow::bail!(
232 "protocol_mismatch: expected browser protocol {}, got {}",
233 BROWSER_PROTOCOL_VERSION,
234 protocol
235 );
236 }
237 Ok(client)
238 }
239
240 async fn call_raw(&mut self, method: &str, params: Value) -> anyhow::Result<Value> {
241 let id = self.next_id;
242 self.next_id = self.next_id.saturating_add(1);
243 let request = BrowserRpcRequest {
244 jsonrpc: "2.0".to_string(),
245 id: json!(id),
246 method: method.to_string(),
247 params,
248 };
249 let raw = serde_json::to_string(&request)?;
250 self.stdin.write_all(raw.as_bytes()).await?;
251 self.stdin.write_all(b"\n").await?;
252 self.stdin.flush().await?;
253
254 let mut line = String::new();
255 let read = self.stdout.read_line(&mut line).await?;
256 if read == 0 {
257 let mut stderr = String::new();
258 let _ = self.stderr.read_to_string(&mut stderr).await;
259 let stderr = stderr.trim();
260 if stderr.is_empty() {
261 anyhow::bail!("browser sidecar closed the stdio connection");
262 }
263 anyhow::bail!(
264 "browser sidecar closed the stdio connection: {}",
265 smoke_excerpt(stderr, 600)
266 );
267 }
268 let response: BrowserRpcResponse =
269 serde_json::from_str(line.trim()).context("invalid browser sidecar response")?;
270 if let Some(error) = response.error {
271 anyhow::bail!("{}", error.message);
272 }
273 response
274 .result
275 .ok_or_else(|| anyhow!("browser sidecar returned an empty result"))
276 }
277
278 async fn call<T: Serialize, R: for<'de> Deserialize<'de>>(
279 &mut self,
280 method: &str,
281 params: T,
282 ) -> anyhow::Result<R> {
283 let value = self.call_raw(method, serde_json::to_value(params)?).await?;
284 serde_json::from_value(value).context("invalid browser sidecar payload")
285 }
286
287 async fn call_value<R: for<'de> Deserialize<'de>>(
288 &mut self,
289 method: &str,
290 params: Value,
291 ) -> anyhow::Result<R> {
292 let value = self.call_raw(method, params).await?;
293 serde_json::from_value(value).context("invalid browser sidecar payload")
294 }
295}
296
297impl BrowserSubsystem {
298 pub fn new(config: BrowserConfig) -> Self {
299 let artifact_root = resolve_shared_paths()
300 .map(|paths| paths.canonical_root.join("browser-artifacts"))
301 .unwrap_or_else(|_| PathBuf::from(".tandem").join("browser-artifacts"));
302 Self {
303 config,
304 status: Arc::new(RwLock::new(BrowserStatus::default())),
305 tools_registered: Arc::new(AtomicBool::new(false)),
306 client: Arc::new(Mutex::new(None)),
307 sessions: Arc::new(RwLock::new(HashMap::new())),
308 artifact_root,
309 }
310 }
311
312 pub fn config(&self) -> &BrowserConfig {
313 &self.config
314 }
315
316 pub async fn install_sidecar(&self) -> anyhow::Result<BrowserSidecarInstallResult> {
317 let mut result = install_browser_sidecar(&self.config).await?;
318 result.status = self.refresh_status().await;
319 Ok(result)
320 }
321
322 pub async fn smoke_test(&self, url: Option<String>) -> anyhow::Result<BrowserSmokeTestResult> {
323 let status = self.status_snapshot().await;
324 if !status.runnable {
325 anyhow::bail!(
326 "browser_not_runnable: run browser doctor first; current status is not runnable"
327 );
328 }
329
330 let target_url = url
331 .map(|value| value.trim().to_string())
332 .filter(|value| !value.is_empty())
333 .unwrap_or_else(|| "https://example.com".to_string());
334 let request = BrowserOpenRequest {
335 url: target_url.clone(),
336 profile_id: None,
337 headless: Some(self.config.headless_default),
338 viewport: Some(BrowserViewport {
339 width: self.config.default_viewport.width,
340 height: self.config.default_viewport.height,
341 }),
342 wait_until: Some("navigation".to_string()),
343 executable_path: self.config.executable_path.clone(),
344 user_data_root: self.config.user_data_root.clone(),
345 allow_no_sandbox: self.config.allow_no_sandbox,
346 headless_default: self.config.headless_default,
347 };
348 let opened: BrowserOpenResult = self.call_sidecar("browser.open", request).await?;
349 let session_id = opened.session_id.clone();
350
351 let result = async {
352 let snapshot: BrowserSnapshotResult = self
353 .call_sidecar(
354 "browser.snapshot",
355 BrowserSnapshotParams {
356 session_id: session_id.clone(),
357 max_elements: Some(25),
358 include_screenshot: false,
359 },
360 )
361 .await?;
362 let extract: BrowserExtractResult = self
363 .call_sidecar(
364 "browser.extract",
365 BrowserExtractParams {
366 session_id: session_id.clone(),
367 format: "visible_text".to_string(),
368 max_bytes: Some(4_000),
369 },
370 )
371 .await?;
372 Ok::<BrowserSmokeTestResult, anyhow::Error>(BrowserSmokeTestResult {
373 ok: true,
374 status,
375 url: target_url,
376 final_url: snapshot.url,
377 title: snapshot.title,
378 load_state: snapshot.load_state,
379 element_count: snapshot.elements.len(),
380 excerpt: Some(smoke_excerpt(&extract.content, 400)),
381 closed: false,
382 })
383 }
384 .await;
385
386 let close_result: BrowserCloseResult = self
387 .call_sidecar(
388 "browser.close",
389 BrowserCloseParams {
390 session_id: session_id.clone(),
391 },
392 )
393 .await
394 .unwrap_or(BrowserCloseResult {
395 session_id,
396 closed: false,
397 });
398
399 let mut smoke = result?;
400 smoke.closed = close_result.closed;
401 Ok(smoke)
402 }
403
404 pub async fn refresh_status(&self) -> BrowserStatus {
405 let config = self.config.clone();
406 let evaluated = tokio::task::spawn_blocking(move || evaluate_browser_status(config))
407 .await
408 .unwrap_or_else(|err| BrowserStatus {
409 enabled: false,
410 runnable: false,
411 headless_default: true,
412 sidecar: Default::default(),
413 browser: Default::default(),
414 blocking_issues: vec![BrowserBlockingIssue {
415 code: "browser_launch_failed".to_string(),
416 message: format!("browser readiness task failed: {}", err),
417 }],
418 recommendations: vec![
419 "Run `tandem-engine browser doctor --json` on the same host.".to_string(),
420 ],
421 install_hints: Vec::new(),
422 last_checked_at_ms: Some(now_ms()),
423 last_error: Some(err.to_string()),
424 });
425 *self.status.write().await = evaluated.clone();
426 evaluated
427 }
428
429 pub async fn status_snapshot(&self) -> BrowserStatus {
430 let current = self.status.read().await.clone();
431 if current
432 .last_checked_at_ms
433 .is_some_and(|ts| now_ms().saturating_sub(ts) <= STATUS_CACHE_MAX_AGE_MS)
434 {
435 current
436 } else {
437 self.refresh_status().await
438 }
439 }
440
441 pub async fn health_summary(&self) -> BrowserHealthSummary {
442 let status = self.status.read().await.clone();
443 BrowserHealthSummary {
444 enabled: status.enabled,
445 runnable: status.runnable,
446 tools_registered: self.tools_registered.load(Ordering::Relaxed),
447 sidecar_found: status.sidecar.found,
448 browser_found: status.browser.found,
449 browser_version: status.browser.version,
450 last_checked_at_ms: status.last_checked_at_ms,
451 last_error: status.last_error,
452 }
453 }
454
455 pub fn set_tools_registered(&self, value: bool) {
456 self.tools_registered.store(value, Ordering::Relaxed);
457 }
458
459 pub async fn register_tools(
460 &self,
461 tools: &ToolRegistry,
462 state: Option<AppState>,
463 ) -> anyhow::Result<()> {
464 tools.unregister_by_prefix("browser_").await;
465 tools
466 .register_tool(
467 "browser_status".to_string(),
468 Arc::new(BrowserTool::new(
469 BrowserToolKind::Status,
470 self.clone(),
471 state.clone(),
472 )),
473 )
474 .await;
475
476 let status = self.status_snapshot().await;
477 if !status.enabled || !status.runnable {
478 self.set_tools_registered(false);
479 return Ok(());
480 }
481
482 for (name, kind) in [
483 ("browser_open", BrowserToolKind::Open),
484 ("browser_navigate", BrowserToolKind::Navigate),
485 ("browser_snapshot", BrowserToolKind::Snapshot),
486 ("browser_click", BrowserToolKind::Click),
487 ("browser_type", BrowserToolKind::Type),
488 ("browser_press", BrowserToolKind::Press),
489 ("browser_wait", BrowserToolKind::Wait),
490 ("browser_extract", BrowserToolKind::Extract),
491 ("browser_screenshot", BrowserToolKind::Screenshot),
492 ("browser_close", BrowserToolKind::Close),
493 ] {
494 tools
495 .register_tool(
496 name.to_string(),
497 Arc::new(BrowserTool::new(kind, self.clone(), state.clone())),
498 )
499 .await;
500 }
501 self.set_tools_registered(true);
502 Ok(())
503 }
504
505 async fn update_last_error(&self, message: impl Into<String>) {
506 let mut status = self.status.write().await;
507 status.last_error = Some(message.into());
508 status.last_checked_at_ms = Some(now_ms());
509 }
510
511 async fn call_sidecar<T: Serialize, R: for<'de> Deserialize<'de>>(
512 &self,
513 method: &str,
514 params: T,
515 ) -> anyhow::Result<R> {
516 let params = serde_json::to_value(params)?;
517 let mut guard = self.client.lock().await;
518 if guard.is_none() {
519 *guard = Some(BrowserSidecarClient::spawn(&self.config).await?);
520 }
521 let result = guard
522 .as_mut()
523 .expect("browser sidecar client initialized")
524 .call_value(method, params.clone())
525 .await;
526 if let Err(err) = &result {
527 *guard = None;
528 self.update_last_error(err.to_string()).await;
529 if err
530 .to_string()
531 .contains("browser sidecar closed the stdio connection")
532 {
533 *guard = Some(BrowserSidecarClient::spawn(&self.config).await?);
534 return guard
535 .as_mut()
536 .expect("browser sidecar client reinitialized")
537 .call_value(method, params)
538 .await;
539 }
540 }
541 result
542 }
543
544 async fn insert_session(
545 &self,
546 browser_session_id: String,
547 owner_session_id: Option<String>,
548 current_url: String,
549 ) {
550 self.sessions.write().await.insert(
551 browser_session_id,
552 ManagedBrowserSession {
553 owner_session_id,
554 current_url,
555 _created_at_ms: now_ms(),
556 updated_at_ms: now_ms(),
557 },
558 );
559 }
560
561 async fn session(&self, browser_session_id: &str) -> Option<ManagedBrowserSession> {
562 self.sessions.read().await.get(browser_session_id).cloned()
563 }
564
565 async fn update_session_url(
566 &self,
567 browser_session_id: &str,
568 current_url: String,
569 ) -> Option<ManagedBrowserSession> {
570 let mut sessions = self.sessions.write().await;
571 let session = sessions.get_mut(browser_session_id)?;
572 session.current_url = current_url;
573 session.updated_at_ms = now_ms();
574 Some(session.clone())
575 }
576
577 async fn remove_session(&self, browser_session_id: &str) -> Option<ManagedBrowserSession> {
578 self.sessions.write().await.remove(browser_session_id)
579 }
580
581 pub async fn close_sessions_for_owner(&self, owner_session_id: &str) -> usize {
582 let session_ids = self
583 .sessions
584 .read()
585 .await
586 .iter()
587 .filter_map(|(session_id, session)| {
588 (session.owner_session_id.as_deref() == Some(owner_session_id))
589 .then_some(session_id.clone())
590 })
591 .collect::<Vec<_>>();
592 self.close_session_ids(session_ids).await
593 }
594
595 pub async fn close_all_sessions(&self) -> usize {
596 let session_ids = self
597 .sessions
598 .read()
599 .await
600 .keys()
601 .cloned()
602 .collect::<Vec<_>>();
603 self.close_session_ids(session_ids).await
604 }
605
606 async fn close_session_ids(&self, session_ids: Vec<String>) -> usize {
607 let mut closed = 0usize;
608 for session_id in session_ids {
609 let _ = self
610 .call_sidecar::<_, BrowserCloseResult>(
611 "browser.close",
612 BrowserCloseParams {
613 session_id: session_id.clone(),
614 },
615 )
616 .await;
617 if self.remove_session(&session_id).await.is_some() {
618 closed += 1;
619 }
620 }
621 closed
622 }
623}
624
625impl BrowserTool {
626 fn new(kind: BrowserToolKind, browser: BrowserSubsystem, state: Option<AppState>) -> Self {
627 Self {
628 kind,
629 browser,
630 state,
631 }
632 }
633
634 async fn execute_impl(&self, args: Value) -> anyhow::Result<ToolResult> {
635 match self.kind {
636 BrowserToolKind::Status => self.execute_status().await,
637 BrowserToolKind::Open => self.execute_open(args).await,
638 BrowserToolKind::Navigate => self.execute_navigate(args).await,
639 BrowserToolKind::Snapshot => self.execute_snapshot(args).await,
640 BrowserToolKind::Click => self.execute_click(args).await,
641 BrowserToolKind::Type => self.execute_type(args).await,
642 BrowserToolKind::Press => self.execute_press(args).await,
643 BrowserToolKind::Wait => self.execute_wait(args).await,
644 BrowserToolKind::Extract => self.execute_extract(args).await,
645 BrowserToolKind::Screenshot => self.execute_screenshot(args).await,
646 BrowserToolKind::Close => self.execute_close(args).await,
647 }
648 }
649
650 async fn execute_status(&self) -> anyhow::Result<ToolResult> {
651 let status = self.browser.status_snapshot().await;
652 ok_tool_result(
653 serde_json::to_value(&status)?,
654 json!({
655 "enabled": status.enabled,
656 "runnable": status.runnable,
657 "sidecar_found": status.sidecar.found,
658 "browser_found": status.browser.found,
659 }),
660 )
661 }
662
663 async fn execute_open(&self, args: Value) -> anyhow::Result<ToolResult> {
664 let ctx = parse_tool_context(&args);
665 let mut request: BrowserOpenRequest =
666 serde_json::from_value(args.clone()).context("invalid browser_open arguments")?;
667 let status = self.browser.status_snapshot().await;
668 if !status.runnable {
669 return browser_not_runnable_result(&status);
670 }
671 ensure_allowed_browser_url(
672 &request.url,
673 &self
674 .effective_allowed_hosts(ctx.model_session_id.as_deref())
675 .await,
676 )?;
677 request.executable_path = self.browser.config.executable_path.clone();
678 request.user_data_root = self.browser.config.user_data_root.clone();
679 request.allow_no_sandbox = self.browser.config.allow_no_sandbox;
680 request.headless_default = self.browser.config.headless_default;
681 if request.viewport.is_none() {
682 request.viewport = Some(BrowserViewport {
683 width: self.browser.config.default_viewport.width,
684 height: self.browser.config.default_viewport.height,
685 });
686 }
687 let result: BrowserOpenResult = self.browser.call_sidecar("browser.open", request).await?;
688 ensure_allowed_browser_url(
689 &result.final_url,
690 &self
691 .effective_allowed_hosts(ctx.model_session_id.as_deref())
692 .await,
693 )
694 .map_err(|err| anyhow!("host_not_allowed: {}", err))?;
695 self.browser
696 .insert_session(
697 result.session_id.clone(),
698 ctx.model_session_id.clone(),
699 result.final_url.clone(),
700 )
701 .await;
702 ok_tool_result(
703 serde_json::to_value(&result)?,
704 json!({
705 "session_id": result.session_id,
706 "url": result.final_url,
707 "headless": result.headless,
708 }),
709 )
710 }
711
712 async fn execute_navigate(&self, args: Value) -> anyhow::Result<ToolResult> {
713 let ctx = parse_tool_context(&args);
714 let params: BrowserNavigateParams =
715 serde_json::from_value(args.clone()).context("invalid browser_navigate arguments")?;
716 let session = self
717 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
718 .await?;
719 ensure_allowed_browser_url(
720 ¶ms.url,
721 &self
722 .effective_allowed_hosts(session.owner_session_id.as_deref())
723 .await,
724 )?;
725 let result: BrowserNavigateResult = self
726 .browser
727 .call_sidecar("browser.navigate", params.clone())
728 .await?;
729 self.enforce_post_navigation(
730 ¶ms.session_id,
731 &result.final_url,
732 session.owner_session_id.as_deref(),
733 )
734 .await?;
735 ok_tool_result(
736 serde_json::to_value(&result)?,
737 json!({
738 "session_id": result.session_id,
739 "url": result.final_url,
740 }),
741 )
742 }
743
744 async fn execute_snapshot(&self, args: Value) -> anyhow::Result<ToolResult> {
745 let ctx = parse_tool_context(&args);
746 let params: BrowserSnapshotParams =
747 serde_json::from_value(args.clone()).context("invalid browser_snapshot arguments")?;
748 let session = self
749 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
750 .await?;
751 self.ensure_page_read_allowed(session.owner_session_id.as_deref(), &session.current_url)
752 .await?;
753 let mut result: BrowserSnapshotResult = self
754 .browser
755 .call_sidecar("browser.snapshot", params.clone())
756 .await?;
757 self.browser
758 .update_session_url(¶ms.session_id, result.url.clone())
759 .await;
760
761 let screenshot_artifact = if let Some(base64) = result.screenshot_base64.take() {
762 Some(
763 self.store_artifact(
764 ctx.model_session_id.as_deref(),
765 ¶ms.session_id,
766 "screenshot",
767 params
768 .include_screenshot
769 .then_some(SNAPSHOT_SCREENSHOT_LABEL.to_string()),
770 "png",
771 &base64::engine::general_purpose::STANDARD
772 .decode(base64.as_bytes())
773 .context("invalid snapshot screenshot payload")?,
774 Some(json!({
775 "source": "browser_snapshot",
776 "url": result.url,
777 })),
778 )
779 .await?,
780 )
781 } else {
782 None
783 };
784 let payload = json!({
785 "session_id": result.session_id,
786 "url": result.url,
787 "title": result.title,
788 "load_state": result.load_state,
789 "viewport": result.viewport,
790 "elements": result.elements,
791 "notices": result.notices,
792 "screenshot_artifact": screenshot_artifact,
793 });
794 ok_tool_result(
795 payload.clone(),
796 json!({
797 "session_id": payload.get("session_id"),
798 "url": payload.get("url"),
799 "element_count": payload.get("elements").and_then(Value::as_array).map(|rows| rows.len()).unwrap_or(0),
800 }),
801 )
802 }
803
804 async fn execute_click(&self, args: Value) -> anyhow::Result<ToolResult> {
805 let ctx = parse_tool_context(&args);
806 let params: tandem_browser::BrowserClickParams =
807 serde_json::from_value(args.clone()).context("invalid browser_click arguments")?;
808 let session = self
809 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
810 .await?;
811 self.ensure_action_allowed(session.owner_session_id.as_deref(), &session.current_url)
812 .await?;
813 let result: BrowserActionResult = self
814 .browser
815 .call_sidecar("browser.click", params.clone())
816 .await?;
817 self.update_action_url(
818 ¶ms.session_id,
819 result.final_url.as_deref(),
820 session.owner_session_id.as_deref(),
821 )
822 .await?;
823 ok_tool_result(
824 serde_json::to_value(&result)?,
825 json!({
826 "session_id": result.session_id,
827 "success": result.success,
828 "url": result.final_url,
829 }),
830 )
831 }
832
833 async fn execute_type(&self, args: Value) -> anyhow::Result<ToolResult> {
834 let ctx = parse_tool_context(&args);
835 let params: BrowserTypeToolArgs =
836 serde_json::from_value(args.clone()).context("invalid browser_type arguments")?;
837 let session = self
838 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
839 .await?;
840 self.ensure_action_allowed(session.owner_session_id.as_deref(), &session.current_url)
841 .await?;
842 let text = resolve_text_input(params.text.clone(), params.secret_ref.clone())?;
843 let request = BrowserTypeParams {
844 session_id: params.session_id.clone(),
845 element_id: params.element_id.clone(),
846 selector: params.selector.clone(),
847 text,
848 replace: params.replace,
849 submit: params.submit,
850 timeout_ms: params.timeout_ms,
851 };
852 let result: BrowserActionResult =
853 self.browser.call_sidecar("browser.type", request).await?;
854 self.update_action_url(
855 ¶ms.session_id,
856 result.final_url.as_deref(),
857 session.owner_session_id.as_deref(),
858 )
859 .await?;
860 ok_tool_result(
861 serde_json::to_value(&result)?,
862 json!({
863 "session_id": result.session_id,
864 "success": result.success,
865 "used_secret_ref": params.secret_ref.is_some(),
866 "url": result.final_url,
867 }),
868 )
869 }
870
871 async fn execute_press(&self, args: Value) -> anyhow::Result<ToolResult> {
872 let ctx = parse_tool_context(&args);
873 let params: BrowserPressParams =
874 serde_json::from_value(args.clone()).context("invalid browser_press arguments")?;
875 let session = self
876 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
877 .await?;
878 self.ensure_action_allowed(session.owner_session_id.as_deref(), &session.current_url)
879 .await?;
880 let result: BrowserActionResult = self
881 .browser
882 .call_sidecar("browser.press", params.clone())
883 .await?;
884 self.update_action_url(
885 ¶ms.session_id,
886 result.final_url.as_deref(),
887 session.owner_session_id.as_deref(),
888 )
889 .await?;
890 ok_tool_result(
891 serde_json::to_value(&result)?,
892 json!({
893 "session_id": result.session_id,
894 "success": result.success,
895 "url": result.final_url,
896 }),
897 )
898 }
899
900 async fn execute_wait(&self, args: Value) -> anyhow::Result<ToolResult> {
901 let ctx = parse_tool_context(&args);
902 let params: BrowserWaitParams =
903 serde_json::from_value(args.clone()).context("invalid browser_wait arguments")?;
904 let session = self
905 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
906 .await?;
907 self.ensure_page_read_allowed(session.owner_session_id.as_deref(), &session.current_url)
908 .await?;
909 let result: BrowserActionResult = self
910 .browser
911 .call_sidecar("browser.wait", params.clone())
912 .await?;
913 self.update_action_url(
914 ¶ms.session_id,
915 result.final_url.as_deref(),
916 session.owner_session_id.as_deref(),
917 )
918 .await?;
919 ok_tool_result(
920 serde_json::to_value(&result)?,
921 json!({
922 "session_id": result.session_id,
923 "success": result.success,
924 "url": result.final_url,
925 }),
926 )
927 }
928
929 async fn execute_extract(&self, args: Value) -> anyhow::Result<ToolResult> {
930 let ctx = parse_tool_context(&args);
931 let params: BrowserExtractParams =
932 serde_json::from_value(args.clone()).context("invalid browser_extract arguments")?;
933 let session = self
934 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
935 .await?;
936 self.ensure_page_read_allowed(session.owner_session_id.as_deref(), &session.current_url)
937 .await?;
938 let result: BrowserExtractResult = self
939 .browser
940 .call_sidecar("browser.extract", params.clone())
941 .await?;
942 let bytes = result.content.as_bytes();
943 let artifact = if bytes.len() > INLINE_EXTRACT_LIMIT_BYTES {
944 Some(
945 self.store_artifact(
946 ctx.model_session_id.as_deref(),
947 ¶ms.session_id,
948 "extract",
949 Some(format!("browser extract ({})", result.format)),
950 extension_for_extract_format(&result.format),
951 bytes,
952 Some(json!({
953 "format": result.format,
954 "truncated": result.truncated,
955 "source": "browser_extract",
956 })),
957 )
958 .await?,
959 )
960 } else {
961 None
962 };
963 let payload = json!({
964 "session_id": result.session_id,
965 "format": result.format,
966 "content": artifact.is_none().then_some(result.content),
967 "truncated": result.truncated,
968 "artifact": artifact,
969 });
970 ok_tool_result(
971 payload.clone(),
972 json!({
973 "session_id": payload.get("session_id"),
974 "format": payload.get("format"),
975 "artifact": payload.get("artifact").is_some(),
976 }),
977 )
978 }
979
980 async fn execute_screenshot(&self, args: Value) -> anyhow::Result<ToolResult> {
981 let ctx = parse_tool_context(&args);
982 let params: BrowserScreenshotParams =
983 serde_json::from_value(args.clone()).context("invalid browser_screenshot arguments")?;
984 let session = self
985 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
986 .await?;
987 self.ensure_page_read_allowed(session.owner_session_id.as_deref(), &session.current_url)
988 .await?;
989 let result: BrowserScreenshotResult = self
990 .browser
991 .call_sidecar("browser.screenshot", params.clone())
992 .await?;
993 let bytes = base64::engine::general_purpose::STANDARD
994 .decode(result.data_base64.as_bytes())
995 .context("invalid screenshot payload")?;
996 let artifact = self
997 .store_artifact(
998 ctx.model_session_id.as_deref(),
999 ¶ms.session_id,
1000 "screenshot",
1001 result.label.clone(),
1002 "png",
1003 &bytes,
1004 Some(json!({
1005 "mime_type": result.mime_type,
1006 "bytes": result.bytes,
1007 "source": "browser_screenshot",
1008 })),
1009 )
1010 .await?;
1011 ok_tool_result(
1012 json!({
1013 "session_id": result.session_id,
1014 "artifact": artifact,
1015 "summary": format!("Saved screenshot artifact ({} bytes).", result.bytes),
1016 }),
1017 json!({
1018 "session_id": result.session_id,
1019 "artifact_id": artifact.artifact_id,
1020 }),
1021 )
1022 }
1023
1024 async fn execute_close(&self, args: Value) -> anyhow::Result<ToolResult> {
1025 let ctx = parse_tool_context(&args);
1026 let params: BrowserCloseParams =
1027 serde_json::from_value(args.clone()).context("invalid browser_close arguments")?;
1028 let _ = self
1029 .load_session(¶ms.session_id, ctx.model_session_id.as_deref())
1030 .await?;
1031 let result: BrowserCloseResult = self
1032 .browser
1033 .call_sidecar("browser.close", params.clone())
1034 .await?;
1035 self.browser.remove_session(¶ms.session_id).await;
1036 ok_tool_result(
1037 serde_json::to_value(&result)?,
1038 json!({
1039 "session_id": result.session_id,
1040 "closed": result.closed,
1041 }),
1042 )
1043 }
1044
1045 async fn load_session(
1046 &self,
1047 browser_session_id: &str,
1048 model_session_id: Option<&str>,
1049 ) -> anyhow::Result<ManagedBrowserSession> {
1050 let session = self
1051 .browser
1052 .session(browser_session_id)
1053 .await
1054 .ok_or_else(|| anyhow!("session `{}` not found", browser_session_id))?;
1055 if let (Some(owner), Some(model_session_id)) =
1056 (session.owner_session_id.as_deref(), model_session_id)
1057 {
1058 if owner != model_session_id {
1059 anyhow::bail!(
1060 "browser session `{}` belongs to a different engine session",
1061 browser_session_id
1062 );
1063 }
1064 }
1065 Ok(session)
1066 }
1067
1068 async fn effective_allowed_hosts(&self, model_session_id: Option<&str>) -> Vec<String> {
1069 if let Some(model_session_id) = model_session_id {
1070 if let Some(state) = self.state.as_ref() {
1071 if let Some(instance) = state
1072 .agent_teams
1073 .instance_for_session(model_session_id)
1074 .await
1075 {
1076 if !instance.capabilities.net_scopes.allow_hosts.is_empty() {
1077 return normalize_allowed_hosts(
1078 instance.capabilities.net_scopes.allow_hosts,
1079 );
1080 }
1081 }
1082 }
1083 }
1084 normalize_allowed_hosts(self.browser.config.allowed_hosts.clone())
1085 }
1086
1087 async fn ensure_page_read_allowed(
1088 &self,
1089 model_session_id: Option<&str>,
1090 current_url: &str,
1091 ) -> anyhow::Result<()> {
1092 ensure_allowed_browser_url(
1093 current_url,
1094 &self.effective_allowed_hosts(model_session_id).await,
1095 )?;
1096 Ok(())
1097 }
1098
1099 async fn ensure_action_allowed(
1100 &self,
1101 model_session_id: Option<&str>,
1102 current_url: &str,
1103 ) -> anyhow::Result<()> {
1104 self.ensure_page_read_allowed(model_session_id, current_url)
1105 .await?;
1106 let host = browser_url_host(current_url)?;
1107 if !is_local_or_private_host(&host)
1108 && !self.external_integrations_allowed(model_session_id).await
1109 {
1110 anyhow::bail!(
1111 "external integrations are disabled for this routine session on host `{}`",
1112 host
1113 );
1114 }
1115 Ok(())
1116 }
1117
1118 async fn external_integrations_allowed(&self, model_session_id: Option<&str>) -> bool {
1119 let Some(model_session_id) = model_session_id else {
1120 return true;
1121 };
1122 let Some(state) = self.state.as_ref() else {
1123 return true;
1124 };
1125 let Some(policy) = state.routine_session_policy(model_session_id).await else {
1126 return true;
1127 };
1128 state
1129 .get_routine(&policy.routine_id)
1130 .await
1131 .map(|routine| routine.external_integrations_allowed)
1132 .unwrap_or(true)
1133 }
1134
1135 async fn enforce_post_navigation(
1136 &self,
1137 browser_session_id: &str,
1138 final_url: &str,
1139 model_session_id: Option<&str>,
1140 ) -> anyhow::Result<()> {
1141 if let Err(err) = ensure_allowed_browser_url(
1142 final_url,
1143 &self.effective_allowed_hosts(model_session_id).await,
1144 ) {
1145 let _ = self
1146 .browser
1147 .call_sidecar::<_, BrowserCloseResult>(
1148 "browser.close",
1149 BrowserCloseParams {
1150 session_id: browser_session_id.to_string(),
1151 },
1152 )
1153 .await;
1154 self.browser.remove_session(browser_session_id).await;
1155 return Err(anyhow!("host_not_allowed: {}", err));
1156 }
1157 self.browser
1158 .update_session_url(browser_session_id, final_url.to_string())
1159 .await;
1160 Ok(())
1161 }
1162
1163 async fn update_action_url(
1164 &self,
1165 browser_session_id: &str,
1166 final_url: Option<&str>,
1167 model_session_id: Option<&str>,
1168 ) -> anyhow::Result<()> {
1169 if let Some(final_url) = final_url {
1170 self.enforce_post_navigation(browser_session_id, final_url, model_session_id)
1171 .await?;
1172 }
1173 Ok(())
1174 }
1175
1176 async fn store_artifact(
1177 &self,
1178 model_session_id: Option<&str>,
1179 browser_session_id: &str,
1180 kind: &str,
1181 label: Option<String>,
1182 extension: &str,
1183 bytes: &[u8],
1184 metadata: Option<Value>,
1185 ) -> anyhow::Result<BrowserArtifactRef> {
1186 fs::create_dir_all(&self.browser.artifact_root).await?;
1187 let artifact_id = format!("artifact-{}", Uuid::new_v4());
1188 let file_name = format!("{artifact_id}.{extension}");
1189 let target = self.browser.artifact_root.join(file_name);
1190 fs::write(&target, bytes)
1191 .await
1192 .with_context(|| format!("failed to write browser artifact `{}`", target.display()))?;
1193 let artifact = BrowserArtifactRef {
1194 artifact_id: artifact_id.clone(),
1195 uri: target.to_string_lossy().to_string(),
1196 kind: kind.to_string(),
1197 label,
1198 created_at_ms: now_ms(),
1199 metadata,
1200 };
1201 self.append_routine_artifact_if_needed(
1202 model_session_id,
1203 artifact.clone(),
1204 browser_session_id,
1205 )
1206 .await;
1207 Ok(artifact)
1208 }
1209
1210 async fn append_routine_artifact_if_needed(
1211 &self,
1212 model_session_id: Option<&str>,
1213 artifact: BrowserArtifactRef,
1214 browser_session_id: &str,
1215 ) {
1216 let Some(model_session_id) = model_session_id else {
1217 return;
1218 };
1219 let Some(state) = self.state.as_ref() else {
1220 return;
1221 };
1222 let Some(policy) = state.routine_session_policy(model_session_id).await else {
1223 return;
1224 };
1225 let run_artifact = RoutineRunArtifact {
1226 artifact_id: artifact.artifact_id.clone(),
1227 uri: artifact.uri.clone(),
1228 kind: artifact.kind.clone(),
1229 label: artifact.label.clone(),
1230 created_at_ms: artifact.created_at_ms,
1231 metadata: artifact.metadata.clone(),
1232 };
1233 let _ = state
1234 .append_routine_run_artifact(&policy.run_id, run_artifact.clone())
1235 .await;
1236 state.event_bus.publish(EngineEvent::new(
1237 "routine.run.artifact_added",
1238 json!({
1239 "runID": policy.run_id,
1240 "routineID": policy.routine_id,
1241 "browserSessionID": browser_session_id,
1242 "artifact": run_artifact,
1243 }),
1244 ));
1245 }
1246}
1247
1248#[async_trait]
1249impl Tool for BrowserTool {
1250 fn schema(&self) -> ToolSchema {
1251 tool_schema(self.kind)
1252 }
1253
1254 async fn execute(&self, args: Value) -> anyhow::Result<ToolResult> {
1255 match self.execute_impl(args).await {
1256 Ok(result) => Ok(result),
1257 Err(err) => {
1258 let message = err.to_string();
1259 let (code, detail) = split_error_code(&message);
1260 Ok(error_tool_result(code, detail.to_string(), None))
1261 }
1262 }
1263 }
1264}
1265
1266impl RuntimeState {
1267 pub async fn browser_status(&self) -> BrowserStatus {
1268 self.browser.status_snapshot().await
1269 }
1270
1271 pub async fn browser_smoke_test(
1272 &self,
1273 url: Option<String>,
1274 ) -> anyhow::Result<BrowserSmokeTestResult> {
1275 self.browser.smoke_test(url).await
1276 }
1277
1278 pub async fn install_browser_sidecar(&self) -> anyhow::Result<BrowserSidecarInstallResult> {
1279 self.browser.install_sidecar().await
1280 }
1281
1282 pub async fn browser_health_summary(&self) -> BrowserHealthSummary {
1283 self.browser.health_summary().await
1284 }
1285
1286 pub async fn close_browser_sessions_for_owner(&self, owner_session_id: &str) -> usize {
1287 self.browser
1288 .close_sessions_for_owner(owner_session_id)
1289 .await
1290 }
1291
1292 pub async fn close_all_browser_sessions(&self) -> usize {
1293 self.browser.close_all_sessions().await
1294 }
1295}
1296
1297impl AppState {
1298 pub async fn browser_status(&self) -> BrowserStatus {
1299 match self.runtime.get() {
1300 Some(runtime) => runtime.browser.status_snapshot().await,
1301 None => BrowserStatus::default(),
1302 }
1303 }
1304
1305 pub async fn browser_smoke_test(
1306 &self,
1307 url: Option<String>,
1308 ) -> anyhow::Result<BrowserSmokeTestResult> {
1309 let Some(runtime) = self.runtime.get() else {
1310 anyhow::bail!("runtime not ready");
1311 };
1312 runtime.browser_smoke_test(url).await
1313 }
1314
1315 pub async fn install_browser_sidecar(&self) -> anyhow::Result<BrowserSidecarInstallResult> {
1316 let Some(runtime) = self.runtime.get() else {
1317 anyhow::bail!("runtime not ready");
1318 };
1319 runtime.install_browser_sidecar().await
1320 }
1321
1322 pub async fn browser_health_summary(&self) -> BrowserHealthSummary {
1323 match self.runtime.get() {
1324 Some(runtime) => runtime.browser.health_summary().await,
1325 None => BrowserHealthSummary::default(),
1326 }
1327 }
1328
1329 pub async fn close_browser_sessions_for_owner(&self, owner_session_id: &str) -> usize {
1330 match self.runtime.get() {
1331 Some(runtime) => {
1332 runtime
1333 .close_browser_sessions_for_owner(owner_session_id)
1334 .await
1335 }
1336 None => 0,
1337 }
1338 }
1339
1340 pub async fn close_all_browser_sessions(&self) -> usize {
1341 match self.runtime.get() {
1342 Some(runtime) => runtime.close_all_browser_sessions().await,
1343 None => 0,
1344 }
1345 }
1346
1347 pub async fn register_browser_tools(&self) -> anyhow::Result<()> {
1348 let Some(runtime) = self.runtime.get() else {
1349 anyhow::bail!("runtime not ready");
1350 };
1351 runtime
1352 .browser
1353 .register_tools(&runtime.tools, Some(self.clone()))
1354 .await
1355 }
1356}
1357
1358fn evaluate_browser_status(config: BrowserConfig) -> BrowserStatus {
1359 let mut status = run_doctor(BrowserDoctorOptions {
1360 enabled: config.enabled,
1361 headless_default: config.headless_default,
1362 allow_no_sandbox: config.allow_no_sandbox,
1363 executable_path: config.executable_path.clone(),
1364 user_data_root: config.user_data_root.clone(),
1365 });
1366 status.headless_default = config.headless_default;
1367 status.sidecar = evaluate_sidecar_status(config.sidecar_path.as_deref());
1368 if config.enabled && !status.sidecar.found {
1369 status.blocking_issues.push(BrowserBlockingIssue {
1370 code: "browser_sidecar_not_found".to_string(),
1371 message: "The tandem-browser sidecar binary was not found on this host.".to_string(),
1372 });
1373 status.recommendations.push(
1374 "Install or bundle `tandem-browser`, or set `TANDEM_BROWSER_SIDECAR` / `browser.sidecar_path`."
1375 .to_string(),
1376 );
1377 }
1378 status.runnable = config.enabled
1379 && status.sidecar.found
1380 && status.browser.found
1381 && status.blocking_issues.is_empty();
1382 status
1383}
1384
1385fn evaluate_sidecar_status(explicit: Option<&str>) -> tandem_browser::BrowserSidecarStatus {
1386 let path = detect_sidecar_binary_path(explicit);
1387 let version = path
1388 .as_ref()
1389 .and_then(|candidate| probe_binary_version(candidate).ok());
1390 tandem_browser::BrowserSidecarStatus {
1391 found: path.is_some(),
1392 path: path.map(|row| row.to_string_lossy().to_string()),
1393 version,
1394 }
1395}
1396
1397fn probe_binary_version(path: &Path) -> anyhow::Result<String> {
1398 let output = std::process::Command::new(path)
1399 .arg("--version")
1400 .output()
1401 .with_context(|| format!("failed to query `{}` version", path.display()))?;
1402 if !output.status.success() {
1403 anyhow::bail!(
1404 "version probe failed: {}",
1405 String::from_utf8_lossy(&output.stderr).trim()
1406 );
1407 }
1408 let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
1409 if stdout.is_empty() {
1410 anyhow::bail!("version probe returned empty stdout");
1411 }
1412 Ok(stdout)
1413}
1414
1415pub async fn install_browser_sidecar(
1416 config: &BrowserConfig,
1417) -> anyhow::Result<BrowserSidecarInstallResult> {
1418 let version = env!("CARGO_PKG_VERSION").to_string();
1419 let release = fetch_release_for_version(&version).await?;
1420 let asset_name = browser_release_asset_name()?;
1421 let asset = release
1422 .assets
1423 .iter()
1424 .find(|candidate| candidate.name == asset_name)
1425 .ok_or_else(|| {
1426 anyhow!(
1427 "release_missing_asset: `{}` not found in {}",
1428 asset_name,
1429 release.tag_name
1430 )
1431 })?;
1432 let install_path = sidecar_install_path(config)?;
1433 let parent = install_path
1434 .parent()
1435 .ok_or_else(|| anyhow!("invalid install path `{}`", install_path.display()))?;
1436 fs::create_dir_all(parent)
1437 .await
1438 .with_context(|| format!("failed to create `{}`", parent.display()))?;
1439
1440 let archive_bytes = download_release_asset(asset).await?;
1441 let downloaded_bytes = archive_bytes.len() as u64;
1442 let install_path_for_unpack = install_path.clone();
1443 let asset_name_for_unpack = asset.name.clone();
1444 let unpacked = tokio::task::spawn_blocking(move || {
1445 unpack_sidecar_archive(
1446 &asset_name_for_unpack,
1447 &archive_bytes,
1448 &install_path_for_unpack,
1449 )
1450 })
1451 .await
1452 .context("browser sidecar install task failed")??;
1453
1454 let status = evaluate_browser_status(config.clone());
1455 Ok(BrowserSidecarInstallResult {
1456 version,
1457 asset_name: asset.name.clone(),
1458 installed_path: unpacked.to_string_lossy().to_string(),
1459 downloaded_bytes: asset.size.max(downloaded_bytes),
1460 status,
1461 })
1462}
1463
1464async fn fetch_release_for_version(version: &str) -> anyhow::Result<GitHubRelease> {
1465 let base = std::env::var(RELEASES_URL_ENV)
1466 .unwrap_or_else(|_| format!("https://api.github.com/repos/{RELEASE_REPO}/releases/tags"));
1467 let url = format!("{}/v{}", base.trim_end_matches('/'), version);
1468 let response = reqwest::Client::new()
1469 .get(&url)
1470 .header(reqwest::header::USER_AGENT, BROWSER_INSTALL_USER_AGENT)
1471 .send()
1472 .await
1473 .with_context(|| format!("failed to fetch release metadata from `{url}`"))?;
1474 let status = response.status();
1475 let body = response.text().await.unwrap_or_default();
1476 if !status.is_success() {
1477 anyhow::bail!("release_lookup_failed: {} {}", status, body.trim());
1478 }
1479 serde_json::from_str::<GitHubRelease>(&body).context("invalid release metadata payload")
1480}
1481
1482async fn download_release_asset(asset: &GitHubAsset) -> anyhow::Result<Vec<u8>> {
1483 let response = reqwest::Client::new()
1484 .get(&asset.browser_download_url)
1485 .header(reqwest::header::USER_AGENT, BROWSER_INSTALL_USER_AGENT)
1486 .send()
1487 .await
1488 .with_context(|| format!("failed to download `{}`", asset.browser_download_url))?;
1489 let status = response.status();
1490 if !status.is_success() {
1491 anyhow::bail!(
1492 "asset_download_failed: {} {}",
1493 status,
1494 asset.browser_download_url
1495 );
1496 }
1497 let bytes = response
1498 .bytes()
1499 .await
1500 .context("failed to read asset bytes")?;
1501 Ok(bytes.to_vec())
1502}
1503
1504fn sidecar_install_path(config: &BrowserConfig) -> anyhow::Result<PathBuf> {
1505 if let Some(explicit) = config
1506 .sidecar_path
1507 .as_deref()
1508 .map(str::trim)
1509 .filter(|value| !value.is_empty())
1510 {
1511 return Ok(PathBuf::from(explicit));
1512 }
1513 managed_sidecar_install_path()
1514}
1515
1516fn managed_sidecar_install_path() -> anyhow::Result<PathBuf> {
1517 let root = resolve_shared_paths()
1518 .map(|paths| paths.canonical_root)
1519 .unwrap_or_else(|_| {
1520 dirs::home_dir()
1521 .map(|home| home.join(".tandem"))
1522 .unwrap_or_else(|| PathBuf::from(".tandem"))
1523 });
1524 Ok(root.join("binaries").join(sidecar_binary_name()))
1525}
1526
1527fn browser_release_asset_name() -> anyhow::Result<String> {
1528 let os = if cfg!(target_os = "windows") {
1529 "windows"
1530 } else if cfg!(target_os = "macos") {
1531 "darwin"
1532 } else if cfg!(target_os = "linux") {
1533 "linux"
1534 } else {
1535 anyhow::bail!("unsupported_os: {}", std::env::consts::OS);
1536 };
1537 let arch = if cfg!(target_arch = "x86_64") {
1538 "x64"
1539 } else if cfg!(target_arch = "aarch64") {
1540 "arm64"
1541 } else {
1542 anyhow::bail!("unsupported_arch: {}", std::env::consts::ARCH);
1543 };
1544 let ext = if cfg!(target_os = "windows") || cfg!(target_os = "macos") {
1545 "zip"
1546 } else {
1547 "tar.gz"
1548 };
1549 Ok(format!("tandem-browser-{os}-{arch}.{ext}"))
1550}
1551
1552fn sidecar_binary_name() -> &'static str {
1553 #[cfg(target_os = "windows")]
1554 {
1555 "tandem-browser.exe"
1556 }
1557 #[cfg(not(target_os = "windows"))]
1558 {
1559 "tandem-browser"
1560 }
1561}
1562
1563fn unpack_sidecar_archive(
1564 asset_name: &str,
1565 archive_bytes: &[u8],
1566 install_path: &Path,
1567) -> anyhow::Result<PathBuf> {
1568 if asset_name.ends_with(".zip") {
1569 let cursor = std::io::Cursor::new(archive_bytes);
1570 let mut archive = zip::ZipArchive::new(cursor).context("invalid zip archive")?;
1571 let binary_present = archive
1572 .file_names()
1573 .any(|name| name == sidecar_binary_name());
1574 let mut file = if binary_present {
1575 archive
1576 .by_name(sidecar_binary_name())
1577 .context("browser binary missing from zip archive")?
1578 } else {
1579 archive
1580 .by_index(0)
1581 .context("browser binary missing from zip archive")?
1582 };
1583 let mut output = std::fs::File::create(install_path)
1584 .with_context(|| format!("failed to create `{}`", install_path.display()))?;
1585 std::io::copy(&mut file, &mut output).context("failed to unpack zip asset")?;
1586 } else if asset_name.ends_with(".tar.gz") {
1587 let cursor = std::io::Cursor::new(archive_bytes);
1588 let decoder = GzDecoder::new(cursor);
1589 let mut archive = tar::Archive::new(decoder);
1590 let mut found = false;
1591 for entry in archive.entries().context("invalid tar archive")? {
1592 let mut entry = entry.context("invalid tar entry")?;
1593 let path = entry.path().context("invalid tar entry path")?;
1594 if path
1595 .file_name()
1596 .and_then(|name| name.to_str())
1597 .is_some_and(|name| name == sidecar_binary_name())
1598 {
1599 entry
1600 .unpack(install_path)
1601 .with_context(|| format!("failed to unpack `{}`", install_path.display()))?;
1602 found = true;
1603 break;
1604 }
1605 }
1606 if !found {
1607 anyhow::bail!("browser binary missing from tar archive");
1608 }
1609 } else {
1610 anyhow::bail!("unsupported archive format `{asset_name}`");
1611 }
1612
1613 #[cfg(not(target_os = "windows"))]
1614 {
1615 use std::os::unix::fs::PermissionsExt;
1616
1617 let mut perms = std::fs::metadata(install_path)
1618 .with_context(|| format!("failed to read `{}` metadata", install_path.display()))?
1619 .permissions();
1620 perms.set_mode(0o755);
1621 std::fs::set_permissions(install_path, perms)
1622 .with_context(|| format!("failed to chmod `{}`", install_path.display()))?;
1623 }
1624
1625 Ok(install_path.to_path_buf())
1626}
1627
1628fn parse_tool_context(args: &Value) -> BrowserToolContext {
1629 serde_json::from_value(args.clone()).unwrap_or(BrowserToolContext {
1630 model_session_id: None,
1631 })
1632}
1633
1634fn ok_tool_result(value: Value, metadata: Value) -> anyhow::Result<ToolResult> {
1635 Ok(ToolResult {
1636 output: serde_json::to_string_pretty(&value)?,
1637 metadata,
1638 })
1639}
1640
1641fn error_tool_result(code: &str, message: String, metadata: Option<Value>) -> ToolResult {
1642 let mut meta = metadata.unwrap_or_else(|| json!({}));
1643 if let Some(obj) = meta.as_object_mut() {
1644 obj.insert("ok".to_string(), Value::Bool(false));
1645 obj.insert("code".to_string(), Value::String(code.to_string()));
1646 obj.insert("message".to_string(), Value::String(message.clone()));
1647 }
1648 ToolResult {
1649 output: message,
1650 metadata: meta,
1651 }
1652}
1653
1654fn split_error_code(message: &str) -> (&str, &str) {
1655 let Some((code, detail)) = message.split_once(':') else {
1656 return ("browser_error", message);
1657 };
1658 let code = code.trim();
1659 if code.is_empty()
1660 || !code
1661 .chars()
1662 .all(|ch| ch.is_ascii_lowercase() || ch == '_' || ch.is_ascii_digit())
1663 {
1664 return ("browser_error", message);
1665 }
1666 (code, detail.trim())
1667}
1668
1669fn smoke_excerpt(content: &str, max_chars: usize) -> String {
1670 let mut excerpt = String::new();
1671 for ch in content.chars().take(max_chars) {
1672 excerpt.push(ch);
1673 }
1674 if content.chars().count() > max_chars {
1675 excerpt.push_str("...");
1676 }
1677 excerpt
1678}
1679
1680fn browser_not_runnable_result(status: &BrowserStatus) -> anyhow::Result<ToolResult> {
1681 ok_tool_result(
1682 serde_json::to_value(status)?,
1683 json!({
1684 "ok": false,
1685 "code": "browser_not_runnable",
1686 "runnable": status.runnable,
1687 "enabled": status.enabled,
1688 }),
1689 )
1690}
1691
1692fn normalize_allowed_hosts(hosts: Vec<String>) -> Vec<String> {
1693 let mut out = Vec::new();
1694 for host in hosts {
1695 let normalized = host.trim().trim_start_matches('.').to_ascii_lowercase();
1696 if normalized.is_empty() {
1697 continue;
1698 }
1699 if !out.iter().any(|existing| existing == &normalized) {
1700 out.push(normalized);
1701 }
1702 }
1703 out
1704}
1705
1706fn browser_url_host(url: &str) -> anyhow::Result<String> {
1707 let parsed =
1708 reqwest::Url::parse(url).with_context(|| format!("invalid browser url `{}`", url))?;
1709 let host = parsed
1710 .host_str()
1711 .ok_or_else(|| anyhow!("url `{}` has no host", url))?;
1712 Ok(host.to_ascii_lowercase())
1713}
1714
1715fn ensure_allowed_browser_url(url: &str, allow_hosts: &[String]) -> anyhow::Result<()> {
1716 let parsed =
1717 reqwest::Url::parse(url).with_context(|| format!("invalid browser url `{}`", url))?;
1718 match parsed.scheme() {
1719 "http" | "https" => {}
1720 other => anyhow::bail!("unsupported_url_scheme: `{}` is not allowed", other),
1721 }
1722 if allow_hosts.is_empty() {
1723 return Ok(());
1724 }
1725 let host = parsed
1726 .host_str()
1727 .ok_or_else(|| anyhow!("url `{}` has no host", url))?
1728 .to_ascii_lowercase();
1729 let allowed = allow_hosts
1730 .iter()
1731 .any(|candidate| host == *candidate || host.ends_with(&format!(".{candidate}")));
1732 if !allowed {
1733 anyhow::bail!("host `{}` is not in the browser allowlist", host);
1734 }
1735 Ok(())
1736}
1737
1738fn is_local_or_private_host(host: &str) -> bool {
1739 if host.eq_ignore_ascii_case("localhost") {
1740 return true;
1741 }
1742 let Ok(ip) = host.parse::<IpAddr>() else {
1743 return false;
1744 };
1745 match ip {
1746 IpAddr::V4(ip) => {
1747 ip.is_loopback()
1748 || ip.is_private()
1749 || ip.is_link_local()
1750 || ip.octets()[0] == 169 && ip.octets()[1] == 254
1751 }
1752 IpAddr::V6(ip) => {
1753 ip == Ipv6Addr::LOCALHOST || ip.is_unique_local() || ip.is_unicast_link_local()
1754 }
1755 }
1756}
1757
1758fn resolve_text_input(text: Option<String>, secret_ref: Option<String>) -> anyhow::Result<String> {
1759 if let Some(secret_ref) = secret_ref
1760 .map(|v| v.trim().to_string())
1761 .filter(|v| !v.is_empty())
1762 {
1763 let value = std::env::var(&secret_ref).with_context(|| {
1764 format!("secret_ref `{}` is not set in the environment", secret_ref)
1765 })?;
1766 if value.trim().is_empty() {
1767 anyhow::bail!("secret_ref `{}` resolved to an empty value", secret_ref);
1768 }
1769 return Ok(value);
1770 }
1771 let text = text.unwrap_or_default();
1772 if text.is_empty() {
1773 anyhow::bail!("browser_type requires either `text` or `secret_ref`");
1774 }
1775 Ok(text)
1776}
1777
1778fn extension_for_extract_format(format: &str) -> &'static str {
1779 match format {
1780 "html" => "html",
1781 "markdown" => "md",
1782 _ => "txt",
1783 }
1784}
1785
1786fn viewport_schema() -> Value {
1787 json!({
1788 "type": "object",
1789 "properties": {
1790 "width": { "type": "integer", "minimum": 1, "maximum": 10000 },
1791 "height": { "type": "integer", "minimum": 1, "maximum": 10000 }
1792 }
1793 })
1794}
1795
1796fn wait_condition_schema() -> Value {
1797 json!({
1798 "type": "object",
1799 "properties": {
1800 "kind": {
1801 "type": "string",
1802 "enum": ["selector", "text", "url", "network_idle", "navigation"]
1803 },
1804 "value": { "type": "string" }
1805 },
1806 "required": ["kind"]
1807 })
1808}
1809
1810fn tool_schema(kind: BrowserToolKind) -> ToolSchema {
1811 match kind {
1812 BrowserToolKind::Status => ToolSchema {
1813 name: "browser_status".to_string(),
1814 description:
1815 "Check browser automation readiness and install guidance. Call this first when browser tools may be unavailable."
1816 .to_string(),
1817 input_schema: json!({ "type": "object", "properties": {} }),
1818 },
1819 BrowserToolKind::Open => ToolSchema {
1820 name: "browser_open".to_string(),
1821 description:
1822 "Open a URL in a browser session. Only http/https are allowed. Omit profile_id for an ephemeral session."
1823 .to_string(),
1824 input_schema: json!({
1825 "type": "object",
1826 "properties": {
1827 "url": { "type": "string" },
1828 "profile_id": { "type": "string" },
1829 "headless": { "type": "boolean" },
1830 "viewport": viewport_schema(),
1831 "wait_until": { "type": "string", "enum": ["navigation", "network_idle"] }
1832 },
1833 "required": ["url"]
1834 }),
1835 },
1836 BrowserToolKind::Navigate => ToolSchema {
1837 name: "browser_navigate".to_string(),
1838 description: "Navigate an existing browser session to a new URL.".to_string(),
1839 input_schema: json!({
1840 "type": "object",
1841 "properties": {
1842 "session_id": { "type": "string" },
1843 "url": { "type": "string" },
1844 "wait_until": { "type": "string", "enum": ["navigation", "network_idle"] }
1845 },
1846 "required": ["session_id", "url"]
1847 }),
1848 },
1849 BrowserToolKind::Snapshot => ToolSchema {
1850 name: "browser_snapshot".to_string(),
1851 description:
1852 "Capture a bounded page summary with stable element_id values. Call this before click/type on a new page or after navigation."
1853 .to_string(),
1854 input_schema: json!({
1855 "type": "object",
1856 "properties": {
1857 "session_id": { "type": "string" },
1858 "max_elements": { "type": "integer", "minimum": 1, "maximum": 200 },
1859 "include_screenshot": { "type": "boolean" }
1860 },
1861 "required": ["session_id"]
1862 }),
1863 },
1864 BrowserToolKind::Click => ToolSchema {
1865 name: "browser_click".to_string(),
1866 description:
1867 "Click a visible page element by element_id when possible. Use wait_for to make navigation and selector waits race-free."
1868 .to_string(),
1869 input_schema: json!({
1870 "type": "object",
1871 "properties": {
1872 "session_id": { "type": "string" },
1873 "element_id": { "type": "string" },
1874 "selector": { "type": "string" },
1875 "wait_for": wait_condition_schema(),
1876 "timeout_ms": { "type": "integer", "minimum": 250, "maximum": 120000 }
1877 },
1878 "required": ["session_id"]
1879 }),
1880 },
1881 BrowserToolKind::Type => ToolSchema {
1882 name: "browser_type".to_string(),
1883 description:
1884 "Type text into an element. Prefer secret_ref over text for credentials; secret_ref resolves from the host environment and is redacted from logs."
1885 .to_string(),
1886 input_schema: json!({
1887 "type": "object",
1888 "properties": {
1889 "session_id": { "type": "string" },
1890 "element_id": { "type": "string" },
1891 "selector": { "type": "string" },
1892 "text": { "type": "string" },
1893 "secret_ref": { "type": "string" },
1894 "replace": { "type": "boolean" },
1895 "submit": { "type": "boolean" },
1896 "timeout_ms": { "type": "integer", "minimum": 250, "maximum": 120000 }
1897 },
1898 "required": ["session_id"]
1899 }),
1900 },
1901 BrowserToolKind::Press => ToolSchema {
1902 name: "browser_press".to_string(),
1903 description: "Dispatch a key press in the active page context.".to_string(),
1904 input_schema: json!({
1905 "type": "object",
1906 "properties": {
1907 "session_id": { "type": "string" },
1908 "key": { "type": "string" },
1909 "wait_for": wait_condition_schema(),
1910 "timeout_ms": { "type": "integer", "minimum": 250, "maximum": 120000 }
1911 },
1912 "required": ["session_id", "key"]
1913 }),
1914 },
1915 BrowserToolKind::Wait => ToolSchema {
1916 name: "browser_wait".to_string(),
1917 description: "Wait for a selector, text, URL fragment, navigation, or network idle.".to_string(),
1918 input_schema: json!({
1919 "type": "object",
1920 "properties": {
1921 "session_id": { "type": "string" },
1922 "condition": wait_condition_schema(),
1923 "timeout_ms": { "type": "integer", "minimum": 250, "maximum": 120000 }
1924 },
1925 "required": ["session_id", "condition"]
1926 }),
1927 },
1928 BrowserToolKind::Extract => ToolSchema {
1929 name: "browser_extract".to_string(),
1930 description:
1931 "Extract page content as visible_text, markdown, or html. Prefer this over screenshots when you need text."
1932 .to_string(),
1933 input_schema: json!({
1934 "type": "object",
1935 "properties": {
1936 "session_id": { "type": "string" },
1937 "format": { "type": "string", "enum": ["visible_text", "markdown", "html"] },
1938 "max_bytes": { "type": "integer", "minimum": 1024, "maximum": 2000000 }
1939 },
1940 "required": ["session_id", "format"]
1941 }),
1942 },
1943 BrowserToolKind::Screenshot => ToolSchema {
1944 name: "browser_screenshot".to_string(),
1945 description: "Capture a screenshot and store it as a browser artifact.".to_string(),
1946 input_schema: json!({
1947 "type": "object",
1948 "properties": {
1949 "session_id": { "type": "string" },
1950 "full_page": { "type": "boolean" },
1951 "label": { "type": "string" }
1952 },
1953 "required": ["session_id"]
1954 }),
1955 },
1956 BrowserToolKind::Close => ToolSchema {
1957 name: "browser_close".to_string(),
1958 description: "Close a browser session and release its resources.".to_string(),
1959 input_schema: json!({
1960 "type": "object",
1961 "properties": {
1962 "session_id": { "type": "string" }
1963 },
1964 "required": ["session_id"]
1965 }),
1966 },
1967 }
1968}
1969
1970#[cfg(test)]
1971mod tests {
1972 use super::*;
1973 use tandem_core::BrowserConfig;
1974 use tandem_tools::ToolRegistry;
1975
1976 #[test]
1977 fn local_and_private_hosts_are_detected() {
1978 assert!(is_local_or_private_host("localhost"));
1979 assert!(is_local_or_private_host("127.0.0.1"));
1980 assert!(is_local_or_private_host("10.1.2.3"));
1981 assert!(is_local_or_private_host("192.168.0.10"));
1982 assert!(!is_local_or_private_host("example.com"));
1983 assert!(!is_local_or_private_host("8.8.8.8"));
1984 }
1985
1986 #[test]
1987 fn allow_host_check_accepts_subdomains() {
1988 let allow_hosts = vec!["example.com".to_string()];
1989 ensure_allowed_browser_url("https://example.com/path", &allow_hosts).expect("root host");
1990 ensure_allowed_browser_url("https://app.example.com/path", &allow_hosts)
1991 .expect("subdomain host");
1992 let err =
1993 ensure_allowed_browser_url("https://example.org/path", &allow_hosts).expect_err("deny");
1994 assert!(err.to_string().contains("allowlist"));
1995 }
1996
1997 #[test]
1998 fn browser_release_asset_name_matches_platform() {
1999 let asset = browser_release_asset_name().expect("asset name");
2000 assert!(asset.starts_with("tandem-browser-"));
2001 if cfg!(target_os = "windows") {
2002 assert!(asset.ends_with(".zip"));
2003 assert!(asset.contains("-windows-"));
2004 } else if cfg!(target_os = "macos") {
2005 assert!(asset.ends_with(".zip"));
2006 assert!(asset.contains("-darwin-"));
2007 } else if cfg!(target_os = "linux") {
2008 assert!(asset.ends_with(".tar.gz"));
2009 assert!(asset.contains("-linux-"));
2010 }
2011 }
2012
2013 #[test]
2014 fn managed_sidecar_path_uses_shared_binaries_dir() {
2015 let temp_root =
2016 std::env::temp_dir().join(format!("tandem-browser-test-{}", Uuid::new_v4()));
2017 std::env::set_var("TANDEM_HOME", &temp_root);
2018
2019 let path = managed_sidecar_install_path().expect("managed path");
2020
2021 assert!(path.starts_with(temp_root.join("binaries")));
2022 assert_eq!(
2023 path.file_name().and_then(|value| value.to_str()),
2024 Some(sidecar_binary_name())
2025 );
2026
2027 std::env::remove_var("TANDEM_HOME");
2028 }
2029
2030 #[tokio::test]
2031 async fn register_tools_keeps_browser_status_available_when_disabled() {
2032 let tools = ToolRegistry::new();
2033 let browser = BrowserSubsystem::new(BrowserConfig::default());
2034
2035 browser
2036 .register_tools(&tools, None)
2037 .await
2038 .expect("register browser tools");
2039
2040 let names = tools
2041 .list()
2042 .await
2043 .into_iter()
2044 .map(|schema| schema.name)
2045 .collect::<Vec<_>>();
2046 assert!(names.iter().any(|name| name == "browser_status"));
2047 assert!(!names.iter().any(|name| name == "browser_open"));
2048 assert!(!browser.health_summary().await.tools_registered);
2049 }
2050
2051 #[tokio::test]
2052 async fn close_sessions_for_owner_removes_matching_sessions() {
2053 let browser = BrowserSubsystem::new(BrowserConfig::default());
2054 browser
2055 .insert_session(
2056 "session-1".to_string(),
2057 Some("owner-1".to_string()),
2058 "https://example.com".to_string(),
2059 )
2060 .await;
2061 browser
2062 .insert_session(
2063 "session-2".to_string(),
2064 Some("owner-2".to_string()),
2065 "https://example.org".to_string(),
2066 )
2067 .await;
2068
2069 let closed = browser.close_sessions_for_owner("owner-1").await;
2070
2071 assert_eq!(closed, 1);
2072 assert!(browser.session("session-1").await.is_none());
2073 assert!(browser.session("session-2").await.is_some());
2074 }
2075}