spider_browser/
spider_browser.rs

1//! SpiderBrowser — main entry point for spider-browser.
2
3use crate::errors::{Result, SpiderError};
4use crate::events::SpiderEventEmitter;
5use crate::page::SpiderPage;
6use crate::protocol::protocol_adapter::{ProtocolAdapter, ProtocolAdapterOptions};
7use crate::protocol::transport::{Transport, TransportOptions};
8use arc_swap::ArcSwap;
9use std::sync::Arc;
10use tokio::sync::mpsc;
11use tracing::info;
12
13#[cfg(feature = "ai")]
14use crate::ai::llm_provider::{create_provider, LLMConfig, LLMProvider};
15
16/// Options for creating a SpiderBrowser instance.
17#[derive(Clone, Debug)]
18pub struct SpiderBrowserOptions {
19    /// Spider API key (required).
20    pub api_key: String,
21    /// WebSocket server URL (default: wss://browser.spider.cloud).
22    pub server_url: Option<String>,
23    /// Browser to use (default: "auto").
24    pub browser: Option<String>,
25    /// Target URL hint for server browser+proxy selection.
26    pub url: Option<String>,
27    /// Captcha handling: "off", "detect", or "solve" (default: "solve").
28    pub captcha: Option<String>,
29    /// Enable smart retry with browser switching (default: true).
30    pub smart_retry: Option<bool>,
31    /// Max retry attempts across all browsers (default: 12).
32    pub max_retries: Option<u32>,
33    /// Stealth level (1-3). 0 = auto-escalate on failure.
34    pub stealth: Option<u32>,
35    /// Maximum stealth level to auto-escalate to (1-3, default: 3).
36    pub max_stealth_levels: Option<u32>,
37    /// WebSocket connect timeout in ms (default: 30000).
38    pub connect_timeout_ms: Option<u64>,
39    /// CDP/BiDi command timeout in ms (default: 30000).
40    pub command_timeout_ms: Option<u64>,
41    /// Timeout for retry attempts (default: 15000).
42    pub retry_timeout_ms: Option<u64>,
43    /// Mark this session as a hedge (parallel attempt).
44    pub hedge: Option<bool>,
45    /// Enable screencast recording (default: false).
46    pub record: Option<bool>,
47    /// Browser mode: "scraping" or "cua".
48    pub mode: Option<String>,
49    /// Country code for geo-located proxy (e.g. "US", "GB", "DE").
50    pub country: Option<String>,
51    /// Custom proxy URL (e.g. "http://user:pass@proxy:8080"). Overrides server hint-based proxy selection.
52    pub proxy_url: Option<String>,
53    /// LLM configuration for AI methods.
54    #[cfg(feature = "ai")]
55    pub llm: Option<LLMConfig>,
56}
57
58impl SpiderBrowserOptions {
59    pub fn new(api_key: impl Into<String>) -> Self {
60        Self {
61            api_key: api_key.into(),
62            server_url: None,
63            browser: None,
64            url: None,
65            captcha: None,
66            smart_retry: None,
67            max_retries: None,
68            stealth: None,
69            max_stealth_levels: None,
70            connect_timeout_ms: None,
71            command_timeout_ms: None,
72            retry_timeout_ms: None,
73            hedge: None,
74            record: None,
75            mode: None,
76            country: None,
77            proxy_url: None,
78            #[cfg(feature = "ai")]
79            llm: None,
80        }
81    }
82}
83
84struct ResolvedOptions {
85    api_key: String,
86    server_url: String,
87    browser: String,
88    url: Option<String>,
89    captcha: String,
90    smart_retry: bool,
91    max_retries: u32,
92    stealth: u32,
93    max_stealth_levels: u32,
94    connect_timeout_ms: u64,
95    command_timeout_ms: u64,
96    retry_timeout_ms: u64,
97    hedge: Option<bool>,
98    record: Option<bool>,
99    mode: Option<String>,
100    country: Option<String>,
101    proxy_url: Option<String>,
102}
103
104/// SpiderBrowser — connects to Spider's pre-warmed browser fleet.
105///
106/// Provides deterministic page control (via SpiderPage) and
107/// AI-powered automation (act, observe, extract, agent).
108pub struct SpiderBrowser {
109    opts: ResolvedOptions,
110    transport: Option<Arc<Transport>>,
111    adapter: Option<Arc<ProtocolAdapter>>,
112    page: Option<Arc<SpiderPage>>,
113    emitter: SpiderEventEmitter,
114    current_url: ArcSwap<Option<String>>,
115    #[cfg(feature = "ai")]
116    llm_provider: Option<Arc<dyn LLMProvider>>,
117    msg_send_tx: Option<mpsc::UnboundedSender<String>>,
118}
119
120impl SpiderBrowser {
121    pub fn new(options: SpiderBrowserOptions) -> Self {
122        let resolved = ResolvedOptions {
123            api_key: options.api_key.clone(),
124            server_url: options.server_url.unwrap_or_else(|| "wss://browser.spider.cloud".into()),
125            browser: options.browser.unwrap_or_else(|| "auto".into()),
126            url: options.url.clone(),
127            captcha: options.captcha.unwrap_or_else(|| "solve".into()),
128            smart_retry: options.smart_retry.unwrap_or(true),
129            max_retries: options.max_retries.unwrap_or(12),
130            stealth: options.stealth.unwrap_or(0),
131            max_stealth_levels: options.max_stealth_levels.unwrap_or(3),
132            connect_timeout_ms: options.connect_timeout_ms.unwrap_or(30_000),
133            command_timeout_ms: options.command_timeout_ms.unwrap_or(30_000),
134            retry_timeout_ms: options.retry_timeout_ms.unwrap_or(15_000),
135            hedge: options.hedge,
136            record: options.record,
137            mode: options.mode.clone(),
138            country: options.country.clone(),
139            proxy_url: options.proxy_url.clone(),
140        };
141
142        #[cfg(feature = "ai")]
143        let llm_provider: Option<Arc<dyn LLMProvider>> =
144            options.llm.map(|config| Arc::from(create_provider(config)));
145
146        Self {
147            opts: resolved,
148            transport: None,
149            adapter: None,
150            page: None,
151            emitter: SpiderEventEmitter::new(),
152            current_url: ArcSwap::from_pointee(options.url),
153            #[cfg(feature = "ai")]
154            llm_provider,
155            msg_send_tx: None,
156        }
157    }
158
159    /// The active page instance for deterministic browser control.
160    pub fn page(&self) -> &SpiderPage {
161        self.page
162            .as_ref()
163            .expect("SpiderBrowser not initialized. Call init() first.")
164    }
165
166    /// Current browser type.
167    pub fn browser(&self) -> String {
168        self.transport
169            .as_ref()
170            .map(|t| t.browser())
171            .unwrap_or_else(|| self.opts.browser.clone())
172    }
173
174    /// Whether the WebSocket is connected.
175    pub fn connected(&self) -> bool {
176        self.transport
177            .as_ref()
178            .map(|t| t.is_connected())
179            .unwrap_or(false)
180    }
181
182    /// Active stealth level.
183    pub fn stealth_level(&self) -> u32 {
184        self.transport
185            .as_ref()
186            .map(|t| t.get_stealth_level())
187            .unwrap_or(self.opts.stealth)
188    }
189
190    /// Credits remaining from last upgrade response.
191    pub fn credits(&self) -> Option<f64> {
192        self.transport.as_ref().and_then(|t| t.upgrade_credits())
193    }
194
195    /// Credits consumed during this session.
196    pub fn session_credits_used(&self) -> Option<f64> {
197        self.transport
198            .as_ref()
199            .and_then(|t| t.session_credits_used())
200    }
201
202    /// Subscribe to events.
203    pub fn on(&self, event: &str, handler: crate::events::EventHandler) {
204        self.emitter.on(event, handler);
205    }
206
207    /// Connect to the browser fleet and initialize the protocol.
208    pub async fn init(&mut self) -> Result<()> {
209        let transport_opts = TransportOptions {
210            api_key: self.opts.api_key.clone(),
211            server_url: self.opts.server_url.clone(),
212            browser: self.opts.browser.clone(),
213            url: self.opts.url.clone(),
214            captcha: Some(self.opts.captcha.clone()),
215            stealth_level: self.opts.stealth,
216            connect_timeout_ms: self.opts.connect_timeout_ms,
217            command_timeout_ms: self.opts.command_timeout_ms,
218            hedge: self.opts.hedge.unwrap_or(false),
219            record: self.opts.record.unwrap_or(false),
220            mode: self.opts.mode.clone(),
221            country: self.opts.country.clone(),
222            proxy_url: self.opts.proxy_url.clone(),
223        };
224
225        let transport = Transport::new(transport_opts, self.emitter.clone());
226        transport.connect(3).await?;
227
228        let active_browser = transport.browser();
229
230        // Take the message receiver and set up routing
231        let mut msg_rx = transport
232            .take_message_rx()
233            .await
234            .ok_or_else(|| SpiderError::Protocol("Message receiver already taken".into()))?;
235
236        let adapter_opts = if self.opts.command_timeout_ms != 30_000 {
237            Some(ProtocolAdapterOptions {
238                command_timeout_ms: Some(self.opts.command_timeout_ms),
239            })
240        } else {
241            None
242        };
243
244        // Create a relay channel that forwards outgoing messages to the transport's WS
245        let (proto_tx, mut proto_rx) = mpsc::unbounded_channel::<String>();
246        let transport_for_relay = Arc::clone(&transport);
247        tokio::spawn(async move {
248            while let Some(data) = proto_rx.recv().await {
249                let _ = transport_for_relay.send(data);
250            }
251        });
252
253        let mut adapter = ProtocolAdapter::new(
254            proto_tx.clone(),
255            self.emitter.clone(),
256            &active_browser,
257            adapter_opts,
258        );
259        adapter.init().await?;
260
261        // Wrap adapter in Arc for sharing between page and AI methods
262        let adapter = Arc::new(adapter);
263        #[cfg(feature = "ai")]
264        let page = SpiderPage::from_arc_with(
265            Arc::clone(&adapter),
266            self.emitter.clone(),
267            self.llm_provider.clone(),
268        );
269        #[cfg(not(feature = "ai"))]
270        let page = SpiderPage::from_arc(Arc::clone(&adapter));
271        let page = Arc::new(page);
272
273        // Spawn message routing task: transport -> adapter.route_message()
274        let page_for_routing = Arc::clone(&page);
275        tokio::spawn(async move {
276            while let Some(data) = msg_rx.recv().await {
277                page_for_routing.route_message(&data);
278            }
279        });
280
281        self.transport = Some(transport);
282        self.adapter = Some(adapter);
283        self.page = Some(page);
284        self.msg_send_tx = Some(proto_tx);
285
286        info!("SpiderBrowser initialized (browser={})", active_browser);
287        Ok(())
288    }
289
290    /// Navigate to a URL with smart retry.
291    pub async fn goto(&self, url: &str) -> Result<()> {
292        self.current_url.store(Arc::new(Some(url.to_string())));
293        self.page().goto(url).await
294    }
295
296    /// Close the connection and clean up resources.
297    pub fn close(&mut self) {
298        if let Some(ref page) = self.page {
299            page.destroy();
300        }
301        if let Some(ref transport) = self.transport {
302            transport.close();
303        }
304        self.emitter.remove_all_listeners();
305        self.page = None;
306        self.adapter = None;
307        self.transport = None;
308        info!("SpiderBrowser closed");
309    }
310
311    // ------------------------------------------------------------------
312    // AI Methods (require "ai" feature + LLM config)
313    // ------------------------------------------------------------------
314
315    /// Execute a single action from natural language.
316    #[cfg(feature = "ai")]
317    pub async fn act(&self, instruction: &str) -> Result<()> {
318        let llm = self.require_llm()?;
319        let adapter = self.require_adapter()?;
320        crate::ai::act::act(adapter, llm.as_ref(), instruction).await
321    }
322
323    /// Discover interactive elements on the page.
324    #[cfg(feature = "ai")]
325    pub async fn observe(
326        &self,
327        instruction: Option<&str>,
328    ) -> Result<Vec<crate::ai::observe::ObserveResult>> {
329        let adapter = self.require_adapter()?;
330        let llm_ref: Option<&dyn LLMProvider> = self.llm_provider.as_ref().map(|b| b.as_ref());
331        crate::ai::observe::observe(adapter, instruction, llm_ref).await
332    }
333
334    /// Extract structured data from the page.
335    #[cfg(feature = "ai")]
336    pub async fn extract<T: serde::de::DeserializeOwned + Send>(
337        &self,
338        instruction: &str,
339    ) -> Result<T> {
340        let llm = self.require_llm()?;
341        let adapter = self.require_adapter()?;
342        crate::ai::extract::extract(adapter, llm.as_ref(), instruction, None).await
343    }
344
345    /// Create an autonomous agent.
346    #[cfg(feature = "ai")]
347    pub fn agent(
348        &self,
349        options: Option<crate::ai::agent::AgentOptions>,
350    ) -> crate::ai::agent::Agent<'_> {
351        let llm = self
352            .llm_provider
353            .as_ref()
354            .expect("LLM not configured. Pass llm option for AI methods.");
355        let adapter = self
356            .adapter
357            .as_ref()
358            .expect("SpiderBrowser not initialized. Call init() first.");
359        crate::ai::agent::Agent::new(adapter, llm.as_ref(), &self.emitter, options)
360    }
361
362    #[cfg(feature = "ai")]
363    fn require_llm(&self) -> Result<&Arc<dyn LLMProvider>> {
364        self.llm_provider.as_ref().ok_or_else(|| {
365            SpiderError::Llm(
366                "LLM not configured. Pass llm option to SpiderBrowser for AI methods.".into(),
367            )
368        })
369    }
370
371    fn require_adapter(&self) -> Result<&ProtocolAdapter> {
372        self.adapter
373            .as_ref()
374            .map(|a| a.as_ref())
375            .ok_or_else(|| {
376                SpiderError::Protocol(
377                    "SpiderBrowser not initialized. Call init() first.".into(),
378                )
379            })
380    }
381}
spider_browser/spider_browser.rs

spider_browser/
spider_browser.rs
