Skip to main content

beecrawl_sdk/
lib.rs

1use reqwest::{Client as HttpClient, Method, StatusCode};
2use serde_json::{json, Map, Value};
3use std::collections::HashMap;
4use std::time::{Duration, Instant};
5use thiserror::Error;
6
7#[derive(Debug, Error)]
8pub enum BeeCrawlError {
9    #[error("BeeCrawl request failed: {0}")]
10    Transport(#[from] reqwest::Error),
11    #[error("BeeCrawl returned invalid JSON: {0}")]
12    InvalidJson(#[from] serde_json::Error),
13    #[error("BeeCrawl request failed with status {status}: {message}")]
14    Api {
15        status: StatusCode,
16        message: String,
17        detail: Value,
18    },
19    #[error("timed out waiting for job {job_id}")]
20    PollTimeout { job_id: String },
21    #[error("invalid base URL: {0}")]
22    InvalidBaseUrl(String),
23}
24
25pub type Result<T> = std::result::Result<T, BeeCrawlError>;
26
27#[derive(Clone, Debug)]
28pub struct BeeCrawlClient {
29    http: HttpClient,
30    base_url: String,
31    api_key: Option<String>,
32}
33
34impl BeeCrawlClient {
35    pub fn new(base_url: impl Into<String>) -> Self {
36        Self::builder(base_url)
37            .build()
38            .expect("default client configuration is valid")
39    }
40
41    pub fn builder(base_url: impl Into<String>) -> BeeCrawlClientBuilder {
42        BeeCrawlClientBuilder {
43            base_url: base_url.into(),
44            api_key: None,
45            timeout: Duration::from_secs(60),
46        }
47    }
48
49    pub async fn scrape(&self, url: &str, options: Value) -> Result<Value> {
50        self.post("/scrape", merge_url(options, url)).await
51    }
52
53    pub async fn map(&self, url: &str, options: Value) -> Result<Value> {
54        self.post("/map", merge_url(options, url)).await
55    }
56
57    pub async fn search(&self, query: &str, options: Value) -> Result<Value> {
58        self.post("/search", merge_field(options, "query", query))
59            .await
60    }
61
62    pub async fn extract(
63        &self,
64        url: &str,
65        schema: HashMap<String, String>,
66        options: Value,
67    ) -> Result<Value> {
68        let mut payload = merge_url(options, url);
69        payload["schema"] = json!(schema);
70        self.post("/extract", payload).await
71    }
72
73    pub async fn crawl(&self, url: &str, options: Value) -> Result<Value> {
74        self.post("/crawl", merge_url(options, url)).await
75    }
76
77    pub async fn batch_scrape(&self, urls: Vec<String>, options: Value) -> Result<Value> {
78        self.post("/batch/scrape", merge_field(options, "urls", urls))
79            .await
80    }
81
82    pub async fn crawl_status(&self, job_id: &str, offset: usize, limit: usize) -> Result<Value> {
83        self.get(
84            &format!("/crawl/{job_id}"),
85            &[("offset", offset.to_string()), ("limit", limit.to_string())],
86        )
87        .await
88    }
89
90    pub async fn batch_scrape_status(
91        &self,
92        job_id: &str,
93        offset: usize,
94        limit: usize,
95    ) -> Result<Value> {
96        self.get(
97            &format!("/batch/scrape/{job_id}"),
98            &[("offset", offset.to_string()), ("limit", limit.to_string())],
99        )
100        .await
101    }
102
103    pub async fn cancel_crawl(&self, job_id: &str) -> Result<Value> {
104        self.request(Method::DELETE, &format!("/crawl/{job_id}"), None, &[])
105            .await
106    }
107
108    pub async fn cancel_batch_scrape(&self, job_id: &str) -> Result<Value> {
109        self.request(
110            Method::DELETE,
111            &format!("/batch/scrape/{job_id}"),
112            None,
113            &[],
114        )
115        .await
116    }
117
118    pub async fn v2_scrape(&self, url: &str, options: Value) -> Result<Value> {
119        self.post("/v2/scrape", merge_url(options, url)).await
120    }
121    pub async fn v2_map(&self, url: &str, options: Value) -> Result<Value> {
122        self.post("/v2/map", merge_url(options, url)).await
123    }
124    pub async fn v2_search(&self, query: &str, options: Value) -> Result<Value> {
125        self.post("/v2/search", merge_field(options, "query", query))
126            .await
127    }
128    pub async fn v2_extract(&self, urls: Vec<String>, options: Value) -> Result<Value> {
129        self.post("/v2/extract", merge_field(options, "urls", urls))
130            .await
131    }
132    pub async fn v2_parse_base64(
133        &self,
134        data: &str,
135        filename: &str,
136        options: Value,
137    ) -> Result<Value> {
138        self.post(
139            "/v2/parse/base64",
140            merge_field(merge_field(options, "base64", data), "filename", filename),
141        )
142        .await
143    }
144    pub async fn v2_parse_reference(&self, upload_ref: &str, options: Value) -> Result<Value> {
145        self.post(
146            "/v2/parse/reference",
147            merge_field(options, "uploadRef", upload_ref),
148        )
149        .await
150    }
151    pub async fn create_parse_upload(&self, filename: &str) -> Result<Value> {
152        self.post("/v2/parse/upload-url", json!({"filename": filename}))
153            .await
154    }
155    pub async fn upload_parse_document(&self, upload_ref: &str, data: Vec<u8>) -> Result<Value> {
156        self.request_bytes(Method::PUT, &format!("/v2/parse/upload/{upload_ref}"), data)
157            .await
158    }
159    pub async fn v2_parse(&self, filename: &str, data: Vec<u8>, options: Value) -> Result<Value> {
160        let form = reqwest::multipart::Form::new()
161            .text("options", options.to_string())
162            .part(
163                "file",
164                reqwest::multipart::Part::bytes(data).file_name(filename.to_string()),
165            );
166        self.request_form("/v2/parse", form).await
167    }
168    pub async fn v2_crawl(&self, url: &str, options: Value) -> Result<Value> {
169        self.post("/v2/crawl", merge_url(options, url)).await
170    }
171    pub async fn v2_batch_scrape(&self, urls: Vec<String>, options: Value) -> Result<Value> {
172        self.post("/v2/batch/scrape", merge_field(options, "urls", urls))
173            .await
174    }
175    pub async fn v2_job_status(
176        &self,
177        kind: &str,
178        job_id: &str,
179        offset: usize,
180        limit: usize,
181    ) -> Result<Value> {
182        self.get(
183            &format!("/v2/{kind}/{job_id}"),
184            &[("offset", offset.to_string()), ("limit", limit.to_string())],
185        )
186        .await
187    }
188    pub async fn v2_job_errors(&self, kind: &str, job_id: &str) -> Result<Value> {
189        self.get(&format!("/v2/{kind}/{job_id}/errors"), &[]).await
190    }
191    pub async fn cancel_v2_job(&self, kind: &str, job_id: &str) -> Result<Value> {
192        self.request(Method::DELETE, &format!("/v2/{kind}/{job_id}"), None, &[])
193            .await
194    }
195    pub async fn active_crawls(&self) -> Result<Value> {
196        self.get("/v2/crawl/active", &[]).await
197    }
198    pub async fn create_browser_session(&self, options: Value) -> Result<Value> {
199        self.post("/v2/browser", options).await
200    }
201    pub async fn browser_sessions(&self) -> Result<Value> {
202        self.get("/v2/browser", &[]).await
203    }
204    pub async fn execute_browser(
205        &self,
206        session_id: &str,
207        code: &str,
208        options: Value,
209    ) -> Result<Value> {
210        self.post(
211            &format!("/v2/browser/{session_id}/execute"),
212            merge_field(options, "code", code),
213        )
214        .await
215    }
216    pub async fn browser_replay(&self, session_id: &str, page_id: Option<&str>) -> Result<Value> {
217        self.get(
218            &format!(
219                "/v2/browser/{session_id}/replay{}",
220                page_id.map(|id| format!("/{id}")).unwrap_or_default()
221            ),
222            &[],
223        )
224        .await
225    }
226    pub async fn delete_browser_session(&self, session_id: &str) -> Result<Value> {
227        self.request(
228            Method::DELETE,
229            &format!("/v2/browser/{session_id}"),
230            None,
231            &[],
232        )
233        .await
234    }
235    pub async fn interact_with_scrape(&self, scrape_id: &str, options: Value) -> Result<Value> {
236        self.post(&format!("/v2/scrape/{scrape_id}/interact"), options)
237            .await
238    }
239    pub async fn delete_scrape_interaction(&self, scrape_id: &str) -> Result<Value> {
240        self.request(
241            Method::DELETE,
242            &format!("/v2/scrape/{scrape_id}/interact"),
243            None,
244            &[],
245        )
246        .await
247    }
248    pub async fn create_agent(&self, prompt: &str, options: Value) -> Result<Value> {
249        self.post("/v2/agent", merge_field(options, "prompt", prompt))
250            .await
251    }
252    pub async fn get_agent(&self, job_id: &str) -> Result<Value> {
253        self.get(&format!("/v2/agent/{job_id}"), &[]).await
254    }
255    pub async fn cancel_agent(&self, job_id: &str) -> Result<Value> {
256        self.request(Method::DELETE, &format!("/v2/agent/{job_id}"), None, &[])
257            .await
258    }
259    pub async fn create_monitor(&self, payload: Value) -> Result<Value> {
260        self.post("/v2/monitor", payload).await
261    }
262    pub async fn list_monitors(&self) -> Result<Value> {
263        self.get("/v2/monitor", &[]).await
264    }
265    pub async fn get_monitor(&self, monitor_id: &str) -> Result<Value> {
266        self.get(&format!("/v2/monitor/{monitor_id}"), &[]).await
267    }
268    pub async fn update_monitor(&self, monitor_id: &str, payload: Value) -> Result<Value> {
269        self.request(
270            Method::PATCH,
271            &format!("/v2/monitor/{monitor_id}"),
272            Some(payload),
273            &[],
274        )
275        .await
276    }
277    pub async fn delete_monitor(&self, monitor_id: &str) -> Result<Value> {
278        self.request(
279            Method::DELETE,
280            &format!("/v2/monitor/{monitor_id}"),
281            None,
282            &[],
283        )
284        .await
285    }
286    pub async fn run_monitor(&self, monitor_id: &str) -> Result<Value> {
287        self.post(&format!("/v2/monitor/{monitor_id}/run"), json!({}))
288            .await
289    }
290    pub async fn monitor_checks(&self, monitor_id: &str, check_id: Option<&str>) -> Result<Value> {
291        self.get(
292            &format!(
293                "/v2/monitor/{monitor_id}/checks{}",
294                check_id.map(|id| format!("/{id}")).unwrap_or_default()
295            ),
296            &[],
297        )
298        .await
299    }
300
301    pub async fn poll_crawl(
302        &self,
303        job_id: &str,
304        offset: usize,
305        limit: usize,
306        interval: Duration,
307        timeout: Duration,
308    ) -> Result<Value> {
309        self.poll(
310            || self.crawl_status(job_id, offset, limit),
311            job_id,
312            interval,
313            timeout,
314        )
315        .await
316    }
317
318    pub async fn poll_batch_scrape(
319        &self,
320        job_id: &str,
321        offset: usize,
322        limit: usize,
323        interval: Duration,
324        timeout: Duration,
325    ) -> Result<Value> {
326        self.poll(
327            || self.batch_scrape_status(job_id, offset, limit),
328            job_id,
329            interval,
330            timeout,
331        )
332        .await
333    }
334
335    async fn poll<F, Fut>(
336        &self,
337        mut status: F,
338        job_id: &str,
339        interval: Duration,
340        timeout: Duration,
341    ) -> Result<Value>
342    where
343        F: FnMut() -> Fut,
344        Fut: std::future::Future<Output = Result<Value>>,
345    {
346        let deadline = Instant::now() + timeout;
347        loop {
348            let result = status().await?;
349            if matches!(
350                result.get("status").and_then(Value::as_str),
351                Some("completed" | "failed" | "cancelled")
352            ) {
353                return Ok(result);
354            }
355            if Instant::now() >= deadline {
356                return Err(BeeCrawlError::PollTimeout {
357                    job_id: job_id.to_string(),
358                });
359            }
360            tokio::time::sleep(interval).await;
361        }
362    }
363
364    async fn post(&self, path: &str, payload: Value) -> Result<Value> {
365        self.request(Method::POST, path, Some(payload), &[]).await
366    }
367
368    async fn get(&self, path: &str, params: &[(&str, String)]) -> Result<Value> {
369        self.request(Method::GET, path, None, params).await
370    }
371
372    async fn request(
373        &self,
374        method: Method,
375        path: &str,
376        payload: Option<Value>,
377        params: &[(&str, String)],
378    ) -> Result<Value> {
379        let mut request = self
380            .http
381            .request(method, format!("{}{}", self.base_url, path));
382        if let Some(api_key) = &self.api_key {
383            request = request.header("X-Web-Extract-Api-Key", api_key);
384        }
385        if !params.is_empty() {
386            request = request.query(params);
387        }
388        if let Some(payload) = payload {
389            request = request.json(&payload);
390        }
391        let response = request.send().await?;
392        let status = response.status();
393        let body = response.text().await?;
394        let payload: Value = serde_json::from_str(&body)?;
395        if !status.is_success() {
396            let detail = payload.get("detail").cloned().unwrap_or(payload);
397            let message = detail
398                .get("message")
399                .and_then(Value::as_str)
400                .unwrap_or("BeeCrawl request failed")
401                .to_string();
402            return Err(BeeCrawlError::Api {
403                status,
404                message,
405                detail,
406            });
407        }
408        Ok(payload)
409    }
410
411    async fn request_bytes(&self, method: Method, path: &str, data: Vec<u8>) -> Result<Value> {
412        let mut request = self
413            .http
414            .request(method, format!("{}{}", self.base_url, path))
415            .body(data);
416        if let Some(api_key) = &self.api_key {
417            request = request.header("X-Web-Extract-Api-Key", api_key);
418        }
419        decode_response(request.send().await?).await
420    }
421
422    async fn request_form(&self, path: &str, form: reqwest::multipart::Form) -> Result<Value> {
423        let mut request = self
424            .http
425            .post(format!("{}{}", self.base_url, path))
426            .multipart(form);
427        if let Some(api_key) = &self.api_key {
428            request = request.header("X-Web-Extract-Api-Key", api_key);
429        }
430        decode_response(request.send().await?).await
431    }
432}
433
434async fn decode_response(response: reqwest::Response) -> Result<Value> {
435    let status = response.status();
436    let body = response.text().await?;
437    let payload: Value = serde_json::from_str(&body)?;
438    if !status.is_success() {
439        let detail = payload.get("detail").cloned().unwrap_or(payload);
440        let message = detail
441            .get("message")
442            .and_then(Value::as_str)
443            .unwrap_or("BeeCrawl request failed")
444            .to_string();
445        return Err(BeeCrawlError::Api {
446            status,
447            message,
448            detail,
449        });
450    }
451    Ok(payload)
452}
453
454#[derive(Clone, Debug)]
455pub struct BeeCrawlClientBuilder {
456    base_url: String,
457    api_key: Option<String>,
458    timeout: Duration,
459}
460
461impl BeeCrawlClientBuilder {
462    pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
463        self.api_key = Some(api_key.into());
464        self
465    }
466
467    pub fn timeout(mut self, timeout: Duration) -> Self {
468        self.timeout = timeout;
469        self
470    }
471
472    pub fn build(self) -> Result<BeeCrawlClient> {
473        let base_url = self.base_url.trim_end_matches('/').to_string();
474        if !(base_url.starts_with("http://") || base_url.starts_with("https://")) {
475            return Err(BeeCrawlError::InvalidBaseUrl(base_url));
476        }
477        Ok(BeeCrawlClient {
478            http: HttpClient::builder().timeout(self.timeout).build()?,
479            base_url,
480            api_key: self.api_key,
481        })
482    }
483}
484
485fn merge_url(options: Value, url: &str) -> Value {
486    merge_field(options, "url", url)
487}
488
489fn merge_field<T: serde::Serialize>(options: Value, field: &str, value: T) -> Value {
490    let mut object = match options {
491        Value::Object(object) => object,
492        _ => Map::new(),
493    };
494    object.insert(field.to_string(), json!(value));
495    Value::Object(object)
496}
497
498#[cfg(test)]
499mod tests {
500    use super::*;
501    use axum::extract::{Request, State};
502    use axum::routing::any;
503    use axum::{Json, Router};
504    use std::sync::{Arc, Mutex};
505
506    #[tokio::test]
507    async fn v2_workflow_and_browser_methods_use_public_routes() {
508        let requests = Arc::new(Mutex::new(Vec::new()));
509        let app = Router::new()
510            .fallback(any(
511                |State(requests): State<Arc<Mutex<Vec<String>>>>, request: Request| async move {
512                    requests.lock().unwrap().push(format!(
513                        "{} {}",
514                        request.method(),
515                        request.uri().path()
516                    ));
517                    Json(json!({"success": true}))
518                },
519            ))
520            .with_state(requests.clone());
521        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
522        let address = listener.local_addr().unwrap();
523        tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
524        let client = BeeCrawlClient::new(format!("http://{address}"));
525
526        client
527            .v2_scrape("https://example.com", json!({}))
528            .await
529            .unwrap();
530        client
531            .create_agent("research", json!({"maxCredits": 2}))
532            .await
533            .unwrap();
534        client
535            .update_monitor("monitor", json!({"enabled": false}))
536            .await
537            .unwrap();
538        client
539            .browser_replay("session", Some("page"))
540            .await
541            .unwrap();
542
543        assert_eq!(
544            *requests.lock().unwrap(),
545            [
546                "POST /v2/scrape",
547                "POST /v2/agent",
548                "PATCH /v2/monitor/monitor",
549                "GET /v2/browser/session/replay/page",
550            ]
551        );
552    }
553}