1use reqwest::{Client as HttpClient, Method, StatusCode};
2use serde_json::{json, Map, Value};
3use std::collections::HashMap;
4use std::time::{Duration, Instant};
5use thiserror::Error;
6
7#[derive(Debug, Error)]
8pub enum BeeCrawlError {
9 #[error("BeeCrawl request failed: {0}")]
10 Transport(#[from] reqwest::Error),
11 #[error("BeeCrawl returned invalid JSON: {0}")]
12 InvalidJson(#[from] serde_json::Error),
13 #[error("BeeCrawl request failed with status {status}: {message}")]
14 Api {
15 status: StatusCode,
16 message: String,
17 detail: Value,
18 },
19 #[error("timed out waiting for job {job_id}")]
20 PollTimeout { job_id: String },
21 #[error("invalid base URL: {0}")]
22 InvalidBaseUrl(String),
23}
24
25pub type Result<T> = std::result::Result<T, BeeCrawlError>;
26
27#[derive(Clone, Debug)]
28pub struct BeeCrawlClient {
29 http: HttpClient,
30 base_url: String,
31 api_key: Option<String>,
32}
33
34impl BeeCrawlClient {
35 pub fn new(base_url: impl Into<String>) -> Self {
36 Self::builder(base_url)
37 .build()
38 .expect("default client configuration is valid")
39 }
40
41 pub fn builder(base_url: impl Into<String>) -> BeeCrawlClientBuilder {
42 BeeCrawlClientBuilder {
43 base_url: base_url.into(),
44 api_key: None,
45 timeout: Duration::from_secs(60),
46 }
47 }
48
49 pub async fn scrape(&self, url: &str, options: Value) -> Result<Value> {
50 self.post("/scrape", merge_url(options, url)).await
51 }
52
53 pub async fn map(&self, url: &str, options: Value) -> Result<Value> {
54 self.post("/map", merge_url(options, url)).await
55 }
56
57 pub async fn search(&self, query: &str, options: Value) -> Result<Value> {
58 self.post("/search", merge_field(options, "query", query))
59 .await
60 }
61
62 pub async fn extract(
63 &self,
64 url: &str,
65 schema: HashMap<String, String>,
66 options: Value,
67 ) -> Result<Value> {
68 let mut payload = merge_url(options, url);
69 payload["schema"] = json!(schema);
70 self.post("/extract", payload).await
71 }
72
73 pub async fn crawl(&self, url: &str, options: Value) -> Result<Value> {
74 self.post("/crawl", merge_url(options, url)).await
75 }
76
77 pub async fn batch_scrape(&self, urls: Vec<String>, options: Value) -> Result<Value> {
78 self.post("/batch/scrape", merge_field(options, "urls", urls))
79 .await
80 }
81
82 pub async fn crawl_status(&self, job_id: &str, offset: usize, limit: usize) -> Result<Value> {
83 self.get(
84 &format!("/crawl/{job_id}"),
85 &[("offset", offset.to_string()), ("limit", limit.to_string())],
86 )
87 .await
88 }
89
90 pub async fn batch_scrape_status(
91 &self,
92 job_id: &str,
93 offset: usize,
94 limit: usize,
95 ) -> Result<Value> {
96 self.get(
97 &format!("/batch/scrape/{job_id}"),
98 &[("offset", offset.to_string()), ("limit", limit.to_string())],
99 )
100 .await
101 }
102
103 pub async fn cancel_crawl(&self, job_id: &str) -> Result<Value> {
104 self.request(Method::DELETE, &format!("/crawl/{job_id}"), None, &[])
105 .await
106 }
107
108 pub async fn cancel_batch_scrape(&self, job_id: &str) -> Result<Value> {
109 self.request(
110 Method::DELETE,
111 &format!("/batch/scrape/{job_id}"),
112 None,
113 &[],
114 )
115 .await
116 }
117
118 pub async fn v2_scrape(&self, url: &str, options: Value) -> Result<Value> {
119 self.post("/v2/scrape", merge_url(options, url)).await
120 }
121 pub async fn v2_map(&self, url: &str, options: Value) -> Result<Value> {
122 self.post("/v2/map", merge_url(options, url)).await
123 }
124 pub async fn v2_search(&self, query: &str, options: Value) -> Result<Value> {
125 self.post("/v2/search", merge_field(options, "query", query))
126 .await
127 }
128 pub async fn v2_extract(&self, urls: Vec<String>, options: Value) -> Result<Value> {
129 self.post("/v2/extract", merge_field(options, "urls", urls))
130 .await
131 }
132 pub async fn v2_parse_base64(
133 &self,
134 data: &str,
135 filename: &str,
136 options: Value,
137 ) -> Result<Value> {
138 self.post(
139 "/v2/parse/base64",
140 merge_field(merge_field(options, "base64", data), "filename", filename),
141 )
142 .await
143 }
144 pub async fn v2_parse_reference(&self, upload_ref: &str, options: Value) -> Result<Value> {
145 self.post(
146 "/v2/parse/reference",
147 merge_field(options, "uploadRef", upload_ref),
148 )
149 .await
150 }
151 pub async fn create_parse_upload(&self, filename: &str) -> Result<Value> {
152 self.post("/v2/parse/upload-url", json!({"filename": filename}))
153 .await
154 }
155 pub async fn upload_parse_document(&self, upload_ref: &str, data: Vec<u8>) -> Result<Value> {
156 self.request_bytes(Method::PUT, &format!("/v2/parse/upload/{upload_ref}"), data)
157 .await
158 }
159 pub async fn v2_parse(&self, filename: &str, data: Vec<u8>, options: Value) -> Result<Value> {
160 let form = reqwest::multipart::Form::new()
161 .text("options", options.to_string())
162 .part(
163 "file",
164 reqwest::multipart::Part::bytes(data).file_name(filename.to_string()),
165 );
166 self.request_form("/v2/parse", form).await
167 }
168 pub async fn v2_crawl(&self, url: &str, options: Value) -> Result<Value> {
169 self.post("/v2/crawl", merge_url(options, url)).await
170 }
171 pub async fn v2_batch_scrape(&self, urls: Vec<String>, options: Value) -> Result<Value> {
172 self.post("/v2/batch/scrape", merge_field(options, "urls", urls))
173 .await
174 }
175 pub async fn v2_job_status(
176 &self,
177 kind: &str,
178 job_id: &str,
179 offset: usize,
180 limit: usize,
181 ) -> Result<Value> {
182 self.get(
183 &format!("/v2/{kind}/{job_id}"),
184 &[("offset", offset.to_string()), ("limit", limit.to_string())],
185 )
186 .await
187 }
188 pub async fn v2_job_errors(&self, kind: &str, job_id: &str) -> Result<Value> {
189 self.get(&format!("/v2/{kind}/{job_id}/errors"), &[]).await
190 }
191 pub async fn cancel_v2_job(&self, kind: &str, job_id: &str) -> Result<Value> {
192 self.request(Method::DELETE, &format!("/v2/{kind}/{job_id}"), None, &[])
193 .await
194 }
195 pub async fn active_crawls(&self) -> Result<Value> {
196 self.get("/v2/crawl/active", &[]).await
197 }
198 pub async fn create_browser_session(&self, options: Value) -> Result<Value> {
199 self.post("/v2/browser", options).await
200 }
201 pub async fn browser_sessions(&self) -> Result<Value> {
202 self.get("/v2/browser", &[]).await
203 }
204 pub async fn execute_browser(
205 &self,
206 session_id: &str,
207 code: &str,
208 options: Value,
209 ) -> Result<Value> {
210 self.post(
211 &format!("/v2/browser/{session_id}/execute"),
212 merge_field(options, "code", code),
213 )
214 .await
215 }
216 pub async fn browser_replay(&self, session_id: &str, page_id: Option<&str>) -> Result<Value> {
217 self.get(
218 &format!(
219 "/v2/browser/{session_id}/replay{}",
220 page_id.map(|id| format!("/{id}")).unwrap_or_default()
221 ),
222 &[],
223 )
224 .await
225 }
226 pub async fn delete_browser_session(&self, session_id: &str) -> Result<Value> {
227 self.request(
228 Method::DELETE,
229 &format!("/v2/browser/{session_id}"),
230 None,
231 &[],
232 )
233 .await
234 }
235 pub async fn interact_with_scrape(&self, scrape_id: &str, options: Value) -> Result<Value> {
236 self.post(&format!("/v2/scrape/{scrape_id}/interact"), options)
237 .await
238 }
239 pub async fn delete_scrape_interaction(&self, scrape_id: &str) -> Result<Value> {
240 self.request(
241 Method::DELETE,
242 &format!("/v2/scrape/{scrape_id}/interact"),
243 None,
244 &[],
245 )
246 .await
247 }
248 pub async fn create_agent(&self, prompt: &str, options: Value) -> Result<Value> {
249 self.post("/v2/agent", merge_field(options, "prompt", prompt))
250 .await
251 }
252 pub async fn get_agent(&self, job_id: &str) -> Result<Value> {
253 self.get(&format!("/v2/agent/{job_id}"), &[]).await
254 }
255 pub async fn cancel_agent(&self, job_id: &str) -> Result<Value> {
256 self.request(Method::DELETE, &format!("/v2/agent/{job_id}"), None, &[])
257 .await
258 }
259 pub async fn create_monitor(&self, payload: Value) -> Result<Value> {
260 self.post("/v2/monitor", payload).await
261 }
262 pub async fn list_monitors(&self) -> Result<Value> {
263 self.get("/v2/monitor", &[]).await
264 }
265 pub async fn get_monitor(&self, monitor_id: &str) -> Result<Value> {
266 self.get(&format!("/v2/monitor/{monitor_id}"), &[]).await
267 }
268 pub async fn update_monitor(&self, monitor_id: &str, payload: Value) -> Result<Value> {
269 self.request(
270 Method::PATCH,
271 &format!("/v2/monitor/{monitor_id}"),
272 Some(payload),
273 &[],
274 )
275 .await
276 }
277 pub async fn delete_monitor(&self, monitor_id: &str) -> Result<Value> {
278 self.request(
279 Method::DELETE,
280 &format!("/v2/monitor/{monitor_id}"),
281 None,
282 &[],
283 )
284 .await
285 }
286 pub async fn run_monitor(&self, monitor_id: &str) -> Result<Value> {
287 self.post(&format!("/v2/monitor/{monitor_id}/run"), json!({}))
288 .await
289 }
290 pub async fn monitor_checks(&self, monitor_id: &str, check_id: Option<&str>) -> Result<Value> {
291 self.get(
292 &format!(
293 "/v2/monitor/{monitor_id}/checks{}",
294 check_id.map(|id| format!("/{id}")).unwrap_or_default()
295 ),
296 &[],
297 )
298 .await
299 }
300
301 pub async fn poll_crawl(
302 &self,
303 job_id: &str,
304 offset: usize,
305 limit: usize,
306 interval: Duration,
307 timeout: Duration,
308 ) -> Result<Value> {
309 self.poll(
310 || self.crawl_status(job_id, offset, limit),
311 job_id,
312 interval,
313 timeout,
314 )
315 .await
316 }
317
318 pub async fn poll_batch_scrape(
319 &self,
320 job_id: &str,
321 offset: usize,
322 limit: usize,
323 interval: Duration,
324 timeout: Duration,
325 ) -> Result<Value> {
326 self.poll(
327 || self.batch_scrape_status(job_id, offset, limit),
328 job_id,
329 interval,
330 timeout,
331 )
332 .await
333 }
334
335 async fn poll<F, Fut>(
336 &self,
337 mut status: F,
338 job_id: &str,
339 interval: Duration,
340 timeout: Duration,
341 ) -> Result<Value>
342 where
343 F: FnMut() -> Fut,
344 Fut: std::future::Future<Output = Result<Value>>,
345 {
346 let deadline = Instant::now() + timeout;
347 loop {
348 let result = status().await?;
349 if matches!(
350 result.get("status").and_then(Value::as_str),
351 Some("completed" | "failed" | "cancelled")
352 ) {
353 return Ok(result);
354 }
355 if Instant::now() >= deadline {
356 return Err(BeeCrawlError::PollTimeout {
357 job_id: job_id.to_string(),
358 });
359 }
360 tokio::time::sleep(interval).await;
361 }
362 }
363
364 async fn post(&self, path: &str, payload: Value) -> Result<Value> {
365 self.request(Method::POST, path, Some(payload), &[]).await
366 }
367
368 async fn get(&self, path: &str, params: &[(&str, String)]) -> Result<Value> {
369 self.request(Method::GET, path, None, params).await
370 }
371
372 async fn request(
373 &self,
374 method: Method,
375 path: &str,
376 payload: Option<Value>,
377 params: &[(&str, String)],
378 ) -> Result<Value> {
379 let mut request = self
380 .http
381 .request(method, format!("{}{}", self.base_url, path));
382 if let Some(api_key) = &self.api_key {
383 request = request.header("X-Web-Extract-Api-Key", api_key);
384 }
385 if !params.is_empty() {
386 request = request.query(params);
387 }
388 if let Some(payload) = payload {
389 request = request.json(&payload);
390 }
391 let response = request.send().await?;
392 let status = response.status();
393 let body = response.text().await?;
394 let payload: Value = serde_json::from_str(&body)?;
395 if !status.is_success() {
396 let detail = payload.get("detail").cloned().unwrap_or(payload);
397 let message = detail
398 .get("message")
399 .and_then(Value::as_str)
400 .unwrap_or("BeeCrawl request failed")
401 .to_string();
402 return Err(BeeCrawlError::Api {
403 status,
404 message,
405 detail,
406 });
407 }
408 Ok(payload)
409 }
410
411 async fn request_bytes(&self, method: Method, path: &str, data: Vec<u8>) -> Result<Value> {
412 let mut request = self
413 .http
414 .request(method, format!("{}{}", self.base_url, path))
415 .body(data);
416 if let Some(api_key) = &self.api_key {
417 request = request.header("X-Web-Extract-Api-Key", api_key);
418 }
419 decode_response(request.send().await?).await
420 }
421
422 async fn request_form(&self, path: &str, form: reqwest::multipart::Form) -> Result<Value> {
423 let mut request = self
424 .http
425 .post(format!("{}{}", self.base_url, path))
426 .multipart(form);
427 if let Some(api_key) = &self.api_key {
428 request = request.header("X-Web-Extract-Api-Key", api_key);
429 }
430 decode_response(request.send().await?).await
431 }
432}
433
434async fn decode_response(response: reqwest::Response) -> Result<Value> {
435 let status = response.status();
436 let body = response.text().await?;
437 let payload: Value = serde_json::from_str(&body)?;
438 if !status.is_success() {
439 let detail = payload.get("detail").cloned().unwrap_or(payload);
440 let message = detail
441 .get("message")
442 .and_then(Value::as_str)
443 .unwrap_or("BeeCrawl request failed")
444 .to_string();
445 return Err(BeeCrawlError::Api {
446 status,
447 message,
448 detail,
449 });
450 }
451 Ok(payload)
452}
453
454#[derive(Clone, Debug)]
455pub struct BeeCrawlClientBuilder {
456 base_url: String,
457 api_key: Option<String>,
458 timeout: Duration,
459}
460
461impl BeeCrawlClientBuilder {
462 pub fn api_key(mut self, api_key: impl Into<String>) -> Self {
463 self.api_key = Some(api_key.into());
464 self
465 }
466
467 pub fn timeout(mut self, timeout: Duration) -> Self {
468 self.timeout = timeout;
469 self
470 }
471
472 pub fn build(self) -> Result<BeeCrawlClient> {
473 let base_url = self.base_url.trim_end_matches('/').to_string();
474 if !(base_url.starts_with("http://") || base_url.starts_with("https://")) {
475 return Err(BeeCrawlError::InvalidBaseUrl(base_url));
476 }
477 Ok(BeeCrawlClient {
478 http: HttpClient::builder().timeout(self.timeout).build()?,
479 base_url,
480 api_key: self.api_key,
481 })
482 }
483}
484
485fn merge_url(options: Value, url: &str) -> Value {
486 merge_field(options, "url", url)
487}
488
489fn merge_field<T: serde::Serialize>(options: Value, field: &str, value: T) -> Value {
490 let mut object = match options {
491 Value::Object(object) => object,
492 _ => Map::new(),
493 };
494 object.insert(field.to_string(), json!(value));
495 Value::Object(object)
496}
497
498#[cfg(test)]
499mod tests {
500 use super::*;
501 use axum::extract::{Request, State};
502 use axum::routing::any;
503 use axum::{Json, Router};
504 use std::sync::{Arc, Mutex};
505
506 #[tokio::test]
507 async fn v2_workflow_and_browser_methods_use_public_routes() {
508 let requests = Arc::new(Mutex::new(Vec::new()));
509 let app = Router::new()
510 .fallback(any(
511 |State(requests): State<Arc<Mutex<Vec<String>>>>, request: Request| async move {
512 requests.lock().unwrap().push(format!(
513 "{} {}",
514 request.method(),
515 request.uri().path()
516 ));
517 Json(json!({"success": true}))
518 },
519 ))
520 .with_state(requests.clone());
521 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
522 let address = listener.local_addr().unwrap();
523 tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
524 let client = BeeCrawlClient::new(format!("http://{address}"));
525
526 client
527 .v2_scrape("https://example.com", json!({}))
528 .await
529 .unwrap();
530 client
531 .create_agent("research", json!({"maxCredits": 2}))
532 .await
533 .unwrap();
534 client
535 .update_monitor("monitor", json!({"enabled": false}))
536 .await
537 .unwrap();
538 client
539 .browser_replay("session", Some("page"))
540 .await
541 .unwrap();
542
543 assert_eq!(
544 *requests.lock().unwrap(),
545 [
546 "POST /v2/scrape",
547 "POST /v2/agent",
548 "PATCH /v2/monitor/monitor",
549 "GET /v2/browser/session/replay/page",
550 ]
551 );
552 }
553}