crawlkit_extensions/firecrawl/
mod.rs

1//! Firecrawl API 客户端
2//!
3//! 通过 [Firecrawl SDK](https://firecrawl.dev) 进行云端网页抓取，支持 JS 渲染。
4//! 适用于 SPA 站点、需要 JavaScript 执行的页面。
5//!
6//! # 示例
7//!
8//! ```rust,no_run
9//! use crawlkit_extensions::firecrawl::FirecrawlClient;
10//!
11//! # #[tokio::main]
12//! # async fn main() -> anyhow::Result<()> {
13//! let client = FirecrawlClient::builder()
14//!     .with_tokens(&["fc-api-token-1", "fc-api-token-2"])
15//!     .build()?;
16//!
17//! let response = client.get("https://example.com", &Default::default()).await?;
18//! println!("{}", response.body);
19//! # Ok(())
20//! # }
21//! ```
22
23use std::collections::HashMap;
24
25use async_trait::async_trait;
26use backon::{ExponentialBuilder, Retryable};
27use crawlkit_core::{CrawlError, HttpClient, Response};
28use rand::Rng;
29
30/// Firecrawl API 客户端
31///
32/// 支持多 token 轮换，适合免费套餐多账号场景。
33pub struct FirecrawlClient {
34    api_tokens: Vec<String>,
35    max_retries: usize,
36}
37
38impl FirecrawlClient {
39    /// 创建 Builder
40    pub fn builder() -> FirecrawlClientBuilder {
41        FirecrawlClientBuilder::default()
42    }
43}
44
45/// FirecrawlClient 构建器
46pub struct FirecrawlClientBuilder {
47    api_tokens: Vec<String>,
48    max_retries: usize,
49}
50
51impl Default for FirecrawlClientBuilder {
52    fn default() -> Self {
53        Self {
54            api_tokens: Vec::new(),
55            max_retries: 3,
56        }
57    }
58}
59
60impl FirecrawlClientBuilder {
61    /// 设置 API token 列表（每次请求随机选择一个）
62    pub fn with_tokens(mut self, tokens: &[impl Into<String> + Clone]) -> Self {
63        self.api_tokens = tokens.iter().map(|t| t.clone().into()).collect();
64        self
65    }
66
67    /// 从环境变量读取 token（逗号分隔多个）
68    pub fn with_tokens_from_env(mut self, env_var: &str) -> Self {
69        if let Ok(val) = std::env::var(env_var) {
70            self.api_tokens = val.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
71        }
72        self
73    }
74
75    /// 设置最大重试次数（默认 3）
76    pub fn with_max_retries(mut self, max_retries: usize) -> Self {
77        self.max_retries = max_retries;
78        self
79    }
80
81    /// 构建 FirecrawlClient
82    ///
83    /// # 错误
84    ///
85    /// 如果未设置任何 token，返回错误。
86    pub fn build(self) -> anyhow::Result<FirecrawlClient> {
87        if self.api_tokens.is_empty() {
88            anyhow::bail!("Firecrawl 至少需要一个 API Token");
89        }
90        Ok(FirecrawlClient {
91            api_tokens: self.api_tokens,
92            max_retries: self.max_retries,
93        })
94    }
95}
96
97#[async_trait]
98impl HttpClient for FirecrawlClient {
99    async fn get(&self, url: &str, _headers: &HashMap<String, String>) -> crawlkit_core::Result<Response> {
100        let tokens = self.api_tokens.clone();
101        let max_retries = self.max_retries;
102
103        let fetch = || {
104            let tokens = tokens.clone();
105            async move {
106                let token = {
107                    let mut rng = rand::rng();
108                    let index = rng.random_range(0..tokens.len());
109                    tokens[index].clone()
110                };
111
112                let client = firecrawl2::Client::new(&token)
113                    .map_err(|e: firecrawl2::error::FirecrawlError| anyhow::anyhow!("Firecrawl 客户端创建失败: {e}"))?;
114
115                let options = firecrawl2::ScrapeOptions {
116                    formats: Some(vec![firecrawl2::Format::Html]),
117                    only_main_content: Some(true),
118                    ..Default::default()
119                };
120
121                let doc = client.scrape(url, options).await
122                    .map_err(|e: firecrawl2::error::FirecrawlError| anyhow::anyhow!("Firecrawl 请求失败: {e}"))?;
123
124                let html = doc.html
125                    .ok_or_else(|| anyhow::anyhow!("Firecrawl 返回 HTML 内容为空"))?;
126
127                Ok::<(String, String), anyhow::Error>((html, url.to_string()))
128            }
129        };
130
131        let (body, final_url) = fetch
132            .retry(&ExponentialBuilder::default().with_max_times(max_retries))
133            .await
134            .map_err(|e| CrawlError::Http(format!("Firecrawl 请求失败（重试 {max_retries} 次）: {e}")))?;
135
136        Ok(Response {
137            url: final_url,
138            status: 200,
139            headers: Default::default(),
140            body,
141        })
142    }
143
144    async fn post(
145        &self,
146        url: &str,
147        headers: &HashMap<String, String>,
148        _body: Vec<u8>,
149    ) -> crawlkit_core::Result<Response> {
150        self.get(url, headers).await
151    }
152
153    fn name(&self) -> &str {
154        "firecrawl"
155    }
156}
crawlkit_extensions/firecrawl/mod.rs

crawlkit_extensions/firecrawl/
mod.rs
