crawlkit_extensions/firecrawl/
mod.rs

1//! Firecrawl API 客户端
2//!
3//! 通过 [Firecrawl SDK](https://firecrawl.dev) 进行云端网页抓取，支持 JS 渲染。
4//! 适用于 SPA 站点、需要 JavaScript 执行的页面。
5//!
6//! # 示例
7//!
8//! ```rust,no_run
9//! use crawlkit_extensions::firecrawl::FirecrawlClient;
10//!
11//! # #[tokio::main]
12//! # async fn main() -> anyhow::Result<()> {
13//! let client = FirecrawlClient::builder()
14//!     .with_tokens(&["fc-api-token-1", "fc-api-token-2"])
15//!     .build()?;
16//!
17//! let response = client.get("https://example.com", &Default::default()).await?;
18//! println!("{}", response.body);
19//! # Ok(())
20//! # }
21//! ```
22
23use std::collections::HashMap;
24
25use async_trait::async_trait;
26use backon::{ExponentialBuilder, Retryable};
27use crawlkit_core::{CrawlError, HttpClient, Response};
28use rand::Rng;
29
30/// Firecrawl API 客户端
31///
32/// 支持多 token 轮换，适合免费套餐多账号场景。
33pub struct FirecrawlClient {
34    name: String,
35    api_tokens: Vec<String>,
36    max_retries: usize,
37}
38
39impl FirecrawlClient {
40    /// 创建 Builder
41    pub fn builder() -> FirecrawlClientBuilder {
42        FirecrawlClientBuilder::default()
43    }
44}
45
46/// FirecrawlClient 构建器
47pub struct FirecrawlClientBuilder {
48    name: String,
49    api_tokens: Vec<String>,
50    max_retries: usize,
51}
52
53impl Default for FirecrawlClientBuilder {
54    fn default() -> Self {
55        Self {
56            name: "firecrawl".to_string(),
57            api_tokens: Vec::new(),
58            max_retries: 3,
59        }
60    }
61}
62
63impl FirecrawlClientBuilder {
64    /// 设置客户端名称（用于日志和 CompositeFetcher 识别，默认 "firecrawl"）
65    pub fn with_name(mut self, name: impl Into<String>) -> Self {
66        self.name = name.into();
67        self
68    }
69
70    /// 设置 API token 列表（每次请求随机选择一个）
71    pub fn with_tokens(mut self, tokens: &[impl Into<String> + Clone]) -> Self {
72        self.api_tokens = tokens.iter().map(|t| t.clone().into()).collect();
73        self
74    }
75
76    /// 从环境变量读取 token（逗号分隔多个）
77    pub fn with_tokens_from_env(mut self, env_var: &str) -> Self {
78        if let Ok(val) = std::env::var(env_var) {
79            self.api_tokens = val.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
80        }
81        self
82    }
83
84    /// 设置最大重试次数（默认 3）
85    pub fn with_max_retries(mut self, max_retries: usize) -> Self {
86        self.max_retries = max_retries;
87        self
88    }
89
90    /// 构建 FirecrawlClient
91    ///
92    /// # 错误
93    ///
94    /// 如果未设置任何 token，返回错误。
95    pub fn build(self) -> anyhow::Result<FirecrawlClient> {
96        if self.api_tokens.is_empty() {
97            anyhow::bail!("Firecrawl 至少需要一个 API Token");
98        }
99        Ok(FirecrawlClient {
100            name: self.name,
101            api_tokens: self.api_tokens,
102            max_retries: self.max_retries,
103        })
104    }
105}
106
107#[async_trait]
108impl HttpClient for FirecrawlClient {
109    async fn get(&self, url: &str, _headers: &HashMap<String, String>) -> crawlkit_core::Result<Response> {
110        let tokens = self.api_tokens.clone();
111        let max_retries = self.max_retries;
112
113        let fetch = || {
114            let tokens = tokens.clone();
115            async move {
116                let token = {
117                    let mut rng = rand::rng();
118                    let index = rng.random_range(0..tokens.len());
119                    tokens[index].clone()
120                };
121
122                let client = firecrawl2::Client::new(&token)
123                    .map_err(|e: firecrawl2::error::FirecrawlError| anyhow::anyhow!("Firecrawl 客户端创建失败: {e}"))?;
124
125                let options = firecrawl2::ScrapeOptions {
126                    formats: Some(vec![firecrawl2::Format::Html]),
127                    only_main_content: Some(true),
128                    ..Default::default()
129                };
130
131                let doc = client.scrape(url, options).await
132                    .map_err(|e: firecrawl2::error::FirecrawlError| anyhow::anyhow!("Firecrawl 请求失败: {e}"))?;
133
134                let html = doc.html
135                    .ok_or_else(|| anyhow::anyhow!("Firecrawl 返回 HTML 内容为空"))?;
136
137                Ok::<(String, String), anyhow::Error>((html, url.to_string()))
138            }
139        };
140
141        let (body, final_url) = fetch
142            .retry(&ExponentialBuilder::default().with_max_times(max_retries))
143            .await
144            .map_err(|e| CrawlError::Http(format!("Firecrawl 请求失败（重试 {max_retries} 次）: {e}")))?;
145
146        Ok(Response {
147            url: final_url,
148            status: 200,
149            headers: Default::default(),
150            body,
151        })
152    }
153
154    async fn post(
155        &self,
156        url: &str,
157        headers: &HashMap<String, String>,
158        _body: Vec<u8>,
159    ) -> crawlkit_core::Result<Response> {
160        self.get(url, headers).await
161    }
162
163    fn name(&self) -> &str {
164        &self.name
165    }
166}
crawlkit_extensions/firecrawl/mod.rs

crawlkit_extensions/firecrawl/
mod.rs
