pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields in requests are ignored for backward/forward compatibility.
4
5use std::collections::BTreeMap;
6
7use serde::{Deserialize, Deserializer, Serialize};
8use serde_json::{Map, Value};
9
10/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
11#[derive(Debug, Clone, PartialEq, Serialize)]
12pub struct Format {
13    #[serde(rename = "type")]
14    pub kind: String,
15    #[serde(flatten)]
16    pub options: Map<String, Value>,
17}
18
19impl Format {
20    pub fn new(kind: &str) -> Self {
21        Self {
22            kind: kind.to_string(),
23            options: Map::new(),
24        }
25    }
26}
27
28impl<'de> Deserialize<'de> for Format {
29    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
30        match Value::deserialize(d)? {
31            Value::String(kind) => Ok(Format::new(&kind)),
32            Value::Object(mut map) => {
33                let kind = match map.remove("type") {
34                    Some(Value::String(k)) => k,
35                    _ => {
36                        return Err(serde::de::Error::custom(
37                            "format object needs a string `type`",
38                        ));
39                    }
40                };
41                Ok(Format { kind, options: map })
42            }
43            _ => Err(serde::de::Error::custom(
44                "format must be a string or an object",
45            )),
46        }
47    }
48}
49
50pub(crate) fn default_formats() -> Vec<Format> {
51    vec![Format::new("markdown")]
52}
53
54pub(crate) fn yes() -> bool {
55    true
56}
57
58#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
59pub struct Location {
60    #[serde(default)]
61    pub country: Option<String>,
62    #[serde(default)]
63    pub languages: Vec<String>,
64}
65
66/// Single page scrape options (`scrapeOptions`).
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct ScrapeOptions {
70    #[serde(default = "default_formats")]
71    pub formats: Vec<Format>,
72    #[serde(default)]
73    pub headers: BTreeMap<String, String>,
74    #[serde(default)]
75    pub include_tags: Vec<String>,
76    #[serde(default)]
77    pub exclude_tags: Vec<String>,
78    #[serde(default = "yes")]
79    pub only_main_content: bool,
80    /// Milliseconds.
81    #[serde(default)]
82    pub timeout: Option<u64>,
83    /// Milliseconds.
84    #[serde(default)]
85    pub wait_for: u64,
86    #[serde(default)]
87    pub mobile: bool,
88    #[serde(default)]
89    pub actions: Vec<Value>,
90    #[serde(default)]
91    pub location: Option<Location>,
92    #[serde(default)]
93    pub parsers: Option<Value>,
94    #[serde(default = "yes")]
95    pub remove_base64_images: bool,
96    #[serde(default = "yes")]
97    pub block_ads: bool,
98    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
99    #[serde(default)]
100    pub proxy: Option<String>,
101    /// PageSift extension: designated named egress (`[[egress]]`).
102    #[serde(default)]
103    pub egress: Option<String>,
104    /// TLS skip verification flag (PageSift always verifies certificates).
105    #[serde(default)]
106    pub skip_tls_verification: Option<bool>,
107    /// Milliseconds.
108    #[serde(default)]
109    pub max_age: Option<u64>,
110    /// Milliseconds.
111    #[serde(default)]
112    pub min_age: Option<u64>,
113    #[serde(default = "yes")]
114    pub store_in_cache: bool,
115
116    // ---- PageSift extensions ----
117    /// `http`, `browser`, `auto` (default).
118    #[serde(default)]
119    pub engine: Option<String>,
120    /// PageSift extension: bounded, stateless same-origin external JS discovery.
121    #[serde(default)]
122    pub discover_scripts: bool,
123    #[serde(default)]
124    pub if_none_match: Option<String>,
125    #[serde(default)]
126    pub if_modified_since: Option<String>,
127    /// Client-side robots.txt bypass flag.
128    #[serde(default)]
129    pub ignore_robots_txt: bool,
130    #[serde(default)]
131    pub store_snapshot: bool,
132    /// BM25 query filter on main content markdown.
133    #[serde(default)]
134    pub query_filter: Option<QueryFilter>,
135    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
136    #[serde(default, skip_serializing_if = "Option::is_none")]
137    pub chunking: Option<Value>,
138}
139
140/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
141#[derive(Debug, Clone, PartialEq, Serialize)]
142#[serde(rename_all = "camelCase")]
143pub struct QueryFilter {
144    pub query: String,
145    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub threshold: Option<f32>,
148    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub relative: Option<f32>,
151    /// Maximum number of blocks to keep.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub top_k: Option<usize>,
154    /// Token ceiling on retained content.
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub max_tokens: Option<usize>,
157    /// Token floor on retained content. Defaults to 1000.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub min_tokens: Option<usize>,
160    /// Prepend ancestor headings before preserved units.
161    pub headings: bool,
162}
163
164impl<'de> Deserialize<'de> for QueryFilter {
165    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
166        #[derive(Deserialize)]
167        #[serde(rename_all = "camelCase")]
168        struct Full {
169            query: String,
170            #[serde(default)]
171            threshold: Option<f32>,
172            #[serde(default)]
173            relative: Option<f32>,
174            #[serde(default)]
175            top_k: Option<usize>,
176            #[serde(default)]
177            max_tokens: Option<usize>,
178            #[serde(default)]
179            min_tokens: Option<usize>,
180            #[serde(default = "yes")]
181            headings: bool,
182        }
183        #[derive(Deserialize)]
184        #[serde(untagged)]
185        enum Repr {
186            Query(String),
187            Full(Full),
188        }
189        Ok(match Repr::deserialize(d)? {
190            Repr::Query(query) => QueryFilter {
191                query,
192                threshold: None,
193                relative: Some(pagesift_extract_defaults::RELATIVE),
194                top_k: None,
195                max_tokens: None,
196                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
197                headings: true,
198            },
199            Repr::Full(f) => QueryFilter {
200                query: f.query,
201                relative: f.relative.or(if f.threshold.is_none() {
202                    Some(pagesift_extract_defaults::RELATIVE)
203                } else {
204                    None
205                }),
206                threshold: f.threshold,
207                top_k: f.top_k,
208                max_tokens: f.max_tokens,
209                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
210                    Some(pagesift_extract_defaults::MIN_TOKENS)
211                } else {
212                    None
213                }),
214                headings: f.headings,
215            },
216        })
217    }
218}
219
220mod pagesift_extract_defaults {
221    pub const RELATIVE: f32 = 0.5;
222    pub const MIN_TOKENS: usize = 1000;
223}
224
225impl Default for ScrapeOptions {
226    fn default() -> Self {
227        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
228    }
229}
230
231impl ScrapeOptions {
232    pub fn wants(&self, kind: &str) -> bool {
233        self.formats.iter().any(|f| f.kind == kind)
234    }
235}
236
237#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
238pub struct ScrapeRequest {
239    pub url: String,
240    #[serde(flatten)]
241    pub options: ScrapeOptions,
242}
243
244#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
245#[serde(rename_all = "camelCase")]
246pub struct BatchScrapeRequest {
247    pub urls: Vec<String>,
248    #[serde(default = "yes")]
249    pub ignore_invalid_urls: bool,
250    #[serde(default)]
251    pub max_concurrency: Option<usize>,
252    #[serde(default)]
253    pub webhook: Option<Value>,
254    #[serde(flatten)]
255    pub options: ScrapeOptions,
256}
257
258fn default_crawl_limit() -> usize {
259    10_000
260}
261
262fn default_sitemap() -> String {
263    "include".into()
264}
265
266#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
267#[serde(rename_all = "camelCase")]
268pub struct CrawlRequest {
269    pub url: String,
270    #[serde(default)]
271    pub include_paths: Vec<String>,
272    #[serde(default)]
273    pub exclude_paths: Vec<String>,
274    #[serde(default)]
275    pub max_discovery_depth: Option<usize>,
276    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
277    #[serde(default, skip_serializing_if = "Option::is_none")]
278    pub max_depth: Option<u64>,
279    #[serde(default = "default_crawl_limit")]
280    pub limit: usize,
281    #[serde(default)]
282    pub crawl_entire_domain: bool,
283    #[serde(default)]
284    pub allow_external_links: bool,
285    /// 没给时：带 `query` 的爬取默认允许同一站点（可注册域名相同）的子域名，广度优先爬取不允许。
286    #[serde(default)]
287    pub allow_subdomains: Option<bool>,
288    #[serde(default)]
289    pub ignore_robots_txt: bool,
290    #[serde(default = "default_sitemap")]
291    pub sitemap: String,
292    #[serde(default)]
293    pub ignore_query_parameters: bool,
294    /// Seconds.
295    #[serde(default)]
296    pub delay: Option<f64>,
297    #[serde(default)]
298    pub max_concurrency: Option<usize>,
299    #[serde(default)]
300    pub scrape_options: ScrapeOptions,
301    #[serde(default)]
302    pub webhook: Option<Value>,
303
304    // ---- PageSift extensions ----
305    #[serde(default, deserialize_with = "query_or_keywords")]
306    pub query: Option<String>,
307    #[serde(default, deserialize_with = "adaptive_options")]
308    pub adaptive: Option<AdaptiveOptions>,
309    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
310    /// 不填时普通爬取是 `auto`，带查询的爬取是 `domain`（相关性打分决定往哪走，路径前缀只会挡路：
311    /// 首页跳到 `/top/` 的站点，招生页在 `/inst/` 下）。
312    #[serde(default)]
313    pub scope: Option<CrawlScope>,
314    /// 带查询的爬取：相关页面上同站点的 PDF/Office 文档也抓回来解析（PageSift 扩展）。
315    #[serde(default)]
316    pub attachments: bool,
317    /// 附件的数量上限，默认等于 `limit`。
318    #[serde(default)]
319    pub attachment_limit: Option<usize>,
320}
321
322/// crawl 的路径范围。
323#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
324#[serde(rename_all = "lowercase")]
325pub enum CrawlScope {
326    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
327    #[default]
328    Auto,
329    /// 只取最窄目录，不放宽。
330    Exact,
331    /// 一开始就放宽到上一级目录（不会到整站根）。
332    Parent,
333    /// 整个主机，等同于 `crawlEntireDomain`。
334    Domain,
335}
336
337fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
338    #[derive(Deserialize)]
339    #[serde(untagged)]
340    enum Repr {
341        Query(String),
342        Keywords(Vec<String>),
343    }
344    Ok(match Option::<Repr>::deserialize(d)? {
345        Some(Repr::Query(q)) => Some(q),
346        Some(Repr::Keywords(k)) => Some(k.join(" ")),
347        None => None,
348    }
349    .filter(|q| !q.trim().is_empty()))
350}
351
352fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
353    #[derive(Deserialize)]
354    #[serde(untagged)]
355    enum Repr {
356        Flag(bool),
357        Options(AdaptiveOptions),
358    }
359    Ok(match Option::<Repr>::deserialize(d)? {
360        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
361        Some(Repr::Flag(false)) | None => None,
362        Some(Repr::Options(o)) => Some(o),
363    })
364}
365
366/// Adaptive crawl stopping conditions (PageSift extension `adaptive`).
367#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
368#[serde(rename_all = "camelCase", default)]
369pub struct AdaptiveOptions {
370    pub coverage_threshold: f64,
371    pub evidence_per_term: usize,
372    pub patience: usize,
373    pub min_gain: f64,
374}
375
376impl Default for AdaptiveOptions {
377    fn default() -> Self {
378        Self {
379            coverage_threshold: 0.9,
380            evidence_per_term: 3,
381            patience: 5,
382            min_gain: 0.05,
383        }
384    }
385}
386
387fn default_map_limit() -> usize {
388    5_000
389}
390
391#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
392#[serde(rename_all = "camelCase")]
393pub struct MapRequest {
394    pub url: String,
395    #[serde(default)]
396    pub search: Option<String>,
397    #[serde(default = "yes")]
398    pub include_subdomains: bool,
399    #[serde(default = "yes")]
400    pub ignore_query_parameters: bool,
401    #[serde(default = "default_sitemap")]
402    pub sitemap: String,
403    #[serde(default = "default_map_limit")]
404    pub limit: usize,
405    #[serde(default)]
406    pub ignore_robots_txt: bool,
407    /// Milliseconds.
408    #[serde(default)]
409    pub timeout: Option<u64>,
410}
411
412fn default_search_limit() -> usize {
413    10
414}
415
416/// `/v1/search` and `/v2/search`.
417#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
418#[serde(rename_all = "camelCase")]
419pub struct SearchRequest {
420    pub query: String,
421    #[serde(default = "default_search_limit")]
422    pub limit: usize,
423    #[serde(default)]
424    pub lang: Option<String>,
425    #[serde(default)]
426    pub country: Option<String>,
427    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
428    #[serde(default)]
429    pub tbs: Option<String>,
430    /// Milliseconds.
431    #[serde(default)]
432    pub timeout: Option<u64>,
433    #[serde(default)]
434    pub scrape_options: Option<ScrapeOptions>,
435}
436
437/// `/v1/extract` and `/v2/extract`.
438#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
439#[serde(rename_all = "camelCase")]
440pub struct ExtractRequest {
441    pub urls: Vec<String>,
442    #[serde(default)]
443    pub prompt: Option<String>,
444    #[serde(default)]
445    pub schema: Option<Value>,
446    #[serde(default)]
447    pub enable_web_search: bool,
448    #[serde(default)]
449    pub ignore_invalid_urls: bool,
450    #[serde(default)]
451    pub scrape_options: Option<ScrapeOptions>,
452}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
