pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields in requests are ignored for backward/forward compatibility.
4
5use std::collections::BTreeMap;
6
7use serde::{Deserialize, Deserializer, Serialize};
8use serde_json::{Map, Value};
9
10/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
11#[derive(Debug, Clone, PartialEq, Serialize)]
12pub struct Format {
13    #[serde(rename = "type")]
14    pub kind: String,
15    #[serde(flatten)]
16    pub options: Map<String, Value>,
17}
18
19impl Format {
20    pub fn new(kind: &str) -> Self {
21        Self {
22            kind: kind.to_string(),
23            options: Map::new(),
24        }
25    }
26}
27
28impl<'de> Deserialize<'de> for Format {
29    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
30        match Value::deserialize(d)? {
31            Value::String(kind) => Ok(Format::new(&kind)),
32            Value::Object(mut map) => {
33                let kind = match map.remove("type") {
34                    Some(Value::String(k)) => k,
35                    _ => {
36                        return Err(serde::de::Error::custom(
37                            "format object needs a string `type`",
38                        ));
39                    }
40                };
41                Ok(Format { kind, options: map })
42            }
43            _ => Err(serde::de::Error::custom(
44                "format must be a string or an object",
45            )),
46        }
47    }
48}
49
50pub(crate) fn default_formats() -> Vec<Format> {
51    vec![Format::new("markdown")]
52}
53
54pub(crate) fn yes() -> bool {
55    true
56}
57
58#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
59pub struct Location {
60    #[serde(default)]
61    pub country: Option<String>,
62    #[serde(default)]
63    pub languages: Vec<String>,
64}
65
66/// Single page scrape options (`scrapeOptions`).
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct ScrapeOptions {
70    #[serde(default = "default_formats")]
71    pub formats: Vec<Format>,
72    #[serde(default)]
73    pub headers: BTreeMap<String, String>,
74    #[serde(default)]
75    pub include_tags: Vec<String>,
76    #[serde(default)]
77    pub exclude_tags: Vec<String>,
78    #[serde(default = "yes")]
79    pub only_main_content: bool,
80    /// Milliseconds.
81    #[serde(default)]
82    pub timeout: Option<u64>,
83    /// Milliseconds.
84    #[serde(default)]
85    pub wait_for: u64,
86    #[serde(default)]
87    pub mobile: bool,
88    #[serde(default)]
89    pub actions: Vec<Value>,
90    #[serde(default)]
91    pub location: Option<Location>,
92    #[serde(default)]
93    pub parsers: Option<Value>,
94    #[serde(default = "yes")]
95    pub remove_base64_images: bool,
96    #[serde(default = "yes")]
97    pub block_ads: bool,
98    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
99    #[serde(default)]
100    pub proxy: Option<String>,
101    /// PageSift extension: designated named egress (`[[egress]]`).
102    #[serde(default)]
103    pub egress: Option<String>,
104    /// TLS skip verification flag (PageSift always verifies certificates).
105    #[serde(default)]
106    pub skip_tls_verification: Option<bool>,
107    /// Milliseconds.
108    #[serde(default)]
109    pub max_age: Option<u64>,
110    /// Milliseconds.
111    #[serde(default)]
112    pub min_age: Option<u64>,
113    #[serde(default = "yes")]
114    pub store_in_cache: bool,
115
116    // ---- PageSift extensions ----
117    /// `http`, `browser`, `auto` (default).
118    #[serde(default)]
119    pub engine: Option<String>,
120    /// PageSift extension: bounded, stateless same-origin external JS discovery.
121    #[serde(default)]
122    pub discover_scripts: bool,
123    #[serde(default)]
124    pub if_none_match: Option<String>,
125    #[serde(default)]
126    pub if_modified_since: Option<String>,
127    /// Client-side robots.txt bypass flag.
128    #[serde(default)]
129    pub ignore_robots_txt: bool,
130    #[serde(default)]
131    pub store_snapshot: bool,
132    /// BM25 query filter on main content markdown.
133    #[serde(default)]
134    pub query_filter: Option<QueryFilter>,
135    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
136    #[serde(default, skip_serializing_if = "Option::is_none")]
137    pub chunking: Option<Value>,
138}
139
140/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
141#[derive(Debug, Clone, PartialEq, Serialize)]
142#[serde(rename_all = "camelCase")]
143pub struct QueryFilter {
144    pub query: String,
145    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub threshold: Option<f32>,
148    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub relative: Option<f32>,
151    /// Maximum number of blocks to keep.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub top_k: Option<usize>,
154    /// Token ceiling on retained content.
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub max_tokens: Option<usize>,
157    /// Token floor on retained content. Defaults to 1000.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub min_tokens: Option<usize>,
160    /// Prepend ancestor headings before preserved units.
161    pub headings: bool,
162}
163
164impl<'de> Deserialize<'de> for QueryFilter {
165    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
166        #[derive(Deserialize)]
167        #[serde(rename_all = "camelCase")]
168        struct Full {
169            query: String,
170            #[serde(default)]
171            threshold: Option<f32>,
172            #[serde(default)]
173            relative: Option<f32>,
174            #[serde(default)]
175            top_k: Option<usize>,
176            #[serde(default)]
177            max_tokens: Option<usize>,
178            #[serde(default)]
179            min_tokens: Option<usize>,
180            #[serde(default = "yes")]
181            headings: bool,
182        }
183        #[derive(Deserialize)]
184        #[serde(untagged)]
185        enum Repr {
186            Query(String),
187            Full(Full),
188        }
189        Ok(match Repr::deserialize(d)? {
190            Repr::Query(query) => QueryFilter {
191                query,
192                threshold: None,
193                relative: Some(pagesift_extract_defaults::RELATIVE),
194                top_k: None,
195                max_tokens: None,
196                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
197                headings: true,
198            },
199            Repr::Full(f) => QueryFilter {
200                query: f.query,
201                relative: f.relative.or(if f.threshold.is_none() {
202                    Some(pagesift_extract_defaults::RELATIVE)
203                } else {
204                    None
205                }),
206                threshold: f.threshold,
207                top_k: f.top_k,
208                max_tokens: f.max_tokens,
209                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
210                    Some(pagesift_extract_defaults::MIN_TOKENS)
211                } else {
212                    None
213                }),
214                headings: f.headings,
215            },
216        })
217    }
218}
219
220mod pagesift_extract_defaults {
221    pub const RELATIVE: f32 = 0.5;
222    pub const MIN_TOKENS: usize = 1000;
223}
224
225impl Default for ScrapeOptions {
226    fn default() -> Self {
227        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
228    }
229}
230
231impl ScrapeOptions {
232    pub fn wants(&self, kind: &str) -> bool {
233        self.formats.iter().any(|f| f.kind == kind)
234    }
235}
236
237#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
238pub struct ScrapeRequest {
239    pub url: String,
240    #[serde(flatten)]
241    pub options: ScrapeOptions,
242}
243
244#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
245#[serde(rename_all = "camelCase")]
246pub struct BatchScrapeRequest {
247    pub urls: Vec<String>,
248    #[serde(default = "yes")]
249    pub ignore_invalid_urls: bool,
250    #[serde(default)]
251    pub max_concurrency: Option<usize>,
252    #[serde(default)]
253    pub webhook: Option<Value>,
254    #[serde(flatten)]
255    pub options: ScrapeOptions,
256}
257
258fn default_crawl_limit() -> usize {
259    10_000
260}
261
262fn default_sitemap() -> String {
263    "include".into()
264}
265
266#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
267#[serde(rename_all = "camelCase")]
268pub struct CrawlRequest {
269    pub url: String,
270    #[serde(default)]
271    pub include_paths: Vec<String>,
272    #[serde(default)]
273    pub exclude_paths: Vec<String>,
274    #[serde(default)]
275    pub max_discovery_depth: Option<usize>,
276    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
277    #[serde(default, skip_serializing_if = "Option::is_none")]
278    pub max_depth: Option<u64>,
279    #[serde(default = "default_crawl_limit")]
280    pub limit: usize,
281    #[serde(default)]
282    pub crawl_entire_domain: bool,
283    #[serde(default)]
284    pub allow_external_links: bool,
285    /// 没给时：带 `query` 的爬取默认允许同一站点（可注册域名相同）的子域名，广度优先爬取不允许。
286    #[serde(default)]
287    pub allow_subdomains: Option<bool>,
288    #[serde(default)]
289    pub ignore_robots_txt: bool,
290    #[serde(default = "default_sitemap")]
291    pub sitemap: String,
292    #[serde(default)]
293    pub ignore_query_parameters: bool,
294    /// Seconds.
295    #[serde(default)]
296    pub delay: Option<f64>,
297    #[serde(default)]
298    pub max_concurrency: Option<usize>,
299    #[serde(default)]
300    pub scrape_options: ScrapeOptions,
301    #[serde(default)]
302    pub webhook: Option<Value>,
303
304    // ---- PageSift extensions ----
305    #[serde(default, deserialize_with = "query_or_keywords")]
306    pub query: Option<String>,
307    #[serde(default, deserialize_with = "adaptive_options")]
308    pub adaptive: Option<AdaptiveOptions>,
309    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
310    #[serde(default)]
311    pub scope: CrawlScope,
312    /// 带查询的爬取：相关页面上同站点的 PDF/Office 文档也抓回来解析（PageSift 扩展）。
313    #[serde(default)]
314    pub attachments: bool,
315    /// 附件的数量上限，默认等于 `limit`。
316    #[serde(default)]
317    pub attachment_limit: Option<usize>,
318}
319
320/// crawl 的路径范围。
321#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
322#[serde(rename_all = "lowercase")]
323pub enum CrawlScope {
324    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
325    #[default]
326    Auto,
327    /// 只取最窄目录，不放宽。
328    Exact,
329    /// 一开始就放宽到上一级目录（不会到整站根）。
330    Parent,
331    /// 整个主机，等同于 `crawlEntireDomain`。
332    Domain,
333}
334
335fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
336    #[derive(Deserialize)]
337    #[serde(untagged)]
338    enum Repr {
339        Query(String),
340        Keywords(Vec<String>),
341    }
342    Ok(match Option::<Repr>::deserialize(d)? {
343        Some(Repr::Query(q)) => Some(q),
344        Some(Repr::Keywords(k)) => Some(k.join(" ")),
345        None => None,
346    }
347    .filter(|q| !q.trim().is_empty()))
348}
349
350fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
351    #[derive(Deserialize)]
352    #[serde(untagged)]
353    enum Repr {
354        Flag(bool),
355        Options(AdaptiveOptions),
356    }
357    Ok(match Option::<Repr>::deserialize(d)? {
358        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
359        Some(Repr::Flag(false)) | None => None,
360        Some(Repr::Options(o)) => Some(o),
361    })
362}
363
364/// Adaptive crawl stopping conditions (PageSift extension `adaptive`).
365#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
366#[serde(rename_all = "camelCase", default)]
367pub struct AdaptiveOptions {
368    pub coverage_threshold: f64,
369    pub evidence_per_term: usize,
370    pub patience: usize,
371    pub min_gain: f64,
372}
373
374impl Default for AdaptiveOptions {
375    fn default() -> Self {
376        Self {
377            coverage_threshold: 0.9,
378            evidence_per_term: 3,
379            patience: 5,
380            min_gain: 0.05,
381        }
382    }
383}
384
385fn default_map_limit() -> usize {
386    5_000
387}
388
389#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
390#[serde(rename_all = "camelCase")]
391pub struct MapRequest {
392    pub url: String,
393    #[serde(default)]
394    pub search: Option<String>,
395    #[serde(default = "yes")]
396    pub include_subdomains: bool,
397    #[serde(default = "yes")]
398    pub ignore_query_parameters: bool,
399    #[serde(default = "default_sitemap")]
400    pub sitemap: String,
401    #[serde(default = "default_map_limit")]
402    pub limit: usize,
403    #[serde(default)]
404    pub ignore_robots_txt: bool,
405    /// Milliseconds.
406    #[serde(default)]
407    pub timeout: Option<u64>,
408}
409
410fn default_search_limit() -> usize {
411    10
412}
413
414/// `/v1/search` and `/v2/search`.
415#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
416#[serde(rename_all = "camelCase")]
417pub struct SearchRequest {
418    pub query: String,
419    #[serde(default = "default_search_limit")]
420    pub limit: usize,
421    #[serde(default)]
422    pub lang: Option<String>,
423    #[serde(default)]
424    pub country: Option<String>,
425    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
426    #[serde(default)]
427    pub tbs: Option<String>,
428    /// Milliseconds.
429    #[serde(default)]
430    pub timeout: Option<u64>,
431    #[serde(default)]
432    pub scrape_options: Option<ScrapeOptions>,
433}
434
435/// `/v1/extract` and `/v2/extract`.
436#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
437#[serde(rename_all = "camelCase")]
438pub struct ExtractRequest {
439    pub urls: Vec<String>,
440    #[serde(default)]
441    pub prompt: Option<String>,
442    #[serde(default)]
443    pub schema: Option<Value>,
444    #[serde(default)]
445    pub enable_web_search: bool,
446    #[serde(default)]
447    pub ignore_invalid_urls: bool,
448    #[serde(default)]
449    pub scrape_options: Option<ScrapeOptions>,
450}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
