pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields in requests are ignored for backward/forward compatibility.
4
5use std::collections::BTreeMap;
6
7use serde::{Deserialize, Deserializer, Serialize};
8use serde_json::{Map, Value};
9
10/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
11#[derive(Debug, Clone, PartialEq, Serialize)]
12pub struct Format {
13    #[serde(rename = "type")]
14    pub kind: String,
15    #[serde(flatten)]
16    pub options: Map<String, Value>,
17}
18
19impl Format {
20    pub fn new(kind: &str) -> Self {
21        Self {
22            kind: kind.to_string(),
23            options: Map::new(),
24        }
25    }
26}
27
28impl<'de> Deserialize<'de> for Format {
29    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
30        match Value::deserialize(d)? {
31            Value::String(kind) => Ok(Format::new(&kind)),
32            Value::Object(mut map) => {
33                let kind = match map.remove("type") {
34                    Some(Value::String(k)) => k,
35                    _ => {
36                        return Err(serde::de::Error::custom(
37                            "format object needs a string `type`",
38                        ));
39                    }
40                };
41                Ok(Format { kind, options: map })
42            }
43            _ => Err(serde::de::Error::custom(
44                "format must be a string or an object",
45            )),
46        }
47    }
48}
49
50pub(crate) fn default_formats() -> Vec<Format> {
51    vec![Format::new("markdown")]
52}
53
54pub(crate) fn yes() -> bool {
55    true
56}
57
58#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
59pub struct Location {
60    #[serde(default)]
61    pub country: Option<String>,
62    #[serde(default)]
63    pub languages: Vec<String>,
64}
65
66/// Single page scrape options (`scrapeOptions`).
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct ScrapeOptions {
70    #[serde(default = "default_formats")]
71    pub formats: Vec<Format>,
72    #[serde(default)]
73    pub headers: BTreeMap<String, String>,
74    #[serde(default)]
75    pub include_tags: Vec<String>,
76    #[serde(default)]
77    pub exclude_tags: Vec<String>,
78    #[serde(default = "yes")]
79    pub only_main_content: bool,
80    /// Milliseconds.
81    #[serde(default)]
82    pub timeout: Option<u64>,
83    /// Milliseconds.
84    #[serde(default)]
85    pub wait_for: u64,
86    #[serde(default)]
87    pub mobile: bool,
88    #[serde(default)]
89    pub actions: Vec<Value>,
90    #[serde(default)]
91    pub location: Option<Location>,
92    #[serde(default)]
93    pub parsers: Option<Value>,
94    #[serde(default = "yes")]
95    pub remove_base64_images: bool,
96    #[serde(default = "yes")]
97    pub block_ads: bool,
98    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
99    #[serde(default)]
100    pub proxy: Option<String>,
101    /// PageSift extension: designated named egress (`[[egress]]`).
102    #[serde(default)]
103    pub egress: Option<String>,
104    /// TLS skip verification flag (PageSift always verifies certificates).
105    #[serde(default)]
106    pub skip_tls_verification: Option<bool>,
107    /// Milliseconds.
108    #[serde(default)]
109    pub max_age: Option<u64>,
110    /// Milliseconds.
111    #[serde(default)]
112    pub min_age: Option<u64>,
113    #[serde(default = "yes")]
114    pub store_in_cache: bool,
115
116    // ---- PageSift extensions ----
117    /// `http`, `browser`, `auto` (default).
118    #[serde(default)]
119    pub engine: Option<String>,
120    /// PageSift extension: bounded, stateless same-origin external JS discovery.
121    #[serde(default)]
122    pub discover_scripts: bool,
123    #[serde(default)]
124    pub if_none_match: Option<String>,
125    #[serde(default)]
126    pub if_modified_since: Option<String>,
127    /// Client-side robots.txt bypass flag.
128    #[serde(default)]
129    pub ignore_robots_txt: bool,
130    #[serde(default)]
131    pub store_snapshot: bool,
132    /// BM25 query filter on main content markdown.
133    #[serde(default)]
134    pub query_filter: Option<QueryFilter>,
135    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
136    #[serde(default, skip_serializing_if = "Option::is_none")]
137    pub chunking: Option<Value>,
138}
139
140/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
141#[derive(Debug, Clone, PartialEq, Serialize)]
142#[serde(rename_all = "camelCase")]
143pub struct QueryFilter {
144    pub query: String,
145    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub threshold: Option<f32>,
148    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub relative: Option<f32>,
151    /// Maximum number of blocks to keep.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub top_k: Option<usize>,
154    /// Token ceiling on retained content.
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub max_tokens: Option<usize>,
157    /// Token floor on retained content. Defaults to 1000.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub min_tokens: Option<usize>,
160    /// Prepend ancestor headings before preserved units.
161    pub headings: bool,
162}
163
164impl<'de> Deserialize<'de> for QueryFilter {
165    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
166        #[derive(Deserialize)]
167        #[serde(rename_all = "camelCase")]
168        struct Full {
169            query: String,
170            #[serde(default)]
171            threshold: Option<f32>,
172            #[serde(default)]
173            relative: Option<f32>,
174            #[serde(default)]
175            top_k: Option<usize>,
176            #[serde(default)]
177            max_tokens: Option<usize>,
178            #[serde(default)]
179            min_tokens: Option<usize>,
180            #[serde(default = "yes")]
181            headings: bool,
182        }
183        #[derive(Deserialize)]
184        #[serde(untagged)]
185        enum Repr {
186            Query(String),
187            Full(Full),
188        }
189        Ok(match Repr::deserialize(d)? {
190            Repr::Query(query) => QueryFilter {
191                query,
192                threshold: None,
193                relative: Some(pagesift_extract_defaults::RELATIVE),
194                top_k: None,
195                max_tokens: None,
196                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
197                headings: true,
198            },
199            Repr::Full(f) => QueryFilter {
200                query: f.query,
201                relative: f.relative.or(if f.threshold.is_none() {
202                    Some(pagesift_extract_defaults::RELATIVE)
203                } else {
204                    None
205                }),
206                threshold: f.threshold,
207                top_k: f.top_k,
208                max_tokens: f.max_tokens,
209                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
210                    Some(pagesift_extract_defaults::MIN_TOKENS)
211                } else {
212                    None
213                }),
214                headings: f.headings,
215            },
216        })
217    }
218}
219
220mod pagesift_extract_defaults {
221    pub const RELATIVE: f32 = 0.5;
222    pub const MIN_TOKENS: usize = 1000;
223}
224
225impl Default for ScrapeOptions {
226    fn default() -> Self {
227        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
228    }
229}
230
231impl ScrapeOptions {
232    pub fn wants(&self, kind: &str) -> bool {
233        self.formats.iter().any(|f| f.kind == kind)
234    }
235}
236
237#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
238pub struct ScrapeRequest {
239    pub url: String,
240    #[serde(flatten)]
241    pub options: ScrapeOptions,
242}
243
244#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
245#[serde(rename_all = "camelCase")]
246pub struct BatchScrapeRequest {
247    pub urls: Vec<String>,
248    #[serde(default = "yes")]
249    pub ignore_invalid_urls: bool,
250    #[serde(default)]
251    pub max_concurrency: Option<usize>,
252    #[serde(default)]
253    pub webhook: Option<Value>,
254    #[serde(flatten)]
255    pub options: ScrapeOptions,
256}
257
258fn default_crawl_limit() -> usize {
259    10_000
260}
261
262fn default_sitemap() -> String {
263    "include".into()
264}
265
266#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
267#[serde(rename_all = "camelCase")]
268pub struct CrawlRequest {
269    pub url: String,
270    #[serde(default)]
271    pub include_paths: Vec<String>,
272    #[serde(default)]
273    pub exclude_paths: Vec<String>,
274    #[serde(default)]
275    pub max_discovery_depth: Option<usize>,
276    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
277    #[serde(default, skip_serializing_if = "Option::is_none")]
278    pub max_depth: Option<u64>,
279    #[serde(default = "default_crawl_limit")]
280    pub limit: usize,
281    #[serde(default)]
282    pub crawl_entire_domain: bool,
283    #[serde(default)]
284    pub allow_external_links: bool,
285    #[serde(default)]
286    pub allow_subdomains: bool,
287    #[serde(default)]
288    pub ignore_robots_txt: bool,
289    #[serde(default = "default_sitemap")]
290    pub sitemap: String,
291    #[serde(default)]
292    pub ignore_query_parameters: bool,
293    /// Seconds.
294    #[serde(default)]
295    pub delay: Option<f64>,
296    #[serde(default)]
297    pub max_concurrency: Option<usize>,
298    #[serde(default)]
299    pub scrape_options: ScrapeOptions,
300    #[serde(default)]
301    pub webhook: Option<Value>,
302
303    // ---- PageSift extensions ----
304    #[serde(default, deserialize_with = "query_or_keywords")]
305    pub query: Option<String>,
306    #[serde(default, deserialize_with = "adaptive_options")]
307    pub adaptive: Option<AdaptiveOptions>,
308    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
309    #[serde(default)]
310    pub scope: CrawlScope,
311}
312
313/// crawl 的路径范围。
314#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
315#[serde(rename_all = "lowercase")]
316pub enum CrawlScope {
317    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
318    #[default]
319    Auto,
320    /// 只取最窄目录，不放宽。
321    Exact,
322    /// 一开始就放宽到上一级目录（不会到整站根）。
323    Parent,
324    /// 整个主机，等同于 `crawlEntireDomain`。
325    Domain,
326}
327
328fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
329    #[derive(Deserialize)]
330    #[serde(untagged)]
331    enum Repr {
332        Query(String),
333        Keywords(Vec<String>),
334    }
335    Ok(match Option::<Repr>::deserialize(d)? {
336        Some(Repr::Query(q)) => Some(q),
337        Some(Repr::Keywords(k)) => Some(k.join(" ")),
338        None => None,
339    }
340    .filter(|q| !q.trim().is_empty()))
341}
342
343fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
344    #[derive(Deserialize)]
345    #[serde(untagged)]
346    enum Repr {
347        Flag(bool),
348        Options(AdaptiveOptions),
349    }
350    Ok(match Option::<Repr>::deserialize(d)? {
351        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
352        Some(Repr::Flag(false)) | None => None,
353        Some(Repr::Options(o)) => Some(o),
354    })
355}
356
357/// Adaptive crawl stopping conditions (PageSift extension `adaptive`).
358#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
359#[serde(rename_all = "camelCase", default)]
360pub struct AdaptiveOptions {
361    pub coverage_threshold: f64,
362    pub evidence_per_term: usize,
363    pub patience: usize,
364    pub min_gain: f64,
365}
366
367impl Default for AdaptiveOptions {
368    fn default() -> Self {
369        Self {
370            coverage_threshold: 0.9,
371            evidence_per_term: 3,
372            patience: 5,
373            min_gain: 0.05,
374        }
375    }
376}
377
378fn default_map_limit() -> usize {
379    5_000
380}
381
382#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
383#[serde(rename_all = "camelCase")]
384pub struct MapRequest {
385    pub url: String,
386    #[serde(default)]
387    pub search: Option<String>,
388    #[serde(default = "yes")]
389    pub include_subdomains: bool,
390    #[serde(default = "yes")]
391    pub ignore_query_parameters: bool,
392    #[serde(default = "default_sitemap")]
393    pub sitemap: String,
394    #[serde(default = "default_map_limit")]
395    pub limit: usize,
396    #[serde(default)]
397    pub ignore_robots_txt: bool,
398    /// Milliseconds.
399    #[serde(default)]
400    pub timeout: Option<u64>,
401}
402
403fn default_search_limit() -> usize {
404    10
405}
406
407/// `/v1/search` and `/v2/search`.
408#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
409#[serde(rename_all = "camelCase")]
410pub struct SearchRequest {
411    pub query: String,
412    #[serde(default = "default_search_limit")]
413    pub limit: usize,
414    #[serde(default)]
415    pub lang: Option<String>,
416    #[serde(default)]
417    pub country: Option<String>,
418    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
419    #[serde(default)]
420    pub tbs: Option<String>,
421    /// Milliseconds.
422    #[serde(default)]
423    pub timeout: Option<u64>,
424    #[serde(default)]
425    pub scrape_options: Option<ScrapeOptions>,
426}
427
428/// `/v1/extract` and `/v2/extract`.
429#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
430#[serde(rename_all = "camelCase")]
431pub struct ExtractRequest {
432    pub urls: Vec<String>,
433    #[serde(default)]
434    pub prompt: Option<String>,
435    #[serde(default)]
436    pub schema: Option<Value>,
437    #[serde(default)]
438    pub enable_web_search: bool,
439    #[serde(default)]
440    pub ignore_invalid_urls: bool,
441    #[serde(default)]
442    pub scrape_options: Option<ScrapeOptions>,
443}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
