pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields in requests are ignored for backward/forward compatibility.
4
5use std::collections::BTreeMap;
6
7use serde::{Deserialize, Deserializer, Serialize};
8use serde_json::{Map, Value};
9
10/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
11#[derive(Debug, Clone, PartialEq, Serialize)]
12pub struct Format {
13    #[serde(rename = "type")]
14    pub kind: String,
15    #[serde(flatten)]
16    pub options: Map<String, Value>,
17}
18
19impl Format {
20    pub fn new(kind: &str) -> Self {
21        Self {
22            kind: kind.to_string(),
23            options: Map::new(),
24        }
25    }
26}
27
28impl<'de> Deserialize<'de> for Format {
29    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
30        match Value::deserialize(d)? {
31            Value::String(kind) => Ok(Format::new(&kind)),
32            Value::Object(mut map) => {
33                let kind = match map.remove("type") {
34                    Some(Value::String(k)) => k,
35                    _ => {
36                        return Err(serde::de::Error::custom(
37                            "format object needs a string `type`",
38                        ));
39                    }
40                };
41                Ok(Format { kind, options: map })
42            }
43            _ => Err(serde::de::Error::custom(
44                "format must be a string or an object",
45            )),
46        }
47    }
48}
49
50pub(crate) fn default_formats() -> Vec<Format> {
51    vec![Format::new("markdown")]
52}
53
54pub(crate) fn yes() -> bool {
55    true
56}
57
58#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
59pub struct Location {
60    #[serde(default)]
61    pub country: Option<String>,
62    #[serde(default)]
63    pub languages: Vec<String>,
64}
65
66/// Single page scrape options (`scrapeOptions`).
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct ScrapeOptions {
70    #[serde(default = "default_formats")]
71    pub formats: Vec<Format>,
72    #[serde(default)]
73    pub headers: BTreeMap<String, String>,
74    #[serde(default)]
75    pub include_tags: Vec<String>,
76    #[serde(default)]
77    pub exclude_tags: Vec<String>,
78    #[serde(default = "yes")]
79    pub only_main_content: bool,
80    /// Milliseconds.
81    #[serde(default)]
82    pub timeout: Option<u64>,
83    /// Milliseconds.
84    #[serde(default)]
85    pub wait_for: u64,
86    #[serde(default)]
87    pub mobile: bool,
88    #[serde(default)]
89    pub actions: Vec<Value>,
90    #[serde(default)]
91    pub location: Option<Location>,
92    #[serde(default)]
93    pub parsers: Option<Value>,
94    #[serde(default = "yes")]
95    pub remove_base64_images: bool,
96    #[serde(default = "yes")]
97    pub block_ads: bool,
98    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
99    #[serde(default)]
100    pub proxy: Option<String>,
101    /// PageSift extension: designated named egress (`[[egress]]`).
102    #[serde(default)]
103    pub egress: Option<String>,
104    /// TLS skip verification flag (PageSift always verifies certificates).
105    #[serde(default)]
106    pub skip_tls_verification: Option<bool>,
107    /// Milliseconds.
108    #[serde(default)]
109    pub max_age: Option<u64>,
110    /// Milliseconds.
111    #[serde(default)]
112    pub min_age: Option<u64>,
113    #[serde(default = "yes")]
114    pub store_in_cache: bool,
115
116    // ---- PageSift extensions ----
117    /// `http`, `browser`, `auto` (default).
118    #[serde(default)]
119    pub engine: Option<String>,
120    /// PageSift extension: bounded, stateless same-origin external JS discovery.
121    #[serde(default)]
122    pub discover_scripts: bool,
123    #[serde(default)]
124    pub if_none_match: Option<String>,
125    #[serde(default)]
126    pub if_modified_since: Option<String>,
127    /// Client-side robots.txt bypass flag.
128    #[serde(default)]
129    pub ignore_robots_txt: bool,
130    #[serde(default)]
131    pub store_snapshot: bool,
132    /// BM25 query filter on main content markdown.
133    #[serde(default)]
134    pub query_filter: Option<QueryFilter>,
135    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
136    #[serde(default, skip_serializing_if = "Option::is_none")]
137    pub chunking: Option<Value>,
138}
139
140/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
141#[derive(Debug, Clone, PartialEq, Serialize)]
142#[serde(rename_all = "camelCase")]
143pub struct QueryFilter {
144    pub query: String,
145    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub threshold: Option<f32>,
148    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub relative: Option<f32>,
151    /// Maximum number of blocks to keep.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub top_k: Option<usize>,
154    /// Token ceiling on retained content.
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub max_tokens: Option<usize>,
157    /// Token floor on retained content. Defaults to 1000.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub min_tokens: Option<usize>,
160    /// Prepend ancestor headings before preserved units.
161    pub headings: bool,
162}
163
164impl<'de> Deserialize<'de> for QueryFilter {
165    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
166        #[derive(Deserialize)]
167        #[serde(rename_all = "camelCase")]
168        struct Full {
169            query: String,
170            #[serde(default)]
171            threshold: Option<f32>,
172            #[serde(default)]
173            relative: Option<f32>,
174            #[serde(default)]
175            top_k: Option<usize>,
176            #[serde(default)]
177            max_tokens: Option<usize>,
178            #[serde(default)]
179            min_tokens: Option<usize>,
180            #[serde(default = "yes")]
181            headings: bool,
182        }
183        #[derive(Deserialize)]
184        #[serde(untagged)]
185        enum Repr {
186            Query(String),
187            Full(Full),
188        }
189        Ok(match Repr::deserialize(d)? {
190            Repr::Query(query) => QueryFilter {
191                query,
192                threshold: None,
193                relative: Some(pagesift_extract_defaults::RELATIVE),
194                top_k: None,
195                max_tokens: None,
196                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
197                headings: true,
198            },
199            Repr::Full(f) => QueryFilter {
200                query: f.query,
201                relative: f.relative.or(if f.threshold.is_none() {
202                    Some(pagesift_extract_defaults::RELATIVE)
203                } else {
204                    None
205                }),
206                threshold: f.threshold,
207                top_k: f.top_k,
208                max_tokens: f.max_tokens,
209                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
210                    Some(pagesift_extract_defaults::MIN_TOKENS)
211                } else {
212                    None
213                }),
214                headings: f.headings,
215            },
216        })
217    }
218}
219
220mod pagesift_extract_defaults {
221    pub const RELATIVE: f32 = 0.5;
222    pub const MIN_TOKENS: usize = 1000;
223}
224
225impl Default for ScrapeOptions {
226    fn default() -> Self {
227        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
228    }
229}
230
231impl ScrapeOptions {
232    pub fn wants(&self, kind: &str) -> bool {
233        self.formats.iter().any(|f| f.kind == kind)
234    }
235}
236
237#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
238pub struct ScrapeRequest {
239    pub url: String,
240    #[serde(flatten)]
241    pub options: ScrapeOptions,
242}
243
244#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
245#[serde(rename_all = "camelCase")]
246pub struct BatchScrapeRequest {
247    pub urls: Vec<String>,
248    #[serde(default = "yes")]
249    pub ignore_invalid_urls: bool,
250    #[serde(default)]
251    pub max_concurrency: Option<usize>,
252    #[serde(default)]
253    pub webhook: Option<Value>,
254    #[serde(flatten)]
255    pub options: ScrapeOptions,
256}
257
258fn default_crawl_limit() -> usize {
259    10_000
260}
261
262fn default_sitemap() -> String {
263    "include".into()
264}
265
266#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
267#[serde(rename_all = "camelCase")]
268pub struct CrawlRequest {
269    pub url: String,
270    #[serde(default)]
271    pub include_paths: Vec<String>,
272    #[serde(default)]
273    pub exclude_paths: Vec<String>,
274    #[serde(default)]
275    pub max_discovery_depth: Option<usize>,
276    #[serde(default = "default_crawl_limit")]
277    pub limit: usize,
278    #[serde(default)]
279    pub crawl_entire_domain: bool,
280    #[serde(default)]
281    pub allow_external_links: bool,
282    #[serde(default)]
283    pub allow_subdomains: bool,
284    #[serde(default)]
285    pub ignore_robots_txt: bool,
286    #[serde(default = "default_sitemap")]
287    pub sitemap: String,
288    #[serde(default)]
289    pub ignore_query_parameters: bool,
290    /// Seconds.
291    #[serde(default)]
292    pub delay: Option<f64>,
293    #[serde(default)]
294    pub max_concurrency: Option<usize>,
295    #[serde(default)]
296    pub scrape_options: ScrapeOptions,
297    #[serde(default)]
298    pub webhook: Option<Value>,
299
300    // ---- PageSift extensions ----
301    #[serde(default, deserialize_with = "query_or_keywords")]
302    pub query: Option<String>,
303    #[serde(default, deserialize_with = "adaptive_options")]
304    pub adaptive: Option<AdaptiveOptions>,
305}
306
307fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
308    #[derive(Deserialize)]
309    #[serde(untagged)]
310    enum Repr {
311        Query(String),
312        Keywords(Vec<String>),
313    }
314    Ok(match Option::<Repr>::deserialize(d)? {
315        Some(Repr::Query(q)) => Some(q),
316        Some(Repr::Keywords(k)) => Some(k.join(" ")),
317        None => None,
318    }
319    .filter(|q| !q.trim().is_empty()))
320}
321
322fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
323    #[derive(Deserialize)]
324    #[serde(untagged)]
325    enum Repr {
326        Flag(bool),
327        Options(AdaptiveOptions),
328    }
329    Ok(match Option::<Repr>::deserialize(d)? {
330        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
331        Some(Repr::Flag(false)) | None => None,
332        Some(Repr::Options(o)) => Some(o),
333    })
334}
335
336/// Adaptive crawl stopping conditions (PageSift extension `adaptive`).
337#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
338#[serde(rename_all = "camelCase", default)]
339pub struct AdaptiveOptions {
340    pub coverage_threshold: f64,
341    pub evidence_per_term: usize,
342    pub patience: usize,
343    pub min_gain: f64,
344}
345
346impl Default for AdaptiveOptions {
347    fn default() -> Self {
348        Self {
349            coverage_threshold: 0.9,
350            evidence_per_term: 3,
351            patience: 5,
352            min_gain: 0.05,
353        }
354    }
355}
356
357fn default_map_limit() -> usize {
358    5_000
359}
360
361#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
362#[serde(rename_all = "camelCase")]
363pub struct MapRequest {
364    pub url: String,
365    #[serde(default)]
366    pub search: Option<String>,
367    #[serde(default = "yes")]
368    pub include_subdomains: bool,
369    #[serde(default = "yes")]
370    pub ignore_query_parameters: bool,
371    #[serde(default = "default_sitemap")]
372    pub sitemap: String,
373    #[serde(default = "default_map_limit")]
374    pub limit: usize,
375    #[serde(default)]
376    pub ignore_robots_txt: bool,
377    /// Milliseconds.
378    #[serde(default)]
379    pub timeout: Option<u64>,
380}
381
382fn default_search_limit() -> usize {
383    10
384}
385
386/// `/v1/search` and `/v2/search`.
387#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
388#[serde(rename_all = "camelCase")]
389pub struct SearchRequest {
390    pub query: String,
391    #[serde(default = "default_search_limit")]
392    pub limit: usize,
393    #[serde(default)]
394    pub lang: Option<String>,
395    #[serde(default)]
396    pub country: Option<String>,
397    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
398    #[serde(default)]
399    pub tbs: Option<String>,
400    /// Milliseconds.
401    #[serde(default)]
402    pub timeout: Option<u64>,
403    #[serde(default)]
404    pub scrape_options: Option<ScrapeOptions>,
405}
406
407/// `/v1/extract` and `/v2/extract`.
408#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
409#[serde(rename_all = "camelCase")]
410pub struct ExtractRequest {
411    pub urls: Vec<String>,
412    #[serde(default)]
413    pub prompt: Option<String>,
414    #[serde(default)]
415    pub schema: Option<Value>,
416    #[serde(default)]
417    pub enable_web_search: bool,
418    #[serde(default)]
419    pub ignore_invalid_urls: bool,
420    #[serde(default)]
421    pub scrape_options: Option<ScrapeOptions>,
422}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
