pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields in requests are ignored for backward/forward compatibility.
4
5use std::collections::BTreeMap;
6
7use serde::{Deserialize, Deserializer, Serialize};
8use serde_json::{Map, Value};
9
10/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
11#[derive(Debug, Clone, PartialEq, Serialize)]
12pub struct Format {
13    #[serde(rename = "type")]
14    pub kind: String,
15    #[serde(flatten)]
16    pub options: Map<String, Value>,
17}
18
19impl Format {
20    pub fn new(kind: &str) -> Self {
21        Self {
22            kind: kind.to_string(),
23            options: Map::new(),
24        }
25    }
26}
27
28impl<'de> Deserialize<'de> for Format {
29    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
30        match Value::deserialize(d)? {
31            Value::String(kind) => Ok(Format::new(&kind)),
32            Value::Object(mut map) => {
33                let kind = match map.remove("type") {
34                    Some(Value::String(k)) => k,
35                    _ => {
36                        return Err(serde::de::Error::custom(
37                            "format object needs a string `type`",
38                        ));
39                    }
40                };
41                Ok(Format { kind, options: map })
42            }
43            _ => Err(serde::de::Error::custom(
44                "format must be a string or an object",
45            )),
46        }
47    }
48}
49
50pub(crate) fn default_formats() -> Vec<Format> {
51    vec![Format::new("markdown")]
52}
53
54pub(crate) fn yes() -> bool {
55    true
56}
57
58#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
59pub struct Location {
60    #[serde(default)]
61    pub country: Option<String>,
62    #[serde(default)]
63    pub languages: Vec<String>,
64}
65
66/// Single page scrape options (`scrapeOptions`).
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct ScrapeOptions {
70    #[serde(default = "default_formats")]
71    pub formats: Vec<Format>,
72    #[serde(default)]
73    pub headers: BTreeMap<String, String>,
74    #[serde(default)]
75    pub include_tags: Vec<String>,
76    #[serde(default)]
77    pub exclude_tags: Vec<String>,
78    #[serde(default = "yes")]
79    pub only_main_content: bool,
80    /// Milliseconds.
81    #[serde(default)]
82    pub timeout: Option<u64>,
83    /// Milliseconds.
84    #[serde(default)]
85    pub wait_for: u64,
86    #[serde(default)]
87    pub mobile: bool,
88    #[serde(default)]
89    pub actions: Vec<Value>,
90    #[serde(default)]
91    pub location: Option<Location>,
92    #[serde(default)]
93    pub parsers: Option<Value>,
94    #[serde(default = "yes")]
95    pub remove_base64_images: bool,
96    #[serde(default = "yes")]
97    pub block_ads: bool,
98    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
99    #[serde(default)]
100    pub proxy: Option<String>,
101    /// PageSift extension: designated named egress (`[[egress]]`).
102    #[serde(default)]
103    pub egress: Option<String>,
104    /// TLS skip verification flag (PageSift always verifies certificates).
105    #[serde(default)]
106    pub skip_tls_verification: Option<bool>,
107    /// Milliseconds.
108    #[serde(default)]
109    pub max_age: Option<u64>,
110    /// Milliseconds.
111    #[serde(default)]
112    pub min_age: Option<u64>,
113    #[serde(default = "yes")]
114    pub store_in_cache: bool,
115
116    // ---- PageSift extensions ----
117    /// `http`, `browser`, `auto` (default).
118    #[serde(default)]
119    pub engine: Option<String>,
120    /// PageSift extension: bounded, stateless same-origin external JS discovery.
121    #[serde(default)]
122    pub discover_scripts: bool,
123    /// PageSift extension: write how the result was produced (render and merge decisions with their
124    /// inputs) to `metadata.pagesift.plan`.
125    #[serde(default)]
126    pub explain: bool,
127    /// PageSift extension: export images as attachments — PDF embedded images in their original
128    /// encoding, Office media, and web page images (fetched politely, robots and SSRF rules apply).
129    #[serde(default)]
130    pub export_images: bool,
131    #[serde(default)]
132    pub if_none_match: Option<String>,
133    #[serde(default)]
134    pub if_modified_since: Option<String>,
135    /// Client-side robots.txt bypass flag.
136    #[serde(default)]
137    pub ignore_robots_txt: bool,
138    #[serde(default)]
139    pub store_snapshot: bool,
140    /// BM25 query filter on main content markdown.
141    #[serde(default)]
142    pub query_filter: Option<QueryFilter>,
143    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
144    #[serde(default, skip_serializing_if = "Option::is_none")]
145    pub chunking: Option<Value>,
146}
147
148/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
149#[derive(Debug, Clone, PartialEq, Serialize)]
150#[serde(rename_all = "camelCase")]
151pub struct QueryFilter {
152    pub query: String,
153    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub threshold: Option<f32>,
156    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
157    #[serde(skip_serializing_if = "Option::is_none")]
158    pub relative: Option<f32>,
159    /// Maximum number of blocks to keep.
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub top_k: Option<usize>,
162    /// Token ceiling on retained content.
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub max_tokens: Option<usize>,
165    /// Token floor on retained content. Defaults to 1000.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub min_tokens: Option<usize>,
168    /// Prepend ancestor headings before preserved units.
169    pub headings: bool,
170}
171
172impl<'de> Deserialize<'de> for QueryFilter {
173    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
174        #[derive(Deserialize)]
175        #[serde(rename_all = "camelCase")]
176        struct Full {
177            query: String,
178            #[serde(default)]
179            threshold: Option<f32>,
180            #[serde(default)]
181            relative: Option<f32>,
182            #[serde(default)]
183            top_k: Option<usize>,
184            #[serde(default)]
185            max_tokens: Option<usize>,
186            #[serde(default)]
187            min_tokens: Option<usize>,
188            #[serde(default = "yes")]
189            headings: bool,
190        }
191        #[derive(Deserialize)]
192        #[serde(untagged)]
193        enum Repr {
194            Query(String),
195            Full(Full),
196        }
197        Ok(match Repr::deserialize(d)? {
198            Repr::Query(query) => QueryFilter {
199                query,
200                threshold: None,
201                relative: Some(pagesift_extract_defaults::RELATIVE),
202                top_k: None,
203                max_tokens: None,
204                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
205                headings: true,
206            },
207            Repr::Full(f) => QueryFilter {
208                query: f.query,
209                relative: f.relative.or(if f.threshold.is_none() {
210                    Some(pagesift_extract_defaults::RELATIVE)
211                } else {
212                    None
213                }),
214                threshold: f.threshold,
215                top_k: f.top_k,
216                max_tokens: f.max_tokens,
217                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
218                    Some(pagesift_extract_defaults::MIN_TOKENS)
219                } else {
220                    None
221                }),
222                headings: f.headings,
223            },
224        })
225    }
226}
227
228mod pagesift_extract_defaults {
229    pub const RELATIVE: f32 = 0.5;
230    pub const MIN_TOKENS: usize = 1000;
231}
232
233impl Default for ScrapeOptions {
234    fn default() -> Self {
235        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
236    }
237}
238
239impl ScrapeOptions {
240    pub fn wants(&self, kind: &str) -> bool {
241        self.formats.iter().any(|f| f.kind == kind)
242    }
243}
244
245#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
246pub struct ScrapeRequest {
247    pub url: String,
248    #[serde(flatten)]
249    pub options: ScrapeOptions,
250}
251
252#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
253#[serde(rename_all = "camelCase")]
254pub struct BatchScrapeRequest {
255    pub urls: Vec<String>,
256    #[serde(default = "yes")]
257    pub ignore_invalid_urls: bool,
258    #[serde(default)]
259    pub max_concurrency: Option<usize>,
260    #[serde(default)]
261    pub webhook: Option<Value>,
262    #[serde(flatten)]
263    pub options: ScrapeOptions,
264}
265
266fn default_crawl_limit() -> usize {
267    10_000
268}
269
270fn default_sitemap() -> String {
271    "include".into()
272}
273
274#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
275#[serde(rename_all = "camelCase")]
276pub struct CrawlRequest {
277    pub url: String,
278    #[serde(default)]
279    pub include_paths: Vec<String>,
280    #[serde(default)]
281    pub exclude_paths: Vec<String>,
282    #[serde(default)]
283    pub max_discovery_depth: Option<usize>,
284    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
285    #[serde(default, skip_serializing_if = "Option::is_none")]
286    pub max_depth: Option<u64>,
287    #[serde(default = "default_crawl_limit")]
288    pub limit: usize,
289    #[serde(default)]
290    pub crawl_entire_domain: bool,
291    #[serde(default)]
292    pub allow_external_links: bool,
293    /// 没给时：带 `query` 的爬取默认允许同一站点（可注册域名相同）的子域名，广度优先爬取不允许。
294    #[serde(default)]
295    pub allow_subdomains: Option<bool>,
296    #[serde(default)]
297    pub ignore_robots_txt: bool,
298    #[serde(default = "default_sitemap")]
299    pub sitemap: String,
300    #[serde(default)]
301    pub ignore_query_parameters: bool,
302    /// Seconds.
303    #[serde(default)]
304    pub delay: Option<f64>,
305    #[serde(default)]
306    pub max_concurrency: Option<usize>,
307    #[serde(default)]
308    pub scrape_options: ScrapeOptions,
309    #[serde(default)]
310    pub webhook: Option<Value>,
311
312    // ---- PageSift extensions ----
313    #[serde(default, deserialize_with = "query_or_keywords")]
314    pub query: Option<String>,
315    #[serde(default, deserialize_with = "adaptive_options")]
316    pub adaptive: Option<AdaptiveOptions>,
317    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
318    /// 不填时普通爬取是 `auto`，带查询的爬取是 `domain`（相关性打分决定往哪走，路径前缀只会挡路：
319    /// 首页跳到 `/top/` 的站点，招生页在 `/inst/` 下）。
320    #[serde(default)]
321    pub scope: Option<CrawlScope>,
322    /// 带查询的爬取：相关页面上同站点的 PDF/Office 文档也抓回来解析（PageSift 扩展）。
323    #[serde(default)]
324    pub attachments: bool,
325    /// 附件的数量上限，默认等于 `limit`。
326    #[serde(default)]
327    pub attachment_limit: Option<usize>,
328}
329
330/// crawl 的路径范围。
331#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
332#[serde(rename_all = "lowercase")]
333pub enum CrawlScope {
334    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
335    #[default]
336    Auto,
337    /// 只取最窄目录，不放宽。
338    Exact,
339    /// 一开始就放宽到上一级目录（不会到整站根）。
340    Parent,
341    /// 整个主机，等同于 `crawlEntireDomain`。
342    Domain,
343}
344
345fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
346    #[derive(Deserialize)]
347    #[serde(untagged)]
348    enum Repr {
349        Query(String),
350        Keywords(Vec<String>),
351    }
352    Ok(match Option::<Repr>::deserialize(d)? {
353        Some(Repr::Query(q)) => Some(q),
354        Some(Repr::Keywords(k)) => Some(k.join(" ")),
355        None => None,
356    }
357    .filter(|q| !q.trim().is_empty()))
358}
359
360fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
361    #[derive(Deserialize)]
362    #[serde(untagged)]
363    enum Repr {
364        Flag(bool),
365        Options(AdaptiveOptions),
366    }
367    Ok(match Option::<Repr>::deserialize(d)? {
368        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
369        Some(Repr::Flag(false)) | None => None,
370        Some(Repr::Options(o)) => Some(o),
371    })
372}
373
374/// Adaptive crawl stopping conditions (PageSift extension `adaptive`). With them the crawl decides
375/// how many pages it needs; `limit` is only a cap.
376#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
377#[serde(rename_all = "camelCase", default)]
378pub struct AdaptiveOptions {
379    /// Share of query terms that must have `evidencePerTerm` pieces of evidence (from at least two
380    /// pages) before the query counts as covered. Once covered, the crawl stops after
381    /// `ceil(patience / 2)` pages in a row that add nothing new (`coverage_reached`).
382    pub coverage_threshold: f64,
383    pub evidence_per_term: usize,
384    /// Pages in a row without new relevant content before the crawl stops (`saturated`). While
385    /// nothing relevant has been found, `2 × patience` pages in a row reached only through links
386    /// without any sign of the query stop it (`no_relevant_pages`).
387    pub patience: usize,
388    /// A page whose new relevant content is less than this share of everything collected so far
389    /// counts as adding nothing new.
390    pub min_gain: f64,
391}
392
393impl Default for AdaptiveOptions {
394    fn default() -> Self {
395        // 请求参数是用户策略，这里是不填时的默认值：
396        Self {
397            // [策略] 先验包 v0：0.81 的现值。覆盖度只决定"覆盖后耐心减半"，不单独停止爬取
398            coverage_threshold: 0.9,
399            // [策略] 先验包 v0：0.81 的现值（每个词要三段不同的证据，单段可能是导航、转载）
400            evidence_per_term: 3,
401            // [策略] 先验包 v0：0.81 的现值；连续这么多页没有新内容才停
402            patience: 5,
403            // [δ] 新内容不到已收集总量的 δ 算"没有新信息"：低于全局风险水平的增量当作噪声
404            // （δ 默认 0.05，与 0.81 的值相同）
405            min_gain: 0.05,
406        }
407    }
408}
409
410fn default_map_limit() -> usize {
411    5_000
412}
413
414#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
415#[serde(rename_all = "camelCase")]
416pub struct MapRequest {
417    pub url: String,
418    #[serde(default)]
419    pub search: Option<String>,
420    #[serde(default = "yes")]
421    pub include_subdomains: bool,
422    #[serde(default = "yes")]
423    pub ignore_query_parameters: bool,
424    #[serde(default = "default_sitemap")]
425    pub sitemap: String,
426    #[serde(default = "default_map_limit")]
427    pub limit: usize,
428    #[serde(default)]
429    pub ignore_robots_txt: bool,
430    /// Milliseconds.
431    #[serde(default)]
432    pub timeout: Option<u64>,
433}
434
435fn default_search_limit() -> usize {
436    10
437}
438
439/// `/v1/search` and `/v2/search`.
440#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
441#[serde(rename_all = "camelCase")]
442pub struct SearchRequest {
443    pub query: String,
444    #[serde(default = "default_search_limit")]
445    pub limit: usize,
446    #[serde(default)]
447    pub lang: Option<String>,
448    #[serde(default)]
449    pub country: Option<String>,
450    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
451    #[serde(default)]
452    pub tbs: Option<String>,
453    /// Milliseconds.
454    #[serde(default)]
455    pub timeout: Option<u64>,
456    #[serde(default)]
457    pub scrape_options: Option<ScrapeOptions>,
458}
459
460/// `/v1/extract` and `/v2/extract`.
461#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
462#[serde(rename_all = "camelCase")]
463pub struct ExtractRequest {
464    pub urls: Vec<String>,
465    #[serde(default)]
466    pub prompt: Option<String>,
467    #[serde(default)]
468    pub schema: Option<Value>,
469    #[serde(default)]
470    pub enable_web_search: bool,
471    #[serde(default)]
472    pub ignore_invalid_urls: bool,
473    #[serde(default)]
474    pub scrape_options: Option<ScrapeOptions>,
475}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
