pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields are accepted by the Firecrawl-compatible routes and listed in `warnings`
4//! (`ignored_fields`, see [`crate::fields`]); pagesift's own entry points reject them.
5
6use std::collections::BTreeMap;
7
8use serde::{Deserialize, Deserializer, Serialize};
9use serde_json::{Map, Value};
10
11/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
12#[derive(Debug, Clone, PartialEq, Serialize)]
13pub struct Format {
14    #[serde(rename = "type")]
15    pub kind: String,
16    #[serde(flatten)]
17    pub options: Map<String, Value>,
18}
19
20impl Format {
21    pub fn new(kind: &str) -> Self {
22        Self {
23            kind: kind.to_string(),
24            options: Map::new(),
25        }
26    }
27}
28
29impl<'de> Deserialize<'de> for Format {
30    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
31        match Value::deserialize(d)? {
32            Value::String(kind) => Ok(Format::new(&kind)),
33            Value::Object(mut map) => {
34                let kind = match map.remove("type") {
35                    Some(Value::String(k)) => k,
36                    _ => {
37                        return Err(serde::de::Error::custom(
38                            "format object needs a string `type`",
39                        ));
40                    }
41                };
42                Ok(Format { kind, options: map })
43            }
44            _ => Err(serde::de::Error::custom(
45                "format must be a string or an object",
46            )),
47        }
48    }
49}
50
51pub(crate) fn default_formats() -> Vec<Format> {
52    vec![Format::new("markdown")]
53}
54
55pub(crate) fn yes() -> bool {
56    true
57}
58
59#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
60pub struct Location {
61    #[serde(default)]
62    pub country: Option<String>,
63    #[serde(default)]
64    pub languages: Vec<String>,
65}
66
67/// Single page scrape options (`scrapeOptions`).
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
69#[serde(rename_all = "camelCase")]
70pub struct ScrapeOptions {
71    #[serde(default = "default_formats")]
72    pub formats: Vec<Format>,
73    #[serde(default)]
74    pub headers: BTreeMap<String, String>,
75    #[serde(default)]
76    pub include_tags: Vec<String>,
77    #[serde(default)]
78    pub exclude_tags: Vec<String>,
79    #[serde(default = "yes")]
80    pub only_main_content: bool,
81    /// Milliseconds.
82    #[serde(default)]
83    pub timeout: Option<u64>,
84    /// Milliseconds.
85    #[serde(default)]
86    pub wait_for: u64,
87    #[serde(default)]
88    pub mobile: bool,
89    #[serde(default)]
90    pub actions: Vec<Value>,
91    #[serde(default)]
92    pub location: Option<Location>,
93    #[serde(default)]
94    pub parsers: Option<Value>,
95    #[serde(default = "yes")]
96    pub remove_base64_images: bool,
97    #[serde(default = "yes")]
98    pub block_ads: bool,
99    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
100    #[serde(default)]
101    pub proxy: Option<String>,
102    /// PageSift extension: designated named egress (`[[egress]]`).
103    #[serde(default)]
104    pub egress: Option<String>,
105    /// TLS skip verification flag (PageSift always verifies certificates).
106    #[serde(default)]
107    pub skip_tls_verification: Option<bool>,
108    /// Milliseconds.
109    #[serde(default)]
110    pub max_age: Option<u64>,
111    /// Milliseconds.
112    #[serde(default)]
113    pub min_age: Option<u64>,
114    #[serde(default = "yes")]
115    pub store_in_cache: bool,
116
117    // ---- PageSift extensions ----
118    /// `http`, `browser`, `auto` (default).
119    #[serde(default)]
120    pub engine: Option<String>,
121    /// PageSift extension: keep the browser page open for `/v2/scrape/{id}/interact`
122    /// (only on `POST /v2/scrape`). Without it the first interact opens the URL again.
123    /// Not serialized when false, so cluster tasks and audit records stay byte-identical.
124    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
125    pub keep_session: bool,
126    /// PageSift extension: bounded, stateless same-origin external JS discovery.
127    #[serde(default)]
128    pub discover_scripts: bool,
129    /// PageSift extension: write how the result was produced (render and merge decisions with their
130    /// inputs) to `metadata.pagesift.plan`.
131    #[serde(default)]
132    pub explain: bool,
133    /// PageSift extension: export images as attachments — PDF embedded images in their original
134    /// encoding, Office media, and web page images (fetched politely, robots and SSRF rules apply).
135    #[serde(default)]
136    pub export_images: bool,
137    #[serde(default)]
138    pub if_none_match: Option<String>,
139    #[serde(default)]
140    pub if_modified_since: Option<String>,
141    /// Client-side robots.txt bypass flag.
142    #[serde(default)]
143    pub ignore_robots_txt: bool,
144    #[serde(default)]
145    pub store_snapshot: bool,
146    /// BM25 query filter on main content markdown.
147    #[serde(default)]
148    pub query_filter: Option<QueryFilter>,
149    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
150    #[serde(default, skip_serializing_if = "Option::is_none")]
151    pub chunking: Option<Value>,
152    /// PageSift extension: when a rendered page counts as ready (Jina's `x-respond-timing`), one of
153    /// [`RESPOND_TIMINGS`]. `html` does not run scripts; the others need the browser.
154    #[serde(default, skip_serializing_if = "Option::is_none")]
155    pub respond_timing: Option<String>,
156}
157
158/// `respondTiming` values, from the earliest to the most complete (Jina Reader's six levels).
159pub const RESPOND_TIMINGS: &[&str] = &[
160    "html",
161    "visible-content",
162    "mutation-idle",
163    "resource-idle",
164    "media-idle",
165    "network-idle",
166];
167
168/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
169#[derive(Debug, Clone, PartialEq, Serialize)]
170#[serde(rename_all = "camelCase")]
171pub struct QueryFilter {
172    pub query: String,
173    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub threshold: Option<f32>,
176    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
177    #[serde(skip_serializing_if = "Option::is_none")]
178    pub relative: Option<f32>,
179    /// Maximum number of blocks to keep.
180    #[serde(skip_serializing_if = "Option::is_none")]
181    pub top_k: Option<usize>,
182    /// Token ceiling on retained content.
183    #[serde(skip_serializing_if = "Option::is_none")]
184    pub max_tokens: Option<usize>,
185    /// Token floor on retained content. Defaults to 1000.
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub min_tokens: Option<usize>,
188    /// Prepend ancestor headings before preserved units.
189    pub headings: bool,
190}
191
192impl<'de> Deserialize<'de> for QueryFilter {
193    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
194        #[derive(Deserialize)]
195        #[serde(rename_all = "camelCase")]
196        struct Full {
197            query: String,
198            #[serde(default)]
199            threshold: Option<f32>,
200            #[serde(default)]
201            relative: Option<f32>,
202            #[serde(default)]
203            top_k: Option<usize>,
204            #[serde(default)]
205            max_tokens: Option<usize>,
206            #[serde(default)]
207            min_tokens: Option<usize>,
208            #[serde(default = "yes")]
209            headings: bool,
210        }
211        #[derive(Deserialize)]
212        #[serde(untagged)]
213        enum Repr {
214            Query(String),
215            Full(Full),
216        }
217        Ok(match Repr::deserialize(d)? {
218            Repr::Query(query) => QueryFilter {
219                query,
220                threshold: None,
221                relative: Some(pagesift_extract_defaults::RELATIVE),
222                top_k: None,
223                max_tokens: None,
224                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
225                headings: true,
226            },
227            Repr::Full(f) => QueryFilter {
228                query: f.query,
229                relative: f.relative.or(if f.threshold.is_none() {
230                    Some(pagesift_extract_defaults::RELATIVE)
231                } else {
232                    None
233                }),
234                threshold: f.threshold,
235                top_k: f.top_k,
236                max_tokens: f.max_tokens,
237                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
238                    Some(pagesift_extract_defaults::MIN_TOKENS)
239                } else {
240                    None
241                }),
242                headings: f.headings,
243            },
244        })
245    }
246}
247
248mod pagesift_extract_defaults {
249    pub const RELATIVE: f32 = 0.5;
250    pub const MIN_TOKENS: usize = 1000;
251}
252
253impl Default for ScrapeOptions {
254    fn default() -> Self {
255        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
256    }
257}
258
259impl ScrapeOptions {
260    pub fn wants(&self, kind: &str) -> bool {
261        self.formats.iter().any(|f| f.kind == kind)
262    }
263}
264
265#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
266pub struct ScrapeRequest {
267    pub url: String,
268    #[serde(flatten)]
269    pub options: ScrapeOptions,
270}
271
272#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
273#[serde(rename_all = "camelCase")]
274pub struct BatchScrapeRequest {
275    pub urls: Vec<String>,
276    #[serde(default = "yes")]
277    pub ignore_invalid_urls: bool,
278    #[serde(default)]
279    pub max_concurrency: Option<usize>,
280    #[serde(default)]
281    pub webhook: Option<Value>,
282    #[serde(flatten)]
283    pub options: ScrapeOptions,
284}
285
286impl BatchScrapeRequest {
287    /// Its own top-level fields, besides the flattened `ScrapeOptions` (serde cannot list fields of
288    /// a struct with `flatten`; a test checks this list against the serialized form).
289    pub const OWN_FIELDS: &'static [&'static str] =
290        &["urls", "ignoreInvalidUrls", "maxConcurrency", "webhook"];
291}
292
293fn default_crawl_limit() -> usize {
294    10_000
295}
296
297fn default_sitemap() -> String {
298    "include".into()
299}
300
301#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
302#[serde(rename_all = "camelCase")]
303pub struct CrawlRequest {
304    pub url: String,
305    #[serde(default)]
306    pub include_paths: Vec<String>,
307    #[serde(default)]
308    pub exclude_paths: Vec<String>,
309    #[serde(default)]
310    pub max_discovery_depth: Option<usize>,
311    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
312    #[serde(default, skip_serializing_if = "Option::is_none")]
313    pub max_depth: Option<u64>,
314    #[serde(default = "default_crawl_limit")]
315    pub limit: usize,
316    #[serde(default)]
317    pub crawl_entire_domain: bool,
318    #[serde(default)]
319    pub allow_external_links: bool,
320    /// 没给时：带 `query` 的爬取默认允许同一站点（可注册域名相同）的子域名，广度优先爬取不允许。
321    #[serde(default)]
322    pub allow_subdomains: Option<bool>,
323    #[serde(default)]
324    pub ignore_robots_txt: bool,
325    #[serde(default = "default_sitemap")]
326    pub sitemap: String,
327    #[serde(default)]
328    pub ignore_query_parameters: bool,
329    /// Seconds.
330    #[serde(default)]
331    pub delay: Option<f64>,
332    #[serde(default)]
333    pub max_concurrency: Option<usize>,
334    #[serde(default)]
335    pub scrape_options: ScrapeOptions,
336    #[serde(default)]
337    pub webhook: Option<Value>,
338
339    // ---- PageSift extensions ----
340    #[serde(default, deserialize_with = "query_or_keywords")]
341    pub query: Option<String>,
342    #[serde(default, deserialize_with = "adaptive_options")]
343    pub adaptive: Option<AdaptiveOptions>,
344    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
345    /// 不填时普通爬取是 `auto`，带查询的爬取是 `domain`（相关性打分决定往哪走，路径前缀只会挡路：
346    /// 首页跳到 `/top/` 的站点，招生页在 `/inst/` 下）。
347    #[serde(default)]
348    pub scope: Option<CrawlScope>,
349    /// 带查询的爬取：相关页面上同站点的 PDF/Office 文档也抓回来解析（PageSift 扩展）。
350    #[serde(default)]
351    pub attachments: bool,
352    /// 附件的数量上限，默认等于 `limit`。
353    #[serde(default)]
354    pub attachment_limit: Option<usize>,
355}
356
357/// crawl 的路径范围。
358#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
359#[serde(rename_all = "lowercase")]
360pub enum CrawlScope {
361    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
362    #[default]
363    Auto,
364    /// 只取最窄目录，不放宽。
365    Exact,
366    /// 一开始就放宽到上一级目录（不会到整站根）。
367    Parent,
368    /// 整个主机，等同于 `crawlEntireDomain`。
369    Domain,
370}
371
372fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
373    #[derive(Deserialize)]
374    #[serde(untagged)]
375    enum Repr {
376        Query(String),
377        Keywords(Vec<String>),
378    }
379    Ok(match Option::<Repr>::deserialize(d)? {
380        Some(Repr::Query(q)) => Some(q),
381        Some(Repr::Keywords(k)) => Some(k.join(" ")),
382        None => None,
383    }
384    .filter(|q| !q.trim().is_empty()))
385}
386
387fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
388    #[derive(Deserialize)]
389    #[serde(untagged)]
390    enum Repr {
391        Flag(bool),
392        Options(AdaptiveOptions),
393    }
394    Ok(match Option::<Repr>::deserialize(d)? {
395        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
396        Some(Repr::Flag(false)) | None => None,
397        Some(Repr::Options(o)) => Some(o),
398    })
399}
400
401/// Adaptive crawl stopping conditions (PageSift extension `adaptive`). With them the crawl decides
402/// how many pages it needs; `limit` is only a cap.
403#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
404#[serde(rename_all = "camelCase", default)]
405pub struct AdaptiveOptions {
406    /// Share of query terms that must have `evidencePerTerm` pieces of evidence (from at least two
407    /// pages) before the query counts as covered. Once covered, the crawl stops after
408    /// `ceil(patience / 2)` pages in a row that add nothing new (`coverage_reached`).
409    pub coverage_threshold: f64,
410    pub evidence_per_term: usize,
411    /// Pages in a row without new relevant content before the crawl stops (`saturated`). While
412    /// nothing relevant has been found, `2 × patience` pages in a row reached only through links
413    /// without any sign of the query stop it (`no_relevant_pages`).
414    pub patience: usize,
415    /// A page whose new relevant content is less than this share of everything collected so far
416    /// counts as adding nothing new.
417    pub min_gain: f64,
418}
419
420impl Default for AdaptiveOptions {
421    fn default() -> Self {
422        // 请求参数是用户策略，这里是不填时的默认值：
423        Self {
424            // [策略] 先验包 v0：0.81 的现值。覆盖度只决定"覆盖后耐心减半"，不单独停止爬取
425            coverage_threshold: 0.9,
426            // [策略] 先验包 v0：0.81 的现值（每个词要三段不同的证据，单段可能是导航、转载）
427            evidence_per_term: 3,
428            // [策略] 先验包 v0：0.81 的现值；连续这么多页没有新内容才停
429            patience: 5,
430            // [δ] 新内容不到已收集总量的 δ 算"没有新信息"：低于全局风险水平的增量当作噪声
431            // （δ 默认 0.05，与 0.81 的值相同）
432            min_gain: 0.05,
433        }
434    }
435}
436
437fn default_map_limit() -> usize {
438    5_000
439}
440
441#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
442#[serde(rename_all = "camelCase")]
443pub struct MapRequest {
444    pub url: String,
445    #[serde(default)]
446    pub search: Option<String>,
447    #[serde(default = "yes")]
448    pub include_subdomains: bool,
449    #[serde(default = "yes")]
450    pub ignore_query_parameters: bool,
451    #[serde(default = "default_sitemap")]
452    pub sitemap: String,
453    #[serde(default = "default_map_limit")]
454    pub limit: usize,
455    #[serde(default)]
456    pub ignore_robots_txt: bool,
457    /// Milliseconds.
458    #[serde(default)]
459    pub timeout: Option<u64>,
460}
461
462fn default_search_limit() -> usize {
463    10
464}
465
466/// `/v1/search` and `/v2/search`.
467#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
468#[serde(rename_all = "camelCase")]
469pub struct SearchRequest {
470    pub query: String,
471    #[serde(default = "default_search_limit")]
472    pub limit: usize,
473    #[serde(default)]
474    pub lang: Option<String>,
475    #[serde(default)]
476    pub country: Option<String>,
477    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
478    #[serde(default)]
479    pub tbs: Option<String>,
480    /// Milliseconds.
481    #[serde(default)]
482    pub timeout: Option<u64>,
483    #[serde(default)]
484    pub scrape_options: Option<ScrapeOptions>,
485}
486
487/// `/v1/extract` and `/v2/extract`.
488#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
489#[serde(rename_all = "camelCase")]
490pub struct ExtractRequest {
491    pub urls: Vec<String>,
492    #[serde(default)]
493    pub prompt: Option<String>,
494    #[serde(default)]
495    pub schema: Option<Value>,
496    #[serde(default)]
497    pub enable_web_search: bool,
498    #[serde(default)]
499    pub ignore_invalid_urls: bool,
500    #[serde(default)]
501    pub scrape_options: Option<ScrapeOptions>,
502}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
