pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields are accepted by the Firecrawl-compatible routes and listed in `warnings`
4//! (`ignored_fields`, see [`crate::fields`]); pagesift's own entry points reject them.
5
6use std::collections::BTreeMap;
7
8use serde::{Deserialize, Deserializer, Serialize};
9use serde_json::{Map, Value};
10
11/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
12#[derive(Debug, Clone, PartialEq, Serialize)]
13pub struct Format {
14    #[serde(rename = "type")]
15    pub kind: String,
16    #[serde(flatten)]
17    pub options: Map<String, Value>,
18}
19
20impl Format {
21    pub fn new(kind: &str) -> Self {
22        Self {
23            kind: kind.to_string(),
24            options: Map::new(),
25        }
26    }
27}
28
29impl<'de> Deserialize<'de> for Format {
30    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
31        match Value::deserialize(d)? {
32            Value::String(kind) => Ok(Format::new(&kind)),
33            Value::Object(mut map) => {
34                let kind = match map.remove("type") {
35                    Some(Value::String(k)) => k,
36                    _ => {
37                        return Err(serde::de::Error::custom(
38                            "format object needs a string `type`",
39                        ));
40                    }
41                };
42                Ok(Format { kind, options: map })
43            }
44            _ => Err(serde::de::Error::custom(
45                "format must be a string or an object",
46            )),
47        }
48    }
49}
50
51pub(crate) fn default_formats() -> Vec<Format> {
52    vec![Format::new("markdown")]
53}
54
55pub(crate) fn yes() -> bool {
56    true
57}
58
59#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
60pub struct Location {
61    #[serde(default)]
62    pub country: Option<String>,
63    #[serde(default)]
64    pub languages: Vec<String>,
65}
66
67/// Single page scrape options (`scrapeOptions`).
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
69#[serde(rename_all = "camelCase")]
70pub struct ScrapeOptions {
71    #[serde(default = "default_formats")]
72    pub formats: Vec<Format>,
73    #[serde(default)]
74    pub headers: BTreeMap<String, String>,
75    #[serde(default)]
76    pub include_tags: Vec<String>,
77    #[serde(default)]
78    pub exclude_tags: Vec<String>,
79    #[serde(default = "yes")]
80    pub only_main_content: bool,
81    /// Milliseconds.
82    #[serde(default)]
83    pub timeout: Option<u64>,
84    /// Milliseconds.
85    #[serde(default)]
86    pub wait_for: u64,
87    #[serde(default)]
88    pub mobile: bool,
89    #[serde(default)]
90    pub actions: Vec<Value>,
91    #[serde(default)]
92    pub location: Option<Location>,
93    #[serde(default)]
94    pub parsers: Option<Value>,
95    #[serde(default = "yes")]
96    pub remove_base64_images: bool,
97    #[serde(default = "yes")]
98    pub block_ads: bool,
99    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
100    #[serde(default)]
101    pub proxy: Option<String>,
102    /// PageSift extension: designated named egress (`[[egress]]`).
103    #[serde(default)]
104    pub egress: Option<String>,
105    /// TLS skip verification flag (PageSift always verifies certificates).
106    #[serde(default)]
107    pub skip_tls_verification: Option<bool>,
108    /// Milliseconds.
109    #[serde(default)]
110    pub max_age: Option<u64>,
111    /// Milliseconds.
112    #[serde(default)]
113    pub min_age: Option<u64>,
114    #[serde(default = "yes")]
115    pub store_in_cache: bool,
116
117    // ---- PageSift extensions ----
118    /// `http`, `browser`, `auto` (default).
119    #[serde(default)]
120    pub engine: Option<String>,
121    /// PageSift extension: keep the browser page open for `/v2/scrape/{id}/interact`
122    /// (only on `POST /v2/scrape`). Without it the first interact opens the URL again.
123    /// Not serialized when false, so cluster tasks and audit records stay byte-identical.
124    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
125    pub keep_session: bool,
126    /// PageSift extension: bounded, stateless same-origin external JS discovery.
127    #[serde(default)]
128    pub discover_scripts: bool,
129    /// Internal: stable caller identity used to scope reusable sessions.
130    /// Never accepted from JSON and never serialized.
131    #[serde(skip)]
132    pub owner: Option<String>,
133    /// PageSift extension: write how the result was produced (render and merge decisions with their
134    /// inputs) to `metadata.pagesift.plan`.
135    #[serde(default)]
136    pub explain: bool,
137    /// PageSift extension: export images as attachments — PDF embedded images in their original
138    /// encoding, Office media, and web page images (fetched politely, robots and SSRF rules apply).
139    #[serde(default)]
140    pub export_images: bool,
141    #[serde(default)]
142    pub if_none_match: Option<String>,
143    #[serde(default)]
144    pub if_modified_since: Option<String>,
145    /// Client-side robots.txt bypass flag.
146    #[serde(default)]
147    pub ignore_robots_txt: bool,
148    #[serde(default)]
149    pub store_snapshot: bool,
150    /// BM25 query filter on main content markdown.
151    #[serde(default)]
152    pub query_filter: Option<QueryFilter>,
153    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
154    #[serde(default, skip_serializing_if = "Option::is_none")]
155    pub chunking: Option<Value>,
156    /// PageSift extension: when a rendered page counts as ready (Jina's `x-respond-timing`), one of
157    /// [`RESPOND_TIMINGS`]. `html` does not run scripts; the others need the browser.
158    #[serde(default, skip_serializing_if = "Option::is_none")]
159    pub respond_timing: Option<String>,
160}
161
162/// `respondTiming` values, from the earliest to the most complete (Jina Reader's six levels).
163pub const RESPOND_TIMINGS: &[&str] = &[
164    "html",
165    "visible-content",
166    "mutation-idle",
167    "resource-idle",
168    "media-idle",
169    "network-idle",
170];
171
172/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
173#[derive(Debug, Clone, PartialEq, Serialize)]
174#[serde(rename_all = "camelCase")]
175pub struct QueryFilter {
176    pub query: String,
177    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub threshold: Option<f32>,
180    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
181    #[serde(skip_serializing_if = "Option::is_none")]
182    pub relative: Option<f32>,
183    /// Maximum number of blocks to keep.
184    #[serde(skip_serializing_if = "Option::is_none")]
185    pub top_k: Option<usize>,
186    /// Token ceiling on retained content.
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub max_tokens: Option<usize>,
189    /// Token floor on retained content. Defaults to 1000.
190    #[serde(skip_serializing_if = "Option::is_none")]
191    pub min_tokens: Option<usize>,
192    /// Prepend ancestor headings before preserved units.
193    pub headings: bool,
194}
195
196impl<'de> Deserialize<'de> for QueryFilter {
197    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
198        #[derive(Deserialize)]
199        #[serde(rename_all = "camelCase")]
200        struct Full {
201            query: String,
202            #[serde(default)]
203            threshold: Option<f32>,
204            #[serde(default)]
205            relative: Option<f32>,
206            #[serde(default)]
207            top_k: Option<usize>,
208            #[serde(default)]
209            max_tokens: Option<usize>,
210            #[serde(default)]
211            min_tokens: Option<usize>,
212            #[serde(default = "yes")]
213            headings: bool,
214        }
215        #[derive(Deserialize)]
216        #[serde(untagged)]
217        enum Repr {
218            Query(String),
219            Full(Full),
220        }
221        Ok(match Repr::deserialize(d)? {
222            Repr::Query(query) => QueryFilter {
223                query,
224                threshold: None,
225                relative: Some(pagesift_extract_defaults::RELATIVE),
226                top_k: None,
227                max_tokens: None,
228                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
229                headings: true,
230            },
231            Repr::Full(f) => QueryFilter {
232                query: f.query,
233                relative: f.relative.or(if f.threshold.is_none() {
234                    Some(pagesift_extract_defaults::RELATIVE)
235                } else {
236                    None
237                }),
238                threshold: f.threshold,
239                top_k: f.top_k,
240                max_tokens: f.max_tokens,
241                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
242                    Some(pagesift_extract_defaults::MIN_TOKENS)
243                } else {
244                    None
245                }),
246                headings: f.headings,
247            },
248        })
249    }
250}
251
252mod pagesift_extract_defaults {
253    pub const RELATIVE: f32 = 0.5;
254    pub const MIN_TOKENS: usize = 1000;
255}
256
257impl Default for ScrapeOptions {
258    fn default() -> Self {
259        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
260    }
261}
262
263impl ScrapeOptions {
264    pub fn wants(&self, kind: &str) -> bool {
265        self.formats.iter().any(|f| f.kind == kind)
266    }
267}
268
269#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
270pub struct ScrapeRequest {
271    pub url: String,
272    #[serde(flatten)]
273    pub options: ScrapeOptions,
274}
275
276#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
277#[serde(rename_all = "camelCase")]
278pub struct BatchScrapeRequest {
279    pub urls: Vec<String>,
280    #[serde(default = "yes")]
281    pub ignore_invalid_urls: bool,
282    #[serde(default)]
283    pub max_concurrency: Option<usize>,
284    #[serde(default)]
285    pub webhook: Option<Value>,
286    #[serde(flatten)]
287    pub options: ScrapeOptions,
288}
289
290impl BatchScrapeRequest {
291    /// Its own top-level fields, besides the flattened `ScrapeOptions` (serde cannot list fields of
292    /// a struct with `flatten`; a test checks this list against the serialized form).
293    pub const OWN_FIELDS: &'static [&'static str] =
294        &["urls", "ignoreInvalidUrls", "maxConcurrency", "webhook"];
295}
296
297fn default_crawl_limit() -> usize {
298    10_000
299}
300
301fn default_sitemap() -> String {
302    "include".into()
303}
304
305#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
306#[serde(rename_all = "camelCase")]
307pub struct CrawlRequest {
308    pub url: String,
309    #[serde(default)]
310    pub include_paths: Vec<String>,
311    #[serde(default)]
312    pub exclude_paths: Vec<String>,
313    #[serde(default)]
314    pub max_discovery_depth: Option<usize>,
315    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
316    #[serde(default, skip_serializing_if = "Option::is_none")]
317    pub max_depth: Option<u64>,
318    #[serde(default = "default_crawl_limit")]
319    pub limit: usize,
320    #[serde(default)]
321    pub crawl_entire_domain: bool,
322    #[serde(default)]
323    pub allow_external_links: bool,
324    /// 没给时：带 `query` 的爬取默认允许同一站点（可注册域名相同）的子域名，广度优先爬取不允许。
325    #[serde(default)]
326    pub allow_subdomains: Option<bool>,
327    #[serde(default)]
328    pub ignore_robots_txt: bool,
329    #[serde(default = "default_sitemap")]
330    pub sitemap: String,
331    #[serde(default)]
332    pub ignore_query_parameters: bool,
333    /// Seconds.
334    #[serde(default)]
335    pub delay: Option<f64>,
336    #[serde(default)]
337    pub max_concurrency: Option<usize>,
338    #[serde(default)]
339    pub scrape_options: ScrapeOptions,
340    #[serde(default)]
341    pub webhook: Option<Value>,
342
343    // ---- PageSift extensions ----
344    #[serde(default, deserialize_with = "query_or_keywords")]
345    pub query: Option<String>,
346    #[serde(default, deserialize_with = "adaptive_options")]
347    pub adaptive: Option<AdaptiveOptions>,
348    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
349    /// 不填时普通爬取是 `auto`，带查询的爬取是 `domain`（相关性打分决定往哪走，路径前缀只会挡路：
350    /// 首页跳到 `/top/` 的站点，招生页在 `/inst/` 下）。
351    #[serde(default)]
352    pub scope: Option<CrawlScope>,
353    /// 带查询的爬取：相关页面上同站点的 PDF/Office 文档也抓回来解析（PageSift 扩展）。
354    #[serde(default)]
355    pub attachments: bool,
356    /// 附件的数量上限，默认等于 `limit`。
357    #[serde(default)]
358    pub attachment_limit: Option<usize>,
359}
360
361/// crawl 的路径范围。
362#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
363#[serde(rename_all = "lowercase")]
364pub enum CrawlScope {
365    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
366    #[default]
367    Auto,
368    /// 只取最窄目录，不放宽。
369    Exact,
370    /// 一开始就放宽到上一级目录（不会到整站根）。
371    Parent,
372    /// 整个主机，等同于 `crawlEntireDomain`。
373    Domain,
374}
375
376fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
377    #[derive(Deserialize)]
378    #[serde(untagged)]
379    enum Repr {
380        Query(String),
381        Keywords(Vec<String>),
382    }
383    Ok(match Option::<Repr>::deserialize(d)? {
384        Some(Repr::Query(q)) => Some(q),
385        Some(Repr::Keywords(k)) => Some(k.join(" ")),
386        None => None,
387    }
388    .filter(|q| !q.trim().is_empty()))
389}
390
391fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
392    #[derive(Deserialize)]
393    #[serde(untagged)]
394    enum Repr {
395        Flag(bool),
396        Options(AdaptiveOptions),
397    }
398    Ok(match Option::<Repr>::deserialize(d)? {
399        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
400        Some(Repr::Flag(false)) | None => None,
401        Some(Repr::Options(o)) => Some(o),
402    })
403}
404
405/// Adaptive crawl stopping conditions (PageSift extension `adaptive`). With them the crawl decides
406/// how many pages it needs; `limit` is only a cap.
407#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
408#[serde(rename_all = "camelCase", default)]
409pub struct AdaptiveOptions {
410    /// Share of query terms that must have `evidencePerTerm` pieces of evidence (from at least two
411    /// pages) before the query counts as covered. Once covered, the crawl stops after
412    /// `ceil(patience / 2)` pages in a row that add nothing new (`coverage_reached`).
413    pub coverage_threshold: f64,
414    pub evidence_per_term: usize,
415    /// Pages in a row without new relevant content before the crawl stops (`saturated`). While
416    /// nothing relevant has been found, `2 × patience` pages in a row reached only through links
417    /// without any sign of the query stop it (`no_relevant_pages`).
418    pub patience: usize,
419    /// A page whose new relevant content is less than this share of everything collected so far
420    /// counts as adding nothing new.
421    pub min_gain: f64,
422}
423
424impl Default for AdaptiveOptions {
425    fn default() -> Self {
426        // 请求参数是用户策略，这里是不填时的默认值：
427        Self {
428            // [策略] 先验包 v0：0.81 的现值。覆盖度只决定"覆盖后耐心减半"，不单独停止爬取
429            coverage_threshold: 0.9,
430            // [策略] 先验包 v0：0.81 的现值（每个词要三段不同的证据，单段可能是导航、转载）
431            evidence_per_term: 3,
432            // [策略] 先验包 v0：0.81 的现值；连续这么多页没有新内容才停
433            patience: 5,
434            // [δ] 新内容不到已收集总量的 δ 算"没有新信息"：低于全局风险水平的增量当作噪声
435            // （δ 默认 0.05，与 0.81 的值相同）
436            min_gain: 0.05,
437        }
438    }
439}
440
441fn default_map_limit() -> usize {
442    5_000
443}
444
445#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
446#[serde(rename_all = "camelCase")]
447pub struct MapRequest {
448    pub url: String,
449    #[serde(default)]
450    pub search: Option<String>,
451    #[serde(default = "yes")]
452    pub include_subdomains: bool,
453    #[serde(default = "yes")]
454    pub ignore_query_parameters: bool,
455    #[serde(default = "default_sitemap")]
456    pub sitemap: String,
457    #[serde(default = "default_map_limit")]
458    pub limit: usize,
459    #[serde(default)]
460    pub ignore_robots_txt: bool,
461    /// Milliseconds.
462    #[serde(default)]
463    pub timeout: Option<u64>,
464}
465
466fn default_search_limit() -> usize {
467    10
468}
469
470/// `/v1/search` and `/v2/search`.
471#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
472#[serde(rename_all = "camelCase")]
473pub struct SearchRequest {
474    pub query: String,
475    #[serde(default = "default_search_limit")]
476    pub limit: usize,
477    #[serde(default)]
478    pub lang: Option<String>,
479    #[serde(default)]
480    pub country: Option<String>,
481    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
482    #[serde(default)]
483    pub tbs: Option<String>,
484    /// Milliseconds.
485    #[serde(default)]
486    pub timeout: Option<u64>,
487    #[serde(default)]
488    pub scrape_options: Option<ScrapeOptions>,
489}
490
491/// `/v1/extract` and `/v2/extract`.
492#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
493#[serde(rename_all = "camelCase")]
494pub struct ExtractRequest {
495    pub urls: Vec<String>,
496    #[serde(default)]
497    pub prompt: Option<String>,
498    #[serde(default)]
499    pub schema: Option<Value>,
500    #[serde(default)]
501    pub enable_web_search: bool,
502    #[serde(default)]
503    pub ignore_invalid_urls: bool,
504    #[serde(default)]
505    pub scrape_options: Option<ScrapeOptions>,
506}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
