pagesift_api/types/
request.rs

1//! Firecrawl v1/v2 request types and options.
2//! Fields and defaults align with Firecrawl source `apps/api/src/controllers/v2/types.ts`.
3//! Unknown fields are accepted by the Firecrawl-compatible routes and listed in `warnings`
4//! (`ignored_fields`, see [`crate::fields`]); pagesift's own entry points reject them.
5
6use std::collections::BTreeMap;
7
8use serde::{Deserialize, Deserializer, Serialize};
9use serde_json::{Map, Value};
10
11/// `formats` item: accepts both `"markdown"` and `{"type": "markdown", ...}`.
12#[derive(Debug, Clone, PartialEq, Serialize)]
13pub struct Format {
14    #[serde(rename = "type")]
15    pub kind: String,
16    #[serde(flatten)]
17    pub options: Map<String, Value>,
18}
19
20impl Format {
21    pub fn new(kind: &str) -> Self {
22        Self {
23            kind: kind.to_string(),
24            options: Map::new(),
25        }
26    }
27}
28
29impl<'de> Deserialize<'de> for Format {
30    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
31        match Value::deserialize(d)? {
32            Value::String(kind) => Ok(Format::new(&kind)),
33            Value::Object(mut map) => {
34                let kind = match map.remove("type") {
35                    Some(Value::String(k)) => k,
36                    _ => {
37                        return Err(serde::de::Error::custom(
38                            "format object needs a string `type`",
39                        ));
40                    }
41                };
42                Ok(Format { kind, options: map })
43            }
44            _ => Err(serde::de::Error::custom(
45                "format must be a string or an object",
46            )),
47        }
48    }
49}
50
51pub(crate) fn default_formats() -> Vec<Format> {
52    vec![Format::new("markdown")]
53}
54
55pub(crate) fn yes() -> bool {
56    true
57}
58
59#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
60pub struct Location {
61    #[serde(default)]
62    pub country: Option<String>,
63    #[serde(default)]
64    pub languages: Vec<String>,
65}
66
67/// Optional presentation for image assets extracted from pages or documents.
68#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
69#[serde(rename_all = "lowercase")]
70pub enum ImageAssetMode {
71    Attach,
72    Embed,
73    Rewrite,
74}
75
76/// Single page scrape options (`scrapeOptions`).
77#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
78#[serde(rename_all = "camelCase")]
79pub struct ScrapeOptions {
80    #[serde(default = "default_formats")]
81    pub formats: Vec<Format>,
82    #[serde(default)]
83    pub headers: BTreeMap<String, String>,
84    #[serde(default)]
85    pub include_tags: Vec<String>,
86    #[serde(default)]
87    pub exclude_tags: Vec<String>,
88    #[serde(default = "yes")]
89    pub only_main_content: bool,
90    /// Milliseconds.
91    #[serde(default)]
92    pub timeout: Option<u64>,
93    /// Milliseconds.
94    #[serde(default)]
95    pub wait_for: u64,
96    #[serde(default)]
97    pub mobile: bool,
98    #[serde(default)]
99    pub actions: Vec<Value>,
100    #[serde(default)]
101    pub location: Option<Location>,
102    #[serde(default)]
103    pub parsers: Option<Value>,
104    #[serde(default = "yes")]
105    pub remove_base64_images: bool,
106    #[serde(default = "yes")]
107    pub block_ads: bool,
108    /// `basic`, `auto` pass through; others return UNSUPPORTED_OPTION.
109    #[serde(default)]
110    pub proxy: Option<String>,
111    /// PageSift extension: designated named egress (`[[egress]]`).
112    #[serde(default)]
113    pub egress: Option<String>,
114    /// TLS skip verification flag (PageSift always verifies certificates).
115    #[serde(default)]
116    pub skip_tls_verification: Option<bool>,
117    /// Milliseconds.
118    #[serde(default)]
119    pub max_age: Option<u64>,
120    /// Milliseconds.
121    #[serde(default)]
122    pub min_age: Option<u64>,
123    #[serde(default = "yes")]
124    pub store_in_cache: bool,
125
126    // ---- PageSift extensions ----
127    /// `http`, `browser`, `auto` (default).
128    #[serde(default)]
129    pub engine: Option<String>,
130    /// PageSift extension: keep the browser page open for `/v2/scrape/{id}/interact`
131    /// (only on `POST /v2/scrape`). Without it the first interact opens the URL again.
132    /// Not serialized when false, so cluster tasks and audit records stay byte-identical.
133    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
134    pub keep_session: bool,
135    /// PageSift extension: bounded, stateless same-origin external JS discovery.
136    #[serde(default)]
137    pub discover_scripts: bool,
138    /// Internal: stable caller identity used to scope reusable sessions.
139    /// Never accepted from JSON and never serialized.
140    #[serde(skip)]
141    pub owner: Option<String>,
142    /// PageSift extension: write how the result was produced (render and merge decisions with their
143    /// inputs) to `metadata.pagesift.plan`.
144    #[serde(default)]
145    pub explain: bool,
146    /// PageSift extension: export images as attachments — PDF embedded images in their original
147    /// encoding, Office media, and web page images (fetched politely, robots and SSRF rules apply).
148    #[serde(default)]
149    pub export_images: bool,
150    /// Optional image asset presentation; absent leaves existing behavior.
151    #[serde(default, skip_serializing_if = "Option::is_none")]
152    pub image_assets: Option<ImageAssetMode>,
153    /// Internal cache scoped to one batch/crawl; not part of the public request.
154    #[serde(skip)]
155    pub asset_fetch_cache: Option<super::AssetFetchCache>,
156    /// Internal channel for provisional extracted Markdown; never serialized.
157    #[serde(skip)]
158    pub markdown_progress: Option<super::MarkdownProgress>,
159    #[serde(default)]
160    pub if_none_match: Option<String>,
161    #[serde(default)]
162    pub if_modified_since: Option<String>,
163    /// Client-side robots.txt bypass flag.
164    #[serde(default)]
165    pub ignore_robots_txt: bool,
166    #[serde(default)]
167    pub store_snapshot: bool,
168    /// BM25 query filter on main content markdown.
169    #[serde(default)]
170    pub query_filter: Option<QueryFilter>,
171    /// PageSift extension: chunking options (`{"type": "recursive", "size": 400}`, etc.).
172    #[serde(default, skip_serializing_if = "Option::is_none")]
173    pub chunking: Option<Value>,
174    /// PageSift extension: when a rendered page counts as ready (Jina's `x-respond-timing`), one of
175    /// [`RESPOND_TIMINGS`]. `html` does not run scripts; the others need the browser.
176    #[serde(default, skip_serializing_if = "Option::is_none")]
177    pub respond_timing: Option<String>,
178}
179
180/// `respondTiming` values, from the earliest to the most complete (Jina Reader's six levels).
181pub const RESPOND_TIMINGS: &[&str] = &[
182    "html",
183    "visible-content",
184    "mutation-idle",
185    "resource-idle",
186    "media-idle",
187    "network-idle",
188];
189
190/// PageSift extension `queryFilter`: filter main content markdown by query relevance.
191#[derive(Debug, Clone, PartialEq, Serialize)]
192#[serde(rename_all = "camelCase")]
193pub struct QueryFilter {
194    pub query: String,
195    /// Absolute threshold: blocks with BM25 score >= threshold are kept.
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub threshold: Option<f32>,
198    /// Relative threshold: score >= ratio * max_score. Defaults to 0.5.
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub relative: Option<f32>,
201    /// Maximum number of blocks to keep.
202    #[serde(skip_serializing_if = "Option::is_none")]
203    pub top_k: Option<usize>,
204    /// Token ceiling on retained content.
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub max_tokens: Option<usize>,
207    /// Token floor on retained content. Defaults to 1000.
208    #[serde(skip_serializing_if = "Option::is_none")]
209    pub min_tokens: Option<usize>,
210    /// Prepend ancestor headings before preserved units.
211    pub headings: bool,
212}
213
214impl<'de> Deserialize<'de> for QueryFilter {
215    fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
216        #[derive(Deserialize)]
217        #[serde(rename_all = "camelCase")]
218        struct Full {
219            query: String,
220            #[serde(default)]
221            threshold: Option<f32>,
222            #[serde(default)]
223            relative: Option<f32>,
224            #[serde(default)]
225            top_k: Option<usize>,
226            #[serde(default)]
227            max_tokens: Option<usize>,
228            #[serde(default)]
229            min_tokens: Option<usize>,
230            #[serde(default = "yes")]
231            headings: bool,
232        }
233        #[derive(Deserialize)]
234        #[serde(untagged)]
235        enum Repr {
236            Query(String),
237            Full(Full),
238        }
239        Ok(match Repr::deserialize(d)? {
240            Repr::Query(query) => QueryFilter {
241                query,
242                threshold: None,
243                relative: Some(pagesift_extract_defaults::RELATIVE),
244                top_k: None,
245                max_tokens: None,
246                min_tokens: Some(pagesift_extract_defaults::MIN_TOKENS),
247                headings: true,
248            },
249            Repr::Full(f) => QueryFilter {
250                query: f.query,
251                relative: f.relative.or(if f.threshold.is_none() {
252                    Some(pagesift_extract_defaults::RELATIVE)
253                } else {
254                    None
255                }),
256                threshold: f.threshold,
257                top_k: f.top_k,
258                max_tokens: f.max_tokens,
259                min_tokens: f.min_tokens.or(if f.threshold.is_none() {
260                    Some(pagesift_extract_defaults::MIN_TOKENS)
261                } else {
262                    None
263                }),
264                headings: f.headings,
265            },
266        })
267    }
268}
269
270mod pagesift_extract_defaults {
271    pub const RELATIVE: f32 = 0.5;
272    pub const MIN_TOKENS: usize = 1000;
273}
274
275impl Default for ScrapeOptions {
276    fn default() -> Self {
277        serde_json::from_value(Value::Object(Map::new())).expect("defaults deserialize")
278    }
279}
280
281impl ScrapeOptions {
282    pub fn wants(&self, kind: &str) -> bool {
283        self.formats.iter().any(|f| f.kind == kind)
284    }
285}
286
287#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
288pub struct ScrapeRequest {
289    pub url: String,
290    #[serde(flatten)]
291    pub options: ScrapeOptions,
292}
293
294#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
295#[serde(rename_all = "camelCase")]
296pub struct BatchScrapeRequest {
297    pub urls: Vec<String>,
298    #[serde(default = "yes")]
299    pub ignore_invalid_urls: bool,
300    #[serde(default)]
301    pub max_concurrency: Option<usize>,
302    #[serde(default)]
303    pub webhook: Option<Value>,
304    #[serde(flatten)]
305    pub options: ScrapeOptions,
306}
307
308impl BatchScrapeRequest {
309    /// Its own top-level fields, besides the flattened `ScrapeOptions` (serde cannot list fields of
310    /// a struct with `flatten`; a test checks this list against the serialized form).
311    pub const OWN_FIELDS: &'static [&'static str] =
312        &["urls", "ignoreInvalidUrls", "maxConcurrency", "webhook"];
313}
314
315fn default_crawl_limit() -> usize {
316    10_000
317}
318
319fn default_sitemap() -> String {
320    "include".into()
321}
322
323#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
324#[serde(rename_all = "camelCase")]
325pub struct CrawlRequest {
326    pub url: String,
327    #[serde(default)]
328    pub include_paths: Vec<String>,
329    #[serde(default)]
330    pub exclude_paths: Vec<String>,
331    #[serde(default)]
332    pub max_discovery_depth: Option<usize>,
333    /// Firecrawl v1 的 `maxDepth`：只做上限校验，抓取深度仍看 `maxDiscoveryDepth`。
334    #[serde(default, skip_serializing_if = "Option::is_none")]
335    pub max_depth: Option<u64>,
336    #[serde(default = "default_crawl_limit")]
337    pub limit: usize,
338    #[serde(default)]
339    pub crawl_entire_domain: bool,
340    #[serde(default)]
341    pub allow_external_links: bool,
342    /// 没给时：带 `query` 的爬取默认允许同一站点（可注册域名相同）的子域名，广度优先爬取不允许。
343    #[serde(default)]
344    pub allow_subdomains: Option<bool>,
345    #[serde(default)]
346    pub ignore_robots_txt: bool,
347    #[serde(default = "default_sitemap")]
348    pub sitemap: String,
349    #[serde(default)]
350    pub ignore_query_parameters: bool,
351    /// Seconds.
352    #[serde(default)]
353    pub delay: Option<f64>,
354    #[serde(default)]
355    pub max_concurrency: Option<usize>,
356    #[serde(default)]
357    pub scrape_options: ScrapeOptions,
358    #[serde(default)]
359    pub webhook: Option<Value>,
360
361    // ---- PageSift extensions ----
362    #[serde(default, deserialize_with = "query_or_keywords")]
363    pub query: Option<String>,
364    #[serde(default, deserialize_with = "adaptive_options")]
365    pub adaptive: Option<AdaptiveOptions>,
366    /// 路径范围怎么定（PageSift 扩展）。`crawlEntireDomain: true` 等同于 `domain`。
367    /// 不填时普通爬取是 `auto`，带查询的爬取是 `domain`（相关性打分决定往哪走，路径前缀只会挡路：
368    /// 首页跳到 `/top/` 的站点，招生页在 `/inst/` 下）。
369    #[serde(default)]
370    pub scope: Option<CrawlScope>,
371    /// 带查询的爬取：相关页面上同站点的 PDF/Office 文档也抓回来解析（PageSift 扩展）。
372    #[serde(default)]
373    pub attachments: bool,
374    /// 附件的数量上限，默认等于 `limit`。
375    #[serde(default)]
376    pub attachment_limit: Option<usize>,
377}
378
379/// crawl 的路径范围。
380#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
381#[serde(rename_all = "lowercase")]
382pub enum CrawlScope {
383    /// 先取种子所在的最窄目录；种子页看起来是叶子页（几乎没有子页链接、却链向很多同级页面）时放宽一级。
384    #[default]
385    Auto,
386    /// 只取最窄目录，不放宽。
387    Exact,
388    /// 一开始就放宽到上一级目录（不会到整站根）。
389    Parent,
390    /// 整个主机，等同于 `crawlEntireDomain`。
391    Domain,
392}
393
394fn query_or_keywords<'de, D: Deserializer<'de>>(d: D) -> Result<Option<String>, D::Error> {
395    #[derive(Deserialize)]
396    #[serde(untagged)]
397    enum Repr {
398        Query(String),
399        Keywords(Vec<String>),
400    }
401    Ok(match Option::<Repr>::deserialize(d)? {
402        Some(Repr::Query(q)) => Some(q),
403        Some(Repr::Keywords(k)) => Some(k.join(" ")),
404        None => None,
405    }
406    .filter(|q| !q.trim().is_empty()))
407}
408
409fn adaptive_options<'de, D: Deserializer<'de>>(d: D) -> Result<Option<AdaptiveOptions>, D::Error> {
410    #[derive(Deserialize)]
411    #[serde(untagged)]
412    enum Repr {
413        Flag(bool),
414        Options(AdaptiveOptions),
415    }
416    Ok(match Option::<Repr>::deserialize(d)? {
417        Some(Repr::Flag(true)) => Some(AdaptiveOptions::default()),
418        Some(Repr::Flag(false)) | None => None,
419        Some(Repr::Options(o)) => Some(o),
420    })
421}
422
423/// Adaptive crawl stopping conditions (PageSift extension `adaptive`). With them the crawl decides
424/// how many pages it needs; `limit` is only a cap.
425#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
426#[serde(rename_all = "camelCase", default)]
427pub struct AdaptiveOptions {
428    /// Share of query terms that must have `evidencePerTerm` pieces of evidence (from at least two
429    /// pages) before the query counts as covered. Once covered, the crawl stops after
430    /// `ceil(patience / 2)` pages in a row that add nothing new (`coverage_reached`).
431    pub coverage_threshold: f64,
432    pub evidence_per_term: usize,
433    /// Pages in a row without new relevant content before the crawl stops (`saturated`). While
434    /// nothing relevant has been found, `2 × patience` pages in a row reached only through links
435    /// without any sign of the query stop it (`no_relevant_pages`).
436    pub patience: usize,
437    /// A page whose new relevant content is less than this share of everything collected so far
438    /// counts as adding nothing new.
439    pub min_gain: f64,
440}
441
442impl Default for AdaptiveOptions {
443    fn default() -> Self {
444        // 请求参数是用户策略，这里是不填时的默认值：
445        Self {
446            // [策略] 先验包 v0：0.81 的现值。覆盖度只决定"覆盖后耐心减半"，不单独停止爬取
447            coverage_threshold: 0.9,
448            // [策略] 先验包 v0：0.81 的现值（每个词要三段不同的证据，单段可能是导航、转载）
449            evidence_per_term: 3,
450            // [策略] 先验包 v0：0.81 的现值；连续这么多页没有新内容才停
451            patience: 5,
452            // [δ] 新内容不到已收集总量的 δ 算"没有新信息"：低于全局风险水平的增量当作噪声
453            // （δ 默认 0.05，与 0.81 的值相同）
454            min_gain: 0.05,
455        }
456    }
457}
458
459fn default_map_limit() -> usize {
460    5_000
461}
462
463#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
464#[serde(rename_all = "camelCase")]
465pub struct MapRequest {
466    pub url: String,
467    #[serde(default)]
468    pub search: Option<String>,
469    #[serde(default = "yes")]
470    pub include_subdomains: bool,
471    #[serde(default = "yes")]
472    pub ignore_query_parameters: bool,
473    #[serde(default = "default_sitemap")]
474    pub sitemap: String,
475    #[serde(default = "default_map_limit")]
476    pub limit: usize,
477    #[serde(default)]
478    pub ignore_robots_txt: bool,
479    /// Milliseconds.
480    #[serde(default)]
481    pub timeout: Option<u64>,
482}
483
484fn default_search_limit() -> usize {
485    10
486}
487
488/// `/v1/search` and `/v2/search`.
489#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
490#[serde(rename_all = "camelCase")]
491pub struct SearchRequest {
492    pub query: String,
493    #[serde(default = "default_search_limit")]
494    pub limit: usize,
495    #[serde(default)]
496    pub lang: Option<String>,
497    #[serde(default)]
498    pub country: Option<String>,
499    /// Time range: `qdr:d`, `qdr:w`, `qdr:m`, `qdr:y`.
500    #[serde(default)]
501    pub tbs: Option<String>,
502    /// Milliseconds.
503    #[serde(default)]
504    pub timeout: Option<u64>,
505    #[serde(default)]
506    pub scrape_options: Option<ScrapeOptions>,
507}
508
509/// `/v1/extract` and `/v2/extract`.
510#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
511#[serde(rename_all = "camelCase")]
512pub struct ExtractRequest {
513    pub urls: Vec<String>,
514    #[serde(default)]
515    pub prompt: Option<String>,
516    #[serde(default)]
517    pub schema: Option<Value>,
518    #[serde(default)]
519    pub enable_web_search: bool,
520    #[serde(default)]
521    pub ignore_invalid_urls: bool,
522    #[serde(default)]
523    pub scrape_options: Option<ScrapeOptions>,
524}
pagesift_api/types/request.rs

pagesift_api/types/
request.rs
