pagesift_api/
validate.rs

1//! 请求校验：绕过类参数和引擎做不到的功能一律明确报错（PRD 6.5.1），不悄悄降级。
2
3use crate::engine::Engine;
4use crate::error::ApiError;
5use crate::types::{PdfParser, ScrapeOptions};
6
7/// URL 最长 8 KB：再长的 URL 浏览器和多数服务器都不接受，只会白白占内存。
8pub const MAX_URL_LEN: usize = 8 * 1024;
9/// 同步 scrape 的 `timeout` 上限（毫秒）；整请求超时按它留余量。
10pub const MAX_SCRAPE_TIMEOUT_MS: u64 = 300_000;
11/// `waitFor` 上限（毫秒）。
12pub const MAX_WAIT_FOR_MS: u64 = 60_000;
13/// crawl 的 `limit` 上限。
14pub const MAX_CRAWL_LIMIT: usize = 100_000;
15/// crawl 的 `maxDiscoveryDepth` / `maxDepth` 上限。
16pub const MAX_CRAWL_DEPTH: u64 = 100;
17/// map 的 `limit` 上限，和 Firecrawl 文档一致。
18pub const MAX_MAP_LIMIT: usize = 100_000;
19/// batch scrape、extract 一次最多带多少个 URL。
20pub const MAX_URLS: usize = 10_000;
21
22#[cfg(test)]
23#[test]
24fn both_remove_base64_images_values_are_accepted() {
25    let engine = crate::test_engine::FakeEngine::open();
26    for keep in [false, true] {
27        let opts = ScrapeOptions {
28            remove_base64_images: keep,
29            ..Default::default()
30        };
31        assert!(check_scrape_options(&opts, &engine).is_ok());
32    }
33}
34
35pub fn check_url(raw: &str) -> Result<(), ApiError> {
36    if raw.len() > MAX_URL_LEN {
37        return Err(ApiError::invalid(format!(
38            "URL is too long: {} bytes, the maximum is {MAX_URL_LEN}",
39            raw.len()
40        )));
41    }
42    match url::Url::parse(raw) {
43        Ok(u) if matches!(u.scheme(), "http" | "https") && u.host().is_some() => Ok(()),
44        Ok(_) => Err(ApiError::invalid(format!(
45            "only http(s) URLs are supported: {raw}"
46        ))),
47        Err(e) => Err(ApiError::invalid(format!("invalid URL {raw:?}: {e}"))),
48    }
49}
50
51pub fn check_scrape_options(opts: &ScrapeOptions, engine: &dyn Engine) -> Result<(), ApiError> {
52    if opts.store_snapshot {
53        return Err(ApiError::unsupported(
54            "storeSnapshot is not supported; use the changeTracking format",
55        ));
56    }
57    if opts
58        .formats
59        .iter()
60        .filter_map(|f| f.options.get("schema"))
61        .any(|schema| !schema.is_object())
62    {
63        return Err(ApiError::invalid("formats[].schema must be an object"));
64    }
65    // `POST /v2/scrape` 和 MCP 的 `pagesift_scrape` 在校验前取走这个字段；走到这里的是别的入口
66    if opts.keep_session {
67        return Err(ApiError::invalid(
68            "keepSession is only available on POST /v2/scrape",
69        ));
70    }
71    if opts.discover_scripts
72        && (opts.engine.as_deref() != Some("http")
73            || !opts.wants("links")
74            || opts.egress.is_some()
75            || !opts.headers.is_empty()
76            || opts.if_none_match.is_some()
77            || opts.if_modified_since.is_some())
78    {
79        return Err(ApiError::unsupported(
80            "discoverScripts requires HTTP engine, links, and no custom headers/egress/conditional request",
81        ));
82    }
83    if let Some(p) = opts.proxy.as_deref()
84        && !matches!(p, "basic" | "auto")
85    {
86        let valid_url = url::Url::parse(p)
87            .ok()
88            .is_some_and(|u| matches!(u.scheme(), "http" | "https" | "socks5" | "socks5h"));
89        if !valid_url {
90            return Err(ApiError::unsupported(format!(
91                "proxy {p:?} is not supported: use \"basic\", \"auto\", or a valid http/https/socks5 URL"
92            )));
93        }
94    }
95    if let Some(e) = opts.engine.as_deref()
96        && !matches!(e, "http" | "browser" | "auto")
97    {
98        return Err(ApiError::invalid(format!(
99            "engine must be http, browser or auto, got {e:?}"
100        )));
101    }
102    if let Some(t) = opts.respond_timing.as_deref() {
103        if !crate::types::RESPOND_TIMINGS.contains(&t) {
104            return Err(ApiError::invalid(format!(
105                "respondTiming must be one of {}",
106                crate::types::RESPOND_TIMINGS.join(", ")
107            )));
108        }
109        if t != "html" && opts.engine.as_deref() == Some("http") {
110            return Err(ApiError::invalid(format!(
111                "respondTiming {t:?} waits for the rendered page and needs engine browser or auto"
112            )));
113        }
114    }
115    if let Some(c) = &opts.chunking {
116        const KINDS: [&str; 8] = [
117            "recursive",
118            "headings",
119            "paragraphs",
120            "sentences",
121            "fixed_tokens",
122            "sliding_window",
123            "overlapping",
124            "regex",
125        ];
126        let kind = c.get("type").and_then(|t| t.as_str());
127        if !kind.is_some_and(|k| KINDS.contains(&k)) {
128            return Err(ApiError::invalid(format!(
129                "chunking.type must be one of {}",
130                KINDS.join(", ")
131            )));
132        }
133    }
134    let supported = engine.supported_formats();
135    if let Some(f) = opts
136        .formats
137        .iter()
138        .find(|f| !supported.contains(&f.kind.as_str()))
139    {
140        return Err(ApiError::unsupported(format!(
141            "format {:?} is not supported by this build (supported: {})",
142            f.kind,
143            supported.join(", ")
144        )));
145    }
146    if !opts.actions.is_empty() {
147        if !engine.supports_actions() {
148            return Err(ApiError::unsupported(engine.actions_unavailable()));
149        }
150        if opts.engine.as_deref() == Some("http") {
151            return Err(ApiError::invalid(
152                "actions run in a browser and cannot be combined with engine \"http\"",
153            ));
154        }
155        if opts.respond_timing.as_deref() == Some("html") {
156            return Err(ApiError::invalid(
157                "actions run in a browser and cannot be combined with respondTiming \"html\"",
158            ));
159        }
160    }
161    if let Some(t) = opts.timeout {
162        if t < 1000 {
163            return Err(ApiError::invalid("timeout must be at least 1000 ms"));
164        }
165        if t > MAX_SCRAPE_TIMEOUT_MS {
166            return Err(ApiError::invalid(format!(
167                "timeout must not exceed {MAX_SCRAPE_TIMEOUT_MS} ms"
168            )));
169        }
170        if opts.wait_for > t / 2 {
171            return Err(ApiError::invalid("waitFor must not exceed half of timeout"));
172        }
173    }
174    if opts.wait_for > MAX_WAIT_FOR_MS {
175        return Err(ApiError::invalid(format!(
176            "waitFor must not exceed {MAX_WAIT_FOR_MS} ms"
177        )));
178    }
179    // 抓到的是 PDF 时才用得上，但写错的 parsers 在抓取前就报出来（parse 也走这里）
180    PdfParser::from_parsers(opts.parsers.as_ref())?;
181    Ok(())
182}
183
184/// URL 数量的上限检查：batch scrape 和 extract 共用。
185pub fn check_url_count(n: usize) -> Result<(), ApiError> {
186    if n > MAX_URLS {
187        return Err(ApiError::invalid(format!(
188            "too many URLs: {n}, the maximum is {MAX_URLS}"
189        )));
190    }
191    Ok(())
192}
193
194/// `/v2/parse` 上传文件的大小上限，和 Firecrawl 一致（50 MB）。
195pub const MAX_PARSE_FILE_BYTES: usize = 50 * 1024 * 1024;
196
197/// `/v2/parse` 的选项：先过和 scrape 相同的检查，再拒绝只对网址有意义的选项
198/// （照 Firecrawl `getUnsupportedParseOptionError`，另加 pagesift 自己的抓取类扩展）。
199pub fn check_parse_options(opts: &ScrapeOptions, engine: &dyn Engine) -> Result<(), ApiError> {
200    check_scrape_options(opts, engine)?;
201    for kind in ["screenshot", "branding", "changeTracking"] {
202        if opts.wants(kind) {
203            return Err(ApiError::unsupported(format!(
204                "parse uploads do not support the {kind} format"
205            )));
206        }
207    }
208    let unsupported = [
209        (!opts.actions.is_empty(), "actions"),
210        (opts.wait_for > 0, "waitFor"),
211        (opts.location.is_some(), "location"),
212        (opts.mobile, "mobile"),
213        (
214            opts.engine.as_deref() == Some("browser"),
215            "engine \"browser\"",
216        ),
217        (opts.egress.is_some(), "egress"),
218        (opts.discover_scripts, "discoverScripts"),
219        (
220            opts.if_none_match.is_some() || opts.if_modified_since.is_some(),
221            "conditional requests",
222        ),
223        (
224            opts.proxy
225                .as_deref()
226                .is_some_and(|p| !matches!(p, "basic" | "auto")),
227            "proxy other than basic or auto",
228        ),
229    ];
230    if let Some((_, what)) = unsupported.iter().find(|(on, _)| *on) {
231        return Err(ApiError::unsupported(format!(
232            "parse uploads do not support {what}"
233        )));
234    }
235    Ok(())
236}
pagesift_api/validate.rs

pagesift_api/
validate.rs
