pagesift_api/
validate.rs

1//! 请求校验：绕过类参数和引擎做不到的功能一律明确报错（PRD 6.5.1），不悄悄降级。
2
3use crate::engine::Engine;
4use crate::error::ApiError;
5use crate::types::{PdfParser, ScrapeOptions};
6
7/// URL 最长 8 KB：再长的 URL 浏览器和多数服务器都不接受，只会白白占内存。
8pub const MAX_URL_LEN: usize = 8 * 1024;
9/// 同步 scrape 的 `timeout` 上限（毫秒）；整请求超时按它留余量。
10pub const MAX_SCRAPE_TIMEOUT_MS: u64 = 300_000;
11/// `waitFor` 上限（毫秒）。
12pub const MAX_WAIT_FOR_MS: u64 = 60_000;
13/// crawl 的 `limit` 上限。
14pub const MAX_CRAWL_LIMIT: usize = 100_000;
15/// crawl 的 `maxDiscoveryDepth` / `maxDepth` 上限。
16pub const MAX_CRAWL_DEPTH: u64 = 100;
17/// map 的 `limit` 上限，和 Firecrawl 文档一致。
18pub const MAX_MAP_LIMIT: usize = 100_000;
19/// batch scrape、extract 一次最多带多少个 URL。
20pub const MAX_URLS: usize = 10_000;
21
22pub fn check_url(raw: &str) -> Result<(), ApiError> {
23    if raw.len() > MAX_URL_LEN {
24        return Err(ApiError::invalid(format!(
25            "URL is too long: {} bytes, the maximum is {MAX_URL_LEN}",
26            raw.len()
27        )));
28    }
29    match url::Url::parse(raw) {
30        Ok(u) if matches!(u.scheme(), "http" | "https") && u.host().is_some() => Ok(()),
31        Ok(_) => Err(ApiError::invalid(format!(
32            "only http(s) URLs are supported: {raw}"
33        ))),
34        Err(e) => Err(ApiError::invalid(format!("invalid URL {raw:?}: {e}"))),
35    }
36}
37
38pub fn check_scrape_options(opts: &ScrapeOptions, engine: &dyn Engine) -> Result<(), ApiError> {
39    if opts.discover_scripts
40        && (opts.engine.as_deref() != Some("http")
41            || !opts.wants("links")
42            || opts.egress.is_some()
43            || !opts.headers.is_empty()
44            || opts.if_none_match.is_some()
45            || opts.if_modified_since.is_some())
46    {
47        return Err(ApiError::unsupported(
48            "discoverScripts requires HTTP engine, links, and no custom headers/egress/conditional request",
49        ));
50    }
51    if let Some(p) = opts.proxy.as_deref()
52        && !matches!(p, "basic" | "auto")
53    {
54        let valid_url = url::Url::parse(p)
55            .ok()
56            .is_some_and(|u| matches!(u.scheme(), "http" | "https" | "socks5" | "socks5h"));
57        if !valid_url {
58            return Err(ApiError::unsupported(format!(
59                "proxy {p:?} is not supported: use \"basic\", \"auto\", or a valid http/https/socks5 URL"
60            )));
61        }
62    }
63    if let Some(e) = opts.engine.as_deref()
64        && !matches!(e, "http" | "browser" | "auto")
65    {
66        return Err(ApiError::invalid(format!(
67            "engine must be http, browser or auto, got {e:?}"
68        )));
69    }
70    if let Some(c) = &opts.chunking {
71        const KINDS: [&str; 8] = [
72            "recursive",
73            "headings",
74            "paragraphs",
75            "sentences",
76            "fixed_tokens",
77            "sliding_window",
78            "overlapping",
79            "regex",
80        ];
81        let kind = c.get("type").and_then(|t| t.as_str());
82        if !kind.is_some_and(|k| KINDS.contains(&k)) {
83            return Err(ApiError::invalid(format!(
84                "chunking.type must be one of {}",
85                KINDS.join(", ")
86            )));
87        }
88    }
89    let supported = engine.supported_formats();
90    if let Some(f) = opts
91        .formats
92        .iter()
93        .find(|f| !supported.contains(&f.kind.as_str()))
94    {
95        return Err(ApiError::unsupported(format!(
96            "format {:?} is not supported by this build (supported: {})",
97            f.kind,
98            supported.join(", ")
99        )));
100    }
101    if !opts.actions.is_empty() && !engine.supports_actions() {
102        return Err(ApiError::unsupported(
103            "actions need browser rendering, which is not available in this build",
104        ));
105    }
106    if let Some(t) = opts.timeout {
107        if t < 1000 {
108            return Err(ApiError::invalid("timeout must be at least 1000 ms"));
109        }
110        if t > MAX_SCRAPE_TIMEOUT_MS {
111            return Err(ApiError::invalid(format!(
112                "timeout must not exceed {MAX_SCRAPE_TIMEOUT_MS} ms"
113            )));
114        }
115        if opts.wait_for > t / 2 {
116            return Err(ApiError::invalid("waitFor must not exceed half of timeout"));
117        }
118    }
119    if opts.wait_for > MAX_WAIT_FOR_MS {
120        return Err(ApiError::invalid(format!(
121            "waitFor must not exceed {MAX_WAIT_FOR_MS} ms"
122        )));
123    }
124    // 抓到的是 PDF 时才用得上，但写错的 parsers 在抓取前就报出来（parse 也走这里）
125    PdfParser::from_parsers(opts.parsers.as_ref())?;
126    Ok(())
127}
128
129/// URL 数量的上限检查：batch scrape 和 extract 共用。
130pub fn check_url_count(n: usize) -> Result<(), ApiError> {
131    if n > MAX_URLS {
132        return Err(ApiError::invalid(format!(
133            "too many URLs: {n}, the maximum is {MAX_URLS}"
134        )));
135    }
136    Ok(())
137}
138
139/// `/v2/parse` 上传文件的大小上限，和 Firecrawl 一致（50 MB）。
140pub const MAX_PARSE_FILE_BYTES: usize = 50 * 1024 * 1024;
141
142/// `/v2/parse` 的选项：先过和 scrape 相同的检查，再拒绝只对网址有意义的选项
143/// （照 Firecrawl `getUnsupportedParseOptionError`，另加 pagesift 自己的抓取类扩展）。
144pub fn check_parse_options(opts: &ScrapeOptions, engine: &dyn Engine) -> Result<(), ApiError> {
145    check_scrape_options(opts, engine)?;
146    for kind in ["screenshot", "branding", "changeTracking"] {
147        if opts.wants(kind) {
148            return Err(ApiError::unsupported(format!(
149                "parse uploads do not support the {kind} format"
150            )));
151        }
152    }
153    let unsupported = [
154        (!opts.actions.is_empty(), "actions"),
155        (opts.wait_for > 0, "waitFor"),
156        (opts.location.is_some(), "location"),
157        (opts.mobile, "mobile"),
158        (
159            opts.engine.as_deref() == Some("browser"),
160            "engine \"browser\"",
161        ),
162        (opts.egress.is_some(), "egress"),
163        (opts.discover_scripts, "discoverScripts"),
164        (
165            opts.if_none_match.is_some() || opts.if_modified_since.is_some(),
166            "conditional requests",
167        ),
168        (
169            opts.proxy
170                .as_deref()
171                .is_some_and(|p| !matches!(p, "basic" | "auto")),
172            "proxy other than basic or auto",
173        ),
174    ];
175    if let Some((_, what)) = unsupported.iter().find(|(on, _)| *on) {
176        return Err(ApiError::unsupported(format!(
177            "parse uploads do not support {what}"
178        )));
179    }
180    Ok(())
181}
pagesift_api/validate.rs

pagesift_api/
validate.rs
