pagesift_api/
validate.rs

1//! 请求校验：绕过类参数和引擎做不到的功能一律明确报错（PRD 6.5.1），不悄悄降级。
2
3use crate::engine::Engine;
4use crate::error::ApiError;
5use crate::types::ScrapeOptions;
6
7pub fn check_url(raw: &str) -> Result<(), ApiError> {
8    match url::Url::parse(raw) {
9        Ok(u) if matches!(u.scheme(), "http" | "https") && u.host().is_some() => Ok(()),
10        Ok(_) => Err(ApiError::invalid(format!(
11            "only http(s) URLs are supported: {raw}"
12        ))),
13        Err(e) => Err(ApiError::invalid(format!("invalid URL {raw:?}: {e}"))),
14    }
15}
16
17pub fn check_scrape_options(opts: &ScrapeOptions, engine: &dyn Engine) -> Result<(), ApiError> {
18    if opts.discover_scripts
19        && (opts.engine.as_deref() != Some("http")
20            || !opts.wants("links")
21            || opts.egress.is_some()
22            || !opts.headers.is_empty()
23            || opts.if_none_match.is_some()
24            || opts.if_modified_since.is_some())
25    {
26        return Err(ApiError::unsupported(
27            "discoverScripts requires HTTP engine, links, and no custom headers/egress/conditional request",
28        ));
29    }
30    if let Some(p) = opts.proxy.as_deref()
31        && !matches!(p, "basic" | "auto")
32    {
33        return Err(ApiError::unsupported(format!(
34            "proxy {p:?} is not supported: pagesift does not bypass anti-bot protections; use \"basic\" or \"auto\""
35        )));
36    }
37    if let Some(e) = opts.engine.as_deref()
38        && !matches!(e, "http" | "browser" | "auto")
39    {
40        return Err(ApiError::invalid(format!(
41            "engine must be http, browser or auto, got {e:?}"
42        )));
43    }
44    if let Some(c) = &opts.chunking {
45        const KINDS: [&str; 8] = [
46            "recursive",
47            "headings",
48            "paragraphs",
49            "sentences",
50            "fixed_tokens",
51            "sliding_window",
52            "overlapping",
53            "regex",
54        ];
55        let kind = c.get("type").and_then(|t| t.as_str());
56        if !kind.is_some_and(|k| KINDS.contains(&k)) {
57            return Err(ApiError::invalid(format!(
58                "chunking.type must be one of {}",
59                KINDS.join(", ")
60            )));
61        }
62    }
63    let supported = engine.supported_formats();
64    if let Some(f) = opts
65        .formats
66        .iter()
67        .find(|f| !supported.contains(&f.kind.as_str()))
68    {
69        return Err(ApiError::unsupported(format!(
70            "format {:?} is not supported by this build (supported: {})",
71            f.kind,
72            supported.join(", ")
73        )));
74    }
75    if !opts.actions.is_empty() && !engine.supports_actions() {
76        return Err(ApiError::unsupported(
77            "actions need browser rendering, which is not available in this build",
78        ));
79    }
80    if let Some(t) = opts.timeout {
81        if t < 1000 {
82            return Err(ApiError::invalid("timeout must be at least 1000 ms"));
83        }
84        if opts.wait_for > t / 2 {
85            return Err(ApiError::invalid("waitFor must not exceed half of timeout"));
86        }
87    }
88    if opts.wait_for > 60_000 {
89        return Err(ApiError::invalid("waitFor must not exceed 60000 ms"));
90    }
91    Ok(())
92}
pagesift_api/validate.rs

pagesift_api/
validate.rs
