pagesift_api/
validate.rs

1//! 请求校验：绕过类参数和引擎做不到的功能一律明确报错（PRD 6.5.1），不悄悄降级。
2
3use crate::engine::Engine;
4use crate::error::ApiError;
5use crate::types::ScrapeOptions;
6
7pub fn check_url(raw: &str) -> Result<(), ApiError> {
8    match url::Url::parse(raw) {
9        Ok(u) if matches!(u.scheme(), "http" | "https") && u.host().is_some() => Ok(()),
10        Ok(_) => Err(ApiError::invalid(format!(
11            "only http(s) URLs are supported: {raw}"
12        ))),
13        Err(e) => Err(ApiError::invalid(format!("invalid URL {raw:?}: {e}"))),
14    }
15}
16
17pub fn check_scrape_options(opts: &ScrapeOptions, engine: &dyn Engine) -> Result<(), ApiError> {
18    if let Some(p) = opts.proxy.as_deref()
19        && !matches!(p, "basic" | "auto")
20    {
21        return Err(ApiError::unsupported(format!(
22            "proxy {p:?} is not supported: pagesift does not bypass anti-bot protections; use \"basic\" or \"auto\""
23        )));
24    }
25    if let Some(e) = opts.engine.as_deref()
26        && !matches!(e, "http" | "browser" | "auto")
27    {
28        return Err(ApiError::invalid(format!(
29            "engine must be http, browser or auto, got {e:?}"
30        )));
31    }
32    if let Some(c) = &opts.chunking {
33        const KINDS: [&str; 8] = [
34            "recursive",
35            "headings",
36            "paragraphs",
37            "sentences",
38            "fixed_tokens",
39            "sliding_window",
40            "overlapping",
41            "regex",
42        ];
43        let kind = c.get("type").and_then(|t| t.as_str());
44        if !kind.is_some_and(|k| KINDS.contains(&k)) {
45            return Err(ApiError::invalid(format!(
46                "chunking.type must be one of {}",
47                KINDS.join(", ")
48            )));
49        }
50    }
51    let supported = engine.supported_formats();
52    if let Some(f) = opts
53        .formats
54        .iter()
55        .find(|f| !supported.contains(&f.kind.as_str()))
56    {
57        return Err(ApiError::unsupported(format!(
58            "format {:?} is not supported by this build (supported: {})",
59            f.kind,
60            supported.join(", ")
61        )));
62    }
63    if !opts.actions.is_empty() && !engine.supports_actions() {
64        return Err(ApiError::unsupported(
65            "actions need browser rendering, which is not available in this build",
66        ));
67    }
68    if let Some(t) = opts.timeout {
69        if t < 1000 {
70            return Err(ApiError::invalid("timeout must be at least 1000 ms"));
71        }
72        if opts.wait_for > t / 2 {
73            return Err(ApiError::invalid("waitFor must not exceed half of timeout"));
74        }
75    }
76    if opts.wait_for > 60_000 {
77        return Err(ApiError::invalid("waitFor must not exceed 60000 ms"));
78    }
79    Ok(())
80}
pagesift_api/validate.rs

pagesift_api/
validate.rs
