crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! Scrape request DTO with input validation

use crate::common::CacheMode;
use once_cell::sync::Lazy;
use regex::Regex;
use sdforge::validator::Validate;
use serde::{Deserialize, Serialize};
use serde_json::Value;

/// Maximum allowed URL length (2048 characters)
pub const MAX_URL_LENGTH: usize = 2048;
/// Maximum number of allowed tags in include/exclude lists
pub const MAX_TAG_COUNT: usize = 50;
/// Maximum number of allowed actions
pub const MAX_ACTION_COUNT: usize = 20;
/// Maximum metadata object depth
pub const MAX_METADATA_DEPTH: usize = 5;

/// URL scheme validation: only http and https are allowed (SSRF mitigation).
static HTTP_URL_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^https?://").unwrap());

fn is_http_url(value: &str) -> Result<(), sdforge::validator::ValidationError> {
    if HTTP_URL_RE.is_match(value) {
        Ok(())
    } else {
        Err(
            sdforge::validator::ValidationError::new("invalid_url_scheme")
                .with_message("URL must start with http:// or https://".into()),
        )
    }
}

/// Validate that metadata JSON nesting depth does not exceed MAX_METADATA_DEPTH.
fn validate_metadata_depth(
    value: &serde_json::Value,
) -> Result<(), sdforge::validator::ValidationError> {
    fn depth(v: &serde_json::Value) -> usize {
        match v {
            serde_json::Value::Object(map) => 1 + map.values().map(depth).max().unwrap_or(0),
            serde_json::Value::Array(arr) => 1 + arr.iter().map(depth).max().unwrap_or(0),
            _ => 0,
        }
    }
    if depth(value) > MAX_METADATA_DEPTH {
        Err(
            sdforge::validator::ValidationError::new("metadata_too_deep").with_message(
                format!("metadata JSON depth must not exceed {}", MAX_METADATA_DEPTH).into(),
            ),
        )
    } else {
        Ok(())
    }
}

/// 爬取请求数据传输对象
///
/// 用于封装客户端发起的网页爬取请求的相关参数
/// 拒绝未知字段以增强安全性
#[derive(Debug, Deserialize, Serialize, Validate)]
#[validate(crate = "sdforge::validator")]
#[serde(deny_unknown_fields)]
pub struct ScrapeRequestDto {
    /// 要爬取的网页URL (仅支持 http/https)
    #[validate(length(min = 1, max = 2048))]
    #[validate(custom(
        function = "is_http_url",
        message = "URL must start with http:// or https://"
    ))]
    pub url: String,
    /// 请求的数据格式列表
    pub formats: Option<Vec<String>>,
    /// 包含的HTML标签列表
    #[validate(length(max = 50, message = "include_tags must have at most 50 entries"))]
    pub include_tags: Option<Vec<String>>,
    /// 排除的HTML标签列表
    #[validate(length(max = 50, message = "exclude_tags must have at most 50 entries"))]
    pub exclude_tags: Option<Vec<String>>,
    /// 回调Webhook地址
    pub webhook: Option<String>,
    /// 提取规则
    pub extraction_rules: Option<
        std::collections::HashMap<
            String,
            crate::domain::services::extraction_service::ExtractionRule,
        >,
    >,
    /// LLM Prompt 提取(自由文本描述需要提取的内容)
    pub extraction_prompt: Option<String>,
    /// JSON Schema 驱动的结构化提取
    pub extraction_schema: Option<serde_json::Value>,
    /// 页面交互动作
    #[validate(length(max = 20, message = "actions must have at most 20 entries"))]
    pub actions: Option<Vec<ScrapeActionDto>>,
    /// 抓取选项
    #[validate(nested)]
    pub options: Option<ScrapeOptionsDto>,
    /// 自定义元数据
    #[validate(custom(function = "validate_metadata_depth"))]
    pub metadata: Option<serde_json::Value>,
    /// 同步等待时长(毫秒,默认 5000,最大 30000)
    #[validate(range(
        min = 0,
        max = 30000,
        message = "sync_wait_ms must be between 0 and 30000"
    ))]
    pub sync_wait_ms: Option<u32>,
}

#[derive(Debug, Deserialize, Serialize, Default, Validate)]
#[validate(crate = "sdforge::validator")]
#[serde(deny_unknown_fields)]
pub struct ScrapeOptionsDto {
    /// 自定义HTTP请求头
    pub headers: Option<Value>,
    /// 等待时间(毫秒)
    #[validate(range(max = 60000, message = "wait_for must be at most 60000ms"))]
    pub wait_for: Option<u64>,
    /// 响应级等待:等待某响应 URL 匹配 regex(regex crate 语法;与 wait_for /
    /// wait_for_response_body 互斥;Playwright 支持,moli 返回 Unsupported)
    pub wait_for_response_url: Option<String>,
    /// 响应级等待:等待某响应体包含指定文本(与 wait_for / wait_for_response_url 互斥;
    /// 单响应体匹配检查上限 10 MiB)
    pub wait_for_response_body: Option<String>,
    /// 超时时间(秒)
    #[validate(range(max = 300, message = "timeout must be at most 300s"))]
    pub timeout: Option<u64>,
    /// 是否需要JavaScript渲染
    pub js_rendering: Option<bool>,
    /// 是否需要截图
    pub screenshot: Option<bool>,
    /// 截图配置
    pub screenshot_options: Option<ScreenshotOptionsDto>,
    /// 是否模拟移动设备
    pub mobile: Option<bool>,
    /// 代理配置 (URL)
    pub proxy: Option<String>,
    /// 是否跳过TLS验证
    pub skip_tls_verification: Option<bool>,
    /// 是否需要TLS指纹对抗
    pub needs_tls_fingerprint: Option<bool>,
    /// 是否使用Fire Engine (CDP)
    pub use_fire_engine: Option<bool>,
    /// 缓存模式
    ///
    /// 控制本次抓取的缓存读写行为。`None`(默认)等价于 `Some(CacheMode::Enabled)`。
    /// 详见 [`crate::common::CacheMode`]。
    ///
    /// 与 `bypass_cache` 的优先级:`bypass_cache=Some(true)` 覆盖 `cache_mode` 为
    /// `Some(CacheMode::Bypass)`(应急绕过读,正常写回)。
    pub cache_mode: Option<CacheMode>,
    /// 应急绕过缓存读
    ///
    /// `Some(true)` → 等价于 `cache_mode=Some(CacheMode::Bypass)`(跳过读,正常写回),
    /// 用于运行时不信任缓存脏数据的应急场景。`Some(false)` 或 `None` → 忽略,按 `cache_mode` 走。
    ///
    /// 详见 [`crate::common::CacheMode::Bypass`]。
    pub bypass_cache: Option<bool>,
    /// 仅转换正文内容
    ///
    /// `Some(true)` 时 Markdown 转换前先经 `ContentExtractionFacade` 提取正文 HTML,
    /// 去除 nav/footer/sidebar/ads 等非主体内容。`None` 或 `Some(false)` 时整页转换。
    ///
    /// 需要 `markdown` 特性 + `formats` 含 `"markdown"` 生效。
    pub only_main_content: Option<bool>,
    /// 是否遵从目标站点 robots.txt
    ///
    /// 缺省回退全局 `robots.scrape_respect_robots`(默认 false)。
    /// 命中 Disallow 时任务置 Failed(error 可观测);robots.txt 获取失败
    /// fail-open 放行(与 crawl 路径一致)。单页抓取不执行 Crawl-delay。
    pub respect_robots: Option<bool>,
    /// 条件请求(增量抓取,scoped 切片)
    ///
    /// `Some(true)` 时 worker 查询同任务的历史结果,携带 `If-None-Match` /
    /// `If-Modified-Since` 发起条件请求;目标站返回 `304` 时复用历史结果
    /// 内容(响应头附 `x-crawlrs-not-modified: true`),不重复传输正文。
    /// 历史结果缺验证器或查询失败时降级为无条件抓取。默认 `None`(关闭)。
    #[serde(default)]
    pub conditional_request: Option<bool>,
    /// 屏蔽广告请求
    ///
    /// `Some(true)` 时浏览器引擎拦截常见广告域请求(engine 层能力,见
    /// `ScrapeRequest.block_ads`)。默认 `None`(关闭)。
    #[serde(default)]
    pub block_ads: Option<bool>,
    /// 屏蔽媒体资源请求
    ///
    /// `Some(true)` 时浏览器引擎拦截图片/视频/音频等媒体资源加载,加速
    /// 纯文本抓取(engine 层能力,见 `ScrapeRequest.block_media`)。
    /// 默认 `None`(关闭)。
    #[serde(default)]
    pub block_media: Option<bool>,
    /// 是否需要 MLLM 视觉理解
    ///
    /// `Some(true)` 时引擎对截图执行多模态大模型分析(需要 `llm` 特性;
    /// 未启用时引擎侧显性报错而非静默跳过)。默认 `None`(关闭)。
    #[serde(default)]
    pub needs_mllm: Option<bool>,
}

impl ScrapeOptionsDto {
    /// 计算 `cache_mode` 与 `bypass_cache` 桥接后的有效缓存模式
    ///
    /// 优先级:`bypass_cache=Some(true)` 覆盖 `cache_mode` 为 `Bypass`;
    /// 其余情况按 `cache_mode` 走;两者皆 `None` 时返回 `None`(等价于 `Enabled`)。
    ///
    /// # 返回
    ///
    /// - `Some(CacheMode::Bypass)`:`bypass_cache=Some(true)`,或 `cache_mode=Some(Bypass)`
    /// - `Some(mode)`:`bypass_cache!=Some(true)` 且 `cache_mode=Some(mode)`
    /// - `None`:两者皆 `None`,由调用方 `unwrap_or_default()` 解析为 `Enabled`
    ///
    /// 此方法消除原 `create_scrape.rs` + `scrape_worker.rs` 中的重复桥接逻辑(DRY 修复)。
    #[must_use]
    pub fn effective_cache_mode(&self) -> Option<CacheMode> {
        if self.bypass_cache.unwrap_or(false) {
            Some(CacheMode::Bypass)
        } else {
            self.cache_mode
        }
    }
}

#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(tag = "type", rename_all = "camelCase")]
pub enum ScrapeActionDto {
    Wait { milliseconds: u64 },
    Click { selector: String },
    Scroll { direction: String },
    Screenshot { full_page: Option<bool> },
    Input { selector: String, text: String },
}

#[derive(Debug, Deserialize, Serialize)]
pub struct ScreenshotOptionsDto {
    pub full_page: Option<bool>,
    pub selector: Option<String>,
    pub quality: Option<u8>,
    pub format: Option<String>,
}
#[cfg(test)]
mod respect_robots_tests {
    use super::*;

    /// respect_robots 缺省为 None(回退全局默认);显式传值正确反序列化。
    #[test]
    fn respect_robots_field_serde() {
        let without: ScrapeRequestDto =
            serde_json::from_str(r#"{"url":"https://text.npr.org"}"#).expect("parse minimal");
        let with_none = without.options.as_ref().and_then(|o| o.respect_robots);
        assert_eq!(with_none, None, "未传 options 时 respect_robots 应为 None");

        let with: ScrapeRequestDto = serde_json::from_str(
            r#"{"url":"https://text.npr.org","options":{"respect_robots":true}}"#,
        )
        .expect("parse with respect_robots");
        assert_eq!(
            with.options.as_ref().unwrap().respect_robots,
            Some(true),
            "respect_robots=true 应正确反序列化"
        );

        let with_false: ScrapeRequestDto = serde_json::from_str(
            r#"{"url":"https://text.npr.org","options":{"respect_robots":false}}"#,
        )
        .expect("parse with respect_robots=false");
        assert_eq!(
            with_false.options.as_ref().unwrap().respect_robots,
            Some(false),
            "respect_robots=false 应正确反序列化(用于覆盖全局默认 true)"
        );
    }

    /// 三分支生效值:请求级 override 优先于全局默认(语义在
    /// `scrape_executor::check_scrape_robots_allowed`,此处固化
    /// `unwrap_or` 的选择逻辑本身)。
    #[test]
    fn respect_robots_override_resolution() {
        let global_default = true;
        let resolve = |override_value: Option<bool>| override_value.unwrap_or(global_default);
        assert!(resolve(Some(true)), "Some(true) 应覆盖全局默认");
        assert!(!resolve(Some(false)), "Some(false) 应覆盖全局默认");
        assert!(resolve(None), "None 应回退全局默认");
    }
}