use crate::common::CacheMode;
use once_cell::sync::Lazy;
use regex::Regex;
use sdforge::validator::Validate;
use serde::{Deserialize, Serialize};
use serde_json::Value;
pub const MAX_URL_LENGTH: usize = 2048;
pub const MAX_TAG_COUNT: usize = 50;
pub const MAX_ACTION_COUNT: usize = 20;
pub const MAX_METADATA_DEPTH: usize = 5;
static HTTP_URL_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^https?://").unwrap());
fn is_http_url(value: &str) -> Result<(), sdforge::validator::ValidationError> {
if HTTP_URL_RE.is_match(value) {
Ok(())
} else {
Err(
sdforge::validator::ValidationError::new("invalid_url_scheme")
.with_message("URL must start with http:// or https://".into()),
)
}
}
fn validate_metadata_depth(
value: &serde_json::Value,
) -> Result<(), sdforge::validator::ValidationError> {
fn depth(v: &serde_json::Value) -> usize {
match v {
serde_json::Value::Object(map) => 1 + map.values().map(depth).max().unwrap_or(0),
serde_json::Value::Array(arr) => 1 + arr.iter().map(depth).max().unwrap_or(0),
_ => 0,
}
}
if depth(value) > MAX_METADATA_DEPTH {
Err(
sdforge::validator::ValidationError::new("metadata_too_deep").with_message(
format!("metadata JSON depth must not exceed {}", MAX_METADATA_DEPTH).into(),
),
)
} else {
Ok(())
}
}
#[derive(Debug, Deserialize, Serialize, Validate)]
#[validate(crate = "sdforge::validator")]
#[serde(deny_unknown_fields)]
pub struct ScrapeRequestDto {
#[validate(length(min = 1, max = 2048))]
#[validate(custom(
function = "is_http_url",
message = "URL must start with http:// or https://"
))]
pub url: String,
pub formats: Option<Vec<String>>,
#[validate(length(max = 50, message = "include_tags must have at most 50 entries"))]
pub include_tags: Option<Vec<String>>,
#[validate(length(max = 50, message = "exclude_tags must have at most 50 entries"))]
pub exclude_tags: Option<Vec<String>>,
pub webhook: Option<String>,
pub extraction_rules: Option<
std::collections::HashMap<
String,
crate::domain::services::extraction_service::ExtractionRule,
>,
>,
pub extraction_prompt: Option<String>,
pub extraction_schema: Option<serde_json::Value>,
#[validate(length(max = 20, message = "actions must have at most 20 entries"))]
pub actions: Option<Vec<ScrapeActionDto>>,
#[validate(nested)]
pub options: Option<ScrapeOptionsDto>,
#[validate(custom(function = "validate_metadata_depth"))]
pub metadata: Option<serde_json::Value>,
#[validate(range(
min = 0,
max = 30000,
message = "sync_wait_ms must be between 0 and 30000"
))]
pub sync_wait_ms: Option<u32>,
}
#[derive(Debug, Deserialize, Serialize, Default, Validate)]
#[validate(crate = "sdforge::validator")]
#[serde(deny_unknown_fields)]
pub struct ScrapeOptionsDto {
pub headers: Option<Value>,
#[validate(range(max = 60000, message = "wait_for must be at most 60000ms"))]
pub wait_for: Option<u64>,
pub wait_for_response_url: Option<String>,
pub wait_for_response_body: Option<String>,
#[validate(range(max = 300, message = "timeout must be at most 300s"))]
pub timeout: Option<u64>,
pub js_rendering: Option<bool>,
pub screenshot: Option<bool>,
pub screenshot_options: Option<ScreenshotOptionsDto>,
pub mobile: Option<bool>,
pub proxy: Option<String>,
pub skip_tls_verification: Option<bool>,
pub needs_tls_fingerprint: Option<bool>,
pub use_fire_engine: Option<bool>,
pub cache_mode: Option<CacheMode>,
pub bypass_cache: Option<bool>,
pub only_main_content: Option<bool>,
pub respect_robots: Option<bool>,
#[serde(default)]
pub conditional_request: Option<bool>,
#[serde(default)]
pub block_ads: Option<bool>,
#[serde(default)]
pub block_media: Option<bool>,
#[serde(default)]
pub needs_mllm: Option<bool>,
}
impl ScrapeOptionsDto {
#[must_use]
pub fn effective_cache_mode(&self) -> Option<CacheMode> {
if self.bypass_cache.unwrap_or(false) {
Some(CacheMode::Bypass)
} else {
self.cache_mode
}
}
}
#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(tag = "type", rename_all = "camelCase")]
pub enum ScrapeActionDto {
Wait { milliseconds: u64 },
Click { selector: String },
Scroll { direction: String },
Screenshot { full_page: Option<bool> },
Input { selector: String, text: String },
}
#[derive(Debug, Deserialize, Serialize)]
pub struct ScreenshotOptionsDto {
pub full_page: Option<bool>,
pub selector: Option<String>,
pub quality: Option<u8>,
pub format: Option<String>,
}
#[cfg(test)]
mod respect_robots_tests {
use super::*;
#[test]
fn respect_robots_field_serde() {
let without: ScrapeRequestDto =
serde_json::from_str(r#"{"url":"https://text.npr.org"}"#).expect("parse minimal");
let with_none = without.options.as_ref().and_then(|o| o.respect_robots);
assert_eq!(with_none, None, "未传 options 时 respect_robots 应为 None");
let with: ScrapeRequestDto = serde_json::from_str(
r#"{"url":"https://text.npr.org","options":{"respect_robots":true}}"#,
)
.expect("parse with respect_robots");
assert_eq!(
with.options.as_ref().unwrap().respect_robots,
Some(true),
"respect_robots=true 应正确反序列化"
);
let with_false: ScrapeRequestDto = serde_json::from_str(
r#"{"url":"https://text.npr.org","options":{"respect_robots":false}}"#,
)
.expect("parse with respect_robots=false");
assert_eq!(
with_false.options.as_ref().unwrap().respect_robots,
Some(false),
"respect_robots=false 应正确反序列化(用于覆盖全局默认 true)"
);
}
#[test]
fn respect_robots_override_resolution() {
let global_default = true;
let resolve = |override_value: Option<bool>| override_value.unwrap_or(global_default);
assert!(resolve(Some(true)), "Some(true) 应覆盖全局默认");
assert!(!resolve(Some(false)), "Some(false) 应覆盖全局默认");
assert!(resolve(None), "None 应回退全局默认");
}
}