use crate::utils::SafeUrl;
use once_cell::sync::Lazy;
use regex::Regex;
use sdforge::validator::Validate;
use serde::{Deserialize, Serialize};
pub const MAX_CRAWL_DEPTH: u32 = 5;
pub const MAX_CONCURRENCY: u32 = 50;
static HTTP_URL_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^https?://").unwrap());
pub(crate) fn is_http_url(value: &str) -> Result<(), sdforge::validator::ValidationError> {
if HTTP_URL_RE.is_match(value) {
Ok(())
} else {
Err(
sdforge::validator::ValidationError::new("invalid_url_scheme")
.with_message("URL must start with http:// or https://".into()),
)
}
}
fn is_http_sitemap_url(value: &str) -> Result<(), sdforge::validator::ValidationError> {
if HTTP_URL_RE.is_match(value) {
Ok(())
} else {
Err(
sdforge::validator::ValidationError::new("invalid_url_scheme")
.with_message(crate::i18n::tr_log("validation-sitemap-url-scheme").into()),
)
}
}
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
#[validate(crate = "sdforge::validator")]
#[serde(deny_unknown_fields)]
pub struct CrawlRequestDto {
#[validate(length(
min = 1,
max = 2048,
message = "URL length must be between 1 and 2048 characters"
))]
#[validate(custom(
function = "is_http_url",
message = "URL must start with http:// or https://"
))]
pub url: String,
#[serde(skip)]
pub validated_url: Option<SafeUrl>,
#[validate(length(
min = 1,
max = 255,
message = "Task name length must be between 1 and 255 characters"
))]
pub name: Option<String>,
#[validate(nested)]
pub config: CrawlConfigDto,
#[validate(range(
min = 0,
max = 30000,
message = "sync_wait_ms must be between 0 and 30000"
))]
pub sync_wait_ms: Option<u32>,
pub expires_at: Option<chrono::DateTime<chrono::Utc>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sitemap_seed: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[validate(custom(function = "is_http_sitemap_url"))]
pub sitemap_url: Option<String>,
}
#[derive(Debug, Deserialize, Serialize, Clone, Validate)]
#[validate(crate = "sdforge::validator")]
#[serde(deny_unknown_fields)]
pub struct CrawlConfigDto {
#[validate(range(max = 5, message = "max_depth must be between 0 and 5"))]
pub max_depth: u32,
pub include_patterns: Option<Vec<String>>,
pub exclude_patterns: Option<Vec<String>>,
pub strategy: Option<String>,
#[validate(range(max = 86400000, message = "crawl_delay_ms must be at most 86400000"))]
pub crawl_delay_ms: Option<u64>,
#[validate(range(max = 50, message = "max_concurrency must be at most 50"))]
pub max_concurrency: Option<u32>,
#[validate(range(max = 100000, message = "max_pages must be at most 100000"))]
pub max_pages: Option<u32>,
pub proxy: Option<String>,
pub headers: Option<serde_json::Value>,
pub extraction_rules: Option<
std::collections::HashMap<
String,
crate::domain::services::extraction_service::ExtractionRule,
>,
>,
pub extraction_prompt: Option<String>,
pub extraction_schema: Option<serde_json::Value>,
}