use std::collections::HashMap;
use std::time::Duration;
use anyhow::{Context, Result};
use serde_json::json;
use crate::application::dto::crawl_request::CrawlConfigDto;
use crate::application::dto::extract_request::ExtractRequestDto;
use crate::common::HttpMethod;
use crate::domain::models::Task;
use crate::engines::engine_client::{ScrapeOptions, ScrapeRequest};
pub fn parse_crawl_payload(task: &Task) -> Result<(crate::domain::id::Id, u32, CrawlConfigDto)> {
let payload = &task.payload;
let crawl_id = match payload.get("crawl_id").and_then(|v| v.as_str()) {
Some(id) => (id).parse::<crate::domain::id::Id>().unwrap_or_default(),
None => {
return Err(anyhow::anyhow!("Missing crawl_id in task payload"));
}
};
let depth = payload.get("depth").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
let config: CrawlConfigDto =
serde_json::from_value(payload.get("config").cloned().unwrap_or(json!({})))?;
Ok((crawl_id, depth, config))
}
pub fn parse_extract_payload(task: &Task) -> Result<(ExtractRequestDto, String)> {
let payload: ExtractRequestDto = serde_json::from_value(task.payload.clone())
.context("Failed to parse extract task input")?;
let url = payload.urls.first().context("No URL provided")?.clone();
Ok((payload, url))
}
pub fn build_crawl_request(
task: &Task,
config: &CrawlConfigDto,
timeout_seconds: u64,
) -> ScrapeRequest {
let mut headers = HashMap::with_capacity(16);
if let Some(h) = &config.headers {
if let Some(obj) = h.as_object() {
for (k, v) in obj {
if k.eq_ignore_ascii_case("host") {
log::warn!(
"ignoring custom Host header for crawl task {} (target host is fixed by URL)",
task.id
);
continue;
}
if let Some(s) = v.as_str() {
headers.insert(k.clone(), s.to_string());
}
}
}
}
ScrapeRequest::new(task.url.clone()).with_options(ScrapeOptions {
method: HttpMethod::Get,
body: None,
headers,
timeout: Duration::from_secs(timeout_seconds),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: config.proxy.clone(),
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: Vec::new(),
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
cache_mode: None,
wait_for: None,
needs_mllm: false,
})
}
pub fn build_extract_request(url: &str, timeout_seconds: u64) -> ScrapeRequest {
ScrapeRequest::new(url.to_string()).with_options(ScrapeOptions {
method: HttpMethod::Get,
body: None,
headers: HashMap::new(),
timeout: Duration::from_secs(timeout_seconds),
needs_js: false,
needs_screenshot: false,
screenshot_config: None,
mobile: false,
proxy: None,
skip_tls_verification: false,
needs_tls_fingerprint: false,
use_fire_engine: false,
actions: vec![],
sync_wait_ms: 0,
block_ads: false,
block_media: false,
session_id: None,
cache_mode: None,
wait_for: None,
needs_mllm: false,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::{TaskStatus, TaskType};
use chrono::Utc;
fn make_task(payload: serde_json::Value) -> Task {
Task {
id: crate::domain::id::Id::generate(),
task_type: TaskType::Scrape,
status: TaskStatus::Queued,
priority: 0,
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
url: "https://example.com".to_string(),
payload,
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
created_at: Utc::now(),
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: Utc::now(),
lock_token: None,
lock_expires_at: None,
expires_at: None,
}
}
#[test]
fn test_parse_crawl_payload_valid() {
let crawl_id = crate::domain::id::Id::generate();
let payload = json!({
"crawl_id": crawl_id.to_string(),
"depth": 2,
"config": { "max_depth": 5 }
});
let task = make_task(payload);
let (id, depth, config) = parse_crawl_payload(&task).unwrap();
assert_eq!(id, crawl_id);
assert_eq!(depth, 2);
assert_eq!(config.max_depth, 5);
}
#[test]
fn test_parse_crawl_payload_missing_crawl_id() {
let payload = json!({ "depth": 0 });
let task = make_task(payload);
assert!(parse_crawl_payload(&task).is_err());
}
#[test]
fn test_parse_crawl_payload_defaults() {
let crawl_id = crate::domain::id::Id::generate();
let payload = json!({
"crawl_id": crawl_id.to_string(),
"config": { "max_depth": 5 }
});
let task = make_task(payload);
let (id, depth, _config) = parse_crawl_payload(&task).unwrap();
assert_eq!(id, crawl_id);
assert_eq!(depth, 0);
}
#[test]
fn test_parse_extract_payload_valid() {
let payload = json!({
"urls": ["https://example.com"],
"rules": null,
"prompt": null,
"schema": null
});
let task = make_task(payload);
let (dto, url) = parse_extract_payload(&task).unwrap();
assert_eq!(url, "https://example.com");
assert!(dto.rules.is_none());
}
#[test]
fn test_parse_extract_payload_no_urls() {
let payload = json!({ "urls": [] });
let task = make_task(payload);
assert!(parse_extract_payload(&task).is_err());
}
#[test]
fn test_build_crawl_request_basic() {
let task = make_task(json!({}));
let config = CrawlConfigDto {
max_depth: 3,
max_concurrency: None,
max_pages: None,
include_patterns: None,
exclude_patterns: None,
headers: None,
proxy: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
strategy: None,
crawl_delay_ms: None,
};
let req = build_crawl_request(&task, &config, 30);
assert_eq!(req.url, "https://example.com");
assert_eq!(req.options.timeout, Duration::from_secs(30));
assert!(!req.options.needs_js);
}
#[test]
fn test_build_crawl_request_with_headers() {
let task = make_task(json!({}));
let config = CrawlConfigDto {
max_depth: 3,
max_concurrency: None,
max_pages: None,
include_patterns: None,
exclude_patterns: None,
headers: Some(json!({"X-Custom": "value"})),
proxy: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
strategy: None,
crawl_delay_ms: None,
};
let req = build_crawl_request(&task, &config, 30);
assert_eq!(
req.options.headers.get("X-Custom").map(|s| s.as_str()),
Some("value")
);
}
#[test]
fn test_build_extract_request_basic() {
let req = build_extract_request("https://example.com/page", 60);
assert_eq!(req.url, "https://example.com/page");
assert_eq!(req.options.timeout, Duration::from_secs(60));
assert!(!req.options.skip_tls_verification);
assert!(req.options.proxy.is_none());
}
#[test]
fn build_crawl_request_drops_custom_host_header() {
let task = make_task(json!({}));
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: Some(json!({
"Host": "internal.vhost",
"X-Test": "ok"
})),
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let request = build_crawl_request(&task, &config, 30);
assert!(
!request
.options
.headers
.keys()
.any(|k| k.eq_ignore_ascii_case("host")),
"custom Host must never reach the engine"
);
assert_eq!(
request.options.headers.get("X-Test").map(String::as_str),
Some("ok"),
"other headers are preserved"
);
}
}