use super::extraction_billing::{ExtractionSettlement, ExtractionSpec};
use super::*;
use crate::domain::models::{CrawlStatus, ScrapeResult, TaskType};
use crate::engines::EngineError;
use crate::utils::coalesce::CoalesceSignal;
use crate::workers::cache_utils::{filter_sensitive_headers, generate_scrape_cache_key};
use crate::workers::coalesce_coordinator::{CoalesceKey, CoalesceResult, RequestCoalescer};
use crate::workers::crawl::{FilterContext, UrlFilter, UrlPatternFilter};
use crate::workers::errors::ScrapeWorkerError;
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::AtomicU64;
use std::sync::Mutex;
use std::time::Duration;
struct MockCacheService {
data: Mutex<HashMap<String, String>>,
get_count: AtomicU64,
set_count: AtomicU64,
}
impl MockCacheService {
fn new() -> Self {
Self {
data: Mutex::new(HashMap::new()),
get_count: AtomicU64::new(0),
set_count: AtomicU64::new(0),
}
}
fn with_entry(key: &str, value: &str) -> Self {
let s = Self::new();
s.data
.lock()
.unwrap()
.insert(key.to_string(), value.to_string());
s
}
fn get_count(&self) -> u64 {
self.get_count.load(Ordering::Relaxed)
}
fn set_count(&self) -> u64 {
self.set_count.load(Ordering::Relaxed)
}
}
#[async_trait::async_trait]
impl CacheService for MockCacheService {
fn get(
&self,
key: &str,
) -> Pin<Box<dyn Future<Output = anyhow::Result<Option<String>>> + Send + '_>> {
self.get_count.fetch_add(1, Ordering::Relaxed);
let data = self.data.lock().unwrap().get(key).cloned();
Box::pin(async move { Ok(data) })
}
fn set(
&self,
key: &str,
value: &str,
_ttl_seconds: u64,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
self.set_count.fetch_add(1, Ordering::Relaxed);
self.data
.lock()
.unwrap()
.insert(key.to_string(), value.to_string());
Box::pin(async { Ok(()) })
}
fn delete(&self, key: &str) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send + '_>> {
self.data.lock().unwrap().remove(key);
Box::pin(async { Ok(()) })
}
fn exists(&self, key: &str) -> Pin<Box<dyn Future<Output = anyhow::Result<bool>> + Send + '_>> {
let exists = self.data.lock().unwrap().contains_key(key);
Box::pin(async move { Ok(exists) })
}
}
fn make_task(payload: Value) -> Task {
Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
payload,
)
}
fn make_coalesce_coordinator(
task_repo: Arc<dyn TaskRepository>,
result_repo: Arc<dyn ScrapeResultRepository>,
) -> Arc<CoalesceCoordinator> {
Arc::new(CoalesceCoordinator::new(
task_repo,
result_repo,
Arc::new(RequestCoalescer::new()),
))
}
fn make_coalesce_coordinator_with_coalescer(
task_repo: Arc<dyn TaskRepository>,
result_repo: Arc<dyn ScrapeResultRepository>,
request_coalescer: Arc<RequestCoalescer>,
) -> Arc<CoalesceCoordinator> {
Arc::new(CoalesceCoordinator::new(
task_repo,
result_repo,
request_coalescer,
))
}
fn parse_dto_for_test(task: &Task) -> Option<ScrapeRequestDto> {
serde_json::from_value(task.payload.clone()).ok()
}
#[cfg(feature = "metrics")]
fn make_test_memory_scheduler() -> Arc<MemoryScheduler> {
use crate::infrastructure::observability::metrics::SystemMonitorTrait;
use std::sync::atomic::{AtomicU64, Ordering};
struct StaticMockMonitor {
bits: AtomicU64,
}
impl SystemMonitorTrait for StaticMockMonitor {
fn cpu_usage(&self) -> f64 {
0.0
}
fn memory_usage(&self) -> f64 {
f64::from_bits(self.bits.load(Ordering::Relaxed))
}
fn is_metrics_stale(&self) -> bool {
false
}
}
Arc::new(MemoryScheduler::new(
Arc::new(StaticMockMonitor {
bits: AtomicU64::new(0.5f64.to_bits()),
}),
0.8,
0.9,
Duration::from_secs(30),
))
}
#[test]
fn test_filter_sensitive_headers_removes_set_cookie() {
let mut headers = HashMap::new();
headers.insert("Set-Cookie".to_string(), "session=abc123".to_string());
headers.insert("Content-Type".to_string(), "text/html".to_string());
filter_sensitive_headers(&mut headers);
assert!(!headers.contains_key("Set-Cookie"));
assert!(headers.contains_key("Content-Type"));
}
#[test]
fn test_filter_sensitive_headers_removes_authorization() {
let mut headers = HashMap::new();
headers.insert("Authorization".to_string(), "Bearer token123".to_string());
headers.insert("X-Custom".to_string(), "value".to_string());
filter_sensitive_headers(&mut headers);
assert!(!headers.contains_key("Authorization"));
assert!(headers.contains_key("X-Custom"));
}
#[test]
fn test_filter_sensitive_headers_case_insensitive() {
let mut headers = HashMap::new();
headers.insert("set-cookie".to_string(), "v1".to_string());
headers.insert("SET-COOKIE".to_string(), "v2".to_string());
headers.insert("Set-Cookie".to_string(), "v3".to_string());
filter_sensitive_headers(&mut headers);
assert!(
!headers.contains_key("set-cookie"),
"lowercase set-cookie should be filtered"
);
assert!(
!headers.contains_key("SET-COOKIE"),
"uppercase SET-COOKIE should be filtered"
);
assert!(
!headers.contains_key("Set-Cookie"),
"mixed-case Set-Cookie should be filtered"
);
}
#[test]
fn test_filter_sensitive_headers_removes_all_sensitive() {
let mut headers = HashMap::new();
headers.insert("Set-Cookie".to_string(), "v".to_string());
headers.insert("Cookie".to_string(), "v".to_string());
headers.insert("Authorization".to_string(), "v".to_string());
headers.insert("Proxy-Authorization".to_string(), "v".to_string());
headers.insert("WWW-Authenticate".to_string(), "v".to_string());
headers.insert("X-Api-Key".to_string(), "v".to_string());
headers.insert("X-Auth-Token".to_string(), "v".to_string());
headers.insert("X-Session-Id".to_string(), "v".to_string());
filter_sensitive_headers(&mut headers);
assert!(
headers.is_empty(),
"all sensitive headers should be removed"
);
}
#[test]
fn test_filter_sensitive_headers_preserves_non_sensitive() {
let mut headers = HashMap::new();
headers.insert("Content-Type".to_string(), "text/html".to_string());
headers.insert("Content-Length".to_string(), "1234".to_string());
headers.insert("Cache-Control".to_string(), "no-cache".to_string());
headers.insert("ETag".to_string(), "abc".to_string());
filter_sensitive_headers(&mut headers);
assert_eq!(headers.len(), 4);
assert!(headers.contains_key("Content-Type"));
assert!(headers.contains_key("Content-Length"));
assert!(headers.contains_key("Cache-Control"));
assert!(headers.contains_key("ETag"));
}
#[test]
fn test_filter_sensitive_headers_empty_input() {
let mut headers: HashMap<String, String> = HashMap::new();
filter_sensitive_headers(&mut headers);
assert!(headers.is_empty());
}
#[test]
fn test_build_scrape_request_minimal_payload_succeeds() {
let task = make_task(json!({"url": "https://example.com"}));
let request =
ScrapeWorker::build_scrape_request(&task).expect("minimal payload with url should succeed");
assert_eq!(request.url, "https://example.com");
assert_eq!(request.options.method, HttpMethod::Get);
assert!(request.options.body.is_none());
assert!(!request.options.needs_js);
assert!(!request.options.needs_screenshot);
assert!(!request.options.mobile);
assert!(request.options.proxy.is_none());
assert!(!request.options.skip_tls_verification);
assert!(!request.options.needs_tls_fingerprint);
assert!(!request.options.use_fire_engine);
assert_eq!(request.options.timeout, Duration::from_secs(30));
assert_eq!(request.options.sync_wait_ms, 0);
assert!(request.options.actions.is_empty());
assert!(request.options.screenshot_config.is_none());
assert!(request.options.headers.is_empty());
}
#[test]
fn test_build_scrape_request_missing_url_fails() {
let task = make_task(json!({"formats": ["html"]}));
assert!(ScrapeWorker::build_scrape_request(&task).is_err());
}
#[test]
fn test_build_scrape_request_non_object_payload_fails() {
let task = make_task(json!(42));
assert!(ScrapeWorker::build_scrape_request(&task).is_err());
}
#[test]
fn test_build_scrape_request_array_payload_fails() {
let task = make_task(json!([1, 2, 3]));
assert!(ScrapeWorker::build_scrape_request(&task).is_err());
}
#[test]
fn test_build_scrape_request_string_payload_fails() {
let task = make_task(json!("not an object"));
assert!(ScrapeWorker::build_scrape_request(&task).is_err());
}
#[test]
fn test_build_scrape_request_unknown_field_fails() {
let task = make_task(json!({"url": "https://example.com", "unknown_field": "value"}));
assert!(ScrapeWorker::build_scrape_request(&task).is_err());
}
#[test]
fn test_build_scrape_request_unknown_option_field_fails() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"bogus": 1}
}));
assert!(ScrapeWorker::build_scrape_request(&task).is_err());
}
#[test]
fn test_build_scrape_request_default_timeout_is_30_seconds() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.timeout, Duration::from_secs(30));
}
#[test]
fn test_build_scrape_request_custom_timeout() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"timeout": 120}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.timeout, Duration::from_secs(120));
}
#[test]
fn test_build_scrape_request_string_headers_are_included() {
let task = make_task(json!({
"url": "https://example.com",
"options": {
"headers": {"X-Custom": "value", "Authorization": "Bearer token"}
}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.headers.len(), 2);
assert_eq!(
request.options.headers.get("X-Custom"),
Some(&"value".to_string())
);
assert_eq!(
request.options.headers.get("Authorization"),
Some(&"Bearer token".to_string())
);
}
#[test]
fn test_build_scrape_request_non_string_headers_are_filtered() {
let task = make_task(json!({
"url": "https://example.com",
"options": {
"headers": {
"X-String": "ok",
"X-Number": 42,
"X-Bool": true,
"X-Null": null,
"X-Object": {"nested": 1}
}
}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.headers.len(), 1);
assert_eq!(
request.options.headers.get("X-String"),
Some(&"ok".to_string())
);
assert!(!request.options.headers.contains_key("X-Number"));
assert!(!request.options.headers.contains_key("X-Bool"));
assert!(!request.options.headers.contains_key("X-Null"));
assert!(!request.options.headers.contains_key("X-Object"));
}
#[test]
fn test_build_scrape_request_empty_headers_map() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"headers": {}}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.headers.is_empty());
}
#[test]
fn test_build_scrape_request_needs_js_false_by_default() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(!request.options.needs_js);
}
#[test]
fn test_build_scrape_request_needs_js_true_from_js_rendering() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"js_rendering": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.needs_js);
}
#[test]
fn test_build_scrape_request_needs_js_false_when_js_rendering_false() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"js_rendering": false}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(!request.options.needs_js);
}
#[test]
fn test_build_scrape_request_needs_js_true_when_actions_non_empty() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "wait", "milliseconds": 500}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.needs_js);
}
#[test]
fn test_build_scrape_request_needs_js_false_when_actions_empty() {
let task = make_task(json!({
"url": "https://example.com",
"actions": []
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(!request.options.needs_js);
}
#[test]
fn test_build_scrape_request_needs_js_true_empty_actions_with_js_rendering() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [],
"options": {"js_rendering": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.needs_js);
}
#[test]
fn test_build_scrape_request_screenshot_false_by_default() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(!request.options.needs_screenshot);
assert!(request.options.screenshot_config.is_none());
}
#[test]
fn test_build_scrape_request_screenshot_true_sets_flag() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"screenshot": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.needs_screenshot);
}
#[test]
fn test_build_scrape_request_screenshot_config_full_page_true() {
let task = make_task(json!({
"url": "https://example.com",
"options": {
"screenshot_options": {
"full_page": true,
"quality": 90,
"format": "png"
}
}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
let config = request
.options
.screenshot_config
.expect("screenshot_config should be set");
assert!(config.full_page);
assert_eq!(config.quality, Some(90));
assert_eq!(config.format, Some("png".to_string()));
assert!(config.selector.is_none());
}
#[test]
fn test_build_scrape_request_screenshot_config_full_page_defaults_to_false() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"screenshot_options": {}}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
let config = request
.options
.screenshot_config
.expect("screenshot_config should be set when screenshot_options is present");
assert!(!config.full_page, "full_page should default to false");
assert!(config.quality.is_none());
assert!(config.format.is_none());
assert!(config.selector.is_none());
}
#[test]
fn test_build_scrape_request_screenshot_config_with_selector() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"screenshot_options": {"selector": "#main"}}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
let config = request
.options
.screenshot_config
.expect("screenshot_config should be set");
assert_eq!(config.selector, Some("#main".to_string()));
}
#[test]
fn test_build_scrape_request_mobile_true() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"mobile": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.mobile);
}
#[test]
fn test_build_scrape_request_mobile_false_by_default() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(!request.options.mobile);
}
#[test]
fn test_build_scrape_request_proxy_set() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"proxy": "http://proxy:8080"}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.proxy, Some("http://proxy:8080".to_string()));
}
#[test]
fn test_build_scrape_request_proxy_none_by_default() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.proxy.is_none());
}
#[test]
fn test_build_scrape_request_skip_tls_verification() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"skip_tls_verification": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.skip_tls_verification);
}
#[test]
fn test_build_scrape_request_needs_tls_fingerprint() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"needs_tls_fingerprint": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.needs_tls_fingerprint);
}
#[test]
fn test_build_scrape_request_use_fire_engine() {
let task = make_task(json!({
"url": "https://example.com",
"options": {"use_fire_engine": true}
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.use_fire_engine);
}
#[test]
fn test_build_scrape_request_sync_wait_ms_default_zero() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.sync_wait_ms, 0);
}
#[test]
fn test_build_scrape_request_sync_wait_ms_set() {
let task = make_task(json!({
"url": "https://example.com",
"sync_wait_ms": 5000
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.sync_wait_ms, 5000);
}
#[test]
fn test_build_scrape_request_method_always_get() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.method, HttpMethod::Get);
}
#[test]
fn test_build_scrape_request_body_always_none() {
let task = make_task(json!({"url": "https://example.com"}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.body.is_none());
}
#[test]
fn test_build_scrape_request_url_comes_from_payload_not_task() {
let mut task = make_task(json!({"url": "https://from-payload.com"}));
task.url = "https://from-task.com".to_string();
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.url, "https://from-payload.com");
}
#[test]
fn test_build_scrape_request_action_wait_mapped() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "wait", "milliseconds": 1500}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.actions.len(), 1);
match &request.options.actions[0] {
PageAction::Wait { milliseconds } => assert_eq!(*milliseconds, 1500),
other => panic!("Expected Wait, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_click_mapped() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "click", "selector": "#submit"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.actions.len(), 1);
match &request.options.actions[0] {
PageAction::Click { selector } => assert_eq!(selector, "#submit"),
other => panic!("Expected Click, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_input_mapped() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "input", "selector": "#search", "text": "rust"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.actions.len(), 1);
match &request.options.actions[0] {
PageAction::Input { selector, text } => {
assert_eq!(selector, "#search");
assert_eq!(text, "rust");
}
other => panic!("Expected Input, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_scroll_down() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "scroll", "direction": "down"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
match &request.options.actions[0] {
PageAction::Scroll { direction } => {
assert_eq!(*direction, ScrollDirection::Down);
}
other => panic!("Expected Scroll Down, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_scroll_up() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "scroll", "direction": "up"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
match &request.options.actions[0] {
PageAction::Scroll { direction } => {
assert_eq!(*direction, ScrollDirection::Up);
}
other => panic!("Expected Scroll Up, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_scroll_top() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "scroll", "direction": "top"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
match &request.options.actions[0] {
PageAction::Scroll { direction } => {
assert_eq!(*direction, ScrollDirection::Top);
}
other => panic!("Expected Scroll Top, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_scroll_bottom() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "scroll", "direction": "bottom"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
match &request.options.actions[0] {
PageAction::Scroll { direction } => {
assert_eq!(*direction, ScrollDirection::Bottom);
}
other => panic!("Expected Scroll Bottom, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_scroll_unknown_direction_defaults_down() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "scroll", "direction": "sideways"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
match &request.options.actions[0] {
PageAction::Scroll { direction } => {
assert_eq!(*direction, ScrollDirection::Down);
}
other => panic!("Expected default Scroll Down, got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_scroll_case_insensitive_direction() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "scroll", "direction": "UP"}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
match &request.options.actions[0] {
PageAction::Scroll { direction } => {
assert_eq!(*direction, ScrollDirection::Up);
}
other => panic!("Expected Scroll Up (case-insensitive), got {:?}", other),
}
}
#[test]
fn test_build_scrape_request_action_screenshot_is_filtered_out() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [{"type": "screenshot", "full_page": true}]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(
request.options.actions.is_empty(),
"screenshot action should be filtered out"
);
assert!(request.options.needs_js);
}
#[test]
fn test_build_scrape_request_multiple_actions_preserve_order() {
let task = make_task(json!({
"url": "https://example.com",
"actions": [
{"type": "wait", "milliseconds": 100},
{"type": "click", "selector": "#btn1"},
{"type": "scroll", "direction": "down"},
{"type": "input", "selector": "#field", "text": "text"},
{"type": "screenshot", "full_page": null}
]
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.options.actions.len(), 4);
assert!(matches!(
request.options.actions[0],
PageAction::Wait { milliseconds: 100 }
));
assert!(matches!(
&request.options.actions[1],
PageAction::Click { selector } if selector == "#btn1"
));
assert!(matches!(
&request.options.actions[2],
PageAction::Scroll { direction } if *direction == ScrollDirection::Down
));
assert!(matches!(
&request.options.actions[3],
PageAction::Input { selector, text } if selector == "#field" && text == "text"
));
}
#[test]
fn test_build_scrape_request_none_actions_yields_empty_vec() {
let task = make_task(json!({
"url": "https://example.com",
"actions": null
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert!(request.options.actions.is_empty());
assert!(!request.options.needs_js);
}
#[test]
fn test_build_scrape_request_all_options_combined() {
let task = make_task(json!({
"url": "https://example.com",
"options": {
"headers": {"Accept": "text/html"},
"timeout": 45,
"js_rendering": true,
"screenshot": true,
"screenshot_options": {"full_page": false, "quality": 50, "format": "jpeg"},
"mobile": true,
"proxy": "http://proxy:3128",
"skip_tls_verification": true,
"needs_tls_fingerprint": true,
"use_fire_engine": true
},
"actions": [{"type": "wait", "milliseconds": 200}],
"sync_wait_ms": 1000
}));
let request = ScrapeWorker::build_scrape_request(&task).expect("should succeed");
assert_eq!(request.url, "https://example.com");
assert_eq!(request.options.timeout, Duration::from_secs(45));
assert_eq!(
request.options.headers.get("Accept"),
Some(&"text/html".to_string())
);
assert!(request.options.needs_js);
assert!(request.options.needs_screenshot);
assert!(request.options.mobile);
assert_eq!(request.options.proxy, Some("http://proxy:3128".to_string()));
assert!(request.options.skip_tls_verification);
assert!(request.options.needs_tls_fingerprint);
assert!(request.options.use_fire_engine);
assert_eq!(request.options.sync_wait_ms, 1000);
assert_eq!(request.options.actions.len(), 1);
let sc = request
.options
.screenshot_config
.expect("screenshot_config should be set");
assert!(!sc.full_page);
assert_eq!(sc.quality, Some(50));
assert_eq!(sc.format, Some("jpeg".to_string()));
}
#[test]
fn test_builder_default_build_fails_with_repository_required() {
let builder = ScrapeWorkerBuilder::default();
let err = match builder.build() {
Err(e) => e,
Ok(_) => panic!("empty builder should fail"),
};
assert_eq!(err, "repository is required");
}
#[test]
fn test_builder_new_equals_default() {
let err_new = match ScrapeWorkerBuilder::new().build() {
Err(e) => e,
Ok(_) => panic!("new() builder should fail"),
};
let err_default = match ScrapeWorkerBuilder::default().build() {
Err(e) => e,
Ok(_) => panic!("default() builder should fail"),
};
assert_eq!(err_new, err_default);
assert_eq!(err_new, "repository is required");
}
#[test]
fn test_builder_with_default_concurrency_limit_does_not_satisfy_required_fields() {
let err = match ScrapeWorkerBuilder::default()
.with_default_concurrency_limit(50)
.build()
{
Err(e) => e,
Ok(_) => panic!("should still fail"),
};
assert_eq!(err, "repository is required");
}
#[test]
fn test_builder_with_default_concurrency_limit_zero() {
let err = match ScrapeWorkerBuilder::default()
.with_default_concurrency_limit(0)
.build()
{
Err(e) => e,
Ok(_) => panic!("should still fail"),
};
assert_eq!(err, "repository is required");
}
#[test]
fn test_builder_default_concurrency_limit_is_ten_by_default() {
let builder = ScrapeWorkerBuilder::default();
let err = match builder.build() {
Err(e) => e,
Ok(_) => panic!("should fail"),
};
assert_eq!(err, "repository is required");
}
#[test]
fn test_scrape_worker_error_from_string_creates_task_error() {
let err: ScrapeWorkerError = "test error".to_string().into();
match err {
ScrapeWorkerError::TaskError(msg) => assert_eq!(msg, "test error"),
other => panic!("Expected TaskError, got {:?}", other),
}
}
#[test]
#[allow(clippy::invalid_regex)] fn test_scrape_worker_error_from_regex_error() {
let regex_err = regex::Regex::new("(unclosed").expect_err("should be invalid");
let err: ScrapeWorkerError = regex_err.into();
match err {
ScrapeWorkerError::RegexError(msg) => assert!(!msg.is_empty()),
other => panic!("Expected RegexError, got {:?}", other),
}
}
#[test]
fn test_scrape_worker_error_from_url_parse_error() {
let url_err = url::Url::parse("not a url").expect_err("should be invalid");
let err: ScrapeWorkerError = url_err.into();
match err {
ScrapeWorkerError::TaskError(msg) => assert!(msg.contains("URL parse error")),
other => panic!("Expected TaskError, got {:?}", other),
}
}
use crate::domain::models::{
Crawl, CreditsTransaction, CreditsTransactionType, DomainError, WebhookEvent,
};
use crate::domain::repositories::credits_repository::CreditsRepositoryError;
use crate::domain::repositories::task_repository::{RepositoryError, TaskQueryParams};
use crate::domain::services::extraction_service::ExtractionRule;
use crate::domain::services::llm::TokenUsage;
use std::collections::HashSet;
struct MockTaskRepository;
#[async_trait::async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(&self, _timeout: chrono::Duration) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct MockScrapeResultRepository;
#[async_trait::async_trait]
impl ScrapeResultRepository for MockScrapeResultRepository {
async fn save(&self, _result: ScrapeResult) -> Result<()> {
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(&self, _team_id: crate::domain::id::Id) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
struct CapturingScrapeResultRepository {
captured: std::sync::Mutex<Option<ScrapeResult>>,
}
impl CapturingScrapeResultRepository {
fn new() -> Self {
Self {
captured: std::sync::Mutex::new(None),
}
}
fn captured(&self) -> Option<ScrapeResult> {
self.captured.lock().ok()?.clone()
}
}
#[async_trait::async_trait]
impl ScrapeResultRepository for CapturingScrapeResultRepository {
async fn save(&self, result: ScrapeResult) -> Result<()> {
let mut guard = self.captured.lock().expect("captured mutex poisoned");
*guard = Some(result);
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(&self, _team_id: crate::domain::id::Id) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
struct MockCrawlRepository;
#[async_trait::async_trait]
impl CrawlRepository for MockCrawlRepository {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Ok(None)
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct MockWebhookService;
#[async_trait::async_trait]
impl WebhookService for MockWebhookService {
async fn send_webhook(&self, _event: &WebhookEvent) -> Result<u16> {
Ok(200)
}
async fn trigger_completion(&self, _task: &Task) -> Result<()> {
Ok(())
}
async fn trigger_failure(&self, _task: &Task, _error_msg: String) -> Result<()> {
Ok(())
}
}
#[derive(Debug, Default)]
struct MockCreditsRepo {
deducted: Arc<std::sync::Mutex<Vec<(crate::domain::id::Id, i64)>>>,
deduction_keys: Arc<std::sync::Mutex<Vec<(crate::domain::id::Id, String)>>>,
}
#[async_trait::async_trait]
impl CreditsRepository for MockCreditsRepo {
async fn get_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, CreditsRepositoryError> {
Ok(100)
}
async fn deduct_credits(
&self,
team_id: crate::domain::id::Id,
amount: i64,
transaction_type: CreditsTransactionType,
_description: String,
reference_id: Option<crate::domain::id::Id>,
) -> Result<(), CreditsRepositoryError> {
self.deducted
.lock()
.unwrap_or_else(|e| e.into_inner())
.push((team_id, amount));
if let Some(task_id) = reference_id {
self.deduction_keys
.lock()
.unwrap_or_else(|e| e.into_inner())
.push((task_id, transaction_type.to_string()));
}
Ok(())
}
async fn has_deduction_for_task(
&self,
task_id: crate::domain::id::Id,
transaction_type: CreditsTransactionType,
) -> Result<bool, CreditsRepositoryError> {
let keys = self
.deduction_keys
.lock()
.unwrap_or_else(|e| e.into_inner());
Ok(keys
.iter()
.any(|(id, ty)| *id == task_id && *ty == transaction_type.to_string()))
}
async fn add_credits(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<i64, CreditsRepositoryError> {
Ok(100)
}
async fn get_transaction_history(
&self,
_team_id: crate::domain::id::Id,
_limit: Option<u32>,
) -> Result<Vec<CreditsTransaction>, CreditsRepositoryError> {
Ok(vec![])
}
async fn initialize_team_credits(
&self,
_team_id: crate::domain::id::Id,
_initial_balance: i64,
) -> Result<i64, CreditsRepositoryError> {
Ok(100)
}
}
struct MockCreateScrapeUseCase;
#[async_trait::async_trait]
impl CreateScrapeUseCaseTrait for MockCreateScrapeUseCase {
async fn execute(&self, _request_dto: ScrapeRequestDto) -> Result<ScrapeResponse, DomainError> {
Ok(ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: String::new(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 0,
final_url: None,
markdown: None,
})
}
}
struct MockRobotsChecker;
#[async_trait::async_trait]
impl RobotsCheckerTrait for MockRobotsChecker {
async fn is_allowed(&self, _url_str: &str, _user_agent: &str) -> Result<bool> {
Ok(true)
}
async fn get_crawl_delay(&self, _url_str: &str, _user_agent: &str) -> Result<Option<Duration>> {
Ok(None)
}
}
struct MockExtractionService;
#[async_trait::async_trait]
impl ExtractionServiceTrait for MockExtractionService {
async fn extract(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> Result<(Value, TokenUsage)> {
Ok((json!({}), TokenUsage::default()))
}
async fn extract_with_schema(
&self,
_html_content: &str,
_schema: &Value,
) -> Result<(Value, TokenUsage)> {
Ok((json!({}), TokenUsage::default()))
}
fn extract_with_selectors(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> Result<Value> {
Ok(json!({}))
}
async fn extract_with_rag(
&self,
_html_content: &str,
_query: &str,
_schema: &Value,
_rag_strategy: &crate::domain::services::rag_strategy::RagExtractionStrategy,
) -> Result<(Value, TokenUsage)> {
Ok((json!({}), TokenUsage::default()))
}
}
async fn build_mock_worker() -> ScrapeWorker {
let engine_client = Arc::new(EngineClient::new());
let settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings for mock worker");
let settings_arc = Arc::new(settings.clone());
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: settings_arc,
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
async fn build_mock_worker_with_cache(cache: Arc<MockCacheService>) -> ScrapeWorker {
let engine_client = Arc::new(EngineClient::new());
let settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings for mock worker");
let settings_arc = Arc::new(settings.clone());
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: settings_arc,
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: cache as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
async fn build_mock_worker_with_capturing_repo(
) -> (ScrapeWorker, Arc<CapturingScrapeResultRepository>) {
let engine_client = Arc::new(EngineClient::new());
let settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings for mock worker");
let settings_arc = Arc::new(settings.clone());
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let capturing_repo = Arc::new(CapturingScrapeResultRepository::new());
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let result_repo: Arc<dyn ScrapeResultRepository> =
capturing_repo.clone() as Arc<dyn ScrapeResultRepository>;
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: settings_arc,
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
(worker, capturing_repo)
}
impl super::ScrapeWorker {
fn should_crawl(&self, url: &str, config: &CrawlConfigDto) -> bool {
if matches!(&config.include_patterns, Some(patterns) if patterns.is_empty()) {
return false;
}
let include = config.include_patterns.clone().unwrap_or_default();
let exclude = config.exclude_patterns.clone().unwrap_or_default();
UrlPatternFilter::new(include, exclude).accept(url, &FilterContext::default())
}
}
#[tokio::test]
async fn test_mock_should_crawl_no_patterns_returns_true() {
let worker = build_mock_worker().await;
let config = make_crawl_config(None, None);
assert!(worker.should_crawl("https://example.com/page1", &config));
assert!(worker.should_crawl("https://other.com/page2", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_include_pattern_match() {
let worker = build_mock_worker().await;
let config = make_crawl_config(Some(vec!["example\\.com".to_string()]), None);
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(worker.should_crawl("https://example.com/sub/page", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_include_pattern_no_match() {
let worker = build_mock_worker().await;
let config = make_crawl_config(Some(vec!["example\\.com".to_string()]), None);
assert!(!worker.should_crawl("https://other.com/page", &config));
assert!(!worker.should_crawl("https://foo.org/path", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_exclude_pattern_match() {
let worker = build_mock_worker().await;
let config = make_crawl_config(None, Some(vec!["blocked".to_string()]));
assert!(!worker.should_crawl("https://example.com/blocked", &config));
assert!(!worker.should_crawl("https://example.com/blocked/page", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_exclude_pattern_no_match() {
let worker = build_mock_worker().await;
let config = make_crawl_config(None, Some(vec!["blocked".to_string()]));
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(worker.should_crawl("https://example.com/allowed", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_both_include_and_exclude() {
let worker = build_mock_worker().await;
let config = make_crawl_config(
Some(vec!["example\\.com".to_string()]),
Some(vec!["blocked".to_string()]),
);
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(!worker.should_crawl("https://example.com/blocked", &config));
assert!(!worker.should_crawl("https://other.com/blocked", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_multiple_include_patterns() {
let worker = build_mock_worker().await;
let config = make_crawl_config(
Some(vec!["example\\.com".to_string(), "test\\.org".to_string()]),
None,
);
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(worker.should_crawl("https://test.org/page", &config));
assert!(!worker.should_crawl("https://other.com/page", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_multiple_exclude_patterns() {
let worker = build_mock_worker().await;
let config = make_crawl_config(None, Some(vec!["blocked".to_string(), "admin".to_string()]));
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(!worker.should_crawl("https://example.com/blocked", &config));
assert!(!worker.should_crawl("https://example.com/admin", &config));
}
#[tokio::test]
#[allow(clippy::invalid_regex)]
async fn test_mock_should_crawl_include_fallback_string_match() {
let worker = build_mock_worker().await;
let config = make_crawl_config(Some(vec!["[unclosed".to_string()]), None);
assert!(worker.should_crawl("https://example.com/[unclosed", &config));
assert!(!worker.should_crawl("https://example.com/other", &config));
}
#[tokio::test]
#[allow(clippy::invalid_regex)]
async fn test_mock_should_crawl_exclude_fallback_string_match() {
let worker = build_mock_worker().await;
let config = make_crawl_config(None, Some(vec!["[unclosed".to_string()]));
assert!(!worker.should_crawl("https://example.com/[unclosed", &config));
assert!(worker.should_crawl("https://example.com/other", &config));
}
#[tokio::test]
async fn test_mock_build_crawl_request_basic() {
let worker = build_mock_worker().await;
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
assert_eq!(request.url, "https://example.com");
assert_eq!(request.options.method, HttpMethod::Get);
assert!(request.options.body.is_none());
assert!(!request.options.needs_js);
assert!(!request.options.needs_screenshot);
assert!(request.options.screenshot_config.is_none());
assert!(!request.options.mobile);
assert!(request.options.proxy.is_none());
assert!(!request.options.skip_tls_verification);
assert!(!request.options.needs_tls_fingerprint);
assert!(!request.options.use_fire_engine);
assert!(request.options.actions.is_empty());
assert_eq!(request.options.sync_wait_ms, 0);
}
#[tokio::test]
async fn test_mock_build_crawl_request_with_headers() {
let worker = build_mock_worker().await;
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: Some(json!({
"Accept": "text/html",
"Authorization": "Bearer token123"
})),
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
assert_eq!(
request.options.headers.get("Accept"),
Some(&"text/html".to_string())
);
assert_eq!(
request.options.headers.get("Authorization"),
Some(&"Bearer token123".to_string())
);
}
#[tokio::test]
async fn test_mock_build_crawl_request_non_string_headers_filtered() {
let worker = build_mock_worker().await;
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: Some(json!({
"X-Number": 42,
"X-Bool": true,
"X-Null": null,
"X-Valid": "ok"
})),
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
assert_eq!(request.options.headers.len(), 1);
assert_eq!(
request.options.headers.get("X-Valid"),
Some(&"ok".to_string())
);
assert!(!request.options.headers.contains_key("X-Number"));
assert!(!request.options.headers.contains_key("X-Bool"));
assert!(!request.options.headers.contains_key("X-Null"));
}
#[tokio::test]
async fn test_mock_build_crawl_request_with_proxy() {
let worker = build_mock_worker().await;
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: Some("http://proxy:3128".to_string()),
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
assert_eq!(request.options.proxy, Some("http://proxy:3128".to_string()));
}
#[tokio::test]
async fn test_mock_build_crawl_request_empty_headers_map() {
let worker = build_mock_worker().await;
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: Some(json!({})),
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
assert!(request.options.headers.is_empty());
}
#[tokio::test]
async fn test_mock_build_extract_request_basic() {
let worker = build_mock_worker().await;
let request = worker.build_extract_request("https://example.com/page");
assert_eq!(request.url, "https://example.com/page");
assert_eq!(request.options.method, HttpMethod::Get);
assert!(request.options.body.is_none());
assert!(request.options.headers.is_empty());
assert!(!request.options.needs_js);
assert!(!request.options.needs_screenshot);
assert!(!request.options.mobile);
assert!(request.options.proxy.is_none());
assert!(!request.options.skip_tls_verification);
assert!(!request.options.needs_tls_fingerprint);
assert!(!request.options.use_fire_engine);
assert!(request.options.actions.is_empty());
}
#[tokio::test]
async fn test_mock_build_extract_request_different_urls() {
let worker = build_mock_worker().await;
let urls = vec![
"https://example.com",
"https://test.org/path",
"http://localhost:8080",
];
for url in &urls {
let request = worker.build_extract_request(url);
assert_eq!(request.url, *url);
}
}
#[tokio::test]
async fn test_mock_trigger_webhook_completion_no_error() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
worker.trigger_webhook(&task, None).await;
}
#[tokio::test]
async fn test_mock_trigger_webhook_failure_with_error_msg() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
worker
.trigger_webhook(&task, Some("Task failed".to_string()))
.await;
}
#[tokio::test]
async fn test_mock_deduct_feature_credits_screenshot_and_proxy() {
let worker = build_mock_worker().await;
worker
.deduct_feature_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
true,
true,
)
.await;
}
#[tokio::test]
async fn test_mock_deduct_feature_credits_screenshot_only() {
let worker = build_mock_worker().await;
worker
.deduct_feature_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
true,
false,
)
.await;
}
#[tokio::test]
async fn test_mock_deduct_feature_credits_proxy_only() {
let worker = build_mock_worker().await;
worker
.deduct_feature_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
false,
true,
)
.await;
}
#[tokio::test]
async fn test_mock_deduct_feature_credits_neither() {
let worker = build_mock_worker().await;
worker
.deduct_feature_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
false,
false,
)
.await;
}
#[tokio::test]
async fn test_mock_save_result_basic() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html>test</html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let result = worker.save_result(&task, &response, None).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_save_result_with_extra_data() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html>test</html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let extra = json!({"title": "Test Page", "links": 5});
let result = worker.save_result(&task, &response, Some(extra)).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_save_result_with_screenshot() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html>test</html>".to_string(),
status_code: 200,
screenshot: Some("base64data".to_string()),
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 200,
final_url: None,
markdown: None,
};
let result = worker.save_result(&task, &response, None).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_process_text_encoding_basic() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html; charset=utf-8".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let result = worker.process_text_encoding(&task, &response).await;
match result {
Ok(content) => assert!(!content.is_empty() || response.content.is_empty()),
Err(_) => { }
}
}
#[tokio::test]
async fn test_mock_update_crawl_completion_status_crawl_not_found() {
let worker = build_mock_worker().await;
worker
.update_crawl_completion_status(crate::domain::id::Id::generate())
.await;
}
#[tokio::test]
async fn test_mock_parse_crawl_payload_valid() {
let worker = build_mock_worker().await;
let crawl_id = crate::domain::id::Id::generate();
let task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 2,
"config": {
"max_depth": 3,
"include_patterns": ["example\\.com"],
"exclude_patterns": ["blocked"],
"strategy": "bfs",
"crawl_delay_ms": 100,
"max_concurrency": 5,
"proxy": "http://proxy:8080",
"headers": {"X-Custom": "value"},
"extraction_rules": {}
}
}));
let (parsed_id, depth, config) = worker.parse_crawl_payload(&task).await.unwrap();
assert_eq!(parsed_id, crawl_id);
assert_eq!(depth, 2);
assert_eq!(config.max_depth, 3);
}
#[tokio::test]
async fn test_mock_parse_crawl_payload_missing_crawl_id() {
let worker = build_mock_worker().await;
let task = make_task(json!({"depth": 1, "config": {}}));
assert!(worker.parse_crawl_payload(&task).await.is_err());
}
#[tokio::test]
async fn test_mock_parse_crawl_payload_default_depth() {
let worker = build_mock_worker().await;
let crawl_id = crate::domain::id::Id::generate();
let task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"config": {"max_depth": 1}
}));
let (_, depth, _) = worker.parse_crawl_payload(&task).await.unwrap();
assert_eq!(depth, 0);
}
#[tokio::test]
async fn test_mock_parse_extract_payload_valid() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"urls": ["https://example.com/page"],
"prompt": "Extract title",
"model": "gpt-4"
}));
let (payload, url) = worker.parse_extract_payload(&task).await.unwrap();
assert_eq!(url, "https://example.com/page");
assert_eq!(payload.urls.len(), 1);
}
#[tokio::test]
async fn test_mock_parse_extract_payload_no_url() {
let worker = build_mock_worker().await;
let task = make_task(json!({"urls": []}));
assert!(worker.parse_extract_payload(&task).await.is_err());
}
#[tokio::test]
async fn test_mock_check_robots_txt_allowed() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
assert!(worker.check_robots_txt(&task).await);
}
#[tokio::test]
async fn test_validate_scrape_request_robots_denied_records_reason() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(DenyingRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let scrape_request = ScrapeRequest::new(task.url.clone());
let dto: ScrapeRequestDto = serde_json::from_value(json!({
"url": task.url,
"options": { "respect_robots": true }
}))
.expect("dto");
let allowed = worker
.validate_scrape_request(&task, &scrape_request, Some(&dto))
.await
.expect("validate should succeed");
assert!(!allowed, "robots Disallow 应拒绝");
let captured = task_repo
.last_updated_task()
.expect("拒绝路径必须写入任务快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("robots.txt") && reason.contains("disallowed"),
"失败原因必须含 robots 语义,实际: {reason}"
);
}
#[tokio::test]
async fn test_validate_scrape_request_guard_rejected_does_not_persist() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
task_repo.reject_guarded_update();
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(DenyingRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let scrape_request = ScrapeRequest::new(task.url.clone());
let dto: ScrapeRequestDto = serde_json::from_value(json!({
"url": task.url,
"options": { "respect_robots": true }
}))
.expect("dto");
let allowed = worker
.validate_scrape_request(&task, &scrape_request, Some(&dto))
.await
.expect("validate should succeed");
assert!(!allowed, "robots Disallow 仍应拒绝");
assert_eq!(task_repo.update_count(), 1, "必须尝试一次守卫式写入");
assert!(
task_repo.last_updated_task().is_none(),
"守卫拒绝时不得产生成功写入快照"
);
}
#[tokio::test]
async fn test_validate_scrape_request_ssrf_proxy_records_reason() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let mut scrape_request = ScrapeRequest::new(task.url.clone());
scrape_request.options.proxy = Some("http://127.0.0.1:8080".to_string());
let allowed = worker
.validate_scrape_request(&task, &scrape_request, None)
.await
.expect("validate should succeed");
assert!(!allowed, "内部代理应被 SSRF 拦截");
let captured = task_repo
.last_updated_task()
.expect("拒绝路径必须写入任务快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("SSRF"),
"失败原因必须标注 SSRF,实际: {reason}"
);
}
#[tokio::test]
async fn test_passes_robots_gate_denied_records_reason() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(DenyingRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let passed = worker
.passes_robots_gate(&task)
.await
.expect("gate should succeed");
assert!(!passed, "crawl robots 拒绝应返回 false");
let captured = task_repo.last_updated_task().expect("必须写入任务快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("robots.txt"),
"crawl 拒绝原因必须含 robots 语义,实际: {reason}"
);
}
#[tokio::test]
async fn test_handle_expired_task_records_reason() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.expires_at = Some(chrono::Utc::now() - chrono::Duration::seconds(1));
let handled = worker
.handle_expired_task(&task)
.await
.expect("expiry handling should succeed");
assert!(handled, "过期任务应被处理");
let captured = task_repo.last_updated_task().expect("必须写入任务快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("expired"),
"过期原因必须落库,实际: {reason}"
);
}
#[tokio::test]
async fn test_handle_expired_crawl_task_increments_failed_tasks() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.task_type = TaskType::Crawl;
task.crawl_id = Some(crate::domain::id::Id::generate());
task.expires_at = Some(chrono::Utc::now() - chrono::Duration::seconds(1));
let handled = worker
.handle_expired_task(&task)
.await
.expect("expiry handling should succeed");
assert!(handled, "过期任务应被处理");
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"过期 crawl 子任务必须计入 failed_tasks"
);
}
#[tokio::test]
async fn test_mock_handle_rules_extraction() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Hello</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let mut rules = HashMap::new();
rules.insert(
"title".to_string(),
ExtractionRule {
selector: Some("h1".to_string()),
attr: None,
is_array: false,
use_llm: None,
llm_prompt: None,
output_format: None,
},
);
let result = worker
.handle_rules_extraction(&mut task, &response, &rules, "https://example.com")
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_handle_prompt_extraction() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello world</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 30,
final_url: None,
markdown: None,
};
let result = worker
.handle_prompt_extraction(
&mut task,
&response,
"Extract the main topic".to_string(),
"https://example.com",
)
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_handle_schema_extraction() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Data</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 20,
final_url: None,
markdown: None,
};
let schema = json!({"type": "object", "properties": {"title": {"type": "string"}}});
let result = worker
.handle_schema_extraction(&mut task, &response, &schema, "https://example.com")
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_save_extract_result_with_data() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "test content".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let result = worker
.save_extract_result(
&mut task,
&response,
Some(json!({"title": "Test"})),
"https://example.com",
)
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_save_extract_result_without_data() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "raw content".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 5,
final_url: None,
markdown: None,
};
let result = worker
.save_extract_result(&mut task, &response, None, "https://example.com")
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_extract_and_queue_links_html_with_links() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
let html = r#"<html><body>
<a href="/page1">Page 1</a>
<a href="https://example.com/page2">Page 2</a>
<a href="https://other.com/page3">Page 3</a>
<a href="mailto:test@example.com">Email</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_extract_and_queue_links_non_html_skipped() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "{\"key\": \"value\"}".to_string(),
status_code: 200,
screenshot: None,
content_type: "application/json".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_extract_and_queue_links_with_include_filter() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
task.url = "https://example.com".to_string();
let html = r#"<html><body>
<a href="https://example.com/page1">Page 1</a>
<a href="https://other.com/page2">Page 2</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(Some(vec!["example\\.com".to_string()]), None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_handle_failure() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let result = worker.handle_failure(&mut task, "mock failure").await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_deduct_token_credits_zero_tokens() {
let worker = build_mock_worker().await;
let usage = TokenUsage::default();
worker
.deduct_token_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
&usage,
"test zero",
)
.await;
}
#[tokio::test]
async fn test_mock_deduct_token_credits_with_tokens() {
let worker = build_mock_worker().await;
let usage = TokenUsage {
prompt_tokens: 100,
completion_tokens: 50,
total_tokens: 150,
};
worker
.deduct_token_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
&usage,
"test with tokens",
)
.await;
}
#[tokio::test]
async fn test_deduct_feature_credits_is_idempotent_per_task() {
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let worker = build_worker_for_success_tests(
Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>,
Arc::new(EngineClient::new()),
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
worker
.deduct_feature_credits(team_id, task_id, true, true)
.await;
worker
.deduct_feature_credits(team_id, task_id, true, true)
.await;
let deductions = deducted_log.lock().unwrap();
assert_eq!(
deductions.len(),
1,
"duplicate feature-credit deduction must be suppressed, got {:?}",
deductions
);
assert_eq!(
deductions[0].1, 3,
"the single deduction should be screenshot(2)+proxy(1)=3 credits"
);
}
#[tokio::test]
async fn test_deduct_token_credits_is_idempotent_per_task() {
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let worker = build_worker_for_success_tests(
Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>,
Arc::new(EngineClient::new()),
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let usage = TokenUsage {
prompt_tokens: 100,
completion_tokens: 50,
total_tokens: 150,
};
worker
.deduct_token_credits(team_id, task_id, &usage, "test idempotent")
.await;
worker
.deduct_token_credits(team_id, task_id, &usage, "test idempotent")
.await;
let deductions = deducted_log.lock().unwrap();
assert_eq!(
deductions.len(),
1,
"duplicate token-credit deduction must be suppressed, got {:?}",
deductions
);
}
#[tokio::test]
async fn test_deduct_feature_and_token_credits_are_independent_per_task() {
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let worker = build_worker_for_success_tests(
Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>,
Arc::new(EngineClient::new()),
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
worker
.deduct_feature_credits(team_id, task_id, true, false)
.await;
let usage = TokenUsage {
prompt_tokens: 100,
completion_tokens: 50,
total_tokens: 150,
};
worker
.deduct_token_credits(team_id, task_id, &usage, "test independent")
.await;
let deductions = deducted_log.lock().unwrap();
assert_eq!(
deductions.len(),
2,
"feature (Scrape) and token (Extract) charges are independent, got {:?}",
deductions
);
}
#[tokio::test]
async fn test_dispatch_extraction_deferred_records_usage_without_deducting() {
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let worker = build_worker_for_success_tests(
Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>,
Arc::new(EngineClient::new()),
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionServiceWithTokens) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({}));
let mut pending = Vec::new();
let data = worker
.dispatch_extraction(
&task,
"<html><body><h1>Title</h1></body></html>",
&task.url,
&ExtractionSpec::Prompt("extract the title"),
ExtractionSettlement::Deferred {
usage_acc: &mut pending,
},
)
.await
.expect("deferred dispatch should succeed");
assert_eq!(data, json!({"title": "Extracted Title"}));
assert_eq!(
pending.len(),
1,
"deferred settlement must queue the usage for post-guard deduction"
);
assert_eq!(pending[0].total_tokens, 150);
assert!(
deducted_log
.lock()
.unwrap_or_else(|e| e.into_inner())
.is_empty(),
"Deferred settlement must not hit the credits repository"
);
let in_memory = worker
.token_usage
.get(&task.team_id)
.map(|v| v.load(Ordering::Relaxed))
.unwrap_or(0);
assert_eq!(
in_memory, 150,
"deferred path must keep real-time in-memory accounting"
);
}
#[tokio::test]
async fn test_dispatch_extraction_immediate_deducts_at_extraction_point() {
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let worker = build_worker_for_success_tests(
Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>,
Arc::new(EngineClient::new()),
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionServiceWithTokens) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({}));
let data = worker
.dispatch_extraction(
&task,
"<html><body><h1>Title</h1></body></html>",
&task.url,
&ExtractionSpec::Prompt("extract the title"),
ExtractionSettlement::Immediate {
description: "Tokens used for test extraction",
},
)
.await
.expect("immediate dispatch should succeed");
assert_eq!(data, json!({"title": "Extracted Title"}));
let deductions = deducted_log.lock().unwrap_or_else(|e| e.into_inner());
assert_eq!(
deductions.len(),
1,
"Immediate settlement must deduct exactly once"
);
assert_eq!(
deductions[0].1, 2,
"150 tokens → ceil(150*10/1000) = 2 credits, got {:?}",
deductions
);
}
#[tokio::test]
async fn test_mock_handle_scrape_success_no_extraction_rules() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_handle_scrape_success_with_extraction_rules() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"url": "https://example.com",
"extraction_rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(matches!(result, Ok(true)));
}
#[tokio::test]
async fn test_handle_scrape_success_returns_false_when_guard_lost() {
struct GuardLostRepo;
#[async_trait::async_trait]
impl TaskRepository for GuardLostRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
let engine_client = Arc::new(EngineClient::new());
let settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings for mock worker");
let settings_arc = Arc::new(settings.clone());
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(GuardLostRepo);
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let credits_repo = Arc::new(MockCreditsRepo::default());
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: credits_repo.clone() as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: settings_arc,
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let task = make_task(json!({
"url": "https://example.com",
"extraction_rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(
matches!(result, Ok(false)),
"guard loss must return Ok(false)"
);
assert!(
credits_repo.deducted.lock().unwrap().is_empty(),
"stale worker must not deduct any credits"
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_handle_scrape_success_with_markdown_format_generates_markdown() {
let (worker, capturing_repo) = build_mock_worker_with_capturing_repo().await;
let task = make_task(json!({
"url": "https://example.com",
"formats": ["markdown"],
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><h1>Title</h1><p>Paragraph text</p></body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(result.is_ok(), "handle_scrape_success should succeed");
let saved = capturing_repo
.captured()
.expect("result should have been saved");
let meta = &saved.meta_data;
assert!(
meta.get("markdown").is_some(),
"meta_data should contain markdown key, got: {meta}"
);
let md = meta
.get("markdown")
.and_then(|v| v.as_str())
.expect("markdown should be a string");
assert!(
md.contains("Title"),
"markdown should contain heading text, got: {md}"
);
assert!(
md.contains("Paragraph text"),
"markdown should contain paragraph text, got: {md}"
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_handle_scrape_success_without_markdown_format_no_markdown() {
let (worker, capturing_repo) = build_mock_worker_with_capturing_repo().await;
let task = make_task(json!({
"url": "https://example.com",
"formats": ["html"],
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><h1>Title</h1></body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(result.is_ok());
let saved = capturing_repo
.captured()
.expect("result should have been saved");
assert!(
saved.meta_data.is_null(),
"meta_data should be null when no markdown requested, got: {}",
saved.meta_data
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_handle_scrape_success_no_formats_field_no_markdown() {
let (worker, capturing_repo) = build_mock_worker_with_capturing_repo().await;
let task = make_task(json!({
"url": "https://example.com",
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><h1>Title</h1></body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(result.is_ok());
let saved = capturing_repo
.captured()
.expect("result should have been saved");
assert!(
saved.meta_data.is_null(),
"meta_data should be null when formats field absent, got: {}",
saved.meta_data
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_save_result_merges_markdown_into_existing_meta_data() {
let (worker, capturing_repo) = build_mock_worker_with_capturing_repo().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: Some("# Hello".to_string()),
};
let existing_meta = json!({"title": "Existing Title"});
let result = worker
.save_result(&task, &response, Some(existing_meta))
.await;
assert!(result.is_ok());
let saved = capturing_repo
.captured()
.expect("result should have been saved");
let meta = &saved.meta_data;
assert!(
meta.get("title").and_then(|v| v.as_str()) == Some("Existing Title"),
"meta_data should preserve existing extracted fields, got: {meta}"
);
assert!(
meta.get("markdown").and_then(|v| v.as_str()) == Some("# Hello"),
"meta_data should contain merged markdown, got: {meta}"
);
}
#[cfg(feature = "content")]
#[tokio::test]
async fn test_save_result_with_markdown_only_creates_markdown_object() {
let (worker, capturing_repo) = build_mock_worker_with_capturing_repo().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: Some("# Generated MD".to_string()),
};
let result = worker.save_result(&task, &response, None).await;
assert!(result.is_ok());
let saved = capturing_repo
.captured()
.expect("result should have been saved");
let meta = &saved.meta_data;
assert!(
meta.is_object(),
"meta_data should be an object when only markdown present, got: {meta}"
);
assert!(
meta.get("markdown").and_then(|v| v.as_str()) == Some("# Generated MD"),
"meta_data should contain markdown, got: {meta}"
);
assert_eq!(
meta.as_object().map(|o| o.len()),
Some(1),
"meta_data should have exactly one key (markdown), got: {meta}"
);
}
#[tokio::test]
async fn test_save_result_without_markdown_does_not_modify_meta_data() {
let (worker, capturing_repo) = build_mock_worker_with_capturing_repo().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let existing_meta = json!({"title": "Only Title"});
let result = worker
.save_result(&task, &response, Some(existing_meta))
.await;
assert!(result.is_ok());
let saved = capturing_repo
.captured()
.expect("result should have been saved");
let meta = &saved.meta_data;
assert!(
meta.get("title").and_then(|v| v.as_str()) == Some("Only Title"),
"meta_data should preserve existing fields, got: {meta}"
);
assert!(
meta.get("markdown").is_none(),
"meta_data should not contain markdown key when response.markdown is None, got: {meta}"
);
}
#[tokio::test]
async fn test_mock_handle_crawl_success_with_link_extraction() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><a href="/page1">Link</a></body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_success(
&task,
response,
crate::domain::id::Id::generate(),
0,
&config,
&request,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_handle_crawl_success_max_depth_no_link_extraction() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><a href="/page1">Link</a></body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let mut config = make_crawl_config(None, None);
config.max_depth = 1;
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_success(
&task,
response,
crate::domain::id::Id::generate(),
1,
&config,
&request,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_handle_crawl_failure_basic() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_failure(
&mut task,
anyhow::anyhow!("Network error"),
crate::domain::id::Id::generate(),
&request,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_process_scrape_task_engine_error() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok()); }
#[tokio::test]
async fn test_process_scrape_task_malformed_payload_marks_failed() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let task_repo_check = task_repo.clone();
let worker = build_worker_for_success_tests(
task_repo as Arc<dyn TaskRepository>,
Arc::new(EngineClient::new()),
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({}));
let result = worker.process_scrape_task(task).await;
assert!(
result.is_ok(),
"畸形 payload 在内部处理为 Failed,process_scrape_task 返回 Ok(())"
);
let captured = task_repo_check
.last_updated_task()
.expect("畸形 payload 必须写入失败快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("Invalid task payload"),
"畸形 payload 原因必须落库,实际: {reason}"
);
assert_eq!(
task_repo_check.mark_completed_count(),
0,
"畸形 payload 不得进入完成路径"
);
}
#[tokio::test]
async fn test_mock_process_scrape_task_with_valid_payload_engine_error() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"url": "https://example.com",
"options": {
"timeout": 10,
"js_rendering": true
}
}));
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_process_crawl_task_invalid_payload() {
let worker = build_mock_worker().await;
let task = make_task(json!({"depth": 1, "config": {}}));
let result = worker.process_crawl_task(task).await;
assert!(result.is_ok()); }
#[tokio::test]
async fn test_mock_process_crawl_task_engine_error() {
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"engine error crawl".to_string(),
"https://example.com/".to_string(),
"https://example.com/".to_string(),
CrawlStatus::Processing,
json!({}),
1, 0, 0, Utc::now(),
Utc::now(),
None,
));
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": {"max_depth": 2}
}));
task.max_retries = 1;
task.attempt_count = 1;
let result = worker.process_crawl_task(task).await;
assert!(result.is_ok());
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"engine failure must increment failed_tasks"
);
assert_eq!(
crawl_repo.update_status_count(),
1,
"sole failed task must finalize crawl as Failed"
);
}
#[tokio::test]
async fn test_process_crawl_task_missing_crawl_row_fails_closed() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new()); let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let missing_id = crate::domain::id::Id::generate();
let mut task = make_task(json!({
"crawl_id": missing_id.to_string(),
"depth": 0,
"config": {"max_depth": 2}
}));
task.crawl_id = Some(missing_id);
assert!(worker.process_crawl_task(task).await.is_ok());
assert_eq!(
task_repo.update_count(),
1,
"missing crawl snapshot must mark the task failed"
);
}
#[tokio::test]
async fn test_process_crawl_task_legacy_payload_blacklist_is_enforced() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"legacy blacklist".to_string(),
"https://example.com/".to_string(),
"https://example.com/".to_string(),
CrawlStatus::Processing,
json!({}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
));
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": {"max_depth": 2},
"domain_blacklist": ["example.com"]
}));
task.crawl_id = Some(crawl_id);
assert!(worker.process_crawl_task(task).await.is_ok());
assert_eq!(
task_repo.update_count(),
1,
"legacy payload blacklist must block the seed"
);
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"legacy rejection must increment failed_tasks"
);
}
#[tokio::test]
async fn test_process_crawl_task_blacklisted_proxy_is_rejected() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"proxy blacklist".to_string(),
"https://allowed.example/".to_string(),
"https://allowed.example/".to_string(),
CrawlStatus::Processing,
json!({"domain_blacklist": ["blocked-proxy.example"]}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
));
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": {
"max_depth": 2,
"proxy": "http://blocked-proxy.example:8080"
}
}));
task.crawl_id = Some(crawl_id);
assert!(worker.process_crawl_task(task).await.is_ok());
assert_eq!(
task_repo.update_count(),
1,
"blacklisted proxy must fail the task"
);
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"rejection must increment failed_tasks"
);
}
#[tokio::test]
async fn test_process_crawl_task_corrupt_blacklist_config_fails_closed() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"corrupt blacklist".to_string(),
"https://example.com/".to_string(),
"https://example.com/".to_string(),
CrawlStatus::Processing,
json!({"domain_blacklist": "not-an-array"}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
));
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": {"max_depth": 2}
}));
task.crawl_id = Some(crawl_id);
assert!(worker.process_crawl_task(task).await.is_ok());
assert_eq!(
task_repo.update_count(),
1,
"corrupt blacklist must mark the task failed"
);
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"rejection on an existing crawl must increment failed_tasks"
);
}
#[tokio::test]
async fn test_mock_process_extract_task_engine_error() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"urls": ["https://example.com/page"]
}));
let result = worker.process_extract_task(task).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_mock_process_extract_task_invalid_payload() {
let worker = build_mock_worker().await;
let task = make_task(json!({"not_a_valid": "field"}));
let result = worker.process_extract_task(task).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_mock_process_extract_task_empty_urls() {
let worker = build_mock_worker().await;
let task = make_task(json!({"urls": []}));
let result = worker.process_extract_task(task).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_builder_new_creates_default() {
let builder = ScrapeWorkerBuilder::new();
assert_eq!(builder.default_concurrency_limit(), 10);
}
#[tokio::test]
async fn test_builder_default_impl() {
let builder = ScrapeWorkerBuilder::default();
assert_eq!(builder.default_concurrency_limit(), 10);
}
#[tokio::test]
async fn test_builder_with_default_concurrency_limit() {
let builder = ScrapeWorkerBuilder::new().with_default_concurrency_limit(50);
assert_eq!(builder.default_concurrency_limit(), 50);
}
#[tokio::test]
async fn test_builder_build_success() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let builder = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>);
#[cfg(feature = "metrics")]
let builder = builder.with_memory_scheduler(make_test_memory_scheduler());
let worker = builder.build();
assert!(worker.is_ok(), "build should succeed with all deps");
let w = worker.unwrap();
assert_eq!(w.default_concurrency_limit, 10);
}
#[tokio::test]
async fn test_builder_build_missing_repository() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "repository is required");
}
#[tokio::test]
async fn test_builder_build_missing_result_repository() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "result_repository is required");
}
#[tokio::test]
async fn test_builder_build_missing_crawl_repository() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "crawl_repository is required");
}
#[tokio::test]
async fn test_builder_build_missing_webhook_service() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "webhook_service is required");
}
#[tokio::test]
async fn test_builder_build_missing_engine_client() {
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "engine_client is required");
}
#[tokio::test]
async fn test_builder_build_missing_team_semaphore() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "team_semaphore is required");
}
#[tokio::test]
async fn test_builder_build_missing_settings() {
let engine_client = Arc::new(EngineClient::new());
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "settings is required");
}
#[tokio::test]
async fn test_builder_build_missing_extraction_service() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "extraction_service is required");
}
#[tokio::test]
async fn test_builder_build_missing_cache_service() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "cache_service is required");
}
#[tokio::test]
async fn test_builder_with_custom_concurrency_limit() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let builder = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.with_default_concurrency_limit(100);
#[cfg(feature = "metrics")]
let builder = builder.with_memory_scheduler(make_test_memory_scheduler());
let worker = builder.build().expect("build should succeed");
assert_eq!(worker.default_concurrency_limit, 100);
}
use crate::bootstrap::infrastructure::init_infrastructure;
use crate::bootstrap::services::init_services;
use crate::common::test_support::testcontainers_fixtures as tcf;
async fn require_docker() -> bool {
tcf::docker_available().await
}
async fn build_scrape_worker() -> anyhow::Result<ScrapeWorker> {
let handle = tcf::DbHandle::start().await?;
let settings = tcf::settings_with_urls(&handle.pg.url)?;
let settings_arc = std::sync::Arc::new(settings.clone());
let infra = init_infrastructure(&settings).await?;
let engines = crate::bootstrap::engines::init_engine_components(
infra.http_client.clone(),
None,
crate::config::settings::ProxyStrategy::RoundRobin,
None,
&settings.engines,
&settings.timeouts.engines,
settings.engines.max_response_body_bytes as usize,
&settings.politeness,
&settings.session,
&settings.observability,
None,
);
let services = init_services(
&infra,
engines.engine_client.clone(),
infra.http_client.clone(),
&settings,
)
.await
.expect("init_services");
let coalesce_coordinator = make_coalesce_coordinator_with_coalescer(
infra.repositories.task_repo.clone() as Arc<dyn TaskRepository>,
infra.repositories.result_repo.clone() as Arc<dyn ScrapeResultRepository>,
services.request_coalescer.clone(),
);
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: infra.repositories.task_repo.clone() as Arc<dyn TaskRepository>,
result_repository: infra.repositories.result_repo.clone()
as Arc<dyn ScrapeResultRepository>,
crawl_repository: infra.repositories.crawl_repo.clone() as Arc<dyn CrawlRepository>,
webhook_service: services.webhook_service.clone(),
credits_repository: infra.repositories.credits_repo.clone() as Arc<dyn CreditsRepository>,
engine_client: engines.engine_client.clone(),
create_scrape_use_case: services.create_scrape_use_case.clone(),
team_semaphore: services.team_semaphore.clone(),
coalesce_coordinator,
robots_checker: services.robots_checker.clone(),
settings: settings_arc,
default_concurrency_limit: settings.concurrency.default_team_limit as usize,
extraction_service: services.extraction_service.clone(),
#[cfg(feature = "content")]
content_extractor: services.content_extractor.clone(),
cache_service: infra.cache_service.clone(),
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
Ok(worker)
}
fn make_crawl_config(
include_patterns: Option<Vec<String>>,
exclude_patterns: Option<Vec<String>>,
) -> CrawlConfigDto {
CrawlConfigDto {
max_depth: 1,
include_patterns,
exclude_patterns,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
}
}
#[tokio::test]
async fn tc_scrape_worker_new_constructs_successfully() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_scrape_worker_new_constructs_successfully");
return;
}
let _garrison_guard = crate::common::test_helpers::acquire_garrison_global_state().await;
let worker = match build_scrape_worker().await {
Ok(w) => w,
Err(e) => {
eprintln!("[skip] failed to build ScrapeWorker: {e}");
return;
}
};
assert_ne!(worker.worker_id, crate::domain::id::Id::nil());
assert!(worker.default_concurrency_limit >= 1);
}
#[tokio::test]
async fn tc_scrape_worker_should_crawl_with_no_patterns() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_scrape_worker_should_crawl_with_no_patterns");
return;
}
let _garrison_guard = crate::common::test_helpers::acquire_garrison_global_state().await;
let worker = match build_scrape_worker().await {
Ok(w) => w,
Err(e) => {
eprintln!("[skip] failed to build ScrapeWorker: {e}");
return;
}
};
let config = make_crawl_config(None, None);
assert!(worker.should_crawl("https://example.com/page1", &config));
}
#[tokio::test]
async fn tc_scrape_worker_should_crawl_with_include_patterns() {
if !require_docker().await {
eprintln!(
"[skip] Docker unavailable — tc_scrape_worker_should_crawl_with_include_patterns"
);
return;
}
let _garrison_guard = crate::common::test_helpers::acquire_garrison_global_state().await;
let worker = match build_scrape_worker().await {
Ok(w) => w,
Err(e) => {
eprintln!("[skip] failed to build ScrapeWorker: {e}");
return;
}
};
let config = make_crawl_config(Some(vec!["example\\.com".to_string()]), None);
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(!worker.should_crawl("https://other.com/page", &config));
}
#[tokio::test]
async fn tc_scrape_worker_should_crawl_with_exclude_patterns() {
if !require_docker().await {
eprintln!(
"[skip] Docker unavailable — tc_scrape_worker_should_crawl_with_exclude_patterns"
);
return;
}
let _garrison_guard = crate::common::test_helpers::acquire_garrison_global_state().await;
let worker = match build_scrape_worker().await {
Ok(w) => w,
Err(e) => {
eprintln!("[skip] failed to build ScrapeWorker: {e}");
return;
}
};
let config = make_crawl_config(None, Some(vec!["blocked".to_string()]));
assert!(worker.should_crawl("https://example.com/page", &config));
assert!(!worker.should_crawl("https://example.com/blocked", &config));
}
#[tokio::test]
async fn tc_scrape_worker_builder_builds_full_worker() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_scrape_worker_builder_builds_full_worker");
return;
}
let _garrison_guard = crate::common::test_helpers::acquire_garrison_global_state().await;
let handle = match tcf::DbHandle::start().await {
Ok(h) => h,
Err(e) => {
eprintln!("[skip] failed to start db container: {e}");
return;
}
};
let settings = tcf::settings_with_urls(&handle.pg.url).unwrap();
let settings_arc = std::sync::Arc::new(settings.clone());
let infra = match init_infrastructure(&settings).await {
Ok(i) => i,
Err(e) => {
eprintln!("[skip] failed to init infrastructure: {e}");
return;
}
};
let engines = crate::bootstrap::engines::init_engine_components(
infra.http_client.clone(),
None,
crate::config::settings::ProxyStrategy::RoundRobin,
None,
&settings.engines,
&settings.timeouts.engines,
settings.engines.max_response_body_bytes as usize,
&settings.politeness,
&settings.session,
&settings.observability,
None,
);
let services = init_services(
&infra,
engines.engine_client.clone(),
infra.http_client.clone(),
&settings,
)
.await
.expect("init_services");
let builder = ScrapeWorkerBuilder::new()
.with_repository(infra.repositories.task_repo.clone() as Arc<dyn TaskRepository>)
.with_result_repository(
infra.repositories.result_repo.clone() as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(infra.repositories.crawl_repo.clone() as Arc<dyn CrawlRepository>)
.with_webhook_service(services.webhook_service.clone())
.with_credits_repository(
infra.repositories.credits_repo.clone() as Arc<dyn CreditsRepository>
)
.with_engine_client(engines.engine_client.clone())
.with_create_scrape_use_case(services.create_scrape_use_case.clone())
.with_team_semaphore(services.team_semaphore.clone())
.with_robots_checker(services.robots_checker.clone())
.with_settings(settings_arc)
.with_default_concurrency_limit(settings.concurrency.default_team_limit as usize)
.with_extraction_service(services.extraction_service.clone())
.with_cache_service(infra.cache_service.clone());
#[cfg(feature = "metrics")]
let builder = builder.with_memory_scheduler(make_test_memory_scheduler());
let worker = builder
.build()
.expect("ScrapeWorkerBuilder::build should succeed with all required deps");
assert_ne!(worker.worker_id, crate::domain::id::Id::nil());
}
#[tokio::test]
async fn tc_scrape_worker_build_crawl_request() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_scrape_worker_build_crawl_request");
return;
}
let _garrison_guard = crate::common::test_helpers::acquire_garrison_global_state().await;
let worker = match build_scrape_worker().await {
Ok(w) => w,
Err(e) => {
eprintln!("[skip] failed to build ScrapeWorker: {e}");
return;
}
};
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
serde_json::json!({}),
);
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
assert_eq!(request.url, "https://example.com");
}
use crate::queue::task_queue::QueueError;
struct MockTaskQueue;
#[async_trait::async_trait]
impl TaskQueue for MockTaskQueue {
async fn enqueue(&self, task: Task) -> Result<Task, QueueError> {
Ok(task)
}
async fn dequeue(&self, _worker_id: crate::domain::id::Id) -> Result<Option<Task>, QueueError> {
Ok(None)
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
struct FailingTaskQueue;
#[async_trait::async_trait]
impl TaskQueue for FailingTaskQueue {
async fn enqueue(&self, task: Task) -> Result<Task, QueueError> {
Ok(task)
}
async fn dequeue(&self, _worker_id: crate::domain::id::Id) -> Result<Option<Task>, QueueError> {
Err(QueueError::Empty)
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
struct MockExtractionServiceWithTokens;
#[async_trait::async_trait]
impl ExtractionServiceTrait for MockExtractionServiceWithTokens {
async fn extract(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> Result<(Value, TokenUsage)> {
Ok((
json!({"title": "Extracted Title"}),
TokenUsage {
prompt_tokens: 100,
completion_tokens: 50,
total_tokens: 150,
},
))
}
async fn extract_with_schema(
&self,
_html_content: &str,
_schema: &Value,
) -> Result<(Value, TokenUsage)> {
Ok((
json!({"data": "value"}),
TokenUsage {
prompt_tokens: 200,
completion_tokens: 100,
total_tokens: 300,
},
))
}
fn extract_with_selectors(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> Result<Value> {
Ok(json!({}))
}
async fn extract_with_rag(
&self,
_html_content: &str,
_query: &str,
_schema: &Value,
_rag_strategy: &crate::domain::services::rag_strategy::RagExtractionStrategy,
) -> Result<(Value, TokenUsage)> {
Ok((json!({}), TokenUsage::default()))
}
}
async fn build_mock_worker_with_tokens() -> ScrapeWorker {
let engine_client = Arc::new(EngineClient::new());
let settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings for mock worker");
let settings_arc = Arc::new(settings.clone());
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: settings_arc,
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionServiceWithTokens)
as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
#[tokio::test]
async fn test_scrape_worker_debug_impl_outputs_fields() {
let worker = build_mock_worker().await;
let debug_str = format!("{:?}", worker);
assert!(debug_str.contains("ScrapeWorker"));
assert!(debug_str.contains("worker_id"));
assert!(debug_str.contains("default_concurrency_limit"));
assert!(debug_str.contains(".."));
}
#[tokio::test]
async fn test_mock_process_next_task_empty_queue_returns_false() {
let worker = build_mock_worker().await;
let queue = MockTaskQueue;
let result = worker.process_next_task(&queue).await;
assert!(result.is_ok());
assert!(!result.unwrap(), "empty queue should return false");
}
#[tokio::test]
async fn test_mock_handle_crawl_success_with_extraction_rules() {
let worker = build_mock_worker_with_tokens().await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let mut rules = HashMap::new();
rules.insert(
"title".to_string(),
ExtractionRule {
selector: Some("h1".to_string()),
attr: None,
is_array: false,
use_llm: None,
llm_prompt: None,
output_format: None,
},
);
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: Some(rules),
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_success(
&task,
response,
crate::domain::id::Id::generate(),
0,
&config,
&request,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_handle_scrape_success_with_token_usage() {
let worker = build_mock_worker_with_tokens().await;
let task = make_task(json!({
"url": "https://example.com",
"extraction_rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_parse_crawl_payload_invalid_crawl_id_defaults_to_nil() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"crawl_id": "not-a-uuid",
"depth": 1,
"config": {"max_depth": 2}
}));
let (crawl_id, depth, _) = worker.parse_crawl_payload(&task).await.unwrap();
assert_eq!(crawl_id, crate::domain::id::Id::nil());
assert_eq!(depth, 1);
}
#[tokio::test]
async fn test_mock_parse_crawl_payload_missing_config_fails() {
let worker = build_mock_worker().await;
let crawl_id = crate::domain::id::Id::generate();
let task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 3
}));
assert!(worker.parse_crawl_payload(&task).await.is_err());
}
#[tokio::test]
async fn test_mock_parse_crawl_payload_invalid_config_json_fails() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"crawl_id": crate::domain::id::Id::generate().to_string(),
"depth": 0,
"config": "not-an-object"
}));
assert!(worker.parse_crawl_payload(&task).await.is_err());
}
#[tokio::test]
async fn test_mock_should_crawl_empty_include_patterns_returns_false() {
let worker = build_mock_worker().await;
let config = make_crawl_config(Some(vec![]), None);
assert!(!worker.should_crawl("https://example.com/page", &config));
}
#[tokio::test]
async fn test_mock_should_crawl_empty_exclude_patterns_returns_true() {
let worker = build_mock_worker().await;
let config = make_crawl_config(None, Some(vec![]));
assert!(worker.should_crawl("https://example.com/page", &config));
}
#[tokio::test]
async fn test_mock_extract_and_queue_links_dfs_strategy() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
task.url = "https://example.com".to_string();
let html = r#"<html><body>
<a href="https://example.com/page1">Page 1</a>
<a href="https://example.com/page2">Page 2</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = CrawlConfigDto {
max_depth: 3,
include_patterns: None,
exclude_patterns: None,
strategy: Some("dfs".to_string()),
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_extract_and_queue_links_filters_self_and_non_http() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
task.url = "https://example.com".to_string();
let html = r#"<html><body>
<a href="https://example.com">Self</a>
<a href="mailto:test@example.com">Email</a>
<a href="javascript:void(0)">JS</a>
<a href="/relative">Relative</a>
<a href="https://other.com/page">Other</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_build_crawl_request_with_extraction_rules() {
let worker = build_mock_worker().await;
let task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Crawl,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
);
let mut rules = HashMap::new();
rules.insert(
"title".to_string(),
ExtractionRule {
selector: Some("h1".to_string()),
attr: None,
is_array: false,
use_llm: None,
llm_prompt: None,
output_format: None,
},
);
let config = CrawlConfigDto {
max_depth: 2,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: Some(rules),
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
assert_eq!(request.url, "https://example.com");
assert_eq!(request.options.method, HttpMethod::Get);
}
#[tokio::test]
async fn test_mock_parse_extract_payload_with_rules() {
let worker = build_mock_worker().await;
let task = make_task(json!({
"urls": ["https://example.com/page"],
"rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let (payload, url) = worker.parse_extract_payload(&task).await.unwrap();
assert_eq!(url, "https://example.com/page");
assert!(payload.rules.is_some());
assert_eq!(payload.rules.as_ref().unwrap().len(), 1);
}
#[tokio::test]
async fn test_mock_handle_rules_extraction_with_tokens() {
let worker = build_mock_worker_with_tokens().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Hello</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 50,
final_url: None,
markdown: None,
};
let mut rules = HashMap::new();
rules.insert(
"title".to_string(),
ExtractionRule {
selector: Some("h1".to_string()),
attr: None,
is_array: false,
use_llm: None,
llm_prompt: None,
output_format: None,
},
);
let result = worker
.handle_rules_extraction(&mut task, &response, &rules, "https://example.com")
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_handle_prompt_extraction_with_tokens() {
let worker = build_mock_worker_with_tokens().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Hello world</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 30,
final_url: None,
markdown: None,
};
let result = worker
.handle_prompt_extraction(
&mut task,
&response,
"Extract the main topic".to_string(),
"https://example.com",
)
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_handle_schema_extraction_with_tokens() {
let worker = build_mock_worker_with_tokens().await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body>Data</body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 20,
final_url: None,
markdown: None,
};
let schema = json!({"type": "object", "properties": {"title": {"type": "string"}}});
let result = worker
.handle_schema_extraction(&mut task, &response, &schema, "https://example.com")
.await;
assert!(result.is_ok());
assert_eq!(task.status, TaskStatus::Completed);
}
#[tokio::test]
async fn test_mock_handle_crawl_failure_with_proxy() {
let worker = build_mock_worker().await;
let mut task = make_task(json!({}));
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: Some("http://proxy:3128".to_string()),
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
};
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_failure(
&mut task,
anyhow::anyhow!("Network error"),
crate::domain::id::Id::generate(),
&request,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_handle_crawl_success_with_screenshot() {
let worker = build_mock_worker().await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><a href="/page1">Link</a></body></html>"#.to_string(),
status_code: 200,
screenshot: Some("base64screenshot".to_string()),
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_success(
&task,
response,
crate::domain::id::Id::generate(),
0,
&config,
&request,
)
.await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_mock_process_text_encoding_json_content() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"{"key": "value"}"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "application/json".to_string(),
headers: HashMap::new(),
response_time_ms: 30,
final_url: None,
markdown: None,
};
let result = worker.process_text_encoding(&task, &response).await;
match result {
Ok(content) => assert!(!content.is_empty() || response.content.is_empty()),
Err(_) => { }
}
}
#[tokio::test]
async fn test_mock_process_text_encoding_empty_content() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: String::new(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 5,
final_url: None,
markdown: None,
};
let result = worker.process_text_encoding(&task, &response).await;
match result {
Ok(content) => assert!(content.is_empty()),
Err(_) => { }
}
}
#[tokio::test]
async fn test_mock_save_result_large_content() {
let worker = build_mock_worker().await;
let task = make_task(json!({"url": "https://example.com"}));
let large_content = "x".repeat(1024 * 1024 + 1);
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: large_content,
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 500,
final_url: None,
markdown: None,
};
let result = worker.save_result(&task, &response, None).await;
assert!(result.is_ok());
}
use crate::engines::router::{EngineRouterTrait, EngineStats};
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
struct ConfigurableTaskRepo {
fail_mark_failed: AtomicBool,
fail_update: AtomicBool,
guard_rejects: AtomicBool,
fail_find_by_id: AtomicBool,
fail_find_existing_urls: AtomicBool,
find_by_id_result: std::sync::Mutex<Option<Task>>,
existing_urls_result: std::sync::Mutex<HashSet<String>>,
mark_failed_count: AtomicU32,
update_count: AtomicU32,
create_count: AtomicU32,
mark_completed_count: AtomicU32,
last_updated_task: std::sync::Mutex<Option<Task>>,
}
impl ConfigurableTaskRepo {
fn new() -> Self {
Self {
fail_mark_failed: AtomicBool::new(false),
fail_update: AtomicBool::new(false),
guard_rejects: AtomicBool::new(false),
fail_find_by_id: AtomicBool::new(false),
fail_find_existing_urls: AtomicBool::new(false),
find_by_id_result: std::sync::Mutex::new(None),
existing_urls_result: std::sync::Mutex::new(HashSet::new()),
mark_failed_count: AtomicU32::new(0),
update_count: AtomicU32::new(0),
create_count: AtomicU32::new(0),
mark_completed_count: AtomicU32::new(0),
last_updated_task: std::sync::Mutex::new(None),
}
}
fn mark_completed_count(&self) -> u32 {
self.mark_completed_count.load(Ordering::SeqCst)
}
fn mark_failed_count(&self) -> u32 {
self.mark_failed_count.load(Ordering::SeqCst)
}
fn update_count(&self) -> u32 {
self.update_count.load(Ordering::SeqCst)
}
fn create_count(&self) -> u32 {
self.create_count.load(Ordering::SeqCst)
}
fn set_existing_urls(&self, urls: HashSet<String>) {
*self.existing_urls_result.lock().unwrap() = urls;
}
fn last_updated_task(&self) -> Option<Task> {
self.last_updated_task.lock().unwrap().clone()
}
fn reject_guarded_update(&self) {
self.guard_rejects.store(true, Ordering::SeqCst);
}
}
#[async_trait::async_trait]
impl TaskRepository for ConfigurableTaskRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
self.create_count.fetch_add(1, Ordering::SeqCst);
Ok(task.clone())
}
async fn find_by_id(&self, id: crate::domain::id::Id) -> Result<Option<Task>, RepositoryError> {
if self.fail_find_by_id.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock find_by_id error"
)));
}
let guard = self.find_by_id_result.lock().unwrap();
Ok(guard.as_ref().filter(|t| t.id == id).cloned())
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
self.update_count.fetch_add(1, Ordering::SeqCst);
*self.last_updated_task.lock().unwrap() = Some(task.clone());
if self.fail_update.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock update error"
)));
}
Ok(task.clone())
}
async fn update_task_guarded(&self, task: &Task) -> Result<u64, RepositoryError> {
self.update_count.fetch_add(1, Ordering::SeqCst);
if self.guard_rejects.load(Ordering::SeqCst) {
return Ok(0);
}
*self.last_updated_task.lock().unwrap() = Some(task.clone());
if self.fail_update.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock update error"
)));
}
Ok(1)
}
async fn requeue_task(
&self,
id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
scheduled_at: Option<chrono::DateTime<chrono::Utc>>,
) -> Result<bool, RepositoryError> {
self.update_count.fetch_add(1, Ordering::SeqCst);
let mut captured = Task::new(
id,
TaskType::Scrape,
crate::domain::id::Id::nil(),
crate::domain::id::Id::nil(),
String::new(),
serde_json::json!({}),
);
captured.status = TaskStatus::Queued;
captured.scheduled_at = scheduled_at;
*self.last_updated_task.lock().unwrap() = Some(captured);
Ok(true)
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
self.mark_completed_count.fetch_add(1, Ordering::SeqCst);
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
self.mark_failed_count.fetch_add(1, Ordering::SeqCst);
if self.fail_mark_failed.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock mark_failed error"
)));
}
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
if self.fail_find_existing_urls.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock find_existing_urls error"
)));
}
Ok(self.existing_urls_result.lock().unwrap().clone())
}
async fn reset_stuck_tasks(&self, _timeout: chrono::Duration) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct ConfigurableCrawlRepo {
crawl: std::sync::Mutex<Option<Crawl>>,
fail_find_by_id: AtomicBool,
fail_increment_completed: AtomicBool,
fail_increment_failed: AtomicBool,
fail_update_status: AtomicBool,
update_status_count: AtomicU32,
failed_increment_count: AtomicU32,
}
impl ConfigurableCrawlRepo {
fn new() -> Self {
Self {
crawl: std::sync::Mutex::new(None),
fail_find_by_id: AtomicBool::new(false),
fail_increment_completed: AtomicBool::new(false),
fail_increment_failed: AtomicBool::new(false),
fail_update_status: AtomicBool::new(false),
update_status_count: AtomicU32::new(0),
failed_increment_count: AtomicU32::new(0),
}
}
fn set_crawl(&self, crawl: Crawl) {
*self.crawl.lock().unwrap() = Some(crawl);
}
fn update_status_count(&self) -> u32 {
self.update_status_count.load(Ordering::SeqCst)
}
fn failed_increment_count(&self) -> u32 {
self.failed_increment_count.load(Ordering::SeqCst)
}
}
#[async_trait::async_trait]
impl CrawlRepository for ConfigurableCrawlRepo {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
if self.fail_find_by_id.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock find_by_id error"
)));
}
Ok(self.crawl.lock().unwrap().clone())
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
if self.fail_increment_completed.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock increment_completed error"
)));
}
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
self.failed_increment_count.fetch_add(1, Ordering::SeqCst);
if self.fail_increment_failed.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock increment_failed error"
)));
}
if let Some(crawl) = self.crawl.lock().unwrap().as_mut() {
crawl.increment_failed_tasks();
crawl.updated_at = Utc::now();
}
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<(), RepositoryError> {
self.update_status_count.fetch_add(1, Ordering::SeqCst);
if self.fail_update_status.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock update_status error"
)));
}
Ok(())
}
async fn finalize_if_all_terminal(
&self,
_id: crate::domain::id::Id,
) -> Result<bool, RepositoryError> {
let eligible = {
let guard = self.crawl.lock().unwrap();
guard.as_ref().is_some_and(|c| {
!matches!(
c.status,
CrawlStatus::Completed | CrawlStatus::Failed | CrawlStatus::Cancelled
) && c.total_tasks() > 0
&& c.completed_tasks() + c.failed_tasks() == c.total_tasks()
})
};
if !eligible {
return Ok(false);
}
self.update_status_count.fetch_add(1, Ordering::SeqCst);
if self.fail_update_status.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock update_status error"
)));
}
let mut guard = self.crawl.lock().unwrap();
let Some(c) = guard.as_mut() else {
return Ok(false);
};
c.status = if c.completed_tasks() == 0 {
CrawlStatus::Failed
} else {
CrawlStatus::Completed
};
c.completed_at = Some(Utc::now());
c.updated_at = Utc::now();
Ok(true)
}
async fn update_status_guarded(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<bool, RepositoryError> {
if self.crawl.lock().unwrap().as_ref().is_some_and(|c| {
matches!(
c.status,
CrawlStatus::Completed | CrawlStatus::Failed | CrawlStatus::Cancelled
)
}) {
return Ok(false);
}
self.update_status_count.fetch_add(1, Ordering::SeqCst);
if self.fail_update_status.load(Ordering::SeqCst) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"Mock update_status error"
)));
}
Ok(true)
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct FailingScrapeResultRepo;
#[async_trait::async_trait]
impl ScrapeResultRepository for FailingScrapeResultRepo {
async fn save(&self, _result: ScrapeResult) -> Result<()> {
Err(anyhow::anyhow!("Mock save error"))
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(&self, _team_id: crate::domain::id::Id) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
struct FailingWebhookService;
#[async_trait::async_trait]
impl WebhookService for FailingWebhookService {
async fn send_webhook(&self, _event: &WebhookEvent) -> Result<u16> {
Err(anyhow::anyhow!("Mock webhook error"))
}
async fn trigger_completion(&self, _task: &Task) -> Result<()> {
Err(anyhow::anyhow!("Mock trigger_completion error"))
}
async fn trigger_failure(&self, _task: &Task, _error_msg: String) -> Result<()> {
Err(anyhow::anyhow!("Mock trigger_failure error"))
}
}
struct FailingCreditsRepo;
#[async_trait::async_trait]
impl CreditsRepository for FailingCreditsRepo {
async fn get_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, CreditsRepositoryError> {
Ok(100)
}
async fn deduct_credits(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), CreditsRepositoryError> {
Err(CreditsRepositoryError::InsufficientCredits {
available: 0,
required: 1,
})
}
async fn add_credits(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<i64, CreditsRepositoryError> {
Ok(100)
}
async fn get_transaction_history(
&self,
_team_id: crate::domain::id::Id,
_limit: Option<u32>,
) -> Result<Vec<CreditsTransaction>, CreditsRepositoryError> {
Ok(vec![])
}
async fn initialize_team_credits(
&self,
_team_id: crate::domain::id::Id,
_initial_balance: i64,
) -> Result<i64, CreditsRepositoryError> {
Ok(_initial_balance)
}
}
struct DenyingRobotsChecker;
#[async_trait::async_trait]
impl RobotsCheckerTrait for DenyingRobotsChecker {
async fn is_allowed(&self, _url_str: &str, _user_agent: &str) -> Result<bool> {
Ok(false)
}
async fn get_crawl_delay(&self, _url_str: &str, _user_agent: &str) -> Result<Option<Duration>> {
Ok(None)
}
}
struct DelayingRobotsChecker;
#[async_trait::async_trait]
impl RobotsCheckerTrait for DelayingRobotsChecker {
async fn is_allowed(&self, _url_str: &str, _user_agent: &str) -> Result<bool> {
Ok(true)
}
async fn get_crawl_delay(&self, _url_str: &str, _user_agent: &str) -> Result<Option<Duration>> {
Ok(Some(Duration::from_millis(10)))
}
}
struct ErroringRobotsChecker;
#[async_trait::async_trait]
impl RobotsCheckerTrait for ErroringRobotsChecker {
async fn is_allowed(&self, _url_str: &str, _user_agent: &str) -> Result<bool> {
Err(anyhow::anyhow!("Mock robots error"))
}
async fn get_crawl_delay(&self, _url_str: &str, _user_agent: &str) -> Result<Option<Duration>> {
Err(anyhow::anyhow!("Mock crawl delay error"))
}
}
struct MockEngineRouter {
error_factory: Box<dyn Fn() -> EngineError + Send + Sync>,
}
impl MockEngineRouter {
fn new(error: EngineError) -> Self {
let error = Arc::new(error);
Self {
error_factory: Box::new(move || match &*error {
EngineError::Timeout(d) => EngineError::Timeout(*d),
EngineError::AllEnginesFailed(s) => EngineError::AllEnginesFailed(s.clone()),
EngineError::Expired => EngineError::Expired,
EngineError::NoEnginesAvailable => EngineError::NoEnginesAvailable,
EngineError::InvalidUrl(s) => EngineError::InvalidUrl(s.clone()),
EngineError::SsrfProtection(s) => EngineError::SsrfProtection(s.clone()),
EngineError::BrowserError(s) => EngineError::BrowserError(s.clone()),
EngineError::AntiBotDetected(s) => EngineError::AntiBotDetected(s.clone()),
EngineError::FeatureToggle(s) => EngineError::FeatureToggle(s.clone()),
EngineError::RequestFailed(s) => EngineError::RequestFailed(s.clone()),
EngineError::Other(s) => EngineError::Other(s.clone()),
EngineError::MemoryConstrained(s) => EngineError::MemoryConstrained(s.clone()),
EngineError::Unsupported(s) => EngineError::Unsupported(s.clone()),
EngineError::Internal(s) => EngineError::Internal(s.clone()),
EngineError::EngineMrtExceeded { engine, mrt } => EngineError::EngineMrtExceeded {
engine: engine.clone(),
mrt: *mrt,
},
}),
}
}
}
#[async_trait::async_trait]
impl EngineRouterTrait for MockEngineRouter {
async fn route(
&self,
_request: &crate::engines::engine_client::InternalScrapeRequest,
) -> Result<crate::engines::engine_client::InternalScrapeResponse, EngineError> {
Err((self.error_factory)())
}
async fn aggregate(
&self,
_request: &crate::engines::engine_client::InternalScrapeRequest,
) -> Result<crate::engines::engine_client::InternalScrapeResponse, EngineError> {
Err((self.error_factory)())
}
fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats> {
std::collections::HashMap::new()
}
fn reset_engine_stats(&self, _engine_name: &str) {}
fn registered_engines(&self) -> Vec<String> {
vec![]
}
}
struct TaskQueueWithTask {
task: std::sync::Mutex<Option<Task>>,
}
impl TaskQueueWithTask {
fn new(task: Task) -> Self {
Self {
task: std::sync::Mutex::new(Some(task)),
}
}
}
#[async_trait::async_trait]
impl TaskQueue for TaskQueueWithTask {
async fn enqueue(&self, task: Task) -> Result<Task, QueueError> {
Ok(task)
}
async fn dequeue(&self, _worker_id: crate::domain::id::Id) -> Result<Option<Task>, QueueError> {
Ok(self.task.lock().unwrap().take())
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
async fn build_configurable_worker(
task_repo: Arc<dyn TaskRepository>,
crawl_repo: Arc<dyn CrawlRepository>,
robots_checker: Arc<dyn RobotsCheckerTrait>,
engine_client: Arc<EngineClient>,
) -> ScrapeWorker {
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for configurable worker");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: crawl_repo,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
async fn build_worker_with_failing_deps(
result_repo: Arc<dyn ScrapeResultRepository>,
webhook_service: Arc<dyn WebhookService>,
credits_repo: Arc<dyn CreditsRepository>,
) -> ScrapeWorker {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for failing deps worker");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service,
credits_repository: credits_repo,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
#[tokio::test]
async fn test_process_task_expired_task_marks_failed_and_triggers_webhook() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({"url": "https://example.com"}));
task.expires_at = Some(Utc::now() - chrono::Duration::seconds(1));
let result = worker.process_task(task).await;
assert!(result.is_ok(), "expired task should return Ok(())");
let captured = task_repo.last_updated_task().expect("必须写入任务快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("expired"),
"过期原因必须落库,实际: {reason}"
);
}
#[tokio::test]
async fn test_process_task_not_expired_proceeds_normally() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({"url": "https://example.com"}));
task.expires_at = Some(Utc::now() + chrono::Duration::hours(1));
let result = worker.process_task(task).await;
assert!(result.is_ok());
assert_eq!(
task_repo.mark_failed_count(),
0,
"mark_failed should not be called for expiration check on non-expired task"
);
}
#[tokio::test]
async fn test_process_task_concurrency_limit_exceeded_reschedules() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for concurrency test");
let team_semaphore = Arc::new(TeamSemaphore::new(0));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo.clone(),
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client: Arc::new(EngineClient::new()),
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let task = make_task(json!({"url": "https://example.com"}));
let result = worker.process_task(task).await;
assert!(result.is_ok(), "rescheduled task should return Ok(())");
assert_eq!(
task_repo.update_count(),
1,
"update should be called to reschedule task"
);
}
#[cfg(feature = "metrics")]
#[tokio::test]
async fn test_process_task_memory_critical_defers_task() {
use crate::infrastructure::observability::metrics::SystemMonitorTrait;
use std::sync::atomic::{AtomicU64, Ordering};
struct CriticalMockMonitor {
bits: AtomicU64,
}
impl SystemMonitorTrait for CriticalMockMonitor {
fn cpu_usage(&self) -> f64 {
0.0
}
fn memory_usage(&self) -> f64 {
f64::from_bits(self.bits.load(Ordering::Relaxed))
}
fn is_metrics_stale(&self) -> bool {
false
}
}
let memory_scheduler: Arc<MemoryScheduler> = Arc::new(MemoryScheduler::new(
Arc::new(CriticalMockMonitor {
bits: AtomicU64::new(0.95f64.to_bits()),
}),
0.8,
0.9,
Duration::from_secs(30),
));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for memory critical test");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(
task_repo.clone() as Arc<dyn TaskRepository>,
result_repo.clone(),
);
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo.clone() as Arc<dyn TaskRepository>,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client: Arc::new(EngineClient::new()),
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
memory_scheduler,
});
let task = make_task(json!({"url": "https://example.com"}));
let original_scheduled_at = task.scheduled_at;
let result = worker.process_task(task).await;
assert!(
result.is_ok(),
"Critical memory should defer task, returning Ok(())"
);
assert_eq!(
task_repo.update_count(),
1,
"Critical memory should trigger one update to reschedule task"
);
assert_eq!(
task_repo.mark_failed_count(),
0,
"Critical memory should NOT mark task as failed"
);
let updated = task_repo
.last_updated_task()
.expect("update should have captured the task");
assert_eq!(
updated.status,
TaskStatus::Queued,
"deferred task should be re-queued"
);
let now = Utc::now();
let scheduled = updated
.scheduled_at
.expect("scheduled_at should be set for deferred task");
assert!(
scheduled > now,
"scheduled_at should be in the future (deferred), got {} <= now {}",
scheduled,
now
);
if let Some(orig) = original_scheduled_at {
assert!(
scheduled > orig,
"deferred scheduled_at should be later than original"
);
}
}
#[cfg(feature = "metrics")]
#[tokio::test]
async fn test_process_task_memory_critical_halves_aimd_before_requeue() {
use crate::domain::services::team_semaphore::AdaptiveParams;
use crate::infrastructure::observability::metrics::SystemMonitorTrait;
use std::sync::atomic::{AtomicU64, Ordering};
struct CriticalMockMonitor {
bits: AtomicU64,
}
impl SystemMonitorTrait for CriticalMockMonitor {
fn cpu_usage(&self) -> f64 {
0.0
}
fn memory_usage(&self) -> f64 {
f64::from_bits(self.bits.load(Ordering::Relaxed))
}
fn is_metrics_stale(&self) -> bool {
false
}
}
let memory_scheduler: Arc<MemoryScheduler> = Arc::new(MemoryScheduler::new(
Arc::new(CriticalMockMonitor {
bits: AtomicU64::new(0.95f64.to_bits()),
}),
0.8,
0.9,
Duration::from_secs(30),
));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for aimd critical test");
let team_semaphore = Arc::new(TeamSemaphore::with_adaptive(AdaptiveParams {
initial: 10,
min_limit: 1,
max_limit: 100,
increase_threshold: 10,
}));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(
task_repo.clone() as Arc<dyn TaskRepository>,
result_repo.clone(),
);
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo.clone() as Arc<dyn TaskRepository>,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client: Arc::new(EngineClient::new()),
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore: team_semaphore.clone(),
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
memory_scheduler,
});
let task = make_task(json!({"url": "https://example.com"}));
let team_id = task.team_id;
let _warm = team_semaphore.try_acquire(team_id).expect("warm permit");
drop(_warm);
assert_eq!(team_semaphore.current_target(team_id), 10, "initial target");
let result = worker.process_task(task).await;
assert!(
result.is_ok(),
"process_task must complete without deadlock"
);
assert_eq!(
team_semaphore.current_target(team_id),
5,
"critical signal must halve AIMD target before requeue"
);
assert_eq!(
task_repo.update_count(),
1,
"existing admit gate must still requeue the task"
);
}
#[cfg(feature = "metrics")]
#[tokio::test]
async fn test_process_task_memory_pressure_defers_task() {
use crate::infrastructure::observability::metrics::SystemMonitorTrait;
use std::sync::atomic::{AtomicU64, Ordering};
struct PressureMockMonitor {
bits: AtomicU64,
}
impl SystemMonitorTrait for PressureMockMonitor {
fn cpu_usage(&self) -> f64 {
0.0
}
fn memory_usage(&self) -> f64 {
f64::from_bits(self.bits.load(Ordering::Relaxed))
}
fn is_metrics_stale(&self) -> bool {
false
}
}
let memory_scheduler: Arc<MemoryScheduler> = Arc::new(MemoryScheduler::new(
Arc::new(PressureMockMonitor {
bits: AtomicU64::new(0.85f64.to_bits()),
}),
0.8,
0.9,
Duration::from_secs(30),
));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for memory pressure test");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(
task_repo.clone() as Arc<dyn TaskRepository>,
result_repo.clone(),
);
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo.clone() as Arc<dyn TaskRepository>,
result_repository: result_repo,
crawl_repository: Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client: Arc::new(EngineClient::new()),
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
memory_scheduler,
});
let task = make_task(json!({"url": "https://example.com"}));
let result = worker.process_task(task).await;
assert!(
result.is_ok(),
"Pressure memory should defer task, returning Ok(())"
);
assert_eq!(
task_repo.update_count(),
1,
"Pressure memory should trigger one update to defer task"
);
assert_eq!(
task_repo.mark_failed_count(),
0,
"Pressure memory should NOT mark task as failed"
);
}
#[cfg(feature = "metrics")]
#[tokio::test]
async fn test_process_task_memory_normal_proceeds() {
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({"url": "https://example.com"}));
let _ = worker.process_task(task).await;
}
#[tokio::test]
async fn test_process_next_task_with_task_returns_true() {
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({"url": "https://example.com"}));
let queue = TaskQueueWithTask::new(task);
let result = worker.process_next_task(&queue).await;
assert!(result.is_ok());
assert!(
result.unwrap(),
"process_next_task should return true when a task is dequeued"
);
}
#[tokio::test]
async fn test_process_scrape_task_timeout_error_marks_failed_directly() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(MockEngineRouter::new(EngineError::Timeout(
Duration::from_secs(30),
)));
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
engine_client,
)
.await;
let mut task = make_task(json!({"url": "https://example.com"}));
task.status = TaskStatus::Active;
task_repo
.find_by_id_result
.lock()
.unwrap()
.replace(task.clone());
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok(), "timeout error should be handled internally");
assert!(
task_repo.update_count() >= 1,
"update should be called for timeout error path"
);
}
#[tokio::test]
async fn test_process_scrape_task_all_engines_failed_error_marks_failed() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(MockEngineRouter::new(
EngineError::AllEnginesFailed("all engines unavailable".to_string()),
));
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
engine_client,
)
.await;
let mut task = make_task(json!({"url": "https://example.com"}));
task.status = TaskStatus::Active;
task_repo
.find_by_id_result
.lock()
.unwrap()
.replace(task.clone());
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok());
assert!(
task_repo.update_count() >= 1,
"update should be called for all_engines_failed path"
);
}
#[tokio::test]
async fn test_process_scrape_task_expired_error_marks_failed() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(MockEngineRouter::new(EngineError::Expired));
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
engine_client,
)
.await;
let mut task = make_task(json!({"url": "https://example.com"}));
task.status = TaskStatus::Active;
task_repo
.find_by_id_result
.lock()
.unwrap()
.replace(task.clone());
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok());
assert!(
task_repo.update_count() >= 1,
"update should be called for expired error path"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_all_tasks_completed() {
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
let crawl = Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
CrawlStatus::Processing,
json!({}),
10, 8, 2, Utc::now(),
Utc::now(),
None,
);
crawl_repo.set_crawl(crawl);
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
worker.update_crawl_completion_status(crawl_id).await;
assert_eq!(
crawl_repo.update_status_count(),
1,
"update_status should be called when all tasks are done"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_not_all_tasks_completed() {
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
let crawl = Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
CrawlStatus::Processing,
json!({}),
10, 3, 2, Utc::now(),
Utc::now(),
None,
);
crawl_repo.set_crawl(crawl);
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
worker.update_crawl_completion_status(crawl_id).await;
assert_eq!(
crawl_repo.update_status_count(),
0,
"update_status should not be called when tasks are still pending"
);
}
#[tokio::test]
async fn test_update_crawl_completion_status_crawl_repo_error() {
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
crawl_repo.fail_find_by_id.store(true, Ordering::SeqCst);
let crawl_id = crate::domain::id::Id::generate();
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
worker.update_crawl_completion_status(crawl_id).await;
assert_eq!(crawl_repo.update_status_count(), 0);
}
#[tokio::test]
async fn test_check_robots_txt_denied_returns_false() {
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(DenyingRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let result = worker.check_robots_txt(&task).await;
assert!(
!result,
"check_robots_txt should return false when robots.txt denies access"
);
}
#[tokio::test]
async fn test_check_robots_txt_with_crawl_delay_returns_true() {
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(DelayingRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let result = worker.check_robots_txt(&task).await;
assert!(
result,
"check_robots_txt should return true when robots.txt allows with delay"
);
}
#[tokio::test]
async fn test_check_robots_txt_error_falls_back_to_allowed() {
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(ErroringRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let result = worker.check_robots_txt(&task).await;
assert!(
result,
"check_robots_txt should fall back to true when robots checker errors"
);
}
#[tokio::test]
async fn test_builder_build_missing_credits_repository() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "credits_repository is required");
}
#[tokio::test]
async fn test_builder_build_missing_create_scrape_use_case() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_team_semaphore(team_semaphore)
.with_robots_checker(Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "create_scrape_use_case is required");
}
#[tokio::test]
async fn test_builder_build_missing_robots_checker() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result = ScrapeWorkerBuilder::new()
.with_repository(Arc::new(MockTaskRepository) as Arc<dyn TaskRepository>)
.with_result_repository(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>
)
.with_crawl_repository(Arc::new(MockCrawlRepository) as Arc<dyn CrawlRepository>)
.with_webhook_service(Arc::new(MockWebhookService) as Arc<dyn WebhookService>)
.with_credits_repository(Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>)
.with_engine_client(engine_client)
.with_create_scrape_use_case(
Arc::new(MockCreateScrapeUseCase) as Arc<dyn CreateScrapeUseCaseTrait>
)
.with_team_semaphore(team_semaphore)
.with_settings(Arc::new(settings))
.with_extraction_service(Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>)
.with_cache_service(Arc::new(MockCacheService::new()) as Arc<dyn CacheService>)
.build();
assert!(result.is_err());
assert_eq!(result.unwrap_err(), "robots_checker is required");
}
#[tokio::test]
async fn test_save_extract_result_failing_result_repo_returns_error() {
let worker = build_worker_with_failing_deps(
Arc::new(FailingScrapeResultRepo) as Arc<dyn ScrapeResultRepository>,
Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
)
.await;
let mut task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "test content".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let result = worker
.save_extract_result(&mut task, &response, None, "https://example.com")
.await;
assert!(
result.is_err(),
"save_extract_result should return error when result_repository.save fails"
);
}
#[tokio::test]
async fn test_handle_failure_returns_error_when_repo_update_fails() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
task_repo.fail_update.store(true, Ordering::SeqCst);
let worker = build_configurable_worker(
task_repo,
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.attempt_count = 100;
task.max_retries = 1;
let result = worker.handle_failure(&mut task, "update failure").await;
assert!(
result.is_err(),
"handle_failure should return error when update fails"
);
}
#[tokio::test]
async fn test_trigger_webhook_failure_does_not_propagate_error() {
let worker = build_worker_with_failing_deps(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>,
Arc::new(FailingWebhookService) as Arc<dyn WebhookService>,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
)
.await;
let task = make_task(json!({}));
worker.trigger_webhook(&task, None).await;
worker
.trigger_webhook(&task, Some("error msg".to_string()))
.await;
}
#[tokio::test]
async fn test_deduct_feature_credits_failure_does_not_propagate_error() {
let worker = build_worker_with_failing_deps(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>,
Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
Arc::new(FailingCreditsRepo) as Arc<dyn CreditsRepository>,
)
.await;
worker
.deduct_feature_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
true,
true,
)
.await;
}
#[tokio::test]
async fn test_deduct_token_credits_failure_does_not_propagate_error() {
let worker = build_worker_with_failing_deps(
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>,
Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
Arc::new(FailingCreditsRepo) as Arc<dyn CreditsRepository>,
)
.await;
let usage = TokenUsage {
prompt_tokens: 100,
completion_tokens: 50,
total_tokens: 150,
};
worker
.deduct_token_credits(
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
&usage,
"test failing deduct",
)
.await;
}
#[tokio::test]
async fn test_handle_crawl_success_save_result_failure_propagates_error() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(FailingScrapeResultRepo);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
CrawlStatus::Processing,
json!({"domain_blacklist": []}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
));
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: crawl_repo.clone() as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body><a href="/page1">Link</a></body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_success(&task, response, crawl_id, 0, &config, &request)
.await;
assert!(
result.is_err(),
"handle_crawl_success should return error when save_result fails"
);
}
#[tokio::test]
async fn test_handle_crawl_success_increment_completed_error_does_not_propagate() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
crawl_repo
.fail_increment_completed
.store(true, Ordering::SeqCst);
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: crawl_repo,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: r#"<html><body>Hello</body></html>"#.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let mut config = make_crawl_config(None, None);
config.max_depth = 0; let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_success(
&task,
response,
crate::domain::id::Id::generate(),
0,
&config,
&request,
)
.await;
assert!(
result.is_ok(),
"handle_crawl_success should not fail when increment_completed_tasks errors"
);
}
#[tokio::test]
async fn test_handle_crawl_failure_increment_failed_error_does_not_propagate() {
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let task_repo: Arc<dyn TaskRepository> = Arc::new(ConfigurableTaskRepo::new());
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
crawl_repo
.fail_increment_failed
.store(true, Ordering::SeqCst);
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: crawl_repo.clone() as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let mut task = make_task(json!({}));
task.attempt_count = 100;
task.max_retries = 1;
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_failure(
&mut task,
anyhow::anyhow!("Network error"),
crate::domain::id::Id::generate(),
&request,
)
.await;
assert!(
result.is_ok(),
"handle_crawl_failure should not fail when increment_failed_tasks errors"
);
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"terminal failure must attempt the crawl failed increment"
);
}
#[tokio::test]
async fn test_handle_crawl_failure_retried_does_not_increment_failed_tasks() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.attempt_count = 0;
task.max_retries = 3;
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_failure(
&mut task,
anyhow::anyhow!("transient network error"),
crate::domain::id::Id::generate(),
&request,
)
.await;
assert!(result.is_ok());
assert_eq!(
crawl_repo.failed_increment_count(),
0,
"retried failure must not increment crawl failed_tasks"
);
assert_eq!(task.status, TaskStatus::Queued, "task should be requeued");
}
#[tokio::test]
async fn test_handle_crawl_failure_terminal_increments_failed_tasks_once() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let crawl_id = crate::domain::id::Id::generate();
let crawl = Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"terminal".into(),
"https://example.com".into(),
"https://example.com".into(),
CrawlStatus::Processing,
json!({}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
);
crawl_repo.set_crawl(crawl);
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.attempt_count = 100;
task.max_retries = 1;
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_failure(
&mut task,
anyhow::anyhow!("permanent failure"),
crawl_id,
&request,
)
.await;
assert!(result.is_ok());
assert_eq!(
crawl_repo.failed_increment_count(),
1,
"terminal failure must increment crawl failed_tasks exactly once"
);
assert_eq!(crawl_repo.update_status_count(), 1);
}
#[tokio::test]
async fn test_process_crawl_task_robots_denied_marks_failed() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let engine_client = Arc::new(EngineClient::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo.clone(),
result_repository: result_repo,
crawl_repository: crawl_repo.clone() as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(DenyingRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
CrawlStatus::Processing,
json!({"domain_blacklist": []}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
));
let task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": {"max_depth": 2}
}));
let result = worker.process_crawl_task(task).await;
assert!(result.is_ok());
let captured = task_repo.last_updated_task().expect("必须写入任务快照");
assert_eq!(captured.status, TaskStatus::Failed);
let reason = captured
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("robots.txt"),
"crawl robots 拒绝原因必须落库,实际: {reason}"
);
assert_eq!(
task_repo.mark_failed_count(),
0,
"仅 mark_failed 会丢失原因,不得使用"
);
}
struct SuccessEngineRouter {
response: crate::engines::engine_client::InternalScrapeResponse,
}
impl SuccessEngineRouter {
fn new() -> Self {
use crate::engines::engine_client::InternalScrapeResponse;
Self {
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>Hello</h1></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
},
}
}
fn with_screenshot(mut self, screenshot: String) -> Self {
self.response.screenshot = Some(screenshot);
self
}
}
#[async_trait::async_trait]
impl EngineRouterTrait for SuccessEngineRouter {
async fn route(
&self,
_request: &crate::engines::engine_client::InternalScrapeRequest,
) -> Result<crate::engines::engine_client::InternalScrapeResponse, EngineError> {
Ok(self.response.clone())
}
async fn aggregate(
&self,
_request: &crate::engines::engine_client::InternalScrapeRequest,
) -> Result<crate::engines::engine_client::InternalScrapeResponse, EngineError> {
Ok(self.response.clone())
}
fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats> {
std::collections::HashMap::new()
}
fn reset_engine_stats(&self, _engine_name: &str) {}
fn registered_engines(&self) -> Vec<String> {
vec!["mock-success-engine".to_string()]
}
}
struct FailingExtractionService;
#[async_trait::async_trait]
impl ExtractionServiceTrait for FailingExtractionService {
async fn extract(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> Result<(Value, TokenUsage)> {
Err(anyhow::anyhow!("Mock extraction failure"))
}
async fn extract_with_schema(
&self,
_html_content: &str,
_schema: &Value,
) -> Result<(Value, TokenUsage)> {
Err(anyhow::anyhow!("Mock extraction failure"))
}
fn extract_with_selectors(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> Result<Value> {
Err(anyhow::anyhow!("Mock extraction failure"))
}
async fn extract_with_rag(
&self,
_html_content: &str,
_query: &str,
_schema: &Value,
_rag_strategy: &crate::domain::services::rag_strategy::RagExtractionStrategy,
) -> Result<(Value, TokenUsage)> {
Err(anyhow::anyhow!("Mock extraction failure"))
}
}
async fn build_worker_for_success_tests(
task_repo: Arc<dyn TaskRepository>,
engine_client: Arc<EngineClient>,
credits_repo: Arc<dyn CreditsRepository>,
extraction_service: Arc<dyn ExtractionServiceTrait>,
) -> ScrapeWorker {
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for success tests");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(MockScrapeResultRepository);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(ConfigurableCrawlRepo::new()) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: credits_repo,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
#[tokio::test]
async fn test_process_scrape_task_success_path_marks_completed() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({"url": "https://example.com"}));
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok(), "success path should return Ok(())");
assert_eq!(
task_repo.mark_completed_count(),
1,
"mark_completed should be called once for successful scrape"
);
}
#[tokio::test]
async fn test_process_scrape_task_success_with_screenshot_and_proxy_deducts_credits() {
let router: Arc<dyn EngineRouterTrait> =
Arc::new(SuccessEngineRouter::new().with_screenshot("base64data".to_string()));
let engine_client = Arc::new(EngineClient::with_router(router));
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"url": "https://example.com",
"options": {
"proxy": "http://proxy.example.com:8080"
}
}));
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok());
let deductions = deducted_log.lock().unwrap();
assert!(
!deductions.is_empty(),
"deduct_credits should be called for screenshot + proxy"
);
let has_extra_credits = deductions.iter().any(|(_, amount)| *amount == 3);
assert!(
has_extra_credits,
"expected a deduct_credits call with amount 3 (screenshot + proxy), got {:?}",
deductions
);
}
#[tokio::test]
async fn test_process_scrape_task_success_handle_scrape_failure_calls_handle_failure() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let team_semaphore = Arc::new(TeamSemaphore::new(10));
let result_repo: Arc<dyn ScrapeResultRepository> = Arc::new(FailingScrapeResultRepo);
let coalesce_coordinator = make_coalesce_coordinator(task_repo.clone(), result_repo.clone());
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo.clone(),
result_repository: result_repo,
crawl_repository: Arc::new(ConfigurableCrawlRepo::new()) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore,
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
});
let task = make_task(json!({"url": "https://example.com"}));
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok(), "handle_failure path should return Ok(())");
}
struct CoalesceSubscribingRouter {
response: crate::engines::engine_client::InternalScrapeResponse,
coalescer: Arc<RequestCoalescer>,
key: CoalesceKey,
receiver: Arc<Mutex<Option<tokio::sync::broadcast::Receiver<CoalesceSignal>>>>,
}
impl CoalesceSubscribingRouter {
fn new(
coalescer: Arc<RequestCoalescer>,
key: CoalesceKey,
) -> (
Self,
Arc<Mutex<Option<tokio::sync::broadcast::Receiver<CoalesceSignal>>>>,
) {
use crate::engines::engine_client::InternalScrapeResponse;
let receiver = Arc::new(Mutex::new(None));
let router = Self {
response: InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 200,
content: "<html><body><h1>Hello</h1></body></html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
},
coalescer,
key,
receiver: receiver.clone(),
};
(router, receiver)
}
fn subscribe(&self) {
match self
.coalescer
.try_start(&self.key, crate::domain::id::Id::generate())
{
CoalesceResult::Wait(rx, _) => {
*self.receiver.lock().unwrap() = Some(rx);
}
CoalesceResult::Proceed(_) => {
panic!("router must observe the worker's in-flight coalesce entry");
}
}
}
}
#[async_trait::async_trait]
impl EngineRouterTrait for CoalesceSubscribingRouter {
async fn route(
&self,
_request: &crate::engines::engine_client::InternalScrapeRequest,
) -> Result<crate::engines::engine_client::InternalScrapeResponse, EngineError> {
self.subscribe();
Ok(self.response.clone())
}
async fn aggregate(
&self,
_request: &crate::engines::engine_client::InternalScrapeRequest,
) -> Result<crate::engines::engine_client::InternalScrapeResponse, EngineError> {
self.subscribe();
Ok(self.response.clone())
}
fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats> {
std::collections::HashMap::new()
}
fn reset_engine_stats(&self, _engine_name: &str) {}
fn registered_engines(&self) -> Vec<String> {
vec!["mock-coalesce-subscriber".to_string()]
}
}
async fn build_worker_with_shared_coalescer(
task_repo: Arc<dyn TaskRepository>,
result_repo: Arc<dyn ScrapeResultRepository>,
coalescer: Arc<RequestCoalescer>,
engine_client: Arc<EngineClient>,
) -> ScrapeWorker {
let settings = crate::bootstrap::config::load_settings()
.expect("Failed to load settings for coalesce tests");
let coalesce_coordinator =
make_coalesce_coordinator_with_coalescer(task_repo.clone(), result_repo.clone(), coalescer);
ScrapeWorker::new(ScrapeWorkerDeps {
repository: task_repo,
result_repository: result_repo,
crawl_repository: Arc::new(ConfigurableCrawlRepo::new()) as Arc<dyn CrawlRepository>,
webhook_service: Arc::new(MockWebhookService) as Arc<dyn WebhookService>,
credits_repository: Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
engine_client,
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase)
as Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore: Arc::new(TeamSemaphore::new(10)),
coalesce_coordinator,
robots_checker: Arc::new(MockRobotsChecker) as Arc<dyn RobotsCheckerTrait>,
settings: Arc::new(settings),
default_concurrency_limit: 10,
extraction_service: Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(MockCacheService::new()) as Arc<dyn CacheService>,
#[cfg(feature = "metrics")]
memory_scheduler: make_test_memory_scheduler(),
})
}
#[tokio::test]
async fn test_process_scrape_task_persist_success_broadcasts_coalesce_success() {
let coalescer = Arc::new(RequestCoalescer::new());
let task = make_task(json!({"url": "https://example.com"}));
let key = CoalesceKey::new(
task.team_id,
"https://example.com",
&ScrapeOptions::default(),
);
let (router, receiver) = CoalesceSubscribingRouter::new(coalescer.clone(), key);
let engine_client = Arc::new(EngineClient::with_router(Arc::new(router)));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_with_shared_coalescer(
task_repo.clone(),
Arc::new(MockScrapeResultRepository) as Arc<dyn ScrapeResultRepository>,
coalescer,
engine_client,
)
.await;
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok(), "success path should return Ok(())");
assert_eq!(
task_repo.mark_completed_count(),
1,
"task must be marked completed after result persistence"
);
let mut rx = receiver
.lock()
.unwrap()
.take()
.expect("router must subscribe while leader is in-flight");
assert_eq!(
rx.try_recv().expect("leader guard must broadcast on drop"),
CoalesceSignal::Completed { success: true },
"persisted result must broadcast success so waiters adopt it"
);
}
#[tokio::test]
async fn test_process_scrape_task_persist_failure_broadcasts_default_failure() {
let coalescer = Arc::new(RequestCoalescer::new());
let task = make_task(json!({"url": "https://example.com"}));
let key = CoalesceKey::new(
task.team_id,
"https://example.com",
&ScrapeOptions::default(),
);
let (router, receiver) = CoalesceSubscribingRouter::new(coalescer.clone(), key);
let engine_client = Arc::new(EngineClient::with_router(Arc::new(router)));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_with_shared_coalescer(
task_repo.clone(),
Arc::new(FailingScrapeResultRepo) as Arc<dyn ScrapeResultRepository>,
coalescer,
engine_client,
)
.await;
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok(), "persistence failure is handled internally");
assert_eq!(
task_repo.mark_completed_count(),
0,
"failed persistence must not mark the task completed"
);
let mut rx = receiver
.lock()
.unwrap()
.take()
.expect("router must subscribe while leader is in-flight");
assert_eq!(
rx.try_recv().expect("leader guard must broadcast on drop"),
CoalesceSignal::Completed { success: false },
"failed persistence must keep the default failure signal"
);
}
#[tokio::test]
async fn test_run_with_empty_queue_loops_and_sleeps() {
let worker = build_mock_worker().await;
let queue = Arc::new(MockTaskQueue) as Arc<dyn TaskQueue>;
let result =
tokio::time::timeout(Duration::from_millis(150), worker.run(Arc::clone(&queue))).await;
assert!(
result.is_err(),
"run() should loop indefinitely; timeout expected"
);
}
#[tokio::test]
async fn test_run_processes_task_then_continues_looping() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({"url": "https://example.com"}));
let queue = Arc::new(TaskQueueWithTask::new(task)) as Arc<dyn TaskQueue>;
let result = tokio::time::timeout(Duration::from_millis(500), worker.run(queue)).await;
assert!(
result.is_err(),
"run() should loop indefinitely after processing"
);
assert!(
task_repo.mark_completed_count() >= 1,
"mark_completed should be called after processing scrape task in run()"
);
}
#[tokio::test]
async fn test_extract_and_queue_links_find_existing_urls_failure_returns_err() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
task_repo
.fail_find_existing_urls
.store(true, Ordering::SeqCst);
let worker = build_configurable_worker(
task_repo,
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let dedup_arc = worker.deduplicator_for_test();
{
let mut dedup = dedup_arc.write();
dedup.insert("https://example.com/page1");
dedup.insert("https://example.com/page2");
}
let mut task = make_task(json!({}));
task.url = "https://example.com".to_string();
let html = r#"<html><body>
<a href="https://example.com/page1">Page 1</a>
<a href="https://example.com/page2">Page 2</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(
result.is_err(),
"extract_and_queue_links should return Err when find_existing_urls fails"
);
}
#[tokio::test]
async fn test_handle_scrape_success_extraction_failure_continues_and_marks_completed() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(FailingExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"url": "https://example.com",
"extraction_rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(
result.is_ok(),
"handle_scrape_success should not fail when extraction fails"
);
assert_eq!(
task_repo.mark_completed_count(),
1,
"mark_completed should be called even when extraction fails"
);
}
#[tokio::test]
async fn test_process_scrape_task_success_without_screenshot_proxy_no_extra_credits() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let credits_repo = Arc::new(MockCreditsRepo::default());
let deducted_log = credits_repo.deducted.clone();
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
credits_repo as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({"url": "https://example.com"}));
let result = worker.process_scrape_task(task).await;
assert!(result.is_ok());
let deductions = deducted_log.lock().unwrap();
assert!(
deductions.is_empty(),
"no deduct_credits calls expected without screenshot/proxy/token-usage, got {:?}",
deductions
);
}
#[tokio::test]
async fn test_run_with_failing_queue_logs_error_and_sleeps() {
let worker = build_mock_worker().await;
let queue = Arc::new(FailingTaskQueue) as Arc<dyn TaskQueue>;
let result =
tokio::time::timeout(Duration::from_millis(150), worker.run(Arc::clone(&queue))).await;
assert!(
result.is_err(),
"run() should loop indefinitely on queue errors; timeout expected"
);
}
#[tokio::test]
async fn test_process_crawl_task_success_calls_handle_crawl_success() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
engine_client,
)
.await;
let crawl_id = crate::domain::id::Id::generate();
crawl_repo.set_crawl(Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
CrawlStatus::Processing,
json!({"domain_blacklist": []}),
1,
0,
0,
Utc::now(),
Utc::now(),
None,
));
let task = make_task(json!({
"crawl_id": crawl_id.to_string(),
"depth": 0,
"config": {"max_depth": 0}
}));
let result = worker.process_crawl_task(task).await;
assert!(result.is_ok(), "success path should return Ok(())");
assert_eq!(
task_repo.mark_completed_count(),
1,
"mark_completed should be called once for successful crawl"
);
}
#[tokio::test]
async fn test_extract_data_with_rules_failure_returns_none_and_logs() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let worker = build_worker_for_success_tests(
Arc::new(ConfigurableTaskRepo::new()),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(FailingExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let mut rules = HashMap::new();
rules.insert(
"title".to_string(),
ExtractionRule {
selector: Some("h1".to_string()),
attr: None,
is_array: false,
use_llm: None,
llm_prompt: None,
output_format: None,
},
);
let config = CrawlConfigDto {
max_depth: 1,
include_patterns: None,
exclude_patterns: None,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: Some(rules),
extraction_prompt: None,
extraction_schema: None,
};
let result = worker
.extract_data_with_rules(&task, &response, &config)
.await;
assert!(
result.is_none(),
"extract_data_with_rules should return None when extraction fails"
);
}
#[tokio::test]
async fn test_handle_crawl_failure_increment_failed_error_logs_and_continues() {
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
crawl_repo
.fail_increment_failed
.store(true, Ordering::SeqCst);
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
crawl_repo,
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.attempt_count = 100;
task.max_retries = 1;
let config = make_crawl_config(None, None);
let request = worker.build_crawl_request(&task, &config);
let result = worker
.handle_crawl_failure(
&mut task,
anyhow::anyhow!("Network error"),
crate::domain::id::Id::generate(),
&request,
)
.await;
assert!(
result.is_ok(),
"handle_crawl_failure should not fail when increment_failed_tasks errors"
);
}
#[tokio::test]
async fn test_handle_failure_guard_loss_returns_stale() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
task_repo.reject_guarded_update();
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.attempt_count = 0;
task.max_retries = 3;
let outcome = worker
.handle_failure(&mut task, "guard lost on retry")
.await
.expect("guard loss must not be an Err");
assert!(matches!(
outcome,
crate::domain::services::retry_handler::HandleFailureResult::Stale
));
let mut terminal = make_task(json!({}));
terminal.attempt_count = 100;
terminal.max_retries = 1;
let outcome = worker
.handle_failure(&mut terminal, "guard lost on terminal")
.await
.expect("guard loss must not be an Err");
assert!(matches!(
outcome,
crate::domain::services::retry_handler::HandleFailureResult::Stale
));
}
#[tokio::test]
async fn test_update_crawl_completion_status_update_status_error_logs_and_continues() {
let crawl_repo = Arc::new(ConfigurableCrawlRepo::new());
crawl_repo.fail_update_status.store(true, Ordering::SeqCst);
let crawl_id = crate::domain::id::Id::generate();
let crawl = Crawl::with_all_fields(
crawl_id,
crate::domain::id::Id::generate(),
"test".to_string(),
"https://example.com".to_string(),
"https://example.com".to_string(),
CrawlStatus::Processing,
json!({}),
10,
8,
2,
Utc::now(),
Utc::now(),
None,
);
crawl_repo.set_crawl(crawl);
let worker = build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
crawl_repo.clone(),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
worker.update_crawl_completion_status(crawl_id).await;
assert_eq!(
crawl_repo.update_status_count(),
1,
"update_status should be called once even when it errors"
);
}
#[tokio::test]
async fn test_process_extract_task_with_rules_calls_handle_rules_extraction() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"urls": ["https://example.com/page"],
"rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let result = worker.process_extract_task(task).await;
assert!(
result.is_ok(),
"process_extract_task with rules should return Ok(())"
);
assert!(
task_repo.update_count() >= 1,
"update should be called to mark task as Completed"
);
}
#[tokio::test]
async fn test_process_extract_task_with_prompt_calls_handle_prompt_extraction() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"urls": ["https://example.com/page"],
"prompt": "Extract the title"
}));
let result = worker.process_extract_task(task).await;
assert!(
result.is_ok(),
"process_extract_task with prompt should return Ok(())"
);
assert!(
task_repo.update_count() >= 1,
"update should be called to mark task as Completed"
);
}
#[tokio::test]
async fn test_process_extract_task_with_schema_calls_handle_schema_extraction() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"urls": ["https://example.com/page"],
"schema": {"type": "object", "properties": {"title": {"type": "string"}}}
}));
let result = worker.process_extract_task(task).await;
assert!(
result.is_ok(),
"process_extract_task with schema should return Ok(())"
);
assert!(
task_repo.update_count() >= 1,
"update should be called to mark task as Completed"
);
}
#[tokio::test]
async fn test_process_extract_task_fallback_saves_raw_result() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(MockCreditsRepo::default()) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionService) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"urls": ["https://example.com/page"]
}));
let result = worker.process_extract_task(task).await;
assert!(
result.is_ok(),
"process_extract_task fallback should return Ok(())"
);
assert!(
task_repo.update_count() >= 1,
"update should be called to mark task as Completed"
);
}
#[tokio::test]
async fn test_extract_and_queue_links_skips_existing_urls() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let mut existing = HashSet::new();
existing.insert("https://example.com/page1".to_string());
task_repo.set_existing_urls(existing);
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let dedup_arc = worker.deduplicator_for_test();
{
let mut dedup = dedup_arc.write();
dedup.insert("https://example.com/page1");
}
let task = make_task(json!({}));
let html = r#"<html><body>
<a href="/page1">Page 1</a>
<a href="/page2">Page 2</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
assert_eq!(
task_repo.create_count(),
1,
"only non-existing URLs should be created (page1 skipped, page2 created)"
);
}
#[tokio::test]
async fn test_extract_and_queue_links_bloom_miss_directly_enqueues() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo.clone(),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let task = make_task(json!({}));
let html = r#"<html><body>
<a href="/page1">Page 1</a>
<a href="/page2">Page 2</a>
<a href="/page3">Page 3</a>
</body></html>"#;
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: html.to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 100,
final_url: None,
markdown: None,
};
let config = make_crawl_config(None, None);
let result = worker
.extract_and_queue_links(
&task,
&response,
crate::domain::id::Id::generate(),
0,
&config,
)
.await;
assert!(result.is_ok());
assert_eq!(
task_repo.create_count(),
3,
"all URLs should be created when Bloom is empty (no DB lookup)"
);
}
#[tokio::test]
async fn test_handle_scrape_success_token_deduct_failure_logs_error() {
let router: Arc<dyn EngineRouterTrait> = Arc::new(SuccessEngineRouter::new());
let engine_client = Arc::new(EngineClient::with_router(router));
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_worker_for_success_tests(
task_repo.clone(),
engine_client,
Arc::new(FailingCreditsRepo) as Arc<dyn CreditsRepository>,
Arc::new(MockExtractionServiceWithTokens) as Arc<dyn ExtractionServiceTrait>,
)
.await;
let task = make_task(json!({
"url": "https://example.com",
"extraction_rules": {
"title": {
"selector": "h1",
"attr": null,
"is_array": false,
"use_llm": null,
"llm_prompt": null,
"output_format": null
}
}
}));
let response = ScrapeResponse {
js_degraded: None,
captured_responses: Vec::new(),
content: "<html><body><h1>Title</h1></body></html>".to_string(),
status_code: 200,
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 10,
final_url: None,
markdown: None,
};
let dto = parse_dto_for_test(&task);
let result = worker
.handle_scrape_success(&task, dto.as_ref(), response)
.await;
assert!(
result.is_ok(),
"handle_scrape_success should not fail when credit deduction fails"
);
assert_eq!(
task_repo.mark_completed_count(),
1,
"mark_completed should still be called"
);
}
#[tokio::test]
async fn test_handle_failure_failed_branch_returns_ok() {
let task_repo = Arc::new(ConfigurableTaskRepo::new());
let worker = build_configurable_worker(
task_repo,
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await;
let mut task = make_task(json!({}));
task.attempt_count = 100;
task.max_retries = 1;
let result = worker.handle_failure(&mut task, "terminal failure").await;
assert!(
result.is_ok(),
"handle_failure should return Ok(()) when task exceeds max_retries"
);
assert_eq!(
task.payload.get("error").and_then(|v| v.as_str()),
Some("terminal failure"),
"terminal path must carry error string into payload"
);
}
struct CountingResultRepo {
existing: Option<ScrapeResult>,
saves: std::sync::atomic::AtomicUsize,
saved_note: std::sync::Mutex<Option<String>>,
}
impl CountingResultRepo {
fn with_existing(existing: Option<ScrapeResult>) -> Self {
Self {
existing,
saves: std::sync::atomic::AtomicUsize::new(0),
saved_note: std::sync::Mutex::new(None),
}
}
fn save_count(&self) -> usize {
self.saves.load(std::sync::atomic::Ordering::SeqCst)
}
}
#[async_trait::async_trait]
impl ScrapeResultRepository for CountingResultRepo {
async fn save(&self, result: ScrapeResult) -> Result<()> {
self.saves.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let note = result
.meta_data
.get("note")
.and_then(|v| v.as_str())
.map(str::to_string);
*self.saved_note.lock().expect("saved_note lock") = note;
assert_eq!(result.status_code, 0, "minimal row uses status_code=0");
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<ScrapeResult>> {
Ok(self.existing.clone())
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(&self, _team_id: crate::domain::id::Id) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
fn make_scrape_result_for_task(task: &Task) -> ScrapeResult {
ScrapeResult {
id: crate::domain::id::Id::generate(),
task_id: task.id,
url: task.url.clone(),
status_code: 200,
content: "<html></html>".to_string(),
content_type: "text/html".to_string(),
headers: serde_json::json!({}),
meta_data: serde_json::json!({}),
screenshot: None,
response_time_ms: 10,
created_at: chrono::Utc::now(),
}
}
#[tokio::test]
async fn test_ensure_result_row_writes_minimal_row_when_missing() {
let task = make_task(json!({}));
let repo = Arc::new(CountingResultRepo::with_existing(None));
let worker = build_worker_with_failing_deps(
repo.clone() as Arc<dyn ScrapeResultRepository>,
Arc::new(MockWebhookService),
Arc::new(MockCreditsRepo::default()),
)
.await;
worker
.ensure_result_row(&task)
.await
.expect("ensure should succeed when row missing");
assert_eq!(repo.save_count(), 1, "missing row must be backfilled once");
assert_eq!(
repo.saved_note.lock().expect("lock").as_deref(),
Some("no content captured")
);
}
#[tokio::test]
async fn test_ensure_result_row_skips_write_when_present() {
let task = make_task(json!({}));
let existing = make_scrape_result_for_task(&task);
let repo = Arc::new(CountingResultRepo::with_existing(Some(existing)));
let worker = build_worker_with_failing_deps(
repo.clone() as Arc<dyn ScrapeResultRepository>,
Arc::new(MockWebhookService),
Arc::new(MockCreditsRepo::default()),
)
.await;
worker
.ensure_result_row(&task)
.await
.expect("ensure should succeed when row present");
assert_eq!(
repo.save_count(),
0,
"present row must not trigger extra write"
);
}
async fn build_should_crawl_worker() -> ScrapeWorker {
build_configurable_worker(
Arc::new(ConfigurableTaskRepo::new()),
Arc::new(ConfigurableCrawlRepo::new()),
Arc::new(MockRobotsChecker),
Arc::new(EngineClient::new()),
)
.await
}
fn build_crawl_config(
include_patterns: Option<Vec<String>>,
exclude_patterns: Option<Vec<String>>,
) -> CrawlConfigDto {
CrawlConfigDto {
max_depth: 3,
include_patterns,
exclude_patterns,
strategy: None,
crawl_delay_ms: None,
max_concurrency: None,
max_pages: None,
proxy: None,
headers: None,
extraction_rules: None,
extraction_prompt: None,
extraction_schema: None,
}
}
#[tokio::test]
async fn test_should_crawl_no_patterns_allows_all() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(None, None);
assert!(worker.should_crawl("https://any-url.com/page", &config));
assert!(worker.should_crawl("https://another.org/deep/path", &config));
}
#[tokio::test]
async fn test_should_crawl_include_regex_matches() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(Some(vec![r".*/blog/.*".to_string()]), None);
assert!(worker.should_crawl("https://example.com/blog/post-1", &config));
assert!(!worker.should_crawl("https://example.com/about", &config));
}
#[tokio::test]
async fn test_should_crawl_include_regex_no_match_returns_false() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(Some(vec![r"^https://specific\.com/.*".to_string()]), None);
assert!(!worker.should_crawl("https://other.com/page", &config));
}
#[tokio::test]
async fn test_should_crawl_exclude_regex_blocks() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(None, Some(vec![r".*/admin/.*".to_string()]));
assert!(!worker.should_crawl("https://example.com/admin/settings", &config));
assert!(worker.should_crawl("https://example.com/public/page", &config));
}
#[tokio::test]
async fn test_should_crawl_invalid_regex_falls_back_to_contains() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(Some(vec!["[unclosed".to_string()]), None);
assert!(worker.should_crawl("https://example.com/[unclosed/page", &config));
assert!(!worker.should_crawl("https://example.com/other", &config));
}
#[tokio::test]
async fn test_should_crawl_include_and_exclude_overlap_exclude_wins() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(
Some(vec![r".*/blog/.*".to_string()]),
Some(vec![r".*/blog/private/.*".to_string()]),
);
assert!(!worker.should_crawl("https://example.com/blog/private/draft", &config));
assert!(worker.should_crawl("https://example.com/blog/public-post", &config));
}
#[tokio::test]
async fn test_should_crawl_multiple_include_patterns_any_match() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(
Some(vec![r".*/docs/.*".to_string(), r".*/api/.*".to_string()]),
None,
);
assert!(worker.should_crawl("https://example.com/docs/intro", &config));
assert!(worker.should_crawl("https://example.com/api/v1/users", &config));
assert!(!worker.should_crawl("https://example.com/home", &config));
}
#[tokio::test]
async fn test_should_crawl_multiple_exclude_patterns_any_blocks() {
let worker = build_should_crawl_worker().await;
let config = build_crawl_config(
None,
Some(vec![
r".*/admin/.*".to_string(),
r".*/internal/.*".to_string(),
]),
);
assert!(!worker.should_crawl("https://example.com/admin/users", &config));
assert!(!worker.should_crawl("https://example.com/internal/debug", &config));
assert!(worker.should_crawl("https://example.com/public/page", &config));
}
fn cache_test_team() -> crate::domain::id::Id {
crate::domain::id::Id::from_u128(0x7e57_0000_0000_0000_0000_0000_0000_0001)
}
#[test]
fn test_generate_scrape_cache_key_format() {
let ctx = CacheContext {
team_id: cache_test_team(),
url: "https://example.com".to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
};
let key = generate_scrape_cache_key(&ctx, &ScrapeOptions::default());
assert!(key.starts_with(&format!("scrape:{}:", cache_test_team())));
assert!(key.contains("https://example.com"));
}
#[test]
fn test_generate_scrape_cache_key_different_methods_produce_different_keys() {
let url = "https://example.com/api";
let get_key = generate_scrape_cache_key(
&CacheContext {
team_id: cache_test_team(),
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
},
&ScrapeOptions::default(),
);
let post_key = generate_scrape_cache_key(
&CacheContext {
team_id: cache_test_team(),
url: url.to_string(),
method: HttpMethod::Post,
mode: CacheMode::Enabled,
},
&ScrapeOptions::default(),
);
assert_ne!(
get_key, post_key,
"GET and POST must have different cache keys"
);
}
#[test]
fn test_generate_scrape_cache_key_different_urls_produce_different_keys() {
let key1 = generate_scrape_cache_key(
&CacheContext {
team_id: cache_test_team(),
url: "https://a.com".to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
},
&ScrapeOptions::default(),
);
let key2 = generate_scrape_cache_key(
&CacheContext {
team_id: cache_test_team(),
url: "https://b.com".to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
},
&ScrapeOptions::default(),
);
assert_ne!(key1, key2);
}
#[tokio::test]
async fn test_try_read_scrape_cache_miss_returns_none() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let ctx = CacheContext {
team_id: cache_test_team(),
url: "https://example.com".to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
};
let key = generate_scrape_cache_key(&ctx, &ScrapeOptions::default());
let result = worker.try_read_scrape_cache(&ctx, &key).await;
assert!(result.is_ok(), "read should succeed even on miss");
assert!(result.unwrap().is_none(), "empty cache should return None");
assert_eq!(cache.get_count(), 1, "get should be called once");
}
#[tokio::test]
async fn test_try_read_scrape_cache_hit_returns_response() {
let url = "https://example.com";
let ctx = CacheContext {
team_id: cache_test_team(),
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::ReadOnly,
};
let key = generate_scrape_cache_key(&ctx, &ScrapeOptions::default());
let cached_response = ScrapeResponse::new(200, "cached content", "text/html");
let cached_json = serde_json::to_string(&cached_response).expect("serialize failed");
let cache = Arc::new(MockCacheService::with_entry(&key, &cached_json));
let worker = build_mock_worker_with_cache(cache.clone()).await;
let result = worker.try_read_scrape_cache(&ctx, &key).await;
assert!(result.is_ok(), "read should succeed on hit");
let resp = result.unwrap().expect("should have cached response");
assert_eq!(resp.status_code, 200);
assert_eq!(resp.content, "cached content");
assert_eq!(cache.get_count(), 1, "get should be called once");
}
#[tokio::test]
async fn test_try_read_scrape_cache_corrupt_data_returns_err() {
let url = "https://example.com";
let ctx = CacheContext {
team_id: cache_test_team(),
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
};
let key = generate_scrape_cache_key(&ctx, &ScrapeOptions::default());
let cache = Arc::new(MockCacheService::with_entry(&key, "{invalid json}"));
let worker = build_mock_worker_with_cache(cache.clone()).await;
let result = worker.try_read_scrape_cache(&ctx, &key).await;
assert!(
result.is_err(),
"corrupt cache should return Err so caller can evict the entry"
);
assert_eq!(cache.get_count(), 1, "get should still be called once");
}
#[tokio::test]
async fn test_try_write_scrape_cache_writes_serialized_response() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let ctx = CacheContext {
team_id: cache_test_team(),
url: "https://example.com/page".to_string(),
method: HttpMethod::Get,
mode: CacheMode::Enabled,
};
let key = generate_scrape_cache_key(&ctx, &ScrapeOptions::default());
let response = ScrapeResponse::new(200, "fresh content", "text/html");
let result = worker.try_write_scrape_cache(&ctx, &key, &response).await;
assert!(result.is_ok(), "write should succeed");
assert_eq!(cache.set_count(), 1, "set should be called once");
}
#[tokio::test]
async fn test_try_write_scrape_cache_key_matches_read_key() {
let url = "https://example.com/sync";
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let ctx = CacheContext {
team_id: cache_test_team(),
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::Bypass,
};
let key = generate_scrape_cache_key(&ctx, &ScrapeOptions::default());
let response = ScrapeResponse::new(200, "written", "text/html");
worker
.try_write_scrape_cache(&ctx, &key, &response)
.await
.unwrap();
assert_eq!(cache.set_count(), 1);
let result = worker.try_read_scrape_cache(&ctx, &key).await;
assert!(result.is_ok());
let resp = result.unwrap().expect("should hit after write");
assert_eq!(resp.content, "written");
}
#[tokio::test]
async fn test_cache_team_isolation_prevents_cross_team_hit() {
let url = "https://example.com/private";
let options = ScrapeOptions::default();
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let ctx_a = CacheContext {
team_id: crate::domain::id::Id::from_u128(1),
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::Bypass,
};
let key_a = generate_scrape_cache_key(&ctx_a, &options);
let response = ScrapeResponse::new(200, "tenant-a content", "text/html");
worker
.try_write_scrape_cache(&ctx_a, &key_a, &response)
.await
.unwrap();
let ctx_b = CacheContext {
team_id: crate::domain::id::Id::from_u128(2),
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::ReadOnly,
};
let key_b = generate_scrape_cache_key(&ctx_b, &options);
assert_ne!(key_a, key_b, "team dimension must change the cache key");
let result = worker.try_read_scrape_cache(&ctx_b, &key_b).await;
assert!(result.is_ok(), "read should succeed on miss");
assert!(
result.unwrap().is_none(),
"tenant B must not read tenant A's cached response"
);
}
#[tokio::test]
async fn test_cache_same_team_needs_js_difference_prevents_hit() {
let url = "https://example.com/spa";
let team = cache_test_team();
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let ctx = CacheContext {
team_id: team,
url: url.to_string(),
method: HttpMethod::Get,
mode: CacheMode::Bypass,
};
let rendered = ScrapeOptions {
needs_js: true,
..Default::default()
};
let key_rendered = generate_scrape_cache_key(&ctx, &rendered);
worker
.try_write_scrape_cache(
&ctx,
&key_rendered,
&ScrapeResponse::new(200, "rendered", "text/html"),
)
.await
.unwrap();
let plain = ScrapeOptions::default();
let key_plain = generate_scrape_cache_key(&ctx, &plain);
assert_ne!(
key_rendered, key_plain,
"needs_js must change the cache key within one tenant"
);
let result = worker.try_read_scrape_cache(&ctx, &key_plain).await;
assert!(result.is_ok(), "read should succeed on miss");
assert!(
result.unwrap().is_none(),
"needs_js=false must not hit the needs_js=true cached response"
);
}
fn make_task_with_cache_mode(url: &str, mode: Option<CacheMode>) -> Task {
let options = match mode {
Some(m) => serde_json::json!({ "cache_mode": m }),
None => serde_json::json!({}),
};
let payload = serde_json::json!({
"url": url,
"options": options,
});
make_task(payload)
}
#[tokio::test]
async fn test_process_scrape_task_disabled_mode_no_cache_read_or_write() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let task = make_task_with_cache_mode("https://example.com", Some(CacheMode::Disabled));
let _ = worker.process_scrape_task(task).await;
assert_eq!(cache.get_count(), 0, "Disabled mode should not read cache");
assert_eq!(cache.set_count(), 0, "Disabled mode should not write cache");
}
#[tokio::test]
async fn test_process_scrape_task_bypass_mode_no_read_but_attempt_write() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let task = make_task_with_cache_mode("https://example.com", Some(CacheMode::Bypass));
let _ = worker.process_scrape_task(task).await;
assert_eq!(cache.get_count(), 0, "Bypass mode should not read cache");
assert_eq!(
cache.set_count(),
0,
"Bypass mode should not write on scrape failure"
);
}
#[tokio::test]
async fn test_process_scrape_task_read_only_mode_reads_cache_no_write() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let task = make_task_with_cache_mode("https://example.com", Some(CacheMode::ReadOnly));
let _ = worker.process_scrape_task(task).await;
assert_eq!(cache.get_count(), 1, "ReadOnly mode should read cache");
assert_eq!(cache.set_count(), 0, "ReadOnly mode should not write cache");
}
#[tokio::test]
async fn test_process_scrape_task_enabled_mode_reads_cache_no_write_on_failure() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let task = make_task_with_cache_mode("https://example.com", Some(CacheMode::Enabled));
let _ = worker.process_scrape_task(task).await;
assert_eq!(cache.get_count(), 1, "Enabled mode should read cache");
assert_eq!(
cache.set_count(),
0,
"Enabled mode should not write on scrape failure"
);
}
#[tokio::test]
async fn test_process_scrape_task_default_mode_reads_cache() {
let cache = Arc::new(MockCacheService::new());
let worker = build_mock_worker_with_cache(cache.clone()).await;
let task = make_task_with_cache_mode("https://example.com", None);
let _ = worker.process_scrape_task(task).await;
assert_eq!(
cache.get_count(),
1,
"Default (Enabled) mode should read cache"
);
assert_eq!(cache.set_count(), 0, "Should not write on scrape failure");
}
#[test]
fn test_map_wait_for_covers_all_branches() {
use crate::workers::scrape_worker::map_wait_for;
assert_eq!(map_wait_for(None), None);
let zero = map_wait_for(Some(0));
assert_eq!(
zero,
Some(crate::engines::engine_client::WaitFor::DomStable(
Duration::from_millis(0)
))
);
let long = map_wait_for(Some(5000));
assert_eq!(
long,
Some(crate::engines::engine_client::WaitFor::DomStable(
Duration::from_millis(5000)
))
);
}