use crate::application::use_cases::create_scrape::CreateScrapeUseCaseTrait;
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::credits_repository::CreditsRepository;
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::domain::services::searxng::SearxngInstanceService;
use crate::domain::services::team_semaphore::TeamSemaphore;
use crate::domain::services::webhook_service::WebhookService;
use crate::engines::engine_client::EngineClient;
use crate::infrastructure::oxcache::CacheService;
use crate::queue::task_queue::TaskQueue;
use crate::workers::coalesce_coordinator::RequestCoalescer;
use crate::workers::coalesce_coordinator::CoalesceCoordinator;
use crate::workers::expiration_worker::ExpirationWorker;
use crate::workers::scrape_worker::{ScrapeWorker, ScrapeWorkerDeps};
use crate::workers::shutdown::ShutdownCoordinator;
use crate::workers::{AbstractWorker, Worker};
use log::{error, info};
use std::sync::Arc;
use tokio::task::JoinHandle;
use crate::config::settings::Settings;
use crate::utils::robots::RobotsCheckerTrait;
#[cfg(feature = "metrics")]
use crate::workers::scheduler::memory_scheduler::MemoryScheduler;
pub struct WorkerManager {
queue: Arc<dyn TaskQueue>,
repository: Arc<dyn TaskRepository>,
result_repository: Arc<dyn ScrapeResultRepository>,
crawl_repository: Arc<dyn CrawlRepository>,
webhook_service: Arc<dyn WebhookService>,
credits_repository: Arc<dyn CreditsRepository>,
engine_client: Arc<EngineClient>,
create_scrape_use_case: Arc<dyn CreateScrapeUseCaseTrait>,
team_semaphore: Arc<TeamSemaphore>,
request_coalescer: Arc<RequestCoalescer>,
coalesce_coordinator: Arc<CoalesceCoordinator>,
robots_checker: Arc<dyn RobotsCheckerTrait>,
settings: Arc<Settings>,
default_concurrency_limit: usize,
handles: Vec<JoinHandle<()>>,
searxng_instances: Option<Arc<dyn SearxngInstanceService>>,
extraction_service:
Arc<dyn crate::domain::services::extraction_service::ExtractionServiceTrait>,
#[cfg(feature = "content")]
content_extractor: Arc<crate::domain::services::content_extractor::ContentExtractionFacade>,
#[cfg(feature = "metrics")]
memory_scheduler: Arc<MemoryScheduler>,
deduplicator: Arc<parking_lot::RwLock<crate::utils::dedup::Deduplicator>>,
cache_service: Arc<dyn CacheService>,
shutdown_coordinator: Arc<ShutdownCoordinator>,
worker_ids: parking_lot::Mutex<Vec<crate::domain::id::Id>>,
}
pub struct WorkerManagerDeps {
pub queue: Arc<dyn TaskQueue>,
pub repository: Arc<dyn TaskRepository>,
pub result_repository: Arc<dyn ScrapeResultRepository>,
pub crawl_repository: Arc<dyn CrawlRepository>,
pub webhook_service: Arc<dyn WebhookService>,
pub credits_repository: Arc<dyn CreditsRepository>,
pub engine_client: Arc<EngineClient>,
pub create_scrape_use_case: Arc<dyn CreateScrapeUseCaseTrait>,
pub team_semaphore: Arc<TeamSemaphore>,
pub request_coalescer: Arc<RequestCoalescer>,
pub robots_checker: Arc<dyn RobotsCheckerTrait>,
pub http_client: Arc<reqwest::Client>,
pub extraction_service:
Arc<dyn crate::domain::services::extraction_service::ExtractionServiceTrait>,
#[cfg(feature = "content")]
pub content_extractor: Arc<crate::domain::services::content_extractor::ContentExtractionFacade>,
pub cache_service: Arc<dyn CacheService>,
pub shutdown_coordinator: Arc<ShutdownCoordinator>,
pub searxng_instances: Option<Arc<dyn SearxngInstanceService>>,
}
pub struct WorkerManagerConfig {
pub settings: Arc<Settings>,
pub default_concurrency_limit: usize,
}
impl WorkerManager {
pub fn new(deps: WorkerManagerDeps, config: WorkerManagerConfig) -> Self {
#[cfg(feature = "metrics")]
let memory_scheduler = {
use crate::infrastructure::observability::metrics::{
shared_system_monitor, SystemMonitorTrait,
};
Arc::new(MemoryScheduler::new(
shared_system_monitor() as Arc<dyn SystemMonitorTrait>,
config.settings.concurrency.mem_pressure_threshold,
config.settings.concurrency.mem_critical_threshold,
std::time::Duration::from_secs(
config.settings.concurrency.critical_timeout_seconds,
),
))
};
let coalesce_coordinator = Arc::new(CoalesceCoordinator::new(
deps.repository.clone(),
deps.result_repository.clone(),
deps.request_coalescer.clone(),
));
Self {
queue: deps.queue,
repository: deps.repository,
result_repository: deps.result_repository,
crawl_repository: deps.crawl_repository,
webhook_service: deps.webhook_service,
credits_repository: deps.credits_repository,
engine_client: deps.engine_client,
create_scrape_use_case: deps.create_scrape_use_case,
team_semaphore: deps.team_semaphore,
coalesce_coordinator,
request_coalescer: deps.request_coalescer,
robots_checker: deps.robots_checker,
settings: config.settings,
default_concurrency_limit: config.default_concurrency_limit,
handles: Vec::new(),
extraction_service: deps.extraction_service,
#[cfg(feature = "content")]
content_extractor: deps.content_extractor,
#[cfg(feature = "metrics")]
memory_scheduler,
deduplicator: Arc::new(parking_lot::RwLock::new(
crate::utils::dedup::Deduplicator::new(),
)),
cache_service: deps.cache_service,
searxng_instances: deps.searxng_instances,
shutdown_coordinator: deps.shutdown_coordinator,
worker_ids: parking_lot::Mutex::new(Vec::new()),
}
}
async fn warmup_deduplicator(&self) {
const PAGE_SIZE: u32 = 5000;
match self
.warmup_dedup_from(|after| self.repository.list_distinct_urls(PAGE_SIZE, after))
.await
{
Ok(total) if total > 0 => {
info!("Deduplicator warmed up with {total} distinct URLs from previous runs");
}
Ok(_) => {}
Err((after_url, e)) => {
error!(
"Deduplicator warmup failed at cursor {after_url:?}: {e}; \
continuing with EMPTY dedup state — already-crawled URLs \
will be re-crawled this run"
);
return;
}
}
if !self.settings.workers.dedup_warmup_include_results {
return;
}
match self
.warmup_dedup_from(|after| self.result_repository.list_distinct_urls(PAGE_SIZE, after))
.await
{
Ok(total) if total > 0 => {
info!(
"Deduplicator warmup: +{total} distinct URLs from scrape_results \
(redirect targets included)"
);
}
Ok(_) => {}
Err((after_url, e)) => {
error!(
"Deduplicator warmup failed on scrape_results phase at cursor \
{after_url:?}: {e}; keeping URLs already warmed from tasks — \
redirect final_urls may be re-crawled this run"
);
}
}
}
async fn warmup_dedup_from<F, Fut, E>(&self, mut fetch: F) -> Result<usize, (Option<String>, E)>
where
F: FnMut(Option<String>) -> Fut,
Fut: std::future::Future<Output = Result<Vec<String>, E>>,
{
const PAGE_SIZE: u32 = 5000;
let mut after_url: Option<String> = None;
let mut total: usize = 0;
loop {
let urls = match fetch(after_url.clone()).await {
Ok(urls) => urls,
Err(e) => return Err((after_url, e)),
};
let page_len = urls.len();
if page_len > 0 {
let mut dedup = self.deduplicator.write();
dedup.extend(urls.iter().map(String::as_str));
total += page_len;
after_url = urls.into_iter().last();
}
if page_len < PAGE_SIZE as usize {
break;
}
}
Ok(total)
}
pub async fn start_workers(&mut self, count: usize) {
self.warmup_deduplicator().await;
self.spawn_expiration_worker();
self.spawn_searxng_workers();
self.spawn_coalescer_purge();
for _ in 0..count {
self.spawn_scrape_worker();
}
}
fn spawn_expiration_worker(&mut self) {
let expiration_processor = Arc::new(ExpirationWorker::new(
self.repository.clone(),
self.result_repository.clone(),
self.settings.retention.scrape_results_days,
));
let expiration_worker =
AbstractWorker::new(expiration_processor, std::time::Duration::from_secs(3600));
self.handles.push(tokio::spawn(async move {
expiration_worker.run().await;
}));
}
fn spawn_searxng_workers(&mut self) {
if let Some(searxng) = self.searxng_instances.clone() {
let collector = AbstractWorker::new(
Arc::new(
crate::workers::searxng_collector_worker::SearxngCollectorWorker::new(
searxng.clone(),
),
),
std::time::Duration::from_secs(
self.settings.search.searxng.collector.interval_seconds,
),
);
self.handles.push(tokio::spawn(async move {
collector.run().await;
}));
let health = AbstractWorker::new(
Arc::new(crate::workers::searxng_health_worker::SearxngHealthWorker::new(searxng)),
std::time::Duration::from_secs(
self.settings.search.searxng.health.interval_seconds,
),
);
self.handles.push(tokio::spawn(async move {
health.run().await;
}));
}
}
fn spawn_coalescer_purge(&mut self) {
let coalescer_for_purge = self.request_coalescer.clone();
let shutdown_for_purge = self.shutdown_coordinator.clone();
self.handles.push(tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
interval.tick().await; loop {
interval.tick().await;
if shutdown_for_purge.is_shutting_down() {
info!("purge_stale loop exiting due to shutdown");
break;
}
let purged = coalescer_for_purge.purge_stale();
if purged > 0 {
info!("purge_stale cleaned up {} zombie coalesce entries", purged);
}
}
}));
}
fn spawn_scrape_worker(&mut self) {
let worker = ScrapeWorker::new(ScrapeWorkerDeps {
repository: self.repository.clone(),
result_repository: self.result_repository.clone(),
crawl_repository: self.crawl_repository.clone(),
webhook_service: self.webhook_service.clone(),
credits_repository: self.credits_repository.clone(),
engine_client: self.engine_client.clone(),
create_scrape_use_case: self.create_scrape_use_case.clone(),
team_semaphore: self.team_semaphore.clone(),
coalesce_coordinator: self.coalesce_coordinator.clone(),
robots_checker: self.robots_checker.clone(),
settings: self.settings.clone(),
default_concurrency_limit: self.default_concurrency_limit,
extraction_service: self.extraction_service.clone(),
#[cfg(feature = "content")]
content_extractor: self.content_extractor.clone(),
cache_service: self.cache_service.clone(),
#[cfg(feature = "metrics")]
memory_scheduler: self.memory_scheduler.clone(),
})
.with_shutdown_coordinator(self.shutdown_coordinator.clone())
.with_deduplicator_opt(Some(self.deduplicator.clone()));
self.worker_ids.lock().push(worker.worker_id());
let queue = self.queue.clone();
let handle = tokio::spawn(async move {
worker.run(queue).await;
});
self.handles.push(handle);
}
pub fn worker_ids(&self) -> Vec<crate::domain::id::Id> {
self.worker_ids.lock().clone()
}
pub fn begin_shutdown(&self) {
self.shutdown_coordinator.trigger();
}
pub async fn wait_for_shutdown(&mut self) {
let _ = self.shutdown_coordinator.wait_for_completion().await;
info!("Shutting down workers...");
let handles = std::mem::take(&mut self.handles);
for handle in &handles {
handle.abort();
}
for handle in handles {
let _ = tokio::time::timeout(std::time::Duration::from_secs(5), async {
let _ = handle.await;
})
.await;
}
info!("Workers shut down successfully");
}
}
impl Drop for WorkerManager {
fn drop(&mut self) {
for handle in &self.handles {
handle.abort();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_worker_manager_config_construction() {
let settings = Arc::new(Settings::default());
let config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 10,
};
assert_eq!(config.default_concurrency_limit, 10);
assert_eq!(Arc::strong_count(&config.settings), 2);
}
#[test]
fn test_worker_manager_config_with_different_concurrency() {
let settings = Arc::new(Settings::default());
let config = WorkerManagerConfig {
settings,
default_concurrency_limit: 50,
};
assert_eq!(config.default_concurrency_limit, 50);
}
#[test]
fn test_worker_manager_config_concurrency_zero() {
let settings = Arc::new(Settings::default());
let config = WorkerManagerConfig {
settings,
default_concurrency_limit: 0,
};
assert_eq!(config.default_concurrency_limit, 0);
}
#[test]
fn test_worker_manager_config_settings_shared() {
let settings = Arc::new(Settings::default());
let config1 = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 5,
};
let config2 = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 15,
};
assert!(Arc::ptr_eq(&config1.settings, &config2.settings));
}
#[test]
fn test_engine_client_can_be_constructed() {
let client = EngineClient::new();
let _cloned = client.clone();
}
#[test]
fn test_team_semaphore_can_be_constructed() {
let sem = TeamSemaphore::new(10);
let team_id = crate::domain::id::Id::generate();
assert!(sem.try_acquire(team_id).is_some());
}
#[test]
fn test_team_semaphore_clone_shares_state() {
let sem = TeamSemaphore::new(1);
let cloned = sem.clone();
let team_id = crate::domain::id::Id::generate();
let _permit = sem
.try_acquire(team_id)
.expect("first acquire should succeed");
assert!(cloned.try_acquire(team_id).is_none());
}
#[test]
fn test_worker_manager_deps_struct_exists() {
fn _assert_deps_type(_deps: WorkerManagerDeps) {}
}
#[test]
fn test_worker_manager_config_struct_exists() {
fn _assert_config_type(_config: WorkerManagerConfig) {}
}
#[test]
fn test_worker_manager_struct_exists() {
fn _assert_manager_type(_manager: WorkerManager) {}
}
#[test]
fn test_worker_manager_config_large_concurrency() {
let settings = Arc::new(Settings::default());
let config = WorkerManagerConfig {
settings,
default_concurrency_limit: usize::MAX,
};
assert_eq!(config.default_concurrency_limit, usize::MAX);
}
#[test]
fn test_worker_manager_config_one_concurrency() {
let settings = Arc::new(Settings::default());
let config = WorkerManagerConfig {
settings,
default_concurrency_limit: 1,
};
assert_eq!(config.default_concurrency_limit, 1);
}
#[test]
fn test_worker_manager_config_settings_arc_count_increases() {
let settings = Arc::new(Settings::default());
let initial_count = Arc::strong_count(&settings);
let _config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 10,
};
let after_count = Arc::strong_count(&settings);
assert_eq!(after_count, initial_count + 1);
}
#[test]
fn test_worker_manager_config_settings_arc_count_decreces_on_drop() {
let settings = Arc::new(Settings::default());
{
let _config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 10,
};
assert!(Arc::strong_count(&settings) > 1);
}
assert_eq!(Arc::strong_count(&settings), 1);
}
#[test]
fn test_multiple_configs_sharing_same_settings() {
let settings = Arc::new(Settings::default());
let configs: Vec<WorkerManagerConfig> = (0..5)
.map(|i| WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: i * 10,
})
.collect();
for config in &configs {
assert!(Arc::ptr_eq(&config.settings, &settings));
}
assert_eq!(configs.len(), 5);
assert_eq!(configs[0].default_concurrency_limit, 0);
assert_eq!(configs[4].default_concurrency_limit, 40);
}
#[test]
fn test_engine_client_clone_preserves_identity() {
let client = EngineClient::new();
let cloned = client.clone();
let _another_clone = cloned.clone();
}
#[test]
fn test_engine_client_multiple_instances() {
let client1 = EngineClient::new();
let client2 = EngineClient::new();
let _both = (client1, client2);
}
#[test]
fn test_settings_default_is_constructible() {
let settings1 = Settings::default();
let settings2 = Settings::default();
let _ = (settings1, settings2);
}
#[test]
fn test_settings_can_be_cloned() {
let settings = Settings::default();
let _cloned = settings.clone();
}
#[tokio::test]
async fn test_team_semaphore_acquire_returns_permit() {
let sem = TeamSemaphore::new(3);
let team_id = crate::domain::id::Id::generate();
let permit = sem.acquire(team_id).await;
assert!(permit.is_ok());
}
#[test]
fn test_team_semaphore_try_acquire_respects_limit() {
let sem = TeamSemaphore::new(1);
let team_id = crate::domain::id::Id::generate();
let p1 = sem.try_acquire(team_id);
assert!(p1.is_some());
let p2 = sem.try_acquire(team_id);
assert!(p2.is_none());
}
#[test]
fn test_worker_manager_deps_has_expected_fields() {
let settings = Arc::new(Settings::default());
let config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 10,
};
let _limit = config.default_concurrency_limit;
let _settings_ref = &config.settings;
}
#[test]
fn test_worker_manager_drop_aborts_handles() {
assert!(std::mem::needs_drop::<WorkerManager>());
}
#[test]
fn test_worker_manager_config_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<WorkerManagerConfig>();
}
#[test]
fn test_worker_manager_config_arc_settings_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<Arc<Settings>>();
}
#[test]
fn test_concurrency_limit_boundary_values() {
let settings = Arc::new(Settings::default());
for &limit in &[0usize, 1, 10, 100, 1000] {
let config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: limit,
};
assert_eq!(config.default_concurrency_limit, limit);
}
}
#[test]
fn test_worker_manager_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<WorkerManager>();
}
#[test]
fn test_worker_manager_deps_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<WorkerManagerDeps>();
}
#[test]
fn test_worker_manager_config_size_is_reasonable() {
let size = std::mem::size_of::<WorkerManagerConfig>();
assert!(size > 0);
assert!(
size <= 32,
"WorkerManagerConfig size {} seems too large",
size
);
}
#[test]
fn test_worker_manager_has_handles_field() {
fn _assert_handles_type(_handles: Vec<JoinHandle<()>>) {}
}
#[test]
fn test_concurrency_limit_powers_of_two() {
let settings = Arc::new(Settings::default());
for &limit in &[1usize, 2, 4, 8, 16, 32, 64, 128, 256] {
let config = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: limit,
};
assert_eq!(config.default_concurrency_limit, limit);
}
}
#[test]
fn test_settings_arc_strong_count_with_many_configs() {
let settings = Arc::new(Settings::default());
let configs: Vec<_> = (0..10)
.map(|_| WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 5,
})
.collect();
assert_eq!(Arc::strong_count(&settings), 11);
assert_eq!(configs.len(), 10);
}
#[test]
fn test_settings_arc_count_decreases_after_config_drop() {
let settings = Arc::new(Settings::default());
let initial = Arc::strong_count(&settings);
{
let _config1 = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 1,
};
let _config2 = WorkerManagerConfig {
settings: settings.clone(),
default_concurrency_limit: 2,
};
assert_eq!(Arc::strong_count(&settings), initial + 2);
}
assert_eq!(Arc::strong_count(&settings), initial);
}
#[test]
fn test_worker_manager_needs_drop_is_true() {
assert!(std::mem::needs_drop::<WorkerManager>());
}
#[test]
fn test_worker_manager_drop_is_not_noop() {
assert!(std::mem::needs_drop::<WorkerManager>());
}
use crate::application::dto::scrape_request::ScrapeRequestDto;
use crate::domain::models::{
Crawl, CrawlStatus, CreditsTransaction, CreditsTransactionType, DomainError, ScrapeResult,
Task, WebhookEvent,
};
use crate::domain::repositories::credits_repository::CreditsRepositoryError;
use crate::domain::repositories::task_repository::{RepositoryError, TaskQueryParams};
use crate::domain::services::extraction_service::{ExtractionRule, ExtractionServiceTrait};
use crate::domain::services::llm::TokenUsage;
use crate::engines::engine_client::ScrapeResponse;
use crate::queue::task_queue::QueueError;
use async_trait::async_trait;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
struct MockTaskQueue;
#[async_trait]
impl TaskQueue for MockTaskQueue {
async fn enqueue(&self, task: Task) -> Result<Task, QueueError> {
Ok(task)
}
async fn dequeue(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, QueueError> {
Ok(None)
}
async fn complete(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn fail(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
async fn cancel(&self, _task_id: crate::domain::id::Id) -> Result<(), QueueError> {
Ok(())
}
}
#[derive(Default)]
struct MockTaskRepository {
warmup_urls: Vec<String>,
}
#[async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn list_distinct_urls(
&self,
_limit: u32,
_after_url: Option<String>,
) -> Result<Vec<String>, RepositoryError> {
Ok(self.warmup_urls.clone())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[derive(Default)]
struct MockScrapeResultRepository {
warmup_urls: Vec<String>,
fail_list_distinct: bool,
}
#[async_trait]
impl ScrapeResultRepository for MockScrapeResultRepository {
async fn save(&self, _result: ScrapeResult) -> anyhow::Result<()> {
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> anyhow::Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> anyhow::Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(
&self,
_team_id: crate::domain::id::Id,
) -> anyhow::Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
async fn list_distinct_urls(
&self,
_limit: u32,
_after_url: Option<String>,
) -> anyhow::Result<Vec<String>> {
if self.fail_list_distinct {
return Err(anyhow::anyhow!("mock results list_distinct_urls failure"));
}
Ok(self.warmup_urls.clone())
}
}
struct MockCrawlRepository;
#[async_trait]
impl CrawlRepository for MockCrawlRepository {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Ok(None)
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct MockWebhookService;
#[async_trait]
impl WebhookService for MockWebhookService {
async fn send_webhook(&self, _event: &WebhookEvent) -> anyhow::Result<u16> {
Ok(200)
}
async fn trigger_completion(&self, _task: &Task) -> anyhow::Result<()> {
Ok(())
}
async fn trigger_failure(&self, _task: &Task, _error_msg: String) -> anyhow::Result<()> {
Ok(())
}
}
struct MockCreditsRepository;
#[async_trait]
impl CreditsRepository for MockCreditsRepository {
async fn get_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, CreditsRepositoryError> {
Ok(0)
}
async fn deduct_credits(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), CreditsRepositoryError> {
Ok(())
}
async fn add_credits(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<i64, CreditsRepositoryError> {
Ok(0)
}
async fn get_transaction_history(
&self,
_team_id: crate::domain::id::Id,
_limit: Option<u32>,
) -> Result<Vec<CreditsTransaction>, CreditsRepositoryError> {
Ok(vec![])
}
async fn initialize_team_credits(
&self,
_team_id: crate::domain::id::Id,
_initial_balance: i64,
) -> Result<i64, CreditsRepositoryError> {
Ok(0)
}
}
struct MockCreateScrapeUseCase;
#[async_trait]
impl CreateScrapeUseCaseTrait for MockCreateScrapeUseCase {
async fn execute(
&self,
_request_dto: ScrapeRequestDto,
) -> Result<ScrapeResponse, DomainError> {
Err(DomainError::EngineError("mock".to_string()))
}
}
struct MockRobotsChecker;
#[async_trait]
impl RobotsCheckerTrait for MockRobotsChecker {
async fn is_allowed(&self, _url_str: &str, _user_agent: &str) -> anyhow::Result<bool> {
Ok(true)
}
async fn get_crawl_delay(
&self,
_url_str: &str,
_user_agent: &str,
) -> anyhow::Result<Option<std::time::Duration>> {
Ok(None)
}
}
struct MockExtractionService;
#[async_trait]
impl ExtractionServiceTrait for MockExtractionService {
async fn extract(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> anyhow::Result<(Value, TokenUsage)> {
Ok((Value::Null, TokenUsage::default()))
}
async fn extract_with_schema(
&self,
_html_content: &str,
_schema: &Value,
) -> anyhow::Result<(Value, TokenUsage)> {
Ok((Value::Null, TokenUsage::default()))
}
fn extract_with_selectors(
&self,
_html_content: &str,
_rules: &HashMap<String, ExtractionRule>,
_base_url: Option<&str>,
) -> anyhow::Result<Value> {
Ok(Value::Null)
}
async fn extract_with_rag(
&self,
_html_content: &str,
_query: &str,
_schema: &Value,
_rag_strategy: &crate::domain::services::rag_strategy::RagExtractionStrategy,
) -> anyhow::Result<(Value, TokenUsage)> {
Ok((Value::Null, TokenUsage::default()))
}
}
fn make_deps() -> WorkerManagerDeps {
WorkerManagerDeps {
queue: Arc::new(MockTaskQueue),
repository: Arc::new(MockTaskRepository::default()),
result_repository: Arc::new(MockScrapeResultRepository::default()),
crawl_repository: Arc::new(MockCrawlRepository),
webhook_service: Arc::new(MockWebhookService),
credits_repository: Arc::new(MockCreditsRepository),
engine_client: Arc::new(EngineClient::new()),
create_scrape_use_case: Arc::new(MockCreateScrapeUseCase),
team_semaphore: Arc::new(TeamSemaphore::new(10)),
request_coalescer: Arc::new(
crate::workers::coalesce_coordinator::RequestCoalescer::new(),
),
robots_checker: Arc::new(MockRobotsChecker),
http_client: Arc::new(reqwest::Client::new()),
extraction_service: Arc::new(MockExtractionService),
#[cfg(feature = "content")]
content_extractor: Arc::new(
crate::domain::services::content_extractor::ContentExtractionFacade::new(None),
),
cache_service: Arc::new(NoopCacheService) as Arc<dyn CacheService>,
shutdown_coordinator: Arc::new(ShutdownCoordinator::default()),
searxng_instances: None,
}
}
struct NoopCacheService;
#[async_trait::async_trait]
impl CacheService for NoopCacheService {
fn get(
&self,
_key: &str,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = anyhow::Result<Option<String>>> + Send + '_>,
> {
Box::pin(async { Ok(None) })
}
fn set(
&self,
_key: &str,
_value: &str,
_ttl_seconds: u64,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
Box::pin(async { Ok(()) })
}
fn delete(
&self,
_key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send + '_>>
{
Box::pin(async { Ok(()) })
}
fn exists(
&self,
_key: &str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<bool>> + Send + '_>>
{
Box::pin(async { Ok(false) })
}
}
fn make_config() -> WorkerManagerConfig {
WorkerManagerConfig {
settings: Arc::new(Settings::default()),
default_concurrency_limit: 10,
}
}
#[tokio::test]
async fn test_worker_manager_new_assigns_fields() {
let manager = WorkerManager::new(make_deps(), make_config());
assert_eq!(manager.default_concurrency_limit, 10);
assert!(
manager.handles.is_empty(),
"new() should start with no handles"
);
}
#[tokio::test]
async fn test_warmup_deduplicator_loads_previous_urls() {
let repo = Arc::new(MockTaskRepository {
warmup_urls: vec![
"https://example.com/a".to_string(),
"https://example.com/b".to_string(),
"https://example.com/c".to_string(),
],
});
let mut deps = make_deps();
deps.repository = repo;
let manager = WorkerManager::new(deps, make_config());
assert_eq!(manager.deduplicator.read().len(), 0);
manager.warmup_deduplicator().await;
let dedup = manager.deduplicator.read();
assert_eq!(dedup.len(), 3, "warmup must load all distinct URLs");
let result = dedup.check("https://example.com/a").unwrap();
assert!(
matches!(
result,
crate::utils::dedup::DedupResult::MaybeExisting { .. }
),
"warmed-up URL must be MaybeExisting, got {result:?}"
);
}
#[tokio::test]
async fn test_warmup_include_results_adds_result_urls() {
let repo = Arc::new(MockTaskRepository {
warmup_urls: vec![
"https://example.com/task-a".to_string(),
"https://example.com/shared".to_string(),
],
});
let result_repo = Arc::new(MockScrapeResultRepository {
warmup_urls: vec![
"https://example.com/shared".to_string(), "https://redirect.example.com/final-1".to_string(),
"https://redirect.example.com/final-2".to_string(),
],
fail_list_distinct: false,
});
let mut deps = make_deps();
deps.repository = repo;
deps.result_repository = result_repo;
let settings = Settings {
workers: crate::config::settings::WorkerSettings {
dedup_warmup_include_results: true,
..Default::default()
},
..Default::default()
};
let manager = WorkerManager::new(
deps,
WorkerManagerConfig {
settings: Arc::new(settings),
default_concurrency_limit: 10,
},
);
assert_eq!(manager.deduplicator.read().len(), 0);
manager.warmup_deduplicator().await;
let dedup = manager.deduplicator.read();
assert_eq!(
dedup.len(),
4,
"warmup must cover the tasks∪results union (shared URL counted once)"
);
let result = dedup.check("https://redirect.example.com/final-1").unwrap();
assert!(
matches!(
result,
crate::utils::dedup::DedupResult::MaybeExisting { .. }
),
"result-table URL must be warmed, got {result:?}"
);
}
#[tokio::test]
async fn test_warmup_default_excludes_results() {
let repo = Arc::new(MockTaskRepository {
warmup_urls: vec![
"https://example.com/task-a".to_string(),
"https://example.com/task-b".to_string(),
],
});
let result_repo = Arc::new(MockScrapeResultRepository {
warmup_urls: vec![
"https://redirect.example.com/final-1".to_string(),
"https://redirect.example.com/final-2".to_string(),
"https://redirect.example.com/final-3".to_string(),
],
fail_list_distinct: false,
});
let mut deps = make_deps();
deps.repository = repo;
deps.result_repository = result_repo;
let manager = WorkerManager::new(deps, make_config());
manager.warmup_deduplicator().await;
let dedup = manager.deduplicator.read();
assert_eq!(
dedup.len(),
2,
"default must warm tasks URLs only (results table excluded)"
);
let result = dedup.check("https://redirect.example.com/final-1").unwrap();
assert!(
matches!(
result,
crate::utils::dedup::DedupResult::DefinitelyNew { .. }
),
"result-table URL must NOT be warmed by default, got {result:?}"
);
}
#[tokio::test]
async fn test_warmup_results_failure_keeps_tasks_state() {
let repo = Arc::new(MockTaskRepository {
warmup_urls: vec![
"https://example.com/task-a".to_string(),
"https://example.com/task-b".to_string(),
],
});
let result_repo = Arc::new(MockScrapeResultRepository {
warmup_urls: vec![],
fail_list_distinct: true,
});
let mut deps = make_deps();
deps.repository = repo;
deps.result_repository = result_repo;
let settings = Settings {
workers: crate::config::settings::WorkerSettings {
dedup_warmup_include_results: true,
..Default::default()
},
..Default::default()
};
let manager = WorkerManager::new(
deps,
WorkerManagerConfig {
settings: Arc::new(settings),
default_concurrency_limit: 10,
},
);
manager.warmup_deduplicator().await;
let dedup = manager.deduplicator.read();
assert_eq!(
dedup.len(),
2,
"tasks-phase state must be retained when results phase fails"
);
}
#[tokio::test]
async fn test_start_workers_zero_count_starts_only_expiration_worker() {
let mut manager = WorkerManager::new(make_deps(), make_config());
manager.start_workers(0).await;
assert_eq!(
manager.handles.len(),
2,
"start_workers(0) should start expiration + purge_stale scheduler"
);
}
#[tokio::test]
async fn test_start_workers_multiple_count() {
let mut manager = WorkerManager::new(make_deps(), make_config());
manager.start_workers(3).await;
assert_eq!(
manager.handles.len(),
5,
"start_workers(3) should start expiration + purge_stale + 3 scrape workers"
);
}
#[tokio::test]
async fn test_start_workers_handles_are_aborted_on_drop() {
let mut manager = WorkerManager::new(make_deps(), make_config());
manager.start_workers(1).await;
assert_eq!(manager.handles.len(), 3);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
assert!(
manager.handles.iter().any(|h| !h.is_finished()),
"at least one worker should be running before drop"
);
drop(manager);
let mut manager2 = WorkerManager::new(make_deps(), make_config());
manager2.start_workers(1).await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let handles: Vec<JoinHandle<()>> = std::mem::take(&mut manager2.handles);
assert!(
handles.iter().any(|h| !h.is_finished()),
"workers should be running before abort"
);
for handle in &handles {
handle.abort();
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
for handle in &handles {
assert!(
handle.is_finished(),
"all handles should be finished after abort"
);
}
}
#[tokio::test]
async fn test_wait_for_shutdown_completes_on_trigger() {
use std::time::Duration;
let mut manager = WorkerManager::new(make_deps(), make_config());
manager.start_workers(1).await;
assert_eq!(manager.handles.len(), 3);
manager.begin_shutdown();
let result =
tokio::time::timeout(Duration::from_secs(5), manager.wait_for_shutdown()).await;
assert!(
result.is_ok(),
"wait_for_shutdown should complete after shutdown trigger"
);
tokio::time::sleep(Duration::from_millis(50)).await;
for handle in &manager.handles {
assert!(
handle.is_finished(),
"handles should be aborted after shutdown"
);
}
}
#[tokio::test]
async fn test_begin_shutdown_sets_coordinator_flag() {
let manager = WorkerManager::new(make_deps(), make_config());
assert!(!manager.shutdown_coordinator.is_shutting_down());
manager.begin_shutdown();
assert!(manager.shutdown_coordinator.is_shutting_down());
}
}