use super::ScrapeWorker;
use super::ScrapeWorkerDeps;
use crate::application::use_cases::create_scrape::CreateScrapeUseCaseTrait;
use crate::config::settings::Settings;
use crate::domain::repositories::crawl_repository::CrawlRepository;
use crate::domain::repositories::credits_repository::CreditsRepository;
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::domain::services::extraction_service::ExtractionServiceTrait;
use crate::domain::services::team_semaphore::TeamSemaphore;
use crate::domain::services::webhook_service::WebhookService;
use crate::engines::engine_client::EngineClient;
use crate::infrastructure::oxcache::CacheService;
use crate::utils::dedup::Deduplicator;
use crate::utils::robots::RobotsCheckerTrait;
use crate::workers::coalesce_coordinator::CoalesceCoordinator;
use crate::workers::coalesce_coordinator::RequestCoalescer;
#[cfg(feature = "metrics")]
use crate::workers::scheduler::memory_scheduler::MemoryScheduler;
use crate::workers::shutdown::ShutdownCoordinator;
use std::sync::Arc;
pub struct ScrapeWorkerBuilder {
repository: Option<Arc<dyn TaskRepository>>,
result_repository: Option<Arc<dyn ScrapeResultRepository>>,
crawl_repository: Option<Arc<dyn CrawlRepository>>,
webhook_service: Option<Arc<dyn WebhookService>>,
credits_repository: Option<Arc<dyn CreditsRepository>>,
engine_client: Option<Arc<EngineClient>>,
create_scrape_use_case: Option<Arc<dyn CreateScrapeUseCaseTrait>>,
team_semaphore: Option<Arc<TeamSemaphore>>,
robots_checker: Option<Arc<dyn RobotsCheckerTrait>>,
settings: Option<Arc<Settings>>,
default_concurrency_limit: usize,
extraction_service: Option<Arc<dyn ExtractionServiceTrait>>,
#[cfg(feature = "content")]
content_extractor:
Option<Arc<crate::domain::services::content_extractor::ContentExtractionFacade>>,
#[cfg(feature = "metrics")]
memory_scheduler: Option<Arc<MemoryScheduler>>,
deduplicator: Option<Arc<parking_lot::RwLock<Deduplicator>>>,
cache_service: Option<Arc<dyn CacheService>>,
shutdown_coordinator: Option<Arc<ShutdownCoordinator>>,
}
impl Default for ScrapeWorkerBuilder {
fn default() -> Self {
Self {
repository: None,
result_repository: None,
crawl_repository: None,
webhook_service: None,
credits_repository: None,
engine_client: None,
create_scrape_use_case: None,
team_semaphore: None,
robots_checker: None,
settings: None,
default_concurrency_limit: 10,
extraction_service: None,
#[cfg(feature = "content")]
content_extractor: None,
#[cfg(feature = "metrics")]
memory_scheduler: None,
deduplicator: None,
cache_service: None,
shutdown_coordinator: None,
}
}
}
impl ScrapeWorkerBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn default_concurrency_limit(&self) -> usize {
self.default_concurrency_limit
}
pub fn with_repository(mut self, repository: Arc<dyn TaskRepository>) -> Self {
self.repository = Some(repository);
self
}
pub fn with_result_repository(
mut self,
result_repository: Arc<dyn ScrapeResultRepository>,
) -> Self {
self.result_repository = Some(result_repository);
self
}
pub fn with_crawl_repository(mut self, crawl_repository: Arc<dyn CrawlRepository>) -> Self {
self.crawl_repository = Some(crawl_repository);
self
}
pub fn with_webhook_service(mut self, webhook_service: Arc<dyn WebhookService>) -> Self {
self.webhook_service = Some(webhook_service);
self
}
pub fn with_credits_repository(
mut self,
credits_repository: Arc<dyn CreditsRepository>,
) -> Self {
self.credits_repository = Some(credits_repository);
self
}
pub fn with_engine_client(mut self, engine_client: Arc<EngineClient>) -> Self {
self.engine_client = Some(engine_client);
self
}
pub fn with_create_scrape_use_case(
mut self,
create_scrape_use_case: Arc<dyn CreateScrapeUseCaseTrait>,
) -> Self {
self.create_scrape_use_case = Some(create_scrape_use_case);
self
}
pub fn with_team_semaphore(mut self, team_semaphore: Arc<TeamSemaphore>) -> Self {
self.team_semaphore = Some(team_semaphore);
self
}
pub fn with_robots_checker(mut self, robots_checker: Arc<dyn RobotsCheckerTrait>) -> Self {
self.robots_checker = Some(robots_checker);
self
}
pub fn with_settings(mut self, settings: Arc<Settings>) -> Self {
self.settings = Some(settings);
self
}
pub fn with_default_concurrency_limit(mut self, limit: usize) -> Self {
self.default_concurrency_limit = limit;
self
}
pub fn with_extraction_service(
mut self,
extraction_service: Arc<dyn ExtractionServiceTrait>,
) -> Self {
self.extraction_service = Some(extraction_service);
self
}
#[cfg(feature = "content")]
pub fn with_content_extractor(
mut self,
content_extractor: Arc<crate::domain::services::content_extractor::ContentExtractionFacade>,
) -> Self {
self.content_extractor = Some(content_extractor);
self
}
pub fn with_cache_service(mut self, cache_service: Arc<dyn CacheService>) -> Self {
self.cache_service = Some(cache_service);
self
}
#[cfg(feature = "metrics")]
pub fn with_memory_scheduler(mut self, memory_scheduler: Arc<MemoryScheduler>) -> Self {
self.memory_scheduler = Some(memory_scheduler);
self
}
pub fn with_deduplicator(
mut self,
deduplicator: Arc<parking_lot::RwLock<Deduplicator>>,
) -> Self {
self.deduplicator = Some(deduplicator);
self
}
pub fn with_shutdown_coordinator(
mut self,
shutdown_coordinator: Arc<ShutdownCoordinator>,
) -> Self {
self.shutdown_coordinator = Some(shutdown_coordinator);
self
}
pub fn build(self) -> Result<ScrapeWorker, &'static str> {
let repository = self.repository.ok_or("repository is required")?;
let result_repository = self
.result_repository
.ok_or("result_repository is required")?;
let crawl_repository = self
.crawl_repository
.ok_or("crawl_repository is required")?;
let webhook_service = self.webhook_service.ok_or("webhook_service is required")?;
let credits_repository = self
.credits_repository
.ok_or("credits_repository is required")?;
let engine_client = self.engine_client.ok_or("engine_client is required")?;
let create_scrape_use_case = self
.create_scrape_use_case
.ok_or("create_scrape_use_case is required")?;
let team_semaphore = self.team_semaphore.ok_or("team_semaphore is required")?;
let robots_checker = self.robots_checker.ok_or("robots_checker is required")?;
let settings = self.settings.ok_or("settings is required")?;
let extraction_service = self
.extraction_service
.ok_or("extraction_service is required")?;
let cache_service = self.cache_service.ok_or("cache_service is required")?;
#[cfg(feature = "content")]
let content_extractor = self.content_extractor.unwrap_or_else(|| {
Arc::new(crate::domain::services::content_extractor::ContentExtractionFacade::new(None))
});
#[cfg(feature = "metrics")]
let memory_scheduler = self
.memory_scheduler
.ok_or("memory_scheduler is required (metrics feature enabled)")?;
let shutdown_coordinator = self
.shutdown_coordinator
.unwrap_or_else(|| Arc::new(ShutdownCoordinator::default()));
let request_coalescer = Arc::new(RequestCoalescer::new());
let coalesce_coordinator = Arc::new(CoalesceCoordinator::new(
repository.clone(),
result_repository.clone(),
request_coalescer,
));
Ok(ScrapeWorker::new(ScrapeWorkerDeps {
repository,
result_repository,
crawl_repository,
webhook_service,
credits_repository,
engine_client,
create_scrape_use_case,
team_semaphore,
coalesce_coordinator,
robots_checker,
settings,
default_concurrency_limit: self.default_concurrency_limit,
extraction_service,
#[cfg(feature = "content")]
content_extractor,
cache_service,
#[cfg(feature = "metrics")]
memory_scheduler,
})
.with_shutdown_coordinator(shutdown_coordinator)
.with_deduplicator_opt(self.deduplicator))
}
}