use std::sync::Arc;
use crate::application::use_cases::crawl_use_case::CrawlUseCase;
use crate::di::{CrawlRsState, CrawlRsStateExt};
use crate::domain::repositories::{
crawl_repository::CrawlRepository, scrape_result_repository::ScrapeResultRepository,
task_repository::TaskRepository,
};
#[cfg(feature = "teams")]
use crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepository;
#[cfg(feature = "teams")]
use crate::domain::services::team_service::TeamService;
#[cfg(feature = "webhook")]
use crate::domain::repositories::webhook_repository::WebhookRepository;
use crate::domain::services::rate_limiting_service::RateLimitingService;
pub trait HandlerState: Clone + Send + Sync + 'static {
fn task_repo(&self) -> Arc<dyn TaskRepository>;
fn crawl_repo(&self) -> Arc<dyn CrawlRepository>;
fn result_repo(&self) -> Arc<dyn ScrapeResultRepository>;
#[cfg(feature = "webhook")]
fn webhook_repo(&self) -> Arc<dyn WebhookRepository>;
#[cfg(feature = "teams")]
fn geo_restriction_repo(&self) -> Arc<dyn GeoRestrictionRepository>;
#[cfg(feature = "teams")]
fn team_service(&self) -> Arc<TeamService>;
fn rate_limiting_service(&self) -> Arc<dyn RateLimitingService>;
}
impl HandlerState for Arc<CrawlRsState> {
fn task_repo(&self) -> Arc<dyn TaskRepository> {
CrawlRsStateExt::task_repo(self)
}
fn crawl_repo(&self) -> Arc<dyn CrawlRepository> {
CrawlRsStateExt::crawl_repo(self)
}
fn result_repo(&self) -> Arc<dyn ScrapeResultRepository> {
CrawlRsStateExt::result_repo(self)
}
#[cfg(feature = "webhook")]
fn webhook_repo(&self) -> Arc<dyn WebhookRepository> {
CrawlRsStateExt::webhook_repo(self)
}
#[cfg(feature = "teams")]
fn geo_restriction_repo(&self) -> Arc<dyn GeoRestrictionRepository> {
CrawlRsStateExt::geo_restriction_repo(self)
}
#[cfg(feature = "teams")]
fn team_service(&self) -> Arc<TeamService> {
CrawlRsStateExt::team_service(self)
}
fn rate_limiting_service(&self) -> Arc<dyn RateLimitingService> {
CrawlRsStateExt::rate_limiting_service(self)
}
}
#[derive(Clone)]
pub struct CrawlHandlerState {
pub crawl_repo: Arc<dyn CrawlRepository>,
pub task_repo: Arc<dyn TaskRepository>,
#[cfg(feature = "webhook")]
pub webhook_repo: Arc<dyn WebhookRepository>,
pub scrape_result_repo: Arc<dyn ScrapeResultRepository>,
#[cfg(feature = "teams")]
pub geo_restriction_repo: Arc<dyn GeoRestrictionRepository>,
#[cfg(feature = "teams")]
pub team_service: Arc<TeamService>,
pub rate_limiting_service: Arc<dyn RateLimitingService>,
}
impl CrawlHandlerState {
#[allow(clippy::too_many_arguments)]
pub fn new(
crawl_repo: Arc<dyn CrawlRepository>,
task_repo: Arc<dyn TaskRepository>,
scrape_result_repo: Arc<dyn ScrapeResultRepository>,
rate_limiting_service: Arc<dyn RateLimitingService>,
#[cfg(feature = "webhook")] webhook_repo: Arc<dyn WebhookRepository>,
#[cfg(feature = "teams")] geo_restriction_repo: Arc<dyn GeoRestrictionRepository>,
#[cfg(feature = "teams")] team_service: Arc<TeamService>,
) -> Self {
Self {
crawl_repo,
task_repo,
scrape_result_repo,
rate_limiting_service,
#[cfg(feature = "webhook")]
webhook_repo,
#[cfg(feature = "teams")]
geo_restriction_repo,
#[cfg(feature = "teams")]
team_service,
}
}
pub fn from_app_state(app_state: &Arc<CrawlRsState>) -> Self {
Self {
crawl_repo: app_state.crawl_repo.clone(),
task_repo: app_state.task_repo.clone(),
scrape_result_repo: app_state.result_repo.clone(),
rate_limiting_service: app_state.rate_limiting_service.clone(),
#[cfg(feature = "webhook")]
webhook_repo: app_state.webhook_repo.clone(),
#[cfg(feature = "teams")]
geo_restriction_repo: app_state.geo_restriction_repo.clone(),
#[cfg(feature = "teams")]
team_service: app_state.team_service.clone(),
}
}
pub fn create_use_case(&self) -> CrawlUseCase {
CrawlUseCase::new(
self.crawl_repo.clone(),
self.task_repo.clone(),
self.scrape_result_repo.clone(),
#[cfg(feature = "teams")]
self.geo_restriction_repo.clone(),
#[cfg(feature = "teams")]
self.team_service.clone(),
)
}
}
impl HandlerState for CrawlHandlerState {
fn task_repo(&self) -> Arc<dyn TaskRepository> {
self.task_repo.clone()
}
fn crawl_repo(&self) -> Arc<dyn CrawlRepository> {
self.crawl_repo.clone()
}
fn result_repo(&self) -> Arc<dyn ScrapeResultRepository> {
self.scrape_result_repo.clone()
}
#[cfg(feature = "webhook")]
fn webhook_repo(&self) -> Arc<dyn WebhookRepository> {
self.webhook_repo.clone()
}
#[cfg(feature = "teams")]
fn geo_restriction_repo(&self) -> Arc<dyn GeoRestrictionRepository> {
self.geo_restriction_repo.clone()
}
#[cfg(feature = "teams")]
fn team_service(&self) -> Arc<TeamService> {
self.team_service.clone()
}
fn rate_limiting_service(&self) -> Arc<dyn RateLimitingService> {
self.rate_limiting_service.clone()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::credits_model::CreditsTransactionType;
use crate::domain::models::ScrapeResult;
use crate::domain::models::{Crawl, Task};
#[cfg(feature = "webhook")]
use crate::domain::models::Webhook;
use crate::domain::repositories::task_repository::{RepositoryError, TaskQueryParams};
#[cfg(feature = "teams")]
use crate::domain::services::geo_location::{GeoLocation, GeoLocationService};
use crate::domain::services::rate_limiting_service::{
BacklogService, ConcurrencyConfig, ConcurrencyControlService, ConcurrencyResult,
QuotaService, RateLimitConfig, RateLimitResult, RateLimitService, RateLimitingError,
};
#[cfg(feature = "teams")]
use crate::domain::services::team_service::TeamGeoRestrictions;
use async_trait::async_trait;
use std::collections::HashSet;
#[cfg(feature = "teams")]
use std::net::IpAddr;
struct MockCrawlRepository;
#[async_trait]
impl CrawlRepository for MockCrawlRepository {
async fn create(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Crawl>, RepositoryError> {
Ok(None)
}
async fn update(&self, crawl: &Crawl) -> Result<Crawl, RepositoryError> {
Ok(crawl.clone())
}
async fn increment_completed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_failed_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn update_status(
&self,
_id: crate::domain::id::Id,
_status: crate::domain::models::CrawlStatus,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn increment_total_tasks(
&self,
_id: crate::domain::id::Id,
) -> Result<(), RepositoryError> {
Ok(())
}
async fn find_by_team_id_paginated(
&self,
_team_id: crate::domain::id::Id,
_limit: u32,
_offset: u32,
) -> Result<Vec<Crawl>, RepositoryError> {
Ok(vec![])
}
async fn count_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct MockTaskRepository;
#[async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[cfg(feature = "webhook")]
struct MockWebhookRepository;
#[cfg(feature = "webhook")]
#[async_trait]
impl WebhookRepository for MockWebhookRepository {
async fn create(&self, webhook: &Webhook) -> Result<Webhook, RepositoryError> {
Ok(webhook.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Webhook>, RepositoryError> {
Ok(None)
}
async fn find_by_team_id(
&self,
_team_id: crate::domain::id::Id,
) -> Result<Vec<Webhook>, RepositoryError> {
Ok(vec![])
}
}
struct MockScrapeResultRepository;
#[async_trait]
impl ScrapeResultRepository for MockScrapeResultRepository {
async fn save(&self, _result: ScrapeResult) -> anyhow::Result<()> {
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> anyhow::Result<Option<ScrapeResult>> {
Ok(None)
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> anyhow::Result<Vec<ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(
&self,
_team_id: crate::domain::id::Id,
) -> anyhow::Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
#[cfg(feature = "teams")]
struct MockGeoRestrictionRepository;
#[cfg(feature = "teams")]
#[async_trait]
impl GeoRestrictionRepository for MockGeoRestrictionRepository {
async fn get_team_restrictions(
&self,
_team_id: crate::domain::id::Id,
) -> Result<
TeamGeoRestrictions,
crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepositoryError,
> {
Ok(TeamGeoRestrictions::default())
}
async fn update_team_restrictions(
&self,
_team_id: crate::domain::id::Id,
_restrictions: &TeamGeoRestrictions,
) -> Result<
(),
crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepositoryError,
> {
Ok(())
}
async fn log_geo_restriction_action(
&self,
_team_id: crate::domain::id::Id,
_ip_address: &str,
_country_code: &str,
_action: &str,
_reason: &str,
) -> Result<
(),
crate::domain::repositories::geo_restriction_repository::GeoRestrictionRepositoryError,
> {
Ok(())
}
}
#[cfg(feature = "teams")]
struct MockGeoLocationService;
#[cfg(feature = "teams")]
#[async_trait]
impl GeoLocationService for MockGeoLocationService {
async fn get_location(&self, _ip: &IpAddr) -> anyhow::Result<GeoLocation> {
Ok(GeoLocation::default())
}
}
struct MockRateLimitingService;
#[async_trait]
impl RateLimitService for MockRateLimitingService {
async fn check_rate_limit(
&self,
_api_key: &str,
_endpoint: &str,
) -> Result<RateLimitResult, RateLimitingError> {
Ok(RateLimitResult::Allowed)
}
async fn get_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<RateLimitConfig, RateLimitingError> {
Ok(RateLimitConfig::default())
}
async fn update_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
_config: RateLimitConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn cleanup_expired_rate_limits(&self) -> Result<u64, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl ConcurrencyControlService for MockRateLimitingService {
async fn check_team_concurrency(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<ConcurrencyResult, RateLimitingError> {
Ok(ConcurrencyResult::Allowed)
}
async fn release_team_concurrency_slot(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_team_current_concurrency(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
async fn get_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<ConcurrencyConfig, RateLimitingError> {
Ok(ConcurrencyConfig::default())
}
async fn update_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
_config: ConcurrencyConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait]
impl BacklogService for MockRateLimitingService {
async fn process_backlog_tasks(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl QuotaService for MockRateLimitingService {
async fn check_and_deduct_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_quota_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, RateLimitingError> {
Ok(0)
}
async fn refund_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
impl RateLimitingService for MockRateLimitingService {}
fn build_test_state() -> CrawlHandlerState {
let crawl_repo: Arc<dyn CrawlRepository> = Arc::new(MockCrawlRepository);
let task_repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository);
let scrape_result_repo: Arc<dyn ScrapeResultRepository> =
Arc::new(MockScrapeResultRepository);
let rate_limiting_service: Arc<dyn RateLimitingService> = Arc::new(MockRateLimitingService);
#[cfg(feature = "teams")]
let geo_restriction_repo: Arc<dyn GeoRestrictionRepository> =
Arc::new(MockGeoRestrictionRepository);
#[cfg(feature = "teams")]
let team_service = Arc::new(TeamService::new(
Arc::new(MockGeoLocationService),
geo_restriction_repo.clone(),
));
#[cfg(feature = "webhook")]
let webhook_repo: Arc<dyn WebhookRepository> = Arc::new(MockWebhookRepository);
CrawlHandlerState::new(
crawl_repo,
task_repo,
scrape_result_repo,
rate_limiting_service,
#[cfg(feature = "webhook")]
webhook_repo,
#[cfg(feature = "teams")]
geo_restriction_repo,
#[cfg(feature = "teams")]
team_service,
)
}
#[test]
fn test_new_stores_all_dependencies() {
let state = build_test_state();
let _ = state.crawl_repo();
let _ = state.task_repo();
let _ = state.result_repo();
#[cfg(feature = "webhook")]
{
let _ = state.webhook_repo();
}
#[cfg(feature = "teams")]
{
let _ = state.geo_restriction_repo();
let _ = state.team_service();
}
let _ = state.rate_limiting_service();
}
#[test]
fn test_handler_state_trait_returns_injected_repositories() {
let state = build_test_state();
assert!(Arc::ptr_eq(&state.crawl_repo, &state.crawl_repo(),));
assert!(Arc::ptr_eq(&state.task_repo, &state.task_repo(),));
#[cfg(feature = "webhook")]
{
assert!(Arc::ptr_eq(&state.webhook_repo, &state.webhook_repo(),));
}
assert!(Arc::ptr_eq(&state.scrape_result_repo, &state.result_repo(),));
#[cfg(feature = "teams")]
{
assert!(Arc::ptr_eq(
&state.geo_restriction_repo,
&state.geo_restriction_repo(),
));
}
}
#[test]
fn test_handler_state_trait_returns_injected_services() {
let state = build_test_state();
#[cfg(feature = "teams")]
{
assert!(Arc::ptr_eq(&state.team_service, &state.team_service(),));
}
assert!(Arc::ptr_eq(
&state.rate_limiting_service,
&state.rate_limiting_service(),
));
}
#[test]
fn test_create_use_case_constructs_without_panic() {
let state = build_test_state();
let _use_case = state.create_use_case();
}
#[test]
fn test_clone_shares_underlying_state() {
let state = build_test_state();
let cloned = state.clone();
assert!(Arc::ptr_eq(&state.crawl_repo, &cloned.crawl_repo));
assert!(Arc::ptr_eq(&state.task_repo, &cloned.task_repo));
#[cfg(feature = "webhook")]
{
assert!(Arc::ptr_eq(&state.webhook_repo, &cloned.webhook_repo));
}
assert!(Arc::ptr_eq(
&state.scrape_result_repo,
&cloned.scrape_result_repo
));
#[cfg(feature = "teams")]
{
assert!(Arc::ptr_eq(
&state.geo_restriction_repo,
&cloned.geo_restriction_repo
));
assert!(Arc::ptr_eq(&state.team_service, &cloned.team_service));
}
assert!(Arc::ptr_eq(
&state.rate_limiting_service,
&cloned.rate_limiting_service
));
}
#[test]
fn test_clone_trait_accessor_returns_same_arc() {
let state = build_test_state();
let cloned = state.clone();
assert!(Arc::ptr_eq(&state.crawl_repo(), &cloned.crawl_repo()));
}
}