use log::info;
use std::sync::Arc;
use crate::application::use_cases::create_scrape::{CreateScrapeUseCase, CreateScrapeUseCaseTrait};
use crate::bootstrap::infrastructure::InfrastructureComponents;
use crate::bootstrap::infrastructure::Repositories;
use crate::config::settings::Settings;
use crate::domain::services::rate_limiting_service::RateLimitingService;
use crate::domain::services::team_semaphore::{AdaptiveParams, TeamSemaphore};
use crate::engines::engine_client::EngineClient;
use crate::presentation::middleware::rate_limit_middleware::RateLimitMiddleware;
use crate::workers::coalesce_coordinator::RequestCoalescer;
#[cfg(feature = "rate-limit")]
use crate::domain::services::rate_limiting_service::{
ConcurrencyConfig, ConcurrencyStrategy, RateLimitConfig, RateLimitStrategy,
};
#[cfg(feature = "rate-limit")]
use crate::infrastructure::services::limiteron_service::{
LimiteronService, RateLimitingConfig, StorageHandle,
};
#[cfg(not(feature = "rate-limit"))]
use crate::infrastructure::services::noop_rate_limiting_service::NoopRateLimitingService;
use dbnexus::DbPool;
pub(super) struct RuntimeServices {
pub(super) team_semaphore: Arc<TeamSemaphore>,
pub(super) request_coalescer: Arc<RequestCoalescer>,
pub(super) rate_limiting_service: Arc<dyn RateLimitingService>,
pub(super) rate_limit_middleware: RateLimitMiddleware,
pub(super) create_scrape_use_case: Arc<dyn CreateScrapeUseCaseTrait>,
#[cfg(feature = "sitemap")]
pub(super) map_use_case: Arc<crate::application::use_cases::map_use_case::MapUseCase>,
}
pub(super) async fn init_runtime_services(
infrastructure: &InfrastructureComponents,
engine_client: &Arc<EngineClient>,
settings: &Settings,
) -> anyhow::Result<RuntimeServices> {
let repositories = &infrastructure.repositories;
crate::workers::crawl::soft404::init_shared(settings.crawl.soft404_max_fingerprints);
let team_semaphore = init_team_semaphore(settings);
let request_coalescer = Arc::new(RequestCoalescer::new());
let rate_limiting_service = init_rate_limiting_service(
repositories,
settings,
Some(infrastructure.db.clone_inner()),
)
.await?;
let rate_limit_middleware = init_rate_limit_middleware(rate_limiting_service.clone());
let create_scrape_use_case: Arc<dyn CreateScrapeUseCaseTrait> =
Arc::new(CreateScrapeUseCase::new(engine_client.clone()));
#[cfg(feature = "sitemap")]
let map_use_case = Arc::new(
crate::application::use_cases::map_use_case::MapUseCase::new(Arc::new(
crate::application::use_cases::map_use_case::EngineSitemapFetcher::new(
engine_client.clone(),
std::time::Duration::from_secs(settings.sitemap.fetch_timeout_secs),
),
)),
);
Ok(RuntimeServices {
team_semaphore,
request_coalescer,
rate_limiting_service,
rate_limit_middleware,
create_scrape_use_case,
#[cfg(feature = "sitemap")]
map_use_case,
})
}
pub(super) fn init_rate_limit_middleware(
rate_limiting_service: Arc<dyn RateLimitingService>,
) -> RateLimitMiddleware {
RateLimitMiddleware::new(rate_limiting_service)
}
pub(super) fn init_team_semaphore(settings: &Settings) -> Arc<TeamSemaphore> {
let default_team_limit = settings.concurrency.default_team_limit as usize;
if settings.concurrency.adaptive_enabled {
let params = AdaptiveParams {
initial: default_team_limit,
..Default::default()
};
info!(
"init_team_semaphore: Adaptive mode enabled (initial={}, min=1, max=100, threshold=10)",
default_team_limit
);
Arc::new(TeamSemaphore::with_adaptive(params))
} else {
Arc::new(TeamSemaphore::new(default_team_limit))
}
}
#[cfg(feature = "rate-limit")]
fn build_rate_limiting_config(settings: &Settings) -> RateLimitingConfig {
let rate_limit_config = RateLimitConfig {
strategy: RateLimitStrategy::TokenBucket,
requests_per_second: settings.rate_limiting.default_rpm / 60,
requests_per_minute: settings.rate_limiting.default_rpm,
requests_per_hour: settings.rate_limiting.default_rpm * 60,
bucket_capacity: Some(settings.rate_limiting.default_rpm),
enabled: settings.rate_limiting.enabled,
};
if let Err(e) = rate_limit_config.validate() {
log::error!("Rate limit configuration error: {}", e);
}
let concurrency_config = ConcurrencyConfig {
strategy: ConcurrencyStrategy::DistributedSemaphore,
max_concurrent_tasks: settings.concurrency.default_team_limit as u32,
max_concurrent_per_team: settings.concurrency.default_team_limit as u32,
lock_timeout_seconds: settings.concurrency.task_lock_duration_seconds,
enabled: true,
};
if let Err(e) = concurrency_config.validate() {
log::error!("Concurrency configuration error: {}", e);
}
RateLimitingConfig {
rate_limit: rate_limit_config,
concurrency: concurrency_config,
backlog_process_interval_seconds: 30,
rate_limit_ttl_seconds: 3600,
}
}
#[cfg(feature = "rate-limit")]
fn select_rate_limit_storage(
settings: &Settings,
db_pool: Option<Arc<DbPool>>,
) -> anyhow::Result<StorageHandle> {
match settings.rate_limiting.storage_backend.as_str() {
"memory" => Ok(StorageHandle::Memory),
"database" => {
#[cfg(all(feature = "db-postgres", feature = "platform"))]
match db_pool {
Some(pool) => Ok(StorageHandle::Database(pool)),
None => {
anyhow::bail!(
"rate_limiting.storage_backend=database but no db pool available; \
refusing to fall back to memory storage (config error)"
);
}
}
#[cfg(not(all(feature = "db-postgres", feature = "platform")))]
{
let _ = db_pool;
anyhow::bail!(
"rate_limiting.storage_backend=database requires db-postgres + platform \
(crawlrs wires the Postgres shared-storage adapter only); \
refusing to fall back to memory storage"
);
}
}
other => {
log::warn!(
"Unknown rate_limiting.storage_backend '{}'; falling back to memory storage",
other
);
Ok(StorageHandle::Memory)
}
}
}
pub(super) async fn init_rate_limiting_service(
repositories: &Repositories,
settings: &Settings,
db_pool: Option<Arc<DbPool>>,
) -> anyhow::Result<Arc<dyn RateLimitingService>> {
#[cfg(feature = "rate-limit")]
{
let rate_limiting_config = build_rate_limiting_config(settings);
let storage = select_rate_limit_storage(settings, db_pool)?;
let service = LimiteronService::new(
repositories.task_repo.clone(),
repositories.tasks_backlog_repo.clone(),
repositories.credits_repo.clone(),
rate_limiting_config,
storage,
)
.await
.expect("Failed to create LimiteronService");
Ok(Arc::new(service))
}
#[cfg(not(feature = "rate-limit"))]
{
let _ = repositories;
let _ = settings;
let _ = db_pool;
log::warn!(
"rate-limit feature disabled, using NoopRateLimitingService — \
all requests are allowed without rate limiting"
);
Ok(Arc::new(NoopRateLimitingService::new()))
}
}