use crate::config::settings::Settings;
use crate::infrastructure::database::dbnexus_connection::DatabasePool;
use crate::infrastructure::dns::DnsCacheService;
use crate::infrastructure::oxcache::{create_cache, CacheService, OxcacheService, SearchCache};
use crate::infrastructure::repositories::{
crawl_repo_impl::CrawlRepositoryImpl, credits_repo_impl::CreditsRepositoryImpl,
scrape_result_repo_impl::ScrapeResultRepositoryImpl,
searxng_instance_repo_impl::SearxngInstanceRepositoryImpl, task_repo_impl::TaskRepositoryImpl,
tasks_backlog_repo_impl::TasksBacklogRepositoryImpl,
webhook_event_repo_impl::WebhookEventRepoImpl, webhook_repo_impl::WebhookRepoImpl,
};
#[cfg(feature = "teams")]
use crate::infrastructure::database::repositories::database_geo_restriction_repo::DatabaseGeoRestrictionRepository;
use anyhow::Result;
use log::info;
use std::net::Ipv4Addr;
use std::sync::Arc;
use std::time::Duration;
#[derive(Clone)]
pub struct Repositories {
pub task_repo: Arc<TaskRepositoryImpl>,
pub result_repo: Arc<ScrapeResultRepositoryImpl>,
pub crawl_repo: Arc<CrawlRepositoryImpl>,
pub webhook_event_repo: Arc<WebhookEventRepoImpl>,
pub webhook_repo: Arc<WebhookRepoImpl>,
pub credits_repo: Arc<CreditsRepositoryImpl>,
#[cfg(feature = "teams")]
pub geo_restriction_repo: Arc<DatabaseGeoRestrictionRepository>,
pub tasks_backlog_repo: Arc<TasksBacklogRepositoryImpl>,
pub searxng_instance_repo: Arc<SearxngInstanceRepositoryImpl>,
}
pub async fn init_database(settings: &Settings) -> Result<Arc<DatabasePool>> {
use crate::infrastructure::database::dbnexus_connection::create_pool;
let pool = create_pool(&settings.database).await?;
let db = DatabasePool {
inner: Arc::new(pool),
stats: Default::default(),
};
let db = Arc::new(db);
info!("Database connection established");
Ok(db)
}
pub fn init_http_client(
settings: &Settings,
dns_cache: Option<Arc<DnsCacheService>>,
) -> Result<Arc<reqwest::Client>> {
let timeout_secs = settings.timeouts.engines.default_timeout_seconds;
let timeout = Duration::from_secs(timeout_secs);
let resolver = match dns_cache {
Some(cache) => {
info!("Using DNS cache for IPv4 resolver");
crate::infrastructure::dns::create_ipv4_only_resolver_with_cache(cache)
}
None => crate::infrastructure::dns::create_ipv4_only_resolver(),
};
let client = reqwest::Client::builder()
.timeout(timeout)
.connect_timeout(crate::utils::http_client::connect_timeout_for(timeout))
.pool_idle_timeout(Duration::from_secs(90))
.local_address(Some(Ipv4Addr::UNSPECIFIED.into()))
.dns_resolver(resolver)
.redirect(crate::utils::http_client::create_ssrf_safe_redirect_policy(
5,
))
.build()?;
let client = Arc::new(client);
info!("HTTP client initialized (timeout: {}s)", timeout_secs);
Ok(client)
}
pub fn init_repositories(db: Arc<DatabasePool>, settings: &Settings) -> Repositories {
let task_repo = Arc::new(TaskRepositoryImpl::new(
db.inner().clone(),
chrono::Duration::seconds(
settings
.concurrency
.task_lock_duration_seconds
.try_into()
.expect("task_lock_duration_seconds exceeds i64 range"),
),
));
let result_repo = Arc::new(ScrapeResultRepositoryImpl::new(db.inner().clone()));
let crawl_repo = Arc::new(CrawlRepositoryImpl::new(db.inner().clone()));
let webhook_event_repo = Arc::new(WebhookEventRepoImpl::new(db.inner().clone()));
let webhook_repo = Arc::new(WebhookRepoImpl::new(db.inner().clone()));
let credits_repo = Arc::new(CreditsRepositoryImpl::new(db.inner().clone()));
#[cfg(feature = "teams")]
let geo_restriction_repo = Arc::new(DatabaseGeoRestrictionRepository::new(db.inner().clone()));
let tasks_backlog_repo = Arc::new(TasksBacklogRepositoryImpl::new(db.inner().clone()));
let searxng_instance_repo = Arc::new(SearxngInstanceRepositoryImpl::new(db.inner().clone()));
Repositories {
task_repo,
result_repo,
crawl_repo,
webhook_event_repo,
webhook_repo,
credits_repo,
#[cfg(feature = "teams")]
geo_restriction_repo,
tasks_backlog_repo,
searxng_instance_repo,
}
}
#[derive(Clone)]
pub struct InfrastructureComponents {
pub db: Arc<DatabasePool>,
pub oxcache: Option<Arc<SearchCache>>,
pub cache_service: Arc<dyn CacheService>,
pub http_client: Arc<reqwest::Client>,
pub repositories: Repositories,
}
pub async fn init_oxcache(settings: &Settings) -> Result<Option<Arc<SearchCache>>> {
if !settings.cache.enabled {
info!("Cache is disabled, skipping oxcache initialization");
return Ok(None);
}
match create_cache(&settings.cache).await {
Ok(cache) => {
info!(
"OxCache initialized (capacity: {}, ttl: {}s)",
settings.cache.memory.capacity, settings.cache.memory.ttl_seconds
);
Ok(Some(cache))
}
Err(e) => {
log::warn!(
"Failed to initialize oxcache: {}. Cache will be disabled.",
e
);
Ok(None)
}
}
}
pub async fn init_cache_service(settings: &Settings) -> Result<Arc<dyn CacheService>> {
if !settings.cache.enabled {
info!("Cache is disabled, skipping cache service initialization");
let service = OxcacheService::build(1, std::time::Duration::from_secs(1))
.await
.map_err(|e| anyhow::anyhow!("Failed to build minimal cache service: {e}"))?;
return Ok(Arc::new(service));
}
let capacity = settings.cache.memory.capacity;
let ttl = std::time::Duration::from_secs(settings.cache.memory.ttl_seconds);
let service = OxcacheService::build_with_backend(
&settings.cache.backend,
if settings.cache.redis_url.is_empty() {
None
} else {
Some(settings.cache.redis_url.as_str())
},
capacity,
ttl,
)
.await
.map_err(|e| anyhow::anyhow!("Failed to build cache service: {e}"))?;
info!(
"Cache service initialized (backend: {}, capacity: {}, ttl: {}s)",
settings.cache.backend, capacity, settings.cache.memory.ttl_seconds
);
Ok(Arc::new(service))
}
pub async fn init_infrastructure(settings: &Settings) -> Result<InfrastructureComponents> {
let db = init_database(settings).await?;
let dns_cache = init_dns_cache_service(settings).await;
let http_client = init_http_client(settings, dns_cache)?;
let repositories = init_repositories(db.clone(), settings);
let oxcache = init_oxcache(settings).await?;
let cache_service = init_cache_service(settings).await?;
Ok(InfrastructureComponents {
db,
oxcache,
cache_service,
http_client,
repositories,
})
}
async fn init_dns_cache_service(settings: &Settings) -> Option<Arc<DnsCacheService>> {
if !settings.cache.enabled {
info!("Cache disabled, DNS cache not initialized");
return None;
}
match crate::infrastructure::oxcache::create_dns_cache(
settings.cache.memory.capacity,
settings.cache.memory.ttl_seconds,
)
.await
{
Ok(cache) => match DnsCacheService::new(cache, settings.cache.memory.ttl_seconds) {
Ok(service) => {
info!(
"DNS cache initialized for IPv4 resolver (capacity: {}, ttl: {}s)",
settings.cache.memory.capacity, settings.cache.memory.ttl_seconds
);
Some(Arc::new(service))
}
Err(e) => {
log::warn!("Failed to create DNS resolver: {}. Using system DNS.", e);
None
}
},
Err(e) => {
log::warn!("Failed to create DNS cache: {}. Using system DNS.", e);
None
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_init_http_client_returns_ok_with_default_settings() {
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let result = init_http_client(&settings, None);
assert!(
result.is_ok(),
"init_http_client should succeed with default settings"
);
}
#[test]
fn test_init_http_client_returns_arc_client() {
let settings = crate::bootstrap::config::load_settings().expect("Failed to load settings");
let client = init_http_client(&settings, None).expect("Should create HTTP client");
let _req = client.get("http://localhost");
}
#[test]
fn test_init_http_client_ignores_proxy_settings() {
let mut settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings");
settings.proxy.enabled = true;
settings.proxy.urls = vec!["http://localhost:10808".to_string()];
let result = init_http_client(&settings, None);
assert!(
result.is_ok(),
"init_http_client should succeed regardless of proxy settings"
);
}
#[tokio::test]
async fn test_init_oxcache_returns_ok_when_cache_disabled() {
let mut settings =
crate::bootstrap::config::load_settings().expect("Failed to load settings");
settings.cache.enabled = false;
let result = init_oxcache(&settings).await;
assert!(
result.is_ok(),
"init_oxcache should return Ok when cache is disabled"
);
let cache = result.expect("init_oxcache should succeed");
assert!(
cache.is_none(),
"oxcache should be None when cache is disabled"
);
}
use crate::common::test_support::testcontainers_fixtures as tcf;
async fn require_docker() -> bool {
tcf::docker_available().await
}
#[tokio::test]
async fn tc_init_database_connects_to_postgres() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_init_database_connects_to_postgres");
return;
}
let pg = match tcf::PgHandle::start().await {
Ok(p) => p,
Err(e) => {
eprintln!("[skip] failed to start postgres container: {e}");
return;
}
};
let settings = tcf::settings_with_urls(&pg.url).unwrap();
let pool = init_database(&settings).await;
assert!(
pool.is_ok(),
"init_database should succeed against a live postgres: {:?}",
pool.err()
);
let pool = pool.unwrap();
let session = pool.get_session("admin").await;
assert!(
session.is_ok(),
"should be able to acquire an admin session from the pool"
);
}
#[tokio::test]
async fn tc_init_database_returns_arc_database_pool() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_init_database_returns_arc_database_pool");
return;
}
let pg = match tcf::PgHandle::start().await {
Ok(p) => p,
Err(e) => {
eprintln!("[skip] failed to start postgres container: {e}");
return;
}
};
let settings = tcf::settings_with_urls(&pg.url).unwrap();
let pool = init_database(&settings)
.await
.expect("pool should be created");
assert!(Arc::strong_count(&pool) >= 1);
let _inner: Arc<dbnexus::DbPool> = pool.inner().clone();
}
#[tokio::test]
async fn tc_init_database_fails_on_invalid_url() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_init_database_fails_on_invalid_url");
return;
}
let settings =
tcf::settings_with_urls("postgres://nobody:nopass@127.0.0.1:1/nonexistent").unwrap();
let result = init_database(&settings).await;
assert!(
result.is_err(),
"init_database should fail when the database URL is unreachable"
);
}
#[tokio::test]
async fn tc_init_repositories_creates_all_repos() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_init_repositories_creates_all_repos");
return;
}
let pg = match tcf::PgHandle::start().await {
Ok(p) => p,
Err(e) => {
eprintln!("[skip] failed to start postgres container: {e}");
return;
}
};
let settings = tcf::settings_with_urls(&pg.url).unwrap();
let db = match init_database(&settings).await {
Ok(d) => d,
Err(e) => {
eprintln!("[skip] failed to init database pool: {e}");
return;
}
};
let repos = init_repositories(db.clone(), &settings);
assert!(Arc::strong_count(&repos.task_repo.clone()) >= 1);
assert!(Arc::strong_count(&repos.result_repo.clone()) >= 1);
assert!(Arc::strong_count(&repos.crawl_repo.clone()) >= 1);
assert!(Arc::strong_count(&repos.webhook_event_repo.clone()) >= 1);
assert!(Arc::strong_count(&repos.webhook_repo.clone()) >= 1);
assert!(Arc::strong_count(&repos.credits_repo.clone()) >= 1);
#[cfg(feature = "teams")]
assert!(Arc::strong_count(&repos.geo_restriction_repo.clone()) >= 1);
assert!(Arc::strong_count(&repos.tasks_backlog_repo.clone()) >= 1);
}
#[tokio::test]
async fn tc_init_infrastructure_full_stack() {
if !require_docker().await {
eprintln!("[skip] Docker unavailable — tc_init_infrastructure_full_stack");
return;
}
let handle = match tcf::DbHandle::start().await {
Ok(h) => h,
Err(e) => {
eprintln!("[skip] failed to start db container: {e}");
return;
}
};
let settings = tcf::settings_with_urls(&handle.pg.url).unwrap();
let infra = init_infrastructure(&settings).await;
assert!(
infra.is_ok(),
"init_infrastructure should succeed against live db: {:?}",
infra.err()
);
let infra = infra.unwrap();
assert!(Arc::strong_count(&infra.db) >= 1);
assert!(Arc::strong_count(&infra.http_client) >= 1);
let _ = &infra.oxcache;
assert!(Arc::strong_count(&infra.repositories.task_repo) >= 1);
}
#[tokio::test]
async fn tc_init_infrastructure_fails_without_db() {
let settings = tcf::settings_with_urls("postgres://127.0.0.1:1/x").unwrap();
let result = init_infrastructure(&settings).await;
assert!(
result.is_err(),
"init_infrastructure should fail when the database is unreachable"
);
}
}