use std::sync::atomic::AtomicUsize;
use std::sync::Arc;
use anyhow::{Context, Result};
use chrono::Utc;
use log::{info, warn};
use tokio::io::{AsyncBufReadExt, BufReader};
use crate::config::{Config, DEFAULT_USER_AGENT};
use crate::error_handling::ProcessingStats;
use crate::fetch::{NetworkContext, ProcessingContext, RuntimeContext};
use crate::initialization::{
init_client, init_host_rate_limiter, init_rate_limiter, init_redirect_client, init_resolver,
init_semaphore,
};
use crate::runtime_metrics::RuntimeMetrics;
use crate::storage::{init_db_pool_with_path, insert_run_metadata, RunMetadata};
use crate::utils::TimingStats;
use super::resources::{ScanResources, UrlSource};
pub type ProgressCallback = Option<Arc<dyn Fn(usize, usize, usize, usize) + Send + Sync>>;
#[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] pub async fn init_scan_resources(
mut config: Config,
) -> Result<(ScanResources, UrlSource, usize, ProgressCallback)> {
config
.validate()
.map_err(|e| anyhow::anyhow!("Configuration validation failed: {e}"))?;
let cache_root = crate::cache_paths::resolve_cache_root(config.cache_dir.as_deref());
let fingerprints_cache = crate::cache_paths::fingerprints_dir(&cache_root);
let geoip_cache = crate::cache_paths::geoip_dir(&cache_root);
let ua_cache = crate::cache_paths::user_agent_dir(&cache_root);
let whois_cache = crate::cache_paths::whois_dir(&cache_root);
log::info!("Cache root: {}", cache_root.display());
if config.user_agent == DEFAULT_USER_AGENT {
let updated_ua = crate::user_agent::get_default_user_agent(Some(&ua_cache)).await;
config.user_agent = updated_ua;
log::debug!("Using auto-updated User-Agent: {}", config.user_agent);
}
let (total_lines, is_stdin) = if config.file.as_os_str() == "-" {
info!("Reading URLs from stdin");
(0, true)
} else {
let file_for_counting = tokio::fs::File::open(&config.file)
.await
.context("Failed to open input file for line counting")?;
let reader = BufReader::new(file_for_counting);
let mut count = 0usize;
let mut counting_lines = reader.lines();
while let Ok(Some(line)) = counting_lines.next_line().await {
let trimmed = line.trim();
if !trimmed.is_empty() && !trimmed.starts_with('#') {
count += 1;
}
}
info!("Total URLs in file: {count}");
(count, false)
};
let url_source = if is_stdin {
use tokio::io::stdin;
UrlSource::Stdin(BufReader::new(stdin()).lines())
} else {
let file = tokio::fs::File::open(&config.file)
.await
.context("Failed to open input file")?;
UrlSource::File(BufReader::new(file).lines())
};
let semaphore = init_semaphore(config.max_concurrency);
let rate_burst = if config.rate_limit_rps > 0 {
let rps_doubled = config.rate_limit_rps.saturating_mul(2);
std::cmp::min(config.max_concurrency, rps_doubled as usize)
} else {
config.max_concurrency
};
let (request_limiter, rate_limiter_shutdown) =
match init_rate_limiter(config.rate_limit_rps, rate_burst) {
Some((limiter, shutdown)) => (Some(limiter), Some(shutdown)),
None => (None, None),
};
let host_limiter = init_host_rate_limiter(config.rate_limit_rps);
let pool_size = crate::storage::pool::sqlite_pool_size(config.max_concurrency);
let pool = init_db_pool_with_path(&config.db_path, pool_size)
.await
.context("Failed to initialize database pool")?;
let resolver = init_resolver().context("Failed to initialize DNS resolver")?;
let client = if let Some(ref overrides) = config.dependency_overrides {
if let Some(ref c) = overrides.http_client {
Arc::new(c.clone())
} else {
init_client(&config, Arc::clone(&resolver))
.await
.context("Failed to initialize HTTP client")?
}
} else {
init_client(&config, Arc::clone(&resolver))
.await
.context("Failed to initialize HTTP client")?
};
let redirect_client = init_redirect_client(&config, Arc::clone(&resolver))
.await
.context("Failed to initialize redirect client")?;
crate::storage::run_migrations(&pool)
.await
.context("Failed to run database migrations")?;
let (enable_whois, whois_client) = if config.enable_whois {
match crate::whois::init_shared_client().await {
Ok(client) => {
info!(
"WHOIS/RDAP lookup enabled (best-effort, {}s timeout, 7-day disk cache)",
crate::config::WHOIS_TIMEOUT_SECS
);
(true, Some(client))
}
Err(e) => {
warn!("Failed to initialize WHOIS/RDAP client: {e}. Continuing without WHOIS.");
(false, None)
}
}
} else {
(false, None)
};
let ruleset = crate::fingerprint::init_ruleset(
config.fingerprints.as_deref(),
Some(&fingerprints_cache),
config.allow_degraded_fingerprints,
)
.await
.context("Failed to initialize fingerprint ruleset")?;
let identity = crate::fingerprint::ruleset_identity_summary(&ruleset);
info!("{identity}");
eprintln!("{identity}");
crate::parse::gitleaks::init_gitleaks().context("Failed to load bundled gitleaks ruleset")?;
let geoip_metadata = match crate::geoip::init_geoip(config.geoip.as_deref(), Some(&geoip_cache))
.await
{
Ok(metadata) => metadata,
Err(e) => {
warn!("Failed to initialize GeoIP database: {e}. Continuing without GeoIP lookup.");
warn!("To enable GeoIP, ensure MAXMIND_LICENSE_KEY in .env is valid and your MaxMind account has GeoLite2 access.");
None
}
};
let start_time_epoch = Utc::now().timestamp_millis();
let run_id = format!("run_{start_time_epoch}");
info!("Starting run: {run_id}");
info!(
"Timeouts: HTTP request={}s, per-URL processing={}s (hardcoded), drain={}s",
config.timeout_seconds,
crate::config::URL_PROCESSING_TIMEOUT.as_secs(),
config.drain_timeout_secs
);
let meta = RunMetadata {
run_id: &run_id,
start_time_ms: start_time_epoch,
version: env!("CARGO_PKG_VERSION"),
fingerprints_source: Some(ruleset.metadata.source.as_str()),
fingerprints_version: Some(ruleset.metadata.version.as_str()),
geoip_version: geoip_metadata.as_ref().map(|m| m.version.as_str()),
};
insert_run_metadata(&pool, &meta)
.await
.context("Failed to insert run metadata")?;
let start_time = std::time::Instant::now();
let error_stats = Arc::new(ProcessingStats::new());
let timing_stats = Arc::new(TimingStats::new());
let runtime_metrics = Arc::new(RuntimeMetrics::default());
let successful_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let failed_urls = Arc::new(AtomicUsize::new(0));
let total_urls_attempted = Arc::new(AtomicUsize::new(0));
let total_urls_in_file = Arc::new(AtomicUsize::new(total_lines));
let progress_callback = config.progress_callback.clone();
let mut runtime = RuntimeContext::new(
error_stats.clone(),
Arc::clone(&timing_stats),
Some(run_id.clone()),
enable_whois,
whois_cache.clone(),
config.scan_external_scripts,
Arc::clone(&runtime_metrics),
config.allow_localhost_for_tests,
);
runtime.whois_client = whois_client;
let shared_ctx = Arc::new(ProcessingContext::new(
NetworkContext::new(
Arc::clone(&client),
Arc::clone(&redirect_client),
Arc::clone(&resolver),
),
Arc::clone(&pool),
runtime,
Arc::clone(&ruleset),
));
let resources = ScanResources {
shared_ctx,
semaphore,
request_limiter,
host_limiter,
rate_limiter_shutdown,
in_flight_urls: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
successful_urls,
skipped_urls,
failed_urls,
total_urls_attempted,
total_urls_in_file,
phase: Arc::new(crate::status_server::AtomicPhase::default()),
throughput_window: Arc::new(crate::status_server::ThroughputWindow::new()),
run_id,
start_time_epoch,
start_time,
_geoip_metadata: geoip_metadata,
config,
};
Ok((resources, url_source, total_lines, progress_callback))
}