mod finalize;
mod init;
mod resources;
mod task;
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex};
use log::warn;
use tokio::task::JoinSet;
use tokio_util::sync::CancellationToken;
use crate::app::{log_progress, validate_and_normalize_url};
use crate::config::{LOGGING_INTERVAL, STATUS_SERVER_LOGGING_INTERVAL_SECS};
use crate::error_handling::ErrorType;
use crate::security::validate_url_safe;
use crate::storage::insert::insert_url_failure;
use crate::storage::models::UrlFailureRecord;
use crate::storage::DbPool;
struct InFlightGuard {
registry: Arc<Mutex<HashSet<String>>>,
url: Option<String>,
}
impl InFlightGuard {
fn register(registry: Arc<Mutex<HashSet<String>>>, url: String) -> Self {
if let Ok(mut set) = registry.lock() {
set.insert(url.clone());
}
Self {
registry,
url: Some(url),
}
}
}
impl Drop for InFlightGuard {
fn drop(&mut self) {
if let Some(url) = self.url.take() {
if let Ok(mut set) = self.registry.lock() {
set.remove(&url);
}
}
}
}
fn domain_from_url_or_raw(url: &str) -> String {
url::Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(str::to_string))
.unwrap_or_else(|| url.to_string())
}
async fn insert_abort_url_failure(
pool: &DbPool,
run_id: Option<&str>,
url: &str,
error_type: ErrorType,
error_message: String,
elapsed_time_seconds: Option<f64>,
) -> bool {
let record = UrlFailureRecord {
url: url.to_string(),
final_url: None,
domain: domain_from_url_or_raw(url),
final_domain: None,
error_type,
error_message,
http_status: None,
retry_count: 0,
elapsed_time_seconds,
timestamp: chrono::Utc::now().timestamp_millis(),
run_id: run_id.map(str::to_string),
redirect_chain: vec![],
response_headers: vec![],
};
match insert_url_failure(pool.as_ref(), &record).await {
Ok(_) => true,
Err(e) => {
log::error!("Failed to insert {error_type} url_failures row for {url}: {e}");
false
}
}
}
async fn record_drain_timeout_failures(
pool: &DbPool,
run_id: &str,
drain_timeout_secs: u64,
abandoned: &[String],
) -> usize {
let mut inserted = 0usize;
for url in abandoned {
let ok = insert_abort_url_failure(
pool,
Some(run_id),
url,
ErrorType::ProcessUrlTimeout,
format!(
"Aborted at scan drain timeout ({drain_timeout_secs}s expired); task did not finish in time"
),
None,
)
.await;
if ok {
inserted += 1;
}
}
inserted
}
pub(super) async fn record_cooperative_cancel_failure(
pool: &DbPool,
run_id: Option<&str>,
url: &str,
) {
let _ = insert_abort_url_failure(
pool,
run_id,
url,
ErrorType::ScanCancelled,
format!("Scan cancelled before URL finished processing: {url}"),
None,
)
.await;
}
pub use resources::{ScanLoopResult, ScanResources, UrlTaskParams};
pub use init::init_scan_resources;
#[derive(Debug, Clone)]
pub struct ScanReport {
pub total_urls: usize,
pub successful: usize,
pub failed: usize,
pub skipped: usize,
pub db_path: PathBuf,
pub run_id: String,
pub elapsed_seconds: f64,
}
#[allow(clippy::type_complexity)] fn invoke_progress_callback(
callback: Option<&Arc<dyn Fn(usize, usize, usize, usize) + Send + Sync>>,
successful: &Arc<std::sync::atomic::AtomicUsize>,
failed: &Arc<std::sync::atomic::AtomicUsize>,
skipped: &Arc<std::sync::atomic::AtomicUsize>,
total: usize,
) {
if let Some(cb) = callback {
cb(
successful.load(Ordering::Relaxed),
failed.load(Ordering::Relaxed),
skipped.load(Ordering::Relaxed),
total,
);
}
}
fn skip_url(
resources: &ScanResources,
progress_callback: &resources::ProgressCallback,
total_lines: usize,
) {
resources
.total_urls_attempted
.fetch_add(1, Ordering::Relaxed);
resources.skipped_urls.fetch_add(1, Ordering::Relaxed);
invoke_progress_callback(
progress_callback.as_ref(),
&resources.successful_urls,
&resources.failed_urls,
&resources.skipped_urls,
total_lines,
);
}
fn build_status_state(resources: &ScanResources) -> crate::status_server::StatusState {
crate::status_server::StatusState {
total_urls: Arc::clone(&resources.total_urls_in_file),
total_urls_attempted: Arc::clone(&resources.total_urls_attempted),
successful_urls: Arc::clone(&resources.successful_urls),
failed_urls: Arc::clone(&resources.failed_urls),
skipped_urls: Arc::clone(&resources.skipped_urls),
start_time: Arc::new(resources.start_time),
error_stats: Arc::clone(&resources.shared_ctx.runtime.error_stats),
timing_stats: Some(Arc::clone(&resources.shared_ctx.runtime.timing_stats)),
request_limiter: resources.request_limiter.as_ref().map(Arc::clone),
runtime_metrics: Arc::clone(&resources.shared_ctx.runtime.runtime_metrics),
run_id: Some(resources.run_id.clone()),
run_start_time_unix_secs: Some({
#[allow(clippy::cast_precision_loss)]
{
(resources.start_time_epoch as f64) / 1000.0
}
}),
phase: Arc::clone(&resources.phase),
throughput_window: Arc::clone(&resources.throughput_window),
max_concurrency: Some(resources.config.max_concurrency),
semaphore: Some(Arc::clone(&resources.semaphore)),
}
}
async fn drain_join_set(
resources: &ScanResources,
tasks: &mut JoinSet<()>,
cancel: &CancellationToken,
) {
let drain_timeout_secs = resources.config.drain_timeout_secs;
let drain_timeout = std::time::Duration::from_secs(drain_timeout_secs);
let drain_deadline = tokio::time::Instant::now() + drain_timeout;
loop {
let remaining = drain_deadline.saturating_duration_since(tokio::time::Instant::now());
if remaining.is_zero() {
let abandoned: Vec<String> = resources
.in_flight_urls
.lock()
.map(|set| set.iter().cloned().collect())
.unwrap_or_default();
let abandoned_count = abandoned.len();
if abandoned_count > 0 {
log::warn!(
"Drain timeout ({drain_timeout_secs}s) reached, recording {abandoned_count} in-flight task(s) as failures and aborting"
);
let inserted = record_drain_timeout_failures(
&resources.shared_ctx.pool,
&resources.run_id,
drain_timeout_secs,
&abandoned,
)
.await;
if inserted < abandoned_count {
log::warn!(
"Recorded {inserted}/{abandoned_count} drain-timeout failures (the rest hit DB errors; see preceding log lines)"
);
}
resources
.failed_urls
.fetch_add(abandoned_count, Ordering::Relaxed);
} else {
log::info!("Drain timeout ({drain_timeout_secs}s) reached, no remaining tasks");
}
tasks.abort_all();
cancel.cancel();
break;
}
match tokio::time::timeout(remaining, tasks.join_next()).await {
Ok(Some(task_result)) => {
if let Err(join_error) = task_result {
resources.failed_urls.fetch_add(1, Ordering::Relaxed);
log::warn!("Failed to join task (panicked): {join_error:?}");
}
}
Ok(None) => break,
#[allow(clippy::needless_continue)]
Err(_) => continue, }
}
}
#[allow(clippy::cognitive_complexity)] #[allow(clippy::too_many_lines)] pub async fn run_scan(
config: crate::config::Config,
) -> Result<ScanReport, crate::error_handling::RunScanError> {
let (resources, mut url_source, total_lines, progress_callback) = init_scan_resources(config)
.await
.map_err(|e| crate::error_handling::RunScanError::Startup(e.into()))?;
let mut status_server = if let Some(port) = resources.config.status_port {
Some(
crate::status_server::spawn_status_server(port, build_status_state(&resources))
.await
.map_err(|e| crate::error_handling::RunScanError::Startup(e.into()))?,
)
} else {
None
};
let cancel = CancellationToken::new();
let cancel_for_signal = cancel.clone();
tokio::spawn(async move {
if tokio::signal::ctrl_c().await.is_ok() {
cancel_for_signal.cancel();
}
});
let cancel_logging = cancel.child_token();
let successful_urls_for_logging = Arc::clone(&resources.successful_urls);
let failed_urls_for_logging = Arc::clone(&resources.failed_urls);
let skipped_urls_for_logging = Arc::clone(&resources.skipped_urls);
let total_urls_for_logging = Arc::clone(&resources.total_urls_in_file);
let start_time = resources.start_time;
let logging_interval_secs = if resources.config.status_port.is_none() {
LOGGING_INTERVAL as u64
} else {
STATUS_SERVER_LOGGING_INTERVAL_SECS
};
let mut logging_task = Some(tokio::task::spawn(async move {
let mut interval =
tokio::time::interval(std::time::Duration::from_secs(logging_interval_secs));
loop {
tokio::select! {
_ = interval.tick() => {
log_progress(
start_time,
&successful_urls_for_logging,
&failed_urls_for_logging,
&skipped_urls_for_logging,
Some(&total_urls_for_logging),
);
}
() = cancel_logging.cancelled() => {
break;
}
}
}
}));
let mut tasks: JoinSet<()> = JoinSet::new();
let mut consecutive_errors = 0;
const MAX_CONSECUTIVE_ERRORS: usize = 10;
loop {
while let Ok(Some(task_result)) =
tokio::time::timeout(std::time::Duration::ZERO, tasks.join_next()).await
{
if let Err(join_error) = task_result {
resources.failed_urls.fetch_add(1, Ordering::Relaxed);
log::warn!("Failed to join task (panicked): {join_error:?}");
}
}
tokio::select! {
line_result = url_source.next_line() => {
let line = match line_result {
Ok(Some(line)) => {
consecutive_errors = 0;
line
}
Ok(None) => break,
Err(e) => {
consecutive_errors += 1;
if consecutive_errors > MAX_CONSECUTIVE_ERRORS {
crate::app::shutdown_gracefully(
cancel.clone(),
logging_task.take(),
resources.rate_limiter_shutdown.clone(),
status_server.take(),
)
.await;
return Err(crate::error_handling::RunScanError::Runtime(
anyhow::anyhow!(
"Too many consecutive read errors ({consecutive_errors}): {e}"
),
));
}
warn!("Failed to read line from input: {e}");
continue;
}
};
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let Some(url) = validate_and_normalize_url(trimmed) else {
skip_url(&resources, &progress_callback, total_lines);
continue;
};
if !resources.config.allow_localhost_for_tests {
if let Err(e) = validate_url_safe(&url) {
warn!("Skipping SSRF-unsafe URL: {e}");
skip_url(&resources, &progress_callback, total_lines);
continue;
}
}
let permit = tokio::select! {
result = Arc::clone(&resources.semaphore).acquire_owned() => {
if let Ok(p) = result { p } else {
warn!("Semaphore closed, skipping URL: {url}");
resources
.total_urls_attempted
.fetch_add(1, Ordering::Relaxed);
resources
.failed_urls
.fetch_add(1, Ordering::Relaxed);
continue;
}
}
() = cancel.cancelled() => {
log::info!("Received interrupt (Ctrl-C), finishing current work and finalizing...");
break;
}
};
resources
.total_urls_attempted
.fetch_add(1, Ordering::Relaxed);
let task_params = UrlTaskParams {
url: Arc::from(url.as_str()),
ctx: Arc::clone(&resources.shared_ctx),
cancel: cancel.clone(),
permit,
request_limiter: resources.request_limiter.as_ref().map(Arc::clone),
host_limiter: resources.host_limiter.as_ref().map(Arc::clone),
successful_urls: Arc::clone(&resources.successful_urls),
skipped_urls: Arc::clone(&resources.skipped_urls),
failed_urls: Arc::clone(&resources.failed_urls),
total_urls_for_callback: total_lines,
progress_callback: progress_callback.clone(),
};
let registry = Arc::clone(&resources.in_flight_urls);
let url_for_registry = url.clone();
tasks.spawn(async move {
let _guard = InFlightGuard::register(registry, url_for_registry);
task::process_url_task(task_params).await;
});
}
() = cancel.cancelled() => {
log::info!("Received interrupt (Ctrl-C), finishing current work and finalizing...");
break;
}
}
}
resources
.phase
.set(crate::status_server::ScanPhase::Draining);
drain_join_set(&resources, &mut tasks, &cancel).await;
let loop_result = ScanLoopResult {
cancel,
logging_task,
status_server,
};
finalize::finalize_scan(resources, loop_result)
.await
.map_err(crate::error_handling::RunScanError::Runtime)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::{Config, FailOn, LogFormat, LogLevel};
use tempfile::NamedTempFile;
#[tokio::test]
async fn test_run_scan_validation_failure() {
let config = Config {
max_concurrency: 0, ..Default::default()
};
let result = run_scan(config).await;
assert!(result.is_err(), "Should fail with invalid configuration");
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains("Configuration validation failed")
|| error_msg.contains("max_concurrency")
|| error_msg.contains("greater than 0"),
"Expected validation error, got: {}",
error_msg
);
}
#[tokio::test]
async fn test_run_scan_file_not_found() {
let config = Config {
file: std::path::PathBuf::from("/nonexistent/file/that/does/not/exist.txt"),
..Default::default()
};
let result = run_scan(config).await;
assert!(result.is_err(), "Should fail when file doesn't exist");
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains("Failed to open input file")
|| error_msg.contains("No such file")
|| error_msg.contains("not found"),
"Expected file not found error, got: {}",
error_msg
);
}
#[tokio::test]
async fn test_run_scan_database_initialization_failure() {
let temp_file = NamedTempFile::new().expect("Failed to create temp file");
let db_path = temp_file.path().join("nested").join("domain_status.db");
let fp = NamedTempFile::new().expect("fp");
std::fs::write(fp.path(), "{}").expect("empty fingerprints");
let config = Config {
file: std::path::PathBuf::from("/dev/null"),
db_path,
fingerprints: Some(fp.path().to_string_lossy().into_owned()),
..Default::default()
};
let result = run_scan(config).await;
assert!(
result.is_err(),
"run_scan should fail when db_path parent is not a directory"
);
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains("database")
|| error_msg.contains("Database")
|| error_msg.contains("Failed to")
|| error_msg.contains("No such file")
|| error_msg.contains("os error")
|| error_msg.contains("unable to open")
|| error_msg.contains("not a directory"),
"Expected database init error, got: {error_msg}"
);
}
#[tokio::test]
async fn test_run_scan_empty_file() {
let temp_input = NamedTempFile::new().expect("Failed to create temp file");
let temp_db = NamedTempFile::new().expect("Failed to create temp DB");
let config = Config {
file: temp_input.path().to_path_buf(),
db_path: temp_db.path().to_path_buf(),
max_concurrency: 30,
timeout_seconds: 10,
rate_limit_rps: 15,
fail_on: FailOn::Never,
fail_on_pct_threshold: 10,
enable_whois: false,
cache_dir: None,
scan_external_scripts: false,
log_level: LogLevel::Info,
log_level_filter_override: None,
log_format: LogFormat::Plain,
user_agent: crate::config::DEFAULT_USER_AGENT.to_string(),
fingerprints: None,
geoip: None,
status_port: None,
log_file: None,
progress_callback: None,
dependency_overrides: None,
allow_localhost_for_tests: false,
drain_timeout_secs: 10,
allow_degraded_fingerprints: true,
};
let result = run_scan(config).await;
match result {
Ok(report) => {
assert_eq!(report.total_urls, 0);
assert_eq!(report.successful, 0);
assert_eq!(report.failed, 0);
assert_eq!(report.skipped, 0);
let pool =
sqlx::SqlitePool::connect(&format!("sqlite:{}", report.db_path.display()))
.await
.expect("connect to test db");
let row: (i64, i64, i64, i64) = sqlx::query_as(
"SELECT total_urls, successful_urls, failed_urls, skipped_urls FROM runs WHERE run_id = ?",
)
.bind(&report.run_id)
.fetch_one(&pool)
.await
.expect("run row should exist");
assert_eq!(
usize::try_from(row.0).expect("total_urls non-negative"),
report.total_urls,
"DB total_urls should match report"
);
assert_eq!(
usize::try_from(row.1).expect("successful_urls non-negative"),
report.successful,
"DB successful_urls should match report"
);
assert_eq!(
usize::try_from(row.2).expect("failed_urls non-negative"),
report.failed,
"DB failed_urls should match report"
);
assert_eq!(
usize::try_from(row.3).expect("skipped_urls non-negative"),
report.skipped,
"DB skipped_urls should match report"
);
}
Err(e) => {
let error_msg = e.to_string();
assert!(
error_msg.contains("database")
|| error_msg.contains("Database")
|| error_msg.contains("migration")
|| error_msg.contains("Failed to initialize"),
"Expected database/setup error for empty file test, got: {}",
error_msg
);
}
}
}
#[tokio::test]
async fn test_run_scan_file_with_comments() {
let temp_input = NamedTempFile::new().expect("Failed to create temp file");
std::fs::write(
temp_input.path(),
"# This is a comment\nhttps://example.com\n# Another comment\n",
)
.expect("Failed to write test file");
let temp_db = NamedTempFile::new().expect("Failed to create temp DB");
let fp_dir = tempfile::TempDir::new().expect("fp dir");
let fp_path = fp_dir.path().join("technologies.json");
std::fs::write(&fp_path, "{}").expect("empty fingerprints");
let config = Config {
file: temp_input.path().to_path_buf(),
db_path: temp_db.path().to_path_buf(),
max_concurrency: 1,
timeout_seconds: 10,
rate_limit_rps: 15,
fail_on: FailOn::Never,
fail_on_pct_threshold: 10,
enable_whois: false,
cache_dir: None,
scan_external_scripts: false,
log_level: LogLevel::Info,
log_level_filter_override: None,
log_format: LogFormat::Plain,
user_agent: crate::config::DEFAULT_USER_AGENT.to_string(),
fingerprints: Some(fp_path.to_string_lossy().into_owned()),
geoip: None,
status_port: None,
log_file: None,
progress_callback: None,
dependency_overrides: None,
allow_localhost_for_tests: false,
drain_timeout_secs: 10,
allow_degraded_fingerprints: false,
};
let report = run_scan(config)
.await
.expect("scan with local fingerprints must not fail init");
assert_eq!(
report.total_urls, 1,
"File with one URL and comments should result in one URL attempted"
);
}
}