use std::sync::atomic::Ordering;
use std::sync::Arc;
use crate::config::{RETRY_MAX_ATTEMPTS, URL_PROCESSING_TIMEOUT};
use crate::error_handling::ErrorType;
use crate::fetch::UrlProcessOutcome;
use crate::initialization::{HostRateLimiter, RateLimiter};
use crate::storage::failure::record_url_failure;
use crate::utils::ProcessUrlResult;
use super::invoke_progress_callback;
use super::resources::{ProgressCallback, UrlTaskParams};
use tokio_util::sync::CancellationToken;
struct TaskProgress<'a> {
successful_urls: &'a Arc<std::sync::atomic::AtomicUsize>,
skipped_urls: &'a Arc<std::sync::atomic::AtomicUsize>,
failed_urls: &'a Arc<std::sync::atomic::AtomicUsize>,
total_urls_for_callback: usize,
progress_callback: &'a ProgressCallback,
}
pub async fn process_url_task(params: UrlTaskParams) {
let UrlTaskParams {
url,
ctx,
cancel,
permit: _permit, request_limiter,
host_limiter,
successful_urls,
skipped_urls,
failed_urls,
total_urls_for_callback,
progress_callback,
} = params;
let progress = TaskProgress {
successful_urls: &successful_urls,
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback,
progress_callback: &progress_callback,
};
if cancel.is_cancelled() {
handle_cancelled(&url, &ctx, &progress).await;
return;
}
if !wait_for_rate_limits(
url.as_ref(),
request_limiter.as_ref(),
host_limiter.as_ref(),
&cancel,
)
.await
{
handle_cancelled(&url, &ctx, &progress).await;
return;
}
if cancel.is_cancelled() {
handle_cancelled(&url, &ctx, &progress).await;
return;
}
let process_start = std::time::Instant::now();
let url_for_logging = Arc::clone(&url);
let result = tokio::select! {
r = tokio::time::timeout(
URL_PROCESSING_TIMEOUT,
crate::utils::process_url(url, ctx.clone()),
) => r,
() = cancel.cancelled() => {
handle_cancelled(&url_for_logging, &ctx, &progress).await;
return;
}
};
match result {
Ok(ProcessUrlResult {
result: Ok(outcome),
..
}) => handle_success(&url_for_logging, outcome, &progress),
Ok(ProcessUrlResult {
result: Err(e),
retry_count,
}) => {
handle_failure(
&url_for_logging,
e,
retry_count,
process_start,
&ctx,
&progress,
)
.await;
}
Err(_) => handle_timeout(&url_for_logging, process_start, &ctx, &progress).await,
}
}
async fn wait_for_rate_limits(
url: &str,
request_limiter: Option<&Arc<RateLimiter>>,
host_limiter: Option<&Arc<HostRateLimiter>>,
cancel: &CancellationToken,
) -> bool {
if let Some(limiter) = request_limiter {
tokio::select! {
() = limiter.acquire() => {}
() = cancel.cancelled() => return false,
}
}
if let Some(limiter) = host_limiter {
tokio::select! {
() = limiter.acquire(url) => {}
() = cancel.cancelled() => return false,
}
}
true
}
async fn handle_cancelled(
url: &Arc<str>,
ctx: &Arc<crate::fetch::ProcessingContext>,
progress: &TaskProgress<'_>,
) {
super::record_cooperative_cancel_failure(
&ctx.pool,
ctx.runtime.run_id.as_deref(),
url.as_ref(),
)
.await;
progress.failed_urls.fetch_add(1, Ordering::Relaxed);
invoke_progress_callback(
progress.progress_callback.as_ref(),
progress.successful_urls,
progress.failed_urls,
progress.skipped_urls,
progress.total_urls_for_callback,
);
ctx.runtime
.error_stats
.increment_error(ErrorType::ScanCancelled);
}
fn handle_success(_url: &Arc<str>, outcome: UrlProcessOutcome, progress: &TaskProgress<'_>) {
match outcome {
UrlProcessOutcome::Inserted => {
progress.successful_urls.fetch_add(1, Ordering::Relaxed);
}
UrlProcessOutcome::Updated => {
progress.skipped_urls.fetch_add(1, Ordering::Relaxed);
}
UrlProcessOutcome::Skipped => {
progress.skipped_urls.fetch_add(1, Ordering::Relaxed);
}
}
invoke_progress_callback(
progress.progress_callback.as_ref(),
progress.successful_urls,
progress.failed_urls,
progress.skipped_urls,
progress.total_urls_for_callback,
);
}
async fn handle_failure(
url: &Arc<str>,
error: anyhow::Error,
retry_count: u32,
process_start: std::time::Instant,
ctx: &Arc<crate::fetch::ProcessingContext>,
progress: &TaskProgress<'_>,
) {
progress.failed_urls.fetch_add(1, Ordering::Relaxed);
let elapsed = process_start.elapsed().as_secs_f64();
invoke_progress_callback(
progress.progress_callback.as_ref(),
progress.successful_urls,
progress.failed_urls,
progress.skipped_urls,
progress.total_urls_for_callback,
);
log::warn!("Failed to process URL {}: {error}", url.as_ref());
let context = crate::storage::failure::extract_failure_context(&error);
record_url_failure(crate::storage::failure::FailureRecordParams {
pool: &ctx.pool,
url: url.as_ref(),
error: &error,
context,
retry_count,
elapsed_time: elapsed,
run_id: ctx.runtime.run_id.as_deref(),
})
.await;
}
async fn handle_timeout(
url: &Arc<str>,
process_start: std::time::Instant,
ctx: &Arc<crate::fetch::ProcessingContext>,
progress: &TaskProgress<'_>,
) {
progress.failed_urls.fetch_add(1, Ordering::Relaxed);
let elapsed = process_start.elapsed().as_secs_f64();
invoke_progress_callback(
progress.progress_callback.as_ref(),
progress.successful_urls,
progress.failed_urls,
progress.skipped_urls,
progress.total_urls_for_callback,
);
log::warn!(
"Failed to process URL {} (timeout after {}s)",
url.as_ref(),
URL_PROCESSING_TIMEOUT.as_secs()
);
let timeout_error = anyhow::anyhow!(
"Process URL timeout after {} seconds for {}",
URL_PROCESSING_TIMEOUT.as_secs(),
url.as_ref()
);
let context = crate::storage::failure::FailureContext {
final_url: None,
redirect_chain: Vec::new(),
response_headers: Vec::new(),
};
#[allow(clippy::cast_possible_truncation)]
record_url_failure(crate::storage::failure::FailureRecordParams {
pool: &ctx.pool,
url: url.as_ref(),
error: &timeout_error,
context,
retry_count: RETRY_MAX_ATTEMPTS as u32 - 1,
elapsed_time: elapsed,
run_id: ctx.runtime.run_id.as_deref(),
})
.await;
ctx.runtime
.error_stats
.increment_error(ErrorType::ProcessUrlTimeout);
}
#[cfg(test)]
mod tests {
use super::super::resources::UrlTaskParams;
use super::*;
use crate::error_handling::ProcessingStats;
use crate::fetch::{NetworkContext, ProcessingContext, RuntimeContext};
use crate::initialization::test_resolver;
use crate::utils::TimingStats;
use std::sync::atomic::AtomicUsize;
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken;
async fn minimal_ctx_with_migrations() -> Arc<ProcessingContext> {
let pool = Arc::new(
sqlx::SqlitePool::connect("sqlite::memory:")
.await
.expect("test pool"),
);
crate::storage::run_migrations(pool.as_ref())
.await
.expect("migrations");
sqlx::query("INSERT OR IGNORE INTO runs (run_id, start_time_ms) VALUES (?, ?)")
.bind("run-1")
.bind(chrono::Utc::now().timestamp_millis())
.execute(pool.as_ref())
.await
.expect("insert run");
let client = Arc::new(reqwest::Client::builder().build().expect("test client"));
let redirect_client = Arc::new(
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("test redirect client"),
);
let ctx = ProcessingContext::new(
NetworkContext::new(client, redirect_client, test_resolver()),
pool,
RuntimeContext::new(
Arc::new(ProcessingStats::new()),
Arc::new(TimingStats::new()),
Some("run-1".to_string()),
false,
std::path::PathBuf::from("/tmp/domain_status_whois_test"),
false,
Arc::new(crate::runtime_metrics::RuntimeMetrics::default()),
true,
),
Arc::new(crate::fingerprint::FingerprintRuleset::empty_for_tests()),
);
Arc::new(ctx)
}
#[tokio::test]
async fn test_handle_success_counts_inserted_url_as_successful() {
let url: Arc<str> = Arc::from("https://example.com");
let successful_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let failed_urls = Arc::new(AtomicUsize::new(0));
let progress = TaskProgress {
successful_urls: &successful_urls,
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &None,
};
handle_success(&url, UrlProcessOutcome::Inserted, &progress);
assert_eq!(successful_urls.load(Ordering::SeqCst), 1);
assert_eq!(skipped_urls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn test_handle_success_counts_skipped_url_separately() {
let url: Arc<str> = Arc::from("https://example.com");
let successful_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let failed_urls = Arc::new(AtomicUsize::new(0));
let progress = TaskProgress {
successful_urls: &successful_urls,
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &None,
};
handle_success(&url, UrlProcessOutcome::Skipped, &progress);
assert_eq!(successful_urls.load(Ordering::SeqCst), 0);
assert_eq!(skipped_urls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_handle_success_counts_updated_as_skipped_for_accounting() {
let url: Arc<str> = Arc::from("https://example.com");
let successful_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let failed_urls = Arc::new(AtomicUsize::new(0));
let progress = TaskProgress {
successful_urls: &successful_urls,
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 2,
progress_callback: &None,
};
handle_success(&url, UrlProcessOutcome::Updated, &progress);
assert_eq!(successful_urls.load(Ordering::SeqCst), 0);
assert_eq!(skipped_urls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_handle_failure_increments_failed_urls_and_invokes_callback() {
let url: Arc<str> = Arc::from("https://example.com/fail");
let failed_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let ctx = minimal_ctx_with_migrations().await;
let progress_calls = Arc::new(AtomicUsize::new(0));
let callback: ProgressCallback = Some(Arc::new({
let progress_calls = Arc::clone(&progress_calls);
move |completed, failed, _skipped, _total| {
progress_calls.store(completed + failed, Ordering::SeqCst);
}
}));
let progress = TaskProgress {
successful_urls: &Arc::new(AtomicUsize::new(0)),
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &callback,
};
let err = anyhow::anyhow!("simulated failure");
handle_failure(&url, err, 0, std::time::Instant::now(), &ctx, &progress).await;
assert_eq!(failed_urls.load(Ordering::SeqCst), 1);
assert_eq!(progress_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_handle_failure_with_429_error_completes_without_panic() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(429))
.mount(&server)
.await;
let resp = reqwest::get(server.uri()).await.unwrap();
let err = resp.error_for_status().unwrap_err();
let anyhow_err = anyhow::anyhow!(err);
let url: Arc<str> = Arc::from(server.uri().as_str());
let failed_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let ctx = minimal_ctx_with_migrations().await;
let progress = TaskProgress {
successful_urls: &Arc::new(AtomicUsize::new(0)),
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &None,
};
handle_failure(
&url,
anyhow_err,
1,
std::time::Instant::now(),
&ctx,
&progress,
)
.await;
assert_eq!(failed_urls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_handle_failure_with_non_429_error_does_not_panic() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("GET"))
.respond_with(wiremock::ResponseTemplate::new(500))
.mount(&server)
.await;
let resp = reqwest::get(server.uri()).await.unwrap();
let err = resp.error_for_status().unwrap_err();
let anyhow_err = anyhow::anyhow!(err);
let url: Arc<str> = Arc::from(server.uri().as_str());
let failed_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let ctx = minimal_ctx_with_migrations().await;
let progress = TaskProgress {
successful_urls: &Arc::new(AtomicUsize::new(0)),
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &None,
};
handle_failure(
&url,
anyhow_err,
0,
std::time::Instant::now(),
&ctx,
&progress,
)
.await;
assert_eq!(failed_urls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_handle_timeout_increments_failed_urls_and_error_stats() {
let url: Arc<str> = Arc::from("https://example.com/timeout");
let failed_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let ctx = minimal_ctx_with_migrations().await;
let progress_calls = Arc::new(AtomicUsize::new(0));
let callback: ProgressCallback = Some(Arc::new({
let progress_calls = Arc::clone(&progress_calls);
move |completed, failed, _skipped, _total| {
progress_calls.store(completed + failed, Ordering::SeqCst);
}
}));
let progress = TaskProgress {
successful_urls: &Arc::new(AtomicUsize::new(0)),
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &callback,
};
handle_timeout(&url, std::time::Instant::now(), &ctx, &progress).await;
assert_eq!(failed_urls.load(Ordering::SeqCst), 1);
assert_eq!(progress_calls.load(Ordering::SeqCst), 1);
assert_eq!(
ctx.runtime
.error_stats
.get_error_count(crate::error_handling::ErrorType::ProcessUrlTimeout),
1
);
}
#[tokio::test]
async fn test_handle_failure_records_failure_when_db_ok() {
let url: Arc<str> = Arc::from("https://example.com/record-fail");
let failed_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let ctx = minimal_ctx_with_migrations().await;
let progress_calls = Arc::new(AtomicUsize::new(0));
let callback: ProgressCallback = Some(Arc::new({
let progress_calls = Arc::clone(&progress_calls);
move |_completed, failed, _skipped, _total| {
progress_calls.store(failed, Ordering::SeqCst);
}
}));
let progress = TaskProgress {
successful_urls: &Arc::new(AtomicUsize::new(0)),
skipped_urls: &skipped_urls,
failed_urls: &failed_urls,
total_urls_for_callback: 1,
progress_callback: &callback,
};
let err = anyhow::anyhow!("simulated failure");
handle_failure(&url, err, 0, std::time::Instant::now(), &ctx, &progress).await;
assert_eq!(failed_urls.load(Ordering::SeqCst), 1);
assert_eq!(progress_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_process_url_task_cancel_persists_url_failure() {
let ctx = minimal_ctx_with_migrations().await;
let cancel = CancellationToken::new();
cancel.cancel();
let semaphore = Arc::new(Semaphore::new(1));
let permit = semaphore.acquire_owned().await.expect("permit");
let successful_urls = Arc::new(AtomicUsize::new(0));
let skipped_urls = Arc::new(AtomicUsize::new(0));
let failed_urls = Arc::new(AtomicUsize::new(0));
process_url_task(UrlTaskParams {
url: Arc::from("https://example.com/cancelled"),
ctx: Arc::clone(&ctx),
cancel,
permit,
request_limiter: None,
host_limiter: None,
successful_urls,
skipped_urls,
failed_urls: Arc::clone(&failed_urls),
total_urls_for_callback: 1,
progress_callback: None,
})
.await;
assert_eq!(failed_urls.load(Ordering::SeqCst), 1);
assert_eq!(
ctx.runtime
.error_stats
.get_error_count(ErrorType::ScanCancelled),
1
);
let count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM url_failures WHERE error_type = ? AND attempted_url = ?",
)
.bind(ErrorType::ScanCancelled.as_str())
.bind("https://example.com/cancelled")
.fetch_one(ctx.pool.as_ref())
.await
.expect("count url_failures");
assert_eq!(count, 1, "cooperative cancel must leave a url_failures row");
}
}