domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
//! Scan finalization and cleanup.
//!
//! This module contains the `finalize_scan` function which handles
//! all cleanup and result aggregation after the main scan loop completes.

use std::sync::atomic::{AtomicUsize, Ordering};

use anyhow::{Context, Result};

use crate::app::statistics::{print_error_statistics, print_partial_failure_counts};
use crate::app::{log_progress, print_timing_statistics, shutdown_gracefully};
use crate::storage::insert::{
    count_run_fact_rows, count_run_partial_failures, saturating_i32_count,
};
use crate::storage::{update_run_stats, RunStats};

use super::{ScanLoopResult, ScanReport, ScanResources};

/// Finalize a scan run and produce the final report.
///
/// This function performs the following finalization steps:
/// 1. Shut down logging and rate limiter tasks
/// 2. Log final progress
/// 3. Update run statistics in database
/// 4. Checkpoint WAL file
/// 5. Close database pool
/// 6. Print error and timing statistics
/// 7. Construct and return the scan report
///
/// # Arguments
///
/// * `resources` - All scan resources (will be consumed)
/// * `loop_result` - Result from the scan loop (cancellation token, logging task)
///
/// # Returns
///
/// Returns a `ScanReport` with final statistics.
///
/// # Errors
///
/// Returns an error if database operations fail.
pub async fn finalize_scan(
    resources: ScanResources,
    loop_result: ScanLoopResult,
) -> Result<ScanReport> {
    resources
        .phase
        .set(crate::status_server::ScanPhase::Finalizing);

    let ScanLoopResult {
        cancel,
        logging_task,
        status_server,
    } = loop_result;

    // Shutdown background tasks
    shutdown_gracefully(
        cancel,
        logging_task,
        resources.rate_limiter_shutdown,
        status_server,
    )
    .await;

    // Log final progress against the input file total (not attempted-only).
    log_progress(
        resources.start_time,
        &resources.successful_urls,
        &resources.failed_urls,
        &resources.skipped_urls,
        Some(&resources.total_urls_in_file),
    );

    let elapsed_seconds = resources.start_time.elapsed().as_secs_f64();

    let FinalRunCounts {
        total_urls,
        successful_urls,
        failed_urls: failed_urls_count,
        skipped_urls: skipped_urls_count,
    } = persist_run_counts(
        resources.shared_ctx.pool.as_ref(),
        &resources.run_id,
        resources.total_urls_attempted.as_ref(),
        resources.successful_urls.as_ref(),
        resources.failed_urls.as_ref(),
        resources.skipped_urls.as_ref(),
        elapsed_seconds,
    )
    .await
    .context("Failed to update run statistics")?;

    let partials =
        count_run_partial_failures(resources.shared_ctx.pool.as_ref(), &resources.run_id)
            .await
            .context("Failed to count url_partial_failures for run")?;
    let live_partials = resources
        .shared_ctx
        .runtime
        .runtime_metrics
        .partial_failure_rows();
    let live_satellite = resources
        .shared_ctx
        .runtime
        .runtime_metrics
        .satellite_insert_errors();
    if i64::try_from(live_partials).unwrap_or(i64::MAX) != partials.total {
        log::warn!(
            "partial_failures atomic={} disagrees with url_partial_failures COUNT(*)={}; logging fact-table count",
            live_partials,
            partials.total
        );
    }
    if i64::try_from(live_satellite).unwrap_or(i64::MAX) != partials.satellite_insert_errors {
        log::warn!(
            "satellite_insert_errors atomic={} disagrees with COUNT(*)={}; logging fact-table count",
            live_satellite,
            partials.satellite_insert_errors
        );
    }
    print_partial_failure_counts(partials.total, partials.satellite_insert_errors);

    // Checkpoint WAL file for clean database state
    if let Err(e) = sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
        .execute(resources.shared_ctx.pool.as_ref())
        .await
    {
        log::warn!("Failed to checkpoint WAL file (this is non-critical): {e}");
    }

    // Close database pool
    resources.shared_ctx.pool.close().await;
    log::debug!("Database pool closed");

    // Print statistics
    print_error_statistics(&resources.shared_ctx.runtime.error_stats);

    let geoip_enabled = crate::geoip::is_enabled();
    print_timing_statistics(
        &resources.shared_ctx.runtime.timing_stats,
        Some(geoip_enabled),
        Some(resources.config.enable_whois),
    );

    // SAFETY: Cast from i32 back to usize for API consistency is safe here.
    // These values came from usize counters and were cast to i32 for database storage.
    // Sign loss cannot occur because URL counts are always non-negative.
    #[allow(clippy::cast_sign_loss)]
    Ok(ScanReport {
        total_urls: total_urls as usize,
        #[allow(clippy::cast_sign_loss)]
        successful: successful_urls as usize,
        #[allow(clippy::cast_sign_loss)]
        failed: failed_urls_count as usize,
        #[allow(clippy::cast_sign_loss)]
        skipped: skipped_urls_count as usize,
        db_path: resources.config.db_path.clone(),
        run_id: resources.run_id,
        elapsed_seconds,
    })
}

struct FinalRunCounts {
    total_urls: i32,
    successful_urls: i32,
    failed_urls: i32,
    skipped_urls: i32,
}

/// Persist `runs` counters. Success/failure come from fact-table `COUNT(*)`;
/// skipped stays on the in-memory atomic (duplicate-domain skips never insert).
async fn persist_run_counts(
    pool: &sqlx::SqlitePool,
    run_id: &str,
    total_urls_attempted: &AtomicUsize,
    successful_urls: &AtomicUsize,
    failed_urls: &AtomicUsize,
    skipped_urls: &AtomicUsize,
    elapsed_seconds: f64,
) -> Result<FinalRunCounts> {
    // SAFETY: Cast from usize to i32 for database storage is acceptable here.
    // These casts represent URL counts processed in a single scan run:
    // 1. Practical limits: Even at 10,000 URLs/sec, processing 2.1B URLs would take 60+ hours
    // 2. Memory constraints: Processing billions of URLs would exhaust system memory first
    // 3. Database schema: SQLite uses INTEGER (i32) for these columns
    // 4. Realistic usage: Typical production runs process 100K-10M URLs, well within i32 range
    #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
    let total_urls = total_urls_attempted.load(Ordering::SeqCst) as i32;
    #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
    let atomic_successful = successful_urls.load(Ordering::SeqCst) as i32;
    #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
    let atomic_failed = failed_urls.load(Ordering::SeqCst) as i32;
    #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
    let skipped_urls = skipped_urls.load(Ordering::SeqCst) as i32;

    let facts = count_run_fact_rows(pool, run_id)
        .await
        .context("Failed to count url_status / url_failures for run")?;
    let successful_urls = saturating_i32_count(facts.successful_urls);
    let failed_urls = saturating_i32_count(facts.failed_urls);

    if facts.successful_urls != i64::from(atomic_successful) {
        log::warn!(
            "successful_urls atomic={} disagrees with url_status COUNT(*)={}; persisting fact-table count",
            atomic_successful,
            facts.successful_urls
        );
    }
    if facts.failed_urls != i64::from(atomic_failed) {
        log::warn!(
            "failed_urls atomic={} disagrees with url_failures COUNT(*)={}; persisting fact-table count",
            atomic_failed,
            facts.failed_urls
        );
    }

    let stats = RunStats {
        run_id,
        total_urls,
        successful_urls,
        failed_urls,
        skipped_urls,
        elapsed_seconds,
    };
    update_run_stats(pool, &stats)
        .await
        .context("Failed to update run statistics")?;

    Ok(FinalRunCounts {
        total_urls,
        successful_urls,
        failed_urls,
        skipped_urls,
    })
}

#[cfg(test)]
mod tests {
    use std::sync::atomic::AtomicUsize;
    use std::sync::Arc;

    use crate::initialization::test_resolver;
    use tokio::sync::Semaphore;
    use tokio_util::sync::CancellationToken;

    use crate::config::Config;
    use crate::error_handling::ProcessingStats;
    use crate::fetch::{NetworkContext, ProcessingContext, RuntimeContext};
    use crate::fingerprint::FingerprintRuleset;
    use crate::runtime_metrics::RuntimeMetrics;
    use crate::storage::test_helpers::create_test_pool as create_test_sqlite_pool;
    use crate::storage::test_helpers::create_test_url_status;
    use crate::storage::{insert_run_metadata, RunMetadata};
    use crate::utils::TimingStats;

    use super::*;

    async fn create_test_pool() -> crate::storage::DbPool {
        Arc::new(create_test_sqlite_pool().await)
    }

    async fn insert_test_run(pool: &sqlx::SqlitePool, run_id: &str) {
        insert_run_metadata(
            pool,
            &RunMetadata {
                run_id,
                start_time_ms: 1_704_067_200_000_i64,
                version: "0.1.0",
                fingerprints_source: Some("test"),
                fingerprints_version: Some("0"),
                geoip_version: None,
            },
        )
        .await
        .expect("insert run metadata");
    }

    async fn seed_url_status_rows(pool: &sqlx::SqlitePool, run_id: &str, count: usize) {
        for i in 0..count {
            let domain = format!("ok{i}.example.com");
            create_test_url_status(pool, &domain, &domain, 200, Some(run_id), 1).await;
        }
    }

    async fn seed_url_failure_rows(pool: &sqlx::SqlitePool, run_id: &str, count: usize) {
        for i in 0..count {
            let domain = format!("fail{i}.example.com");
            sqlx::query(
                "INSERT INTO url_failures (
                    attempted_url, initial_domain, error_type, error_message,
                    retry_count, observed_at_ms, run_id
                ) VALUES (?, ?, 'timeout', 'timed out', 0, 1, ?)",
            )
            .bind(format!("https://{domain}/"))
            .bind(&domain)
            .bind(run_id)
            .execute(pool)
            .await
            .expect("insert url_failures row");
        }
    }

    async fn finalize_resources(
        pool: crate::storage::DbPool,
        run_id: &str,
        attempted: usize,
        atomic_success: usize,
        atomic_failed: usize,
        atomic_skipped: usize,
    ) -> (ScanResources, ScanLoopResult) {
        let start_time_ms = 1_704_067_200_000_i64;
        let client = Arc::new(reqwest::Client::builder().build().expect("http client"));
        let redirect_client = Arc::new(
            reqwest::Client::builder()
                .redirect(reqwest::redirect::Policy::none())
                .build()
                .expect("redirect client"),
        );
        let error_stats = Arc::new(ProcessingStats::new());
        let timing_stats = Arc::new(TimingStats::new());
        let shared_ctx = Arc::new(ProcessingContext::new(
            NetworkContext::new(client, redirect_client, test_resolver()),
            Arc::clone(&pool),
            RuntimeContext::new(
                Arc::clone(&error_stats),
                Arc::clone(&timing_stats),
                Some(run_id.to_string()),
                false,
                std::path::PathBuf::from("/tmp/domain_status_whois_test"),
                false,
                Arc::new(RuntimeMetrics::default()),
                true,
            ),
            Arc::new(FingerprintRuleset::empty_for_tests()),
        ));

        let resources = ScanResources {
            shared_ctx,
            semaphore: Arc::new(Semaphore::new(1)),
            request_limiter: None,
            host_limiter: None,
            rate_limiter_shutdown: None,
            in_flight_urls: Arc::new(std::sync::Mutex::new(std::collections::HashSet::new())),
            successful_urls: Arc::new(AtomicUsize::new(atomic_success)),
            skipped_urls: Arc::new(AtomicUsize::new(atomic_skipped)),
            failed_urls: Arc::new(AtomicUsize::new(atomic_failed)),
            total_urls_attempted: Arc::new(AtomicUsize::new(attempted)),
            total_urls_in_file: Arc::new(AtomicUsize::new(attempted)),
            phase: Arc::new(crate::status_server::AtomicPhase::default()),
            throughput_window: Arc::new(crate::status_server::ThroughputWindow::new()),
            run_id: run_id.to_string(),
            start_time_epoch: start_time_ms,
            start_time: std::time::Instant::now(),
            _geoip_metadata: None,
            config: Config {
                db_path: std::path::PathBuf::from(":memory:"),
                enable_whois: false,
                cache_dir: None,
                scan_external_scripts: false,
                ..Config::default()
            },
        };
        let loop_result = ScanLoopResult {
            cancel: CancellationToken::new(),
            logging_task: None,
            status_server: None,
        };
        (resources, loop_result)
    }

    /// `finalize_scan` returns a `ScanReport` with fact-table success/failure counts.
    #[tokio::test]
    async fn test_finalize_scan_returns_correct_report() {
        let pool = create_test_pool().await;
        let run_id = "run_finalize_test_123";
        insert_test_run(pool.as_ref(), run_id).await;
        seed_url_status_rows(pool.as_ref(), run_id, 8).await;
        seed_url_failure_rows(pool.as_ref(), run_id, 2).await;
        let (resources, loop_result) = finalize_resources(pool, run_id, 10, 8, 2, 0).await;

        let report = finalize_scan(resources, loop_result)
            .await
            .expect("finalize_scan should succeed");

        assert_eq!(report.total_urls, 10);
        assert_eq!(report.successful, 8);
        assert_eq!(report.failed, 2);
        assert_eq!(report.skipped, 0);
        assert_eq!(report.run_id, run_id);
        assert!(report.elapsed_seconds >= 0.0);
    }

    #[tokio::test]
    async fn test_finalize_scan_persists_fact_counts_when_atomics_disagree() {
        let pool = create_test_pool().await;
        let run_id = "run_finalize_disagree";
        insert_test_run(pool.as_ref(), run_id).await;
        seed_url_status_rows(pool.as_ref(), run_id, 3).await;
        seed_url_failure_rows(pool.as_ref(), run_id, 1).await;
        let (resources, loop_result) = finalize_resources(pool, run_id, 10, 8, 2, 4).await;

        let report = finalize_scan(resources, loop_result)
            .await
            .expect("finalize_scan should succeed");

        assert_eq!(report.total_urls, 10);
        assert_eq!(report.successful, 3);
        assert_eq!(report.failed, 1);
        assert_eq!(report.skipped, 4);
    }
}