domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
//! Error categorization and retry strategy.
//!
//! This module provides functions to categorize errors and configure retry strategies.

use std::time::Duration;

use super::stats::ProcessingStats;
use super::types::ErrorType;

/// Exponential backoff capped at `max_delay`, limited to `remaining` yields.
struct ExponentialBackoff {
    current_ms: u64,
    factor: u64,
    max_delay: Duration,
    remaining: usize,
}

impl Iterator for ExponentialBackoff {
    type Item = Duration;

    fn next(&mut self) -> Option<Duration> {
        if self.remaining == 0 {
            return None;
        }
        self.remaining -= 1;
        let delay = Duration::from_millis(self.current_ms).min(self.max_delay);
        self.current_ms = self.current_ms.saturating_mul(self.factor);
        Some(delay)
    }
}

/// Creates an exponential backoff retry strategy.
///
/// Returns a retry strategy configured with:
/// - Initial delay: `RETRY_INITIAL_DELAY_MS` milliseconds
/// - Backoff factor: `RETRY_FACTOR` (doubles delay each retry)
/// - Maximum delay: `RETRY_MAX_DELAY_SECS` seconds
/// - Maximum attempts: `RETRY_MAX_ATTEMPTS` (prevents infinite retries)
///
/// The iterator is limited to `RETRY_MAX_ATTEMPTS` items to prevent
/// infinite retries and keep the budget inside `URL_PROCESSING_TIMEOUT`.
pub fn get_retry_strategy() -> impl Iterator<Item = Duration> {
    ExponentialBackoff {
        current_ms: crate::config::RETRY_INITIAL_DELAY_MS,
        factor: crate::config::RETRY_FACTOR,
        max_delay: Duration::from_secs(crate::config::RETRY_MAX_DELAY_SECS),
        remaining: crate::config::RETRY_MAX_ATTEMPTS,
    }
}

/// Categorizes a `reqwest::Error` into an `ErrorType`.
///
/// This is the unified error categorization logic used by both
/// `update_error_stats` and `extract_error_type` to ensure consistency.
///
/// # Arguments
///
/// * `error` - The `reqwest::Error` to categorize
///
/// # Returns
///
/// The appropriate `ErrorType` for the error.
pub fn categorize_reqwest_error(error: &reqwest::Error) -> ErrorType {
    // Check HTTP status codes first
    if let Some(status) = error.status() {
        match status.as_u16() {
            // Client errors (4xx) - track common ones specifically
            400 => return ErrorType::HttpRequestBadRequest,
            401 => return ErrorType::HttpRequestUnauthorized,
            403 => return ErrorType::HttpRequestBotDetectionError,
            404 => return ErrorType::HttpRequestNotFound,
            429 => return ErrorType::HttpRequestTooManyRequests,
            // Server errors (5xx) - track common ones specifically
            500 => return ErrorType::HttpRequestInternalServerError,
            502 => return ErrorType::HttpRequestBadGateway,
            503 => return ErrorType::HttpRequestServiceUnavailable,
            504 => return ErrorType::HttpRequestGatewayTimeout,
            // Less common status codes (406, 521, etc.) fall through to generic categories
            // Other client errors (4xx) - use generic format
            _ if status.is_client_error() => {
                return ErrorType::HttpRequestOtherError;
            }
            // Other server errors (5xx) - use generic format
            _ if status.is_server_error() => {
                return ErrorType::HttpRequestOtherError;
            }
            _ => {
                // Non-standard status codes - fall through to check error type
            }
        }
    }

    // Check reqwest error types
    if error.is_builder() {
        ErrorType::HttpRequestBuilderError
    } else if error.is_redirect() {
        ErrorType::HttpRequestRedirectError
    } else if error.is_status() {
        ErrorType::HttpRequestStatusError
    } else if error.is_timeout() {
        ErrorType::HttpRequestTimeoutError
    } else if error.is_request() {
        ErrorType::HttpRequestRequestError
    } else if error.is_connect() {
        ErrorType::HttpRequestConnectError
    } else if error.is_body() {
        ErrorType::HttpRequestBodyError
    } else if error.is_decode() {
        ErrorType::HttpRequestDecodeError
    } else {
        ErrorType::HttpRequestOtherError
    }
}

/// Updates processing statistics based on a `reqwest::Error`.
///
/// Analyzes the error and increments the appropriate `ErrorType` counter.
/// Handles both HTTP status errors (e.g., 429 Too Many Requests) and network-level
/// errors (timeouts, connection failures, etc.).
///
/// # Arguments
///
/// * `stats` - The processing statistics tracker to update
/// * `error` - The `reqwest::Error` to categorize and record
pub fn update_error_stats(stats: &ProcessingStats, error: &reqwest::Error) {
    use super::ReqwestErrorExt;
    stats.increment_error(error.categorize());
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::error_handling::ReqwestErrorExt;
    use std::time::Duration;

    #[test]
    fn test_get_retry_strategy_initial_delay() {
        let strategy = get_retry_strategy();
        let first_delay = strategy
            .take(1)
            .next()
            .expect("Retry strategy should always yield at least one delay");

        // First delay should be RETRY_INITIAL_DELAY_MS (or capped at max).
        let expected_ms = u128::from(crate::config::RETRY_INITIAL_DELAY_MS);
        let actual_ms = first_delay.as_millis();
        assert!(
            actual_ms >= expected_ms,
            "Expected delay >= {}ms, got {}ms",
            expected_ms,
            actual_ms
        );
    }

    #[test]
    fn test_get_retry_strategy_exponential_backoff() {
        let strategy = get_retry_strategy();
        let delays: Vec<Duration> = strategy.take(5).collect();

        // Verify delays increase (exponential backoff or capped at max)
        for i in 1..delays.len() {
            let prev = delays[i - 1].as_millis();
            let curr = delays[i].as_millis();
            // Delay should increase (or stay at max)
            assert!(curr >= prev, "Delay should increase: {} >= {}", curr, prev);

            // If not at max, should be approximately double
            let max_delay_ms = u128::from(crate::config::RETRY_MAX_DELAY_SECS * 1000);
            if curr < max_delay_ms {
                // SAFETY: Cast u128 to f64 for retry backoff ratio test
                // - Duration delays are in milliseconds (typically 10-5000ms for retries)
                // - Max delay is RETRY_MAX_DELAY_SECS (config) * 1000, typically 5000ms
                // - f64 has 53 bits of precision, can exactly represent integers up to 2^53
                // - Test delays (< 10^6 ms) are well within f64 precision range
                // - This is test-only code for ratio validation
                // - Precision loss for delays > 2^53 ms is acceptable (test would still work)
                #[allow(clippy::cast_precision_loss)]
                let ratio = curr as f64 / prev as f64;
                // Allow wide tolerance - backoff doubles each step until max_delay
                assert!(
                    (1.0..=3.0).contains(&ratio),
                    "Backoff factor should be reasonable: {} / {} = {}",
                    curr,
                    prev,
                    ratio
                );
            }
        }
    }

    #[test]
    fn test_get_retry_strategy_max_delay() {
        let strategy = get_retry_strategy();
        let max_delay_ms = crate::config::RETRY_MAX_DELAY_SECS * 1000;

        // All delays should be <= max_delay
        for delay in strategy {
            assert!(
                delay.as_millis() <= u128::from(max_delay_ms),
                "Delay {}ms exceeds max {}ms",
                delay.as_millis(),
                max_delay_ms
            );
        }
    }

    #[test]
    fn test_get_retry_strategy_max_attempts() {
        let strategy = get_retry_strategy();
        let count = strategy.count();

        // Should be limited to RETRY_MAX_ATTEMPTS
        assert_eq!(count, crate::config::RETRY_MAX_ATTEMPTS);
    }

    #[tokio::test]
    async fn test_update_error_stats() {
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let stats = ProcessingStats::new();
        let server = Server::run();

        // Test with 404 error
        server.expect(
            Expectation::matching(request::method_path("GET", "/404"))
                .respond_with(status_code(404).body("Not Found")),
        );

        let client = reqwest::Client::new();
        let url = format!("http://{}/404", server.addr());
        let response = client.get(&url).send().await.unwrap();
        let error = response.error_for_status().unwrap_err();

        update_error_stats(&stats, &error);

        assert_eq!(
            stats.get_error_count(ErrorType::HttpRequestNotFound),
            1,
            "404 error should increment HttpRequestNotFound counter"
        );
    }

    // Test individual status codes (using separate tests to avoid lifetime issues).
    // Each test asserts both categorization and retriability contract (ReqwestErrorExt::is_retriable).
    #[tokio::test]
    async fn test_categorize_reqwest_error_400() {
        test_status_code(400, ErrorType::HttpRequestBadRequest, false).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_401() {
        test_status_code(401, ErrorType::HttpRequestUnauthorized, false).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_403() {
        test_status_code(403, ErrorType::HttpRequestBotDetectionError, false).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_404() {
        test_status_code(404, ErrorType::HttpRequestNotFound, false).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_429() {
        test_status_code(429, ErrorType::HttpRequestTooManyRequests, true).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_500() {
        test_status_code(500, ErrorType::HttpRequestInternalServerError, true).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_502() {
        test_status_code(502, ErrorType::HttpRequestBadGateway, true).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_503() {
        test_status_code(503, ErrorType::HttpRequestServiceUnavailable, true).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_504() {
        test_status_code(504, ErrorType::HttpRequestGatewayTimeout, true).await;
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_521() {
        // 521 (Cloudflare) is now categorized as HttpRequestOtherError (server error)
        test_status_code(521, ErrorType::HttpRequestOtherError, true).await;
    }

    // Helper function to test a single status code (categorization + retriability contract).
    async fn test_status_code(code: u16, expected_error_type: ErrorType, expected_retriable: bool) {
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let server = Server::run();
        let client = reqwest::Client::new();

        // Use a static path pattern to avoid lifetime issues
        let path = match code {
            400 => "/400",
            401 => "/401",
            403 => "/403",
            404 => "/404",
            429 => "/429",
            500 => "/500",
            502 => "/502",
            503 => "/503",
            504 => "/504",
            521 => "/521",
            _ => panic!("Unsupported status code in test"),
        };

        server.expect(
            Expectation::matching(request::method_path("GET", path))
                .respond_with(status_code(code).body("Error")),
        );

        let url = format!("http://{}{}", server.addr(), path);
        let response = client.get(&url).send().await.unwrap();
        let error = response.error_for_status().unwrap_err();
        let categorized = categorize_reqwest_error(&error);

        assert_eq!(
            categorized, expected_error_type,
            "Status code {} should be categorized as {:?}",
            code, expected_error_type
        );
        assert_eq!(
            error.is_retriable(),
            expected_retriable,
            "Status code {} retriability should be {}",
            code,
            expected_retriable
        );
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_other_client_errors() {
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let server = Server::run();
        server.expect(
            Expectation::matching(request::method_path("GET", "/418"))
                .respond_with(status_code(418).body("I'm a teapot")), // 418 is a 4xx but not specifically handled
        );

        let client = reqwest::Client::new();
        let url = format!("http://{}/418", server.addr());
        let response = client.get(&url).send().await.unwrap();
        let error = response.error_for_status().unwrap_err();
        let categorized = categorize_reqwest_error(&error);

        assert_eq!(
            categorized,
            ErrorType::HttpRequestOtherError,
            "Other 4xx errors should be categorized as HttpRequestOtherError"
        );
        assert!(
            !error.is_retriable(),
            "Other 4xx (418) should not be retriable"
        );
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_other_server_errors() {
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let server = Server::run();
        server.expect(
            Expectation::matching(request::method_path("GET", "/507"))
                .respond_with(status_code(507).body("Insufficient Storage")), // 507 is a 5xx but not specifically handled
        );

        let client = reqwest::Client::new();
        let url = format!("http://{}/507", server.addr());
        let response = client.get(&url).send().await.unwrap();
        let error = response.error_for_status().unwrap_err();
        let categorized = categorize_reqwest_error(&error);

        assert_eq!(
            categorized,
            ErrorType::HttpRequestOtherError,
            "Other 5xx errors should be categorized as HttpRequestOtherError"
        );
        assert!(error.is_retriable(), "Other 5xx (507) should be retriable");
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_timeout() {
        use std::time::Duration;

        // Test timeout by using a very short timeout on a non-existent server
        // This will trigger a timeout error
        // Use a slightly longer timeout to be more reliable on CI
        let client = reqwest::Client::builder()
            .timeout(Duration::from_millis(50))
            .connect_timeout(Duration::from_millis(10))
            .build()
            .unwrap();

        // Use a URL that won't respond quickly (or at all)
        // TEST-NET-1 (RFC 3330) - should not respond
        let error = client.get("http://192.0.2.1:80").send().await.unwrap_err();

        let categorized = categorize_reqwest_error(&error);

        // Timeout errors should be categorized as HttpRequestTimeoutError
        // (may also be ConnectError depending on timing, but timeout is more likely)
        assert!(
            categorized == ErrorType::HttpRequestTimeoutError
                || categorized == ErrorType::HttpRequestConnectError,
            "Timeout errors should be categorized as TimeoutError or ConnectError, got {:?}",
            categorized
        );
        assert!(
            error.is_retriable(),
            "Timeout/connect errors should be retriable"
        );
    }

    #[tokio::test]
    async fn test_categorize_reqwest_error_connect() {
        use std::time::Duration;

        // Create a client that tries to connect to a non-existent server
        // Use a longer timeout to ensure we get a connection error rather than timeout
        // on slower CI environments
        let client = reqwest::Client::builder()
            .timeout(Duration::from_millis(500))
            .connect_timeout(Duration::from_millis(200))
            .build()
            .unwrap();

        // Use a reserved IP address that should never respond (RFC 3330 - TEST-NET-1)
        let error = client
            .get("http://192.0.2.1:80") // TEST-NET-1, should not respond
            .send()
            .await
            .unwrap_err();

        let categorized = categorize_reqwest_error(&error);

        // Connection errors can be categorized as connect, timeout, or request error
        // depending on the specific error and timing (especially on CI)
        assert!(
            categorized == ErrorType::HttpRequestConnectError
                || categorized == ErrorType::HttpRequestTimeoutError
                || categorized == ErrorType::HttpRequestRequestError,
            "Connection errors should be categorized as ConnectError, TimeoutError, or RequestError, got {:?}",
            categorized
        );
        assert!(
            error.is_retriable(),
            "Connection errors should be retriable"
        );
    }
}