domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
//! Error retriability and retry logic.

use std::future::Future;
use std::time::Duration;

use anyhow::{Error, Result};

use crate::config::MAX_NETWORK_DOWNLOAD_RETRIES;
use crate::error_handling::ReqwestErrorExt;

/// Determines if an error is retriable (should be retried).
///
/// This function categorizes errors into retriable (transient) and non-retriable (permanent)
/// categories to guide retry logic. Only transient errors that might succeed on retry
/// should be retried.
///
/// # Retriable Errors
///
/// - Network timeouts (`reqwest::Error::is_timeout()`)
/// - Connection failures (`reqwest::Error::is_connect()`)
/// - Request errors (`reqwest::Error::is_request()`)
/// - Server errors (5xx HTTP status codes)
/// - Rate limiting (429 Too Many Requests)
/// - DNS resolution failures
///
/// # Non-Retriable Errors
///
/// - Client errors (4xx HTTP status codes, except 429)
/// - URL parsing errors
/// - Database errors
/// - Redirect errors
/// - Decode errors
///
/// # Implementation Details
///
/// Uses error chain inspection to properly identify error types without relying on
/// fragile string matching. HTTP retriability is delegated to
/// [`ReqwestErrorExt::is_retriable`] so there is a single source of truth.
///
/// **Default behavior:** Unrecognized errors are NOT retried (fail-fast). Only
/// explicitly-identified transient errors trigger retries.
pub(crate) fn is_retriable_error(error: &Error) -> bool {
    // Check error chain for specific error types
    for cause in error.chain() {
        // Check for reqwest errors (HTTP client errors) — single source of truth
        if let Some(reqwest_err) = cause.downcast_ref::<reqwest::Error>() {
            return reqwest_err.is_retriable();
        }

        // Check for URL parsing errors (not retriable)
        if cause.downcast_ref::<url::ParseError>().is_some() {
            return false;
        }

        // Check for database errors (not retriable)
        if cause.downcast_ref::<sqlx::Error>().is_some() {
            return false;
        }

        // Check for DNS resolution errors (retriable) via structured type, not string matching.
        if cause
            .downcast_ref::<hickory_resolver::net::NetError>()
            .is_some()
        {
            return true;
        }
    }

    // Default: do NOT retry unknown errors.
    false
}

/// Retries an async network download with exponential backoff.
///
/// Uses [`MAX_NETWORK_DOWNLOAD_RETRIES`] attempts. Delay between attempts is
/// `base_delay_secs << (attempt - 1)` (e.g. base 1 → 1s, 2s, 4s; base 2 → 2s, 4s, 8s).
pub(crate) async fn with_network_download_retry<T, F, Fut>(
    label: &str,
    base_delay_secs: u64,
    mut op: F,
) -> Result<T>
where
    F: FnMut() -> Fut,
    Fut: Future<Output = Result<T>>,
{
    let mut last_error = None;
    for attempt in 1..=MAX_NETWORK_DOWNLOAD_RETRIES {
        match op().await {
            Ok(value) => return Ok(value),
            Err(e) => {
                last_error = Some(e);
                if attempt < MAX_NETWORK_DOWNLOAD_RETRIES {
                    log::warn!(
                        "Failed {label} (attempt {attempt}/{MAX_NETWORK_DOWNLOAD_RETRIES}), retrying..."
                    );
                    tokio::time::sleep(Duration::from_secs(base_delay_secs << (attempt - 1))).await;
                }
            }
        }
    }

    Err(last_error.unwrap_or_else(|| {
        anyhow::anyhow!("Failed {label} after {MAX_NETWORK_DOWNLOAD_RETRIES} attempts")
    }))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_is_retriable_error_timeout_string_only() {
        // A string-only "timeout" error without a real reqwest::Error type
        // is unrecognized and should NOT be retried (fail-fast for unknowns)
        let err = anyhow::anyhow!("Request timeout");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_url_parse() {
        // Test URL parse error - when converted to anyhow::Error, it should be detected
        // Note: The downcast might not work if anyhow wraps it, so we test the actual behavior
        let parse_err = url::ParseError::EmptyHost;
        let err: anyhow::Error = parse_err.into();
        // The function should detect url::ParseError via downcast_ref
        // If downcast doesn't work, the error message check might catch it, but
        // for now, we test that it's handled (may need to adjust based on actual behavior)
        let result = is_retriable_error(&err);
        // URL parse errors should not be retriable
        // If this fails, it means downcast isn't working and we need to fix the implementation
        assert!(!result, "URL parse error should not be retriable");
    }

    #[test]
    fn test_is_retriable_error_database() {
        // Create a database error directly (not wrapped)
        let db_err = sqlx::Error::PoolClosed;
        let err: anyhow::Error = db_err.into();
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_404() {
        let err = anyhow::anyhow!("404 not found");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_403() {
        let err = anyhow::anyhow!("403 forbidden");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_401() {
        let err = anyhow::anyhow!("401 unauthorized");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_dns_typed() {
        // Only NetError (structured) is retriable; plain strings are not
        let resolve_err = hickory_resolver::net::NetError::from("lookup failed");
        let err: anyhow::Error = resolve_err.into();
        assert!(is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_dns_string_not_retriable() {
        // Plain string "DNS lookup failed" is not retriable (no type downcast)
        let err = anyhow::anyhow!("DNS lookup failed");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_resolve_string_not_retriable() {
        let err = anyhow::anyhow!("Failed to resolve hostname");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_unknown() {
        // Unknown errors should NOT be retried — fail fast for unrecognized errors
        let err = anyhow::anyhow!("Some unknown error");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_empty() {
        // Empty error message should NOT be retried
        let err = anyhow::anyhow!("");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_5xx_string_only() {
        // Plain strings like "500 internal server error" are NOT matched by status code logic.
        // Real 5xx errors come from reqwest with proper status codes (tested in async tests below).
        let err = anyhow::anyhow!("500 internal server error");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_429_string_only() {
        // Same: plain "429" string doesn't trigger reqwest downcast.
        // Real 429 errors are tested via httptest mock server below.
        let err = anyhow::anyhow!("429 too many requests");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_400() {
        // "400 bad request" as a plain string (not a real reqwest::Error with status code)
        // doesn't match any known pattern, so it's treated as unknown → not retried.
        // Real 400 errors come from reqwest with status codes and are caught on line 62.
        let err = anyhow::anyhow!("400 bad request");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_redirect() {
        // Plain string "Redirect error" is unrecognized → not retried.
        // Real redirect errors come from reqwest and are caught via is_redirect().
        let err = anyhow::anyhow!("Redirect error");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_connection_failure_string() {
        // Plain string "Connection failed" is unrecognized → not retried.
        // Real connection errors come from reqwest and are caught via is_connect().
        let err = anyhow::anyhow!("Connection failed");
        assert!(!is_retriable_error(&err));
    }

    #[test]
    fn test_is_retriable_error_timeout_message_string() {
        // Plain string "Request timed out" is unrecognized → not retried.
        // Real timeouts come from reqwest and are caught via is_timeout().
        let err = anyhow::anyhow!("Request timed out");
        assert!(!is_retriable_error(&err));
    }

    // Note: Error chain tests removed because anyhow's error wrapping makes downcast
    // behavior unpredictable in chains. The core functionality is tested above with
    // direct error types, which is the realistic use case.

    #[test]
    fn test_is_retriable_error_mixed_chain() {
        // Test error chain with multiple error types
        // This tests that the first matching error type in the chain determines retriability
        let db_err = sqlx::Error::PoolClosed;
        let err: anyhow::Error = db_err.into();
        let wrapped = err.context("Additional context");

        // Database error should be non-retriable even when wrapped
        assert!(!is_retriable_error(&wrapped));
    }

    #[test]
    fn test_is_retriable_error_chain_order() {
        // Test that error chain order matters (first match wins)
        // If a non-retriable error is wrapped by a retriable-looking message,
        // the downcast should catch the non-retriable error first
        let parse_err = url::ParseError::EmptyHost;
        let err: anyhow::Error = parse_err.into();
        let wrapped = err.context("Some other context"); // Don't use DNS message to avoid confusion

        // URL parse error should be detected via downcast, not message
        // Note: The function checks downcast_ref first, so URL parse errors should be non-retriable
        assert!(!is_retriable_error(&wrapped));
    }

    #[test]
    fn test_is_retriable_error_message_case_insensitive() {
        // Test that error message matching is case-insensitive
        let err1 = anyhow::anyhow!("404 NOT FOUND");
        let err2 = anyhow::anyhow!("Not Found");
        let err3 = anyhow::anyhow!("FORBIDDEN");

        // All should be non-retriable (case-insensitive matching)
        assert!(!is_retriable_error(&err1));
        // Note: "Not Found" without "404" might not match the pattern
        // The function checks for "404", "403", "401" specifically
        let _result2 = is_retriable_error(&err2);
        let result3 = is_retriable_error(&err3);
        // "FORBIDDEN" should match "forbidden" pattern
        assert!(!result3, "FORBIDDEN should be non-retriable");
    }

    #[test]
    fn test_is_retriable_error_partial_message_match() {
        // No string-based 4xx/DNS matching; only typed detection
        let err1 = anyhow::anyhow!("Error: 404 page not found");
        let err2 = anyhow::anyhow!("HTTP 500 internal server error occurred");
        let err3 = anyhow::anyhow!("DNS resolution failed for domain");

        assert!(!is_retriable_error(&err1)); // unknown → false
        assert!(!is_retriable_error(&err2)); // unknown → false
        assert!(!is_retriable_error(&err3)); // no NetError type → false
    }

    #[test]
    fn test_is_retriable_error_nested_context_resolve_error() {
        // NetError in chain (even with context) is retriable
        let resolve_err = hickory_resolver::net::NetError::from("lookup failed");
        let err: anyhow::Error = resolve_err.into();
        let err = err.context("First context").context("Second context");

        assert!(is_retriable_error(&err));
    }

    #[tokio::test]
    async fn test_is_retriable_error_reqwest_status_429() {
        // Test that 429 status code is correctly identified as retriable
        // This is critical - rate limits should be retried with backoff
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let server = Server::run();
        server.expect(
            Expectation::matching(request::method_path("GET", "/429"))
                .respond_with(status_code(429).body("Too Many Requests")),
        );

        let client = reqwest::Client::new();
        let url = server.url("/429").to_string();
        let response = client
            .get(&url)
            .send()
            .await
            .expect("Failed to create test request");
        let reqwest_err = response.error_for_status().unwrap_err();
        let error: anyhow::Error = reqwest_err.into();

        // 429 should be retriable (line 57-58)
        assert!(is_retriable_error(&error));
    }

    #[tokio::test]
    async fn test_is_retriable_error_reqwest_status_5xx() {
        // Test that 5xx status codes are correctly identified as retriable
        // This is critical - server errors are temporary and should be retried
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let server = Server::run();
        server.expect(
            Expectation::matching(request::method_path("GET", "/500"))
                .respond_with(status_code(500).body("Internal Server Error")),
        );

        let client = reqwest::Client::new();
        let url = server.url("/500").to_string();
        let response = client
            .get(&url)
            .send()
            .await
            .expect("Failed to create test request");
        let reqwest_err = response.error_for_status().unwrap_err();
        let error: anyhow::Error = reqwest_err.into();

        // 500 should be retriable (line 67-68)
        assert!(is_retriable_error(&error));
    }

    #[tokio::test]
    async fn test_is_retriable_error_reqwest_status_4xx_not_429() {
        // Test that 4xx status codes (except 429) are NOT retriable
        // This is critical - client errors are permanent and should not be retried
        use httptest::{matchers::*, responders::*, Expectation, Server};

        let server = Server::run();
        server.expect(
            Expectation::matching(request::method_path("GET", "/400"))
                .respond_with(status_code(400).body("Bad Request")),
        );

        let client = reqwest::Client::new();
        let url = server.url("/400").to_string();
        let response = client
            .get(&url)
            .send()
            .await
            .expect("Failed to create test request");
        let reqwest_err = response.error_for_status().unwrap_err();
        let error: anyhow::Error = reqwest_err.into();

        // 400 should NOT be retriable (line 62-63)
        assert!(!is_retriable_error(&error));
    }

    #[tokio::test]
    async fn test_is_retriable_error_reqwest_timeout() {
        // Test that reqwest timeout errors are retriable
        // This is critical - timeouts are transient network issues
        // Create a timeout by using a very short timeout
        let client = reqwest::Client::builder()
            .timeout(std::time::Duration::from_millis(1))
            .build()
            .expect("Failed to create client");

        // Use a URL that will timeout (connection to closed port)
        let error = client.get("http://127.0.0.1:1/").send().await.unwrap_err();
        let error: anyhow::Error = error.into();

        // Timeout errors should be retriable (line 73)
        // Note: May also be connect error, which is also retriable
        assert!(is_retriable_error(&error));
    }

    #[test]
    fn test_is_retriable_error_message_no_string_matching() {
        // Plain strings are not retriable; only typed NetError is
        let dns_error = anyhow::anyhow!("DNS resolution failed");
        let resolve_error = anyhow::anyhow!("Failed to resolve hostname");
        let lookup_error = anyhow::anyhow!("DNS lookup failed");

        assert!(!is_retriable_error(&dns_error));
        assert!(!is_retriable_error(&resolve_error));
        assert!(!is_retriable_error(&lookup_error));
    }

    #[test]
    fn test_is_retriable_error_unknown_defaults_to_not_retriable() {
        // Unknown errors default to NOT retriable — fail fast rather than silently retry.
        let unknown_error = anyhow::anyhow!("Some completely unknown error type");
        assert!(!is_retriable_error(&unknown_error));
    }
}