crawlex 1.0.6

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

//! Pre-network freshness skip — seeds a `pages` row through the real
//! SQLite storage write path, then asserts `evaluate_freshness` returns
//! the exact `Fresh` outcome the crawler uses to short-circuit the fetch.

use bytes::Bytes;
use crawlex::cache_validator::{evaluate_freshness, CacheValidationStatus};
use crawlex::storage::ArtifactStorage;
use http::HeaderMap;
use url::Url;

async fn seed(
    url: &str,
    last_modified: Option<&str>,
) -> (tempfile::TempDir, crawlex::storage::PageCacheMetadata) {
    let tmp = tempfile::tempdir().unwrap();
    let db_path = tmp.path().join("freshness.db");
    let sq = crawlex::storage::sqlite::SqliteStorage::open(&db_path).unwrap();
    let parsed = Url::parse(url).unwrap();
    let mut headers = HeaderMap::new();
    if let Some(lm) = last_modified {
        headers.insert("last-modified", lm.parse().unwrap());
    }
    sq.save_raw_response(
        &parsed,
        &parsed,
        200,
        &headers,
        &Bytes::from_static(b"<html></html>"),
        false,
    )
    .await
    .unwrap();
    let mut meta = None;
    for _ in 0..50 {
        if let Some(m) = sq.page_cache_metadata(&parsed).await.unwrap() {
            meta = Some(m);
            break;
        }
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    }
    (tmp, meta.expect("seeded row should be visible"))
}

#[tokio::test]
async fn cache_max_age_skips_fresh_row() {
    let (_tmp, meta) = seed("https://example.test/recent", None).await;
    let outcome = evaluate_freshness(&meta, Some(3600), None);
    assert_eq!(outcome.status, CacheValidationStatus::Fresh);
    assert_eq!(outcome.reason, "fresh-by-max-age");
}

#[tokio::test]
async fn modified_since_skips_old_last_modified() {
    let (_tmp, meta) = seed(
        "https://example.test/stale-lm",
        Some("Sun, 06 Nov 1994 08:49:37 GMT"),
    )
    .await;
    let outcome = evaluate_freshness(&meta, None, Some(1_700_000_000));
    assert_eq!(outcome.status, CacheValidationStatus::Fresh);
    assert_eq!(outcome.reason, "unmodified-since");
}

#[tokio::test]
async fn defaults_do_not_short_circuit() {
    let (_tmp, meta) = seed("https://example.test/no-knobs", None).await;
    let outcome = evaluate_freshness(&meta, None, None);
    assert_eq!(outcome.status, CacheValidationStatus::Unknown);
}

#[tokio::test]
async fn sqlite_cache_metadata_lookup_uses_canonical_url_aliases() {
    let tmp = tempfile::tempdir().unwrap();
    let db_path = tmp.path().join("freshness-alias.db");
    let sq = crawlex::storage::sqlite::SqliteStorage::open(&db_path).unwrap();
    let saved = Url::parse("https://www.example.test/store/index.html?utm_source=x&a=1").unwrap();
    let lookup = Url::parse("http://example.test/store/?a=1&utm_medium=y").unwrap();

    let mut headers = HeaderMap::new();
    headers.insert("etag", "\"alias-v1\"".parse().unwrap());
    sq.save_raw_response(
        &saved,
        &saved,
        200,
        &headers,
        &Bytes::from_static(b"<html><head><title>A</title></head><body></body></html>"),
        false,
    )
    .await
    .unwrap();

    for _ in 0..50 {
        if let Some(m) = sq.page_cache_metadata(&lookup).await.unwrap() {
            assert_eq!(m.etag.as_deref(), Some("\"alias-v1\""));
            return;
        }
        tokio::time::sleep(std::time::Duration::from_millis(20)).await;
    }
    panic!("canonical alias cache metadata was not visible");
}