crawlex 1.0.6

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

//! LinkRel source — Hot tier (B3).
//!
//! Reads `<link rel="...">` tags. Preconnect / dns-prefetch hints to
//! well-known CDN / analytics / payment hosts produce supporting
//! evidence (lower weight than ScriptSrc — a preconnect proves the
//! site *plans* to use the host, not necessarily that it does).
//! Manifest links are evidence the host ships a PWA.

use scraper::{Html, Selector};

use crate::fingerprint::detection::{Category, Detection, Evidence, EvidenceSource, Tier, Vendor};
use crate::fingerprint::target::TargetContext;

use super::Source;

#[derive(Default)]
pub struct LinkRelSource;

impl LinkRelSource {
    pub fn new() -> Self {
        Self
    }
}

/// (host-substring lower, weight, category, vendor).
/// Used for preconnect / dns-prefetch href values.
const HOST_TABLE: &[(&str, u8, Category, Vendor)] = &[
    ("googletagmanager.com", 6, Category::TagManager, Vendor::Gtm),
    (
        "google-analytics.com",
        6,
        Category::Analytics,
        Vendor::GoogleAnalytics,
    ),
    ("cdn.shopify.com", 6, Category::Ecommerce, Vendor::Shopify),
    ("cloudflare.com", 4, Category::Cdn, Vendor::Cloudflare),
    ("akamaihd.net", 4, Category::Cdn, Vendor::Akamai),
    ("akamaized.net", 4, Category::Cdn, Vendor::Akamai),
    ("cloudfront.net", 4, Category::Cdn, Vendor::CloudFront),
    ("vercel-storage.com", 6, Category::Cdn, Vendor::Vercel),
    ("js.stripe.com", 6, Category::Payment, Vendor::Stripe),
    ("paypalobjects.com", 6, Category::Payment, Vendor::Paypal),
];

impl Source for LinkRelSource {
    fn name(&self) -> &'static str {
        "link_rel"
    }

    fn tier(&self) -> Tier {
        Tier::Hot
    }

    fn analyze(&self, ctx: &TargetContext<'_>) -> Vec<Detection> {
        let body_str = match std::str::from_utf8(ctx.body) {
            Ok(s) => s,
            Err(_) => return Vec::new(),
        };
        let doc = Html::parse_document(body_str);
        let sel = Selector::parse("link[rel][href]").unwrap();

        let mut seen: std::collections::HashSet<(Category, Vendor)> =
            std::collections::HashSet::new();
        let mut out: Vec<Detection> = Vec::new();

        for link in doc.select(&sel) {
            let Some(rel) = link.value().attr("rel") else {
                continue;
            };
            let rel_lower = rel.to_ascii_lowercase();
            // Only care about preconnect / dns-prefetch / preload here.
            if !rel_lower.contains("preconnect")
                && !rel_lower.contains("dns-prefetch")
                && !rel_lower.contains("preload")
            {
                continue;
            }
            let Some(href) = link.value().attr("href") else {
                continue;
            };
            let lower = href.to_ascii_lowercase();
            for (needle, weight, cat, vendor) in HOST_TABLE {
                if lower.contains(needle) && seen.insert((*cat, *vendor)) {
                    out.push(Detection::from_single(
                        *cat,
                        *vendor,
                        Evidence::new(
                            EvidenceSource::LinkRel,
                            format!("link rel={rel_lower} href '{href}'"),
                            *weight,
                        ),
                    ));
                }
            }
        }
        out
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use http::HeaderMap;
    use url::Url;

    fn ctx_with<'a>(headers: &'a HeaderMap, url: &'a Url, body: &'a [u8]) -> TargetContext<'a> {
        TargetContext::http_only(url, 200, headers, body)
    }

    #[test]
    fn detects_preconnect_to_gtm() {
        let h = HeaderMap::new();
        let u: Url = "https://example.com/".parse().unwrap();
        let body = br#"<html><head><link rel="preconnect" href="https://www.googletagmanager.com"></head></html>"#;
        let dets = LinkRelSource::new().analyze(&ctx_with(&h, &u, body));
        assert!(dets.iter().any(|d| d.vendor == Vendor::Gtm));
    }

    #[test]
    fn detects_dns_prefetch_cloudflare() {
        let h = HeaderMap::new();
        let u: Url = "https://example.com/".parse().unwrap();
        let body =
            br#"<html><head><link rel="dns-prefetch" href="//cdnjs.cloudflare.com"></head></html>"#;
        let dets = LinkRelSource::new().analyze(&ctx_with(&h, &u, body));
        assert!(dets.iter().any(|d| d.vendor == Vendor::Cloudflare));
    }

    #[test]
    fn no_detection_for_stylesheet_link() {
        let h = HeaderMap::new();
        let u: Url = "https://example.com/".parse().unwrap();
        let body = br#"<html><head><link rel="stylesheet" href="https://www.googletagmanager.com/x.css"></head></html>"#;
        let dets = LinkRelSource::new().analyze(&ctx_with(&h, &u, body));
        assert!(dets.is_empty());
    }
}