crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

//! Peer certificate extraction from BoringSSL TLS streams.
//!
//! During a successful handshake the server cert chain is available via
//! `SslRef::peer_certificate()`. We pull CN, SANs, issuer, validity window,
//! and a SHA-256 fingerprint. SANs feed the crawler frontier as new
//! subdomains; the fingerprint lets us cluster hosts by exact cert.

use boring::hash::MessageDigest;
use boring::ssl::SslRef;
use boring::x509::{GeneralNameRef, X509};
use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PeerCert {
    pub subject_cn: Option<String>,
    pub issuer_cn: Option<String>,
    pub sans: Vec<String>,
    pub not_before: Option<String>,
    pub not_after: Option<String>,
    pub sha256: Option<String>,
}

pub fn extract(ssl: &SslRef) -> Option<PeerCert> {
    let cert: X509 = ssl.peer_certificate()?;
    let subject_cn = name_cn(cert.subject_name());
    let issuer_cn = name_cn(cert.issuer_name());

    let mut sans: Vec<String> = Vec::new();
    if let Some(stack) = cert.subject_alt_names() {
        for gn in stack.iter() {
            if let Some(dns) = general_name_dns(gn) {
                sans.push(dns.to_ascii_lowercase());
            }
        }
    }
    sans.sort();
    sans.dedup();

    let sha256 = cert
        .digest(MessageDigest::sha256())
        .ok()
        .map(|d| hex::encode(d.as_ref()));

    Some(PeerCert {
        subject_cn,
        issuer_cn,
        sans,
        not_before: Some(cert.not_before().to_string()),
        not_after: Some(cert.not_after().to_string()),
        sha256,
    })
}

fn name_cn(name: &boring::x509::X509NameRef) -> Option<String> {
    for entry in name.entries() {
        if entry.object().nid() == boring::nid::Nid::COMMONNAME {
            if let Ok(s) = entry.data().as_utf8() {
                return Some(s.to_string());
            }
        }
    }
    None
}

fn general_name_dns(gn: &GeneralNameRef) -> Option<&str> {
    gn.dnsname()
}