crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

pub mod asset_refs;
pub mod assets;
pub mod cert;
pub mod dns;
pub mod favicon;
pub mod graph;
pub mod js_endpoints;
pub mod links;
pub mod network_probe;
pub mod pipeline;
pub mod pwa;
pub mod robots_paths;
pub mod security_txt;
pub mod sitemap;
pub mod subdomains;
pub mod tech_fingerprint;
pub mod types;
pub mod wayback;
pub mod well_known;
pub mod whois;

// Adapter modules — one Discoverer adapter per topic. Free functions in
// the topic modules above stay where they are; the adapters wrap them.
pub mod adapters;

pub use assets::{classify_url, classify_with_mime, AssetKind};
pub use graph::DiscoveryGraph;
pub use links::extract_links;
pub use pipeline::{Discoverer, DiscoveryError, DiscoveryPipeline};
pub use types::{DiscoveryContext, DiscoveryFeatures, DnsKind, Finding, SubdomainSource};