stealthscraper-rs 0.4.0

A stealthy web scraping library providing TLS JA4 fingerprint impersonation and headless browser automation.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

#![warn(missing_docs)]
#![forbid(unsafe_code)]
//! Stealthy Rust web scraping that defeats modern bot protection (Cloudflare,
//! Akamai, DataDome) on two fronts at once:
//!
//! - **JavaScript / CDP probing** — a real headless Chrome instance is driven via
//!   the Chrome DevTools Protocol, with stealth scripts masking `navigator`,
//!   WebGL, Canvas, and Audio fingerprints.
//! - **Network (JA3/JA4) fingerprinting** — a local MITM proxy
//!   ([`TlsSpoofingProxy`]) re-emits the browser's traffic through `wreq` so the
//!   TLS `ClientHello` and HTTP/2 settings match the impersonated browser.
//!
//! # Capabilities
//!
//! - **Challenge handling** ([`challenge`]) — classify a page with [`detect`] and
//!   choose a retry/rotate [`Action`] via [`MitigationPolicy`].
//! - **Proxy rotation** ([`proxy_pool`]) — a rotatable [`ProxyPool`]; the MITM
//!   upstream client is hot-swapped so the egress IP changes without relaunching
//!   the browser.
//! - **Geo/locale consistency** ([`geo`]) — derive `Accept-Language`,
//!   `navigator.languages`, and the timezone from the egress proxy's country so
//!   the IP and locale never contradict each other.
//! - **Profile rotation** — relaunch under a fresh [`BrowserProfile`] when the
//!   fingerprint identity itself is burned (the `browser` feature's `CloudScraper`).
//! - **Session state** ([`state`]) — per-domain outcomes and cooldowns behind a
//!   [`StateStore`]; durable with the `persistence` feature.
//! - **Observability** ([`events`]) — a [`ScraperEvent`] / [`EventSink`] stream.
//!
//! # Feature flags
//!
//! - `browser` *(off by default)* — the headless-Chrome API (`CloudScraper`,
//!   `solve_challenge`, profile rotation, human-behavior helpers). Required for
//!   the quick start below.
//! - `persistence` *(off by default)* — the durable, `redb`-backed state store.
//!
//! With no features enabled the crate builds only the pure, dependency-light core
//! ([`challenge`], [`proxy_pool`], [`geo`], the [`state`] model, [`events`]) for
//! embedding into your own pipeline.
//!
//! # Quick start
//!
#![cfg_attr(feature = "browser", doc = "```no_run")]
#![cfg_attr(not(feature = "browser"), doc = "```ignore")]
//! use stealthscraper_rs::{BrowserProfile, CloudScraper};
//!
//! # #[tokio::main]
//! # async fn main() -> Result<(), stealthscraper_rs::Error> {
//! // The builder spins up the MITM proxy and launches a stealth browser whose
//! // JA4 fingerprint matches the chosen profile.
//! let scraper = CloudScraper::builder()
//!     .profile(BrowserProfile::random())
//!     .build()
//!     .await?;
//!
//! let tab = scraper.new_stealth_tab()?;
//! tab.navigate_to("https://protected.example.com").expect("navigate");
//! tab.wait_until_navigated().expect("wait");
//!
//! // Detect and wait out / solve any bot-protection challenge on the page.
//! let signal = scraper.solve_challenge(&tab)?;
//! println!("page cleared (challenge: {:?})", signal.kind);
//! # Ok(())
//! # }
//! ```
//!
//! `solve_challenge` is synchronous and blocking; on an async runtime call it from
//! `tokio::task::spawn_blocking` and run the proxy on a multi-threaded runtime.

/// Emulation of human-like interaction patterns (typing delays, mouse curves).
#[cfg(feature = "browser")]
pub mod behavior;
/// Pure detection and mitigation policy for bot-protection challenges.
pub mod challenge;
/// Strong typed Error enums for the scraper and underlying HTTP proxy.
pub mod error;
/// Observability events and sinks emitted during a scrape.
pub mod events;
/// Geo/locale consistency: country codes, locale table, and a resolver port.
pub mod geo;
/// Management of browser fingerprints, user agents, and localized hardware characteristics.
pub mod profile;
/// Local MITM TLS spoofing proxy using Hyper and Rustls.
pub mod proxy;
/// Rotatable pool of upstream proxies with selection strategy (pure domain logic).
pub mod proxy_pool;
/// Core headless Chrome browser lifecycle and orchestration.
#[cfg(feature = "browser")]
pub mod scraper;
/// Automated solvers for bypassing common JavaScript challenges.
#[cfg(feature = "browser")]
pub mod solver;
/// Per-domain session state: model, store port, and adapters.
pub mod state;
/// Injection scripts to mask navigator and WebGL hooks.
pub mod stealth;

pub use challenge::{
    Action, ChallengeKind, ChallengeSignal, Confidence, DetectionInput, MitigationPolicy, detect,
};
pub use error::Error;
pub use events::{EventSink, LogEventSink, NoopEventSink, ScraperEvent};
pub use geo::{CountryCode, GeoResolver, Locale};
pub use profile::BrowserProfile;
pub use proxy::TlsSpoofingProxy;
pub use proxy_pool::{ProxyPool, RotationStrategy};
#[cfg(feature = "browser")]
pub use scraper::{CloudScraper, CloudScraperBuilder};
#[cfg(feature = "browser")]
pub use solver::GenericSolver;
pub use state::{DomainState, InMemoryStateStore, Outcome, StateStore};