stealthscraper_rs/
lib.rs

1#![warn(missing_docs)]
2#![forbid(unsafe_code)]
3//! Stealthy Rust web scraping that defeats modern bot protection (Cloudflare,
4//! Akamai, DataDome) on two fronts at once:
5//!
6//! - **JavaScript / CDP probing** — a real headless Chrome instance is driven via
7//!   the Chrome DevTools Protocol, with stealth scripts masking `navigator`,
8//!   WebGL, Canvas, and Audio fingerprints.
9//! - **Network (JA3/JA4) fingerprinting** — a local MITM proxy
10//!   ([`TlsSpoofingProxy`]) re-emits the browser's traffic through `wreq` so the
11//!   TLS `ClientHello` and HTTP/2 settings match the impersonated browser.
12//!
13//! # Capabilities
14//!
15//! - **Challenge handling** ([`challenge`]) — classify a page with [`detect`] and
16//!   choose a retry/rotate [`Action`] via [`MitigationPolicy`].
17//! - **Proxy rotation** ([`proxy_pool`]) — a rotatable [`ProxyPool`]; the MITM
18//!   upstream client is hot-swapped so the egress IP changes without relaunching
19//!   the browser.
20//! - **Geo/locale consistency** ([`geo`]) — derive `Accept-Language`,
21//!   `navigator.languages`, and the timezone from the egress proxy's country so
22//!   the IP and locale never contradict each other.
23//! - **Profile rotation** — relaunch under a fresh [`BrowserProfile`] when the
24//!   fingerprint identity itself is burned (the `browser` feature's `CloudScraper`).
25//! - **Session state** ([`state`]) — per-domain outcomes and cooldowns behind a
26//!   [`StateStore`]; durable with the `persistence` feature.
27//! - **Observability** ([`events`]) — a [`ScraperEvent`] / [`EventSink`] stream.
28//!
29//! # Feature flags
30//!
31//! - `browser` *(off by default)* — the headless-Chrome API (`CloudScraper`,
32//!   `solve_challenge`, profile rotation, human-behavior helpers). Required for
33//!   the quick start below.
34//! - `persistence` *(off by default)* — the durable, `redb`-backed state store.
35//!
36//! With no features enabled the crate builds only the pure, dependency-light core
37//! ([`challenge`], [`proxy_pool`], [`geo`], the [`state`] model, [`events`]) for
38//! embedding into your own pipeline.
39//!
40//! # Quick start
41//!
42#![cfg_attr(feature = "browser", doc = "```no_run")]
43#![cfg_attr(not(feature = "browser"), doc = "```ignore")]
44//! use stealthscraper_rs::{BrowserProfile, CloudScraper};
45//!
46//! # #[tokio::main]
47//! # async fn main() -> Result<(), stealthscraper_rs::Error> {
48//! // The builder spins up the MITM proxy and launches a stealth browser whose
49//! // JA4 fingerprint matches the chosen profile.
50//! let scraper = CloudScraper::builder()
51//!     .profile(BrowserProfile::random())
52//!     .build()
53//!     .await?;
54//!
55//! let tab = scraper.new_stealth_tab()?;
56//! tab.navigate_to("https://protected.example.com").expect("navigate");
57//! tab.wait_until_navigated().expect("wait");
58//!
59//! // Detect and wait out / solve any bot-protection challenge on the page.
60//! let signal = scraper.solve_challenge(&tab)?;
61//! println!("page cleared (challenge: {:?})", signal.kind);
62//! # Ok(())
63//! # }
64//! ```
65//!
66//! `solve_challenge` is synchronous and blocking; on an async runtime call it from
67//! `tokio::task::spawn_blocking` and run the proxy on a multi-threaded runtime.
68
69/// Emulation of human-like interaction patterns (typing delays, mouse curves).
70#[cfg(feature = "browser")]
71pub mod behavior;
72/// Pure detection and mitigation policy for bot-protection challenges.
73pub mod challenge;
74/// Strong typed Error enums for the scraper and underlying HTTP proxy.
75pub mod error;
76/// Observability events and sinks emitted during a scrape.
77pub mod events;
78/// Geo/locale consistency: country codes, locale table, and a resolver port.
79pub mod geo;
80/// Management of browser fingerprints, user agents, and localized hardware characteristics.
81pub mod profile;
82/// Local MITM TLS spoofing proxy using Hyper and Rustls.
83pub mod proxy;
84/// Rotatable pool of upstream proxies with selection strategy (pure domain logic).
85pub mod proxy_pool;
86/// Core headless Chrome browser lifecycle and orchestration.
87#[cfg(feature = "browser")]
88pub mod scraper;
89/// Automated solvers for bypassing common JavaScript challenges.
90#[cfg(feature = "browser")]
91pub mod solver;
92/// Per-domain session state: model, store port, and adapters.
93pub mod state;
94/// Injection scripts to mask navigator and WebGL hooks.
95pub mod stealth;
96
97pub use challenge::{
98    Action, ChallengeKind, ChallengeSignal, Confidence, DetectionInput, MitigationPolicy, detect,
99};
100pub use error::Error;
101pub use events::{EventSink, LogEventSink, NoopEventSink, ScraperEvent};
102pub use geo::{CountryCode, GeoResolver, Locale};
103pub use profile::BrowserProfile;
104pub use proxy::TlsSpoofingProxy;
105pub use proxy_pool::{ProxyPool, RotationStrategy};
106#[cfg(feature = "browser")]
107pub use scraper::{CloudScraper, CloudScraperBuilder};
108#[cfg(feature = "browser")]
109pub use solver::GenericSolver;
110pub use state::{DomainState, InMemoryStateStore, Outcome, StateStore};
stealthscraper_rs/lib.rs

stealthscraper_rs/
lib.rs
