rust_scrapling/
lib.rs

1//! RUSTScrapling — a Rust port of the [Scrapling](https://github.com/D4Vinci/Scrapling)
2//! web-scraping framework.
3//!
4//! The crate is organized in three layers, each usable on its own:
5//!
6//! - **[`parser`]** — HTML parsing and data extraction. [`Selector`] wraps a
7//!   parsed DOM node and offers CSS selection (including Parsel-style
8//!   `::text` / `::attr(name)` pseudo-elements via [`Selector::css_get`] /
9//!   [`Selector::css_getall`]), text extraction, DOM navigation, regex
10//!   helpers, JSON parsing, and *adaptive element relocation* — a saved
11//!   element's "unique properties" can be used to find it again by
12//!   similarity after the page layout changes.
13//! - **[`fetchers`]** — async HTTP. [`Fetcher`] is a reqwest-backed client
14//!   configured through [`FetcherConfig`] (timeouts, retries, redirects,
15//!   single/per-protocol/rotating proxies, response-size caps) that sends
16//!   browser-like *stealth headers* by default, and returns a [`Response`]
17//!   with charset-aware body decoding and direct parser integration.
18//! - **[`spiders`]** — crawling. Implement the [`Spider`] trait (or use a
19//!   ready-made template: [`CrawlSpider`], [`SitemapSpider`],
20//!   [`ShopifySpider`]) and run it with [`CrawlerEngine`]. The engine
21//!   handles concurrency limits (global and per-domain), request
22//!   deduplication, robots.txt compliance (RFC 9309 matching plus
23//!   `Crawl-delay`), AutoThrottle adaptive per-domain delays,
24//!   blocked-response retries, checkpoints for pause/resume, and a
25//!   development-mode response cache for offline iteration. Results come
26//!   back as a [`CrawlResult`] with live stats and an [`ItemList`]
27//!   exportable to CSV, JSON, JSON Lines, or XML.
28//!
29//! # Quickstart: parsing HTML
30//!
31//! ```
32//! use rust_scrapling::Selector;
33//!
34//! let html = r#"<html><body>
35//!   <ul>
36//!     <li class="item"><a href="/p/1">Widget</a> <span class="price">9.99</span></li>
37//!     <li class="item"><a href="/p/2">Gadget</a> <span class="price">19.99</span></li>
38//!   </ul>
39//! </body></html>"#;
40//!
41//! let page = Selector::from_html(html);
42//!
43//! // Parsel-style `::text` / `::attr()` pseudo-elements:
44//! let names: Vec<String> = page
45//!     .css_getall("li.item a::text")
46//!     .into_iter()
47//!     .map(|t| t.into_string())
48//!     .collect();
49//! assert_eq!(names, ["Widget", "Gadget"]);
50//!
51//! let first_href = page.css_get("li.item a::attr(href)").unwrap();
52//! assert_eq!(first_href.as_str(), "/p/1");
53//!
54//! // Plain CSS queries return `Selectors` for further navigation:
55//! let prices = page.css(".price");
56//! assert_eq!(prices.len(), 2);
57//! assert_eq!(prices[0].text().as_str(), "9.99");
58//! ```
59//!
60//! # Fetching a page
61//!
62//! ```no_run
63//! use rust_scrapling::{Fetcher, FetcherConfig};
64//!
65//! #[tokio::main]
66//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
67//!     let config = FetcherConfig::builder()
68//!         .timeout(20)
69//!         .stealth(true) // browser-like headers (the default)
70//!         .build();
71//!     let fetcher = Fetcher::new(config)?;
72//!
73//!     let response = fetcher.get("https://example.com").await?;
74//!     println!("status: {}", response.status());
75//!     if let Some(title) = response.selector().css_get("title::text") {
76//!         println!("title: {}", title);
77//!     }
78//!     Ok(())
79//! }
80//! ```
81//!
82//! # A minimal spider
83//!
84//! ```no_run
85//! use async_trait::async_trait;
86//! use rust_scrapling::spiders::response::SpiderResponse;
87//! use rust_scrapling::spiders::session::SessionManager;
88//! use rust_scrapling::{CrawlerEngine, FetcherConfig, Spider, SpiderRequest};
89//! use std::sync::Arc;
90//!
91//! struct QuotesSpider;
92//!
93//! #[async_trait]
94//! impl Spider for QuotesSpider {
95//!     fn name(&self) -> &str {
96//!         "quotes"
97//!     }
98//!
99//!     fn start_urls(&self) -> Vec<String> {
100//!         vec!["https://quotes.toscrape.com/".to_string()]
101//!     }
102//!
103//!     async fn parse(
104//!         &self,
105//!         response: SpiderResponse,
106//!     ) -> (Vec<serde_json::Value>, Vec<SpiderRequest>) {
107//!         let items = response
108//!             .css_getall(".quote .text::text")
109//!             .into_iter()
110//!             .map(|text| serde_json::json!({ "quote": text.as_str() }))
111//!             .collect();
112//!         (items, Vec::new()) // no follow-up requests
113//!     }
114//! }
115//!
116//! #[tokio::main]
117//! async fn main() {
118//!     let sessions = SessionManager::new(FetcherConfig::default());
119//!     let engine = CrawlerEngine::new(Arc::new(QuotesSpider), sessions, None)
120//!         .expect("HTTP client should build");
121//!     let result = engine.crawl().await;
122//!     println!("scraped {} items", result.items.len());
123//! }
124//! ```
125//!
126//! # Feature highlights
127//!
128//! - CSS selectors with `::text` / `::attr(name)` extraction
129//!   ([`Selector::css_get`] / [`Selector::css_getall`])
130//! - Adaptive element relocation that survives page-layout changes
131//!   ([`parser::adaptive`])
132//! - Stealth (browser-like) request headers, on by default
133//!   ([`FetcherConfig::build_headers`])
134//! - robots.txt compliance with RFC 9309 longest-match rules and
135//!   `Crawl-delay` support ([`spiders::robots`])
136//! - AutoThrottle: adaptive per-domain crawl delays ([`spiders::throttle`])
137//! - Checkpoints with pause/resume ([`spiders::checkpoint`],
138//!   [`CrawlerEngine::request_pause`])
139//! - Development-mode response cache for offline replays
140//!   ([`spiders::cache`])
141//! - Item export to CSV / JSON / JSON Lines / XML ([`ItemList`])
142
143#![warn(missing_docs)]
144
145pub mod core;
146pub mod fetchers;
147pub mod parser;
148pub mod spiders;
149
150// Re-export primary types at crate root
151pub use fetchers::client::Fetcher;
152pub use fetchers::config::FetcherConfig;
153pub use fetchers::response::Response;
154pub use parser::{Selector, Selectors};
155pub use spiders::engine::CrawlerEngine;
156pub use spiders::links::LinkExtractor;
157pub use spiders::request::SpiderRequest;
158pub use spiders::result::{CrawlResult, CrawlStats, ItemList};
159pub use spiders::spider::Spider;
160pub use spiders::templates::{CrawlRule, CrawlSpider, ShopifySpider, SitemapSpider};
rust_scrapling/lib.rs

rust_scrapling/
lib.rs
