spider_middleware/lib.rs
1//! # spider-middleware
2//!
3//! Built-in middleware implementations for the `spider-lib` framework.
4//!
5//! Provides middlewares for rate limiting, retries, user-agent rotation, cookies, and more.
6//!
7//! ## Example
8//!
9//! ```rust,ignore
10//! use spider_middleware::rate_limit::RateLimitMiddleware;
11//! use spider_middleware::retry::RetryMiddleware;
12//!
13//! let crawler = CrawlerBuilder::new(MySpider)
14//! .add_middleware(RateLimitMiddleware::default())
15//! .add_middleware(RetryMiddleware::new())
16//! .build()
17//! .await?;
18//! ```
19
20pub mod middleware;
21pub mod rate_limit;
22pub mod referer;
23pub mod request;
24pub mod retry;
25
26pub use spider_util::request::Request;
27pub use spider_util::response::Response;
28
29pub mod prelude;
30
31#[cfg(feature = "middleware-user-agent")]
32pub mod user_agent;
33
34#[cfg(feature = "middleware-cookies")]
35pub mod cookies;
36
37#[cfg(feature = "middleware-cache")]
38pub mod http_cache;
39
40#[cfg(feature = "middleware-proxy")]
41pub mod proxy;
42
43#[cfg(feature = "middleware-robots")]
44pub mod robots_txt;