spider_middleware/lib.rs
1//! # spider-middleware
2//!
3//! Built-in middleware implementations for the `spider-lib` framework.
4//!
5//! Provides middlewares for rate limiting, retries, user-agent rotation, cookies, and more.
6//!
7//! ## Example
8//!
9//! ```rust,ignore
10//! use spider_middleware::rate_limit::RateLimitMiddleware;
11//! use spider_middleware::retry::RetryMiddleware;
12//!
13//! let crawler = CrawlerBuilder::new(MySpider)
14//! .add_middleware(RateLimitMiddleware::default())
15//! .add_middleware(RetryMiddleware::new())
16//! .build()
17//! .await?;
18//! ```
19
20pub mod middleware;
21pub mod rate_limit;
22pub mod referer;
23pub mod retry;
24
25pub use spider_util::request::{Body, Request};
26pub use spider_util::response::Response;
27
28pub mod prelude;
29
30#[cfg(feature = "middleware-user-agent")]
31pub mod user_agent;
32
33#[cfg(feature = "middleware-cookies")]
34pub mod cookies;
35
36#[cfg(feature = "middleware-cache")]
37pub mod http_cache;
38
39#[cfg(feature = "middleware-proxy")]
40pub mod proxy;
41
42#[cfg(feature = "middleware-robots")]
43pub mod robots;