lychee_lib/ratelimit/
mod.rs

1//! Per-host rate limiting and concurrency control.
2//!
3//! This module provides adaptive rate limiting for HTTP requests on a per-host basis.
4//! It prevents overwhelming servers with too many concurrent requests and respects
5//! server-provided rate limit headers.
6//!
7//! # Architecture
8//!
9//! - [`HostKey`]: Represents a hostname/domain for rate limiting
10//! - [`Host`]: Manages rate limiting, concurrency, and caching for a specific host
11//! - [`HostPool`]: Coordinates multiple hosts and routes requests appropriately
12//! - [`HostConfig`]: Configuration for per-host behavior
13//! - [`HostStats`]: Statistics tracking for each host
14
15mod config;
16mod headers;
17mod host;
18mod pool;
19
20pub use config::{HostConfig, HostConfigs, RateLimitConfig};
21pub use host::{Host, HostKey, HostStats, HostStatsMap};
22use http::HeaderMap;
23pub use pool::{ClientMap, HostPool};
24use reqwest::Response;
25use url::Url;
26
27use crate::{ErrorKind, Result};
28
29/// The result of a HTTP request, used for internal per-host caching.
30/// This abstraction exists, because [`Response`] cannot easily be cached
31/// since it does not implement [`Clone`].
32#[derive(Debug, Clone)]
33pub(crate) struct CacheableResponse {
34    /// HTTP status code of the response.
35    pub(crate) status: reqwest::StatusCode,
36    /// Response body text. Only populated when `needs_body` was `true` in
37    /// [`HostPool::execute_request`].
38    pub(crate) text: Option<String>,
39    /// Response headers.
40    pub(crate) headers: HeaderMap,
41    /// Final URL after any redirects.
42    pub(crate) url: Url,
43}
44
45impl CacheableResponse {
46    async fn from_response(response: Response, needs_body: bool) -> Result<Self> {
47        let status = response.status();
48        let headers = response.headers().clone();
49        let url = response.url().clone();
50        let text = if needs_body {
51            Some(response.text().await.map_err(ErrorKind::ReadResponseBody)?)
52        } else {
53            None
54        };
55
56        Ok(Self {
57            status,
58            text,
59            headers,
60            url,
61        })
62    }
63}
lychee_lib/ratelimit/mod.rs

lychee_lib/ratelimit/
mod.rs
