rover/fetcher/
mod.rs

1//! HTTP fetching, charset detection, SSRF enforcement.
2
3pub mod cache_control;
4pub mod cached;
5pub mod canonical;
6pub mod charset;
7pub mod client;
8pub mod dns;
9pub mod fetch;
10pub mod har;
11#[cfg(feature = "headless")]
12pub mod headless;
13pub mod ssrf;
14pub mod ttl;
15
16pub mod concurrency;
17pub mod rate_limit;
18pub mod retry;
19pub mod robots;
20
21pub use cached::{
22    CacheStatus, CachedFetch, ExtractResult, FetchOptions, HeadlessMode, fetch_with_cache,
23};
24pub use fetch::FetchedPage;
25
26use thiserror::Error;
27
28#[derive(Debug, Error)]
29pub enum FetcherError {
30    #[error("ssrf violation: {0}")]
31    Ssrf(#[from] ssrf::SsrfError),
32
33    #[error("http error: {0}")]
34    Http(#[from] reqwest::Error),
35
36    #[error("invalid url: {0}")]
37    Url(#[from] url::ParseError),
38
39    #[error("dns lookup failed for {host}: {source}")]
40    Dns {
41        host: String,
42        source: std::io::Error,
43    },
44
45    #[error("response decoding failed")]
46    Decode,
47
48    #[error("HTTP {status} from {url}")]
49    Status { status: u16, url: String },
50
51    #[error("storage error: {0}")]
52    Storage(#[from] crate::storage::StorageError),
53
54    #[error("extractor error: {0}")]
55    Extract(#[from] crate::extractor::ExtractorError),
56
57    #[error("retries exhausted after {attempts} attempts; last error: {last}")]
58    RetryExhausted {
59        attempts: u8,
60        last: Box<FetcherError>,
61    },
62
63    #[error("rate limited: server requested wait of {retry_after_secs}s")]
64    RateLimited { retry_after_secs: u64 },
65
66    #[error("robots.txt disallows {url} for user-agent {ua}")]
67    RobotsDisallowed { url: String, ua: String },
68
69    #[error("robots.txt fetch failed for {host}: {source}")]
70    RobotsFetchFailed {
71        host: String,
72        #[source]
73        source: Box<FetcherError>,
74    },
75
76    #[error("fetch deferred to retry task {task_id}")]
77    Deferred { task_id: String },
78
79    #[error("headless feature not compiled into this binary")]
80    HeadlessFeatureNotCompiled,
81
82    #[error("headless renderer is not wired into this fetcher")]
83    HeadlessRendererUnavailable,
84
85    #[cfg(feature = "headless")]
86    #[error("headless render failed: {0}")]
87    Headless(#[from] crate::fetcher::headless::HeadlessError),
88}
rover/fetcher/mod.rs

rover/fetcher/
mod.rs
