rover-fetch 0.2.0

An MCP server for fetching and prepping web content for LLM agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

//! HTTP fetching, charset detection, SSRF enforcement.

pub mod cache_control;
pub mod cached;
pub mod canonical;
pub mod charset;
pub mod client;
pub mod dns;
pub mod fetch;
pub mod har;
#[cfg(feature = "headless")]
pub mod headless;
pub mod ssrf;
pub mod ttl;

pub mod concurrency;
pub mod rate_limit;
pub mod retry;
pub mod robots;

pub use cached::{
    CacheStatus, CachedFetch, ExtractResult, FetchOptions, HeadlessMode, fetch_with_cache,
};
pub use fetch::FetchedPage;

use thiserror::Error;

#[derive(Debug, Error)]
pub enum FetcherError {
    #[error("ssrf violation: {0}")]
    Ssrf(#[from] ssrf::SsrfError),

    #[error("http error: {0}")]
    Http(#[from] reqwest::Error),

    #[error("invalid url: {0}")]
    Url(#[from] url::ParseError),

    #[error("dns lookup failed for {host}: {source}")]
    Dns {
        host: String,
        source: std::io::Error,
    },

    #[error("response decoding failed")]
    Decode,

    #[error("HTTP {status} from {url}")]
    Status { status: u16, url: String },

    #[error("storage error: {0}")]
    Storage(#[from] crate::storage::StorageError),

    #[error("extractor error: {0}")]
    Extract(#[from] crate::extractor::ExtractorError),

    #[error("retries exhausted after {attempts} attempts; last error: {last}")]
    RetryExhausted {
        attempts: u8,
        last: Box<FetcherError>,
    },

    #[error("rate limited: server requested wait of {retry_after_secs}s")]
    RateLimited { retry_after_secs: u64 },

    #[error("robots.txt disallows {url} for user-agent {ua}")]
    RobotsDisallowed { url: String, ua: String },

    #[error("robots.txt fetch failed for {host}: {source}")]
    RobotsFetchFailed {
        host: String,
        #[source]
        source: Box<FetcherError>,
    },

    #[error("fetch deferred to retry task {task_id}")]
    Deferred { task_id: String },

    #[error("headless feature not compiled into this binary")]
    HeadlessFeatureNotCompiled,

    #[error("headless renderer is not wired into this fetcher")]
    HeadlessRendererUnavailable,

    #[cfg(feature = "headless")]
    #[error("headless render failed: {0}")]
    Headless(#[from] crate::fetcher::headless::HeadlessError),
}