spire_core/crawler/robots/
mod.rs

1//! robots.txt, fetched once per origin and obeyed per RFC 9309.
2
3mod cache;
4mod source;
5
6use std::fmt;
7use std::sync::Arc;
8use std::time::Duration;
9
10pub(crate) use cache::{RobotsCache, Verdict};
11pub(crate) use source::{BackendSource, RobotsSource};
12
13use crate::backend::Backend;
14use crate::crawler::Tag;
15use crate::crawler::connector::Connector;
16
17/// How a crawl fetches and obeys robots.txt, see
18/// [`Crawler::with_robots`](crate::crawler::Crawler::with_robots).
19///
20/// robots.txt is fetched once per origin (scheme, host and port) before its
21/// first request, and handled as RFC 9309 describes:
22///
23/// - Its rules for [`user agent`](Self::new) decide what is crawled;
24///   disallowed requests are skipped, and `Crawl-delay` slows the host down.
25/// - A `4xx` other than `429`, or too many redirects, allows everything.
26/// - A `5xx`, a `429` or a network failure disallows everything while
27///   robots.txt is retried with backoff. Once the
28///   [retries](Self::with_unreachable_retries) are spent, the origin's
29///   requests are skipped.
30/// - Rules older than the [maximum age](Self::with_max_age) are fetched
31///   again; until then, and if fetching fails, the old rules apply.
32///
33/// robots.txt is fetched with the crawl's backend unless
34/// [`with_backend`](Self::with_backend) sets another, e.g. an HTTP client
35/// for a browser crawl. [`with_sitemaps`](Self::with_sitemaps) also queues
36/// the sitemaps it lists.
37///
38/// ```
39/// use std::time::Duration;
40///
41/// use spire_core::backend::Noop;
42/// use spire_core::crawler::Robots;
43///
44/// let robots = Robots::new("MyBot")
45///     .with_backend(Noop)
46///     .with_max_age(Duration::from_secs(3600));
47/// ```
48#[derive(Clone)]
49pub struct Robots {
50    pub(crate) agent: Arc<str>,
51    pub(crate) source: Option<Arc<dyn RobotsSource>>,
52    pub(crate) max_age: Duration,
53    pub(crate) retries: u32,
54    pub(crate) retry_delay: Duration,
55    pub(crate) sitemaps: Option<Tag>,
56}
57
58impl Robots {
59    /// Obeys robots.txt for `user_agent`, fetching it with the crawl's
60    /// backend, keeping it for 24 hours and retrying an unreachable one 3
61    /// times.
62    pub fn new(user_agent: impl Into<Arc<str>>) -> Self {
63        Self {
64            agent: user_agent.into(),
65            source: None,
66            max_age: Duration::from_hours(24),
67            retries: 3,
68            retry_delay: Duration::from_secs(30),
69            sitemaps: None,
70        }
71    }
72
73    /// Fetches robots.txt with `backend` instead of the crawl's.
74    #[must_use]
75    pub fn with_backend<B: Backend>(mut self, backend: B) -> Self {
76        let connector = Connector::new(Arc::new(backend));
77        self.source = Some(Arc::new(BackendSource::new(Arc::new(connector))));
78        self
79    }
80
81    /// Fetches robots.txt again once its rules are older than `age`.
82    /// RFC 9309 asks for at most 24 hours.
83    #[must_use]
84    pub const fn with_max_age(mut self, age: Duration) -> Self {
85        self.max_age = age;
86        self
87    }
88
89    /// Retries an unreachable robots.txt `retries` times, 30 seconds apart
90    /// and doubling, before skipping the origin's requests.
91    #[must_use]
92    pub const fn with_unreachable_retries(mut self, retries: u32) -> Self {
93        self.retries = retries;
94        self
95    }
96
97    /// Queues the sitemaps an origin's robots.txt lists in `Sitemap:` lines,
98    /// tagged `tag`, the first time it is fetched.
99    ///
100    /// They go through the frontier like any other seed, so its scope,
101    /// deduplication and depth limits apply. Route the tag to a handler that
102    /// parses them, e.g. `FollowSitemap` in `spire`.
103    #[must_use]
104    pub fn with_sitemaps(mut self, tag: impl Into<Tag>) -> Self {
105        self.sitemaps = Some(tag.into());
106        self
107    }
108
109    /// The delay before the `attempt`-th retry, starting at 1.
110    pub(crate) fn retry_delay(&self, attempt: u32) -> Duration {
111        let exponent = attempt.saturating_sub(1).min(16);
112        self.retry_delay
113            .saturating_mul(1 << exponent)
114            .min(Duration::from_mins(10))
115    }
116}
117
118impl From<&str> for Robots {
119    fn from(user_agent: &str) -> Self {
120        Self::new(user_agent)
121    }
122}
123
124impl From<String> for Robots {
125    fn from(user_agent: String) -> Self {
126        Self::new(user_agent)
127    }
128}
129
130impl fmt::Debug for Robots {
131    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
132        f.debug_struct("Robots")
133            .field("agent", &self.agent)
134            .field("own_backend", &self.source.is_some())
135            .field("max_age", &self.max_age)
136            .field("retries", &self.retries)
137            .field("retry_delay", &self.retry_delay)
138            .field("sitemaps", &self.sitemaps)
139            .finish()
140    }
141}
spire_core/crawler/robots/mod.rs

spire_core/crawler/robots/
mod.rs
