spire_core/crawler/robots/
mod.rs

1//! robots.txt, fetched once per origin and obeyed per RFC 9309.
2
3mod cache;
4mod source;
5
6use std::fmt;
7use std::sync::Arc;
8use std::time::Duration;
9
10pub(crate) use cache::{RobotsCache, Verdict};
11pub(crate) use source::{BackendSource, RobotsSource};
12
13use crate::backend::Backend;
14use crate::request::Tag;
15
16/// How a crawl fetches and obeys robots.txt, see
17/// [`Crawler::with_robots`](crate::Crawler::with_robots).
18///
19/// robots.txt is fetched once per origin (scheme, host and port) before its
20/// first request, and handled as RFC 9309 describes:
21///
22/// - Its rules for [`user agent`](Self::new) decide what is crawled;
23///   disallowed requests are skipped, and `Crawl-delay` slows the host down.
24/// - A `4xx` other than `429`, or too many redirects, allows everything.
25/// - A `5xx`, a `429` or a network failure disallows everything while
26///   robots.txt is retried with backoff. Once the
27///   [retries](Self::with_unreachable_retries) are spent, the origin's
28///   requests are skipped.
29/// - Rules older than the [maximum age](Self::with_max_age) are fetched
30///   again; until then, and if fetching fails, the old rules apply.
31///
32/// robots.txt is fetched with the crawl's backend unless
33/// [`with_backend`](Self::with_backend) sets another, e.g. an HTTP client
34/// for a browser crawl. [`with_sitemaps`](Self::with_sitemaps) also queues
35/// the sitemaps it lists.
36///
37/// ```
38/// use std::time::Duration;
39///
40/// use spire_core::backend::Noop;
41/// use spire_core::crawler::Robots;
42///
43/// let robots = Robots::new("MyBot")
44///     .with_backend(Noop)
45///     .with_max_age(Duration::from_secs(3600));
46/// ```
47#[derive(Clone)]
48pub struct Robots {
49    pub(crate) agent: Arc<str>,
50    pub(crate) source: Option<Arc<dyn RobotsSource>>,
51    pub(crate) max_age: Duration,
52    pub(crate) retries: u32,
53    pub(crate) retry_delay: Duration,
54    pub(crate) sitemaps: Option<Tag>,
55}
56
57impl Robots {
58    /// Obeys robots.txt for `user_agent`, fetching it with the crawl's
59    /// backend, keeping it for 24 hours and retrying an unreachable one 3
60    /// times.
61    pub fn new(user_agent: impl Into<Arc<str>>) -> Self {
62        Self {
63            agent: user_agent.into(),
64            source: None,
65            max_age: Duration::from_hours(24),
66            retries: 3,
67            retry_delay: Duration::from_secs(30),
68            sitemaps: None,
69        }
70    }
71
72    /// Fetches robots.txt with `backend` instead of the crawl's.
73    #[must_use]
74    pub fn with_backend<B: Backend>(mut self, backend: B) -> Self {
75        self.source = Some(Arc::new(BackendSource::new(Arc::new(backend))));
76        self
77    }
78
79    /// Fetches robots.txt again once its rules are older than `age`.
80    /// RFC 9309 asks for at most 24 hours.
81    #[must_use]
82    pub const fn with_max_age(mut self, age: Duration) -> Self {
83        self.max_age = age;
84        self
85    }
86
87    /// Retries an unreachable robots.txt `retries` times, 30 seconds apart
88    /// and doubling, before skipping the origin's requests.
89    #[must_use]
90    pub const fn with_unreachable_retries(mut self, retries: u32) -> Self {
91        self.retries = retries;
92        self
93    }
94
95    /// Queues the sitemaps an origin's robots.txt lists in `Sitemap:` lines,
96    /// tagged `tag`, the first time it is fetched.
97    ///
98    /// They go through the frontier like any other seed, so its scope,
99    /// deduplication and depth limits apply. Route the tag to a handler that
100    /// parses them, e.g. `FollowSitemap` in `spire`.
101    #[must_use]
102    pub fn with_sitemaps(mut self, tag: impl Into<Tag>) -> Self {
103        self.sitemaps = Some(tag.into());
104        self
105    }
106
107    /// The delay before the `attempt`-th retry, starting at 1.
108    pub(crate) fn retry_delay(&self, attempt: u32) -> Duration {
109        let exponent = attempt.saturating_sub(1).min(16);
110        self.retry_delay
111            .saturating_mul(1 << exponent)
112            .min(Duration::from_mins(10))
113    }
114}
115
116impl From<&str> for Robots {
117    fn from(user_agent: &str) -> Self {
118        Self::new(user_agent)
119    }
120}
121
122impl From<String> for Robots {
123    fn from(user_agent: String) -> Self {
124        Self::new(user_agent)
125    }
126}
127
128impl fmt::Debug for Robots {
129    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130        f.debug_struct("Robots")
131            .field("agent", &self.agent)
132            .field("own_backend", &self.source.is_some())
133            .field("max_age", &self.max_age)
134            .field("retries", &self.retries)
135            .field("retry_delay", &self.retry_delay)
136            .field("sitemaps", &self.sitemaps)
137            .finish()
138    }
139}
spire_core/crawler/robots/mod.rs

spire_core/crawler/robots/
mod.rs
