spire_core/crawler/robots/
mod.rs

1//! robots.txt, fetched once per origin and obeyed per RFC 9309.
2
3mod cache;
4mod source;
5
6use std::fmt;
7use std::sync::Arc;
8use std::time::Duration;
9
10pub(crate) use cache::{RobotsCache, Verdict};
11pub(crate) use source::{BackendSource, RobotsSource};
12
13use crate::backend::Backend;
14
15/// How a crawl fetches and obeys robots.txt, see
16/// [`Crawler::with_robots`](crate::Crawler::with_robots).
17///
18/// robots.txt is fetched once per origin (scheme, host and port) before its
19/// first request, and handled as RFC 9309 describes:
20///
21/// - Its rules for [`user agent`](Self::new) decide what is crawled;
22///   disallowed requests are skipped, and `Crawl-delay` slows the host down.
23/// - A `4xx` other than `429`, or too many redirects, allows everything.
24/// - A `5xx`, a `429` or a network failure disallows everything while
25///   robots.txt is retried with backoff. Once the
26///   [retries](Self::with_unreachable_retries) are spent, the origin's
27///   requests are skipped.
28/// - Rules older than the [maximum age](Self::with_max_age) are fetched
29///   again; until then, and if fetching fails, the old rules apply.
30///
31/// robots.txt is fetched with the crawl's backend unless
32/// [`with_backend`](Self::with_backend) sets another, e.g. an HTTP client
33/// for a browser crawl.
34///
35/// ```
36/// use std::time::Duration;
37///
38/// use spire_core::backend::Noop;
39/// use spire_core::crawler::Robots;
40///
41/// let robots = Robots::new("MyBot")
42///     .with_backend(Noop)
43///     .with_max_age(Duration::from_secs(3600));
44/// ```
45#[derive(Clone)]
46pub struct Robots {
47    pub(crate) agent: Arc<str>,
48    pub(crate) source: Option<Arc<dyn RobotsSource>>,
49    pub(crate) max_age: Duration,
50    pub(crate) retries: u32,
51    pub(crate) retry_delay: Duration,
52}
53
54impl Robots {
55    /// Obeys robots.txt for `user_agent`, fetching it with the crawl's
56    /// backend, keeping it for 24 hours and retrying an unreachable one 3
57    /// times.
58    pub fn new(user_agent: impl Into<Arc<str>>) -> Self {
59        Self {
60            agent: user_agent.into(),
61            source: None,
62            max_age: Duration::from_hours(24),
63            retries: 3,
64            retry_delay: Duration::from_secs(30),
65        }
66    }
67
68    /// Fetches robots.txt with `backend` instead of the crawl's.
69    #[must_use]
70    pub fn with_backend<B: Backend>(mut self, backend: B) -> Self {
71        self.source = Some(Arc::new(BackendSource::new(Arc::new(backend))));
72        self
73    }
74
75    /// Fetches robots.txt again once its rules are older than `age`.
76    /// RFC 9309 asks for at most 24 hours.
77    #[must_use]
78    pub const fn with_max_age(mut self, age: Duration) -> Self {
79        self.max_age = age;
80        self
81    }
82
83    /// Retries an unreachable robots.txt `retries` times, 30 seconds apart
84    /// and doubling, before skipping the origin's requests.
85    #[must_use]
86    pub const fn with_unreachable_retries(mut self, retries: u32) -> Self {
87        self.retries = retries;
88        self
89    }
90
91    /// The delay before the `attempt`-th retry, starting at 1.
92    pub(crate) fn retry_delay(&self, attempt: u32) -> Duration {
93        let exponent = attempt.saturating_sub(1).min(16);
94        self.retry_delay
95            .saturating_mul(1 << exponent)
96            .min(Duration::from_mins(10))
97    }
98}
99
100impl From<&str> for Robots {
101    fn from(user_agent: &str) -> Self {
102        Self::new(user_agent)
103    }
104}
105
106impl From<String> for Robots {
107    fn from(user_agent: String) -> Self {
108        Self::new(user_agent)
109    }
110}
111
112impl fmt::Debug for Robots {
113    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
114        f.debug_struct("Robots")
115            .field("agent", &self.agent)
116            .field("own_backend", &self.source.is_some())
117            .field("max_age", &self.max_age)
118            .field("retries", &self.retries)
119            .field("retry_delay", &self.retry_delay)
120            .finish()
121    }
122}
spire_core/crawler/robots/mod.rs

spire_core/crawler/robots/
mod.rs
