Skip to main content

rustai_core/http/
robots.rs

1//! A small, strict `robots.txt` reader.
2//!
3//! Being a good citizen is not optional for a library people will point at
4//! other people's servers, so this is on by default. The implementation follows
5//! the parts of RFC 9309 that matter in practice: user-agent grouping, longest
6//! matching rule wins, `Allow` beats `Disallow` on equal length, `*` and `$`
7//! wildcards, and `Crawl-delay` as a courtesy extension.
8
9/// Parsed rules for one user-agent.
10#[derive(Debug, Clone, Default, PartialEq)]
11pub struct Robots {
12    rules: Vec<(bool, String)>,
13    /// Seconds the host asked us to wait between requests, if stated.
14    pub crawl_delay: Option<f64>,
15    /// Sitemaps advertised in the file.
16    pub sitemaps: Vec<String>,
17}
18
19impl Robots {
20    /// An empty policy that allows everything.
21    pub fn allow_all() -> Self {
22        Robots::default()
23    }
24
25    /// Parse `robots.txt`, keeping the group that applies to `agent`.
26    ///
27    /// A group naming our agent explicitly always wins over the `*` group, as
28    /// the RFC requires, even when `*` appears later in the file.
29    pub fn parse(body: &str, agent: &str) -> Self {
30        let agent_lc = agent.to_lowercase();
31        let mut specific: Vec<(bool, String)> = Vec::new();
32        let mut wildcard: Vec<(bool, String)> = Vec::new();
33        let mut specific_delay = None;
34        let mut wildcard_delay = None;
35        let mut sitemaps = Vec::new();
36
37        // Which groups the current run of `User-agent:` lines opened.
38        let mut in_specific = false;
39        let mut in_wildcard = false;
40        let mut last_was_agent = false;
41
42        for line in body.lines() {
43            let line = line.split('#').next().unwrap_or("").trim();
44            if line.is_empty() {
45                continue;
46            }
47            let Some((key, value)) = line.split_once(':') else { continue };
48            let key = key.trim().to_lowercase();
49            let value = value.trim();
50
51            match key.as_str() {
52                "user-agent" => {
53                    if !last_was_agent {
54                        in_specific = false;
55                        in_wildcard = false;
56                    }
57                    let v = value.to_lowercase();
58                    if v == "*" {
59                        in_wildcard = true;
60                    } else if agent_lc.contains(&v) || v.contains(&agent_lc) {
61                        in_specific = true;
62                    }
63                    last_was_agent = true;
64                    continue;
65                }
66                "sitemap" => {
67                    sitemaps.push(value.to_string());
68                    last_was_agent = false;
69                    continue;
70                }
71                _ => {}
72            }
73            last_was_agent = false;
74
75            match key.as_str() {
76                "disallow" | "allow" => {
77                    let allow = key == "allow";
78                    // `Disallow:` with an empty value means "allow everything".
79                    if value.is_empty() && !allow {
80                        continue;
81                    }
82                    if in_specific {
83                        specific.push((allow, value.to_string()));
84                    }
85                    if in_wildcard {
86                        wildcard.push((allow, value.to_string()));
87                    }
88                }
89                "crawl-delay" => {
90                    let d = value.parse::<f64>().ok().filter(|d| d.is_finite() && *d >= 0.0);
91                    if in_specific {
92                        specific_delay = d;
93                    }
94                    if in_wildcard {
95                        wildcard_delay = d;
96                    }
97                }
98                _ => {}
99            }
100        }
101
102        if specific.is_empty() && specific_delay.is_none() {
103            Robots { rules: wildcard, crawl_delay: wildcard_delay, sitemaps }
104        } else {
105            Robots { rules: specific, crawl_delay: specific_delay, sitemaps }
106        }
107    }
108
109    /// May we fetch this path (including query string)?
110    pub fn allows(&self, path: &str) -> bool {
111        let mut best: Option<(usize, bool)> = None;
112        for (allow, pattern) in &self.rules {
113            if !matches_pattern(pattern, path) {
114                continue;
115            }
116            let len = pattern.len();
117            match best {
118                // Equal specificity: `Allow` wins, per RFC 9309 ยง2.2.2.
119                Some((blen, ballow)) if blen > len || (blen == len && ballow) => {}
120                _ => best = Some((len, *allow)),
121            }
122        }
123        best.map(|(_, allow)| allow).unwrap_or(true)
124    }
125}
126
127/// Glob matching limited to the two wildcards robots.txt defines.
128fn matches_pattern(pattern: &str, path: &str) -> bool {
129    let anchored = pattern.ends_with('$');
130    let pattern = pattern.strip_suffix('$').unwrap_or(pattern);
131
132    let mut pos = 0usize;
133    let mut parts = pattern.split('*');
134    // The first segment must match at the very start.
135    if let Some(first) = parts.next() {
136        if !path[pos..].starts_with(first) {
137            return false;
138        }
139        pos += first.len();
140    }
141    let segments: Vec<&str> = parts.collect();
142    for (i, seg) in segments.iter().enumerate() {
143        let last = i + 1 == segments.len();
144        if seg.is_empty() {
145            if last && anchored {
146                return true; // trailing `*$`
147            }
148            continue;
149        }
150        if last && anchored {
151            return path[pos..].ends_with(seg);
152        }
153        match path[pos..].find(seg) {
154            Some(at) => pos += at + seg.len(),
155            None => return false,
156        }
157    }
158    if anchored && segments.is_empty() { pos == path.len() } else { true }
159}
160
161#[cfg(test)]
162mod tests {
163    use super::*;
164
165    const BODY: &str = r#"
166        # comment
167        User-agent: *
168        Disallow: /private/
169        Allow: /private/public/
170        Crawl-delay: 2
171
172        User-agent: rustai
173        Disallow: /nope
174        Disallow: /*.pdf$
175
176        Sitemap: https://example.com/sitemap.xml
177    "#;
178
179    #[test]
180    fn specific_group_wins_over_wildcard() {
181        let r = Robots::parse(BODY, "rustai/0.1");
182        assert!(r.allows("/private/secret"), "wildcard group should not apply");
183        assert!(!r.allows("/nope"));
184    }
185
186    #[test]
187    fn wildcard_group_applies_to_others() {
188        let r = Robots::parse(BODY, "SomeOtherBot");
189        assert!(!r.allows("/private/secret"));
190        assert!(r.allows("/private/public/page"));
191        assert_eq!(r.crawl_delay, Some(2.0));
192    }
193
194    #[test]
195    fn longest_match_wins_and_allow_breaks_ties() {
196        let r = Robots::parse("User-agent: *\nDisallow: /a/\nAllow: /a/", "x");
197        assert!(r.allows("/a/b"));
198    }
199
200    #[test]
201    fn dollar_anchors_the_end() {
202        let r = Robots::parse(BODY, "rustai");
203        assert!(!r.allows("/docs/manual.pdf"));
204        assert!(r.allows("/docs/manual.pdf?x=1"));
205    }
206
207    #[test]
208    fn empty_disallow_allows_everything() {
209        let r = Robots::parse("User-agent: *\nDisallow:", "x");
210        assert!(r.allows("/anything"));
211    }
212
213    #[test]
214    fn sitemaps_are_collected() {
215        let r = Robots::parse(BODY, "x");
216        assert_eq!(r.sitemaps, ["https://example.com/sitemap.xml"]);
217    }
218
219    #[test]
220    fn a_wildcard_crawl_delay_is_picked_up() {
221        // arXiv publishes exactly this, and honouring it is why a second
222        // request to the same host through one client waits fifteen seconds.
223        let r = Robots::parse("User-agent: *\nCrawl-delay: 15\nAllow: /list\n", "rustai");
224        assert_eq!(r.crawl_delay, Some(15.0));
225    }
226
227    #[test]
228    fn a_crawl_delay_in_a_comment_is_not_a_directive() {
229        // Wikipedia's robots.txt discusses crawl-delay in prose. Reading that
230        // as a directive would stall every request to it.
231        let r = Robots::parse(
232            "# semrushbot respects crawl-delay directives\nUser-agent: *\nDisallow: /w/\n",
233            "rustai",
234        );
235        assert_eq!(r.crawl_delay, None);
236    }
237
238    #[test]
239    fn missing_file_allows_all() {
240        assert!(Robots::allow_all().allows("/anything"));
241    }
242}