rustai_core/http/
robots.rs1#[derive(Debug, Clone, Default, PartialEq)]
11pub struct Robots {
12 rules: Vec<(bool, String)>,
13 pub crawl_delay: Option<f64>,
15 pub sitemaps: Vec<String>,
17}
18
19impl Robots {
20 pub fn allow_all() -> Self {
22 Robots::default()
23 }
24
25 pub fn parse(body: &str, agent: &str) -> Self {
30 let agent_lc = agent.to_lowercase();
31 let mut specific: Vec<(bool, String)> = Vec::new();
32 let mut wildcard: Vec<(bool, String)> = Vec::new();
33 let mut specific_delay = None;
34 let mut wildcard_delay = None;
35 let mut sitemaps = Vec::new();
36
37 let mut in_specific = false;
39 let mut in_wildcard = false;
40 let mut last_was_agent = false;
41
42 for line in body.lines() {
43 let line = line.split('#').next().unwrap_or("").trim();
44 if line.is_empty() {
45 continue;
46 }
47 let Some((key, value)) = line.split_once(':') else { continue };
48 let key = key.trim().to_lowercase();
49 let value = value.trim();
50
51 match key.as_str() {
52 "user-agent" => {
53 if !last_was_agent {
54 in_specific = false;
55 in_wildcard = false;
56 }
57 let v = value.to_lowercase();
58 if v == "*" {
59 in_wildcard = true;
60 } else if agent_lc.contains(&v) || v.contains(&agent_lc) {
61 in_specific = true;
62 }
63 last_was_agent = true;
64 continue;
65 }
66 "sitemap" => {
67 sitemaps.push(value.to_string());
68 last_was_agent = false;
69 continue;
70 }
71 _ => {}
72 }
73 last_was_agent = false;
74
75 match key.as_str() {
76 "disallow" | "allow" => {
77 let allow = key == "allow";
78 if value.is_empty() && !allow {
80 continue;
81 }
82 if in_specific {
83 specific.push((allow, value.to_string()));
84 }
85 if in_wildcard {
86 wildcard.push((allow, value.to_string()));
87 }
88 }
89 "crawl-delay" => {
90 let d = value.parse::<f64>().ok().filter(|d| d.is_finite() && *d >= 0.0);
91 if in_specific {
92 specific_delay = d;
93 }
94 if in_wildcard {
95 wildcard_delay = d;
96 }
97 }
98 _ => {}
99 }
100 }
101
102 if specific.is_empty() && specific_delay.is_none() {
103 Robots { rules: wildcard, crawl_delay: wildcard_delay, sitemaps }
104 } else {
105 Robots { rules: specific, crawl_delay: specific_delay, sitemaps }
106 }
107 }
108
109 pub fn allows(&self, path: &str) -> bool {
111 let mut best: Option<(usize, bool)> = None;
112 for (allow, pattern) in &self.rules {
113 if !matches_pattern(pattern, path) {
114 continue;
115 }
116 let len = pattern.len();
117 match best {
118 Some((blen, ballow)) if blen > len || (blen == len && ballow) => {}
120 _ => best = Some((len, *allow)),
121 }
122 }
123 best.map(|(_, allow)| allow).unwrap_or(true)
124 }
125}
126
127fn matches_pattern(pattern: &str, path: &str) -> bool {
129 let anchored = pattern.ends_with('$');
130 let pattern = pattern.strip_suffix('$').unwrap_or(pattern);
131
132 let mut pos = 0usize;
133 let mut parts = pattern.split('*');
134 if let Some(first) = parts.next() {
136 if !path[pos..].starts_with(first) {
137 return false;
138 }
139 pos += first.len();
140 }
141 let segments: Vec<&str> = parts.collect();
142 for (i, seg) in segments.iter().enumerate() {
143 let last = i + 1 == segments.len();
144 if seg.is_empty() {
145 if last && anchored {
146 return true; }
148 continue;
149 }
150 if last && anchored {
151 return path[pos..].ends_with(seg);
152 }
153 match path[pos..].find(seg) {
154 Some(at) => pos += at + seg.len(),
155 None => return false,
156 }
157 }
158 if anchored && segments.is_empty() { pos == path.len() } else { true }
159}
160
161#[cfg(test)]
162mod tests {
163 use super::*;
164
165 const BODY: &str = r#"
166 # comment
167 User-agent: *
168 Disallow: /private/
169 Allow: /private/public/
170 Crawl-delay: 2
171
172 User-agent: rustai
173 Disallow: /nope
174 Disallow: /*.pdf$
175
176 Sitemap: https://example.com/sitemap.xml
177 "#;
178
179 #[test]
180 fn specific_group_wins_over_wildcard() {
181 let r = Robots::parse(BODY, "rustai/0.1");
182 assert!(r.allows("/private/secret"), "wildcard group should not apply");
183 assert!(!r.allows("/nope"));
184 }
185
186 #[test]
187 fn wildcard_group_applies_to_others() {
188 let r = Robots::parse(BODY, "SomeOtherBot");
189 assert!(!r.allows("/private/secret"));
190 assert!(r.allows("/private/public/page"));
191 assert_eq!(r.crawl_delay, Some(2.0));
192 }
193
194 #[test]
195 fn longest_match_wins_and_allow_breaks_ties() {
196 let r = Robots::parse("User-agent: *\nDisallow: /a/\nAllow: /a/", "x");
197 assert!(r.allows("/a/b"));
198 }
199
200 #[test]
201 fn dollar_anchors_the_end() {
202 let r = Robots::parse(BODY, "rustai");
203 assert!(!r.allows("/docs/manual.pdf"));
204 assert!(r.allows("/docs/manual.pdf?x=1"));
205 }
206
207 #[test]
208 fn empty_disallow_allows_everything() {
209 let r = Robots::parse("User-agent: *\nDisallow:", "x");
210 assert!(r.allows("/anything"));
211 }
212
213 #[test]
214 fn sitemaps_are_collected() {
215 let r = Robots::parse(BODY, "x");
216 assert_eq!(r.sitemaps, ["https://example.com/sitemap.xml"]);
217 }
218
219 #[test]
220 fn a_wildcard_crawl_delay_is_picked_up() {
221 let r = Robots::parse("User-agent: *\nCrawl-delay: 15\nAllow: /list\n", "rustai");
224 assert_eq!(r.crawl_delay, Some(15.0));
225 }
226
227 #[test]
228 fn a_crawl_delay_in_a_comment_is_not_a_directive() {
229 let r = Robots::parse(
232 "# semrushbot respects crawl-delay directives\nUser-agent: *\nDisallow: /w/\n",
233 "rustai",
234 );
235 assert_eq!(r.crawl_delay, None);
236 }
237
238 #[test]
239 fn missing_file_allows_all() {
240 assert!(Robots::allow_all().allows("/anything"));
241 }
242}