1mod fetch;
2mod pattern;
3
4use std::collections::HashSet;
5use std::time::Duration;
6
7use url::Url;
8
9#[cfg(feature = "html")]
10pub(crate) use pattern::normalize;
11pub(crate) use pattern::{Pattern, path_bytes};
12
13const ROBOTS_PATH: &str = "/robots.txt";
14const BYTE_ORDER_MARK: char = '\u{feff}';
15const ANY_AGENT: &str = "*";
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
18#[non_exhaustive]
19pub enum Purpose {
20 Search,
21 AiInput,
22 AiTrain,
23}
24
25const USAGE_LABELS: [(&str, Purpose); 3] = [
26 ("search", Purpose::Search),
27 ("ai-use", Purpose::AiInput),
28 ("train-ai", Purpose::AiTrain),
29];
30
31const SIGNAL_VALUES: [(&str, bool); 2] = [("yes", true), ("no", false)];
32const USAGE_VALUES: [(&str, bool); 2] = [("y", true), ("n", false)];
33
34impl Purpose {
35 const ALL: [Purpose; 3] = [Purpose::Search, Purpose::AiInput, Purpose::AiTrain];
36
37 #[must_use]
38 pub fn as_str(self) -> &'static str {
39 match self {
40 Purpose::Search => "search",
41 Purpose::AiInput => "ai-input",
42 Purpose::AiTrain => "ai-train",
43 }
44 }
45
46 fn parse(key: &str) -> Option<Purpose> {
47 lookup(
48 &Purpose::ALL.map(|purpose| (purpose.as_str(), purpose)),
49 key,
50 )
51 }
52
53 fn from_label(label: &str) -> Option<Purpose> {
54 lookup(&USAGE_LABELS, label)
55 }
56}
57
58pub(crate) fn lookup<T: Copy>(table: &[(&str, T)], key: &str) -> Option<T> {
59 table
60 .iter()
61 .find(|(name, _)| name.eq_ignore_ascii_case(key.trim()))
62 .map(|(_, value)| *value)
63}
64
65#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
66#[non_exhaustive]
67pub enum RobotsSource {
68 Parsed,
69 Unavailable,
70 Unreachable,
71}
72
73#[derive(Debug, Clone)]
74pub struct Robots {
75 source: RobotsSource,
76 rules: Vec<Rule>,
77 crawl_delay: Option<Duration>,
78 sitemaps: Vec<String>,
79 uses: Vec<UseRule>,
80}
81
82#[derive(Debug, Clone)]
83struct UseRule {
84 pattern: Option<Pattern>,
85 purpose: Purpose,
86 allowed: bool,
87}
88
89#[derive(Debug, Clone)]
90struct Rule {
91 pattern: Pattern,
92 allow: bool,
93}
94
95#[derive(Clone, Copy, PartialEq, Eq)]
96enum Field {
97 UserAgent,
98 Allow,
99 Disallow,
100 CrawlDelay,
101 Sitemap,
102 ContentSignal,
103 ContentUsage,
104}
105
106const FIELDS: [(&str, Field); 11] = [
107 ("user-agent", Field::UserAgent),
108 ("useragent", Field::UserAgent),
109 ("user agent", Field::UserAgent),
110 ("allow", Field::Allow),
111 ("disallow", Field::Disallow),
112 ("dissallow", Field::Disallow),
113 ("crawl-delay", Field::CrawlDelay),
114 ("sitemap", Field::Sitemap),
115 ("site-map", Field::Sitemap),
116 ("content-signal", Field::ContentSignal),
117 ("content-usage", Field::ContentUsage),
118];
119
120impl Field {
121 fn parse(key: &str) -> Option<Field> {
122 lookup(&FIELDS, key)
123 }
124}
125
126#[derive(Clone, PartialEq, Eq, Hash)]
127enum Agent {
128 Any,
129 Named(String),
130}
131
132impl Agent {
133 fn parse(token: &str) -> Option<Agent> {
134 if token.trim() == ANY_AGENT {
135 return Some(Agent::Any);
136 }
137 let name = product_token(token).to_ascii_lowercase();
138 (!name.is_empty()).then_some(Agent::Named(name))
139 }
140}
141
142pub(crate) fn product_token(agent: &str) -> &str {
143 let agent = agent.trim();
144 let end = agent
145 .find(|c: char| !(c.is_ascii_alphabetic() || matches!(c, '_' | '-')))
146 .unwrap_or(agent.len());
147 &agent[..end]
148}
149
150#[derive(Default)]
151struct Group {
152 agents: HashSet<Agent>,
153 rules: Vec<Rule>,
154 crawl_delay: Option<Duration>,
155 uses: Vec<UseRule>,
156}
157
158impl Group {
159 fn apply(&mut self, field: Field, value: &str) {
160 match field {
161 Field::Allow | Field::Disallow => {
162 if let Some(pattern) = Pattern::parse(value) {
163 let allow = field == Field::Allow;
164 self.rules.push(Rule { pattern, allow });
165 }
166 }
167 Field::CrawlDelay => {
168 let seconds = value
169 .parse::<f64>()
170 .ok()
171 .filter(|s| s.is_finite() && *s >= 0.0);
172 self.crawl_delay = self
173 .crawl_delay
174 .or(seconds.map(|s| Duration::try_from_secs_f64(s).unwrap_or(Duration::MAX)));
175 }
176 Field::ContentSignal => {
177 let prefs = value
178 .split(',')
179 .filter_map(|entry| pref(entry, signal_key, &SIGNAL_VALUES));
180 self.uses.extend(prefs.map(|(purpose, allowed)| UseRule {
181 pattern: None,
182 purpose,
183 allowed,
184 }));
185 }
186 Field::ContentUsage => {
187 let (pattern, prefs) = scoped_usage(value);
188 self.uses
189 .extend(usage_prefs(prefs).map(|(purpose, allowed)| UseRule {
190 pattern: pattern.clone(),
191 purpose,
192 allowed,
193 }));
194 }
195 Field::UserAgent | Field::Sitemap => {}
196 }
197 }
198}
199
200fn signal_key(key: &str) -> Option<Purpose> {
201 Purpose::parse(key.trim())
202}
203
204fn pref(
205 entry: &str,
206 purpose: fn(&str) -> Option<Purpose>,
207 values: &[(&str, bool)],
208) -> Option<(Purpose, bool)> {
209 let (key, value) = entry.split_once('=')?;
210 let value = value.split(';').next().unwrap_or_default();
211 Some((purpose(key)?, lookup(values, value)?))
212}
213
214fn usage_prefs(prefs: &str) -> impl Iterator<Item = (Purpose, bool)> + '_ {
215 prefs
216 .split(',')
217 .filter_map(|entry| pref(entry, Purpose::from_label, &USAGE_VALUES))
218}
219
220fn scoped_usage(value: &str) -> (Option<Pattern>, &str) {
221 match value.split_once(char::is_whitespace) {
222 Some((path, prefs)) if path.starts_with(['/', '*']) => (Pattern::parse(path), prefs),
223 _ => (None, value),
224 }
225}
226
227#[cfg(feature = "html")]
228pub(crate) fn header_uses(value: &str) -> impl Iterator<Item = (Purpose, bool)> + '_ {
229 usage_prefs(value)
230}
231
232impl Robots {
233 #[must_use]
234 pub fn parse(text: &str, agent: &str) -> Robots {
235 let mut groups: Vec<Group> = Vec::new();
236 let mut sitemaps = Vec::new();
237 let mut collecting_agents = false;
238 for line in text.trim_start_matches(BYTE_ORDER_MARK).split(['\n', '\r']) {
239 let line = line.split('#').next().unwrap_or_default();
240 let Some((key, value)) = line.split_once(':') else {
241 continue;
242 };
243 let Some(field) = Field::parse(key.trim()) else {
244 continue;
245 };
246 let value = value.trim();
247 match field {
248 Field::UserAgent => {
249 if !collecting_agents {
250 groups.push(Group::default());
251 collecting_agents = true;
252 }
253 if let (Some(group), Some(agent)) = (groups.last_mut(), Agent::parse(value)) {
254 group.agents.insert(agent);
255 }
256 }
257 Field::Sitemap if !value.is_empty() => sitemaps.push(value.to_owned()),
258 Field::Sitemap => {}
259 field => {
260 collecting_agents = false;
261 if let Some(group) = groups.last_mut() {
262 group.apply(field, value);
263 }
264 }
265 }
266 }
267 let wanted = match Agent::parse(agent) {
268 Some(named) if groups.iter().any(|group| group.agents.contains(&named)) => named,
269 _ => Agent::Any,
270 };
271 let mut robots = Robots::empty(RobotsSource::Parsed);
272 robots.sitemaps = sitemaps;
273 for group in groups
274 .into_iter()
275 .filter(|group| group.agents.contains(&wanted))
276 {
277 robots.rules.extend(group.rules);
278 robots.crawl_delay = robots.crawl_delay.max(group.crawl_delay);
279 robots.uses.extend(group.uses);
280 }
281 robots
282 }
283
284 pub(crate) fn empty(source: RobotsSource) -> Robots {
285 Robots {
286 source,
287 rules: Vec::new(),
288 crawl_delay: None,
289 sitemaps: Vec::new(),
290 uses: Vec::new(),
291 }
292 }
293
294 #[must_use]
295 pub fn allows(&self, url: &Url) -> bool {
296 if url.path() == ROBOTS_PATH {
297 return true;
298 }
299 match self.source {
300 RobotsSource::Unavailable => true,
301 RobotsSource::Unreachable => false,
302 RobotsSource::Parsed => {
303 let path = path_bytes(url);
304 self.rules
305 .iter()
306 .filter(|rule| rule.pattern.matches(&path))
307 .max_by_key(|rule| (rule.pattern.specificity(), rule.allow))
308 .is_none_or(|rule| rule.allow)
309 }
310 }
311 }
312
313 #[must_use]
314 pub fn allows_use(&self, url: &Url, purpose: Purpose) -> bool {
315 let path = path_bytes(url);
316 self.uses
317 .iter()
318 .filter(|rule| rule.purpose == purpose)
319 .filter_map(|rule| match &rule.pattern {
320 None => Some((0, rule.allowed)),
321 Some(pattern) if pattern.matches(&path) => {
322 Some((pattern.specificity(), rule.allowed))
323 }
324 Some(_) => None,
325 })
326 .max_by_key(|(specificity, allowed)| (*specificity, !allowed))
327 .is_none_or(|(_, allowed)| allowed)
328 }
329
330 #[must_use]
331 pub fn crawl_delay(&self) -> Option<Duration> {
332 self.crawl_delay
333 }
334
335 #[must_use]
336 pub fn sitemaps(&self) -> &[String] {
337 &self.sitemaps
338 }
339
340 #[must_use]
341 pub fn source(&self) -> RobotsSource {
342 self.source
343 }
344}