fetchium_core/http/
robots.rs1use parking_lot::Mutex;
7use std::collections::HashMap;
8use std::time::{Duration, Instant};
9
10#[derive(Debug, Clone)]
12pub struct RobotRules {
13 pub disallowed: Vec<String>,
15 pub crawl_delay: Option<f64>,
17 pub fetched_at: Instant,
19}
20
21impl RobotRules {
22 pub fn allows(&self, path: &str) -> bool {
24 for prefix in &self.disallowed {
25 if path.starts_with(prefix.as_str()) {
26 return false;
27 }
28 }
29 true
30 }
31
32 fn is_stale(&self, ttl: Duration) -> bool {
33 self.fetched_at.elapsed() > ttl
34 }
35}
36
37pub struct RobotsCache {
39 inner: Mutex<HashMap<String, RobotRules>>,
40 ttl: Duration,
41}
42
43impl Default for RobotsCache {
44 fn default() -> Self {
45 Self::new(Duration::from_secs(86400)) }
47}
48
49impl RobotsCache {
50 pub fn new(ttl: Duration) -> Self {
52 Self {
53 inner: Mutex::new(HashMap::new()),
54 ttl,
55 }
56 }
57
58 pub fn get(&self, domain: &str) -> Option<RobotRules> {
60 let lock = self.inner.lock();
61 let rules = lock.get(domain)?;
62 if rules.is_stale(self.ttl) {
63 None
64 } else {
65 Some(rules.clone())
66 }
67 }
68
69 pub fn insert(&self, domain: &str, rules: RobotRules) {
71 self.inner.lock().insert(domain.to_string(), rules);
72 }
73
74 pub fn is_allowed(&self, url: &str) -> bool {
77 let Ok(parsed) = url::Url::parse(url) else {
78 return true;
79 };
80 let domain = match parsed.host_str() {
81 Some(h) => h.to_string(),
82 None => return true,
83 };
84 let path = parsed.path();
85 match self.get(&domain) {
86 Some(rules) => rules.allows(path),
87 None => true, }
89 }
90}
91
92pub fn parse_robots_txt(content: &str) -> RobotRules {
104 let mut disallowed = Vec::new();
105 let mut crawl_delay: Option<f64> = None;
106 let mut in_wildcard_block = false;
107
108 for line in content.lines() {
109 let line = line.split('#').next().unwrap_or("").trim();
110 if line.is_empty() {
111 continue;
112 }
113 if let Some(agent) = line.strip_prefix("User-agent:") {
114 let agent = agent.trim();
115 in_wildcard_block = agent == "*" || agent.eq_ignore_ascii_case("fetchium");
116 } else if in_wildcard_block {
117 if let Some(path) = line.strip_prefix("Disallow:") {
118 let path = path.trim();
119 if !path.is_empty() {
120 disallowed.push(path.to_string());
121 }
122 } else if let Some(delay) = line.strip_prefix("Crawl-delay:") {
123 crawl_delay = delay.trim().parse::<f64>().ok();
124 }
125 }
126 }
127
128 RobotRules {
129 disallowed,
130 crawl_delay,
131 fetched_at: Instant::now(),
132 }
133}
134
135#[cfg(test)]
136mod tests {
137 use super::*;
138
139 #[test]
140 fn parse_basic_disallow() {
141 let content = "User-agent: *\nDisallow: /admin/\nDisallow: /private/\n";
142 let rules = parse_robots_txt(content);
143 assert!(!rules.allows("/admin/panel"));
144 assert!(!rules.allows("/private/data"));
145 assert!(rules.allows("/public/page"));
146 assert!(rules.allows("/"));
147 }
148
149 #[test]
150 fn parse_crawl_delay() {
151 let content = "User-agent: *\nDisallow: /\nCrawl-delay: 2.5\n";
152 let rules = parse_robots_txt(content);
153 assert_eq!(rules.crawl_delay, Some(2.5));
154 }
155
156 #[test]
157 fn parse_comments_stripped() {
158 let content = "User-agent: * # all bots\nDisallow: /secret # private\n";
159 let rules = parse_robots_txt(content);
160 assert!(!rules.allows("/secret"));
161 }
162
163 #[test]
164 fn empty_disallow_means_allow_all() {
165 let content = "User-agent: *\nDisallow:\n";
166 let rules = parse_robots_txt(content);
167 assert!(rules.allows("/anything"));
168 }
169
170 #[test]
171 fn cache_is_allowed_fail_open_when_missing() {
172 let cache = RobotsCache::default();
173 assert!(cache.is_allowed("https://example.com/page"));
175 }
176
177 #[test]
178 fn cache_respects_disallow() {
179 let cache = RobotsCache::default();
180 let rules = parse_robots_txt("User-agent: *\nDisallow: /admin/\n");
181 cache.insert("example.com", rules);
182 assert!(!cache.is_allowed("https://example.com/admin/panel"));
183 assert!(cache.is_allowed("https://example.com/public/page"));
184 }
185}