Skip to main content

fetchium_core/http/
robots.rs

1//! robots.txt parser and per-domain cache (PRD §41).
2//!
3//! Fetches, caches, and respects robots.txt for all domains.
4//! Cache entries expire after 24 hours (configurable).
5
6use parking_lot::Mutex;
7use std::collections::HashMap;
8use std::time::{Duration, Instant};
9
10/// A parsed robots.txt directive set for a single user-agent.
11#[derive(Debug, Clone)]
12pub struct RobotRules {
13    /// Disallowed path prefixes.
14    pub disallowed: Vec<String>,
15    /// Optional crawl delay in seconds.
16    pub crawl_delay: Option<f64>,
17    /// When these rules were fetched.
18    pub fetched_at: Instant,
19}
20
21impl RobotRules {
22    /// Return `true` if the given path is allowed by these rules.
23    pub fn allows(&self, path: &str) -> bool {
24        for prefix in &self.disallowed {
25            if path.starts_with(prefix.as_str()) {
26                return false;
27            }
28        }
29        true
30    }
31
32    fn is_stale(&self, ttl: Duration) -> bool {
33        self.fetched_at.elapsed() > ttl
34    }
35}
36
37/// In-memory cache for robots.txt rules, keyed by domain.
38pub struct RobotsCache {
39    inner: Mutex<HashMap<String, RobotRules>>,
40    ttl: Duration,
41}
42
43impl Default for RobotsCache {
44    fn default() -> Self {
45        Self::new(Duration::from_secs(86400)) // 24h default TTL
46    }
47}
48
49impl RobotsCache {
50    /// Create a new cache with the given TTL for rules.
51    pub fn new(ttl: Duration) -> Self {
52        Self {
53            inner: Mutex::new(HashMap::new()),
54            ttl,
55        }
56    }
57
58    /// Get cached rules for a domain, or `None` if missing/stale.
59    pub fn get(&self, domain: &str) -> Option<RobotRules> {
60        let lock = self.inner.lock();
61        let rules = lock.get(domain)?;
62        if rules.is_stale(self.ttl) {
63            None
64        } else {
65            Some(rules.clone())
66        }
67    }
68
69    /// Insert or replace rules for a domain.
70    pub fn insert(&self, domain: &str, rules: RobotRules) {
71        self.inner.lock().insert(domain.to_string(), rules);
72    }
73
74    /// Check whether a URL is allowed. Returns `true` if no cached rules
75    /// exist (fail-open — don't block on missing data).
76    pub fn is_allowed(&self, url: &str) -> bool {
77        let Ok(parsed) = url::Url::parse(url) else {
78            return true;
79        };
80        let domain = match parsed.host_str() {
81            Some(h) => h.to_string(),
82            None => return true,
83        };
84        let path = parsed.path();
85        match self.get(&domain) {
86            Some(rules) => rules.allows(path),
87            None => true, // fail-open: no cached rules → allow
88        }
89    }
90}
91
92/// Parse the text content of a robots.txt file for `user-agent: *`.
93///
94/// Returns a [`RobotRules`] with disallowed paths and optional crawl delay.
95///
96/// # Examples
97/// ```rust
98/// use fetchium_core::http::robots::parse_robots_txt;
99/// let rules = parse_robots_txt("User-agent: *\nDisallow: /private/\nDisallow: /admin/");
100/// assert!(!rules.allows("/private/secret"));
101/// assert!(rules.allows("/public/page"));
102/// ```
103pub fn parse_robots_txt(content: &str) -> RobotRules {
104    let mut disallowed = Vec::new();
105    let mut crawl_delay: Option<f64> = None;
106    let mut in_wildcard_block = false;
107
108    for line in content.lines() {
109        let line = line.split('#').next().unwrap_or("").trim();
110        if line.is_empty() {
111            continue;
112        }
113        if let Some(agent) = line.strip_prefix("User-agent:") {
114            let agent = agent.trim();
115            in_wildcard_block = agent == "*" || agent.eq_ignore_ascii_case("fetchium");
116        } else if in_wildcard_block {
117            if let Some(path) = line.strip_prefix("Disallow:") {
118                let path = path.trim();
119                if !path.is_empty() {
120                    disallowed.push(path.to_string());
121                }
122            } else if let Some(delay) = line.strip_prefix("Crawl-delay:") {
123                crawl_delay = delay.trim().parse::<f64>().ok();
124            }
125        }
126    }
127
128    RobotRules {
129        disallowed,
130        crawl_delay,
131        fetched_at: Instant::now(),
132    }
133}
134
135#[cfg(test)]
136mod tests {
137    use super::*;
138
139    #[test]
140    fn parse_basic_disallow() {
141        let content = "User-agent: *\nDisallow: /admin/\nDisallow: /private/\n";
142        let rules = parse_robots_txt(content);
143        assert!(!rules.allows("/admin/panel"));
144        assert!(!rules.allows("/private/data"));
145        assert!(rules.allows("/public/page"));
146        assert!(rules.allows("/"));
147    }
148
149    #[test]
150    fn parse_crawl_delay() {
151        let content = "User-agent: *\nDisallow: /\nCrawl-delay: 2.5\n";
152        let rules = parse_robots_txt(content);
153        assert_eq!(rules.crawl_delay, Some(2.5));
154    }
155
156    #[test]
157    fn parse_comments_stripped() {
158        let content = "User-agent: * # all bots\nDisallow: /secret # private\n";
159        let rules = parse_robots_txt(content);
160        assert!(!rules.allows("/secret"));
161    }
162
163    #[test]
164    fn empty_disallow_means_allow_all() {
165        let content = "User-agent: *\nDisallow:\n";
166        let rules = parse_robots_txt(content);
167        assert!(rules.allows("/anything"));
168    }
169
170    #[test]
171    fn cache_is_allowed_fail_open_when_missing() {
172        let cache = RobotsCache::default();
173        // No cached rules → fail-open (allow)
174        assert!(cache.is_allowed("https://example.com/page"));
175    }
176
177    #[test]
178    fn cache_respects_disallow() {
179        let cache = RobotsCache::default();
180        let rules = parse_robots_txt("User-agent: *\nDisallow: /admin/\n");
181        cache.insert("example.com", rules);
182        assert!(!cache.is_allowed("https://example.com/admin/panel"));
183        assert!(cache.is_allowed("https://example.com/public/page"));
184    }
185}