use parking_lot::Mutex;
use std::collections::HashMap;
use std::time::{Duration, Instant};
#[derive(Debug, Clone)]
pub struct RobotRules {
pub disallowed: Vec<String>,
pub crawl_delay: Option<f64>,
pub fetched_at: Instant,
}
impl RobotRules {
pub fn allows(&self, path: &str) -> bool {
for prefix in &self.disallowed {
if path.starts_with(prefix.as_str()) {
return false;
}
}
true
}
fn is_stale(&self, ttl: Duration) -> bool {
self.fetched_at.elapsed() > ttl
}
}
pub struct RobotsCache {
inner: Mutex<HashMap<String, RobotRules>>,
ttl: Duration,
}
impl Default for RobotsCache {
fn default() -> Self {
Self::new(Duration::from_secs(86400)) }
}
impl RobotsCache {
pub fn new(ttl: Duration) -> Self {
Self {
inner: Mutex::new(HashMap::new()),
ttl,
}
}
pub fn get(&self, domain: &str) -> Option<RobotRules> {
let lock = self.inner.lock();
let rules = lock.get(domain)?;
if rules.is_stale(self.ttl) {
None
} else {
Some(rules.clone())
}
}
pub fn insert(&self, domain: &str, rules: RobotRules) {
self.inner.lock().insert(domain.to_string(), rules);
}
pub fn is_allowed(&self, url: &str) -> bool {
let Ok(parsed) = url::Url::parse(url) else {
return true;
};
let domain = match parsed.host_str() {
Some(h) => h.to_string(),
None => return true,
};
let path = parsed.path();
match self.get(&domain) {
Some(rules) => rules.allows(path),
None => true, }
}
}
pub fn parse_robots_txt(content: &str) -> RobotRules {
let mut disallowed = Vec::new();
let mut crawl_delay: Option<f64> = None;
let mut in_wildcard_block = false;
for line in content.lines() {
let line = line.split('#').next().unwrap_or("").trim();
if line.is_empty() {
continue;
}
if let Some(agent) = line.strip_prefix("User-agent:") {
let agent = agent.trim();
in_wildcard_block = agent == "*" || agent.eq_ignore_ascii_case("fetchium");
} else if in_wildcard_block {
if let Some(path) = line.strip_prefix("Disallow:") {
let path = path.trim();
if !path.is_empty() {
disallowed.push(path.to_string());
}
} else if let Some(delay) = line.strip_prefix("Crawl-delay:") {
crawl_delay = delay.trim().parse::<f64>().ok();
}
}
}
RobotRules {
disallowed,
crawl_delay,
fetched_at: Instant::now(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_basic_disallow() {
let content = "User-agent: *\nDisallow: /admin/\nDisallow: /private/\n";
let rules = parse_robots_txt(content);
assert!(!rules.allows("/admin/panel"));
assert!(!rules.allows("/private/data"));
assert!(rules.allows("/public/page"));
assert!(rules.allows("/"));
}
#[test]
fn parse_crawl_delay() {
let content = "User-agent: *\nDisallow: /\nCrawl-delay: 2.5\n";
let rules = parse_robots_txt(content);
assert_eq!(rules.crawl_delay, Some(2.5));
}
#[test]
fn parse_comments_stripped() {
let content = "User-agent: * # all bots\nDisallow: /secret # private\n";
let rules = parse_robots_txt(content);
assert!(!rules.allows("/secret"));
}
#[test]
fn empty_disallow_means_allow_all() {
let content = "User-agent: *\nDisallow:\n";
let rules = parse_robots_txt(content);
assert!(rules.allows("/anything"));
}
#[test]
fn cache_is_allowed_fail_open_when_missing() {
let cache = RobotsCache::default();
assert!(cache.is_allowed("https://example.com/page"));
}
#[test]
fn cache_respects_disallow() {
let cache = RobotsCache::default();
let rules = parse_robots_txt("User-agent: *\nDisallow: /admin/\n");
cache.insert("example.com", rules);
assert!(!cache.is_allowed("https://example.com/admin/panel"));
assert!(cache.is_allowed("https://example.com/public/page"));
}
}