use crate::impl_basic_error_conversions;
use crate::infrastructure::oxcache::CacheService;
use crate::utils::retry_policy::RetryPolicy;
use anyhow::Result;
use robotstxt::DefaultMatcher;
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use thiserror::Error;
use tokio::sync::Mutex;
use url::Url;
use async_trait::async_trait;
#[derive(Error, Debug)]
pub enum RobotsCheckerError {
#[error("Cache lock acquisition failed: {0}")]
CacheLockError(String),
#[error("URL parse failed: {0}")]
UrlParseError(String),
#[error("Validation failed: {0}")]
ValidationError(String),
}
impl_basic_error_conversions!(RobotsCheckerError, ValidationError);
impl From<crate::infrastructure::security::ssrf::SsrfError> for RobotsCheckerError {
fn from(err: crate::infrastructure::security::ssrf::SsrfError) -> Self {
RobotsCheckerError::ValidationError(err.to_string())
}
}
#[derive(Default, Clone)]
pub struct CacheStats {
pub hits: Arc<AtomicU64>,
pub misses: Arc<AtomicU64>,
}
impl CacheStats {
pub fn hits(&self) -> u64 {
self.hits.load(Ordering::Relaxed)
}
pub fn misses(&self) -> u64 {
self.misses.load(Ordering::Relaxed)
}
pub fn record_hit(&self) {
self.hits.fetch_add(1, Ordering::Relaxed);
}
pub fn record_miss(&self) {
self.misses.fetch_add(1, Ordering::Relaxed);
}
}
#[async_trait]
pub trait RobotsCheckerTrait: Send + Sync {
async fn is_allowed(&self, url_str: &str, user_agent: &str) -> Result<bool>;
async fn get_crawl_delay(&self, url_str: &str, user_agent: &str) -> Result<Option<Duration>>;
}
#[derive(Clone)]
struct CachedRobots {
content: Arc<str>,
expires_at: Instant,
fetch_failed: bool,
}
#[derive(Clone)]
struct FetchedRobots {
content: Arc<str>,
fetch_failed: bool,
}
const MAX_MEMORY_CACHE_ENTRIES: usize = 4096;
const MAX_MEMORY_CACHE_BYTES: usize = 16 * 1024 * 1024;
#[derive(Default)]
struct MemoryCache {
entries: HashMap<String, CachedRobots>,
total_bytes: usize,
}
impl MemoryCache {
fn remove(&mut self, key: &str) -> Option<CachedRobots> {
let removed = self.entries.remove(key);
if let Some(entry) = &removed {
self.total_bytes = self.total_bytes.saturating_sub(entry.content.len());
}
removed
}
fn insert(&mut self, key: String, value: CachedRobots) {
self.remove(&key);
let incoming = value.content.len();
if incoming > MAX_MEMORY_CACHE_BYTES {
return;
}
if self.total_bytes + incoming > MAX_MEMORY_CACHE_BYTES
|| self.entries.len() >= MAX_MEMORY_CACHE_ENTRIES
{
self.make_room(incoming);
}
if self.total_bytes + incoming > MAX_MEMORY_CACHE_BYTES
|| self.entries.len() >= MAX_MEMORY_CACHE_ENTRIES
{
return;
}
self.total_bytes += incoming;
self.entries.insert(key, value);
}
fn make_room(&mut self, incoming: usize) {
let mut candidates: Vec<(Instant, String)> = self
.entries
.iter()
.map(|(key, entry)| (entry.expires_at, key.clone()))
.collect();
candidates.sort_by_key(|(expires_at, _)| *expires_at);
for (_expires_at, key) in candidates {
if self.total_bytes + incoming <= MAX_MEMORY_CACHE_BYTES
&& self.entries.len() < MAX_MEMORY_CACHE_ENTRIES
{
break;
}
self.remove(&key);
}
}
}
const ROBOTS_FAILURE_CACHE_TTL: Duration = Duration::from_secs(60);
const MAX_ROBOTS_BODY_BYTES: usize = 512 * 1024;
const MAX_ROBOTS_PREALLOC_BYTES: usize = 64 * 1024;
#[derive(Clone)]
pub struct RobotsChecker {
fetcher: Arc<dyn RobotsFetcher>,
user_agent: String,
memory_cache: Arc<Mutex<MemoryCache>>,
cache_service: Option<Arc<dyn CacheService>>,
cache_ttl: Duration,
retry_policy: RetryPolicy,
fail_closed: bool,
cache_stats: Arc<CacheStats>,
}
#[async_trait]
impl RobotsCheckerTrait for RobotsChecker {
async fn is_allowed(&self, url_str: &str, user_agent: &str) -> Result<bool> {
let fetched = self.get_robots_content(url_str).await?;
if fetched.fetch_failed && self.fail_closed {
return Ok(false);
}
let url = Url::parse(url_str)?;
let mut matcher = DefaultMatcher::default();
let path_and_query = match url.query() {
Some(q) => format!("{}?{}", url.path(), q),
None => url.path().to_string(),
};
Ok(matcher.one_agent_allowed_by_robots(&fetched.content, user_agent, &path_and_query))
}
async fn get_crawl_delay(&self, url_str: &str, user_agent: &str) -> Result<Option<Duration>> {
let fetched = self.get_robots_content(url_str).await?;
Ok(self.parse_crawl_delay(&fetched.content, user_agent))
}
}
impl RobotsChecker {
pub fn new(
http_client: Arc<reqwest::Client>,
settings: &crate::config::settings::RobotsSettings,
cache_service: Option<Arc<dyn CacheService>>,
cache_stats: Option<Arc<CacheStats>>,
) -> Self {
Self {
fetcher: Arc::new(HttpRobotsFetcher {
client: http_client,
}),
user_agent: settings.user_agent.clone(),
memory_cache: Arc::new(Mutex::new(MemoryCache::default())),
cache_service,
cache_ttl: Duration::from_secs(settings.cache_ttl_secs),
retry_policy: RetryPolicy {
max_retries: settings.fetch_max_retries,
initial_backoff: Duration::from_secs(settings.fetch_initial_backoff_secs),
max_backoff: Duration::from_secs(settings.fetch_max_backoff_secs),
..Default::default()
},
fail_closed: settings.fail_closed,
cache_stats: cache_stats.unwrap_or_else(|| Arc::new(CacheStats::default())),
}
}
pub fn user_agent(&self) -> &str {
&self.user_agent
}
async fn get_robots_content(&self, url_str: &str) -> Result<FetchedRobots, RobotsCheckerError> {
let robots_url = build_robots_url(url_str)?;
if let Some(hit) = self.try_memory_cache(&robots_url).await {
return Ok(hit);
}
self.cache_stats.record_miss();
let cache_key = format!("robots_cache:{}", robots_url);
if let Some(hit) = self.try_cache_service(&cache_key, &robots_url).await {
return Ok(hit);
}
crate::infrastructure::security::ssrf::validate_url(&robots_url).await?;
match self.fetch_robots_text(&robots_url).await {
RobotsFetch::FailClosed | RobotsFetch::Rejected => Ok(FetchedRobots {
content: Arc::from(""),
fetch_failed: true,
}),
RobotsFetch::Content(content) => {
let content: Arc<str> = Arc::from(content);
self.store_fetched_content(&robots_url, &cache_key, &content)
.await;
Ok(FetchedRobots {
content,
fetch_failed: false,
})
}
}
}
async fn store_fetched_content(&self, robots_url: &str, cache_key: &str, content: &Arc<str>) {
self.store_memory_cache(robots_url, content).await;
if let Some(ref cache_service) = self.cache_service {
let _ = cache_service.set(cache_key, content.as_ref(), 86400).await;
}
}
async fn try_memory_cache(&self, robots_url: &str) -> Option<FetchedRobots> {
let mut cache = self.memory_cache.lock().await;
if let Some(cached) = cache.entries.get(robots_url) {
if cached.expires_at > Instant::now() {
let fetched = FetchedRobots {
content: Arc::clone(&cached.content),
fetch_failed: cached.fetch_failed,
};
drop(cache);
self.cache_stats.record_hit();
return Some(fetched);
}
cache.remove(robots_url);
}
None
}
async fn try_cache_service(&self, cache_key: &str, robots_url: &str) -> Option<FetchedRobots> {
if let Some(ref cache_service) = self.cache_service {
if let Ok(Some(content)) = cache_service.get(cache_key).await {
let content: Arc<str> = Arc::from(content.as_str());
let mut cache = self.memory_cache.lock().await;
cache.insert(
robots_url.to_string(),
CachedRobots {
content: Arc::clone(&content),
expires_at: Instant::now() + self.cache_ttl,
fetch_failed: false,
},
);
drop(cache);
self.cache_stats.record_hit();
return Some(FetchedRobots {
content,
fetch_failed: false,
});
}
}
None
}
async fn fetch_robots_text(&self, robots_url: &str) -> RobotsFetch {
let mut attempt = 0;
let mut last_error = None;
let mut rejected = false;
while attempt < self.retry_policy.max_retries {
attempt += 1;
match self.fetch_robots_once(robots_url).await {
RobotsFetchAttempt::Content(content) => {
return RobotsFetch::Content(content);
}
RobotsFetchAttempt::Retryable(err) => {
last_error = Some(err);
}
RobotsFetchAttempt::Rejected(err) => {
log::warn!("robots fetch rejected (no retry): {}", err);
last_error = Some(err);
rejected = true;
break;
}
}
if attempt < self.retry_policy.max_retries {
let backoff = self.retry_policy.calculate_backoff(attempt);
tokio::time::sleep(backoff).await;
}
}
if rejected {
self.store_failure_marker(robots_url).await;
return RobotsFetch::Rejected;
}
match last_error {
Some(err) => self.handle_robots_fetch_error(robots_url, err).await,
None => RobotsFetch::Content(String::new()),
}
}
async fn fetch_robots_once(&self, robots_url: &str) -> RobotsFetchAttempt {
self.fetcher.fetch(robots_url, &self.user_agent).await
}
async fn store_failure_marker(&self, robots_url: &str) {
let mut cache = self.memory_cache.lock().await;
cache.insert(
robots_url.to_string(),
CachedRobots {
content: Arc::from(""),
expires_at: Instant::now() + ROBOTS_FAILURE_CACHE_TTL,
fetch_failed: true,
},
);
}
async fn handle_robots_fetch_error(&self, robots_url: &str, err: anyhow::Error) -> RobotsFetch {
if self.fail_closed {
log::warn!(
"{}",
crate::i18n::tr_log_args(
"robots-fetch-fail-closed",
&[("url", robots_url.to_string()), ("error", err.to_string()),],
)
);
self.store_failure_marker(robots_url).await;
return RobotsFetch::FailClosed;
}
log::warn!("Failed to fetch robots.txt from {}: {}", robots_url, err);
RobotsFetch::Content(String::new())
}
async fn store_memory_cache(&self, robots_url: &str, content: &Arc<str>) {
let mut cache = self.memory_cache.lock().await;
cache.insert(
robots_url.to_string(),
CachedRobots {
content: Arc::clone(content),
expires_at: Instant::now() + self.cache_ttl, fetch_failed: false,
},
);
}
fn parse_crawl_delay(&self, content: &str, user_agent: &str) -> Option<Duration> {
let mut current_agent_matched = false;
let mut delay: Option<f64> = None;
let mut specific_agent_found = false;
for line in content.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
match classify_robots_line(line) {
RobotsLine::UserAgent(agent) => {
if agent == "*" {
current_agent_matched = !specific_agent_found;
} else if user_agent.to_lowercase().contains(&agent.to_lowercase()) {
current_agent_matched = true;
specific_agent_found = true;
delay = None;
} else {
current_agent_matched = false;
}
}
RobotsLine::CrawlDelay(raw) if current_agent_matched => {
apply_crawl_delay_value(&mut delay, raw);
}
RobotsLine::Other | RobotsLine::CrawlDelay(_) => {}
}
}
delay.map(Duration::from_secs_f64)
}
pub fn get_cache_stats(&self) -> (u64, u64) {
(self.cache_stats.hits(), self.cache_stats.misses())
}
}
fn build_robots_url(url_str: &str) -> Result<String, RobotsCheckerError> {
let url = Url::parse(url_str).map_err(|e| RobotsCheckerError::UrlParseError(e.to_string()))?;
let host = url
.host_str()
.ok_or_else(|| RobotsCheckerError::UrlParseError("Invalid URL: no host".to_string()))?;
let scheme = url.scheme();
let port = url.port_or_known_default().unwrap_or(80);
Ok(format!("{}://{}:{}/robots.txt", scheme, host, port))
}
enum RobotsFetch {
FailClosed,
Content(String),
Rejected,
}
pub enum RobotsFetchAttempt {
Content(String),
Retryable(anyhow::Error),
Rejected(anyhow::Error),
}
#[async_trait]
pub(crate) trait RobotsFetcher: Send + Sync {
async fn fetch(&self, robots_url: &str, user_agent: &str) -> RobotsFetchAttempt;
}
struct HttpRobotsFetcher {
client: Arc<reqwest::Client>,
}
#[async_trait]
impl RobotsFetcher for HttpRobotsFetcher {
async fn fetch(&self, robots_url: &str, user_agent: &str) -> RobotsFetchAttempt {
let response = self
.client
.get(robots_url)
.header(reqwest::header::USER_AGENT, user_agent.to_string())
.timeout(std::time::Duration::from_secs(5))
.send()
.await;
match response {
Ok(mut resp) => {
match resp.remote_addr() {
Some(addr)
if crate::infrastructure::security::ssrf::is_private_ip(addr.ip()) =>
{
return RobotsFetchAttempt::Rejected(anyhow::anyhow!(
"Robots fetch resolved to private/reserved address: {}",
addr.ip()
));
}
Some(_) => {}
None => {
log::debug!(
"robots fetch remote_addr unavailable; relying on DNS validation"
);
}
}
let status = resp.status().as_u16();
if (200..300).contains(&status) {
let capacity =
resp.content_length()
.unwrap_or(0)
.min(MAX_ROBOTS_PREALLOC_BYTES as u64) as usize;
let mut body: Vec<u8> = Vec::with_capacity(capacity);
loop {
match resp.chunk().await {
Ok(Some(chunk)) => {
if body.len() + chunk.len() > MAX_ROBOTS_BODY_BYTES {
return RobotsFetchAttempt::Rejected(anyhow::anyhow!(
"robots.txt exceeds {} bytes",
MAX_ROBOTS_BODY_BYTES
));
}
body.extend_from_slice(&chunk);
}
Ok(None) => break,
Err(e) => {
return RobotsFetchAttempt::Retryable(anyhow::anyhow!(
"Read body failed: {e}"
));
}
}
}
RobotsFetchAttempt::Content(match String::from_utf8(body) {
Ok(text) => text,
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
})
} else if status == 404 {
RobotsFetchAttempt::Content(String::new())
} else if status >= 500 {
RobotsFetchAttempt::Retryable(anyhow::anyhow!("Server error: {status}"))
} else {
RobotsFetchAttempt::Content(String::new())
}
}
Err(e) => RobotsFetchAttempt::Retryable(anyhow::anyhow!("Request failed: {e}")),
}
}
}
enum RobotsLine<'a> {
UserAgent(&'a str),
CrawlDelay(&'a str),
Other,
}
fn classify_robots_line(line: &str) -> RobotsLine<'_> {
let lower_line = line.to_lowercase();
if lower_line.starts_with("user-agent:") {
RobotsLine::UserAgent(line[11..].trim())
} else if lower_line.starts_with("crawl-delay:") {
RobotsLine::CrawlDelay(line[12..].trim())
} else {
RobotsLine::Other
}
}
fn apply_crawl_delay_value(delay: &mut Option<f64>, raw: &str) {
if let Ok(d) = raw.parse::<f64>() {
if d.is_finite() && d >= 0.0 {
*delay = Some(d);
} else {
log::warn!("Ignoring non-finite or negative Crawl-delay value: {}", d);
}
}
}
#[cfg(test)]
#[path = "tests/robots/mod.rs"]
mod tests;