Struct RobotsChecker 

pub struct RobotsChecker { /* private fields */ }

Robots.txt checker

Implementations

impl RobotsChecker

pub fn new(user_agent: &str, cache_ttl_secs: u64) -> Self

New checker

pub fn set_enabled(&mut self, enabled: bool)

Enable/disable robots.txt respect

pub fn is_enabled(&self) -> bool

Is robots.txt respected?

pub fn robots_url(url: &Url) -> Option<Url>

URL of the robots.txt for a site

pub fn cache_robots(&self, url: &Url, content: &str)

Caches a robots.txt

pub fn is_allowed(&self, url: &Url, robots_content: Option<&str>) -> bool

Checks if a URL is allowed

pub fn get_crawl_delay(&self, url: &Url) -> Option<Duration>

Retrieves the crawl-delay

pub fn cache(&self) -> &RobotsCache

Access to the cache