Module politeness 

Politeness - Robots.txt and throttling

Re-exports

pub use robots::RobotsChecker;

pub use robots::RobotsCache;

pub use throttle::DomainThrottler;

pub use throttle::ThrottleState;

Modules

robots

Robots - Parsing and respecting robots.txt (RFC 9309)

throttle

Throttle - Rate limiting per domain