argus-robots 0.1.0

Robots.txt parsing and caching for web crawling
Documentation 
1
2
3
4
5
6
7
8
9

mod cache;
mod parser;

pub use cache::RobotsCache;
pub use parser::{RobotsTxt, Rule};

pub fn is_allowed(_url: &str) -> bool {
    true
}