robotxt 0.6.1

The implementation of the Robots.txt (or URL exclusion) protocol with the support of crawl-delay, sitemap and universal match extensions.
Documentation 
1
2
3
4
5
6
7
8
9

pub use create::create_url;
pub(crate) use normal::normalize_path;

mod create;
mod normal;

/// Google currently enforces a `robots.txt` file size limit of 500 kibibytes (KiB).
/// See [How Google interprets Robots.txt](https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt).
pub const BYTE_LIMIT: usize = 512_000;