1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

#![cfg_attr(feature = "clippy", allow(items_after_statements))]
#![cfg_attr(feature = "clippy", allow(match_bool))]

#![cfg_attr(feature = "release", deny(warnings))]


//! # Standard
//!
//! [A Standard for Robot Exclusion](http://www.robotstxt.org/orig.html)
//!
//! * User-agent
//! * Disallow
//!
//! # Additions
//!
//! * Allow
//! * Crawl-delay
//! * Request-rate
//! * Sitemap
//! * Host


pub extern crate url;

pub mod prelude;

pub mod builder;
pub use builder::*;

pub mod matcher;
pub use matcher::*;

pub mod parts;
pub use parts::*;

pub mod parse;
pub use parse::*;

pub mod render;
pub use render::*;