robots_txt/lib.rs
1#![allow(clippy::items_after_statements)]
2#![allow(clippy::match_bool)]
3#![cfg_attr(feature = "release", deny(warnings))]
4
5//! # Standard
6//!
7//! [A Standard for Robot Exclusion](http://www.robotstxt.org/orig.html)
8//!
9//! * User-agent
10//! * Disallow
11//!
12//! # Additions
13//!
14//! * Allow
15//! * Crawl-delay
16//! * Request-rate
17//! * Sitemap
18//! * Host
19
20pub mod builder;
21pub mod matcher;
22pub mod parse;
23pub mod parts;
24
25pub use self::parts::Robots;