Crate robots_txt [−] [src]
Standard
A Standard for Robot Exclusion
- User-agent
- Disallow
Additions
- Allow
- Crawl-delay
- Request-rate
- Sitemap
- Host
Reexports
pub extern crate url; |
pub use builder::*; |
pub use matcher::*; |
pub use parts::*; |
pub use parse::*; |
pub use render::*; |
Modules
builder | |
matcher | |
parse | |
parts | |
prelude | |
render |