Crate robots_txt [−] [src]
Standard
A Standard for Robot Exclusion
- User-agent
- Disallow
Additions
- Allow
- Crawl-delay
- Request-rate
- Sitemap
- Host
Reexports
pub extern crate url; |
pub use builder::*; |
pub use matcher::*; |
pub use parts::*; |
pub use parse::*; |
pub use render::*; |
Modules
| builder | |
| matcher | |
| parse | |
| parts | |
| prelude | |
| render |