robotparser_fork/
lib.rs

1//! robots.txt parser for Rust
2//!
3//! The robots.txt Exclusion Protocol is implemented as specified in
4//! <https://www.robotstxt.org/norobots-rfc.txt>
5//!
6//! # Installation
7//!
8//! Add it to your ``Cargo.toml``:
9//!
10//! ```toml
11//! [dependencies]
12//! robotparser = "0.11"
13//! ```
14//!
15//!
16//! # Examples
17//!
18//! ```rust
19//! use robotparser::http::RobotsTxtClient;
20//! use robotparser::service::RobotsTxtService;
21//! use reqwest::blocking::Client;
22//! use url::Url;
23//!
24//! let client = Client::new();
25//! let robots_txt_url = Url::parse("https://www.python.org/robots.txt").unwrap();
26//! let robots_txt = client.fetch_robots_txt(robots_txt_url.origin()).unwrap().get_result();
27//! let fetch_url = Url::parse("https://www.python.org/robots.txt").unwrap();
28//! assert!(robots_txt.can_fetch("*", &fetch_url));
29//! ```
30
31/// Request builder & response parsers for other http libraries.
32pub mod http;
33/// Contains models of robots.txt file.
34pub mod model;
35/// Contains robots.txt parsers.
36pub mod parser;
37/// Contains robots.txt services.
38pub mod service;