robotparser_fork/
http.rs

1//! # Supported libraries
2//! To enable support for the required library, you need to add this feature to your `Cargo.toml`.
3//! Now only one library is supported - `reqwest`.
4//! But you can also add support for other libraries.
5
6use url::Origin;
7#[cfg(feature = "reqwest")]
8/// Support for reqwest library.
9pub mod reqwest;
10
11/// User agent of this crate.
12pub const DEFAULT_USER_AGENT: &str = "robotparser-rs (https://crates.io/crates/robotparser)";
13
14/// Trait to fetch and parse the robots.txt file.
15/// Must be implemented on http-client.
16pub trait RobotsTxtClient {
17    type Result;
18    fn fetch_robots_txt(&self, origin: Origin) -> Self::Result;
19}
robotparser_fork/http.rs

robotparser_fork/
http.rs
