robotxt 0.5.0

The implementation of the Robots.txt (or URL exclusion) protocol with the support of crawl-delay, sitemap and universal match extensions.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

#![forbid(unsafe_code)]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![doc = include_str!("./README.md")]

#[derive(Debug, thiserror::Error)]
pub enum Error {
    ///
    #[error("url parsing error: {0}")]
    Url(#[from] url::ParseError),
}

mod paths;
pub use paths::*;

#[cfg(feature = "builder")]
#[cfg_attr(docsrs, doc(cfg(feature = "builder")))]
mod build;
#[cfg(feature = "builder")]
pub use build::*;

#[cfg(feature = "parser")]
#[cfg_attr(docsrs, doc(cfg(feature = "parser")))]
mod parse;
#[cfg(feature = "parser")]
pub use parse::*;

// Re-exports
pub use url;

#[doc(hidden)]
pub mod prelude {
    pub use super::paths::*;
    pub use super::Error;

    #[cfg(feature = "builder")]
    pub use super::build::*;
    #[cfg(feature = "parser")]
    pub use super::parse::*;
}