robotxt 0.6.1

The implementation of the Robots.txt (or URL exclusion) protocol with the support of crawl-delay, sitemap and universal match extensions.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

use std::collections::HashSet;
use std::fmt;

use url::Url;

pub use crate::build::group::GroupBuilder;
use crate::build::split::format_comment;

mod group;
mod split;

/// The set of formatted `user-agent` groups that can be written
/// in the `robots.txt` compliant format.
#[derive(Debug, Default, Clone)]
pub struct RobotsBuilder {
    groups: Vec<GroupBuilder>,
    sitemaps: HashSet<Url>,
    header: Option<String>,
    footer: Option<String>,
}

impl RobotsBuilder {
    /// Creates a new [`RobotsBuilder`] with default settings.
    pub fn new() -> Self {
        Self::default()
    }

    /// Adds a global header, usually used for permissions or legal notices.
    ///
    /// ```
    /// use robotxt::RobotsBuilder;
    ///
    /// let txt = RobotsBuilder::default()
    ///     .header("Note: Stop right there!")
    ///     .group(["*"], |u| u.disallow("/"))
    ///     .group(["foobot"], |u| u.allow("/"));
    /// ```
    pub fn header(mut self, header: &str) -> Self {
        self.header = Some(header.to_string());
        self
    }

    /// Adds a new `user-agent` group from the provided list of user-agents.
    ///
    /// ```
    /// use robotxt::RobotsBuilder;
    ///
    /// let txt = RobotsBuilder::default()
    ///     .group(["*"], |u| u.disallow("/"))
    ///     .group(["foobot"], |u| u.allow("/"));
    /// ```
    pub fn group<'a>(
        mut self,
        group: impl IntoIterator<Item = &'a str>,
        factory: impl FnOnce(GroupBuilder) -> GroupBuilder,
    ) -> Self {
        let section = GroupBuilder::from_iter(group);
        self.groups.push(factory(section));
        self
    }

    /// Adds the `Sitemap` directive from the URL address.
    ///
    /// ```
    /// use url::Url;
    /// use robotxt::RobotsBuilder;
    ///
    /// let txt = RobotsBuilder::default()
    ///     .sitemap("https://example.com/sitemap_1.xml".try_into().unwrap())
    ///     .sitemap("https://example.com/sitemap_1.xml".try_into().unwrap());
    /// ```
    pub fn sitemap(mut self, sitemap: Url) -> Self {
        self.sitemaps.insert(sitemap);
        self
    }

    /// Adds a global footer, usually used for notices.
    ///
    /// ```
    /// use robotxt::RobotsBuilder;
    ///
    /// let txt = RobotsBuilder::default()
    ///     .group(["*"], |u| u.disallow("/"))
    ///     .group(["foobot"], |u| u.allow("/"))
    ///     .footer("Note: Have a nice day!");
    /// ```
    pub fn footer(mut self, footer: &str) -> Self {
        self.footer = Some(footer.to_string());
        self
    }

    /// Parses the constructed output.
    /// See [`Robots::from_bytes`].
    ///
    /// [`Robots::from_bytes`]: crate::Robots::from_bytes
    #[cfg(feature = "parser")]
    #[cfg_attr(docsrs, doc(cfg(feature = "parser")))]
    pub fn parse(&self, user_agent: &str) -> crate::Robots {
        let txt = self.to_string();
        crate::Robots::from_bytes(txt.as_bytes(), user_agent)
    }
}

impl fmt::Display for RobotsBuilder {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let header = self.header.as_ref().map(|h| format_comment(h));
        let footer = self.footer.as_ref().map(|f| format_comment(f));

        let groups = self.groups.iter().map(|u| u.to_string());
        let groups = groups.collect::<Vec<_>>().join("\n\n");

        let result = [header, Some(groups), footer];
        let result = result.iter().filter_map(|u| u.clone());
        let result = result.collect::<Vec<_>>().join("\n\n");
        write!(f, "{}", result.as_str())
    }
}

#[cfg(test)]
mod builder {
    use crate::{Result, RobotsBuilder};

    #[test]
    fn readme() -> Result<()> {
        let txt = RobotsBuilder::default()
            .header("Robots.txt: Start")
            .group(["foobot"], |u| {
                u.crawl_delay(5)
                    .header("Rules for Foobot: Start")
                    .allow("/example/yeah.txt")
                    .disallow("/example/nope.txt")
                    .footer("Rules for Foobot: End")
            })
            .group(["barbot", "nombot"], |u| {
                u.crawl_delay(2)
                    .disallow("/example/yeah.txt")
                    .disallow("/example/nope.txt")
            })
            .sitemap("https://example.com/sitemap_1.xml".try_into()?)
            .sitemap("https://example.com/sitemap_2.xml".try_into()?)
            .footer("Robots.txt: End");

        println!("{}", txt.to_string());
        Ok(())
    }
}