muffy 0.3.12

The static website validator
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

use robotxt::Robots;
use url::Url;

const USER_AGENT: &str = "MuffyBot";

pub struct RobotList {
    db: Robots,
}

impl RobotList {
    pub fn parse(source: &str) -> Self {
        Self {
            db: Robots::from_bytes(source.as_bytes(), USER_AGENT),
        }
    }

    pub fn is_allowed(&self, path: &str) -> bool {
        self.db.is_relative_allowed(path)
    }

    pub fn sitemaps(&self) -> impl Iterator<Item = &str> {
        self.db.sitemaps().iter().map(Url::as_str)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use indoc::indoc;
    use pretty_assertions::assert_eq;

    #[test]
    fn parse_sitemaps() {
        let source = indoc! {"
            user-agent: *
            sitemap: https://example.com/primary.xml\r
            sitemap: https://example.com/secondary.xml
            Sitemap: https://example.com/tertiary.xml
        "};
        let list = RobotList::parse(source);

        assert_eq!(
            list.sitemaps().collect::<Vec<_>>(),
            vec![
                "https://example.com/primary.xml",
                "https://example.com/secondary.xml",
                "https://example.com/tertiary.xml",
            ]
        );
    }

    #[test]
    fn parse_sitemaps_not_normalized() {
        let source = indoc! {"
            sitemap:https://example.com/no-space.xml
             sitemap: https://example.com/leading-space.xml
            SITEMAP: https://example.com/upper.xml
            Sitemap: https://example.com/title.xml
        "};
        let list = RobotList::parse(source);

        assert_eq!(
            list.sitemaps().collect::<Vec<_>>(),
            vec![
                "https://example.com/no-space.xml",
                "https://example.com/leading-space.xml",
                "https://example.com/upper.xml",
                "https://example.com/title.xml"
            ]
        );
    }

    #[test]
    fn disallow_path() {
        let source = indoc! {"
            user-agent: MuffyBot
            disallow: /private
        "};
        let list = RobotList::parse(source);

        assert!(list.is_allowed("/public"));
        assert!(!list.is_allowed("/private"));
    }

    #[test]
    fn allow_path() {
        let source = indoc! {"
            user-agent: MuffyBot
            disallow: /private
            allow: /private/public
        "};
        let list = RobotList::parse(source);

        assert!(list.is_allowed("/private/public"));
        assert!(!list.is_allowed("/private/secret"));
    }
}