siteone-crawler 2.5.1

Website crawler and QA toolkit in Rust for security, performance, SEO, and accessibility audits, offline cloning, markdown export, sitemap generation, cache warming, and CI/CD gating — one dependency-free binary for all major platforms, 10 tools in one.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

// SiteOne Crawler - SeoAndOpenGraphResult
// (c) Jan Reges <jan.reges@siteone.cz>

use super::heading_tree_item::HeadingTreeItem;

pub const ROBOTS_INDEX: i32 = 1;
pub const ROBOTS_NOINDEX: i32 = 0;
pub const ROBOTS_FOLLOW: i32 = 1;
pub const ROBOTS_NOFOLLOW: i32 = 2;

#[derive(Debug, Clone)]
pub struct SeoAndOpenGraphResult {
    pub url_uq_id: String,
    pub url_path_and_query: String,
    /// Host of the page itself, used to detect off-domain canonical links.
    pub page_host: Option<String>,

    pub title: Option<String>,
    pub description: Option<String>,
    pub keywords: Option<String>,
    pub h1: Option<String>,
    pub canonical: Option<String>,

    pub robots_index: Option<i32>,
    pub robots_follow: Option<i32>,
    pub denied_by_robots_txt: bool,

    pub og_title: Option<String>,
    pub og_type: Option<String>,
    pub og_image: Option<String>,
    pub og_url: Option<String>,
    pub og_description: Option<String>,
    pub og_site_name: Option<String>,

    pub twitter_card: Option<String>,
    pub twitter_site: Option<String>,
    pub twitter_creator: Option<String>,
    pub twitter_title: Option<String>,
    pub twitter_description: Option<String>,
    pub twitter_image: Option<String>,

    pub heading_tree_items: Vec<HeadingTreeItem>,
    pub headings_count: usize,
    pub headings_errors_count: usize,
}

impl SeoAndOpenGraphResult {
    pub fn new(url_uq_id: String, url_path_and_query: String) -> Self {
        Self {
            url_uq_id,
            url_path_and_query,
            page_host: None,
            title: None,
            description: None,
            keywords: None,
            h1: None,
            canonical: None,
            robots_index: None,
            robots_follow: None,
            denied_by_robots_txt: false,
            og_title: None,
            og_type: None,
            og_image: None,
            og_url: None,
            og_description: None,
            og_site_name: None,
            twitter_card: None,
            twitter_site: None,
            twitter_creator: None,
            twitter_title: None,
            twitter_description: None,
            twitter_image: None,
            heading_tree_items: Vec::new(),
            headings_count: 0,
            headings_errors_count: 0,
        }
    }

    /// Check if a URL is denied by robots.txt for our crawler's user agent.
    ///
    /// Delegates to the shared [`RobotsTxt`](crate::engine::robots_txt::RobotsTxt) parser so the
    /// SEO "Indexing" column uses the exact same rules as the crawler itself: user-agent scoping
    /// (`*` / `SiteOne-Crawler`), `Allow:` overrides, and wildcard/`$`-anchor matching.
    ///
    /// Historically this did a naive global scan of every `Disallow:` line regardless of its
    /// `User-agent:` block, so a single `Disallow: /` targeting another bot (e.g. GPTBot/CCBot)
    /// flagged every page as denied — the v2.4.0 "all pages DENY" regression (issue #105).
    pub fn is_denied_by_robots_txt(url: &str, robots_txt_content: &str) -> bool {
        if robots_txt_content.is_empty() {
            return false;
        }
        !crate::engine::robots_txt::RobotsTxt::parse(robots_txt_content).is_allowed(url)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Regression test for the v2.4.0 SEO "Indexing" column reporting every page as
    /// `DENY (robots.txt)` (issue #105). A `Disallow: /` under a *specific* bot's
    /// user-agent block (e.g. GPTBot/CCBot) must not deny pages for our crawler — only
    /// rules under `*` / `SiteOne-Crawler` apply.
    #[test]
    fn disallow_all_for_other_bot_does_not_deny_our_crawler() {
        let robots = "\
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: *
Disallow: /checkout/cart
";
        // Normal pages stay allowed despite GPTBot/CCBot being fully blocked.
        assert!(!SeoAndOpenGraphResult::is_denied_by_robots_txt(
            "https://example.com/about",
            robots
        ));
        assert!(!SeoAndOpenGraphResult::is_denied_by_robots_txt(
            "https://example.com/",
            robots
        ));
        // Paths disallowed for `*` are still correctly denied.
        assert!(SeoAndOpenGraphResult::is_denied_by_robots_txt(
            "https://example.com/checkout/cart",
            robots
        ));
    }

    /// `Allow:` rules must override a broader `Disallow:` (the naive scanner ignored them).
    #[test]
    fn allow_overrides_disallow() {
        let robots = "\
User-agent: *
Disallow: /admin/
Allow: /admin/public/
";
        assert!(SeoAndOpenGraphResult::is_denied_by_robots_txt(
            "https://example.com/admin/secret",
            robots
        ));
        assert!(!SeoAndOpenGraphResult::is_denied_by_robots_txt(
            "https://example.com/admin/public/page",
            robots
        ));
    }

    #[test]
    fn empty_robots_txt_denies_nothing() {
        assert!(!SeoAndOpenGraphResult::is_denied_by_robots_txt(
            "https://example.com/anything",
            ""
        ));
    }
}