siteone-crawler 2.5.1

Website crawler and QA toolkit in Rust for security, performance, SEO, and accessibility audits, offline cloning, markdown export, sitemap generation, cache warming, and CI/CD gating — one dependency-free binary for all major platforms, 10 tools in one.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

// SiteOne Crawler - AnalyzerStats
// (c) Jan Reges <jan.reges@siteone.cz>

use std::collections::HashMap;

#[derive(Debug, Clone, Default)]
pub struct AnalyzerStats {
    /// analysis_name -> severity -> set of subject hashes (or just counted entries)
    severity_counts_per_analysis: HashMap<String, SeverityCounts>,
}

#[derive(Debug, Clone, Default)]
struct SeverityCounts {
    ok: HashMap<String, bool>,
    notice: HashMap<String, bool>,
    warning: HashMap<String, bool>,
    critical: HashMap<String, bool>,
}

impl AnalyzerStats {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn add_ok(&mut self, analysis_name: &str, subject: Option<&str>) {
        self.add_result(analysis_name, "ok", subject);
    }

    pub fn add_warning(&mut self, analysis_name: &str, subject: Option<&str>) {
        self.add_result(analysis_name, "warning", subject);
    }

    pub fn add_critical(&mut self, analysis_name: &str, subject: Option<&str>) {
        self.add_result(analysis_name, "critical", subject);
    }

    pub fn add_notice(&mut self, analysis_name: &str, subject: Option<&str>) {
        self.add_result(analysis_name, "notice", subject);
    }

    pub fn to_table_data(&self) -> Vec<HashMap<String, String>> {
        let mut result = Vec::new();
        for (analysis_name, counts) in &self.severity_counts_per_analysis {
            let mut row = HashMap::new();
            row.insert("analysisName".to_string(), analysis_name.clone());
            row.insert("ok".to_string(), counts.ok.len().to_string());
            row.insert("notice".to_string(), counts.notice.len().to_string());
            row.insert("warning".to_string(), counts.warning.len().to_string());
            row.insert("critical".to_string(), counts.critical.len().to_string());
            result.push(row);
        }
        result
    }

    fn add_result(&mut self, analysis_name: &str, severity: &str, subject: Option<&str>) {
        let counts = self
            .severity_counts_per_analysis
            .entry(analysis_name.to_string())
            .or_default();

        let subject_hash = subject.map(|s| {
            use md5::{Digest, Md5};
            let mut hasher = Md5::new();
            hasher.update(s.trim().as_bytes());
            let result = hasher.finalize();
            crate::utils::to_lower_hex(result)[..10].to_string()
        });

        let map = match severity {
            "ok" => &mut counts.ok,
            "notice" => &mut counts.notice,
            "warning" => &mut counts.warning,
            "critical" => &mut counts.critical,
            _ => return,
        };

        if let Some(hash) = subject_hash {
            map.insert(hash, true);
        } else {
            // Use a unique key based on current count
            let key = format!("_auto_{}", map.len());
            map.insert(key, true);
        }
    }
}