siteone-crawler 2.1.0

Website crawler and QA toolkit in Rust for security, performance, SEO, and accessibility audits, offline cloning, markdown export, sitemap generation, cache warming, and CI/CD gating — one dependency-free binary for all major platforms, 10 tools in one.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

// SiteOne Crawler - SvelteProcessor
// (c) Jan Reges <jan.reges@siteone.cz>
//
// Handles SvelteKit specific patterns.

use once_cell::sync::Lazy;
use regex::Regex;

use crate::content_processor::base_processor::ProcessorConfig;
use crate::content_processor::content_processor::ContentProcessor;
use crate::engine::found_urls::FoundUrls;
use crate::engine::parsed_url::ParsedUrl;
use crate::types::ContentTypeId;

static RE_SVELTE_TAG: Lazy<Regex> = Lazy::new(|| Regex::new(r"(?i)<svelte:[^>]+>\s*").unwrap());

pub struct SvelteProcessor {
    #[allow(dead_code)]
    config: ProcessorConfig,
    debug_mode: bool,
}

impl SvelteProcessor {
    pub fn new(config: ProcessorConfig) -> Self {
        Self {
            config,
            debug_mode: false,
        }
    }
}

impl ContentProcessor for SvelteProcessor {
    fn find_urls(&self, _content: &str, _source_url: &ParsedUrl) -> Option<FoundUrls> {
        // SvelteProcessor doesn't extract URLs
        None
    }

    fn apply_content_changes_before_url_parsing(
        &self,
        _content: &mut String,
        _content_type: ContentTypeId,
        _url: &ParsedUrl,
    ) {
        // No changes needed before URL parsing in SvelteProcessor
    }

    fn apply_content_changes_for_offline_version(
        &self,
        content: &mut String,
        _content_type: ContentTypeId,
        _url: &ParsedUrl,
        _remove_unwanted_code: bool,
    ) {
        // Remove <svelte:*> tags for offline version
        if content.contains("<svelte:") {
            *content = RE_SVELTE_TAG.replace_all(content, "").to_string();
        }
    }

    fn is_content_type_relevant(&self, content_type: ContentTypeId) -> bool {
        // SvelteProcessor is only relevant for HTML (overrides the base relevantContentTypes)
        content_type == ContentTypeId::Html
    }

    fn get_name(&self) -> &str {
        "SvelteProcessor"
    }

    fn set_debug_mode(&mut self, debug_mode: bool) {
        self.debug_mode = debug_mode;
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn make_config() -> ProcessorConfig {
        ProcessorConfig::new(ParsedUrl::parse("https://example.com/", None))
    }

    #[test]
    fn test_remove_svelte_tags() {
        let processor = SvelteProcessor::new(make_config());
        let mut content = r#"<html><head><svelte:head></svelte:head></head><body>test</body></html>"#.to_string();
        let url = ParsedUrl::parse("https://example.com/", None);
        processor.apply_content_changes_for_offline_version(&mut content, ContentTypeId::Html, &url, false);
        assert!(!content.contains("<svelte:"));
    }

    #[test]
    fn test_is_relevant_only_for_html() {
        let processor = SvelteProcessor::new(make_config());
        assert!(processor.is_content_type_relevant(ContentTypeId::Html));
        assert!(!processor.is_content_type_relevant(ContentTypeId::Script));
        assert!(!processor.is_content_type_relevant(ContentTypeId::Stylesheet));
    }
}