siteone-crawler 2.0.2

Website crawler and QA toolkit in Rust for security, performance, SEO, and accessibility audits, offline cloning, markdown export, sitemap generation, cache warming, and CI/CD gating — one dependency-free binary for all major platforms, 10 tools in one.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

// SiteOne Crawler - Main entry point
// (c) Jan Reges <jan.reges@siteone.cz>

use siteone_crawler::engine::initiator::Initiator;
use siteone_crawler::utils;

#[tokio::main]
async fn main() {
    // Install the default crypto provider for rustls (needed by SSL/TLS analyzer)
    let _ = rustls::crypto::ring::default_provider().install_default();

    // Force ANSI color output
    utils::force_enabled_colors();

    // Set timezone early, before tokio runtime spawns threads.
    // We check argv directly to avoid duplicating full option parsing.
    {
        let argv: Vec<String> = std::env::args().collect();
        for i in 0..argv.len() {
            if let Some(tz) = argv[i].strip_prefix("--timezone=") {
                // SAFETY: Called before any threads are spawned by the runtime
                unsafe {
                    std::env::set_var("TZ", tz);
                }
                break;
            } else if argv[i] == "--timezone" && i + 1 < argv.len() {
                unsafe {
                    std::env::set_var("TZ", &argv[i + 1]);
                }
                break;
            }
        }
    }

    let argv: Vec<String> = std::env::args().collect();

    // Create initiator (parses CLI args, handles --help/--version)
    // On error: show ERROR, then help, then ERROR again
    let initiator = match Initiator::new(&argv) {
        Ok(i) => i,
        Err(e) => {
            // Extract inner message (strip "Config error: " prefix for display)
            let msg = match &e {
                siteone_crawler::error::CrawlerError::Config(inner) => inner.clone(),
                other => other.to_string(),
            };
            eprint!("{}", utils::get_color_text(&format!("ERROR: {}", msg), "red", false));
            Initiator::print_help();
            eprintln!(
                "{}",
                utils::get_color_text(&format!("\nERROR: {}\n", msg), "red", false)
            );
            std::process::exit(101);
        }
    };

    // Check for serve mode (built-in HTTP server for browsing exports)
    let serve_markdown = initiator.get_options().serve_markdown_dir.clone();
    let serve_offline = initiator.get_options().serve_offline_dir.clone();
    let serve_port = initiator.get_options().serve_port as u16;
    let serve_bind = initiator.get_options().serve_bind_address.clone();

    if let Some(dir) = serve_markdown {
        siteone_crawler::server::run(
            std::path::PathBuf::from(dir),
            siteone_crawler::server::ServeMode::Markdown,
            serve_port,
            &serve_bind,
        )
        .await;
        return;
    }
    if let Some(dir) = serve_offline {
        siteone_crawler::server::run(
            std::path::PathBuf::from(dir),
            siteone_crawler::server::ServeMode::Offline,
            serve_port,
            &serve_bind,
        )
        .await;
        return;
    }

    // Create manager from initiator
    let mut manager = match initiator.create_manager() {
        Ok(m) => m,
        Err(e) => {
            eprintln!("Error initializing crawler: {}", e);
            std::process::exit(1);
        }
    };

    // Run the crawler
    match manager.run().await {
        Ok(exit_code) => {
            if exit_code != 0 {
                std::process::exit(exit_code);
            }
        }
        Err(e) => {
            eprintln!("Crawler error: {}", e);
            std::process::exit(1);
        }
    }
}