siteprobe 0.5.0

Siteprobe is a Rust-based CLI tool that fetches all URLs from a given `sitemap.xml` url, checks their existence, and generates a performance report. It supports various features such as authentication, concurrency control, caching bypass, and more.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

mod metrics;
mod network;
mod options;
mod report;
mod sitemap;
mod storage;
mod utils;

use std::error::Error;
use std::process::ExitCode;
use std::sync::Arc;

use crate::sitemap::{fetch_and_generate_report, get_sitemap_urls};
use clap::Parser;
use console::style;
use tokio::time::Instant;

#[tokio::main]
async fn main() -> Result<ExitCode, Box<dyn Error>> {
    // Parse terminal arguments.
    let options = options::Cli::parse();

    // Build the HTTP client.
    let client = Arc::new(network::build_client(&options)?);
    let start_time = Instant::now();

    // Fetch all URLs from the sitemap.
    let urls = get_sitemap_urls(options.sitemap_url.as_str(), &client)
        .await
        .map_err(|e| {
            eprintln!("{} {}", style("[ERROR]").red(), e);
            std::process::exit(1);
        })?;

    // Fetch URLs concurrently and generate a report.
    let report = fetch_and_generate_report(urls, &client, &options, &start_time).await?;

    // Display the report.
    report.show_text_report(&options);

    // Optionally, write the report to CSV file.
    if let Some(path) = options.report_path.as_ref() {
        report.write_csv_report(path)?;
    }

    // Optionally, write the report to JSON file.
    if let Some(path) = options.report_path_json.as_ref() {
        report.write_json_report(&options, path)?;
    }

    Ok(ExitCode::SUCCESS)
}