awesome-sails-benchmarks 0.3.0

Benchmarking for Sails programs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

// This file is part of Gear.

// Copyright (C) 2026 Gear Technologies Inc.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

use anyhow::Result;
use awesome_sails_benchmarks::{BenchStorage, ComparisonConfig, ReportBuilder};
use clap::Parser;
use std::{fs, path::PathBuf};

#[derive(Parser)]
#[command(
    name = "bench-analyzer",
    about = "A CLI tool to compare two benchmark JSON files and detect performance regressions."
)]
struct Cli {
    /// Path to the current benchmark results (the ones you just generated).
    #[arg(long)]
    current: PathBuf,
    /// Path to the baseline benchmark results (the reference file to compare against).
    #[arg(long)]
    other: PathBuf,
    /// Path to save the comparison report in Markdown format.
    #[arg(long)]
    output: Option<PathBuf>,
    /// Custom regression threshold percentage (e.g. 5.0).
    #[arg(long)]
    threshold: Option<f64>,
    /// Exit with code 1 if a significant regression is detected.
    #[arg(long)]
    fail_on_regression: bool,
    /// Enable strict mode: the tool will fail if ANY metric deviates from baseline
    /// by more than the threshold (including improvements).
    /// Useful for CI self-checks to ensure benchmark stability.
    #[arg(long, requires = "threshold")]
    strict: bool,
}

fn main() -> Result<()> {
    let cli = Cli::parse();

    let storage_cur = BenchStorage::at_path(&cli.current);
    let storage_oth = BenchStorage::at_path(&cli.other);

    let mut config = ComparisonConfig::default();
    if let Some(t) = cli.threshold {
        config.regressed_strong = t;
    }

    let diffs = storage_cur.compare(&storage_oth, &config)?;

    let report = ReportBuilder::new(diffs).with_config(config).build();

    // 1. CLI Output (ASCII Table)
    println!("{}", report.as_ascii());

    // 2. Markdown Output
    if let Some(out_path) = cli.output {
        fs::write(out_path, report.as_markdown().to_string())?;
    }

    if cli.fail_on_regression || cli.strict {
        let failed = if cli.strict {
            report.has_any_deviation(cli.threshold.unwrap())
        } else {
            report.has_significant_regression()
        };

        if failed {
            eprintln!("\nError: Benchmark performance regression detected!");
            std::process::exit(1);
        }
    }

    Ok(())
}