agentcarousel 0.2.3

Evaluate agents and skills with YAML fixtures, run cases (mock or live), and keep run rows in SQLite for reports and evidence export.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

use agentcarousel_reporters::{
    diff_runs, fetch_run, list_runs, print_diff, print_json, print_terminal, RunListing,
};
use clap::{Parser, Subcommand};

use super::config::ResolvedConfig;
use super::exit_codes::ExitCode;

/// List/show/diff runs in the local history DB (same DB as test/eval; see config).
#[derive(Debug, Parser)]
pub struct ReportArgs {
    #[command(subcommand)]
    command: ReportCommand,
}

#[derive(Debug, Subcommand)]
enum ReportCommand {
    /// Recent run ids (newest first).
    List {
        #[arg(short = 'l', long, default_value_t = 20)]
        limit: usize,
        #[arg(short = 'j', long)]
        json: bool,
    },
    /// One run (terminal or --json).
    Show {
        run_id: String,
        #[arg(short = 'j', long)]
        json: bool,
    },
    /// Compare two runs (regressions vs configured threshold).
    Diff { run_id_a: String, run_id_b: String },
}

pub fn run_report(args: ReportArgs, config: &ResolvedConfig) -> i32 {
    match args.command {
        ReportCommand::List { limit, json } => report_list(limit, json),
        ReportCommand::Show { run_id, json } => report_show(&run_id, json),
        ReportCommand::Diff { run_id_a, run_id_b } => {
            report_diff(&run_id_a, &run_id_b, config.report.regression_threshold)
        }
    }
}

fn report_list(limit: usize, json: bool) -> i32 {
    match list_runs(limit) {
        Ok(runs) => {
            if json {
                let payload =
                    serde_json::to_string_pretty(&runs).unwrap_or_else(|_| "[]".to_string());
                println!("{payload}");
            } else {
                print_list(&runs);
            }
            ExitCode::Ok.as_i32()
        }
        Err(err) => {
            eprintln!("error: {err}");
            ExitCode::RuntimeError.as_i32()
        }
    }
}

fn report_show(run_id: &str, json: bool) -> i32 {
    match fetch_run(run_id) {
        Ok(run) => {
            if json {
                print_json(&run);
            } else {
                print_terminal(&run);
            }
            ExitCode::Ok.as_i32()
        }
        Err(err) => {
            eprintln!("error: {err}");
            ExitCode::RuntimeError.as_i32()
        }
    }
}

fn report_diff(run_id_a: &str, run_id_b: &str, threshold: f32) -> i32 {
    let run_a = match fetch_run(run_id_a) {
        Ok(run) => run,
        Err(err) => {
            eprintln!("error: {err}");
            return ExitCode::RuntimeError.as_i32();
        }
    };
    let run_b = match fetch_run(run_id_b) {
        Ok(run) => run,
        Err(err) => {
            eprintln!("error: {err}");
            return ExitCode::RuntimeError.as_i32();
        }
    };

    let diff = diff_runs(&run_a, &run_b, threshold);
    print_diff(&diff);
    if diff.has_regressions {
        ExitCode::Failed.as_i32()
    } else {
        ExitCode::Ok.as_i32()
    }
}

fn print_list(runs: &[RunListing]) {
    if runs.is_empty() {
        println!("no runs recorded");
        return;
    }
    for run in runs {
        println!("{}  {}", run.id, run.started_at.to_rfc3339());
    }
}