1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
use anyhow::anyhow;
use anyhow::Result;
use clap::{error::ErrorKind::ArgumentConflict, Args, Parser};
use clap::{CommandFactory, Subcommand};
use std::path::PathBuf;
use crate::audit;
use crate::basic_measure::measure;
use crate::config::bump_epoch;
use crate::data::ReductionFunc;
use crate::git_interop::{prune, pull, push};
use crate::measurement_storage::add;
use crate::reporting::report;
#[derive(Parser)]
#[command(version)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Args)]
struct CliMeasurement {
/// Name of the measurement
#[arg(short = 'm', long = "measurement", value_parser=parse_spaceless_string)]
name: String,
/// Key-value pairs separated by '='
#[arg(short, long, value_parser=parse_key_value)]
key_value: Vec<(String, String)>,
}
#[derive(Args)]
struct CliReportHistory {
/// Limit the number of previous commits considered.
/// HEAD is included in this count.
#[arg(short = 'n', long, default_value = "40")]
max_count: usize,
}
#[derive(Subcommand)]
enum Commands {
/// Measure the runtime of the supplied command (in nanoseconds)
Measure {
/// Repetitions
#[arg(short = 'n', long, value_parser=clap::value_parser!(u16).range(1..), default_value = "1")]
repetitions: u16,
#[command(flatten)]
measurement: CliMeasurement,
/// Command to measure
#[arg(required(true), last(true))]
command: Vec<String>,
},
/// Add single measurement
Add {
/// Measured value to be added
value: f64,
#[command(flatten)]
measurement: CliMeasurement,
},
/// Publish performance results to remote
Push {},
/// Pull performance results from remote
Pull {},
/// Create an HTML performance report
Report {
/// HTML output file
#[arg(short, long, default_value = "output.html")]
output: PathBuf,
#[command(flatten)]
report_history: CliReportHistory,
/// Select an individual measurements instead of all
#[arg(short, long)]
measurement: Vec<String>,
/// Key-value pairs separated by '=', select only matching measurements
#[arg(short, long, value_parser=parse_key_value)]
key_value: Vec<(String, String)>,
/// Create individual traces in the graph by grouping with the value of this selector
#[arg(short, long, value_parser=parse_spaceless_string)]
separate_by: Option<String>,
/// What to aggregate the measurements in each group with
#[arg(short, long)]
aggregate_by: Option<ReductionFunc>,
},
/// For a given measurement, check perfomance deviations of the HEAD commit
/// against `<n>` previous commits. Group previous results and aggregate their
/// results before comparison.
Audit {
#[arg(short, long, value_parser=parse_spaceless_string)]
measurement: String,
#[command(flatten)]
report_history: CliReportHistory,
/// Key-value pair separated by "=" with no whitespaces to subselect measurements
#[arg(short, long, value_parser=parse_key_value)]
selectors: Vec<(String, String)>,
/// Minimum number of measurements needed. If less, pass test and assume
/// more measurements are needed.
/// A minimum of two historic measurements are needed for proper evaluation of standard
/// deviation.
// TODO(kaihowl) fix up min value and default_value
#[arg(long, value_parser=clap::value_parser!(u16).range(1..), default_value="2")]
min_measurements: u16,
/// What to aggregate the measurements in each group with
#[arg(short, long, default_value = "min")]
aggregate_by: ReductionFunc,
/// Multiple of the stddev after which a outlier is detected.
/// If the HEAD measurement is within `[mean-<d>*sigma; mean+<d>*sigma]`,
/// it is considered acceptable.
#[arg(short = 'd', long, default_value = "4.0")]
sigma: f64,
},
/// Accept HEAD commit's measurement for audit, even if outside of range.
/// This is allows to accept expected performance changes.
/// This is accomplished by starting a new epoch for the given measurement.
/// The epoch is configured in the git perf config file.
/// A change to the epoch therefore has to be committed and will result in a new HEAD for which
/// new measurements have to be taken.
BumpEpoch {
#[arg(short = 'm', long = "measurement", value_parser=parse_spaceless_string)]
measurement: String,
},
/// Remove all performance measurements for non-existent/unreachable objects.
/// Will refuse to work if run on a shallow clone.
Prune {},
/// Generate the manpage content
#[command(hide = true)]
Manpage {},
}
fn parse_key_value(s: &str) -> Result<(String, String)> {
let pos = s
.find('=')
.ok_or_else(|| anyhow!("invalid key=value: no '=' found in '{}'", s))?;
let key = parse_spaceless_string(&s[..pos])?;
let value = parse_spaceless_string(&s[pos + 1..])?;
Ok((key, value))
}
fn parse_spaceless_string(s: &str) -> Result<String> {
if s.split_whitespace().count() > 1 {
Err(anyhow!("invalid string/key/value: found space in '{}'", s))
} else {
Ok(String::from(s))
}
}
pub fn handle_calls() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Measure {
repetitions,
command,
measurement,
} => Ok(measure(
&measurement.name,
repetitions,
&command,
&measurement.key_value,
)?),
Commands::Add { value, measurement } => {
Ok(add(&measurement.name, value, &measurement.key_value)?)
}
Commands::Push {} => Ok(push(None)?),
Commands::Pull {} => Ok(pull(None)?),
Commands::Report {
output,
separate_by,
report_history,
measurement,
key_value,
aggregate_by,
} => Ok(report(
output,
separate_by,
report_history.max_count,
&measurement,
&key_value,
aggregate_by,
)?),
Commands::Audit {
measurement,
report_history,
selectors,
min_measurements,
aggregate_by,
sigma,
} => {
if report_history.max_count < min_measurements.into() {
Cli::command().error(ArgumentConflict, format!("The minimal number of measurements ({}) cannot be more than the maximum number of measurements ({})", min_measurements, report_history.max_count)).exit()
}
Ok(audit::audit(
&measurement,
report_history.max_count,
min_measurements,
&selectors,
aggregate_by,
sigma,
)?)
}
Commands::BumpEpoch { measurement } => Ok(bump_epoch(&measurement)?),
Commands::Prune {} => Ok(prune()?),
Commands::Manpage {} => {
generate_manpage().expect("Man page generation failed");
Ok(())
}
}
}
fn generate_manpage() -> Result<()> {
let man = clap_mangen::Man::new(Cli::command());
man.render(&mut std::io::stdout())?;
// TODO(kaihowl) this does not look very nice. Fix it.
for command in Cli::command()
.get_subcommands()
.filter(|c| !c.is_hide_set())
{
let man = clap_mangen::Man::new(command.clone());
man.render(&mut std::io::stdout())?
}
Ok(())
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn verify_cli() {
Cli::command().debug_assert()
}
}