comnum_spec/
cli.rs

1//! The command-line tool `comnum`: run declarative Monte Carlo campaigns.
2//!
3//! ```text
4//! comnum run SPEC... [--out DIR] [--threads N] [--fast-dev-run] [--profile] [--quiet]
5//! comnum check SPEC...
6//! comnum verify SPEC... [--out DIR] [--threads N] [--quiet]
7//! comnum conditions SPEC
8//! comnum blocks [NAME...] [--json]
9//! comnum schema
10//! ```
11//!
12//! `run` writes the CSV and JSON reports given by the specification, or
13//! `DIR/<name>.csv` and `DIR/<name>.json` (`DIR` = `results` by default), and prints
14//! every metric with its confidence interval (for a specification with a `search`, the
15//! operating point of every combination of the other axes); `--profile` adds the mean
16//! time of every block ([`profile`](crate::profile)). `check` builds the link of every
17//! condition and runs one trial of each, without writing anything (for a specification
18//! with a `search`, at the lower end of its range for every combination of the other
19//! axes). `verify` runs the specifications and checks their expected results (closed
20//! forms, published curves), prints every check, writes the reports of the run and
21//! `DIR/<name>.verify.json`, and fails if a check fails. `conditions` lists the
22//! conditions of the grid; `blocks` documents the blocks of the links (their parameters
23//! with types, units and accepted values, and their ports; as JSON with `--json`);
24//! `schema` prints the JSON Schema of a block of a link, for editors and graphical
25//! front ends.
26//!
27//! The binary `comnum` of this crate is [`main`] on the arguments of the process; a crate
28//! that adds blocks ([`blocks::register`]) wraps the same function in its own binary,
29//! so that its command lists, checks and runs them with the others.
30
31use crate::{RunOptions, Spec, blocks, run, search, verify};
32use std::path::PathBuf;
33use std::process::ExitCode;
34
35const USAGE: &str = "usage:
36  comnum run SPEC... [--out DIR] [--threads N] [--fast-dev-run] [--profile] [--quiet]
37  comnum check SPEC...
38  comnum verify SPEC... [--out DIR] [--threads N] [--quiet]
39  comnum conditions SPEC
40  comnum blocks [NAME...] [--json]
41  comnum schema
42
43SPEC is a YAML, JSON or TOML specification (see the documentation of the crate comnum-spec).";
44
45#[allow(clippy::struct_excessive_bools, reason = "the flags of the command line")]
46struct Args {
47    command: String,
48    files: Vec<PathBuf>,
49    out: PathBuf,
50    threads: Option<usize>,
51    fast_dev_run: bool,
52    profile: bool,
53    quiet: bool,
54    json: bool,
55}
56
57fn parse_args(args: impl IntoIterator<Item = String>) -> Result<Args, String> {
58    let mut it = args.into_iter();
59    let command = it.next().ok_or_else(|| USAGE.to_string())?;
60    let mut a = Args {
61        command,
62        files: Vec::new(),
63        out: PathBuf::from("results"),
64        threads: None,
65        fast_dev_run: false,
66        profile: false,
67        quiet: false,
68        json: false,
69    };
70    while let Some(arg) = it.next() {
71        match arg.as_str() {
72            "--out" => a.out = PathBuf::from(it.next().ok_or("--out needs a directory")?),
73            "--threads" => {
74                let n = it.next().ok_or("--threads needs a number")?;
75                a.threads = Some(n.parse().map_err(|_| format!("--threads: {n:?} is not a number"))?);
76            }
77            "--fast-dev-run" => a.fast_dev_run = true,
78            "--profile" => a.profile = true,
79            "--quiet" => a.quiet = true,
80            "--json" => a.json = true,
81            "-h" | "--help" => return Err(USAGE.to_string()),
82            s if s.starts_with("--") => return Err(format!("unknown option {s}\n\n{USAGE}")),
83            s => a.files.push(PathBuf::from(s)),
84        }
85    }
86    Ok(a)
87}
88
89/// The options of `run` and `verify`: the threads, the progress and the reports, those
90/// of the specification or `DIR/<name>.csv` and `DIR/<name>.json`.
91fn report_options(spec: &Spec, a: &Args) -> RunOptions {
92    RunOptions {
93        threads: a.threads,
94        progress: Some(!a.quiet),
95        csv: Some(spec.csv.clone().unwrap_or_else(|| a.out.join(format!("{}.csv", spec.name)))),
96        json: Some(spec.json.clone().unwrap_or_else(|| a.out.join(format!("{}.json", spec.name)))),
97        ..RunOptions::default()
98    }
99}
100
101/// Say where the reports went, unless `--quiet`.
102fn written(opts: &RunOptions, a: &Args) {
103    if let (false, Some(csv), Some(json)) = (a.quiet, &opts.csv, &opts.json) {
104        eprintln!("written {} and {}", csv.display(), json.display());
105    }
106}
107
108fn run_one(path: &PathBuf, a: &Args) -> comnum_core::Result<()> {
109    let spec = Spec::from_path(path)?;
110    let opts =
111        RunOptions { fast_dev_run: a.fast_dev_run, profile: a.profile.then_some(true), ..report_options(&spec, a) };
112    if let Some(sp) = &spec.search {
113        if !a.quiet {
114            eprintln!(
115                "{}: {} (search of {} for {} = {:e})",
116                spec.name, spec.description, sp.along, sp.metric, sp.target
117            );
118        }
119        for r in search::search(&spec, &opts)? {
120            println!(
121                "{} | {} = {:.4} [{:.4}, {:.4}] | {} evaluations{}",
122                r.condition.label(),
123                r.along,
124                r.value,
125                r.lo,
126                r.hi,
127                r.evaluations.len(),
128                if r.converged { "" } else { " (not converged)" }
129            );
130        }
131        written(&opts, a);
132        return Ok(());
133    }
134    if !a.quiet {
135        eprintln!("{}: {} ({} conditions)", spec.name, spec.description, spec.conditions().len());
136    }
137    let results = run(&spec, &opts)?;
138    for r in &results {
139        let metrics: Vec<String> = r
140            .record
141            .iter()
142            .map(|(k, e)| {
143                let (lo, hi) = e.interval(spec.confidence);
144                format!("{k}={:.4e} [{lo:.3e}, {hi:.3e}]", e.value())
145            })
146            .collect();
147        println!("{} | {} | {} trials | {}", r.condition.label(), r.status.as_str(), r.trials, metrics.join("  "));
148    }
149    let profile = crate::profile_of(&spec, &results)?;
150    if !profile.is_empty() {
151        println!("\nprofile: mean time per trial over {} conditions", results.len());
152        for row in &profile {
153            println!("{:>24}  {:>10.3} ms  {:>5.1} %", row.label, row.time_ms, 100.0 * row.share);
154        }
155    }
156    written(&opts, a);
157    Ok(())
158}
159
160/// Run a specification and check its expectations; returns the number of failures.
161fn verify_one(path: &PathBuf, a: &Args) -> comnum_core::Result<usize> {
162    let spec = Spec::from_path(path)?;
163    // the reports of the run, as with `run`
164    let opts = report_options(&spec, a);
165    let (_, checks) = verify::verify_spec(&spec, &opts)?;
166    let failed = checks.iter().filter(|c| !c.passed).count();
167    println!("{}: {} checks, {} failed", spec.name, checks.len(), failed);
168    for c in &checks {
169        println!(
170            "  {} {} | {} | {}: expected {:.4e}, measured {:.4e} [{:.3e}, {:.3e}]",
171            if c.passed { "ok  " } else { "FAIL" },
172            c.name,
173            c.condition,
174            c.metric,
175            c.expected,
176            c.measured.value,
177            c.measured.lo,
178            c.measured.hi
179        );
180    }
181    verify::write_report(&a.out.join(format!("{}.verify.json", spec.name)), &spec, &opts, &checks)?;
182    Ok(failed)
183}
184
185fn check_one(path: &PathBuf) -> comnum_core::Result<()> {
186    let spec = Spec::from_path(path)?;
187    // a search: the first point it evaluates, for every combination of the other axes
188    let mut spec = search::first_points(&spec).unwrap_or(spec);
189    // nothing is written
190    (spec.csv, spec.json) = (None, None);
191    let opts = RunOptions { fast_dev_run: true, batch: Some(1), progress: Some(false), ..RunOptions::default() };
192    let r = run(&spec, &opts)?;
193    let names: Vec<&String> = r.first().map(|c| c.record.iter().map(|(k, _)| k).collect()).unwrap_or_default();
194    println!("{}: ok, {} conditions, {} blocks, metrics {:?}", path.display(), r.len(), spec.link.len(), names);
195    Ok(())
196}
197
198/// Pretty JSON, or the error as a configuration error.
199fn to_json(v: &impl serde::Serialize) -> comnum_core::Result<String> {
200    serde_json::to_string_pretty(v).map_err(|e| comnum_core::Error::Config(format!("cannot serialise the report: {e}")))
201}
202
203/// Run the command line on its arguments (without the program name); the exit code is
204/// 0 on success, 1 on a failure and 2 on a usage error.
205pub fn main(args: impl IntoIterator<Item = String>) -> ExitCode {
206    let a = match parse_args(args) {
207        Ok(a) => a,
208        Err(msg) => {
209            eprintln!("{msg}");
210            return ExitCode::from(2);
211        }
212    };
213    if ["run", "check", "verify", "conditions"].contains(&a.command.as_str()) && a.files.is_empty() {
214        eprintln!("no specification given\n\n{USAGE}");
215        return ExitCode::from(2);
216    }
217    let result: comnum_core::Result<()> = match a.command.as_str() {
218        "blocks" => {
219            let all = blocks::registry();
220            let wanted: Vec<String> = a.files.iter().map(|f| f.display().to_string()).collect();
221            let shown: Vec<_> =
222                all.iter().filter(|b| wanted.is_empty() || wanted.iter().any(|w| w == b.name)).collect();
223            if shown.is_empty() {
224                Err(comnum_core::Error::Config(format!("no block {wanted:?}")))
225            } else if a.json {
226                let all: Vec<_> = shown.iter().map(|b| b.to_json()).collect();
227                to_json(&all).map(|j| println!("{j}"))
228            } else {
229                for b in shown {
230                    println!("{}", b.describe());
231                }
232                Ok(())
233            }
234        }
235        "schema" => to_json(&blocks::json_schema()).map(|j| println!("{j}")),
236        "conditions" => a.files.iter().try_for_each(|f| {
237            let spec = Spec::from_path(f)?;
238            for (i, c) in spec.conditions().iter().enumerate() {
239                println!("{i}: {}", c.label());
240            }
241            Ok(())
242        }),
243        "check" => a.files.iter().try_for_each(check_one),
244        "run" => a.files.iter().try_for_each(|f| run_one(f, &a)),
245        "verify" => {
246            let mut failed = 0;
247            let r = a.files.iter().try_for_each(|f| verify_one(f, &a).map(|n| failed += n));
248            if r.is_ok() && failed > 0 {
249                eprintln!("{failed} checks failed");
250                return ExitCode::FAILURE;
251            }
252            r
253        }
254        other => {
255            eprintln!("unknown command {other:?}\n\n{USAGE}");
256            return ExitCode::from(2);
257        }
258    };
259    match result {
260        Ok(()) => ExitCode::SUCCESS,
261        Err(e) => {
262            eprintln!("error: {e}");
263            ExitCode::FAILURE
264        }
265    }
266}
comnum_spec/cli.rs

comnum_spec/
cli.rs
