comnum_spec/cli/
mod.rs

1//! The command-line tool `comnum`: run declarative Monte Carlo campaigns.
2//!
3//! ```text
4//! comnum run SPEC... [--out DIR] [--threads N] [--fast-dev-run] [--profile] [--quiet] [--json] [--max-seconds S]
5//! comnum check SPEC... [--json]
6//! comnum verify SPEC... [--out DIR] [--threads N] [--quiet] [--json] [--max-seconds S]
7//! comnum conditions SPEC
8//! comnum blocks [NAME...] [--json]
9//! comnum schema
10//! comnum guide
11//! ```
12//!
13//! `run` writes the CSV and JSON reports given by the specification, or
14//! `DIR/<name>.csv` and `DIR/<name>.json` (`DIR` = `results` by default), and prints
15//! every metric with its confidence interval (for a specification with a `search`, the
16//! operating point of every combination of the other axes); `--profile` adds the mean
17//! time of every block ([`profile`](crate::profile)). `check` builds the link of every
18//! condition and runs one trial of each, without writing anything (for a specification
19//! with a `search`, at the lower end of its range for every combination of the other
20//! axes). `verify` runs the specifications and checks their expected results (closed
21//! forms, published curves), prints every check, writes the reports of the run and
22//! `DIR/<name>.verify.json`, and fails if a check fails. `conditions` lists the
23//! conditions of the grid; `blocks` documents the blocks of the links (their parameters
24//! with types, units and accepted values, and their ports; as JSON with `--json`);
25//! `schema` prints the JSON Schema of a block of a link, for editors and graphical
26//! front ends; `guide` prints the guide for programs and agents
27//! (`GUIDE_AGENTS.md` of the crate).
28//!
29//! For programs and agents ([`agent`]): `--json` prints one compact JSON document per
30//! specification on standard output (the results with their confidence intervals and
31//! their provenance, the checks of `verify`) and the errors as JSON with a kind, a hint
32//! and a suggestion for a misspelt block or parameter; `--max-seconds S` bounds the
33//! wall time of a run (300 s by default with `--json`, 0 for none), spread over its
34//! conditions, and marks the conditions it cut short (`time_limited`: their results
35//! then depend on the machine). A specification `-` is read from standard input (YAML
36//! or JSON).
37//!
38//! The binary `comnum` of this crate is [`main`] on the arguments of the process; a crate
39//! that adds blocks ([`blocks::register`]) wraps the same function in its own binary,
40//! so that its command lists, checks and runs them with the others.
41
42use crate::{Format, RunOptions, Spec, blocks, run, search, verify};
43use comnum_sim::stopping::Stop;
44use serde_json::json;
45use std::path::PathBuf;
46use std::process::ExitCode;
47
48pub mod agent;
49
50const USAGE: &str = "usage:
51  comnum run SPEC... [--out DIR] [--threads N] [--fast-dev-run] [--profile] [--quiet] [--json] [--max-seconds S]
52  comnum check SPEC... [--json]
53  comnum verify SPEC... [--out DIR] [--threads N] [--quiet] [--json] [--max-seconds S]
54  comnum conditions SPEC
55  comnum blocks [NAME...] [--json]
56  comnum schema
57  comnum guide
58
59SPEC is a YAML, JSON or TOML specification (see the documentation of the crate comnum-spec), or -
60for standard input. --json: one JSON document per specification on standard output, for programs
61and agents (a time budget of 300 s unless --max-seconds is given; --max-seconds 0 for none).";
62
63#[allow(clippy::struct_excessive_bools, reason = "the flags of the command line")]
64struct Args {
65    command: String,
66    files: Vec<PathBuf>,
67    out: PathBuf,
68    threads: Option<usize>,
69    fast_dev_run: bool,
70    profile: bool,
71    quiet: bool,
72    json: bool,
73    max_seconds: Option<f64>,
74}
75
76fn parse_args(args: impl IntoIterator<Item = String>) -> Result<Args, String> {
77    let mut it = args.into_iter();
78    let command = it.next().ok_or_else(|| USAGE.to_string())?;
79    let mut a = Args {
80        command,
81        files: Vec::new(),
82        out: PathBuf::from("results"),
83        threads: None,
84        fast_dev_run: false,
85        profile: false,
86        quiet: false,
87        json: false,
88        max_seconds: None,
89    };
90    while let Some(arg) = it.next() {
91        match arg.as_str() {
92            "--out" => a.out = PathBuf::from(it.next().ok_or("--out needs a directory")?),
93            "--threads" => {
94                let n = it.next().ok_or("--threads needs a number")?;
95                a.threads = Some(n.parse().map_err(|_| format!("--threads: {n:?} is not a number"))?);
96            }
97            "--fast-dev-run" => a.fast_dev_run = true,
98            "--profile" => a.profile = true,
99            "--quiet" => a.quiet = true,
100            "--json" => a.json = true,
101            "--max-seconds" => {
102                let s = it.next().ok_or("--max-seconds needs a number of seconds")?;
103                let v: f64 = s.parse().map_err(|_| format!("--max-seconds: {s:?} is not a number"))?;
104                if !(v.is_finite() && v >= 0.0) {
105                    return Err(format!("--max-seconds: {s} must be a number of seconds >= 0"));
106                }
107                a.max_seconds = Some(v);
108            }
109            "-h" | "--help" => return Err(USAGE.to_string()),
110            s if s.starts_with("--") => return Err(format!("unknown option {s}\n\n{USAGE}")),
111            "-" => a.files.push(PathBuf::from("-")),
112            s => a.files.push(PathBuf::from(s)),
113        }
114    }
115    Ok(a)
116}
117
118/// The options of `run` and `verify`: the threads, the progress and the reports, those
119/// of the specification or `DIR/<name>.csv` and `DIR/<name>.json`.
120fn report_options(spec: &Spec, a: &Args) -> RunOptions {
121    RunOptions {
122        threads: a.threads,
123        progress: Some(!a.quiet && !a.json),
124        csv: Some(spec.csv.clone().unwrap_or_else(|| a.out.join(format!("{}.csv", spec.name)))),
125        json: Some(spec.json.clone().unwrap_or_else(|| a.out.join(format!("{}.json", spec.name)))),
126        ..RunOptions::default()
127    }
128}
129
130/// Say where the reports went, unless `--quiet`.
131fn written(opts: &RunOptions, a: &Args) {
132    if let (false, false, Some(csv), Some(json)) = (a.quiet, a.json, &opts.csv, &opts.json) {
133        eprintln!("written {} and {}", csv.display(), json.display());
134    }
135}
136
137/// Read a specification from a file, or from standard input for `-`.
138fn load(path: &PathBuf) -> comnum_core::Result<Spec> {
139    if path.as_os_str() == "-" {
140        let mut text = String::new();
141        std::io::Read::read_to_string(&mut std::io::stdin(), &mut text)?;
142        return Spec::parse(&text, Format::Yaml).map_err(|e| e.context("standard input"));
143    }
144    Spec::from_path(path).map_err(|e| match e {
145        comnum_core::Error::Io(io) => {
146            comnum_core::Error::Io(std::io::Error::new(io.kind(), format!("{}: {io}", path.display())))
147        }
148        other => other,
149    })
150}
151
152/// The specification with the time budget of the options, and the rule of the
153/// specification when a budget applies.
154fn budgeted(path: &PathBuf, a: &Args) -> comnum_core::Result<(Spec, Option<Stop>)> {
155    let mut spec = load(path)?;
156    let original = agent::budget(a.max_seconds, a.json).map(|s| agent::apply_budget(&mut spec, s));
157    Ok((spec, original))
158}
159
160fn run_one(path: &PathBuf, a: &Args) -> comnum_core::Result<()> {
161    let (spec, original) = budgeted(path, a)?;
162    let opts =
163        RunOptions { fast_dev_run: a.fast_dev_run, profile: a.profile.then_some(true), ..report_options(&spec, a) };
164    let started = std::time::Instant::now();
165    if a.json {
166        let mut doc = json!({
167            "ok": true,
168            "command": "run",
169            "spec": spec.name,
170            "description": spec.description,
171            "provenance": agent::provenance(&spec, &opts),
172            "budget_s": agent::budget(a.max_seconds, a.json),
173        });
174        if spec.search.is_some() {
175            let found = search::search(&spec, &opts)?;
176            doc["operating_points"] = found.iter().map(agent::operating_point).collect();
177        } else {
178            let results = run(&spec, &opts)?;
179            doc["conditions"] =
180                results.iter().map(|r| agent::condition(r, spec.confidence, original.as_ref())).collect();
181        }
182        doc["wall_s"] = json!(started.elapsed().as_secs_f64());
183        println!("{doc}");
184        return Ok(());
185    }
186    if let Some(sp) = &spec.search {
187        if !a.quiet {
188            eprintln!(
189                "{}: {} (search of {} for {} = {:e})",
190                spec.name, spec.description, sp.along, sp.metric, sp.target
191            );
192        }
193        for r in search::search(&spec, &opts)? {
194            println!(
195                "{} | {} = {:.4} [{:.4}, {:.4}] | {} evaluations{}",
196                r.condition.label(),
197                r.along,
198                r.value,
199                r.lo,
200                r.hi,
201                r.evaluations.len(),
202                if r.converged { "" } else { " (not converged)" }
203            );
204        }
205        written(&opts, a);
206        return Ok(());
207    }
208    if !a.quiet {
209        eprintln!("{}: {} ({} conditions)", spec.name, spec.description, spec.conditions().len());
210    }
211    let results = run(&spec, &opts)?;
212    for r in &results {
213        let metrics: Vec<String> = r
214            .record
215            .iter()
216            .map(|(k, e)| {
217                let (lo, hi) = e.interval(spec.confidence);
218                format!("{k}={:.4e} [{lo:.3e}, {hi:.3e}]", e.value())
219            })
220            .collect();
221        println!("{} | {} | {} trials | {}", r.condition.label(), r.status.as_str(), r.trials, metrics.join("  "));
222    }
223    let profile = crate::profile_of(&spec, &results)?;
224    if !profile.is_empty() {
225        println!("\nprofile: mean time per trial over {} conditions", results.len());
226        for row in &profile {
227            println!("{:>24}  {:>10.3} ms  {:>5.1} %", row.label, row.time_ms, 100.0 * row.share);
228        }
229    }
230    written(&opts, a);
231    Ok(())
232}
233
234/// Run a specification and check its expectations; returns the number of failures.
235fn verify_one(path: &PathBuf, a: &Args) -> comnum_core::Result<usize> {
236    let (spec, _) = budgeted(path, a)?;
237    // the reports of the run, as with `run`
238    let opts = report_options(&spec, a);
239    let (_, checks) = verify::verify_spec(&spec, &opts)?;
240    let failed = checks.iter().filter(|c| !c.passed).count();
241    let report = a.out.join(format!("{}.verify.json", spec.name));
242    verify::write_report(&report, &spec, &opts, &checks)?;
243    if a.json {
244        let doc = json!({
245            "ok": failed == 0,
246            "command": "verify",
247            "spec": spec.name,
248            "provenance": agent::provenance(&spec, &opts),
249            "budget_s": agent::budget(a.max_seconds, a.json),
250            "checks": checks.iter().map(agent::check_json).collect::<Vec<_>>(),
251            "failed": failed,
252            "report": report.display().to_string(),
253        });
254        println!("{doc}");
255        return Ok(failed);
256    }
257    println!("{}: {} checks, {} failed", spec.name, checks.len(), failed);
258    for c in &checks {
259        println!(
260            "  {} {} | {} | {}: expected {:.4e}, measured {:.4e} [{:.3e}, {:.3e}]",
261            if c.passed { "ok  " } else { "FAIL" },
262            c.name,
263            c.condition,
264            c.metric,
265            c.expected,
266            c.measured.value,
267            c.measured.lo,
268            c.measured.hi
269        );
270    }
271    Ok(failed)
272}
273
274fn check_one(path: &PathBuf, json_out: bool) -> comnum_core::Result<()> {
275    let spec = load(path)?;
276    // a search: the first point it evaluates, for every combination of the other axes
277    let mut spec = search::first_points(&spec).unwrap_or(spec);
278    // nothing is written
279    (spec.csv, spec.json) = (None, None);
280    let opts = RunOptions { fast_dev_run: true, batch: Some(1), progress: Some(false), ..RunOptions::default() };
281    let r = run(&spec, &opts)?;
282    let names: Vec<&String> = r.first().map(|c| c.record.iter().map(|(k, _)| k).collect()).unwrap_or_default();
283    if json_out {
284        let doc = json!({
285            "ok": true,
286            "command": "check",
287            "spec": spec.name,
288            "conditions": r.len(),
289            "blocks": spec.link.len(),
290            "metrics": names,
291        });
292        println!("{doc}");
293    } else {
294        println!("{}: ok, {} conditions, {} blocks, metrics {:?}", path.display(), r.len(), spec.link.len(), names);
295    }
296    Ok(())
297}
298
299/// Pretty JSON, or the error as a configuration error.
300fn to_json(v: &impl serde::Serialize) -> comnum_core::Result<String> {
301    serde_json::to_string_pretty(v).map_err(|e| comnum_core::Error::Config(format!("cannot serialise the report: {e}")))
302}
303
304/// Run the command line on its arguments (without the program name); the exit code is
305/// 0 on success, 1 on a failure and 2 on a usage error.
306pub fn main(args: impl IntoIterator<Item = String>) -> ExitCode {
307    let args: Vec<String> = args.into_iter().collect();
308    let json_out = args.iter().any(|s| s == "--json");
309    let usage = |msg: &str| {
310        if json_out {
311            let first = msg.lines().next().unwrap_or(msg);
312            println!("{}", json!({"ok": false, "command": args.first(), "error": agent::error("usage", first)}));
313        } else {
314            eprintln!("{msg}");
315        }
316        ExitCode::from(2)
317    };
318    let a = match parse_args(args.clone()) {
319        Ok(a) => a,
320        Err(msg) => return usage(&msg),
321    };
322    if ["run", "check", "verify", "conditions"].contains(&a.command.as_str()) && a.files.is_empty() {
323        return usage(&format!("no specification given\n\n{USAGE}"));
324    }
325    if a.json && ["run", "check", "verify"].contains(&a.command.as_str()) {
326        return main_json(&a);
327    }
328    let result: comnum_core::Result<()> = match a.command.as_str() {
329        "blocks" => {
330            let all = blocks::registry();
331            let wanted: Vec<String> = a.files.iter().map(|f| f.display().to_string()).collect();
332            let shown: Vec<_> =
333                all.iter().filter(|b| wanted.is_empty() || wanted.iter().any(|w| w == b.name)).collect();
334            if shown.is_empty() {
335                Err(comnum_core::Error::Config(format!("no block {wanted:?}")))
336            } else if a.json {
337                let all: Vec<_> = shown.iter().map(|b| b.to_json()).collect();
338                to_json(&all).map(|j| println!("{j}"))
339            } else {
340                for b in shown {
341                    println!("{}", b.describe());
342                }
343                Ok(())
344            }
345        }
346        "schema" => to_json(&blocks::json_schema()).map(|j| println!("{j}")),
347        "guide" => {
348            print!("{}", include_str!("../../GUIDE_AGENTS.md"));
349            Ok(())
350        }
351        "conditions" => a.files.iter().try_for_each(|f| {
352            let spec = load(f)?;
353            for (i, c) in spec.conditions().iter().enumerate() {
354                println!("{i}: {}", c.label());
355            }
356            Ok(())
357        }),
358        "check" => a.files.iter().try_for_each(|f| check_one(f, a.json)),
359        "run" => a.files.iter().try_for_each(|f| run_one(f, &a)),
360        "verify" => {
361            let mut failed = 0;
362            let r = a.files.iter().try_for_each(|f| verify_one(f, &a).map(|n| failed += n));
363            if r.is_ok() && failed > 0 {
364                eprintln!("{failed} checks failed");
365                return ExitCode::FAILURE;
366            }
367            r
368        }
369        other => return usage(&format!("unknown command {other:?}\n\n{USAGE}")),
370    };
371    match result {
372        Ok(()) => ExitCode::SUCCESS,
373        Err(e) => {
374            eprintln!("error: {e}");
375            ExitCode::FAILURE
376        }
377    }
378}
379
380/// `run`, `check` and `verify` with `--json`: one line per specification, an error as a
381/// line of its own (the other specifications still run); the exit code is 1 if one
382/// failed.
383fn main_json(a: &Args) -> ExitCode {
384    let mut failed = false;
385    for f in &a.files {
386        let r = match a.command.as_str() {
387            "run" => run_one(f, a).map(|()| 0),
388            "check" => check_one(f, true).map(|()| 0),
389            _ => verify_one(f, a),
390        };
391        match r {
392            Ok(n) => failed |= n > 0,
393            Err(e) => {
394                failed = true;
395                let message = e.to_string();
396                println!(
397                    "{}",
398                    json!({"ok": false, "command": a.command, "file": f.display().to_string(), "error": agent::error(agent::error_kind(&e), &message)})
399                );
400            }
401        }
402    }
403    if failed { ExitCode::FAILURE } else { ExitCode::SUCCESS }
404}
comnum_spec/cli/mod.rs

comnum_spec/cli/
mod.rs
