comnum_sim/
loggers.rs

1//! Loggers: where the results go.
2//!
3//! A [`Logger`] receives every [`ConditionResult`] as soon as the condition ends, and
4//! the [`RunReport`] at the end of the run.
5//!
6//! - [`CsvLogger`] writes one row per condition **as it completes** (and flushes), so
7//!   that a long campaign leaves usable partial results. The columns are those of
8//!   [`ConditionResult::row`]: `index`, the keys of the condition, `status`, `trials`,
9//!   `wall_s`, then for every metric `<m>`, `<m>_lo`, `<m>_hi` (the confidence
10//!   interval) and `<m>_count`, `<m>_total` (errors and trials of a rate, or the number
11//!   of values of a mean); those of the first condition that ran.
12//! - [`JsonLogger`] writes the full report at the end of the run: the run information
13//!   ([`run_header`]: scenario, seed, batch, threads, stopping rule, confidence, library
14//!   version and commit, configuration, start time), the wall time, and every condition
15//!   with its metrics
16//!   and, under `dropped`, the number of values each metric dropped (non-finite or of
17//!   the other kind, see [`Record::dropped`](crate::scenario::Record::dropped); an
18//!   empty mapping when none was).
19//!
20//! # Examples
21//! ```
22//! use comnum_sim::prelude::*;
23//! use comnum_mapping::{constellation::{Constellation, ConstellationKind}, source as sources};
24//! use comnum_channel::awgn as channels;
25//! use comnum_metrics as metrics;
26//! use comnum_sim::scenario::from_fn;
27//! let dir = std::env::temp_dir().join("comnum-doc-loggers");
28//! std::fs::create_dir_all(&dir).unwrap();
29//! let s = from_fn("toss", |_, rng, rec| { rec.errors("p", rng.bit() as u64, 1); Ok(()) });
30//! Runner::new()
31//!     .stop(Stop::max_trials(64))
32//!     .logger(CsvLogger::new(dir.join("toss.csv")))
33//!     .logger(JsonLogger::new(dir.join("toss.json")))
34//!     .run(&s, &Grid::new().axis("n", [1usize, 2]))
35//!     .unwrap();
36//! let csv = std::fs::read_to_string(dir.join("toss.csv")).unwrap();
37//! assert!(csv.starts_with("index,n,status,trials,wall_s,p,p_lo,p_hi,p_count,p_total\n"));
38//! assert_eq!(csv.lines().count(), 3);
39//! let json = std::fs::read_to_string(dir.join("toss.json")).unwrap();
40//! assert!(json.contains("\"scenario\": \"toss\""));
41//! ```
42
43use crate::grid::Value;
44use crate::json;
45use crate::runner::{ConditionResult, RunInfo, RunReport};
46use crate::stats::Estimate;
47use comnum_core::Result;
48use std::fs::File;
49use std::io::{BufWriter, Write};
50use std::path::{Path, PathBuf};
51
52/// A destination of the results.
53pub trait Logger: Send {
54    /// Before the first condition.
55    ///
56    /// # Errors
57    /// [`comnum_core::Error::Io`] if the output cannot be created.
58    fn start(&mut self, _info: &RunInfo) -> Result<()> {
59        Ok(())
60    }
61    /// After every condition (`confidence`: level of the intervals).
62    ///
63    /// # Errors
64    /// [`comnum_core::Error::Io`] if the output cannot be written.
65    fn log(&mut self, result: &ConditionResult, confidence: f64) -> Result<()>;
66    /// After the last condition.
67    ///
68    /// # Errors
69    /// [`comnum_core::Error::Io`] if the output cannot be written.
70    fn finish(&mut self, _report: &RunReport) -> Result<()> {
71        Ok(())
72    }
73}
74
75/// A CSV field (RFC 4180): `s` itself, or quoted with its quotes doubled when it holds
76/// a comma, a quote or a line break.
77///
78/// # Examples
79/// ```
80/// use comnum_sim::loggers::csv_field;
81/// assert_eq!(csv_field("snr_db"), "snr_db");
82/// assert_eq!(csv_field("a, \"b\""), "\"a, \"\"b\"\"\"");
83/// ```
84pub fn csv_field(s: &str) -> String {
85    if s.contains([',', '"', '\n']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
86}
87
88/// Create the parent directories of `path` (none for a bare file name).
89///
90/// # Errors
91/// [`comnum_core::Error::Io`] if a directory cannot be created.
92pub fn create_parent_dir(path: &Path) -> Result<()> {
93    if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
94        std::fs::create_dir_all(dir)?;
95    }
96    Ok(())
97}
98
99/// One CSV row per condition ([`ConditionResult::row`]), written as the conditions
100/// complete.
101///
102/// The header is written with the first condition that ran trials (skipped conditions
103/// before it are held back and written right after the header). The values of the
104/// condition are written as they are, the wall time to the microsecond, the estimates
105/// and their intervals in scientific notation (empty when not finite), the counts as
106/// integers.
107pub struct CsvLogger {
108    path: PathBuf,
109    out: Option<BufWriter<File>>,
110    header: bool,
111    keys: Vec<String>,
112    columns: Vec<String>,
113    pending: Vec<(ConditionResult, f64)>,
114}
115
116impl CsvLogger {
117    /// A logger writing to `path` (created or truncated at the start of the run; the
118    /// parent directories are created).
119    pub fn new(path: impl Into<PathBuf>) -> CsvLogger {
120        CsvLogger {
121            path: path.into(),
122            out: None,
123            header: false,
124            keys: Vec::new(),
125            columns: Vec::new(),
126            pending: Vec::new(),
127        }
128    }
129
130    fn write_header(&mut self, r: &ConditionResult, confidence: f64) -> Result<()> {
131        self.keys = r.condition.keys().map(str::to_string).collect();
132        self.columns = r.row(confidence).into_iter().map(|(k, _)| k).collect();
133        let header: Vec<String> = self.columns.iter().map(|k| csv_field(k)).collect();
134        if let Some(out) = self.out.as_mut() {
135            writeln!(out, "{}", header.join(","))?;
136        }
137        self.header = true;
138        Ok(())
139    }
140
141    /// A cell of the row: a value of the condition or of the head as it is, the wall
142    /// time to the microsecond, an estimate or a bound in scientific notation.
143    fn cell(column: &str, v: &Value, metric: bool) -> String {
144        match v {
145            Value::F64(x) if !metric && column == "wall_s" => format!("{x:.6}"),
146            Value::F64(x) if metric && x.is_finite() => format!("{x:e}"),
147            Value::F64(_) if metric => String::new(),
148            other => csv_field(&other.to_string()),
149        }
150    }
151
152    fn write_row(&mut self, r: &ConditionResult, confidence: f64) -> Result<()> {
153        let cells = r.row(confidence);
154        // the head of the row (index, condition, status, trials, wall_s) by position, the
155        // condition by its keys and the metrics by their names (a metric may have the
156        // name of a variable: snr_db)
157        let (head, metrics) = cells.split_at(4 + r.condition.entries().len());
158        let (columns_head, columns_metrics) = self.columns.split_at(4 + self.keys.len());
159        let condition = &head[1..head.len() - 3];
160        let mut row = vec![Self::cell("index", &head[0].1, false)];
161        row.extend(
162            self.keys.iter().map(|k| {
163                condition.iter().find(|(c, _)| c == k).map(|(_, v)| Self::cell(k, v, false)).unwrap_or_default()
164            }),
165        );
166        row.extend(head[head.len() - 3..].iter().map(|(k, v)| Self::cell(k, v, false)));
167        debug_assert_eq!(row.len(), columns_head.len());
168        row.extend(
169            columns_metrics
170                .iter()
171                .map(|c| metrics.iter().find(|(k, _)| k == c).map(|(_, v)| Self::cell(c, v, true)).unwrap_or_default()),
172        );
173        if let Some(out) = self.out.as_mut() {
174            writeln!(out, "{}", row.join(","))?;
175            out.flush()?;
176        }
177        Ok(())
178    }
179}
180
181impl Logger for CsvLogger {
182    fn start(&mut self, _: &RunInfo) -> Result<()> {
183        create_parent_dir(&self.path)?;
184        self.out = Some(BufWriter::new(File::create(&self.path)?));
185        self.header = false;
186        self.pending.clear();
187        Ok(())
188    }
189
190    fn log(&mut self, r: &ConditionResult, confidence: f64) -> Result<()> {
191        if !self.header {
192            if r.trials == 0 {
193                self.pending.push((r.clone(), confidence));
194                return Ok(());
195            }
196            self.write_header(r, confidence)?;
197            for (p, c) in std::mem::take(&mut self.pending) {
198                self.write_row(&p, c)?;
199            }
200        }
201        self.write_row(r, confidence)
202    }
203
204    fn finish(&mut self, _: &RunReport) -> Result<()> {
205        if !self.header {
206            if let Some((first, c)) = self.pending.first().cloned() {
207                self.write_header(&first, c)?;
208            }
209            for (p, c) in std::mem::take(&mut self.pending) {
210                self.write_row(&p, c)?;
211            }
212        }
213        if let Some(out) = self.out.as_mut() {
214            out.flush()?;
215        }
216        Ok(())
217    }
218}
219
220/// The full report as JSON, written at the end of the run.
221pub struct JsonLogger {
222    path: PathBuf,
223}
224
225impl JsonLogger {
226    /// A logger writing to `path` at the end of the run.
227    pub fn new(path: impl Into<PathBuf>) -> JsonLogger {
228        JsonLogger { path: path.into() }
229    }
230}
231
232/// A value of a condition as JSON (a number, a string or a boolean).
233///
234/// # Examples
235/// ```
236/// use comnum_sim::grid::Value;
237/// assert_eq!(comnum_sim::loggers::value_json(&Value::Str("qam".into())), "\"qam\"");
238/// assert_eq!(comnum_sim::loggers::value_json(&Value::F64(f64::NAN)), "null");
239/// ```
240pub fn value_json(v: &Value) -> String {
241    match v {
242        Value::F64(x) => json::number(*x),
243        Value::I64(x) => x.to_string(),
244        Value::Str(s) => json::string(s),
245        Value::Bool(b) => b.to_string(),
246    }
247}
248
249fn estimate_json(e: &Estimate, confidence: f64) -> String {
250    let (lo, hi) = e.interval(confidence);
251    let mut f = vec![("value", json::number(e.value())), ("lo", json::number(lo)), ("hi", json::number(hi))];
252    match e {
253        Estimate::Rate(r) => {
254            f.push(("kind", json::string("rate")));
255            f.push(("errors", r.errors.to_string()));
256            f.push(("total", r.total.to_string()));
257        }
258        Estimate::Mean(m) => {
259            f.push(("kind", json::string("mean")));
260            f.push(("count", m.count().to_string()));
261            f.push(("std", json::number(m.std())));
262        }
263    }
264    json::object(&f)
265}
266
267/// The fields of the run information that head every JSON report (of a run, of a search
268/// and of a verification), encoded: `scenario`, `seed`, `batch`, `threads`, `stop`,
269/// `confidence`, `version`, `commit`, `config` (the configuration as a JSON document,
270/// `null` if unknown) and `started_unix_s`.
271pub fn run_header(i: &RunInfo) -> Vec<(&'static str, String)> {
272    vec![
273        ("scenario", json::string(&i.scenario)),
274        ("seed", i.seed.to_string()),
275        ("batch", i.batch.to_string()),
276        ("threads", i.threads.to_string()),
277        ("stop", json::string(&i.stop)),
278        ("confidence", json::number(i.confidence)),
279        ("version", json::string(&i.version)),
280        ("commit", json::string(&i.commit)),
281        ("config", i.config.clone().unwrap_or_else(|| "null".into())),
282        ("started_unix_s", i.started_unix_s.to_string()),
283    ]
284}
285
286/// Write a JSON document to `path`, its parent directories created.
287///
288/// # Errors
289/// [`comnum_core::Error::Io`] if the file cannot be written.
290pub fn write_json(path: &Path, doc: &str) -> Result<()> {
291    create_parent_dir(path)?;
292    std::fs::write(path, format!("{doc}\n"))?;
293    Ok(())
294}
295
296impl Logger for JsonLogger {
297    fn log(&mut self, _: &ConditionResult, _: f64) -> Result<()> {
298        Ok(())
299    }
300
301    fn finish(&mut self, rep: &RunReport) -> Result<()> {
302        let i = &rep.info;
303        let conditions: Vec<String> = rep
304            .results
305            .iter()
306            .map(|r| {
307                let cond: Vec<(&str, String)> =
308                    r.condition.entries().iter().map(|(k, v)| (k.as_str(), value_json(v))).collect();
309                let metrics: Vec<(&str, String)> =
310                    r.record.iter().map(|(k, e)| (k.as_str(), estimate_json(e, i.confidence))).collect();
311                let dropped: Vec<(&str, String)> =
312                    r.record.dropped_iter().map(|(k, n)| (k.as_str(), n.to_string())).collect();
313                json::object(&[
314                    ("index", r.index.to_string()),
315                    ("condition", json::object(&cond)),
316                    ("status", json::string(r.status.as_str())),
317                    ("trials", r.trials.to_string()),
318                    ("wall_s", json::number(r.wall_s)),
319                    ("metrics", json::object(&metrics)),
320                    ("dropped", json::object(&dropped)),
321                ])
322            })
323            .collect();
324        let mut fields = run_header(i);
325        fields.push(("wall_s", json::number(rep.wall_s)));
326        fields.push(("conditions", json::array(&conditions)));
327        write_json(&self.path, &json::object(&fields))
328    }
329}
330
331#[cfg(test)]
332mod tests {
333    use super::*;
334    use crate::prelude::*;
335    use crate::scenario::from_fn;
336
337    #[test]
338    fn the_json_report_lists_the_dropped_values() {
339        let dir = std::env::temp_dir().join(format!("comnum-dropped-{}", std::process::id()));
340        let path = dir.join("r.json");
341        let s = from_fn("drops", |_, _, rec| {
342            rec.value("snr_db", f64::INFINITY);
343            rec.value("evm", 0.1);
344            Ok(())
345        });
346        Runner::new().batch(4).stop(Stop::max_trials(6)).logger(JsonLogger::new(&path)).run(&s, &Grid::new()).unwrap();
347        let json = std::fs::read_to_string(&path).unwrap();
348        assert!(json.contains("\"dropped\": {\"snr_db\": 6}"), "{json}");
349        std::fs::remove_dir_all(&dir).unwrap();
350    }
351}
comnum_sim/loggers.rs

comnum_sim/
loggers.rs
