henad_explore/output/
summary_csv.rs

1//! Writer of `summary.csv`, statistics over the replicates of each config, rebuilt from `runs.csv`.
2//!
3//! A row holds a config's id, block and values, its run counts, the mean tick its runs ended on, and five columns
4//! for each reducer `R` of `runs.csv`: `R:mean`, `R:sd`, `R:n`, `R:ci95_low` and `R:ci95_high`. Statistics cover the
5//! runs that did not fail. `n` counts the finite values, and a statistic with too few of them is an empty cell.
6
7use std::collections::BTreeMap;
8use std::collections::btree_map::Entry;
9use std::fmt;
10use std::io::{self, BufRead, Write};
11use std::ops::Range;
12
13use henad_core::explore::outcome::RunStatus;
14use henad_core::explore::summary::{ReplicateSummary, SummaryAccumulator};
15use henad_core::export::csv::{CsvError, escape_field, fmt_f64, parse_records};
16
17use crate::output::read::{RecordScan, shifted};
18use crate::output::runs_csv::{NOTE_COLUMN, OUTCOME_COLUMNS};
19
20/// Statistics written for each reducer, as suffixes of its column name.
21pub const STATISTICS: [&str; 5] = ["mean", "sd", "n", "ci95_low", "ci95_high"];
22
23/// Text of a `runs.csv` that cannot be summarized.
24#[derive(Debug)]
25pub enum SummaryError {
26    /// Text that is not valid CSV.
27    Csv(CsvError),
28    /// A header without the column `column`.
29    MissingColumn {
30        /// Name of the missing column.
31        column: &'static str,
32    },
33    /// Record `record_number`, with a different number of fields from the header.
34    FieldCount {
35        /// Number of the record, counting the header as record 1.
36        record_number: usize,
37        /// Number of fields in the record.
38        found: usize,
39        /// Number of fields in the header.
40        expected: usize,
41    },
42    /// Field `column` of record `record_number`, holding `text` that the column cannot accept.
43    BadField {
44        /// Number of the record, counting the header as record 1.
45        record_number: usize,
46        /// Name of the column holding the field.
47        column: String,
48        /// Text of the field after CSV unescaping.
49        text: String,
50    },
51    /// Reading `runs.csv` failed, or it is not UTF-8.
52    Read(io::Error),
53    /// Writing the summary failed.
54    Io(io::Error),
55}
56
57impl fmt::Display for SummaryError {
58    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
59        match self {
60            Self::Csv(_) => f.write_str("runs.csv is not valid CSV"),
61            Self::Read(_) => f.write_str("cannot read runs.csv"),
62            Self::MissingColumn { column } => write!(f, "runs.csv has no '{column}' column"),
63            Self::FieldCount {
64                record_number,
65                found,
66                expected,
67            } => write!(
68                f,
69                "record {record_number} of runs.csv has {found} fields, expected {expected}"
70            ),
71            Self::BadField {
72                record_number,
73                column,
74                text,
75            } => {
76                write!(
77                    f,
78                    "record {record_number} of runs.csv has invalid value '{text}' in column '{column}'"
79                )
80            }
81            Self::Io(_) => f.write_str("cannot write the summary"),
82        }
83    }
84}
85
86impl std::error::Error for SummaryError {
87    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
88        match self {
89            Self::Csv(error) => Some(error),
90            Self::Read(error) | Self::Io(error) => Some(error),
91            Self::MissingColumn { .. } | Self::FieldCount { .. } | Self::BadField { .. } => None,
92        }
93    }
94}
95
96impl From<io::Error> for SummaryError {
97    fn from(error: io::Error) -> Self {
98        Self::Io(error)
99    }
100}
101
102/// Positions of the columns of a `runs.csv` header that the summary reads.
103///
104/// Note that a parameter can share its name with a later column. Those columns are found from the end of the header.
105struct RunsLayout {
106    config_id: usize,
107    block: usize,
108    /// Parameter and action columns, between `run_key` and `status`.
109    values: Range<usize>,
110    status: usize,
111    ticks: usize,
112    /// Reducer columns, between `steps_per_s` and `note`.
113    reducers: Range<usize>,
114}
115
116impl RunsLayout {
117    /// Finds the columns in `header`. Each column after the parameters is searched for after the column before it, and
118    /// from the end of the header, so the ranges between them never run backwards.
119    fn read(header: &[String]) -> Result<Self, SummaryError> {
120        let find_first = |column: &'static str| {
121            header
122                .iter()
123                .position(|name| name == column)
124                .ok_or(SummaryError::MissingColumn { column })
125        };
126        let find_last_from = |start: usize, column: &'static str| {
127            header
128                .get(start..)
129                .and_then(|rest| rest.iter().rposition(|name| name == column))
130                .map(|position| start + position)
131                .ok_or(SummaryError::MissingColumn { column })
132        };
133        let last_outcome = OUTCOME_COLUMNS[OUTCOME_COLUMNS.len() - 1];
134        let config_id = find_first("config_id")?;
135        let block = find_first("block")?;
136        let values_start = find_first("run_key")? + 1;
137        let status = find_last_from(values_start, "status")?;
138        let reducers_start = find_last_from(status, last_outcome)? + 1;
139        Ok(Self {
140            config_id,
141            block,
142            values: values_start..status,
143            status,
144            ticks: find_last_from(status, "ticks")?,
145            reducers: reducers_start..find_last_from(reducers_start, NOTE_COLUMN)?,
146        })
147    }
148}
149
150/// Reader of the records of a CSV text, one at a time.
151struct RecordReader<R> {
152    source: R,
153    /// Text of the record being read.
154    text: Vec<u8>,
155    /// Number of line feeds read so far.
156    line_feeds: usize,
157}
158
159impl<R: BufRead> RecordReader<R> {
160    fn new(source: R) -> Self {
161        Self {
162            source,
163            text: Vec::new(),
164            line_feeds: 0,
165        }
166    }
167
168    /// Returns the fields of the next record, or `None` at the end of the text. The last record's line ending is
169    /// optional.
170    fn next_record(&mut self) -> Result<Option<Vec<String>>, SummaryError> {
171        self.text.clear();
172        let lines_before = self.line_feeds;
173        let mut scan = RecordScan::FieldStart;
174        let mut record_ended = false;
175        while !record_ended {
176            let start = self.text.len();
177            // A read stops after the first line feed, the one byte that can end a record.
178            if self
179                .source
180                .read_until(b'\n', &mut self.text)
181                .map_err(SummaryError::Read)?
182                == 0
183            {
184                break;
185            }
186            for &byte in &self.text[start..] {
187                (scan, record_ended) = scan.advance(byte);
188            }
189            if self.text.ends_with(b"\n") {
190                self.line_feeds += 1;
191            }
192        }
193        if self.text.is_empty() {
194            return Ok(None);
195        }
196        let text = std::str::from_utf8(&self.text)
197            .map_err(|error| SummaryError::Read(io::Error::new(io::ErrorKind::InvalidData, error)))?;
198        let record = parse_records(text)
199            .map_err(|error| SummaryError::Csv(shifted(&error, lines_before)))?
200            .into_iter()
201            .next();
202        Ok(record)
203    }
204}
205
206/// Reads `runs`, a `runs.csv`, writes the summary of every config in it to `dest`, and returns `dest`.
207///
208/// The runs are read one record at a time. Configs are written in id order. Note that the statistics depend on the
209/// order of the runs in `runs`, down to the last bit.
210///
211/// # Errors
212///
213/// Returns [`SummaryError`] when `runs` cannot be read or is not a `runs.csv`, or a write fails.
214pub(crate) fn write_summary<R: BufRead, W: Write>(runs: R, mut dest: W) -> Result<W, SummaryError> {
215    let mut records = RecordReader::new(runs);
216    let Some(header) = records.next_record()? else {
217        return Err(SummaryError::MissingColumn { column: "run_id" });
218    };
219    let layout = RunsLayout::read(&header)?;
220    let reducer_names = &header[layout.reducers.clone()];
221
222    let mut accumulator = SummaryAccumulator::new(reducer_names.len());
223    // Block and value cells of each config, escaped and joined, as its first run records them.
224    let mut configs: BTreeMap<u64, String> = BTreeMap::new();
225    let mut reducers = Vec::with_capacity(reducer_names.len());
226    let mut record_number = 1;
227    while let Some(row) = records.next_record()? {
228        record_number += 1;
229        if row.len() != header.len() {
230            return Err(SummaryError::FieldCount {
231                record_number,
232                found: row.len(),
233                expected: header.len(),
234            });
235        }
236        let bad_field = |column: usize| SummaryError::BadField {
237            record_number,
238            column: header[column].clone(),
239            text: row[column].clone(),
240        };
241        let config_id: u64 = row[layout.config_id]
242            .parse()
243            .ok()
244            .ok_or_else(|| bad_field(layout.config_id))?;
245        let status: RunStatus = row[layout.status]
246            .parse()
247            .ok()
248            .ok_or_else(|| bad_field(layout.status))?;
249        let ticks: u64 = row[layout.ticks].parse().ok().ok_or_else(|| bad_field(layout.ticks))?;
250        reducers.clear();
251        for column in layout.reducers.clone() {
252            let value = match row[column].as_str() {
253                "" => None,
254                text => Some(text.parse::<f64>().ok().ok_or_else(|| bad_field(column))?),
255            };
256            reducers.push(value);
257        }
258        accumulator.push(config_id, status, ticks, &reducers);
259        if let Entry::Vacant(entry) = configs.entry(config_id) {
260            let mut cells = escape_field(&row[layout.block]);
261            for value in &row[layout.values.clone()] {
262                cells.push(',');
263                cells.push_str(&escape_field(value));
264            }
265            entry.insert(cells);
266        }
267    }
268
269    let mut columns: Vec<String> = ["config_id", "block"].map(str::to_owned).to_vec();
270    columns.extend(header[layout.values.clone()].iter().map(|name| escape_field(name)));
271    columns.extend(["runs", "ok", "failed", "ticks:mean"].map(str::to_owned));
272    for name in reducer_names {
273        columns.extend(STATISTICS.map(|statistic| escape_field(&format!("{name}:{statistic}"))));
274    }
275    writeln!(dest, "{}", columns.join(","))?;
276    for summary in accumulator.rows() {
277        let Some(config_cells) = configs.remove(&summary.config_id) else {
278            continue;
279        };
280        let mut cells = vec![summary.config_id.to_string(), config_cells];
281        cells.extend([summary.runs, summary.ok, summary.failed].map(|count| count.to_string()));
282        cells.push(optional_cell(summary.ticks.mean));
283        for reducer in &summary.reducers {
284            cells.extend(statistic_cells(reducer));
285        }
286        writeln!(dest, "{}", cells.join(","))?;
287    }
288    dest.flush()?;
289    Ok(dest)
290}
291
292/// Returns the cells of `summary` in [`STATISTICS`] order.
293fn statistic_cells(summary: &ReplicateSummary) -> [String; 5] {
294    [
295        optional_cell(summary.mean),
296        optional_cell(summary.standard_deviation),
297        summary.n.to_string(),
298        optional_cell(summary.ci95.map(|(low, _)| low)),
299        optional_cell(summary.ci95.map(|(_, high)| high)),
300    ]
301}
302
303/// Returns `value` as a cell, empty for `None`.
304fn optional_cell(value: Option<f64>) -> String {
305    value.map_or_else(String::new, fmt_f64)
306}
307
308#[cfg(test)]
309mod tests {
310    use std::io::BufReader;
311
312    use henad_core::export::csv::CsvError;
313
314    use super::{SummaryError, write_summary};
315
316    const RUNS: &str = "\
317run_id,config_id,block,rep,seed,run_key,rate,\"a, b\",status,stop_reason,ticks,population,build_ms,wall_ms,steps_per_s,Infected:max,\"Speed.[0, 1):min\",note
3180,0,0,0,11,00000000000000aa,0.1,x,ok,steps,10,64,1,2,5000,1,,
3191,0,0,1,12,00000000000000ab,0.1,x,ok,steps,10,64,1,2,5000,2,4,
3202,0,0,2,13,00000000000000ac,0.1,x,non_finite,steps,10,64,1,2,5000,3,,Infected is not finite at tick 5
3213,0,0,3,14,00000000000000ad,0.1,x,ok,steps,10,64,1,2,5000,4,,
3224,1,1,0,11,00000000000000ba,0.2,\"y, z\",panicked,fault,3,64,1,2,1500,99,,\"while stepping, it panicked\"
3235,1,1,1,12,00000000000000bb,0.2,\"y, z\",ok,steps,10,64,1,2,5000,7,1,
324";
325
326    fn summary(runs: &str) -> Result<String, SummaryError> {
327        write_summary(runs.as_bytes(), Vec::new()).map(|bytes| String::from_utf8(bytes).expect("the rows are UTF-8"))
328    }
329
330    #[test]
331    fn a_record_read_across_many_reads_is_summarized_as_one() {
332        let runs = RUNS.replace(
333            "x,ok,steps,10,64,1,2,5000,1,,",
334            "\"x\",ok,steps,10,64,1,2,5000,1,,\"two\nlines\"",
335        );
336        let whole = summary(&runs).expect("a valid runs.csv");
337        let byte_by_byte =
338            write_summary(BufReader::with_capacity(1, runs.as_bytes()), Vec::new()).expect("a valid runs.csv");
339        assert_eq!(String::from_utf8(byte_by_byte).expect("the rows are UTF-8"), whole);
340        assert_eq!(
341            whole,
342            summary(RUNS).expect("a valid runs.csv"),
343            "quotes change no value"
344        );
345
346        let stray = runs.replace("5,1,1,1,12,", "5,1,1,1,1\"2,");
347        assert!(
348            matches!(
349                summary(&stray),
350                Err(SummaryError::Csv(CsvError::MisplacedQuote { line: 8 }))
351            ),
352            "the line counts from the start of the file, the quoted line feed included"
353        );
354    }
355
356    #[test]
357    fn a_header_out_of_order_is_refused() {
358        let note_first = RUNS
359            .replacen(",note\n", ",Late\n", 1)
360            .replacen("run_key,", "run_key,note,", 1);
361        assert!(matches!(
362            summary(note_first.lines().next().expect("a header")),
363            Err(SummaryError::MissingColumn { column: "note" })
364        ));
365        let status_first = "status,run_id,config_id,block,rep,seed,run_key,stop_reason,ticks,population,build_ms,wall_ms,steps_per_s,note\n";
366        assert!(matches!(
367            summary(status_first),
368            Err(SummaryError::MissingColumn { column: "status" })
369        ));
370    }
371
372    #[test]
373    fn the_summary_matches_hand_computed_statistics() {
374        let text = summary(RUNS).expect("a valid runs.csv");
375        let lines: Vec<&str> = text.lines().collect();
376        assert_eq!(
377            lines[0],
378            "config_id,block,rate,\"a, b\",runs,ok,failed,ticks:mean,\
379             Infected:max:mean,Infected:max:sd,Infected:max:n,Infected:max:ci95_low,Infected:max:ci95_high,\
380             \"Speed.[0, 1):min:mean\",\"Speed.[0, 1):min:sd\",\"Speed.[0, 1):min:n\",\
381             \"Speed.[0, 1):min:ci95_low\",\"Speed.[0, 1):min:ci95_high\""
382        );
383        let first: Vec<&str> = lines[1].split(',').collect();
384        assert_eq!(first[..8], ["0", "0", "0.1", "x", "4", "3", "0", "10"]);
385        assert_eq!(first[8], "2.5", "the mean of 1, 2, 3 and 4");
386        let sd: f64 = first[9].parse().expect("a spread");
387        assert!((sd - 1.290_994).abs() < 1e-6, "sd {sd}");
388        assert_eq!(first[10], "4");
389        let low: f64 = first[11].parse().expect("an interval");
390        assert!((2.5 - low - 2.054_260).abs() < 1e-6, "half-width {}", 2.5 - low);
391        assert_eq!(first[13..], ["4", "", "1", "", ""], "one finite value has no spread");
392
393        assert_eq!(
394            lines[2], "1,1,0.2,\"y, z\",2,1,1,10,7,,1,,,1,,1,,",
395            "the failed run is counted and left out"
396        );
397        assert_eq!(lines.len(), 3);
398    }
399
400    #[test]
401    fn a_parameter_named_like_a_later_column_is_read_as_a_parameter() {
402        let runs = "\
403run_id,config_id,block,rep,seed,run_key,ticks,status,note,status,stop_reason,ticks,population,build_ms,wall_ms,steps_per_s,Infected:max,note
4040,0,0,0,11,00000000000000aa,5,x,y,ok,steps,10,64,1,2,5000,1,
4051,0,0,1,12,00000000000000ab,5,x,y,ok,steps,10,64,1,2,5000,3,
406";
407        let text = summary(runs).expect("a valid runs.csv");
408        let lines: Vec<&str> = text.lines().collect();
409        assert!(
410            lines[0].starts_with("config_id,block,ticks,status,note,runs,ok,failed,ticks:mean,Infected:max:mean,"),
411            "{}",
412            lines[0]
413        );
414        let first: Vec<&str> = lines[1].split(',').collect();
415        assert_eq!(first[..10], ["0", "0", "5", "x", "y", "2", "2", "0", "10", "2"]);
416    }
417
418    #[test]
419    fn a_table_that_is_not_runs_csv_is_refused() {
420        assert!(matches!(
421            summary("run_id,config_id\n0,0\n"),
422            Err(SummaryError::MissingColumn { column: "block" })
423        ));
424        let short = RUNS.replace("5,1,1,1,12,", "5,1,1,12,");
425        let error = summary(&short).expect_err("a short record");
426        assert!(matches!(
427            error,
428            SummaryError::FieldCount {
429                record_number: 7,
430                found: 17,
431                expected: 18
432            }
433        ));
434        assert_eq!(error.to_string(), "record 7 of runs.csv has 17 fields, expected 18");
435        let bad = RUNS.replace("ok,steps,10,64,1,2,5000,4,", "ok,steps,ten,64,1,2,5000,4,");
436        assert!(
437            matches!(summary(&bad), Err(SummaryError::BadField { record_number: 5, column, .. }) if column == "ticks")
438        );
439    }
440}
henad_explore/output/summary_csv.rs

henad_explore/output/
summary_csv.rs
