Skip to main content

rich_data/
lib.rs

1//! # rich-data
2//!
3//! Tabular data for [rs-rich](https://github.com/buchochelliq-labs/rs-rich-cli):
4//! one row source contract with adapters behind it, so a file, a stream of
5//! records or a query result becomes a table without hand-written glue. Not
6//! a port of anything upstream; it builds on `rich-ext`'s public API only.
7//!
8//! - [`RowSource`]: column names, an optional [`Schema`] (`rich_ext`'s
9//!   format-neutral model) and rows of [`Value`]s, one at a time. [`Rows`]
10//!   holds them in memory and renders through `rich_ext`'s
11//!   [`TableData`](rich_ext::table::TableData).
12//! - Adapters: [`csv`] (CSV and TSV through the port of Python's `csv`
13//!   sniffer and reader that `rich --csv` uses), [`jsonl`] (JSON Lines),
14//!   [`serialize`] (any `serde::Serialize` rows), and `arrow` (Arrow
15//!   `RecordBatch`es, behind the off-by-default `arrow` feature).
16//! - `er`: an ER diagram of a [`Schema`] or of SQL DDL, through
17//!   `rs-rich-diagram`, behind the off-by-default `er` feature.
18//! - [`infer`]: column types (integers, floats, booleans, dates, timestamps,
19//!   nulls, text) with the evidence for each, on request.
20//! - [`stats`]: per-column count, nulls, distinct values, min, max, mean,
21//!   median, quantiles and top values, as a table or under each heading.
22//! - [`window`]: rows for `rich_ext`'s virtualised tables: [`Rows`] by
23//!   index, and [`RowWindow`](window::RowWindow), one window of a
24//!   forward-only source read in constant memory.
25//! - [`sql`]: a query result set ([`ResultSet`](sql::ResultSet)) with typed
26//!   alignment, `NULL` marked apart from empty text, and a row count.
27//! - [`profile`]: a bounded profile of any row source (types, nulls,
28//!   distinct values, statistics, histograms or top values, and a
29//!   missing-value map), sampled when the input is long, as JSON or drawn.
30//! - [`quality`]: data quality check results (pass, warn, fail, error, with
31//!   observed and expected values and failing rows) as a report.
32//!
33//! Conditional styles for the tables these rows become are
34//! `rich_ext::table::rules`.
35//!
36//! ```
37//! use rich::Console;
38//! use rich_data::{csv, infer::Inferrer, RowSource};
39//!
40//! let mut rows = csv::CsvReader::new()
41//!     .read("service,p99,up\nweb,120,true\napi,35.5,false\n")
42//!     .unwrap();
43//! assert_eq!(rows.columns(), ["service", "p99", "up"]);
44//!
45//! // Types are only inferred when asked, and every guess carries its evidence.
46//! let inference = Inferrer::new().infer(&rows);
47//! assert_eq!(inference.columns()[1].data_type().to_string(), "float");
48//! assert_eq!(inference.columns()[1].evidence().floats, 2);
49//! inference.apply(&mut rows);
50//!
51//! let console = Console::builder().width(40).build();
52//! let out = console.render_export(&rows.to_table_data());
53//! assert!(out.contains("│ api     │ 35.5 │ false │"));
54//! ```
55
56pub mod csv;
57#[cfg(feature = "er")]
58pub mod er;
59pub mod infer;
60pub mod jsonl;
61pub mod profile;
62pub mod quality;
63mod record;
64pub mod serialize;
65pub mod sql;
66pub mod stats;
67pub mod window;
68
69#[cfg(feature = "arrow")]
70pub mod arrow;
71
72use std::fmt;
73
74use rich::Justify;
75use rich_ext::table::{Column, TableData};
76
77pub use record::RecordSource;
78pub use rich_ext::schema::{DataType, Field, Schema};
79pub use rich_ext::table::Value;
80
81/// One row of cells.
82pub type Row = Vec<Value>;
83
84/// Why rows could not be read: what went wrong, and where when the source
85/// knows (a 1-based line or record number).
86#[derive(Clone, Debug, PartialEq, Eq)]
87pub struct DataError {
88    line: Option<usize>,
89    message: String,
90}
91
92impl DataError {
93    /// An error with no position.
94    pub fn new(message: impl Into<String>) -> Self {
95        DataError {
96            line: None,
97            message: message.into(),
98        }
99    }
100
101    /// An error at a 1-based line or record.
102    pub fn at(line: usize, message: impl Into<String>) -> Self {
103        DataError {
104            line: Some(line),
105            message: message.into(),
106        }
107    }
108
109    /// The 1-based line or record, when known.
110    pub fn line(&self) -> Option<usize> {
111        self.line
112    }
113
114    /// The message, without the position.
115    pub fn message(&self) -> &str {
116        &self.message
117    }
118}
119
120impl fmt::Display for DataError {
121    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
122        match self.line {
123            Some(line) => write!(f, "line {line}: {}", self.message),
124            None => f.write_str(&self.message),
125        }
126    }
127}
128
129impl std::error::Error for DataError {}
130
131/// The row source contract: column names, an optional schema, and rows.
132///
133/// A source yields rows one at a time, so a renderer or a profile can stop
134/// early or hold one window of a long input; [`collect_rows`] reads the rest
135/// into [`Rows`]. Every row a source yields has exactly one cell per column.
136///
137/// [`collect_rows`]: RowSource::collect_rows
138pub trait RowSource {
139    /// The column names, in order.
140    fn columns(&self) -> &[String];
141
142    /// What the source knows about the columns' types, if anything. A
143    /// source's schema has one field per column, in the same order.
144    fn schema(&self) -> Option<&Schema> {
145        None
146    }
147
148    /// The next row, `None` at the end, or why it could not be read.
149    fn next_row(&mut self) -> Option<Result<Row, DataError>>;
150
151    /// Read every remaining row into memory.
152    fn collect_rows(mut self) -> Result<Rows, DataError>
153    where
154        Self: Sized,
155    {
156        let mut rows = Rows::new(self.columns().to_vec());
157        rows.schema = self.schema().cloned();
158        while let Some(row) = self.next_row() {
159            rows.push(row?);
160        }
161        Ok(rows)
162    }
163}
164
165/// Rows held in memory: column names, an optional schema, and the rows.
166#[derive(Clone, Debug, Default)]
167pub struct Rows {
168    columns: Vec<String>,
169    schema: Option<Schema>,
170    rows: Vec<Row>,
171}
172
173impl Rows {
174    /// No rows under `columns`.
175    pub fn new(columns: impl IntoIterator<Item = impl Into<String>>) -> Self {
176        Rows {
177            columns: columns.into_iter().map(Into::into).collect(),
178            schema: None,
179            rows: Vec::new(),
180        }
181    }
182
183    /// Attach a schema (one field per column).
184    pub fn with_schema(mut self, schema: Schema) -> Self {
185        self.schema = Some(schema);
186        self
187    }
188
189    /// Replace the schema.
190    pub fn set_schema(&mut self, schema: Option<Schema>) {
191        self.schema = schema;
192    }
193
194    /// Append a row: missing cells are null, and extra cells are dropped.
195    pub fn push(&mut self, row: impl IntoIterator<Item = Value>) -> &mut Self {
196        let mut row: Row = row.into_iter().take(self.columns.len()).collect();
197        row.resize(self.columns.len(), Value::Null);
198        self.rows.push(row);
199        self
200    }
201
202    /// The column names.
203    pub fn columns(&self) -> &[String] {
204        &self.columns
205    }
206
207    /// The schema, if any.
208    pub fn schema(&self) -> Option<&Schema> {
209        self.schema.as_ref()
210    }
211
212    /// The rows, in order.
213    pub fn rows(&self) -> &[Row] {
214        &self.rows
215    }
216
217    /// The rows, to change in place. Keep one cell per column.
218    pub fn rows_mut(&mut self) -> &mut [Row] {
219        &mut self.rows
220    }
221
222    /// The number of rows.
223    pub fn len(&self) -> usize {
224        self.rows.len()
225    }
226
227    /// Whether there are no rows.
228    pub fn is_empty(&self) -> bool {
229        self.rows.is_empty()
230    }
231
232    /// The position of the first column called `name`.
233    pub fn column_index(&self, name: &str) -> Option<usize> {
234        self.columns.iter().position(|c| c == name)
235    }
236
237    /// Every cell of the column at `index`, top to bottom.
238    pub fn column(&self, index: usize) -> impl Iterator<Item = &Value> {
239        self.rows.iter().filter_map(move |row| row.get(index))
240    }
241
242    /// The columns, schema and rows.
243    pub fn into_parts(self) -> (Vec<String>, Option<Schema>, Vec<Row>) {
244        (self.columns, self.schema, self.rows)
245    }
246
247    /// A row source over these rows.
248    pub fn into_source(self) -> RowsSource {
249        RowsSource {
250            columns: self.columns,
251            schema: self.schema,
252            rows: self.rows.into_iter(),
253        }
254    }
255
256    /// A table of these rows: one column per name, numeric columns (by the
257    /// schema) right-justified. Sort, group, total or style it with
258    /// `rich_ext::table`.
259    pub fn to_table_data(&self) -> TableData {
260        let columns = self.columns.iter().enumerate().map(|(index, name)| {
261            let numeric = self
262                .schema
263                .as_ref()
264                .and_then(|s| s.fields().get(index))
265                .is_some_and(|f| f.data_type().is_numeric());
266            let column = Column::new(name.clone());
267            if numeric {
268                column.justify(Justify::Right)
269            } else {
270                column
271            }
272        });
273        let mut data = TableData::new(columns);
274        data.extend(self.rows.iter().cloned());
275        data
276    }
277}
278
279/// [`Rows`] as a [`RowSource`], from [`Rows::into_source`].
280#[derive(Debug)]
281pub struct RowsSource {
282    columns: Vec<String>,
283    schema: Option<Schema>,
284    rows: std::vec::IntoIter<Row>,
285}
286
287impl RowSource for RowsSource {
288    fn columns(&self) -> &[String] {
289        &self.columns
290    }
291
292    fn schema(&self) -> Option<&Schema> {
293        self.schema.as_ref()
294    }
295
296    fn next_row(&mut self) -> Option<Result<Row, DataError>> {
297        self.rows.next().map(Ok)
298    }
299}
300
301#[cfg(test)]
302mod tests {
303    use super::*;
304
305    #[test]
306    fn rows_pad_and_cut_to_their_columns() {
307        let mut rows = Rows::new(["a", "b"]);
308        rows.push([Value::Int(1)]);
309        rows.push([Value::Int(1), Value::Int(2), Value::Int(3)]);
310        assert_eq!(rows.rows()[0], [Value::Int(1), Value::Null]);
311        assert_eq!(rows.rows()[1], [Value::Int(1), Value::Int(2)]);
312        assert_eq!(rows.column_index("b"), Some(1));
313        assert_eq!(rows.column(0).count(), 2);
314    }
315
316    #[test]
317    fn a_source_collects_back_into_the_same_rows() {
318        let mut rows =
319            Rows::new(["x"]).with_schema(Schema::new([Field::new("x", DataType::Integer)]));
320        rows.push([Value::Int(7)]);
321        let back = rows.clone().into_source().collect_rows().unwrap();
322        assert_eq!(back.rows(), rows.rows());
323        assert_eq!(back.schema(), rows.schema());
324    }
325
326    #[test]
327    fn errors_name_their_line() {
328        assert_eq!(DataError::at(3, "bad").to_string(), "line 3: bad");
329        assert_eq!(DataError::new("bad").to_string(), "bad");
330    }
331}