rich_data/lib.rs
1//! # rich-data
2//!
3//! Tabular data for [rs-rich](https://github.com/buchochelliq-labs/rs-rich-cli):
4//! one row source contract with adapters behind it, so a file, a stream of
5//! records or a query result becomes a table without hand-written glue. Not
6//! a port of anything upstream; it builds on `rich-ext`'s public API only.
7//!
8//! - [`RowSource`]: column names, an optional [`Schema`] (`rich_ext`'s
9//! format-neutral model) and rows of [`Value`]s, one at a time. [`Rows`]
10//! holds them in memory and renders through `rich_ext`'s
11//! [`TableData`](rich_ext::table::TableData).
12//! - Adapters: [`csv`] (CSV and TSV through the port of Python's `csv`
13//! sniffer and reader that `rich --csv` uses), [`jsonl`] (JSON Lines),
14//! [`serialize`] (any `serde::Serialize` rows), and `arrow` (Arrow
15//! `RecordBatch`es, behind the off-by-default `arrow` feature).
16//! - `er`: an ER diagram of a [`Schema`] or of SQL DDL, through
17//! `rs-rich-diagram`, behind the off-by-default `er` feature.
18//! - [`infer`]: column types (integers, floats, booleans, dates, timestamps,
19//! nulls, text) with the evidence for each, on request.
20//! - [`stats`]: per-column count, nulls, distinct values, min, max, mean,
21//! median, quantiles and top values, as a table or under each heading.
22//! - [`window`]: rows for `rich_ext`'s virtualised tables: [`Rows`] by
23//! index, and [`RowWindow`](window::RowWindow), one window of a
24//! forward-only source read in constant memory.
25//! - [`sql`]: a query result set ([`ResultSet`](sql::ResultSet)) with typed
26//! alignment, `NULL` marked apart from empty text, and a row count.
27//! - [`profile`]: a bounded profile of any row source (types, nulls,
28//! distinct values, statistics, histograms or top values, and a
29//! missing-value map), sampled when the input is long, as JSON or drawn;
30//! and of JSON records path by path, with presence and type mixes.
31//! - [`quality`]: data quality check results (pass, warn, fail, error, with
32//! observed and expected values and failing rows) as a report.
33//!
34//! Conditional styles for the tables these rows become are
35//! `rich_ext::table::rules`.
36//!
37//! ```
38//! use rich::Console;
39//! use rich_data::{csv, infer::Inferrer, RowSource};
40//!
41//! let mut rows = csv::CsvReader::new()
42//! .read("service,p99,up\nweb,120,true\napi,35.5,false\n")
43//! .unwrap();
44//! assert_eq!(rows.columns(), ["service", "p99", "up"]);
45//!
46//! // Types are only inferred when asked, and every guess carries its evidence.
47//! let inference = Inferrer::new().infer(&rows);
48//! assert_eq!(inference.columns()[1].data_type().to_string(), "float");
49//! assert_eq!(inference.columns()[1].evidence().floats, 2);
50//! inference.apply(&mut rows);
51//!
52//! let console = Console::builder().width(40).build();
53//! let out = console.render_export(&rows.to_table_data());
54//! assert!(out.contains("│ api │ 35.5 │ false │"));
55//! ```
56
57pub mod csv;
58#[cfg(feature = "er")]
59pub mod er;
60pub mod infer;
61pub mod jsonl;
62pub mod profile;
63pub mod quality;
64mod record;
65pub mod serialize;
66pub mod sql;
67pub mod stats;
68pub mod window;
69
70#[cfg(feature = "arrow")]
71pub mod arrow;
72
73use std::fmt;
74
75use rich::Justify;
76use rich_ext::table::{Column, TableData};
77
78pub use record::RecordSource;
79pub use rich_ext::schema::{DataType, Field, Schema};
80pub use rich_ext::table::Value;
81
82/// One row of cells.
83pub type Row = Vec<Value>;
84
85/// Why rows could not be read: what went wrong, and where when the source
86/// knows (a 1-based line or record number).
87#[derive(Clone, Debug, PartialEq, Eq)]
88pub struct DataError {
89 line: Option<usize>,
90 message: String,
91}
92
93impl DataError {
94 /// An error with no position.
95 pub fn new(message: impl Into<String>) -> Self {
96 DataError {
97 line: None,
98 message: message.into(),
99 }
100 }
101
102 /// An error at a 1-based line or record.
103 pub fn at(line: usize, message: impl Into<String>) -> Self {
104 DataError {
105 line: Some(line),
106 message: message.into(),
107 }
108 }
109
110 /// The 1-based line or record, when known.
111 pub fn line(&self) -> Option<usize> {
112 self.line
113 }
114
115 /// The message, without the position.
116 pub fn message(&self) -> &str {
117 &self.message
118 }
119}
120
121impl fmt::Display for DataError {
122 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
123 match self.line {
124 Some(line) => write!(f, "line {line}: {}", self.message),
125 None => f.write_str(&self.message),
126 }
127 }
128}
129
130impl std::error::Error for DataError {}
131
132/// The row source contract: column names, an optional schema, and rows.
133///
134/// A source yields rows one at a time, so a renderer or a profile can stop
135/// early or hold one window of a long input; [`collect_rows`] reads the rest
136/// into [`Rows`]. Every row a source yields has exactly one cell per column.
137///
138/// [`collect_rows`]: RowSource::collect_rows
139pub trait RowSource {
140 /// The column names, in order.
141 fn columns(&self) -> &[String];
142
143 /// What the source knows about the columns' types, if anything. A
144 /// source's schema has one field per column, in the same order.
145 fn schema(&self) -> Option<&Schema> {
146 None
147 }
148
149 /// The next row, `None` at the end, or why it could not be read.
150 fn next_row(&mut self) -> Option<Result<Row, DataError>>;
151
152 /// Read every remaining row into memory.
153 fn collect_rows(mut self) -> Result<Rows, DataError>
154 where
155 Self: Sized,
156 {
157 let mut rows = Rows::new(self.columns().to_vec());
158 rows.schema = self.schema().cloned();
159 while let Some(row) = self.next_row() {
160 rows.push(row?);
161 }
162 Ok(rows)
163 }
164}
165
166/// Rows held in memory: column names, an optional schema, and the rows.
167#[derive(Clone, Debug, Default)]
168pub struct Rows {
169 columns: Vec<String>,
170 schema: Option<Schema>,
171 rows: Vec<Row>,
172}
173
174impl Rows {
175 /// No rows under `columns`.
176 pub fn new(columns: impl IntoIterator<Item = impl Into<String>>) -> Self {
177 Rows {
178 columns: columns.into_iter().map(Into::into).collect(),
179 schema: None,
180 rows: Vec::new(),
181 }
182 }
183
184 /// Attach a schema (one field per column).
185 pub fn with_schema(mut self, schema: Schema) -> Self {
186 self.schema = Some(schema);
187 self
188 }
189
190 /// Replace the schema.
191 pub fn set_schema(&mut self, schema: Option<Schema>) {
192 self.schema = schema;
193 }
194
195 /// Append a row: missing cells are null, and extra cells are dropped.
196 pub fn push(&mut self, row: impl IntoIterator<Item = Value>) -> &mut Self {
197 let mut row: Row = row.into_iter().take(self.columns.len()).collect();
198 row.resize(self.columns.len(), Value::Null);
199 self.rows.push(row);
200 self
201 }
202
203 /// The column names.
204 pub fn columns(&self) -> &[String] {
205 &self.columns
206 }
207
208 /// The schema, if any.
209 pub fn schema(&self) -> Option<&Schema> {
210 self.schema.as_ref()
211 }
212
213 /// The rows, in order.
214 pub fn rows(&self) -> &[Row] {
215 &self.rows
216 }
217
218 /// The rows, to change in place. Keep one cell per column.
219 pub fn rows_mut(&mut self) -> &mut [Row] {
220 &mut self.rows
221 }
222
223 /// The number of rows.
224 pub fn len(&self) -> usize {
225 self.rows.len()
226 }
227
228 /// Whether there are no rows.
229 pub fn is_empty(&self) -> bool {
230 self.rows.is_empty()
231 }
232
233 /// The position of the first column called `name`.
234 pub fn column_index(&self, name: &str) -> Option<usize> {
235 self.columns.iter().position(|c| c == name)
236 }
237
238 /// Every cell of the column at `index`, top to bottom.
239 pub fn column(&self, index: usize) -> impl Iterator<Item = &Value> {
240 self.rows.iter().filter_map(move |row| row.get(index))
241 }
242
243 /// The columns, schema and rows.
244 pub fn into_parts(self) -> (Vec<String>, Option<Schema>, Vec<Row>) {
245 (self.columns, self.schema, self.rows)
246 }
247
248 /// A row source over these rows.
249 pub fn into_source(self) -> RowsSource {
250 RowsSource {
251 columns: self.columns,
252 schema: self.schema,
253 rows: self.rows.into_iter(),
254 }
255 }
256
257 /// A table of these rows: one column per name, numeric columns (by the
258 /// schema) right-justified. Sort, group, total or style it with
259 /// `rich_ext::table`.
260 pub fn to_table_data(&self) -> TableData {
261 let columns = self.columns.iter().enumerate().map(|(index, name)| {
262 let numeric = self
263 .schema
264 .as_ref()
265 .and_then(|s| s.fields().get(index))
266 .is_some_and(|f| f.data_type().is_numeric());
267 let column = Column::new(name.clone());
268 if numeric {
269 column.justify(Justify::Right)
270 } else {
271 column
272 }
273 });
274 let mut data = TableData::new(columns);
275 data.extend(self.rows.iter().cloned());
276 data
277 }
278}
279
280/// [`Rows`] as a [`RowSource`], from [`Rows::into_source`].
281#[derive(Debug)]
282pub struct RowsSource {
283 columns: Vec<String>,
284 schema: Option<Schema>,
285 rows: std::vec::IntoIter<Row>,
286}
287
288impl RowSource for RowsSource {
289 fn columns(&self) -> &[String] {
290 &self.columns
291 }
292
293 fn schema(&self) -> Option<&Schema> {
294 self.schema.as_ref()
295 }
296
297 fn next_row(&mut self) -> Option<Result<Row, DataError>> {
298 self.rows.next().map(Ok)
299 }
300}
301
302#[cfg(test)]
303mod tests {
304 use super::*;
305
306 #[test]
307 fn rows_pad_and_cut_to_their_columns() {
308 let mut rows = Rows::new(["a", "b"]);
309 rows.push([Value::Int(1)]);
310 rows.push([Value::Int(1), Value::Int(2), Value::Int(3)]);
311 assert_eq!(rows.rows()[0], [Value::Int(1), Value::Null]);
312 assert_eq!(rows.rows()[1], [Value::Int(1), Value::Int(2)]);
313 assert_eq!(rows.column_index("b"), Some(1));
314 assert_eq!(rows.column(0).count(), 2);
315 }
316
317 #[test]
318 fn a_source_collects_back_into_the_same_rows() {
319 let mut rows =
320 Rows::new(["x"]).with_schema(Schema::new([Field::new("x", DataType::Integer)]));
321 rows.push([Value::Int(7)]);
322 let back = rows.clone().into_source().collect_rows().unwrap();
323 assert_eq!(back.rows(), rows.rows());
324 assert_eq!(back.schema(), rows.schema());
325 }
326
327 #[test]
328 fn errors_name_their_line() {
329 assert_eq!(DataError::at(3, "bad").to_string(), "line 3: bad");
330 assert_eq!(DataError::new("bad").to_string(), "bad");
331 }
332}