densa_data/lib.rs
1//! Live Arrow tables: rows identified by a key, updated by record batches, and views that
2//! stay sorted and filtered as the rows change.
3//!
4//! [`LiveTable`] takes Arrow [`RecordBatch`](arrow_array::RecordBatch)es from a feed, a
5//! query engine or a file, and upserts them by a key column; [`ViewSpec`] describes a view
6//! (a [`Filter`] and sort columns), which the table keeps up to date incrementally instead of
7//! recomputing it; [`GroupSpec`] does the same for grouped aggregates (count, sum, min, max,
8//! mean). A view's rows are [`RowRef`]s into [`LiveTable::batches`], so a display
9//! layer (such as `densa-arrow`'s grid) reads values straight from the Arrow arrays. Each
10//! key's row is a [`RecordId`] while the key lives: a selection can follow it through
11//! updates, compactions and re-sorted views ([`View::position`]). [`LiveTable::snapshot`]
12//! hands the live rows to anything else that reads Arrow, such as a SQL engine. A
13//! [`Window`] keeps a table to its recent rows, by a time column.
14//!
15//! This crate has no user-interface dependency.
16//!
17//! ```
18//! use std::sync::Arc;
19//! use arrow_array::{ArrayRef, Float64Array, RecordBatch, StringArray};
20//! use densa_data::{CompareOp, Filter, LiveTable, Order, Scalar, ViewSpec};
21//!
22//! let quotes = |symbols: Vec<&str>, prices: Vec<f64>| {
23//! RecordBatch::try_from_iter([
24//! ("symbol", Arc::new(StringArray::from(symbols)) as ArrayRef),
25//! ("price", Arc::new(Float64Array::from(prices)) as ArrayRef),
26//! ])
27//! };
28//! let mut table = LiveTable::new(quotes(vec![], vec![])?.schema(), "symbol")?;
29//! let expensive = table.add_view(
30//! ViewSpec::new()
31//! .filter(Filter::compare("price", CompareOp::Gt, Scalar::Float(100.0)))
32//! .sort_by("price", Order::Descending),
33//! )?;
34//!
35//! table.upsert(quotes(vec!["AAA", "BBB", "CCC"], vec![150.0, 90.0, 120.0])?)?;
36//! table.upsert(quotes(vec!["BBB"], vec![200.0])?)?; // BBB rises: it enters the view
37//!
38//! let view = table.view(expensive).unwrap();
39//! assert_eq!(view.len(), 3); // BBB 200, AAA 150, CCC 120
40//! # Ok::<(), Box<dyn std::error::Error>>(())
41//! ```
42
43mod dictionary;
44mod error;
45mod filter;
46mod group;
47mod key;
48mod snapshot;
49mod sorted;
50mod table;
51mod value;
52mod view;
53mod window;
54
55pub use arrow_array;
56pub use arrow_schema;
57pub use error::DataError;
58pub use filter::{CompareOp, Filter, Scalar};
59pub use group::{Aggregate, GroupSpec, Groups};
60pub use table::{Changes, GroupsId, LiveTable, RecordId, RowRef, TableOptions, View, ViewId};
61pub use view::{Order, ViewSpec};
62pub use window::Window;
63
64/// A row's internal identity: stable for as long as its key is live.
65type RowId = u32;