datui_lib/formats/
pushdown.rs

1//! Sources that read a window of rows themselves, and sources that filter and sort.
2//!
3//! Polars hands an anonymous scan no row offset, so a window deep in a view of one would
4//! read every row before it. A [`Windowed`] source starts the window where it is asked
5//! to. A [`Pushdown`] source goes further: it runs the sidebar's filters and sort itself
6//! (a SQLite table, whose indexes do the work) and gives the view as a frame, a window
7//! of it at a time, and a count.
8
9use std::sync::Arc;
10
11use polars::prelude::{Expr, LazyFrame, Operator, PolarsResult};
12
13use crate::app::modals::filter_modal::FilterStatement;
14
15/// A source that reads rows `[start, start + len)` of a view without the ones before.
16///
17/// `Any`, so a reader whose source the app asks more of than rows can find its own
18/// again: an audio file's, whose signal a quality run checks.
19pub trait Windowed: Send + Sync + std::any::Any {
20    /// The rows as a frame, read when it is collected.
21    fn window(&self, start: usize, len: usize) -> PolarsResult<LazyFrame>;
22}
23
24/// How many rows a view holds, counted by its source. Blocks.
25pub type Counter = Arc<dyn Fn() -> PolarsResult<usize> + Send + Sync>;
26
27/// A view the source runs itself.
28#[derive(Clone)]
29pub struct PushedView {
30    /// Every row of the view, in its order.
31    pub lf: LazyFrame,
32    /// The view's rows a window at a time.
33    pub window: Arc<dyn Windowed>,
34    /// Its row count.
35    pub counter: Counter,
36}
37
38/// A source that filters and sorts. What it notices while reading is said through
39/// [`Pushdown::notes`], which may grow once a pass over the whole source is done.
40pub trait Pushdown: Send + Sync {
41    /// The view with `filters` and the sort `sort` (column, descending; nulls last and
42    /// ties in the source's order), or the source's order backward when `reversed` and
43    /// there is no sort. `None` when the source cannot run it, and Polars does.
44    fn view(
45        &self,
46        filters: &[FilterStatement],
47        sort: &[(String, bool)],
48        reversed: bool,
49    ) -> Option<PushedView>;
50
51    /// What reading the source has found to say.
52    fn notes(&self) -> Vec<crate::notes::Note>;
53
54    /// The source itself, for its tests.
55    #[cfg(test)]
56    fn as_any(&self) -> &dyn std::any::Any;
57}
58
59/// What of a predicate Polars pushed into an anonymous scan the scan can evaluate.
60///
61/// A sort with a limit (a top-k) pushes a dynamic predicate, its running bound, into
62/// the scan as an `Expr::Display`, which panics when it is turned back into a plan
63/// (pola-rs/polars#28629; #28643 strips it for Python IO plugins only). The bound is
64/// a hint the top-k applies itself, so the terms holding it are dropped.
65pub(crate) fn evaluable(predicate: Option<Expr>) -> Option<Expr> {
66    fn terms(e: Expr, out: &mut Vec<Expr>) {
67        match e {
68            Expr::BinaryExpr {
69                left,
70                op: Operator::And | Operator::LogicalAnd,
71                right,
72            } => {
73                terms(Arc::unwrap_or_clone(left), out);
74                terms(Arc::unwrap_or_clone(right), out);
75            }
76            e => out.push(e),
77        }
78    }
79    let mut all = Vec::new();
80    terms(predicate?, &mut all);
81    all.into_iter()
82        .filter(|term| !term.into_iter().any(|e| matches!(e, Expr::Display { .. })))
83        .reduce(|left, right| left.and(right))
84}
85
86#[cfg(test)]
87mod tests {
88    use super::*;
89    use polars::prelude::{col, lit};
90
91    fn bound() -> Expr {
92        Expr::Display {
93            inputs: vec![col("id")],
94            fmt_str: Box::new("dynamic_pred: 1".into()),
95        }
96    }
97
98    #[test]
99    fn a_top_k_bound_is_dropped_from_a_pushed_predicate() {
100        let kept = col("id").lt(lit(4));
101        assert_eq!(
102            evaluable(Some(kept.clone().and(bound()))),
103            Some(kept.clone())
104        );
105        assert_eq!(
106            evaluable(Some(bound().and(kept.clone()))),
107            Some(kept.clone())
108        );
109        assert_eq!(evaluable(Some(bound())), None);
110        assert_eq!(evaluable(Some(kept.clone())), Some(kept));
111        assert_eq!(evaluable(None), None);
112    }
113}
datui_lib/formats/pushdown.rs

datui_lib/formats/
pushdown.rs
