datui_lib/
pushdown.rs

1//! Sources that read a window of rows themselves, and sources that filter and sort.
2//!
3//! Polars hands an anonymous scan no row offset, so a window deep in a view of one would
4//! read every row before it. A [`Windowed`] source starts the window where it is asked
5//! to. A [`Pushdown`] source goes further: it runs the sidebar's filters and sort itself
6//! (a SQLite table, whose indexes do the work) and gives the view as a frame, a window
7//! of it at a time, and a count.
8
9use std::sync::Arc;
10
11use polars::prelude::{Expr, LazyFrame, Operator, PolarsResult};
12
13use crate::filter_modal::FilterStatement;
14
15/// A source that reads rows `[start, start + len)` of a view without the ones before.
16///
17/// `Any`, so a reader whose source the app asks more of than rows can find its own
18/// again: an audio file's, whose signal a quality run checks.
19pub trait Windowed: Send + Sync + std::any::Any {
20    /// The rows as a frame, read when it is collected.
21    fn window(&self, start: usize, len: usize) -> PolarsResult<LazyFrame>;
22}
23
24/// How many rows a view holds, counted by its source. Blocks.
25pub type Counter = Arc<dyn Fn() -> PolarsResult<usize> + Send + Sync>;
26
27/// A view the source runs itself.
28#[derive(Clone)]
29pub struct PushedView {
30    /// Every row of the view, in its order.
31    pub lf: LazyFrame,
32    /// The view's rows a window at a time.
33    pub window: Arc<dyn Windowed>,
34    /// Its row count.
35    pub counter: Counter,
36}
37
38/// A source that filters and sorts. What it notices while reading is said through
39/// [`Pushdown::notes`], which may grow once a pass over the whole source is done.
40pub trait Pushdown: Send + Sync {
41    /// The view with `filters` and the sort `sort` (column, descending; nulls last and
42    /// ties in the source's order), or the source's order backward when `reversed` and
43    /// there is no sort. `None` when the source cannot run it, and Polars does.
44    fn view(
45        &self,
46        filters: &[FilterStatement],
47        sort: &[(String, bool)],
48        reversed: bool,
49    ) -> Option<PushedView>;
50
51    /// What reading the source has found to say.
52    fn notes(&self) -> Vec<crate::notes::Note>;
53
54    /// The source itself, for its tests.
55    fn as_any(&self) -> &dyn std::any::Any;
56}
57
58/// What of a predicate Polars pushed into an anonymous scan the scan can evaluate.
59///
60/// A sort with a limit (a top-k) pushes a dynamic predicate, its running bound, into
61/// the scan as an `Expr::Display`, which panics when it is turned back into a plan
62/// (pola-rs/polars#28629; #28643 strips it for Python IO plugins only). The bound is
63/// a hint the top-k applies itself, so the terms holding it are dropped.
64pub(crate) fn evaluable(predicate: Option<Expr>) -> Option<Expr> {
65    fn terms(e: Expr, out: &mut Vec<Expr>) {
66        match e {
67            Expr::BinaryExpr {
68                left,
69                op: Operator::And | Operator::LogicalAnd,
70                right,
71            } => {
72                terms(Arc::unwrap_or_clone(left), out);
73                terms(Arc::unwrap_or_clone(right), out);
74            }
75            e => out.push(e),
76        }
77    }
78    let mut all = Vec::new();
79    terms(predicate?, &mut all);
80    all.into_iter()
81        .filter(|term| !term.into_iter().any(|e| matches!(e, Expr::Display { .. })))
82        .reduce(|left, right| left.and(right))
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88    use polars::prelude::{col, lit};
89
90    fn bound() -> Expr {
91        Expr::Display {
92            inputs: vec![col("id")],
93            fmt_str: Box::new("dynamic_pred: 1".into()),
94        }
95    }
96
97    #[test]
98    fn a_top_k_bound_is_dropped_from_a_pushed_predicate() {
99        let kept = col("id").lt(lit(4));
100        assert_eq!(
101            evaluable(Some(kept.clone().and(bound()))),
102            Some(kept.clone())
103        );
104        assert_eq!(
105            evaluable(Some(bound().and(kept.clone()))),
106            Some(kept.clone())
107        );
108        assert_eq!(evaluable(Some(bound())), None);
109        assert_eq!(evaluable(Some(kept.clone())), Some(kept));
110        assert_eq!(evaluable(None), None);
111    }
112}
datui_lib/pushdown.rs

datui_lib/
pushdown.rs
