Skip to main content

rypipe_core/
plan.rs

1use arrow::datatypes::TimeUnit;
2use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
3
4/// The storage type for a column.
5#[derive(Clone, Debug, PartialEq, Eq)]
6pub enum FieldType {
7    String,
8    Int64,
9    Float64,
10    Boolean,
11    Dictionary,
12    /// Calendar date: days since the Unix epoch.
13    Date32,
14    /// Point in time with an explicit unit.
15    Timestamp(TimeUnit),
16}
17
18impl FieldType {
19    #[allow(clippy::should_implement_trait)]
20    pub fn from_str(s: &str) -> Option<Self> {
21        match s {
22            "string" => Some(FieldType::String),
23            "int64" => Some(FieldType::Int64),
24            "float64" => Some(FieldType::Float64),
25            "bool" | "boolean" => Some(FieldType::Boolean),
26            "dictionary" => Some(FieldType::Dictionary),
27            "date32" => Some(FieldType::Date32),
28            "timestamp" => Some(FieldType::Timestamp(TimeUnit::Microsecond)),
29            "timestamp[s]" => Some(FieldType::Timestamp(TimeUnit::Second)),
30            "timestamp[ms]" => Some(FieldType::Timestamp(TimeUnit::Millisecond)),
31            "timestamp[us]" | "timestamp[µs]" => Some(FieldType::Timestamp(TimeUnit::Microsecond)),
32            "timestamp[ns]" => Some(FieldType::Timestamp(TimeUnit::Nanosecond)),
33            _ => None,
34        }
35    }
36}
37
38/// A compiled execution plan that controls field renaming, dropping,
39/// type assignment, dictionary encoding, row filtering, and column ordering.
40/// Default (empty) is a no-op.
41#[derive(Clone, Debug, Default)]
42pub struct ExecutionPlan {
43    /// Map from raw field name to output column name.
44    pub field_map: HashMap<String, String>,
45    /// Set of raw field names to drop entirely.
46    pub drop_fields: HashSet<String>,
47    /// Explicit type overrides per output column name.
48    pub field_types: HashMap<String, FieldType>,
49    /// Set of output column names to dict-encode.
50    pub dictionary_columns: HashSet<String>,
51    /// Optional row filter predicate.
52    pub filter: Option<FilterPredicate>,
53    /// Desired output column order (names in order).  Columns not listed here
54    /// appear after all listed columns in first-appearance order.  If empty,
55    /// first-appearance order is used.
56    pub schema_order: Vec<String>,
57    /// When true, string columns with low cardinality are automatically
58    /// upgraded to dictionary encoding during parse.
59    pub auto_dict: bool,
60    /// Auto-dict tuning: maximum fraction of rows allowed as distinct values
61    /// for an upgrade. Defaults to `0.05` when `None`.
62    pub dict_threshold: Option<f64>,
63    /// Auto-dict tuning: maximum dictionary size (distinct values) allowed for
64    /// an upgrade. Defaults to `256` when `None`.
65    pub dict_max_size: Option<usize>,
66}
67
68impl ExecutionPlan {
69    pub fn new() -> Self {
70        Self::default()
71    }
72
73    /// Rename a raw field to an output column name.
74    pub fn rename(mut self, raw: impl Into<String>, output: impl Into<String>) -> Self {
75        self.field_map.insert(raw.into(), output.into());
76        self
77    }
78
79    /// Drop a single raw/output field.
80    pub fn drop(mut self, field: impl Into<String>) -> Self {
81        self.drop_fields.insert(field.into());
82        self
83    }
84
85    /// Drop many raw/output fields.
86    pub fn drop_many<I, S>(mut self, fields: I) -> Self
87    where
88        I: IntoIterator<Item = S>,
89        S: Into<String>,
90    {
91        self.drop_fields.extend(fields.into_iter().map(Into::into));
92        self
93    }
94
95    /// Set the storage type for an output column.
96    pub fn type_as(mut self, field: impl Into<String>, field_type: FieldType) -> Self {
97        self.field_types.insert(field.into(), field_type);
98        self
99    }
100
101    /// Dict-encode an output column.
102    pub fn dictionary(mut self, field: impl Into<String>) -> Self {
103        self.dictionary_columns.insert(field.into());
104        self
105    }
106
107    /// Keep only rows where `field == value` (string comparison, per-row).
108    pub fn filter_eq(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
109        self.filter = Some(FilterPredicate::Equal {
110            field: field.into(),
111            value: value.into(),
112        });
113        self
114    }
115
116    /// Keep only rows where `field != value` (string comparison, per-row).
117    pub fn filter_ne(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
118        self.filter = Some(FilterPredicate::NotEqual {
119            field: field.into(),
120            value: value.into(),
121        });
122        self
123    }
124
125    /// Keep only rows where `field_a op field_b` (native-typed, per-row).
126    pub fn filter_compare(
127        mut self,
128        field_a: impl Into<String>,
129        op: CompareOp,
130        field_b: impl Into<String>,
131    ) -> Self {
132        self.filter = Some(FilterPredicate::Compare {
133            field_a: field_a.into(),
134            op,
135            field_b: field_b.into(),
136        });
137        self
138    }
139
140    /// Set the output column order. Columns not listed appear after the listed
141    /// ones in first-appearance order.
142    pub fn schema_order<I, S>(mut self, order: I) -> Self
143    where
144        I: IntoIterator<Item = S>,
145        S: Into<String>,
146    {
147        self.schema_order = order.into_iter().map(Into::into).collect();
148        self
149    }
150
151    /// Enable or disable automatic dictionary upgrade for low-cardinality
152    /// string columns.
153    pub fn with_auto_dict(mut self, yes: bool) -> Self {
154        self.auto_dict = yes;
155        self
156    }
157
158    /// Override the auto-dict distinct-ratio threshold (default `0.05`).
159    pub fn with_dict_threshold(mut self, threshold: f64) -> Self {
160        self.dict_threshold = Some(threshold);
161        self
162    }
163
164    /// Override the auto-dict maximum dictionary size (default `256`).
165    pub fn with_dict_max_size(mut self, max_size: usize) -> Self {
166        self.dict_max_size = Some(max_size);
167        self
168    }
169
170    /// Determine the storage type for an output column name.
171    pub fn column_type(&self, name: &str) -> FieldType {
172        if let Some(ft) = self.field_types.get(name) {
173            return ft.clone();
174        }
175        if self.dictionary_columns.contains(name) {
176            return FieldType::Dictionary;
177        }
178        FieldType::String
179    }
180
181    /// Resolve a raw field name to its output column name.
182    /// Returns `None` if the field should be dropped.
183    ///
184    /// Application order: rename first, then drop, matching left-to-right
185    /// pipeline semantics.
186    pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str> {
187        let resolved = self.field_map.get(raw).map_or(raw, |s| s.as_str());
188        if self.drop_fields.contains(resolved) {
189            return None;
190        }
191        Some(resolved)
192    }
193}
194
195/// Comparison operator for column-to-column filters (evaluated post-reduce).
196#[derive(Clone, Copy, Debug, PartialEq, Eq)]
197pub enum CompareOp {
198    Gt,
199    Lt,
200    Ge,
201    Le,
202    Eq,
203    Ne,
204}
205
206impl CompareOp {
207    #[allow(clippy::should_implement_trait)]
208    pub fn from_str(s: &str) -> Option<Self> {
209        match s {
210            ">" | "gt" => Some(CompareOp::Gt),
211            "<" | "lt" => Some(CompareOp::Lt),
212            ">=" | "ge" => Some(CompareOp::Ge),
213            "<=" | "le" => Some(CompareOp::Le),
214            "==" | "eq" => Some(CompareOp::Eq),
215            "!=" | "ne" => Some(CompareOp::Ne),
216            _ => None,
217        }
218    }
219}
220
221/// A filter predicate evaluated per-row during parsing.
222///
223// Leaf predicates (`Equal`, `NotEqual`, `Compare`) can be composed into a
224// tree with `And`, `Or`, and `Not`. Evaluation is recursive with
225// short-circuiting; a row is kept only if the whole tree passes.
226#[derive(Clone, Debug, PartialEq)]
227pub enum FilterPredicate {
228    /// Keep row if `field_value != value` (string comparison, per-row).
229    NotEqual { field: String, value: String },
230    /// Keep row if `field_value == value` (string comparison, per-row).
231    Equal { field: String, value: String },
232    /// Column-to-column comparison, evaluated natively per-row during
233    /// parsing with numeric promotion (Int64 vs Float64 widens).
234    Compare {
235        field_a: String,
236        op: CompareOp,
237        field_b: String,
238    },
239    /// Keep row if both sub-predicates pass. Short-circuits on the first
240    /// failure.
241    And(Box<FilterPredicate>, Box<FilterPredicate>),
242    /// Keep row if either sub-predicate passes. Short-circuits on the first
243    /// success.
244    Or(Box<FilterPredicate>, Box<FilterPredicate>),
245    /// Keep row if the sub-predicate fails.
246    Not(Box<FilterPredicate>),
247}
248
249impl FilterPredicate {
250    /// Combine two predicates with logical AND.
251    pub fn all(a: FilterPredicate, b: FilterPredicate) -> Self {
252        FilterPredicate::And(Box::new(a), Box::new(b))
253    }
254
255    /// Combine two predicates with logical OR.
256    pub fn any(a: FilterPredicate, b: FilterPredicate) -> Self {
257        FilterPredicate::Or(Box::new(a), Box::new(b))
258    }
259
260    /// Negate a predicate.
261    #[allow(clippy::should_implement_trait)]
262    pub fn not(inner: FilterPredicate) -> Self {
263        FilterPredicate::Not(Box::new(inner))
264    }
265}
266
267impl FilterPredicate {
268    /// Check whether a partial row passes the filter.
269    /// `columns`/`field_index` contain all builders; `row_index` is the current
270    /// row number. Returns true to keep the row.
271    ///
272    /// * `Equal`/`NotEqual` use string comparison on the stored value.
273    /// * `Compare` is evaluated natively per-row against typed values with
274    ///   numeric promotion (`Int64` vs `Float64` widens to f64); any other
275    ///   type mismatch or a null operand fails the row.
276    /// * `And`/`Or` short-circuit; `Not` negates. A missing field fails an
277    ///   inner leaf, which `Not` can flip back to a keep.
278    pub(crate) fn check(
279        &self,
280        columns: &[crate::columnar::ColumnBuilder],
281        field_index: &HashMap<String, usize>,
282        row_index: usize,
283        plan: &ExecutionPlan,
284    ) -> bool {
285        match self {
286            FilterPredicate::NotEqual { field, value } => {
287                let actual = get_value(columns, field_index, field, plan, row_index);
288                actual.as_deref() != Some(value.as_str())
289            }
290            FilterPredicate::Equal { field, value } => {
291                let actual = get_value(columns, field_index, field, plan, row_index);
292                actual.as_deref() == Some(value.as_str())
293            }
294            FilterPredicate::Compare {
295                field_a,
296                op,
297                field_b,
298            } => {
299                let va = get_column(columns, field_index, resolve(field_a, plan))
300                    .and_then(|b| b.get_typed_value(row_index));
301                let vb = get_column(columns, field_index, resolve(field_b, plan))
302                    .and_then(|b| b.get_typed_value(row_index));
303                match (va, vb) {
304                    (Some(a), Some(b)) => compare_typed(&a, *op, &b),
305                    _ => false,
306                }
307            }
308            FilterPredicate::And(a, b) => {
309                // Evaluate the operand with the earlier field first for
310                // short-circuit benefit (C2: reorder by document position).
311                let (first, second) =
312                    if pred_ordinal(a, field_index, plan) <= pred_ordinal(b, field_index, plan) {
313                        (a.as_ref(), b.as_ref())
314                    } else {
315                        (b.as_ref(), a.as_ref())
316                    };
317                first.check(columns, field_index, row_index, plan)
318                    && second.check(columns, field_index, row_index, plan)
319            }
320            FilterPredicate::Or(a, b) => {
321                let (first, second) =
322                    if pred_ordinal(a, field_index, plan) <= pred_ordinal(b, field_index, plan) {
323                        (a.as_ref(), b.as_ref())
324                    } else {
325                        (b.as_ref(), a.as_ref())
326                    };
327                first.check(columns, field_index, row_index, plan)
328                    || second.check(columns, field_index, row_index, plan)
329            }
330            FilterPredicate::Not(inner) => !inner.check(columns, field_index, row_index, plan),
331        }
332    }
333}
334
335/// Resolve a filter field name to its output column name.
336fn resolve<'a>(field: &'a str, plan: &'a ExecutionPlan) -> &'a str {
337    plan.field_map.get(field).map_or(field, |s| s.as_str())
338}
339
340fn get_column<'a>(
341    columns: &'a [crate::columnar::ColumnBuilder],
342    field_index: &HashMap<String, usize>,
343    name: &str,
344) -> Option<&'a crate::columnar::ColumnBuilder> {
345    field_index.get(name).map(|&i| &columns[i])
346}
347
348/// Fetch a stored value formatted as a string for Equal/NotEqual checks.
349/// Tries zero-allocation `get_filter_view` first (String/Dict columns);
350/// falls back to `get_filter_value` for typed columns that need formatting.
351fn get_value(
352    columns: &[crate::columnar::ColumnBuilder],
353    field_index: &HashMap<String, usize>,
354    field: &str,
355    plan: &ExecutionPlan,
356    row_index: usize,
357) -> Option<String> {
358    let col = get_column(columns, field_index, resolve(field, plan))?;
359    // Fast path: borrowed &str for String/Dictionary columns (no allocation).
360    if let Some(s) = col.get_filter_view(row_index) {
361        return Some(s.to_owned());
362    }
363    // Slow path: typed columns need formatting (allocates).
364    col.get_filter_value(row_index)
365}
366
367/// Native-typed comparison with numeric promotion. Mixed Int64/Float64
368/// operands are widened to f64; any other type mismatch yields false.
369/// Nulls never reach here (checked by the caller).
370fn compare_typed(
371    a: &crate::columnar::TypedValue<'_>,
372    op: CompareOp,
373    b: &crate::columnar::TypedValue<'_>,
374) -> bool {
375    use crate::columnar::TypedValue as T;
376    // Numeric promotion: Int64 <-> Float64.
377    let num_pair: Option<(f64, f64)> = match (a, b) {
378        (T::Int64(x), T::Int64(y)) => Some((*x as f64, *y as f64)),
379        (T::Float64(x), T::Float64(y)) => Some((*x, *y)),
380        (T::Int64(x), T::Float64(y)) | (T::Float64(y), T::Int64(x)) => Some((*x as f64, *y)),
381        _ => None,
382    };
383    if let Some((x, y)) = num_pair {
384        return apply_op(op, x.partial_cmp(&y));
385    }
386    match (a, b) {
387        (T::Str(x), T::Str(y)) => apply_op(op, x.cmp(y).into()),
388        (T::Bool(x), T::Bool(y)) => apply_op(op, x.cmp(y).into()),
389        // Temporal types compare by their raw integer; mixed Date32/Timestamp
390        // or differing timestamp units are not comparable.
391        (T::Date32(x), T::Date32(y)) => apply_op(op, x.cmp(y).into()),
392        (T::Timestamp(x), T::Timestamp(y)) => apply_op(op, x.cmp(y).into()),
393        _ => false,
394    }
395}
396
397/// Apply an operator to an `Ordering`, treating `None` (NaN) as no-match.
398fn apply_op(op: CompareOp, ord: Option<std::cmp::Ordering>) -> bool {
399    use std::cmp::Ordering::*;
400    let Some(ord) = ord else { return false };
401    match op {
402        CompareOp::Gt => ord == Greater,
403        CompareOp::Lt => ord == Less,
404        CompareOp::Ge => ord != Less,
405        CompareOp::Le => ord != Greater,
406        CompareOp::Eq => ord == Equal,
407        CompareOp::Ne => ord != Equal,
408    }
409}
410
411/// Return the minimum field ordinal for a predicate, used to order
412/// `And`/`Or` operands by document position (C2). Higher ordinal = later
413/// in the document. Returns `usize::MAX` for predicates without a
414/// resolvable field (should evaluate last).
415fn pred_ordinal(
416    pred: &FilterPredicate,
417    field_index: &HashMap<String, usize>,
418    plan: &ExecutionPlan,
419) -> usize {
420    match pred {
421        FilterPredicate::Equal { field, .. } | FilterPredicate::NotEqual { field, .. } => {
422            field_index
423                .get(resolve(field, plan))
424                .copied()
425                .unwrap_or(usize::MAX)
426        }
427        FilterPredicate::Compare {
428            field_a, field_b, ..
429        } => {
430            let a = field_index
431                .get(resolve(field_a, plan))
432                .copied()
433                .unwrap_or(usize::MAX);
434            let b = field_index
435                .get(resolve(field_b, plan))
436                .copied()
437                .unwrap_or(usize::MAX);
438            a.min(b)
439        }
440        FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
441            pred_ordinal(a, field_index, plan).min(pred_ordinal(b, field_index, plan))
442        }
443        FilterPredicate::Not(inner) => pred_ordinal(inner, field_index, plan),
444    }
445}