Skip to main content

rypipe_core/
plan.rs

1use arrow::datatypes::TimeUnit;
2use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
3
4/// The storage type for a column.
5#[derive(Clone, Debug, PartialEq, Eq)]
6pub enum FieldType {
7    String,
8    Int64,
9    Float64,
10    Boolean,
11    Dictionary,
12    /// Calendar date: days since the Unix epoch.
13    Date32,
14    /// Point in time with an explicit unit and an optional chrono format
15    /// string for parsing non-ISO input (e.g. `timestamp[ms,format=%Y%m%d]`).
16    Timestamp(TimeUnit, Option<Box<str>>),
17    /// Fixed-precision decimal with given scale (default 18).
18    Decimal128(u8),
19}
20
21impl std::str::FromStr for FieldType {
22    type Err = ();
23    fn from_str(s: &str) -> Result<Self, Self::Err> {
24        match s {
25            "string" => Ok(FieldType::String),
26            "int64" => Ok(FieldType::Int64),
27            "float64" => Ok(FieldType::Float64),
28            "bool" | "boolean" => Ok(FieldType::Boolean),
29            "dictionary" => Ok(FieldType::Dictionary),
30            "date32" => Ok(FieldType::Date32),
31            s if s.starts_with("timestamp") => parse_timestamp_spec(s).ok_or(()),
32            "decimal128" => Ok(FieldType::Decimal128(18)),
33            s if s.starts_with("decimal128(") => {
34                let scale = s
35                    .trim_start_matches("decimal128(")
36                    .trim_end_matches(')')
37                    .parse::<u8>()
38                    .map_err(|_| ())?;
39                Ok(FieldType::Decimal128(scale))
40            }
41            _ => Err(()),
42        }
43    }
44}
45
46/// Parse a timestamp type spec: `timestamp`, `timestamp[unit]`, or
47/// `timestamp[unit,format=…]` (unit defaults to `us`; format is a chrono
48/// format string tried before the built-in ISO layouts).
49fn parse_timestamp_spec(s: &str) -> Option<FieldType> {
50    let rest = s.strip_prefix("timestamp")?;
51    if rest.is_empty() {
52        return Some(FieldType::Timestamp(TimeUnit::Microsecond, None));
53    }
54    let inner = rest.strip_prefix('[')?.strip_suffix(']')?;
55    let (unit_str, format) = match inner.split_once(',') {
56        Some((unit, opt)) => {
57            let fmt = opt.strip_prefix("format=")?;
58            if fmt.is_empty() {
59                return None;
60            }
61            (unit, Some(fmt))
62        }
63        None => {
64            if let Some(fmt) = inner.strip_prefix("format=") {
65                if fmt.is_empty() {
66                    return None;
67                }
68                ("us", Some(fmt))
69            } else {
70                (inner, None)
71            }
72        }
73    };
74    let unit = match unit_str {
75        "" | "us" | "µs" => TimeUnit::Microsecond,
76        "s" => TimeUnit::Second,
77        "ms" => TimeUnit::Millisecond,
78        "ns" => TimeUnit::Nanosecond,
79        _ => return None,
80    };
81    let format: Option<Box<str>> =
82        format.and_then(|f| if f.is_empty() { None } else { Some(f.into()) });
83    Some(FieldType::Timestamp(unit, format))
84}
85
86/// A compiled execution plan that controls field renaming, dropping,
87/// type assignment, dictionary encoding, row filtering, and column ordering.
88/// Default (empty) is a no-op.
89#[derive(Clone, Default)]
90pub struct ExecutionPlan {
91    /// Map from raw field name to output column name.
92    pub field_map: HashMap<String, String>,
93    /// Set of raw field names to drop entirely.
94    pub drop_fields: HashSet<String>,
95    /// Explicit type overrides per output column name.
96    pub field_types: HashMap<String, FieldType>,
97    /// Set of output column names to dict-encode.
98    pub dictionary_columns: HashSet<String>,
99    /// Optional row filter predicate.
100    pub filter: Option<FilterPredicate>,
101    /// Desired output column order (names in order).  Columns not listed here
102    /// appear after all listed columns in first-appearance order.  If empty,
103    /// first-appearance order is used.
104    pub schema_order: Vec<String>,
105    /// When true, string columns with low cardinality are automatically
106    /// upgraded to dictionary encoding during parse.
107    pub auto_dict: bool,
108    /// Auto-dict tuning: maximum fraction of rows allowed as distinct values
109    /// for an upgrade. Defaults to `0.05` when `None`.
110    pub dict_threshold: Option<f64>,
111    /// Auto-dict tuning: maximum dictionary size (distinct values) allowed for
112    /// an upgrade. Defaults to `256` when `None`.
113    pub dict_max_size: Option<usize>,
114    /// When true, a non-null value that fails to parse into its declared
115    /// `field_types` type aborts the parse with an error instead of silently
116    /// becoming null. Genuine nulls (missing fields) stay null regardless.
117    pub strict_types: bool,
118    /// Cap on the number of bounded-streaming batches/chunks. Defaults to
119    /// `MAX_SPLIT_CHUNKS` (100,000) when `None`.
120    pub max_split_chunks: Option<usize>,
121    /// Optional row observer; hooks fire from parse threads during the read.
122    pub observer: Option<std::sync::Arc<dyn crate::RowObserver>>,
123}
124
125impl std::fmt::Debug for ExecutionPlan {
126    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
127        f.debug_struct("ExecutionPlan")
128            .field("field_map", &self.field_map)
129            .field("drop_fields", &self.drop_fields)
130            .field("field_types", &self.field_types)
131            .field("dictionary_columns", &self.dictionary_columns)
132            .field("filter", &self.filter)
133            .field("schema_order", &self.schema_order)
134            .field("auto_dict", &self.auto_dict)
135            .field("dict_threshold", &self.dict_threshold)
136            .field("dict_max_size", &self.dict_max_size)
137            .field("strict_types", &self.strict_types)
138            .field("max_split_chunks", &self.max_split_chunks)
139            .field("observer", &self.observer.is_some())
140            .finish()
141    }
142}
143
144impl ExecutionPlan {
145    pub fn new() -> Self {
146        Self::default()
147    }
148
149    /// Rename a raw field to an output column name.
150    pub fn rename(mut self, raw: impl Into<String>, output: impl Into<String>) -> Self {
151        self.field_map.insert(raw.into(), output.into());
152        self
153    }
154
155    /// Drop a single raw/output field.
156    pub fn drop(mut self, field: impl Into<String>) -> Self {
157        self.drop_fields.insert(field.into());
158        self
159    }
160
161    /// Drop many raw/output fields.
162    pub fn drop_many<I, S>(mut self, fields: I) -> Self
163    where
164        I: IntoIterator<Item = S>,
165        S: Into<String>,
166    {
167        self.drop_fields.extend(fields.into_iter().map(Into::into));
168        self
169    }
170
171    /// Set the storage type for an output column.
172    pub fn type_as(mut self, field: impl Into<String>, field_type: FieldType) -> Self {
173        self.field_types.insert(field.into(), field_type);
174        self
175    }
176
177    /// Dict-encode an output column.
178    pub fn dictionary(mut self, field: impl Into<String>) -> Self {
179        self.dictionary_columns.insert(field.into());
180        self
181    }
182
183    /// Keep only rows where `field == value` (string comparison, per-row).
184    pub fn filter_eq(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
185        self.filter = Some(FilterPredicate::Equal {
186            field: field.into(),
187            value: value.into(),
188        });
189        self
190    }
191
192    /// Keep only rows where `field != value` (string comparison, per-row).
193    pub fn filter_ne(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
194        self.filter = Some(FilterPredicate::NotEqual {
195            field: field.into(),
196            value: value.into(),
197        });
198        self
199    }
200
201    /// Keep only rows where `field_a op field_b` (native-typed, per-row).
202    pub fn filter_compare(
203        mut self,
204        field_a: impl Into<String>,
205        op: CompareOp,
206        field_b: impl Into<String>,
207    ) -> Self {
208        self.filter = Some(FilterPredicate::Compare {
209            field_a: field_a.into(),
210            op,
211            field_b: field_b.into(),
212        });
213        self
214    }
215
216    /// Declare the output schema as a **projection + order**: the output
217    /// contains exactly the listed columns, in the listed order.  Fields in
218    /// the data but not listed are skipped during parsing (never
219    /// materialized); listed columns absent from the data come out
220    /// null-filled.  `field_map` renames apply before matching.
221    pub fn schema_order<I, S>(mut self, order: I) -> Self
222    where
223        I: IntoIterator<Item = S>,
224        S: Into<String>,
225    {
226        self.schema_order = order.into_iter().map(Into::into).collect();
227        self
228    }
229
230    /// Enable or disable automatic dictionary upgrade for low-cardinality
231    /// string columns.
232    pub fn with_auto_dict(mut self, yes: bool) -> Self {
233        self.auto_dict = yes;
234        self
235    }
236
237    /// Override the auto-dict distinct-ratio threshold (default `0.05`).
238    pub fn with_dict_threshold(mut self, threshold: f64) -> Self {
239        self.dict_threshold = Some(threshold);
240        self
241    }
242
243    /// Override the auto-dict maximum dictionary size (default `256`).
244    pub fn with_dict_max_size(mut self, max_size: usize) -> Self {
245        self.dict_max_size = Some(max_size);
246        self
247    }
248
249    /// Enable strict typing: a non-null value that cannot be parsed into its
250    /// `field_types`-declared type becomes an error at `finish()` instead of
251    /// a silent null.
252    pub fn with_strict_types(mut self, yes: bool) -> Self {
253        self.strict_types = yes;
254        self
255    }
256
257    /// Override the bounded-streaming split cap (default 100,000 chunks).
258    pub fn with_max_split_chunks(mut self, cap: usize) -> Self {
259        self.max_split_chunks = Some(cap);
260        self
261    }
262
263    /// Attach a row observer; hooks fire from parse threads during the read.
264    pub fn with_observer(mut self, observer: std::sync::Arc<dyn crate::RowObserver>) -> Self {
265        self.observer = Some(observer);
266        self
267    }
268
269    /// Determine the storage type for an output column name.
270    pub fn column_type(&self, name: &str) -> FieldType {
271        if let Some(ft) = self.field_types.get(name) {
272            return ft.clone();
273        }
274        if self.dictionary_columns.contains(name) {
275            return FieldType::Dictionary;
276        }
277        FieldType::String
278    }
279
280    /// Resolve a raw field name to its output column name.
281    /// Returns `None` if the field should be dropped.
282    ///
283    /// Application order: rename first, then drop, then schema projection,
284    /// matching left-to-right pipeline semantics.
285    ///
286    /// When `schema_order` is non-empty it acts as a **projection**: only the
287    /// listed columns are materialized, so any other resolved name returns
288    /// `None` here (which makes `ColumnarSink::wants`/`resolve` reject the
289    /// field and lets adapters skip extracting it).  Fields referenced by the
290    /// row filter are kept even when absent from `schema_order` so predicates
291    /// still evaluate correctly; they are projected out of the final batch.
292    pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str> {
293        let resolved = self.field_map.get(raw).map_or(raw, |s| s.as_str());
294        if self.drop_fields.contains(resolved) {
295            return None;
296        }
297        if !self.schema_order.is_empty()
298            && !self.schema_order.iter().any(|n| n == resolved)
299            && !self.filter_references(resolved)
300        {
301            return None;
302        }
303        Some(resolved)
304    }
305
306    /// Whether the row filter references `resolved` (a post-rename output
307    /// name).  Used by [`Self::resolve_field`] to keep predicate fields out
308    /// of the schema projection.
309    fn filter_references(&self, resolved: &str) -> bool {
310        fn matches(pred: &FilterPredicate, plan: &ExecutionPlan, resolved: &str) -> bool {
311            // Rename only: drop/projection must not apply here (recursion).
312            fn ren<'x>(plan: &'x ExecutionPlan, f: &'x String) -> &'x str {
313                plan.field_map.get(f).map_or(f.as_str(), |s| s.as_str())
314            }
315            match pred {
316                FilterPredicate::Equal { field, .. }
317                | FilterPredicate::NotEqual { field, .. }
318                | FilterPredicate::CompareLiteral { field, .. }
319                | FilterPredicate::StartsWith { field, .. }
320                | FilterPredicate::EndsWith { field, .. }
321                | FilterPredicate::In { field, .. }
322                | FilterPredicate::NotIn { field, .. }
323                | FilterPredicate::NotField { field, .. }
324                | FilterPredicate::ArithmeticCompare { field, .. }
325                | FilterPredicate::Strip { field, .. }
326                | FilterPredicate::Lower { field, .. }
327                | FilterPredicate::Upper { field, .. }
328                | FilterPredicate::Replace { field, .. }
329                | FilterPredicate::Length { field, .. }
330                | FilterPredicate::Contains { field, .. }
331                | FilterPredicate::IsNull { field, .. }
332                | FilterPredicate::IsType { field, .. }
333                | FilterPredicate::Regex { field, .. } => ren(plan, field) == resolved,
334                FilterPredicate::Always(_) => false,
335                FilterPredicate::Compare {
336                    field_a, field_b, ..
337                } => ren(plan, field_a) == resolved || ren(plan, field_b) == resolved,
338                FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
339                    matches(a, plan, resolved) || matches(b, plan, resolved)
340                }
341                FilterPredicate::Not(inner) => matches(inner, plan, resolved),
342            }
343        }
344        self.filter
345            .as_ref()
346            .is_some_and(|f| matches(f, self, resolved))
347    }
348}
349
350/// Comparison operator for column-to-column filters (evaluated post-reduce).
351#[derive(Clone, Copy, Debug, PartialEq, Eq)]
352pub enum CompareOp {
353    Gt,
354    Lt,
355    Ge,
356    Le,
357    Eq,
358    Ne,
359}
360
361/// Arithmetic operator for field expressions.
362#[derive(Clone, Copy, Debug, PartialEq, Eq)]
363pub enum ArithOp {
364    Add,
365    Sub,
366    Mul,
367    Div,
368}
369
370impl std::str::FromStr for ArithOp {
371    type Err = ();
372    fn from_str(s: &str) -> Result<Self, Self::Err> {
373        match s {
374            "+" | "add" => Ok(ArithOp::Add),
375            "-" | "sub" => Ok(ArithOp::Sub),
376            "*" | "mul" => Ok(ArithOp::Mul),
377            "/" | "div" => Ok(ArithOp::Div),
378            _ => Err(()),
379        }
380    }
381}
382
383impl std::str::FromStr for CompareOp {
384    type Err = ();
385    fn from_str(s: &str) -> Result<Self, Self::Err> {
386        match s {
387            ">" | "gt" => Ok(CompareOp::Gt),
388            "<" | "lt" => Ok(CompareOp::Lt),
389            ">=" | "ge" => Ok(CompareOp::Ge),
390            "<=" | "le" => Ok(CompareOp::Le),
391            "==" | "eq" => Ok(CompareOp::Eq),
392            "!=" | "ne" => Ok(CompareOp::Ne),
393            _ => Err(()),
394        }
395    }
396}
397
398/// A filter predicate evaluated per-row during parsing.
399///
400// Leaf predicates (`Equal`, `NotEqual`, `Compare`, `CompareLiteral`) can be
401// composed into a tree with `And`, `Or`, and `Not`. Evaluation is recursive
402// with short-circuiting; a row is kept only if the whole tree passes.
403#[derive(Clone, Debug, PartialEq)]
404pub enum FilterPredicate {
405    /// Keep row if `field_value != value` (string comparison, per-row).
406    NotEqual { field: String, value: String },
407    /// Keep row if `field_value == value` (string comparison, per-row).
408    Equal { field: String, value: String },
409    /// Column-to-column comparison, evaluated natively per-row during
410    /// parsing with numeric promotion (Int64 vs Float64 widens).
411    Compare {
412        field_a: String,
413        op: CompareOp,
414        field_b: String,
415    },
416    /// Column-to-literal comparison with ordering. Uses typed comparison
417    /// with numeric promotion (e.g. Int64 field > Float64 literal).
418    CompareLiteral {
419        field: String,
420        op: CompareOp,
421        value: String,
422    },
423    /// Keep row if `field_value.starts_with(value)`.
424    StartsWith { field: String, value: String },
425    /// Keep row if `field_value.ends_with(value)`.
426    EndsWith { field: String, value: String },
427    /// Keep row if `field_value` is in the given set.
428    In { field: String, values: Vec<String> },
429    /// Keep row if `field_value` is not in the given set.
430    NotIn { field: String, values: Vec<String> },
431    /// Always keep or reject the row.
432    Always(bool),
433    /// Keep row if the field is falsy (None, empty string, or missing).
434    NotField { field: String },
435    /// Arithmetic compare: field <arith_op> <arith_value> <cmp_op> <cmp_value>
436    ArithmeticCompare {
437        field: String,
438        arith_op: ArithOp,
439        arith_value: f64,
440        cmp_op: CompareOp,
441        cmp_value: String,
442    },
443    /// Keep row if `field_value.strip()` satisfies the comparison.
444    Strip {
445        field: String,
446        op: CompareOp,
447        value: String,
448    },
449    /// Keep row if `field_value.to_lowercase()` satisfies the comparison.
450    Lower {
451        field: String,
452        op: CompareOp,
453        value: String,
454    },
455    /// Keep row if `field_value.to_uppercase()` satisfies the comparison.
456    Upper {
457        field: String,
458        op: CompareOp,
459        value: String,
460    },
461    /// Keep row if `field_value.replace(old, new)` satisfies the comparison.
462    Replace {
463        field: String,
464        old: String,
465        new: String,
466        op: CompareOp,
467        value: String,
468    },
469    /// Keep row if `field_value.len()` satisfies the comparison (numeric).
470    Length {
471        field: String,
472        op: CompareOp,
473        value: String,
474    },
475    /// Keep row if `field_value.contains(value)`.
476    Contains { field: String, value: String },
477    /// Keep row if the field is null or missing.
478    IsNull { field: String },
479    /// Keep row if the field is of the given type (e.g., Int64, Float64, String).
480    IsType {
481        field: String,
482        field_type: FieldType,
483    },
484    /// Keep row if `field_value` matches the regular expression.
485    Regex { field: String, re: RegexSpec },
486    /// Keep row if both sub-predicates pass. Short-circuits on the first
487    /// failure.
488    And(Box<FilterPredicate>, Box<FilterPredicate>),
489    /// Keep row if either sub-predicate passes. Short-circuits on the first
490    /// success.
491    Or(Box<FilterPredicate>, Box<FilterPredicate>),
492    /// Keep row if the sub-predicate fails.
493    Not(Box<FilterPredicate>),
494}
495
496/// A compiled regex plus its pattern. Equality and debug output use the
497/// pattern string; the compiled form is the execution representation.
498#[derive(Clone)]
499pub struct RegexSpec {
500    pub pattern: String,
501    pub compiled: regex::Regex,
502}
503
504impl RegexSpec {
505    pub fn new(pattern: impl Into<String>) -> std::result::Result<Self, regex::Error> {
506        let pattern = pattern.into();
507        let compiled = regex::Regex::new(&pattern)?;
508        Ok(Self { pattern, compiled })
509    }
510}
511
512impl std::fmt::Debug for RegexSpec {
513    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
514        f.debug_struct("RegexSpec")
515            .field("pattern", &self.pattern)
516            .finish()
517    }
518}
519
520impl PartialEq for RegexSpec {
521    fn eq(&self, other: &Self) -> bool {
522        self.pattern == other.pattern
523    }
524}
525
526impl FilterPredicate {
527    /// Combine two predicates with logical AND.
528    pub fn all(a: FilterPredicate, b: FilterPredicate) -> Self {
529        FilterPredicate::And(Box::new(a), Box::new(b))
530    }
531
532    /// Combine two predicates with logical OR.
533    pub fn any(a: FilterPredicate, b: FilterPredicate) -> Self {
534        FilterPredicate::Or(Box::new(a), Box::new(b))
535    }
536
537    /// Negate a predicate.
538    #[allow(clippy::should_implement_trait)]
539    pub fn not(inner: FilterPredicate) -> Self {
540        FilterPredicate::Not(Box::new(inner))
541    }
542}
543
544impl FilterPredicate {
545    /// Check whether a partial row passes the filter.
546    /// `columns`/`field_index` contain all builders; `row_index` is the current
547    /// row number. Returns true to keep the row.
548    ///
549    /// * `Equal`/`NotEqual` use string comparison on the stored value.
550    /// * `Compare` is evaluated natively per-row against typed values with
551    ///   numeric promotion (`Int64` vs `Float64` widens to f64); any other
552    ///   type mismatch or a null operand fails the row.
553    /// * `And`/`Or` short-circuit; `Not` negates. A missing field fails an
554    ///   inner leaf, which `Not` can flip back to a keep.
555    pub(crate) fn check(
556        &self,
557        columns: &[crate::columnar::ColumnBuilder],
558        field_index: &HashMap<String, usize>,
559        row_index: usize,
560        plan: &ExecutionPlan,
561    ) -> bool {
562        match self {
563            FilterPredicate::NotEqual { field, value } => {
564                let actual = get_value(columns, field_index, field, plan, row_index);
565                actual.as_deref() != Some(value.as_str())
566            }
567            FilterPredicate::Equal { field, value } => {
568                let actual = get_value(columns, field_index, field, plan, row_index);
569                actual.as_deref() == Some(value.as_str())
570            }
571            FilterPredicate::Compare {
572                field_a,
573                op,
574                field_b,
575            } => {
576                let va = get_column(columns, field_index, resolve(field_a, plan))
577                    .and_then(|b| b.get_typed_value(row_index));
578                let vb = get_column(columns, field_index, resolve(field_b, plan))
579                    .and_then(|b| b.get_typed_value(row_index));
580                match (va, vb) {
581                    (Some(a), Some(b)) => compare_typed(&a, *op, &b),
582                    _ => false,
583                }
584            }
585            FilterPredicate::CompareLiteral { field, op, value } => {
586                let va = get_column(columns, field_index, resolve(field, plan))
587                    .and_then(|b| b.get_typed_value(row_index));
588                if let Some(a) = va {
589                    if let Some(b) = typed_from_literal(&a, value) {
590                        return compare_typed(&a, *op, &b);
591                    }
592                }
593                false
594            }
595            FilterPredicate::StartsWith { field, value } => {
596                let actual = get_value(columns, field_index, field, plan, row_index);
597                match actual {
598                    Some(s) => s.starts_with(value.as_str()),
599                    None => false,
600                }
601            }
602            FilterPredicate::EndsWith { field, value } => {
603                let actual = get_value(columns, field_index, field, plan, row_index);
604                match actual {
605                    Some(s) => s.ends_with(value.as_str()),
606                    None => false,
607                }
608            }
609            FilterPredicate::In { field, values } => {
610                let actual = get_value(columns, field_index, field, plan, row_index);
611                match actual {
612                    Some(s) => values.contains(&s),
613                    None => false,
614                }
615            }
616            FilterPredicate::NotIn { field, values } => {
617                let actual = get_value(columns, field_index, field, plan, row_index);
618                match actual {
619                    Some(s) => !values.contains(&s),
620                    None => true,
621                }
622            }
623            FilterPredicate::Always(keep) => *keep,
624            FilterPredicate::NotField { field } => {
625                let actual = get_value(columns, field_index, field, plan, row_index);
626                match actual {
627                    Some(s) => s.is_empty(),
628                    None => true,
629                }
630            }
631            FilterPredicate::ArithmeticCompare {
632                field,
633                arith_op,
634                arith_value,
635                cmp_op,
636                cmp_value,
637            } => {
638                let actual = get_column(columns, field_index, resolve(field, plan))
639                    .and_then(|b| b.get_typed_value(row_index));
640                if let Some(a) = actual {
641                    let field_f64 = match &a {
642                        crate::columnar::TypedValue::Int64(v) => *v as f64,
643                        crate::columnar::TypedValue::Float64(v) => *v,
644                        crate::columnar::TypedValue::Str(s) => s.parse::<f64>().unwrap_or(0.0),
645                        _ => return false,
646                    };
647                    let result = match arith_op {
648                        ArithOp::Add => field_f64 + arith_value,
649                        ArithOp::Sub => field_f64 - arith_value,
650                        ArithOp::Mul => field_f64 * arith_value,
651                        ArithOp::Div => field_f64 / arith_value,
652                    };
653                    let cmp_f64 = cmp_value.parse::<f64>().unwrap_or(0.0);
654                    return apply_op(*cmp_op, result.partial_cmp(&cmp_f64));
655                }
656                false
657            }
658            FilterPredicate::Strip { field, op, value } => {
659                let actual = get_value(columns, field_index, field, plan, row_index);
660                match actual {
661                    Some(s) => {
662                        let transformed = s.trim().to_string();
663                        apply_op(*op, transformed.as_str().partial_cmp(value.as_str()))
664                    }
665                    None => false,
666                }
667            }
668            FilterPredicate::Lower { field, op, value } => {
669                let actual = get_value(columns, field_index, field, plan, row_index);
670                match actual {
671                    Some(s) => {
672                        let transformed = s.to_lowercase();
673                        apply_op(*op, transformed.as_str().partial_cmp(value.as_str()))
674                    }
675                    None => false,
676                }
677            }
678            FilterPredicate::Upper { field, op, value } => {
679                let actual = get_value(columns, field_index, field, plan, row_index);
680                match actual {
681                    Some(s) => {
682                        let transformed = s.to_uppercase();
683                        apply_op(*op, transformed.as_str().partial_cmp(value.as_str()))
684                    }
685                    None => false,
686                }
687            }
688            FilterPredicate::Replace {
689                field,
690                old,
691                new,
692                op,
693                value,
694            } => {
695                let actual = get_value(columns, field_index, field, plan, row_index);
696                match actual {
697                    Some(s) => {
698                        let transformed = s.replace(old.as_str(), new.as_str());
699                        apply_op(*op, transformed.as_str().partial_cmp(value.as_str()))
700                    }
701                    None => false,
702                }
703            }
704            FilterPredicate::Length { field, op, value } => {
705                let actual = get_value(columns, field_index, field, plan, row_index);
706                match actual {
707                    Some(s) => {
708                        let len = s.len() as f64;
709                        let cmp_val = value.parse::<f64>().unwrap_or(0.0);
710                        apply_op(*op, len.partial_cmp(&cmp_val))
711                    }
712                    None => false,
713                }
714            }
715            FilterPredicate::Contains { field, value } => {
716                let actual = get_value(columns, field_index, field, plan, row_index);
717                match actual {
718                    Some(s) => s.contains(value.as_str()),
719                    None => false,
720                }
721            }
722            FilterPredicate::IsNull { field } => {
723                let actual = get_value(columns, field_index, field, plan, row_index);
724                actual.is_none()
725            }
726            FilterPredicate::Regex { field, re } => {
727                let actual = get_value(columns, field_index, field, plan, row_index);
728                match actual {
729                    Some(s) => re.compiled.is_match(s.as_ref()),
730                    None => false,
731                }
732            }
733            FilterPredicate::IsType { field, field_type } => {
734                let resolved = resolve(field, plan);
735                let col = get_column(columns, field_index, resolved);
736                match col {
737                    None => false,
738                    Some(c) => c.is_type_at(row_index, field_type),
739                }
740            }
741            FilterPredicate::And(a, b) => {
742                // Evaluate the operand with the earlier field first for
743                // short-circuit benefit (C2: reorder by document position).
744                let (first, second) =
745                    if pred_ordinal(a, field_index, plan) <= pred_ordinal(b, field_index, plan) {
746                        (a.as_ref(), b.as_ref())
747                    } else {
748                        (b.as_ref(), a.as_ref())
749                    };
750                first.check(columns, field_index, row_index, plan)
751                    && second.check(columns, field_index, row_index, plan)
752            }
753            FilterPredicate::Or(a, b) => {
754                let (first, second) =
755                    if pred_ordinal(a, field_index, plan) <= pred_ordinal(b, field_index, plan) {
756                        (a.as_ref(), b.as_ref())
757                    } else {
758                        (b.as_ref(), a.as_ref())
759                    };
760                first.check(columns, field_index, row_index, plan)
761                    || second.check(columns, field_index, row_index, plan)
762            }
763            FilterPredicate::Not(inner) => !inner.check(columns, field_index, row_index, plan),
764        }
765    }
766}
767
768/// Resolve a filter field name to its output column name.
769fn resolve<'a>(field: &'a str, plan: &'a ExecutionPlan) -> &'a str {
770    plan.field_map.get(field).map_or(field, |s| s.as_str())
771}
772
773fn get_column<'a>(
774    columns: &'a [crate::columnar::ColumnBuilder],
775    field_index: &HashMap<String, usize>,
776    name: &str,
777) -> Option<&'a crate::columnar::ColumnBuilder> {
778    field_index.get(name).map(|&i| &columns[i])
779}
780
781/// Fetch a stored value formatted as a string for Equal/NotEqual checks.
782/// Tries zero-allocation `get_filter_view` first (String/Dict columns);
783/// falls back to `get_filter_value` for typed columns that need formatting.
784fn get_value(
785    columns: &[crate::columnar::ColumnBuilder],
786    field_index: &HashMap<String, usize>,
787    field: &str,
788    plan: &ExecutionPlan,
789    row_index: usize,
790) -> Option<String> {
791    let col = get_column(columns, field_index, resolve(field, plan))?;
792    // Fast path: borrowed &str for String/Dictionary columns (no allocation).
793    if let Some(s) = col.get_filter_view(row_index) {
794        return Some(s.to_owned());
795    }
796    // Slow path: typed columns need formatting (allocates).
797    col.get_filter_value(row_index)
798}
799
800/// Native-typed comparison with numeric promotion. Mixed Int64/Float64
801/// operands are widened to f64; any other type mismatch yields false.
802/// Nulls never reach here (checked by the caller).
803fn compare_typed(
804    a: &crate::columnar::TypedValue<'_>,
805    op: CompareOp,
806    b: &crate::columnar::TypedValue<'_>,
807) -> bool {
808    use crate::columnar::TypedValue as T;
809    // Numeric promotion: Int64 <-> Float64.
810    let num_pair: Option<(f64, f64)> = match (a, b) {
811        (T::Int64(x), T::Int64(y)) => Some((*x as f64, *y as f64)),
812        (T::Float64(x), T::Float64(y)) => Some((*x, *y)),
813        (T::Int64(x), T::Float64(y)) | (T::Float64(y), T::Int64(x)) => Some((*x as f64, *y)),
814        _ => None,
815    };
816    if let Some((x, y)) = num_pair {
817        return apply_op(op, x.partial_cmp(&y));
818    }
819    match (a, b) {
820        (T::Str(x), T::Str(y)) => apply_op(op, x.cmp(y).into()),
821        (T::Bool(x), T::Bool(y)) => apply_op(op, x.cmp(y).into()),
822        // Temporal types compare by their raw integer; mixed Date32/Timestamp
823        // or differing timestamp units are not comparable.
824        (T::Date32(x), T::Date32(y)) => apply_op(op, x.cmp(y).into()),
825        (T::Timestamp(x), T::Timestamp(y)) => apply_op(op, x.cmp(y).into()),
826        _ => false,
827    }
828}
829
830/// Apply an operator to an `Ordering`, treating `None` (NaN) as no-match.
831fn apply_op(op: CompareOp, ord: Option<std::cmp::Ordering>) -> bool {
832    use std::cmp::Ordering::*;
833    let Some(ord) = ord else { return false };
834    match op {
835        CompareOp::Gt => ord == Greater,
836        CompareOp::Lt => ord == Less,
837        CompareOp::Ge => ord != Less,
838        CompareOp::Le => ord != Greater,
839        CompareOp::Eq => ord == Equal,
840        CompareOp::Ne => ord != Equal,
841    }
842}
843
844/// Parse a string literal into a `TypedValue` matching the operand's type.
845/// Used by `CompareLiteral` to coerce the literal to the field's type.
846fn typed_from_literal<'a>(
847    sample: &crate::columnar::TypedValue<'a>,
848    literal: &'a str,
849) -> Option<crate::columnar::TypedValue<'a>> {
850    use crate::columnar::TypedValue as T;
851    match sample {
852        T::Str(_) => Some(T::Str(literal)),
853        T::Int64(_) => literal.parse::<i64>().ok().map(T::Int64),
854        T::Float64(_) => literal.parse::<f64>().ok().map(T::Float64),
855        T::Bool(_) => match literal {
856            "true" | "True" | "TRUE" => Some(T::Bool(true)),
857            "false" | "False" | "FALSE" => Some(T::Bool(false)),
858            _ => None,
859        },
860        T::Date32(_) => literal.parse::<i32>().ok().map(T::Date32),
861        T::Timestamp(_) => literal.parse::<i64>().ok().map(T::Timestamp),
862    }
863}
864
865/// Return the minimum field ordinal for a predicate, used to order
866/// `And`/`Or` operands by document position (C2). Higher ordinal = later
867/// in the document. Returns `usize::MAX` for predicates without a
868/// resolvable field (should evaluate last).
869fn pred_ordinal(
870    pred: &FilterPredicate,
871    field_index: &HashMap<String, usize>,
872    plan: &ExecutionPlan,
873) -> usize {
874    match pred {
875        FilterPredicate::Equal { field, .. }
876        | FilterPredicate::NotEqual { field, .. }
877        | FilterPredicate::CompareLiteral { field, .. }
878        | FilterPredicate::StartsWith { field, .. }
879        | FilterPredicate::EndsWith { field, .. }
880        | FilterPredicate::In { field, .. }
881        | FilterPredicate::NotIn { field, .. }
882        | FilterPredicate::NotField { field, .. }
883        | FilterPredicate::ArithmeticCompare { field, .. }
884        | FilterPredicate::Strip { field, .. }
885        | FilterPredicate::Lower { field, .. }
886        | FilterPredicate::Upper { field, .. }
887        | FilterPredicate::Replace { field, .. }
888        | FilterPredicate::Length { field, .. }
889        | FilterPredicate::Contains { field, .. }
890        | FilterPredicate::IsNull { field, .. }
891        | FilterPredicate::IsType { field, .. }
892        | FilterPredicate::Regex { field, .. } => field_index
893            .get(resolve(field, plan))
894            .copied()
895            .unwrap_or(usize::MAX),
896        FilterPredicate::Always(_) => usize::MAX,
897        FilterPredicate::Compare {
898            field_a, field_b, ..
899        } => {
900            let a = field_index
901                .get(resolve(field_a, plan))
902                .copied()
903                .unwrap_or(usize::MAX);
904            let b = field_index
905                .get(resolve(field_b, plan))
906                .copied()
907                .unwrap_or(usize::MAX);
908            a.min(b)
909        }
910        FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
911            pred_ordinal(a, field_index, plan).min(pred_ordinal(b, field_index, plan))
912        }
913        FilterPredicate::Not(inner) => pred_ordinal(inner, field_index, plan),
914    }
915}
916
917#[cfg(test)]
918mod tests {
919    use super::*;
920
921    #[test]
922    fn test_timestamp_spec_plain() {
923        assert_eq!(
924            "timestamp".parse::<FieldType>(),
925            Ok(FieldType::Timestamp(TimeUnit::Microsecond, None))
926        );
927        assert_eq!(
928            "timestamp[ns]".parse::<FieldType>(),
929            Ok(FieldType::Timestamp(TimeUnit::Nanosecond, None))
930        );
931    }
932
933    #[test]
934    fn test_timestamp_spec_format() {
935        assert_eq!(
936            "timestamp[ms,format=%Y%m%d %H:%M]".parse::<FieldType>(),
937            Ok(FieldType::Timestamp(
938                TimeUnit::Millisecond,
939                Some("%Y%m%d %H:%M".into())
940            ))
941        );
942        // Unit omitted: defaults to microseconds.
943        assert_eq!(
944            "timestamp[format=%d/%m/%Y]".parse::<FieldType>(),
945            Ok(FieldType::Timestamp(
946                TimeUnit::Microsecond,
947                Some("%d/%m/%Y".into())
948            ))
949        );
950        // Unknown unit or unknown option key: rejected.
951        assert_eq!("timestamp[fortnights]".parse::<FieldType>(), Err(()));
952        assert_eq!("timestamp[us,color=red]".parse::<FieldType>(), Err(()));
953        assert_eq!("timestamp[us,format=]".parse::<FieldType>(), Err(()));
954    }
955}