Skip to main content

rypipe_core/
plan.rs

1use arrow::datatypes::TimeUnit;
2use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
3
4mod predicate;
5pub(crate) use predicate::compare_decimal128;
6pub use predicate::{ArithOp, CompareOp, FilterPredicate, RegexSpec, TrimMode};
7
8/// The storage type for a column.
9#[derive(Clone, Debug, PartialEq, Eq)]
10pub enum FieldType {
11    String,
12    Int64,
13    Float64,
14    Boolean,
15    Dictionary,
16    /// Calendar date: days since the Unix epoch.
17    Date32,
18    /// Point in time with an explicit unit and an optional chrono format
19    /// string for parsing non-ISO input (e.g. `timestamp[ms,format=%Y%m%d]`).
20    Timestamp(TimeUnit, Option<Box<str>>),
21    /// Fixed-precision decimal with given scale (default 18).
22    Decimal128(u8),
23}
24
25impl std::str::FromStr for FieldType {
26    type Err = String;
27    fn from_str(s: &str) -> Result<Self, Self::Err> {
28        match s {
29            "string" => Ok(FieldType::String),
30            "int64" => Ok(FieldType::Int64),
31            "float64" => Ok(FieldType::Float64),
32            "bool" | "boolean" => Ok(FieldType::Boolean),
33            "dictionary" => Ok(FieldType::Dictionary),
34            "date32" => Ok(FieldType::Date32),
35            s if s.starts_with("timestamp") => {
36                parse_timestamp_spec(s).ok_or_else(|| format!("invalid timestamp spec: {s:?}"))
37            }
38            "decimal128" => Ok(FieldType::Decimal128(18)),
39            s if s.starts_with("decimal128(") => {
40                let inner = s
41                    .strip_prefix("decimal128(")
42                    .and_then(|v| v.strip_suffix(')'))
43                    .ok_or_else(|| format!("invalid decimal128 spec: {s:?}"))?;
44                let scale = inner
45                    .parse::<u8>()
46                    .map_err(|e| format!("invalid decimal128 scale: {e}"))?;
47                if scale > 38 {
48                    return Err(format!("decimal128 scale exceeds 38: {scale}"));
49                }
50                Ok(FieldType::Decimal128(scale))
51            }
52            _ => Err(format!("unknown field type: {s:?}")),
53        }
54    }
55}
56
57/// Parse a timestamp type spec: `timestamp`, `timestamp[unit]`, or
58/// `timestamp[unit,format=…]` (unit defaults to `us`; format is a chrono
59/// format string tried before the built-in ISO layouts).
60fn parse_timestamp_spec(s: &str) -> Option<FieldType> {
61    let rest = s.strip_prefix("timestamp")?;
62    if rest.is_empty() {
63        return Some(FieldType::Timestamp(TimeUnit::Microsecond, None));
64    }
65    let inner = rest.strip_prefix('[')?.strip_suffix(']')?;
66    let (unit_str, format) = match inner.split_once(',') {
67        Some((unit, opt)) => {
68            let fmt = opt.strip_prefix("format=")?;
69            if fmt.is_empty() {
70                return None;
71            }
72            (unit, Some(fmt))
73        }
74        None => {
75            if let Some(fmt) = inner.strip_prefix("format=") {
76                if fmt.is_empty() {
77                    return None;
78                }
79                ("us", Some(fmt))
80            } else {
81                (inner, None)
82            }
83        }
84    };
85    let unit = match unit_str {
86        "" | "us" | "µs" => TimeUnit::Microsecond,
87        "s" => TimeUnit::Second,
88        "ms" => TimeUnit::Millisecond,
89        "ns" => TimeUnit::Nanosecond,
90        _ => return None,
91    };
92    let format: Option<Box<str>> =
93        format.and_then(|f| if f.is_empty() { None } else { Some(f.into()) });
94    Some(FieldType::Timestamp(unit, format))
95}
96
97/// A compiled execution plan that controls field renaming, dropping,
98/// type assignment, dictionary encoding, row filtering, and column ordering.
99/// Default (empty) is a no-op.
100#[derive(Clone, Default)]
101pub struct ExecutionPlan {
102    /// Map from raw field name to output column name.
103    pub field_map: HashMap<String, String>,
104    /// Set of raw field names to drop entirely.
105    pub drop_fields: HashSet<String>,
106    /// Explicit type overrides per output column name.
107    pub field_types: HashMap<String, FieldType>,
108    /// Set of output column names to dict-encode.
109    pub dictionary_columns: HashSet<String>,
110    /// Optional row filter predicate.
111    pub filter: Option<FilterPredicate>,
112    /// Desired output column order (names in order).  Columns not listed here
113    /// appear after all listed columns in first-appearance order.  If empty,
114    /// first-appearance order is used.
115    pub schema_order: Vec<String>,
116    /// When true, string columns with low cardinality are automatically
117    /// upgraded to dictionary encoding during parse.
118    pub auto_dict: bool,
119    /// Auto-dict tuning: maximum fraction of rows allowed as distinct values
120    /// for an upgrade. Defaults to `0.05` when `None`.
121    pub dict_threshold: Option<f64>,
122    /// Auto-dict tuning: maximum dictionary size (distinct values) allowed for
123    /// an upgrade. Defaults to `256` when `None`.
124    pub dict_max_size: Option<usize>,
125    /// When true, a non-null value that fails to parse into its declared
126    /// `field_types` type aborts the parse with an error instead of silently
127    /// becoming null. Genuine nulls (missing fields) stay null regardless.
128    pub strict_types: bool,
129    /// Cap on the number of bounded-streaming batches/chunks. Defaults to
130    /// `MAX_SPLIT_CHUNKS` (100,000) when `None`.
131    pub max_split_chunks: Option<usize>,
132    /// Optional row observer; hooks fire from parse threads during the read.
133    pub observer: Option<std::sync::Arc<dyn crate::RowObserver>>,
134}
135
136impl std::fmt::Debug for ExecutionPlan {
137    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
138        f.debug_struct("ExecutionPlan")
139            .field("field_map", &self.field_map)
140            .field("drop_fields", &self.drop_fields)
141            .field("field_types", &self.field_types)
142            .field("dictionary_columns", &self.dictionary_columns)
143            .field("filter", &self.filter)
144            .field("schema_order", &self.schema_order)
145            .field("auto_dict", &self.auto_dict)
146            .field("dict_threshold", &self.dict_threshold)
147            .field("dict_max_size", &self.dict_max_size)
148            .field("strict_types", &self.strict_types)
149            .field("max_split_chunks", &self.max_split_chunks)
150            .field("observer", &self.observer.is_some())
151            .finish()
152    }
153}
154
155impl ExecutionPlan {
156    pub fn new() -> Self {
157        Self::default()
158    }
159
160    /// Rename a raw field to an output column name.
161    pub fn rename(mut self, raw: impl Into<String>, output: impl Into<String>) -> Self {
162        self.field_map.insert(raw.into(), output.into());
163        self
164    }
165
166    /// Drop a single raw/output field.
167    pub fn drop(mut self, field: impl Into<String>) -> Self {
168        self.drop_fields.insert(field.into());
169        self
170    }
171
172    /// Drop many raw/output fields.
173    pub fn drop_many<I, S>(mut self, fields: I) -> Self
174    where
175        I: IntoIterator<Item = S>,
176        S: Into<String>,
177    {
178        self.drop_fields.extend(fields.into_iter().map(Into::into));
179        self
180    }
181
182    /// Set the storage type for an output column.
183    pub fn type_as(mut self, field: impl Into<String>, field_type: FieldType) -> Self {
184        self.field_types.insert(field.into(), field_type);
185        self
186    }
187
188    /// Dict-encode an output column.
189    pub fn dictionary(mut self, field: impl Into<String>) -> Self {
190        self.dictionary_columns.insert(field.into());
191        self
192    }
193
194    /// Keep only rows where `field == value` (string comparison, per-row).
195    pub fn filter_eq(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
196        self.filter = Some(FilterPredicate::Equal {
197            field: field.into(),
198            value: value.into(),
199        });
200        self
201    }
202
203    /// Keep only rows where `field != value` (string comparison, per-row).
204    pub fn filter_ne(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
205        self.filter = Some(FilterPredicate::NotEqual {
206            field: field.into(),
207            value: value.into(),
208        });
209        self
210    }
211
212    /// Keep only rows where `field_a op field_b` (native-typed, per-row).
213    pub fn filter_compare(
214        mut self,
215        field_a: impl Into<String>,
216        op: CompareOp,
217        field_b: impl Into<String>,
218    ) -> Self {
219        self.filter = Some(FilterPredicate::Compare {
220            field_a: field_a.into(),
221            op,
222            field_b: field_b.into(),
223        });
224        self
225    }
226
227    /// Declare the output schema as a **projection + order**: the output
228    /// contains exactly the listed columns, in the listed order.  Fields in
229    /// the data but not listed are skipped during parsing (never
230    /// materialized); listed columns absent from the data come out
231    /// null-filled.  `field_map` renames apply before matching.
232    pub fn schema_order<I, S>(mut self, order: I) -> Self
233    where
234        I: IntoIterator<Item = S>,
235        S: Into<String>,
236    {
237        self.schema_order = order.into_iter().map(Into::into).collect();
238        self
239    }
240
241    /// Enable or disable automatic dictionary upgrade for low-cardinality
242    /// string columns.
243    pub fn with_auto_dict(mut self, yes: bool) -> Self {
244        self.auto_dict = yes;
245        self
246    }
247
248    /// Override the auto-dict distinct-ratio threshold (default `0.05`).
249    pub fn with_dict_threshold(mut self, threshold: f64) -> Self {
250        self.dict_threshold = Some(threshold);
251        self
252    }
253
254    /// Override the auto-dict maximum dictionary size (default `256`).
255    pub fn with_dict_max_size(mut self, max_size: usize) -> Self {
256        self.dict_max_size = Some(max_size);
257        self
258    }
259
260    /// Enable strict typing: a non-null value that cannot be parsed into its
261    /// `field_types`-declared type becomes an error at `finish()` instead of
262    /// a silent null.
263    pub fn with_strict_types(mut self, yes: bool) -> Self {
264        self.strict_types = yes;
265        self
266    }
267
268    /// Override the bounded-streaming split cap (default 100,000 chunks).
269    pub fn with_max_split_chunks(mut self, cap: usize) -> Self {
270        self.max_split_chunks = Some(cap);
271        self
272    }
273
274    /// Attach a row observer; hooks fire from parse threads during the read.
275    pub fn with_observer(mut self, observer: std::sync::Arc<dyn crate::RowObserver>) -> Self {
276        self.observer = Some(observer);
277        self
278    }
279
280    /// Determine the storage type for an output column name.
281    pub fn column_type(&self, name: &str) -> FieldType {
282        if let Some(ft) = self.field_types.get(name) {
283            return ft.clone();
284        }
285        if self.dictionary_columns.contains(name) {
286            return FieldType::Dictionary;
287        }
288        FieldType::String
289    }
290
291    /// Resolve a raw field name to its output column name.
292    /// Returns `None` if the field should be dropped.
293    ///
294    /// Application order: rename first, then drop, then schema projection,
295    /// matching left-to-right pipeline semantics.
296    ///
297    /// When `schema_order` is non-empty it acts as a **projection**: only the
298    /// listed columns are materialized, so any other resolved name returns
299    /// `None` here (which makes `ColumnarSink::wants`/`resolve` reject the
300    /// field and lets adapters skip extracting it).  Fields referenced by the
301    /// row filter are kept even when absent from `schema_order` so predicates
302    /// still evaluate correctly; they are projected out of the final batch.
303    pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str> {
304        let resolved = self.field_map.get(raw).map_or(raw, |s| s.as_str());
305        if self.drop_fields.contains(resolved) {
306            return None;
307        }
308        if !self.schema_order.is_empty()
309            && !self.schema_order.iter().any(|n| n == resolved)
310            && !self.filter_references(resolved)
311        {
312            return None;
313        }
314        Some(resolved)
315    }
316
317    /// Whether the row filter references `resolved` (a post-rename output
318    /// name).  Used by [`Self::resolve_field`] to keep predicate fields out
319    /// of the schema projection.
320    fn filter_references(&self, resolved: &str) -> bool {
321        fn matches(pred: &FilterPredicate, plan: &ExecutionPlan, resolved: &str) -> bool {
322            // Rename only: drop/projection must not apply here (recursion).
323            fn ren<'x>(plan: &'x ExecutionPlan, f: &'x String) -> &'x str {
324                plan.field_map.get(f).map_or(f.as_str(), |s| s.as_str())
325            }
326            match pred {
327                FilterPredicate::Equal { field, .. }
328                | FilterPredicate::NotEqual { field, .. }
329                | FilterPredicate::CompareLiteral { field, .. }
330                | FilterPredicate::StartsWith { field, .. }
331                | FilterPredicate::EndsWith { field, .. }
332                | FilterPredicate::In { field, .. }
333                | FilterPredicate::NotIn { field, .. }
334                | FilterPredicate::NotField { field, .. }
335                | FilterPredicate::ArithmeticCompare { field, .. }
336                | FilterPredicate::Strip { field, .. }
337                | FilterPredicate::Lower { field, .. }
338                | FilterPredicate::Upper { field, .. }
339                | FilterPredicate::Replace { field, .. }
340                | FilterPredicate::Length { field, .. }
341                | FilterPredicate::Contains { field, .. }
342                | FilterPredicate::IsNull { field, .. }
343                | FilterPredicate::IsType { field, .. }
344                | FilterPredicate::Regex { field, .. } => ren(plan, field) == resolved,
345                FilterPredicate::Always(_) => false,
346                FilterPredicate::Compare {
347                    field_a, field_b, ..
348                } => ren(plan, field_a) == resolved || ren(plan, field_b) == resolved,
349                FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
350                    matches(a, plan, resolved) || matches(b, plan, resolved)
351                }
352                FilterPredicate::Not(inner) => matches(inner, plan, resolved),
353            }
354        }
355        self.filter
356            .as_ref()
357            .is_some_and(|f| matches(f, self, resolved))
358    }
359}
360
361#[cfg(test)]
362mod tests {
363    use super::*;
364
365    #[test]
366    fn test_timestamp_spec_plain() {
367        assert_eq!(
368            "timestamp".parse::<FieldType>(),
369            Ok(FieldType::Timestamp(TimeUnit::Microsecond, None))
370        );
371        assert_eq!(
372            "timestamp[ns]".parse::<FieldType>(),
373            Ok(FieldType::Timestamp(TimeUnit::Nanosecond, None))
374        );
375    }
376
377    #[test]
378    fn test_timestamp_spec_format() {
379        assert_eq!(
380            "timestamp[ms,format=%Y%m%d %H:%M]".parse::<FieldType>(),
381            Ok(FieldType::Timestamp(
382                TimeUnit::Millisecond,
383                Some("%Y%m%d %H:%M".into())
384            ))
385        );
386        assert_eq!(
387            "timestamp[format=%d/%m/%Y]".parse::<FieldType>(),
388            Ok(FieldType::Timestamp(
389                TimeUnit::Microsecond,
390                Some("%d/%m/%Y".into())
391            ))
392        );
393        assert!("timestamp[fortnights]".parse::<FieldType>().is_err());
394        assert!("timestamp[us,color=red]".parse::<FieldType>().is_err());
395        assert!("timestamp[us,format=]".parse::<FieldType>().is_err());
396    }
397
398    #[test]
399    fn decimal128_spec_requires_closing_delimiter_and_valid_scale() {
400        assert_eq!(
401            "decimal128(38)".parse::<FieldType>(),
402            Ok(FieldType::Decimal128(38))
403        );
404        assert!("decimal128(38".parse::<FieldType>().is_err());
405        assert!("decimal128(39)".parse::<FieldType>().is_err());
406    }
407}