Skip to main content

rypipe_core/
plan.rs

1use arrow::datatypes::TimeUnit;
2use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
3
4mod predicate;
5pub(crate) use predicate::compare_decimal128;
6pub use predicate::{ArithOp, CompareOp, FilterPredicate, RegexSpec, TrimMode};
7
8/// The storage type for a column.
9#[derive(Clone, Debug, PartialEq, Eq)]
10pub enum FieldType {
11    String,
12    Int64,
13    Float64,
14    Boolean,
15    Dictionary,
16    /// Calendar date: days since the Unix epoch.
17    Date32,
18    /// Point in time with an explicit unit and an optional chrono format
19    /// string for parsing non-ISO input (e.g. `timestamp[ms,format=%Y%m%d]`).
20    Timestamp(TimeUnit, Option<Box<str>>),
21    /// Fixed-precision decimal with given scale (default 18).
22    Decimal128(u8),
23}
24
25impl std::str::FromStr for FieldType {
26    type Err = String;
27    fn from_str(s: &str) -> Result<Self, Self::Err> {
28        match s {
29            "string" => Ok(FieldType::String),
30            "int64" => Ok(FieldType::Int64),
31            "float64" => Ok(FieldType::Float64),
32            "bool" | "boolean" => Ok(FieldType::Boolean),
33            "dictionary" => Ok(FieldType::Dictionary),
34            "date32" => Ok(FieldType::Date32),
35            s if s.starts_with("timestamp") => {
36                parse_timestamp_spec(s).ok_or_else(|| format!("invalid timestamp spec: {s:?}"))
37            }
38            "decimal128" => Ok(FieldType::Decimal128(18)),
39            s if s.starts_with("decimal128(") => {
40                let inner = s
41                    .strip_prefix("decimal128(")
42                    .and_then(|v| v.strip_suffix(')'))
43                    .ok_or_else(|| format!("invalid decimal128 spec: {s:?}"))?;
44                let scale = inner
45                    .parse::<u8>()
46                    .map_err(|e| format!("invalid decimal128 scale: {e}"))?;
47                if scale > 38 {
48                    return Err(format!("decimal128 scale exceeds 38: {scale}"));
49                }
50                Ok(FieldType::Decimal128(scale))
51            }
52            _ => Err(format!("unknown field type: {s:?}")),
53        }
54    }
55}
56
57/// Parse a timestamp type spec: `timestamp`, `timestamp[unit]`, or
58/// `timestamp[unit,format=…]` (unit defaults to `us`; format is a chrono
59/// format string tried before the built-in ISO layouts).
60fn parse_timestamp_spec(s: &str) -> Option<FieldType> {
61    let rest = s.strip_prefix("timestamp")?;
62    if rest.is_empty() {
63        return Some(FieldType::Timestamp(TimeUnit::Microsecond, None));
64    }
65    let inner = rest.strip_prefix('[')?.strip_suffix(']')?;
66    let (unit_str, format) = match inner.split_once(',') {
67        Some((unit, opt)) => {
68            let fmt = opt.strip_prefix("format=")?;
69            if fmt.is_empty() {
70                return None;
71            }
72            (unit, Some(fmt))
73        }
74        None => {
75            if let Some(fmt) = inner.strip_prefix("format=") {
76                if fmt.is_empty() {
77                    return None;
78                }
79                ("us", Some(fmt))
80            } else {
81                (inner, None)
82            }
83        }
84    };
85    let unit = match unit_str {
86        "" | "us" | "µs" => TimeUnit::Microsecond,
87        "s" => TimeUnit::Second,
88        "ms" => TimeUnit::Millisecond,
89        "ns" => TimeUnit::Nanosecond,
90        _ => return None,
91    };
92    let format: Option<Box<str>> =
93        format.and_then(|f| if f.is_empty() { None } else { Some(f.into()) });
94    Some(FieldType::Timestamp(unit, format))
95}
96
97/// A compiled execution plan that controls field renaming, dropping,
98/// type assignment, dictionary encoding, row filtering, and column ordering.
99/// Default (empty) is a no-op.
100#[derive(Clone, Default)]
101pub struct ExecutionPlan {
102    /// Map from raw field name to output column name.
103    pub field_map: HashMap<String, String>,
104    /// Set of raw field names to drop entirely.
105    pub drop_fields: HashSet<String>,
106    /// Explicit type overrides per output column name.
107    pub field_types: HashMap<String, FieldType>,
108    /// Set of output column names to dict-encode.
109    pub dictionary_columns: HashSet<String>,
110    /// Optional row filter predicate.
111    pub filter: Option<FilterPredicate>,
112    /// Desired output column order (names in order).  Columns not listed here
113    /// appear after all listed columns in first-appearance order.  If empty,
114    /// first-appearance order is used.
115    pub schema_order: Vec<String>,
116    /// When true, string columns with low cardinality are automatically
117    /// upgraded to dictionary encoding during parse.
118    pub auto_dict: bool,
119    /// Auto-dict tuning: maximum fraction of rows allowed as distinct values
120    /// for an upgrade. Defaults to `0.05` when `None`.
121    pub dict_threshold: Option<f64>,
122    /// Auto-dict tuning: maximum dictionary size (distinct values) allowed for
123    /// an upgrade. Defaults to `256` when `None`.
124    pub dict_max_size: Option<usize>,
125    /// When true, a non-null value that fails to parse into its declared
126    /// `field_types` type aborts the parse with an error instead of silently
127    /// becoming null. Genuine nulls (missing fields) stay null regardless.
128    pub strict_types: bool,
129    /// Cap on the number of bounded-streaming batches/chunks. Defaults to
130    /// `MAX_SPLIT_CHUNKS` (100,000) when `None`.
131    pub max_split_chunks: Option<usize>,
132    /// Opt-in chunk-size floor for split planning. `None` uses the engine
133    /// default ([`crate::MIN_CHUNK_BYTES`], 2 MiB). Lower it for many small
134    /// cores (more, smaller chunks); raise it for fewer, larger chunks.
135    pub min_chunk_bytes: Option<usize>,
136    /// Optional row observer; hooks fire from parse threads during the read.
137    pub observer: Option<std::sync::Arc<dyn crate::RowObserver>>,
138}
139
140impl std::fmt::Debug for ExecutionPlan {
141    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
142        f.debug_struct("ExecutionPlan")
143            .field("field_map", &self.field_map)
144            .field("drop_fields", &self.drop_fields)
145            .field("field_types", &self.field_types)
146            .field("dictionary_columns", &self.dictionary_columns)
147            .field("filter", &self.filter)
148            .field("schema_order", &self.schema_order)
149            .field("auto_dict", &self.auto_dict)
150            .field("dict_threshold", &self.dict_threshold)
151            .field("dict_max_size", &self.dict_max_size)
152            .field("strict_types", &self.strict_types)
153            .field("max_split_chunks", &self.max_split_chunks)
154            .field("min_chunk_bytes", &self.min_chunk_bytes)
155            .field("observer", &self.observer.is_some())
156            .finish()
157    }
158}
159
160impl ExecutionPlan {
161    pub fn new() -> Self {
162        Self::default()
163    }
164
165    /// Rename a raw field to an output column name.
166    pub fn rename(mut self, raw: impl Into<String>, output: impl Into<String>) -> Self {
167        self.field_map.insert(raw.into(), output.into());
168        self
169    }
170
171    /// Drop a single raw/output field.
172    pub fn drop(mut self, field: impl Into<String>) -> Self {
173        self.drop_fields.insert(field.into());
174        self
175    }
176
177    /// Drop many raw/output fields.
178    pub fn drop_many<I, S>(mut self, fields: I) -> Self
179    where
180        I: IntoIterator<Item = S>,
181        S: Into<String>,
182    {
183        self.drop_fields.extend(fields.into_iter().map(Into::into));
184        self
185    }
186
187    /// Set the storage type for an output column.
188    pub fn type_as(mut self, field: impl Into<String>, field_type: FieldType) -> Self {
189        self.field_types.insert(field.into(), field_type);
190        self
191    }
192
193    /// Dict-encode an output column.
194    pub fn dictionary(mut self, field: impl Into<String>) -> Self {
195        self.dictionary_columns.insert(field.into());
196        self
197    }
198
199    /// Keep only rows where `field == value` (string comparison, per-row).
200    pub fn filter_eq(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
201        self.filter = Some(FilterPredicate::Equal {
202            field: field.into(),
203            value: value.into(),
204        });
205        self
206    }
207
208    /// Keep only rows where `field != value` (string comparison, per-row).
209    pub fn filter_ne(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
210        self.filter = Some(FilterPredicate::NotEqual {
211            field: field.into(),
212            value: value.into(),
213        });
214        self
215    }
216
217    /// Keep only rows where `field_a op field_b` (native-typed, per-row).
218    pub fn filter_compare(
219        mut self,
220        field_a: impl Into<String>,
221        op: CompareOp,
222        field_b: impl Into<String>,
223    ) -> Self {
224        self.filter = Some(FilterPredicate::Compare {
225            field_a: field_a.into(),
226            op,
227            field_b: field_b.into(),
228        });
229        self
230    }
231
232    /// Declare the output schema as a **projection + order**: the output
233    /// contains exactly the listed columns, in the listed order.  Fields in
234    /// the data but not listed are skipped during parsing (never
235    /// materialized); listed columns absent from the data come out
236    /// null-filled.  `field_map` renames apply before matching.
237    pub fn schema_order<I, S>(mut self, order: I) -> Self
238    where
239        I: IntoIterator<Item = S>,
240        S: Into<String>,
241    {
242        self.schema_order = order.into_iter().map(Into::into).collect();
243        self
244    }
245
246    /// Enable or disable automatic dictionary upgrade for low-cardinality
247    /// string columns.
248    pub fn with_auto_dict(mut self, yes: bool) -> Self {
249        self.auto_dict = yes;
250        self
251    }
252
253    /// Override the auto-dict distinct-ratio threshold (default `0.05`).
254    pub fn with_dict_threshold(mut self, threshold: f64) -> Self {
255        self.dict_threshold = Some(threshold);
256        self
257    }
258
259    /// Override the auto-dict maximum dictionary size (default `256`).
260    pub fn with_dict_max_size(mut self, max_size: usize) -> Self {
261        self.dict_max_size = Some(max_size);
262        self
263    }
264
265    /// Enable strict typing: a non-null value that cannot be parsed into its
266    /// `field_types`-declared type becomes an error at `finish()` instead of
267    /// a silent null.
268    pub fn with_strict_types(mut self, yes: bool) -> Self {
269        self.strict_types = yes;
270        self
271    }
272
273    /// Override the bounded-streaming split cap (default 100,000 chunks).
274    pub fn with_max_split_chunks(mut self, cap: usize) -> Self {
275        self.max_split_chunks = Some(cap);
276        self
277    }
278
279    /// Override the split-planning chunk-size floor (default 2 MiB).
280    pub fn with_min_chunk_bytes(mut self, bytes: usize) -> Self {
281        self.min_chunk_bytes = Some(bytes.max(1));
282        self
283    }
284
285    /// Attach a row observer; hooks fire from parse threads during the read.
286    pub fn with_observer(mut self, observer: std::sync::Arc<dyn crate::RowObserver>) -> Self {
287        self.observer = Some(observer);
288        self
289    }
290
291    /// Determine the storage type for an output column name.
292    pub fn column_type(&self, name: &str) -> FieldType {
293        if let Some(ft) = self.field_types.get(name) {
294            return ft.clone();
295        }
296        if self.dictionary_columns.contains(name) {
297            return FieldType::Dictionary;
298        }
299        FieldType::String
300    }
301
302    /// Resolve a raw field name to its output column name.
303    /// Returns `None` if the field should be dropped.
304    ///
305    /// Application order: rename first, then drop, then schema projection,
306    /// matching left-to-right pipeline semantics.
307    ///
308    /// When `schema_order` is non-empty it acts as a **projection**: only the
309    /// listed columns are materialized, so any other resolved name returns
310    /// `None` here (which makes `ColumnarSink::wants`/`resolve` reject the
311    /// field and lets adapters skip extracting it).  Fields referenced by the
312    /// row filter are kept even when absent from `schema_order` so predicates
313    /// still evaluate correctly; they are projected out of the final batch.
314    pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str> {
315        let resolved = self.field_map.get(raw).map_or(raw, |s| s.as_str());
316        if self.drop_fields.contains(resolved) {
317            return None;
318        }
319        if !self.schema_order.is_empty()
320            && !self.schema_order.iter().any(|n| n == resolved)
321            && !self.filter_references(resolved)
322        {
323            return None;
324        }
325        Some(resolved)
326    }
327
328    /// Whether the row filter references `resolved` (a post-rename output
329    /// name).  Used by [`Self::resolve_field`] to keep predicate fields out
330    /// of the schema projection.
331    fn filter_references(&self, resolved: &str) -> bool {
332        fn matches(pred: &FilterPredicate, plan: &ExecutionPlan, resolved: &str) -> bool {
333            // Rename only: drop/projection must not apply here (recursion).
334            fn ren<'x>(plan: &'x ExecutionPlan, f: &'x String) -> &'x str {
335                plan.field_map.get(f).map_or(f.as_str(), |s| s.as_str())
336            }
337            match pred {
338                FilterPredicate::Equal { field, .. }
339                | FilterPredicate::NotEqual { field, .. }
340                | FilterPredicate::CompareLiteral { field, .. }
341                | FilterPredicate::StartsWith { field, .. }
342                | FilterPredicate::EndsWith { field, .. }
343                | FilterPredicate::In { field, .. }
344                | FilterPredicate::NotIn { field, .. }
345                | FilterPredicate::NotField { field, .. }
346                | FilterPredicate::ArithmeticCompare { field, .. }
347                | FilterPredicate::Strip { field, .. }
348                | FilterPredicate::Lower { field, .. }
349                | FilterPredicate::Upper { field, .. }
350                | FilterPredicate::Replace { field, .. }
351                | FilterPredicate::Length { field, .. }
352                | FilterPredicate::Contains { field, .. }
353                | FilterPredicate::IsNull { field, .. }
354                | FilterPredicate::IsType { field, .. }
355                | FilterPredicate::Regex { field, .. } => ren(plan, field) == resolved,
356                FilterPredicate::Always(_) => false,
357                FilterPredicate::Compare {
358                    field_a, field_b, ..
359                } => ren(plan, field_a) == resolved || ren(plan, field_b) == resolved,
360                FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
361                    matches(a, plan, resolved) || matches(b, plan, resolved)
362                }
363                FilterPredicate::Not(inner) => matches(inner, plan, resolved),
364            }
365        }
366        self.filter
367            .as_ref()
368            .is_some_and(|f| matches(f, self, resolved))
369    }
370}
371
372#[cfg(test)]
373mod tests {
374    use super::*;
375
376    #[test]
377    fn test_timestamp_spec_plain() {
378        assert_eq!(
379            "timestamp".parse::<FieldType>(),
380            Ok(FieldType::Timestamp(TimeUnit::Microsecond, None))
381        );
382        assert_eq!(
383            "timestamp[ns]".parse::<FieldType>(),
384            Ok(FieldType::Timestamp(TimeUnit::Nanosecond, None))
385        );
386    }
387
388    #[test]
389    fn test_timestamp_spec_format() {
390        assert_eq!(
391            "timestamp[ms,format=%Y%m%d %H:%M]".parse::<FieldType>(),
392            Ok(FieldType::Timestamp(
393                TimeUnit::Millisecond,
394                Some("%Y%m%d %H:%M".into())
395            ))
396        );
397        assert_eq!(
398            "timestamp[format=%d/%m/%Y]".parse::<FieldType>(),
399            Ok(FieldType::Timestamp(
400                TimeUnit::Microsecond,
401                Some("%d/%m/%Y".into())
402            ))
403        );
404        assert!("timestamp[fortnights]".parse::<FieldType>().is_err());
405        assert!("timestamp[us,color=red]".parse::<FieldType>().is_err());
406        assert!("timestamp[us,format=]".parse::<FieldType>().is_err());
407    }
408
409    #[test]
410    fn decimal128_spec_requires_closing_delimiter_and_valid_scale() {
411        assert_eq!(
412            "decimal128(38)".parse::<FieldType>(),
413            Ok(FieldType::Decimal128(38))
414        );
415        assert!("decimal128(38".parse::<FieldType>().is_err());
416        assert!("decimal128(39)".parse::<FieldType>().is_err());
417    }
418}