1use arrow::datatypes::TimeUnit;
2use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
3
4mod predicate;
5pub(crate) use predicate::compare_decimal128;
6pub use predicate::{ArithOp, CompareOp, FilterPredicate, RegexSpec, TrimMode};
7
8#[derive(Clone, Debug, PartialEq, Eq)]
10pub enum FieldType {
11 String,
12 Int64,
13 Float64,
14 Boolean,
15 Dictionary,
16 Date32,
18 Timestamp(TimeUnit, Option<Box<str>>),
21 Decimal128(u8),
23}
24
25impl std::str::FromStr for FieldType {
26 type Err = String;
27 fn from_str(s: &str) -> Result<Self, Self::Err> {
28 match s {
29 "string" => Ok(FieldType::String),
30 "int64" => Ok(FieldType::Int64),
31 "float64" => Ok(FieldType::Float64),
32 "bool" | "boolean" => Ok(FieldType::Boolean),
33 "dictionary" => Ok(FieldType::Dictionary),
34 "date32" => Ok(FieldType::Date32),
35 s if s.starts_with("timestamp") => {
36 parse_timestamp_spec(s).ok_or_else(|| format!("invalid timestamp spec: {s:?}"))
37 }
38 "decimal128" => Ok(FieldType::Decimal128(18)),
39 s if s.starts_with("decimal128(") => {
40 let inner = s
41 .strip_prefix("decimal128(")
42 .and_then(|v| v.strip_suffix(')'))
43 .ok_or_else(|| format!("invalid decimal128 spec: {s:?}"))?;
44 let scale = inner
45 .parse::<u8>()
46 .map_err(|e| format!("invalid decimal128 scale: {e}"))?;
47 if scale > 38 {
48 return Err(format!("decimal128 scale exceeds 38: {scale}"));
49 }
50 Ok(FieldType::Decimal128(scale))
51 }
52 _ => Err(format!("unknown field type: {s:?}")),
53 }
54 }
55}
56
57fn parse_timestamp_spec(s: &str) -> Option<FieldType> {
61 let rest = s.strip_prefix("timestamp")?;
62 if rest.is_empty() {
63 return Some(FieldType::Timestamp(TimeUnit::Microsecond, None));
64 }
65 let inner = rest.strip_prefix('[')?.strip_suffix(']')?;
66 let (unit_str, format) = match inner.split_once(',') {
67 Some((unit, opt)) => {
68 let fmt = opt.strip_prefix("format=")?;
69 if fmt.is_empty() {
70 return None;
71 }
72 (unit, Some(fmt))
73 }
74 None => {
75 if let Some(fmt) = inner.strip_prefix("format=") {
76 if fmt.is_empty() {
77 return None;
78 }
79 ("us", Some(fmt))
80 } else {
81 (inner, None)
82 }
83 }
84 };
85 let unit = match unit_str {
86 "" | "us" | "µs" => TimeUnit::Microsecond,
87 "s" => TimeUnit::Second,
88 "ms" => TimeUnit::Millisecond,
89 "ns" => TimeUnit::Nanosecond,
90 _ => return None,
91 };
92 let format: Option<Box<str>> =
93 format.and_then(|f| if f.is_empty() { None } else { Some(f.into()) });
94 Some(FieldType::Timestamp(unit, format))
95}
96
97#[derive(Clone, Default)]
101pub struct ExecutionPlan {
102 pub field_map: HashMap<String, String>,
104 pub drop_fields: HashSet<String>,
106 pub field_types: HashMap<String, FieldType>,
108 pub dictionary_columns: HashSet<String>,
110 pub filter: Option<FilterPredicate>,
112 pub schema_order: Vec<String>,
116 pub auto_dict: bool,
119 pub dict_threshold: Option<f64>,
122 pub dict_max_size: Option<usize>,
125 pub strict_types: bool,
129 pub max_split_chunks: Option<usize>,
132 pub min_chunk_bytes: Option<usize>,
136 pub observer: Option<std::sync::Arc<dyn crate::RowObserver>>,
138}
139
140impl std::fmt::Debug for ExecutionPlan {
141 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
142 f.debug_struct("ExecutionPlan")
143 .field("field_map", &self.field_map)
144 .field("drop_fields", &self.drop_fields)
145 .field("field_types", &self.field_types)
146 .field("dictionary_columns", &self.dictionary_columns)
147 .field("filter", &self.filter)
148 .field("schema_order", &self.schema_order)
149 .field("auto_dict", &self.auto_dict)
150 .field("dict_threshold", &self.dict_threshold)
151 .field("dict_max_size", &self.dict_max_size)
152 .field("strict_types", &self.strict_types)
153 .field("max_split_chunks", &self.max_split_chunks)
154 .field("min_chunk_bytes", &self.min_chunk_bytes)
155 .field("observer", &self.observer.is_some())
156 .finish()
157 }
158}
159
160impl ExecutionPlan {
161 pub fn new() -> Self {
162 Self::default()
163 }
164
165 pub fn rename(mut self, raw: impl Into<String>, output: impl Into<String>) -> Self {
167 self.field_map.insert(raw.into(), output.into());
168 self
169 }
170
171 pub fn drop(mut self, field: impl Into<String>) -> Self {
173 self.drop_fields.insert(field.into());
174 self
175 }
176
177 pub fn drop_many<I, S>(mut self, fields: I) -> Self
179 where
180 I: IntoIterator<Item = S>,
181 S: Into<String>,
182 {
183 self.drop_fields.extend(fields.into_iter().map(Into::into));
184 self
185 }
186
187 pub fn type_as(mut self, field: impl Into<String>, field_type: FieldType) -> Self {
189 self.field_types.insert(field.into(), field_type);
190 self
191 }
192
193 pub fn dictionary(mut self, field: impl Into<String>) -> Self {
195 self.dictionary_columns.insert(field.into());
196 self
197 }
198
199 pub fn filter_eq(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
201 self.filter = Some(FilterPredicate::Equal {
202 field: field.into(),
203 value: value.into(),
204 });
205 self
206 }
207
208 pub fn filter_ne(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
210 self.filter = Some(FilterPredicate::NotEqual {
211 field: field.into(),
212 value: value.into(),
213 });
214 self
215 }
216
217 pub fn filter_compare(
219 mut self,
220 field_a: impl Into<String>,
221 op: CompareOp,
222 field_b: impl Into<String>,
223 ) -> Self {
224 self.filter = Some(FilterPredicate::Compare {
225 field_a: field_a.into(),
226 op,
227 field_b: field_b.into(),
228 });
229 self
230 }
231
232 pub fn schema_order<I, S>(mut self, order: I) -> Self
238 where
239 I: IntoIterator<Item = S>,
240 S: Into<String>,
241 {
242 self.schema_order = order.into_iter().map(Into::into).collect();
243 self
244 }
245
246 pub fn with_auto_dict(mut self, yes: bool) -> Self {
249 self.auto_dict = yes;
250 self
251 }
252
253 pub fn with_dict_threshold(mut self, threshold: f64) -> Self {
255 self.dict_threshold = Some(threshold);
256 self
257 }
258
259 pub fn with_dict_max_size(mut self, max_size: usize) -> Self {
261 self.dict_max_size = Some(max_size);
262 self
263 }
264
265 pub fn with_strict_types(mut self, yes: bool) -> Self {
269 self.strict_types = yes;
270 self
271 }
272
273 pub fn with_max_split_chunks(mut self, cap: usize) -> Self {
275 self.max_split_chunks = Some(cap);
276 self
277 }
278
279 pub fn with_min_chunk_bytes(mut self, bytes: usize) -> Self {
281 self.min_chunk_bytes = Some(bytes.max(1));
282 self
283 }
284
285 pub fn with_observer(mut self, observer: std::sync::Arc<dyn crate::RowObserver>) -> Self {
287 self.observer = Some(observer);
288 self
289 }
290
291 pub fn column_type(&self, name: &str) -> FieldType {
293 if let Some(ft) = self.field_types.get(name) {
294 return ft.clone();
295 }
296 if self.dictionary_columns.contains(name) {
297 return FieldType::Dictionary;
298 }
299 FieldType::String
300 }
301
302 pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str> {
315 let resolved = self.field_map.get(raw).map_or(raw, |s| s.as_str());
316 if self.drop_fields.contains(resolved) {
317 return None;
318 }
319 if !self.schema_order.is_empty()
320 && !self.schema_order.iter().any(|n| n == resolved)
321 && !self.filter_references(resolved)
322 {
323 return None;
324 }
325 Some(resolved)
326 }
327
328 fn filter_references(&self, resolved: &str) -> bool {
332 fn matches(pred: &FilterPredicate, plan: &ExecutionPlan, resolved: &str) -> bool {
333 fn ren<'x>(plan: &'x ExecutionPlan, f: &'x String) -> &'x str {
335 plan.field_map.get(f).map_or(f.as_str(), |s| s.as_str())
336 }
337 match pred {
338 FilterPredicate::Equal { field, .. }
339 | FilterPredicate::NotEqual { field, .. }
340 | FilterPredicate::CompareLiteral { field, .. }
341 | FilterPredicate::StartsWith { field, .. }
342 | FilterPredicate::EndsWith { field, .. }
343 | FilterPredicate::In { field, .. }
344 | FilterPredicate::NotIn { field, .. }
345 | FilterPredicate::NotField { field, .. }
346 | FilterPredicate::ArithmeticCompare { field, .. }
347 | FilterPredicate::Strip { field, .. }
348 | FilterPredicate::Lower { field, .. }
349 | FilterPredicate::Upper { field, .. }
350 | FilterPredicate::Replace { field, .. }
351 | FilterPredicate::Length { field, .. }
352 | FilterPredicate::Contains { field, .. }
353 | FilterPredicate::IsNull { field, .. }
354 | FilterPredicate::IsType { field, .. }
355 | FilterPredicate::Regex { field, .. } => ren(plan, field) == resolved,
356 FilterPredicate::Always(_) => false,
357 FilterPredicate::Compare {
358 field_a, field_b, ..
359 } => ren(plan, field_a) == resolved || ren(plan, field_b) == resolved,
360 FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
361 matches(a, plan, resolved) || matches(b, plan, resolved)
362 }
363 FilterPredicate::Not(inner) => matches(inner, plan, resolved),
364 }
365 }
366 self.filter
367 .as_ref()
368 .is_some_and(|f| matches(f, self, resolved))
369 }
370}
371
372#[cfg(test)]
373mod tests {
374 use super::*;
375
376 #[test]
377 fn test_timestamp_spec_plain() {
378 assert_eq!(
379 "timestamp".parse::<FieldType>(),
380 Ok(FieldType::Timestamp(TimeUnit::Microsecond, None))
381 );
382 assert_eq!(
383 "timestamp[ns]".parse::<FieldType>(),
384 Ok(FieldType::Timestamp(TimeUnit::Nanosecond, None))
385 );
386 }
387
388 #[test]
389 fn test_timestamp_spec_format() {
390 assert_eq!(
391 "timestamp[ms,format=%Y%m%d %H:%M]".parse::<FieldType>(),
392 Ok(FieldType::Timestamp(
393 TimeUnit::Millisecond,
394 Some("%Y%m%d %H:%M".into())
395 ))
396 );
397 assert_eq!(
398 "timestamp[format=%d/%m/%Y]".parse::<FieldType>(),
399 Ok(FieldType::Timestamp(
400 TimeUnit::Microsecond,
401 Some("%d/%m/%Y".into())
402 ))
403 );
404 assert!("timestamp[fortnights]".parse::<FieldType>().is_err());
405 assert!("timestamp[us,color=red]".parse::<FieldType>().is_err());
406 assert!("timestamp[us,format=]".parse::<FieldType>().is_err());
407 }
408
409 #[test]
410 fn decimal128_spec_requires_closing_delimiter_and_valid_scale() {
411 assert_eq!(
412 "decimal128(38)".parse::<FieldType>(),
413 Ok(FieldType::Decimal128(38))
414 );
415 assert!("decimal128(38".parse::<FieldType>().is_err());
416 assert!("decimal128(39)".parse::<FieldType>().is_err());
417 }
418}