1use arrow::datatypes::TimeUnit;
2use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet};
3
4mod predicate;
5pub(crate) use predicate::compare_decimal128;
6pub use predicate::{ArithOp, CompareOp, FilterPredicate, RegexSpec, TrimMode};
7
8#[derive(Clone, Debug, PartialEq, Eq)]
10pub enum FieldType {
11 String,
12 Int64,
13 Float64,
14 Boolean,
15 Dictionary,
16 Date32,
18 Timestamp(TimeUnit, Option<Box<str>>),
21 Decimal128(u8),
23}
24
25impl std::str::FromStr for FieldType {
26 type Err = String;
27 fn from_str(s: &str) -> Result<Self, Self::Err> {
28 match s {
29 "string" => Ok(FieldType::String),
30 "int64" => Ok(FieldType::Int64),
31 "float64" => Ok(FieldType::Float64),
32 "bool" | "boolean" => Ok(FieldType::Boolean),
33 "dictionary" => Ok(FieldType::Dictionary),
34 "date32" => Ok(FieldType::Date32),
35 s if s.starts_with("timestamp") => {
36 parse_timestamp_spec(s).ok_or_else(|| format!("invalid timestamp spec: {s:?}"))
37 }
38 "decimal128" => Ok(FieldType::Decimal128(18)),
39 s if s.starts_with("decimal128(") => {
40 let inner = s
41 .strip_prefix("decimal128(")
42 .and_then(|v| v.strip_suffix(')'))
43 .ok_or_else(|| format!("invalid decimal128 spec: {s:?}"))?;
44 let scale = inner
45 .parse::<u8>()
46 .map_err(|e| format!("invalid decimal128 scale: {e}"))?;
47 if scale > 38 {
48 return Err(format!("decimal128 scale exceeds 38: {scale}"));
49 }
50 Ok(FieldType::Decimal128(scale))
51 }
52 _ => Err(format!("unknown field type: {s:?}")),
53 }
54 }
55}
56
57fn parse_timestamp_spec(s: &str) -> Option<FieldType> {
61 let rest = s.strip_prefix("timestamp")?;
62 if rest.is_empty() {
63 return Some(FieldType::Timestamp(TimeUnit::Microsecond, None));
64 }
65 let inner = rest.strip_prefix('[')?.strip_suffix(']')?;
66 let (unit_str, format) = match inner.split_once(',') {
67 Some((unit, opt)) => {
68 let fmt = opt.strip_prefix("format=")?;
69 if fmt.is_empty() {
70 return None;
71 }
72 (unit, Some(fmt))
73 }
74 None => {
75 if let Some(fmt) = inner.strip_prefix("format=") {
76 if fmt.is_empty() {
77 return None;
78 }
79 ("us", Some(fmt))
80 } else {
81 (inner, None)
82 }
83 }
84 };
85 let unit = match unit_str {
86 "" | "us" | "µs" => TimeUnit::Microsecond,
87 "s" => TimeUnit::Second,
88 "ms" => TimeUnit::Millisecond,
89 "ns" => TimeUnit::Nanosecond,
90 _ => return None,
91 };
92 let format: Option<Box<str>> =
93 format.and_then(|f| if f.is_empty() { None } else { Some(f.into()) });
94 Some(FieldType::Timestamp(unit, format))
95}
96
97#[derive(Clone, Default)]
101pub struct ExecutionPlan {
102 pub field_map: HashMap<String, String>,
104 pub drop_fields: HashSet<String>,
106 pub field_types: HashMap<String, FieldType>,
108 pub dictionary_columns: HashSet<String>,
110 pub filter: Option<FilterPredicate>,
112 pub schema_order: Vec<String>,
116 pub auto_dict: bool,
119 pub dict_threshold: Option<f64>,
122 pub dict_max_size: Option<usize>,
125 pub strict_types: bool,
129 pub max_split_chunks: Option<usize>,
132 pub observer: Option<std::sync::Arc<dyn crate::RowObserver>>,
134}
135
136impl std::fmt::Debug for ExecutionPlan {
137 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
138 f.debug_struct("ExecutionPlan")
139 .field("field_map", &self.field_map)
140 .field("drop_fields", &self.drop_fields)
141 .field("field_types", &self.field_types)
142 .field("dictionary_columns", &self.dictionary_columns)
143 .field("filter", &self.filter)
144 .field("schema_order", &self.schema_order)
145 .field("auto_dict", &self.auto_dict)
146 .field("dict_threshold", &self.dict_threshold)
147 .field("dict_max_size", &self.dict_max_size)
148 .field("strict_types", &self.strict_types)
149 .field("max_split_chunks", &self.max_split_chunks)
150 .field("observer", &self.observer.is_some())
151 .finish()
152 }
153}
154
155impl ExecutionPlan {
156 pub fn new() -> Self {
157 Self::default()
158 }
159
160 pub fn rename(mut self, raw: impl Into<String>, output: impl Into<String>) -> Self {
162 self.field_map.insert(raw.into(), output.into());
163 self
164 }
165
166 pub fn drop(mut self, field: impl Into<String>) -> Self {
168 self.drop_fields.insert(field.into());
169 self
170 }
171
172 pub fn drop_many<I, S>(mut self, fields: I) -> Self
174 where
175 I: IntoIterator<Item = S>,
176 S: Into<String>,
177 {
178 self.drop_fields.extend(fields.into_iter().map(Into::into));
179 self
180 }
181
182 pub fn type_as(mut self, field: impl Into<String>, field_type: FieldType) -> Self {
184 self.field_types.insert(field.into(), field_type);
185 self
186 }
187
188 pub fn dictionary(mut self, field: impl Into<String>) -> Self {
190 self.dictionary_columns.insert(field.into());
191 self
192 }
193
194 pub fn filter_eq(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
196 self.filter = Some(FilterPredicate::Equal {
197 field: field.into(),
198 value: value.into(),
199 });
200 self
201 }
202
203 pub fn filter_ne(mut self, field: impl Into<String>, value: impl Into<String>) -> Self {
205 self.filter = Some(FilterPredicate::NotEqual {
206 field: field.into(),
207 value: value.into(),
208 });
209 self
210 }
211
212 pub fn filter_compare(
214 mut self,
215 field_a: impl Into<String>,
216 op: CompareOp,
217 field_b: impl Into<String>,
218 ) -> Self {
219 self.filter = Some(FilterPredicate::Compare {
220 field_a: field_a.into(),
221 op,
222 field_b: field_b.into(),
223 });
224 self
225 }
226
227 pub fn schema_order<I, S>(mut self, order: I) -> Self
233 where
234 I: IntoIterator<Item = S>,
235 S: Into<String>,
236 {
237 self.schema_order = order.into_iter().map(Into::into).collect();
238 self
239 }
240
241 pub fn with_auto_dict(mut self, yes: bool) -> Self {
244 self.auto_dict = yes;
245 self
246 }
247
248 pub fn with_dict_threshold(mut self, threshold: f64) -> Self {
250 self.dict_threshold = Some(threshold);
251 self
252 }
253
254 pub fn with_dict_max_size(mut self, max_size: usize) -> Self {
256 self.dict_max_size = Some(max_size);
257 self
258 }
259
260 pub fn with_strict_types(mut self, yes: bool) -> Self {
264 self.strict_types = yes;
265 self
266 }
267
268 pub fn with_max_split_chunks(mut self, cap: usize) -> Self {
270 self.max_split_chunks = Some(cap);
271 self
272 }
273
274 pub fn with_observer(mut self, observer: std::sync::Arc<dyn crate::RowObserver>) -> Self {
276 self.observer = Some(observer);
277 self
278 }
279
280 pub fn column_type(&self, name: &str) -> FieldType {
282 if let Some(ft) = self.field_types.get(name) {
283 return ft.clone();
284 }
285 if self.dictionary_columns.contains(name) {
286 return FieldType::Dictionary;
287 }
288 FieldType::String
289 }
290
291 pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str> {
304 let resolved = self.field_map.get(raw).map_or(raw, |s| s.as_str());
305 if self.drop_fields.contains(resolved) {
306 return None;
307 }
308 if !self.schema_order.is_empty()
309 && !self.schema_order.iter().any(|n| n == resolved)
310 && !self.filter_references(resolved)
311 {
312 return None;
313 }
314 Some(resolved)
315 }
316
317 fn filter_references(&self, resolved: &str) -> bool {
321 fn matches(pred: &FilterPredicate, plan: &ExecutionPlan, resolved: &str) -> bool {
322 fn ren<'x>(plan: &'x ExecutionPlan, f: &'x String) -> &'x str {
324 plan.field_map.get(f).map_or(f.as_str(), |s| s.as_str())
325 }
326 match pred {
327 FilterPredicate::Equal { field, .. }
328 | FilterPredicate::NotEqual { field, .. }
329 | FilterPredicate::CompareLiteral { field, .. }
330 | FilterPredicate::StartsWith { field, .. }
331 | FilterPredicate::EndsWith { field, .. }
332 | FilterPredicate::In { field, .. }
333 | FilterPredicate::NotIn { field, .. }
334 | FilterPredicate::NotField { field, .. }
335 | FilterPredicate::ArithmeticCompare { field, .. }
336 | FilterPredicate::Strip { field, .. }
337 | FilterPredicate::Lower { field, .. }
338 | FilterPredicate::Upper { field, .. }
339 | FilterPredicate::Replace { field, .. }
340 | FilterPredicate::Length { field, .. }
341 | FilterPredicate::Contains { field, .. }
342 | FilterPredicate::IsNull { field, .. }
343 | FilterPredicate::IsType { field, .. }
344 | FilterPredicate::Regex { field, .. } => ren(plan, field) == resolved,
345 FilterPredicate::Always(_) => false,
346 FilterPredicate::Compare {
347 field_a, field_b, ..
348 } => ren(plan, field_a) == resolved || ren(plan, field_b) == resolved,
349 FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
350 matches(a, plan, resolved) || matches(b, plan, resolved)
351 }
352 FilterPredicate::Not(inner) => matches(inner, plan, resolved),
353 }
354 }
355 self.filter
356 .as_ref()
357 .is_some_and(|f| matches(f, self, resolved))
358 }
359}
360
361#[cfg(test)]
362mod tests {
363 use super::*;
364
365 #[test]
366 fn test_timestamp_spec_plain() {
367 assert_eq!(
368 "timestamp".parse::<FieldType>(),
369 Ok(FieldType::Timestamp(TimeUnit::Microsecond, None))
370 );
371 assert_eq!(
372 "timestamp[ns]".parse::<FieldType>(),
373 Ok(FieldType::Timestamp(TimeUnit::Nanosecond, None))
374 );
375 }
376
377 #[test]
378 fn test_timestamp_spec_format() {
379 assert_eq!(
380 "timestamp[ms,format=%Y%m%d %H:%M]".parse::<FieldType>(),
381 Ok(FieldType::Timestamp(
382 TimeUnit::Millisecond,
383 Some("%Y%m%d %H:%M".into())
384 ))
385 );
386 assert_eq!(
387 "timestamp[format=%d/%m/%Y]".parse::<FieldType>(),
388 Ok(FieldType::Timestamp(
389 TimeUnit::Microsecond,
390 Some("%d/%m/%Y".into())
391 ))
392 );
393 assert!("timestamp[fortnights]".parse::<FieldType>().is_err());
394 assert!("timestamp[us,color=red]".parse::<FieldType>().is_err());
395 assert!("timestamp[us,format=]".parse::<FieldType>().is_err());
396 }
397
398 #[test]
399 fn decimal128_spec_requires_closing_delimiter_and_valid_scale() {
400 assert_eq!(
401 "decimal128(38)".parse::<FieldType>(),
402 Ok(FieldType::Decimal128(38))
403 );
404 assert!("decimal128(38".parse::<FieldType>().is_err());
405 assert!("decimal128(39)".parse::<FieldType>().is_err());
406 }
407}