1#[cfg(test)]
2use std::borrow::Cow;
3use std::sync::Arc;
4
5#[cfg(feature = "profile")]
6use std::sync::atomic::{AtomicUsize, Ordering};
7
8#[cfg(feature = "profile")]
10pub static RESOLVE_AND_PUT_COUNT: AtomicUsize = AtomicUsize::new(0);
11#[cfg(feature = "profile")]
13pub static PREDICATE_EVALUATIONS: AtomicUsize = AtomicUsize::new(0);
14#[cfg(feature = "profile")]
16pub static PREDICATE_FAILS: AtomicUsize = AtomicUsize::new(0);
17#[cfg(feature = "profile")]
19pub static PREDICATE_UNDECIDED: AtomicUsize = AtomicUsize::new(0);
20#[cfg(feature = "profile")]
22pub static IS_PRED_TRUE: AtomicUsize = AtomicUsize::new(0);
23#[cfg(feature = "profile")]
25pub static IS_PRED_FALSE: AtomicUsize = AtomicUsize::new(0);
26
27use arrow::datatypes::{Field as ArrowField, Schema};
28use arrow::record_batch::RecordBatch;
29use rustc_hash::FxHashMap as HashMap;
30
31use smallvec::SmallVec;
32
33use crate::columnar::ColumnBuilder;
34use crate::decoder::ColumnarSink;
35use crate::engine::predicate::{PredicateState, RowBuffer};
36use crate::plan::{ExecutionPlan, FieldType, FilterPredicate};
37use crate::value::Value;
38use crate::Result;
39
40pub struct TableBuilder {
44 pub(crate) columns: Vec<ColumnBuilder>,
45 pub(crate) field_index: HashMap<String, usize>,
46 pub(crate) column_order: Vec<String>,
47 pub(crate) row_count: usize,
48 pub(crate) estimated_rows: usize,
49 pub(crate) plan: Arc<ExecutionPlan>,
50 pub(crate) row_dirty: Vec<u64>,
55 pub(crate) frozen: Option<std::sync::Arc<crate::schema::FrozenSchema>>,
59 pub(crate) unknown_error: Option<String>,
60 pub(crate) row_buf: Option<Box<RowBuffer>>,
64 pub(crate) cached_wanted_mask: u64,
72 pub(crate) ordinal_expect: Vec<Option<(u32, Vec<u8>)>>,
77 pub(crate) current_ordinal: u32,
80 pub(crate) strict_error: Option<crate::Error>,
84 pub(crate) rows_accepted: usize,
87 pub(crate) rows_rejected: usize,
89}
90
91impl TableBuilder {
92 pub fn new() -> Self {
93 Self {
94 columns: Vec::new(),
95 field_index: HashMap::default(),
96 column_order: Vec::new(),
97 row_count: 0,
98 estimated_rows: 0,
99 plan: Arc::new(ExecutionPlan::new()),
100 row_dirty: Vec::new(),
101 frozen: None,
102 unknown_error: None,
103 row_buf: None,
104 cached_wanted_mask: 0,
105 ordinal_expect: Vec::new(),
106 current_ordinal: 0,
107 strict_error: None,
108 rows_accepted: 0,
109 rows_rejected: 0,
110 }
111 }
112
113 pub fn with_capacity(cap: usize) -> Self {
114 Self {
115 columns: Vec::new(),
116 field_index: HashMap::default(),
117 column_order: Vec::new(),
118 row_count: 0,
119 estimated_rows: cap,
120 plan: Arc::new(ExecutionPlan::new()),
121 row_dirty: Vec::new(),
122 frozen: None,
123 unknown_error: None,
124 row_buf: None,
125 cached_wanted_mask: 0,
126 ordinal_expect: Vec::new(),
127 current_ordinal: 0,
128 strict_error: None,
129 rows_accepted: 0,
130 rows_rejected: 0,
131 }
132 }
133
134 pub fn with_plan(cap: usize, plan: Arc<ExecutionPlan>) -> Self {
135 let mut b = Self {
136 columns: Vec::new(),
137 field_index: HashMap::default(),
138 column_order: Vec::new(),
139 row_count: 0,
140 estimated_rows: cap,
141 plan: Arc::clone(&plan),
142 row_dirty: Vec::new(),
143 frozen: None,
144 unknown_error: None,
145 row_buf: if plan.filter.is_some() {
146 Some(Box::new(RowBuffer {
147 fields: SmallVec::new(),
148 state: PredicateState::Undecided,
149 predicate_mask: SmallVec::new(),
150 direct: false,
151 predicate_ordinal: None,
152 buffer_worthwhile: true,
153 pred_names: SmallVec::new(),
154 }))
155 } else {
156 None
157 },
158 cached_wanted_mask: 0,
159 ordinal_expect: Vec::new(),
160 current_ordinal: 0,
161 strict_error: None,
162 rows_accepted: 0,
163 rows_rejected: 0,
164 };
165 b.predeclare_schema_columns();
166 b.cached_wanted_mask = b.compute_wanted_mask();
167 b
168 }
169
170 fn predeclare_schema_columns(&mut self) {
176 if self.plan.schema_order.is_empty() {
177 return;
178 }
179 let mut names: Vec<String> = Vec::new();
180 for name in &self.plan.schema_order {
181 if !self.plan.drop_fields.contains(name) {
185 names.push(name.clone());
186 }
187 }
188 if let Some(ref filter) = self.plan.filter {
191 let mut pred_names: SmallVec<[String; 4]> = SmallVec::new();
192 Self::collect_predicate_field_names(filter, &self.plan, &mut pred_names);
193 names.extend(pred_names);
194 }
195 for name in names {
196 if self.field_index.contains_key(&name) || self.plan.drop_fields.contains(&name) {
197 continue;
198 }
199 let ty = self.plan.column_type(&name);
200 let col = ColumnBuilder::with_capacity(self.estimated_rows.max(64), &ty);
201 self.columns.push(col);
202 self.field_index
203 .insert(name.clone(), self.columns.len() - 1);
204 self.column_order.push(name);
205 }
206 let words = self.columns.len().div_ceil(64);
207 self.row_dirty.resize(words, 0);
208 }
209
210 fn compute_wanted_mask(&self) -> u64 {
217 if self.plan.schema_order.is_empty() {
218 return 0;
219 }
220 let mut mask = 0u64;
221 for &idx in self.field_index.values() {
222 if idx >= 64 {
223 return 0;
224 }
225 mask |= 1u64 << idx;
226 }
227 mask
228 }
229
230 pub fn ensure_schema(&mut self, schema: &crate::schema::FrozenSchema) -> Result<()> {
236 for (slot, name) in schema.column_names().iter().enumerate() {
237 let name_str: &str = name;
238 if self.field_index.contains_key(name_str) {
239 continue; }
241 let ty = schema.column_types()[slot].clone();
242 let col = ColumnBuilder::with_capacity(self.estimated_rows, &ty);
243 self.columns.push(col);
244 self.field_index
245 .insert(name_str.to_string(), self.columns.len() - 1);
246 self.column_order.push(name_str.to_string());
247 }
248 let words = self.columns.len().div_ceil(64);
250 self.row_dirty.resize(words, 0);
251 self.frozen = Some(std::sync::Arc::new(schema.clone()));
256 Ok(())
257 }
258
259 pub(crate) fn get_column(&self, name: &str) -> Option<&ColumnBuilder> {
261 self.field_index.get(name).map(|&i| &self.columns[i])
262 }
263
264 pub(crate) fn get_column_mut(&mut self, name: &str) -> Option<&mut ColumnBuilder> {
266 if let Some(&i) = self.field_index.get(name) {
267 Some(&mut self.columns[i])
268 } else {
269 None
270 }
271 }
272
273 pub(crate) fn bytes_used(&self) -> usize {
274 self.columns.iter().map(|c| c.bytes_used()).sum::<usize>()
275 + self.column_order.iter().map(|s| s.len()).sum::<usize>()
276 + self.row_dirty.len() * 8
277 }
278
279 pub(crate) fn split_off(&mut self, n: usize) -> Self {
285 assert!(n <= self.row_count, "split_off beyond row_count");
286 assert!(n > 0);
287 let mut other = Self {
288 columns: Vec::with_capacity(self.columns.len()),
289 field_index: self.field_index.clone(),
290 column_order: self.column_order.clone(),
291 row_count: n,
292 estimated_rows: n,
293 plan: Arc::clone(&self.plan),
294 row_dirty: vec![0; self.columns.len().div_ceil(64)],
295 frozen: self.frozen.clone(),
296 unknown_error: None,
297 cached_wanted_mask: self.cached_wanted_mask,
298 row_buf: if self.plan.filter.is_some() {
299 Some(Box::new(RowBuffer {
300 fields: SmallVec::new(),
301 state: PredicateState::Undecided,
302 predicate_mask: self
303 .row_buf
304 .as_ref()
305 .map_or_else(SmallVec::new, |b| b.predicate_mask.clone()),
306 direct: false,
307 predicate_ordinal: None,
308 buffer_worthwhile: true,
309 pred_names: self
310 .row_buf
311 .as_ref()
312 .map_or_else(SmallVec::new, |b| b.pred_names.clone()),
313 }))
314 } else {
315 None
316 },
317 ordinal_expect: Vec::new(),
318 current_ordinal: 0,
319 strict_error: None,
320 rows_accepted: std::mem::take(&mut self.rows_accepted),
323 rows_rejected: std::mem::take(&mut self.rows_rejected),
324 };
325 for (idx, col) in self.columns.iter_mut().enumerate() {
326 let drain = col.split_off(n);
327 other.columns.push(drain);
328 let _ = idx;
330 }
331 self.row_count -= n;
332 self.row_dirty = vec![0; self.columns.len().div_ceil(64)];
334 other
336 }
337
338 pub(crate) fn take_column(&mut self, name: &str) -> Option<ColumnBuilder> {
341 let idx = self.field_index.remove(name)?;
342 let new_len = self.columns.len().div_ceil(64);
347 self.row_dirty.resize(new_len, 0);
348 self.row_dirty.fill(0);
350 let _ = idx; let last = self.columns.len() - 1;
354 let col = if idx == last {
355 self.columns.pop().unwrap()
356 } else {
357 let col = self.columns.swap_remove(idx);
358 let old_last = self.columns.len(); let moved_name = self.field_index.iter().find_map(|(k, &v)| {
363 if v == old_last {
364 Some(k.clone())
365 } else {
366 None
367 }
368 });
369 if let Some(k) = moved_name {
370 self.field_index.insert(k, idx);
371 }
372 col
373 };
374 Some(col)
375 }
376
377 pub fn num_rows(&self) -> usize {
378 self.row_count
379 }
380
381 pub fn num_columns(&self) -> usize {
382 self.column_order.len()
383 }
384
385 pub fn column_names(&self) -> &[String] {
386 &self.column_order
387 }
388
389 pub fn column_diagnostics(&self) -> Vec<(String, usize, usize)> {
391 self.column_order
392 .iter()
393 .map(|name| {
394 let idx = self.field_index[name];
395 let col = &self.columns[idx];
396 (name.clone(), col.bytes_used(), col.capacity_bytes())
397 })
398 .collect()
399 }
400
401 pub fn estimated_rows(&self) -> usize {
403 self.estimated_rows
404 }
405
406 pub fn finish(&mut self) -> Result<RecordBatch> {
410 ColumnarSink::finish(self)
411 }
412
413 pub fn reset(&mut self) {
415 self.columns.clear();
416 self.field_index.clear();
417 self.column_order.clear();
418 self.row_dirty.fill(0);
419 self.row_count = 0;
420 if let Some(ref mut buf) = self.row_buf {
421 buf.fields.clear();
422 buf.state = PredicateState::Undecided;
423 }
424 self.unknown_error = None;
426 self.strict_error = None;
427 self.rows_accepted = 0;
428 self.rows_rejected = 0;
429 }
430
431 pub fn normalize(&mut self) {
434 for b in &mut self.columns {
435 while b.len() > self.row_count {
436 b.pop();
437 }
438 }
439 self.row_dirty.fill(0);
441 }
442
443 pub fn auto_dict_upgrade(&mut self) {
446 if self.plan.auto_dict {
447 let max_ratio = self.plan.dict_threshold.unwrap_or(0.05);
448 let max_size = self.plan.dict_max_size.unwrap_or(256);
449 for b in &mut self.columns {
450 b.try_upgrade_to_dict(512, max_ratio, max_size);
451 }
452 }
453 }
454
455 pub fn sort_columns(&mut self) {
459 if self.plan.schema_order.is_empty() {
460 return;
461 }
462 let order = &self.plan.schema_order;
463 let rank = |name: &String| order.iter().position(|n| n == name).unwrap_or(usize::MAX);
464 self.column_order.sort_by_key(rank);
465 }
466
467 pub(crate) fn schema_insert_index(&self, name: &str) -> usize {
468 let order = &self.plan.schema_order;
469 if order.is_empty() {
470 return self.column_order.len();
471 }
472 let pos = order.iter().position(|n| n == name);
473 match pos {
474 Some(p) => self
475 .column_order
476 .iter()
477 .position(|existing| {
478 order
479 .iter()
480 .position(|n| n == existing)
481 .is_some_and(|ep| ep > p)
482 })
483 .unwrap_or(self.column_order.len()),
484 None => self.column_order.len(),
485 }
486 }
487
488 fn ensure_column_idx(&mut self, name: &str) -> usize {
492 if let Some(&idx) = self.field_index.get(name) {
493 return idx;
494 }
495 if self.frozen.is_some() {
502 if let Some(frozen) = &self.frozen {
503 if frozen.is_exact() {
504 if self.unknown_error.is_none() {
506 self.unknown_error = Some(format!(
507 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
508 name,
509 frozen.num_columns(),
510 frozen.is_exact()
511 ));
512 }
513 return 0;
514 }
515 }
517 }
518 let est = self.estimated_rows.max(64);
519 let col_type = self.plan.column_type(name);
520 let mut b = ColumnBuilder::with_capacity(est, &col_type);
521 let backfill = if self.row_buf.as_ref().is_some_and(|b| b.direct) {
526 self.row_count.saturating_sub(1)
527 } else {
528 self.row_count
529 };
530 for _ in 0..backfill {
531 b.push(None);
532 }
533 let idx = self.columns.len();
534 self.columns.push(b);
535 self.field_index.insert(name.to_owned(), idx);
536 let needed = self.columns.len().div_ceil(64);
538 if self.row_dirty.len() < needed {
539 self.row_dirty.resize(needed, 0);
540 }
541 let order_idx = self.schema_insert_index(name);
542 self.column_order.insert(order_idx, name.to_owned());
543 idx
544 }
545
546 fn record_strict_violation_at(&mut self, idx: usize, value: &Value<'_>) {
552 if !self.plan.strict_types || self.strict_error.is_some() {
553 return;
554 }
555 let Value::Str(s) = value else {
556 return;
557 };
558 let Some(name) = self.column_order.get(idx) else {
559 return;
560 };
561 let Some(ft) = self.plan.field_types.get(name) else {
562 return;
563 };
564 if crate::columnar::parses_as(s, ft) {
565 return;
566 }
567 let truncated: String = s.chars().take(64).collect();
568 self.strict_error = Some(crate::Error::Plan(format!(
569 "strict_types: value {truncated:?} in column {name:?} at row {} \
570 does not parse as declared type {ft:?}",
571 self.row_count
572 )));
573 }
574
575 #[inline]
578 fn notify_put_field(&self, idx: usize, value: &Value<'_>) {
579 if let Some(ref obs) = self.plan.observer {
580 if let Some(name) = self.column_order.get(idx) {
581 obs.on_put_field(self.row_count, name, idx, value);
582 }
583 }
584 }
585
586 #[inline]
589 fn push_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
590 if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
597 if let Some(frozen) = &self.frozen {
598 if frozen.is_exact() {
599 if self.unknown_error.is_none() {
601 self.unknown_error = Some(format!(
602 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
603 resolved_name,
604 frozen.num_columns(),
605 frozen.is_exact()
606 ));
607 }
608 return;
609 }
610 }
612 }
613 let idx = self.ensure_column_idx(resolved_name);
614 self.record_strict_violation_at(idx, &value);
615 self.notify_put_field(idx, &value);
616 if self.row_count == 0 {
618 let ord = self.current_ordinal as usize;
619 if ord >= self.ordinal_expect.len() {
620 self.ordinal_expect.resize_with(ord + 1, || None);
621 }
622 self.ordinal_expect[ord] = Some((idx as u32, resolved_name.as_bytes().to_vec()));
623 }
624 self.current_ordinal += 1;
625 let word = idx / 64;
626 let bit = idx % 64;
627 self.row_dirty[word] |= 1u64 << bit;
628 let b = &mut self.columns[idx];
629 let row_count = self.row_count;
630 if b.len() > row_count {
631 b.pop();
632 }
633 b.push_value(value);
634 }
635
636 #[inline]
639 fn push_field(&mut self, name: &str, value: Value<'_>) {
640 if self.plan.field_map.is_empty()
643 && self.plan.drop_fields.is_empty()
644 && self.plan.schema_order.is_empty()
645 {
646 self.push_field_resolved(name, value);
647 return;
648 }
649 if let Some(idx) = Self::resolve_and_slot(&self.plan, &self.field_index, name) {
652 self.record_strict_violation_at(idx, &value);
653 self.notify_put_field(idx, &value);
654 if self.row_count == 0 {
656 let ord = self.current_ordinal as usize;
657 if ord >= self.ordinal_expect.len() {
658 self.ordinal_expect.resize_with(ord + 1, || None);
659 }
660 let resolved = self.plan.resolve_field(name).unwrap_or(name);
662 self.ordinal_expect[ord] = Some((idx as u32, resolved.as_bytes().to_vec()));
663 }
664 self.current_ordinal += 1;
665 let word = idx / 64;
666 let bit = idx % 64;
667 self.row_dirty[word] |= 1u64 << bit;
668 let b = &mut self.columns[idx];
669 let row_count = self.row_count;
670 if b.len() > row_count {
671 b.pop();
672 }
673 b.push_value(value);
674 } else {
675 if let Some(resolved) = self.plan.resolve_field(name) {
678 let owned = resolved.to_owned();
679 self.push_field_resolved(&owned, value);
680 }
681 }
682 }
683
684 #[inline]
689 fn resolve_and_slot(
690 plan: &ExecutionPlan,
691 field_index: &HashMap<String, usize>,
692 name: &str,
693 ) -> Option<usize> {
694 let resolved = plan.resolve_field(name)?;
695 field_index.get(resolved).copied()
696 }
697
698 #[doc(hidden)]
702 pub fn advance_row(&mut self) {
703 self.row_count += 1;
704 }
705
706 fn finish_row(&mut self) {
711 let ncols = self.columns.len();
713 let full_words = ncols / 64;
715 let rem_bits = ncols % 64;
716 let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
717 && (rem_bits == 0
718 || self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
719 if is_full {
720 self.row_dirty.fill(0);
721 } else {
722 for (i, b) in self.columns.iter_mut().enumerate() {
723 let word = i / 64;
724 let bit = i % 64;
725 let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
726 if !is_set {
727 b.push(None);
728 }
729 }
730 self.row_dirty.fill(0);
731 }
732
733 if let Some(ref filter) = self.plan.filter {
734 if !filter.check(&self.columns, &self.field_index, self.row_count, &self.plan) {
735 for b in &mut self.columns {
736 b.pop();
737 }
738 self.rows_rejected += 1;
739 if let Some(ref obs) = self.plan.observer {
740 obs.on_row_rejected(self.row_count);
741 }
742 return;
743 }
744 }
745
746 self.row_count += 1;
747 self.rows_accepted += 1;
748 if let Some(ref obs) = self.plan.observer {
749 obs.on_row_accepted(self.row_count - 1);
750 }
751 }
752
753 fn build_predicate_mask(&mut self) {
757 let buf = match self.row_buf {
758 Some(ref mut b) => b,
759 None => return,
760 };
761 if !buf.pred_names.is_empty() {
762 return; }
764 let mut names: SmallVec<[String; 4]> = SmallVec::new();
766 if let Some(ref f) = self.plan.filter.clone() {
767 Self::collect_predicate_field_names(f, &self.plan, &mut names);
768 }
769 buf.pred_names.clone_from(&names);
771 let ncols = self.columns.len();
773 let words = ncols.div_ceil(64);
774 buf.predicate_mask.resize(words, 0);
775 for name in &names {
777 if let Some(&slot) = self.field_index.get(name.as_str()) {
778 let word = slot / 64;
779 let bit = slot % 64;
780 if word < buf.predicate_mask.len() {
781 buf.predicate_mask[word] |= 1u64 << bit;
782 }
783 }
784 }
785 }
786
787 #[inline]
792 fn mark_predicate_slot(&mut self, slot: u32) {
793 if self.plan.filter.is_none() {
794 return;
795 }
796 let needs_build = self.row_buf.as_ref().is_some_and(|buf| {
800 let word = slot as usize / 64;
801 let bit = slot as usize % 64;
802 if word < buf.predicate_mask.len() && (buf.predicate_mask[word] >> bit) & 1 == 1 {
803 return false; }
805 buf.pred_names.is_empty() });
807 if needs_build {
808 self.build_predicate_mask();
809 }
810 if let Some(ref buf) = self.row_buf {
812 let word = slot as usize / 64;
813 let bit = slot as usize % 64;
814 if word < buf.predicate_mask.len() && (buf.predicate_mask[word] >> bit) & 1 == 1 {
815 return; }
817 }
818 if let Some(col_name) = self.column_order.get(slot as usize) {
821 let is_pred = self
822 .row_buf
823 .as_ref()
824 .is_some_and(|buf| buf.pred_names.iter().any(|n| n == col_name));
825 if is_pred {
826 let buf = self.row_buf.as_mut().unwrap();
827 let word = slot as usize / 64;
828 let bit = slot as usize % 64;
829 if word >= buf.predicate_mask.len() {
830 buf.predicate_mask.resize(word + 1, 0);
831 }
832 buf.predicate_mask[word] |= 1u64 << bit;
833 }
834 }
835 }
836
837 fn collect_predicate_field_names(
838 pred: &FilterPredicate,
839 plan: &ExecutionPlan,
840 names: &mut SmallVec<[String; 4]>,
841 ) {
842 match pred {
843 FilterPredicate::Equal { field, .. }
844 | FilterPredicate::NotEqual { field, .. }
845 | FilterPredicate::CompareLiteral { field, .. }
846 | FilterPredicate::StartsWith { field, .. }
847 | FilterPredicate::EndsWith { field, .. }
848 | FilterPredicate::In { field, .. }
849 | FilterPredicate::NotIn { field, .. }
850 | FilterPredicate::NotField { field, .. }
851 | FilterPredicate::ArithmeticCompare { field, .. }
852 | FilterPredicate::Strip { field, .. }
853 | FilterPredicate::Lower { field, .. }
854 | FilterPredicate::Upper { field, .. }
855 | FilterPredicate::Replace { field, .. }
856 | FilterPredicate::Length { field, .. }
857 | FilterPredicate::Contains { field, .. }
858 | FilterPredicate::IsNull { field, .. }
859 | FilterPredicate::IsType { field, .. }
860 | FilterPredicate::Regex { field, .. } => {
861 let resolved = plan.resolve_field(field).unwrap_or(field);
862 names.push(resolved.to_string());
863 }
864 FilterPredicate::Always(_) => {}
865 FilterPredicate::Compare {
866 field_a, field_b, ..
867 } => {
868 for f in [field_a, field_b] {
869 let resolved = plan.resolve_field(f).unwrap_or(f);
870 names.push(resolved.to_string());
871 }
872 }
873 FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
874 Self::collect_predicate_field_names(a, plan, names);
875 Self::collect_predicate_field_names(b, plan, names);
876 }
877 FilterPredicate::Not(inner) => Self::collect_predicate_field_names(inner, plan, names),
878 }
879 }
880
881 #[inline]
882 fn is_predicate_slot(&self, slot: u32) -> bool {
883 self.row_buf.as_ref().is_some_and(|b| {
884 let word = slot as usize / 64;
885 let bit = slot as usize % 64;
886 word < b.predicate_mask.len() && (b.predicate_mask[word] >> bit) & 1 == 1
887 })
888 }
889
890 fn get_buffered_value(&self, field: &str) -> Option<&Value<'static>> {
891 let buf = self.row_buf.as_ref()?;
892 let resolved = self.plan.resolve_field(field)?;
893 let slot = *self.field_index.get(resolved)? as u32;
894 for (s, v) in buf.fields.iter().rev() {
895 if *s == slot {
896 return Some(v);
897 }
898 }
899 None
900 }
901
902 fn get_buffered_str(&self, field: &str) -> Option<String> {
903 match self.get_buffered_value(field) {
904 Some(Value::Str(s)) => Some(s.to_string()),
905 Some(Value::Int64(i)) => Some(i.to_string()),
906 Some(Value::Float64(f)) => Some(f.to_string()),
907 Some(Value::Bool(b)) => Some(b.to_string()),
908 Some(v) => Some(format!("{:?}", v)),
909 None => None,
910 }
911 }
912
913 fn evaluate_predicate_state(&self) -> PredicateState {
914 let Some(ref pred) = self.plan.filter else {
915 return PredicateState::Pass;
916 };
917 #[cfg(feature = "profile")]
918 {
919 crate::engine::PREDICATE_EVALUATIONS.fetch_add(1, Ordering::Relaxed);
920 }
921 let result = Self::eval_predicate(pred, self);
922 #[cfg(feature = "profile")]
923 match result {
924 PredicateState::Fail => {
925 crate::engine::PREDICATE_FAILS.fetch_add(1, Ordering::Relaxed);
926 }
927 PredicateState::Undecided => {
928 crate::engine::PREDICATE_UNDECIDED.fetch_add(1, Ordering::Relaxed);
929 }
930 _ => {}
931 }
932 result
933 }
934
935 fn eval_predicate(pred: &FilterPredicate, tb: &TableBuilder) -> PredicateState {
936 match pred {
937 FilterPredicate::Equal { field, value } => match tb.get_buffered_str(field) {
938 Some(actual) => {
939 if actual == *value {
940 PredicateState::Pass
941 } else {
942 PredicateState::Fail
943 }
944 }
945 None => PredicateState::Undecided,
946 },
947 FilterPredicate::NotEqual { field, value } => match tb.get_buffered_str(field) {
948 Some(actual) => {
949 if actual != *value {
950 PredicateState::Pass
951 } else {
952 PredicateState::Fail
953 }
954 }
955 None => PredicateState::Undecided,
956 },
957 FilterPredicate::Compare {
958 field_a,
959 op,
960 field_b,
961 } => {
962 let normalize = |field: &str, value: &Value<'static>| {
963 let field = tb.plan.resolve_field(field).unwrap_or(field);
964 match tb.plan.column_type(field) {
965 FieldType::Int64 => match value.as_str() {
966 Some(s) => lexical::parse::<i64, _>(s.as_bytes())
967 .ok()
968 .map(Value::Int64),
969 _ => Some(value.clone()),
970 },
971 FieldType::Float64 => match value.as_str() {
972 Some(s) => lexical::parse::<f64, _>(s.as_bytes())
973 .ok()
974 .map(Value::Float64),
975 _ => Some(value.clone()),
976 },
977 FieldType::Boolean => match value.as_str() {
978 Some(s) => s.parse::<bool>().ok().map(Value::Bool),
979 _ => Some(value.clone()),
980 },
981 FieldType::Date32 => match value.as_str() {
982 Some(s) => crate::columnar::parse_date32(s).map(Value::Date32),
983 _ => Some(value.clone()),
984 },
985 FieldType::Timestamp(unit, fmt) => match value.as_str() {
986 Some(s) => crate::columnar::parse_timestamp(s, unit, fmt.as_deref())
987 .map(Value::Timestamp),
988 _ => Some(value.clone()),
989 },
990 FieldType::String | FieldType::Dictionary => Some(value.clone()),
991 FieldType::Decimal128(_) => value.as_str().and_then(|s| {
992 s.trim()
993 .parse::<i128>()
994 .ok()
995 .map(|n| Value::Int64(n as i64))
996 }),
997 }
998 };
999 let va = tb
1000 .get_buffered_value(field_a)
1001 .and_then(|value| normalize(field_a, value));
1002 let vb = tb
1003 .get_buffered_value(field_b)
1004 .and_then(|value| normalize(field_b, value));
1005 match (va, vb) {
1006 (Some(ref a), Some(ref b)) => {
1007 let ord = match (a, b) {
1008 (crate::value::Value::Int64(ai), crate::value::Value::Int64(bi)) => {
1009 Some(ai.cmp(bi))
1010 }
1011 (
1012 crate::value::Value::Float64(af),
1013 crate::value::Value::Float64(bf),
1014 ) => af.partial_cmp(bf),
1015 (crate::value::Value::Int64(ai), crate::value::Value::Float64(bf)) => {
1016 (*ai as f64).partial_cmp(bf)
1017 }
1018 (crate::value::Value::Float64(af), crate::value::Value::Int64(bi)) => {
1019 af.partial_cmp(&(*bi as f64))
1020 }
1021 (crate::value::Value::Str(a), crate::value::Value::Str(b)) => {
1022 let resolved_a =
1023 tb.plan.resolve_field(field_a).unwrap_or(field_a.as_str());
1024 let resolved_b =
1025 tb.plan.resolve_field(field_b).unwrap_or(field_b.as_str());
1026 let type_a = tb.plan.field_types.get(resolved_a);
1027 let type_b = tb.plan.field_types.get(resolved_b);
1028 match (type_a, type_b) {
1029 (
1030 Some(crate::plan::FieldType::Int64),
1031 Some(crate::plan::FieldType::Int64),
1032 ) => {
1033 let ai: i64 =
1034 lexical::parse(a.as_bytes()).ok().unwrap_or(0);
1035 let bi: i64 =
1036 lexical::parse(b.as_bytes()).ok().unwrap_or(0);
1037 Some(ai.cmp(&bi))
1038 }
1039 (
1040 Some(crate::plan::FieldType::Float64),
1041 Some(crate::plan::FieldType::Float64),
1042 ) => {
1043 let af: f64 =
1044 lexical::parse(a.as_bytes()).ok().unwrap_or(0.0);
1045 let bf: f64 =
1046 lexical::parse(b.as_bytes()).ok().unwrap_or(0.0);
1047 af.partial_cmp(&bf)
1048 }
1049 (
1050 Some(crate::plan::FieldType::Int64),
1051 Some(crate::plan::FieldType::Float64),
1052 ) => {
1053 let ai: f64 = lexical::parse::<i64, _>(a.as_bytes())
1054 .ok()
1055 .unwrap_or(0)
1056 as f64;
1057 let bf: f64 =
1058 lexical::parse(b.as_bytes()).ok().unwrap_or(0.0);
1059 ai.partial_cmp(&bf)
1060 }
1061 (
1062 Some(crate::plan::FieldType::Float64),
1063 Some(crate::plan::FieldType::Int64),
1064 ) => {
1065 let af: f64 =
1066 lexical::parse(a.as_bytes()).ok().unwrap_or(0.0);
1067 let bi: f64 = lexical::parse::<i64, _>(b.as_bytes())
1068 .ok()
1069 .unwrap_or(0)
1070 as f64;
1071 af.partial_cmp(&bi)
1072 }
1073 (Some(_), None) | (None, Some(_)) => None,
1076 (None, None) => Some(a.cmp(b)),
1078 (
1080 Some(crate::plan::FieldType::Timestamp(ua, fa)),
1081 Some(crate::plan::FieldType::Timestamp(ub, fb)),
1082 ) => {
1083 let ta = crate::columnar::parse_timestamp(
1084 a.as_ref(),
1085 *ua,
1086 fa.as_deref(),
1087 );
1088 let tb_val = crate::columnar::parse_timestamp(
1089 b.as_ref(),
1090 *ub,
1091 fb.as_deref(),
1092 );
1093 match (ta, tb_val) {
1094 (Some(ai), Some(bi)) => Some(ai.cmp(&bi)),
1095 _ => None,
1096 }
1097 }
1098 (
1100 Some(crate::plan::FieldType::Date32),
1101 Some(crate::plan::FieldType::Date32),
1102 ) => {
1103 let da = crate::columnar::parse_date32(a.as_ref());
1104 let db = crate::columnar::parse_date32(b.as_ref());
1105 match (da, db) {
1106 (Some(ai), Some(bi)) => Some(ai.cmp(&bi)),
1107 _ => None,
1108 }
1109 }
1110 _ => None,
1112 }
1113 }
1114 (crate::value::Value::Bool(a), crate::value::Value::Bool(b)) => {
1115 Some(a.cmp(b))
1116 }
1117 _ => {
1118 let av = format!("{:?}", a);
1119 let bv = format!("{:?}", b);
1120 Some(av.cmp(&bv))
1121 }
1122 };
1123 let pass = ord.is_some_and(|ord| match op {
1124 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1125 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1126 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1127 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1128 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1129 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1130 });
1131 if pass {
1132 PredicateState::Pass
1133 } else {
1134 PredicateState::Fail
1135 }
1136 }
1137 _ => PredicateState::Undecided,
1138 }
1139 }
1140 FilterPredicate::CompareLiteral { field, op, value } => {
1141 let va = tb.get_buffered_value(field).and_then(|v| {
1142 let resolved = tb.plan.resolve_field(field).unwrap_or(field);
1143 match tb.plan.column_type(resolved) {
1144 crate::plan::FieldType::Int64 => match v.as_str() {
1145 Some(s) => lexical::parse::<i64, _>(s.as_bytes())
1146 .ok()
1147 .map(crate::value::Value::Int64),
1148 _ => Some(v.clone()),
1149 },
1150 crate::plan::FieldType::Float64 => match v.as_str() {
1151 Some(s) => lexical::parse::<f64, _>(s.as_bytes())
1152 .ok()
1153 .map(crate::value::Value::Float64),
1154 _ => Some(v.clone()),
1155 },
1156 crate::plan::FieldType::Boolean => match v.as_str() {
1157 Some(s) => s.parse::<bool>().ok().map(crate::value::Value::Bool),
1158 _ => Some(v.clone()),
1159 },
1160 _ => Some(v.clone()),
1161 }
1162 });
1163 let vb = match tb
1164 .plan
1165 .column_type(tb.plan.resolve_field(field).unwrap_or(field))
1166 {
1167 crate::plan::FieldType::Int64 => lexical::parse::<i64, _>(value.as_bytes())
1168 .ok()
1169 .map(crate::value::Value::Int64),
1170 crate::plan::FieldType::Float64 => lexical::parse::<f64, _>(value.as_bytes())
1171 .ok()
1172 .map(crate::value::Value::Float64),
1173 crate::plan::FieldType::Boolean => {
1174 value.parse::<bool>().ok().map(crate::value::Value::Bool)
1175 }
1176 _ => Some(crate::value::Value::Str(std::borrow::Cow::Borrowed(value))),
1177 };
1178 match (va, vb) {
1179 (Some(ref a), Some(ref b)) => {
1180 let ord = match (a, b) {
1181 (crate::value::Value::Int64(ai), crate::value::Value::Int64(bi)) => {
1182 Some(ai.cmp(bi))
1183 }
1184 (
1185 crate::value::Value::Float64(af),
1186 crate::value::Value::Float64(bf),
1187 ) => af.partial_cmp(bf),
1188 (crate::value::Value::Int64(ai), crate::value::Value::Float64(bf)) => {
1189 (*ai as f64).partial_cmp(bf)
1190 }
1191 (crate::value::Value::Float64(af), crate::value::Value::Int64(bi)) => {
1192 af.partial_cmp(&(*bi as f64))
1193 }
1194 (crate::value::Value::Str(a), crate::value::Value::Str(b)) => {
1195 Some(a.cmp(b))
1196 }
1197 (crate::value::Value::Bool(a), crate::value::Value::Bool(b)) => {
1198 Some(a.cmp(b))
1199 }
1200 _ => None,
1201 };
1202 let pass = ord.is_some_and(|ord| match op {
1203 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1204 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1205 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1206 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1207 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1208 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1209 });
1210 if pass {
1211 PredicateState::Pass
1212 } else {
1213 PredicateState::Fail
1214 }
1215 }
1216 _ => PredicateState::Undecided,
1217 }
1218 }
1219 FilterPredicate::StartsWith { field, value } => match tb.get_buffered_str(field) {
1220 Some(actual) => {
1221 if actual.starts_with(value.as_str()) {
1222 PredicateState::Pass
1223 } else {
1224 PredicateState::Fail
1225 }
1226 }
1227 None => PredicateState::Undecided,
1228 },
1229 FilterPredicate::EndsWith { field, value } => match tb.get_buffered_str(field) {
1230 Some(actual) => {
1231 if actual.ends_with(value.as_str()) {
1232 PredicateState::Pass
1233 } else {
1234 PredicateState::Fail
1235 }
1236 }
1237 None => PredicateState::Undecided,
1238 },
1239 FilterPredicate::In { field, values } => match tb.get_buffered_str(field) {
1240 Some(actual) => {
1241 if values.contains(&actual) {
1242 PredicateState::Pass
1243 } else {
1244 PredicateState::Fail
1245 }
1246 }
1247 None => PredicateState::Fail,
1248 },
1249 FilterPredicate::NotIn { field, values } => match tb.get_buffered_str(field) {
1250 Some(actual) => {
1251 if !values.contains(&actual) {
1252 PredicateState::Pass
1253 } else {
1254 PredicateState::Fail
1255 }
1256 }
1257 None => PredicateState::Pass,
1258 },
1259 FilterPredicate::Always(keep) => {
1260 if *keep {
1261 PredicateState::Pass
1262 } else {
1263 PredicateState::Fail
1264 }
1265 }
1266 FilterPredicate::NotField { field } => match tb.get_buffered_str(field) {
1267 Some(actual) => {
1268 if actual.is_empty() {
1269 PredicateState::Pass
1270 } else {
1271 PredicateState::Fail
1272 }
1273 }
1274 None => PredicateState::Pass,
1275 },
1276 FilterPredicate::ArithmeticCompare {
1277 field,
1278 arith_op,
1279 arith_value,
1280 cmp_op,
1281 cmp_value,
1282 } => match tb.get_buffered_str(field) {
1283 Some(actual) => {
1284 let field_f64 = actual.parse::<f64>().unwrap_or(0.0);
1285 let result = match arith_op {
1286 crate::plan::ArithOp::Add => field_f64 + arith_value,
1287 crate::plan::ArithOp::Sub => field_f64 - arith_value,
1288 crate::plan::ArithOp::Mul => field_f64 * arith_value,
1289 crate::plan::ArithOp::Div => field_f64 / arith_value,
1290 };
1291 let cmp_f64 = cmp_value.parse::<f64>().unwrap_or(0.0);
1292 match result.partial_cmp(&cmp_f64) {
1293 Some(ord) => {
1294 let pass = match cmp_op {
1295 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1296 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1297 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1298 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1299 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1300 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1301 };
1302 if pass {
1303 PredicateState::Pass
1304 } else {
1305 PredicateState::Fail
1306 }
1307 }
1308 None => PredicateState::Fail,
1309 }
1310 }
1311 None => PredicateState::Undecided,
1312 },
1313 FilterPredicate::Strip { field, op, value } => match tb.get_buffered_str(field) {
1314 Some(actual) => {
1315 let transformed = actual.trim().to_string();
1316 match transformed.as_str().partial_cmp(value.as_str()) {
1317 Some(ord) => {
1318 let pass = match op {
1319 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1320 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1321 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1322 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1323 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1324 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1325 };
1326 if pass {
1327 PredicateState::Pass
1328 } else {
1329 PredicateState::Fail
1330 }
1331 }
1332 None => PredicateState::Fail,
1333 }
1334 }
1335 None => PredicateState::Undecided,
1336 },
1337 FilterPredicate::Lower { field, op, value } => match tb.get_buffered_str(field) {
1338 Some(actual) => {
1339 let transformed = actual.to_lowercase();
1340 match transformed.as_str().partial_cmp(value.as_str()) {
1341 Some(ord) => {
1342 let pass = match op {
1343 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1344 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1345 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1346 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1347 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1348 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1349 };
1350 if pass {
1351 PredicateState::Pass
1352 } else {
1353 PredicateState::Fail
1354 }
1355 }
1356 None => PredicateState::Fail,
1357 }
1358 }
1359 None => PredicateState::Undecided,
1360 },
1361 FilterPredicate::Upper { field, op, value } => match tb.get_buffered_str(field) {
1362 Some(actual) => {
1363 let transformed = actual.to_uppercase();
1364 match transformed.as_str().partial_cmp(value.as_str()) {
1365 Some(ord) => {
1366 let pass = match op {
1367 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1368 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1369 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1370 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1371 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1372 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1373 };
1374 if pass {
1375 PredicateState::Pass
1376 } else {
1377 PredicateState::Fail
1378 }
1379 }
1380 None => PredicateState::Fail,
1381 }
1382 }
1383 None => PredicateState::Undecided,
1384 },
1385 FilterPredicate::Replace {
1386 field,
1387 old,
1388 new,
1389 op,
1390 value,
1391 } => match tb.get_buffered_str(field) {
1392 Some(actual) => {
1393 let transformed = actual.replace(old.as_str(), new.as_str());
1394 match transformed.as_str().partial_cmp(value.as_str()) {
1395 Some(ord) => {
1396 let pass = match op {
1397 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1398 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1399 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1400 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1401 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1402 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1403 };
1404 if pass {
1405 PredicateState::Pass
1406 } else {
1407 PredicateState::Fail
1408 }
1409 }
1410 None => PredicateState::Fail,
1411 }
1412 }
1413 None => PredicateState::Undecided,
1414 },
1415 FilterPredicate::Length { field, op, value } => match tb.get_buffered_str(field) {
1416 Some(actual) => {
1417 let len = actual.len() as f64;
1418 let cmp_val = value.parse::<f64>().unwrap_or(0.0);
1419 match len.partial_cmp(&cmp_val) {
1420 Some(ord) => {
1421 let pass = match op {
1422 crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1423 crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1424 crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1425 crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1426 crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1427 crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1428 };
1429 if pass {
1430 PredicateState::Pass
1431 } else {
1432 PredicateState::Fail
1433 }
1434 }
1435 None => PredicateState::Fail,
1436 }
1437 }
1438 None => PredicateState::Undecided,
1439 },
1440 FilterPredicate::Contains { field, value } => match tb.get_buffered_str(field) {
1441 Some(actual) => {
1442 if actual.contains(value.as_str()) {
1443 PredicateState::Pass
1444 } else {
1445 PredicateState::Fail
1446 }
1447 }
1448 None => PredicateState::Undecided,
1449 },
1450 FilterPredicate::Regex { field, re } => match tb.get_buffered_str(field) {
1451 Some(actual) => {
1452 if re.compiled.is_match(actual.as_ref()) {
1453 PredicateState::Pass
1454 } else {
1455 PredicateState::Fail
1456 }
1457 }
1458 None => PredicateState::Undecided,
1459 },
1460 FilterPredicate::IsNull { field } => match tb.get_buffered_str(field) {
1461 Some(_) => PredicateState::Fail,
1462 None => PredicateState::Pass,
1463 },
1464 FilterPredicate::IsType { field, field_type } => {
1465 let resolved = tb.plan.resolve_field(field).unwrap_or(field);
1467 let declared = tb.plan.column_type(resolved);
1468 if declared == *field_type {
1469 if tb.get_buffered_value(field).is_some() {
1471 return PredicateState::Pass;
1472 }
1473 }
1474 if let Some(val) = tb.get_buffered_value(field) {
1476 match field_type {
1477 crate::plan::FieldType::Int64 => {
1478 if let Some(s) = val.as_str() {
1479 if s.parse::<i64>().is_ok() {
1480 return PredicateState::Pass;
1481 }
1482 }
1483 }
1484 crate::plan::FieldType::Float64 => {
1485 if let Some(s) = val.as_str() {
1486 if s.parse::<f64>().is_ok() {
1487 return PredicateState::Pass;
1488 }
1489 }
1490 }
1491 crate::plan::FieldType::Boolean => {
1492 if let Some(s) = val.as_str() {
1493 if matches!(
1494 s.to_lowercase().as_str(),
1495 "true" | "false" | "1" | "0" | "yes" | "no"
1496 ) {
1497 return PredicateState::Pass;
1498 }
1499 }
1500 }
1501 crate::plan::FieldType::String | crate::plan::FieldType::Dictionary => {
1502 return PredicateState::Pass;
1503 }
1504 _ => {}
1505 }
1506 }
1507 PredicateState::Fail
1508 }
1509 FilterPredicate::And(a, b) => {
1510 let sa = Self::eval_predicate(a, tb);
1511 let sb = Self::eval_predicate(b, tb);
1512 match (sa, sb) {
1513 (PredicateState::Fail, _) | (_, PredicateState::Fail) => PredicateState::Fail,
1514 (PredicateState::Pass, PredicateState::Pass) => PredicateState::Pass,
1515 _ => PredicateState::Undecided,
1516 }
1517 }
1518 FilterPredicate::Or(a, b) => {
1519 let sa = Self::eval_predicate(a, tb);
1520 let sb = Self::eval_predicate(b, tb);
1521 match (sa, sb) {
1522 (PredicateState::Pass, _) | (_, PredicateState::Pass) => PredicateState::Pass,
1523 (PredicateState::Fail, PredicateState::Fail) => PredicateState::Fail,
1524 _ => PredicateState::Undecided,
1525 }
1526 }
1527 FilterPredicate::Not(inner) => match Self::eval_predicate(inner, tb) {
1528 PredicateState::Pass => PredicateState::Fail,
1529 PredicateState::Fail => PredicateState::Pass,
1530 PredicateState::Undecided => PredicateState::Undecided,
1531 },
1532 }
1533 }
1534
1535 fn drain_buffered(&mut self, pass: bool) {
1536 let observer = self.plan.observer.clone();
1537 let buf = match self.row_buf {
1538 Some(ref mut b) => b,
1539 None => return,
1540 };
1541 if pass {
1542 let ncols = self.columns.len();
1546 let words = ncols.div_ceil(64);
1547 if self.row_dirty.len() < words {
1548 self.row_dirty.resize(words, 0);
1549 }
1550 let mut seen: u64 = 0;
1551 let mut seen_extra: SmallVec<[u64; 1]> = SmallVec::new();
1552 if words > 1 {
1553 seen_extra.resize(words - 1, 0);
1554 }
1555 for &(slot, ref val) in buf.fields.iter().rev() {
1556 let word = slot as usize / 64;
1557 let bit = slot as usize % 64;
1558 let mask = 1u64 << bit;
1559 let already = if word == 0 {
1560 seen & mask != 0
1561 } else {
1562 seen_extra.get(word - 1).is_some_and(|w| w & mask != 0)
1563 };
1564 if already {
1565 continue;
1566 }
1567 if word == 0 {
1568 seen |= mask;
1569 } else if let Some(w) = seen_extra.get_mut(word - 1) {
1570 *w |= mask;
1571 }
1572 self.row_dirty[word] |= mask;
1573 let b = &mut self.columns[slot as usize];
1574 if b.len() > self.row_count {
1575 b.pop();
1576 }
1577 b.push_value(val.clone());
1578 if let Some(ref obs) = observer {
1581 if let Some(name) = self.column_order.get(slot as usize) {
1582 obs.on_put_field(self.row_count, name, slot as usize, val);
1583 }
1584 }
1585 }
1586 if !buf.direct {
1591 let ncols = self.columns.len();
1592 let full_words = ncols / 64;
1593 let rem_bits = ncols % 64;
1594 let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
1595 && (rem_bits == 0
1596 || self.row_dirty.get(full_words).copied().unwrap_or(0)
1597 == (1u64 << rem_bits) - 1);
1598 if is_full {
1599 self.row_dirty.fill(0);
1600 } else {
1601 for (i, b) in self.columns.iter_mut().enumerate() {
1602 let word = i / 64;
1603 let bit = i % 64;
1604 let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
1605 if !is_set {
1606 b.push(None);
1607 }
1608 }
1609 self.row_dirty.fill(0);
1610 }
1611 } else {
1612 }
1615 self.row_count += 1;
1616 self.rows_accepted += 1;
1617 if let Some(ref obs) = observer {
1618 obs.on_row_accepted(self.row_count - 1);
1619 }
1620 } else {
1621 buf.fields.clear();
1623 self.row_dirty.fill(0);
1624 self.rows_rejected += 1;
1625 if let Some(ref obs) = observer {
1626 obs.on_row_rejected(self.row_count);
1627 }
1628 }
1629 if !buf.fields.is_empty() {
1631 buf.fields.clear();
1632 }
1633 buf.state = PredicateState::Undecided;
1634 }
1635
1636 fn evaluate_against_null(&self) -> PredicateState {
1637 let Some(ref pred) = self.plan.filter else {
1642 return PredicateState::Pass;
1643 };
1644 Self::eval_predicate_with_null(pred, self)
1645 }
1646
1647 fn eval_predicate_with_null(pred: &FilterPredicate, tb: &TableBuilder) -> PredicateState {
1648 match pred {
1649 FilterPredicate::Equal { field, value } => match tb.get_buffered_str(field) {
1650 Some(actual) => {
1651 if actual == *value {
1652 PredicateState::Pass
1653 } else {
1654 PredicateState::Fail
1655 }
1656 }
1657 None => PredicateState::Fail, },
1659 FilterPredicate::NotEqual { field, value } => match tb.get_buffered_str(field) {
1660 Some(actual) => {
1661 if actual != *value {
1662 PredicateState::Pass
1663 } else {
1664 PredicateState::Fail
1665 }
1666 }
1667 None => PredicateState::Pass, },
1669 FilterPredicate::Compare { .. }
1670 | FilterPredicate::CompareLiteral { .. }
1671 | FilterPredicate::StartsWith { .. }
1672 | FilterPredicate::EndsWith { .. }
1673 | FilterPredicate::In { .. }
1674 | FilterPredicate::NotIn { .. }
1675 | FilterPredicate::NotField { .. }
1676 | FilterPredicate::ArithmeticCompare { .. }
1677 | FilterPredicate::Strip { .. }
1678 | FilterPredicate::Lower { .. }
1679 | FilterPredicate::Upper { .. }
1680 | FilterPredicate::Replace { .. }
1681 | FilterPredicate::Length { .. }
1682 | FilterPredicate::Contains { .. }
1683 | FilterPredicate::IsNull { .. }
1684 | FilterPredicate::IsType { .. }
1685 | FilterPredicate::Regex { .. } => match Self::eval_predicate(pred, tb) {
1686 PredicateState::Undecided => PredicateState::Fail,
1687 other => other,
1688 },
1689 FilterPredicate::Always(keep) => {
1690 if *keep {
1691 PredicateState::Pass
1692 } else {
1693 PredicateState::Fail
1694 }
1695 }
1696 FilterPredicate::And(a, b) => {
1697 let sa = Self::eval_predicate_with_null(a, tb);
1698 let sb = Self::eval_predicate_with_null(b, tb);
1699 match (sa, sb) {
1700 (PredicateState::Fail, _) | (_, PredicateState::Fail) => PredicateState::Fail,
1701 (PredicateState::Pass, PredicateState::Pass) => PredicateState::Pass,
1702 _ => PredicateState::Undecided, }
1704 }
1705 FilterPredicate::Or(a, b) => {
1706 let sa = Self::eval_predicate_with_null(a, tb);
1707 let sb = Self::eval_predicate_with_null(b, tb);
1708 match (sa, sb) {
1709 (PredicateState::Pass, _) | (_, PredicateState::Pass) => PredicateState::Pass,
1710 (PredicateState::Fail, PredicateState::Fail) => PredicateState::Fail,
1711 _ => PredicateState::Fail,
1712 }
1713 }
1714 FilterPredicate::Not(inner) => match Self::eval_predicate_with_null(inner, tb) {
1715 PredicateState::Pass => PredicateState::Fail,
1716 PredicateState::Fail => PredicateState::Pass,
1717 PredicateState::Undecided => PredicateState::Fail, },
1719 }
1720 }
1721
1722 fn null_fill_missing(&mut self) {
1725 let ncols = self.columns.len();
1726 let full_words = ncols / 64;
1727 let rem_bits = ncols % 64;
1728 let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
1729 && (rem_bits == 0
1730 || self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
1731 if is_full {
1732 self.row_dirty.fill(0);
1733 } else {
1734 for (i, b) in self.columns.iter_mut().enumerate() {
1735 let word = i / 64;
1736 let bit = i % 64;
1737 let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
1738 if !is_set {
1739 b.push(None);
1740 }
1741 }
1742 self.row_dirty.fill(0);
1743 }
1744 }
1747}
1748
1749impl Default for TableBuilder {
1750 fn default() -> Self {
1751 Self::new()
1752 }
1753}
1754
1755impl ColumnarSink for TableBuilder {
1756 #[inline]
1757 fn begin_row(&mut self) {
1758 if let Some(ref obs) = self.plan.observer {
1759 obs.on_begin_row(self.row_count);
1760 }
1761 self.current_ordinal = 0;
1762 if let Some(ref mut buf) = self.row_buf {
1763 if buf.predicate_ordinal.is_some() && buf.buffer_worthwhile && self.row_count > 0 {
1769 let ncols = self
1772 .frozen
1773 .as_ref()
1774 .map(|f| f.num_columns() as u32)
1775 .or_else(|| {
1776 (!self.plan.schema_order.is_empty())
1777 .then(|| self.plan.schema_order.len() as u32)
1778 })
1779 .unwrap_or(self.columns.len() as u32);
1780 if let Some(ordinal) = buf.predicate_ordinal {
1781 if ordinal >= ncols * 4 / 5 {
1782 buf.buffer_worthwhile = false;
1783 }
1784 }
1785 }
1786 buf.fields.clear();
1787 buf.state = PredicateState::Undecided;
1788 buf.direct = false;
1789 }
1790 }
1791
1792 #[inline]
1793 fn put_field(&mut self, name: &str, value: Value<'_>) {
1794 if self.plan.filter.is_none() {
1795 self.push_field(name, value);
1796 return;
1797 }
1798 let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
1800 if !buf_worthwhile {
1801 self.push_field(name, value);
1802 return;
1803 }
1804 let resolved = match self.plan.resolve_field(name) {
1806 Some(r) => r.to_owned(),
1807 None => return,
1808 };
1809 if self.frozen.is_some()
1811 && !self.field_index.contains_key(&resolved)
1812 && self.unknown_error.is_none()
1813 {
1814 if let Some(ref frozen) = self.frozen {
1815 if frozen.is_exact()
1816 && !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
1817 {
1818 self.unknown_error = Some(format!(
1819 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1820 resolved,
1821 frozen.num_columns(),
1822 frozen.is_exact()
1823 ));
1824 return;
1825 }
1826 }
1827 }
1828 let slot = self.ensure_column_idx(&resolved) as u32;
1829 self.record_strict_violation_at(slot as usize, &value);
1830 self.mark_predicate_slot(slot);
1831 let val_static: Value<'static> = value.into_static();
1832 let is_pred = self.is_predicate_slot(slot);
1833 #[cfg(feature = "profile")]
1834 if is_pred {
1835 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1836 } else {
1837 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1838 }
1839 if let Some(ref mut buf) = self.row_buf {
1841 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1842 buf.fields[pos].1 = val_static;
1843 } else {
1844 buf.fields.push((slot, val_static));
1845 }
1846 }
1847 if is_pred {
1848 if let Some(ref mut buf) = self.row_buf {
1850 if buf.predicate_ordinal.is_none() {
1851 buf.predicate_ordinal = Some(slot);
1852 }
1853 }
1854 let state = self.evaluate_predicate_state();
1855 if let Some(ref mut buf) = self.row_buf {
1856 buf.state = state;
1857 }
1858 }
1859 }
1860
1861 #[inline]
1862 fn end_row(&mut self) {
1863 if self.plan.filter.is_none() {
1864 self.finish_row();
1865 return;
1866 }
1867 let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
1870 if !buf_worthwhile {
1871 self.finish_row();
1875 return;
1876 }
1877 let (state, direct) = self
1879 .row_buf
1880 .as_ref()
1881 .map_or((PredicateState::Pass, false), |b| (b.state, b.direct));
1882 if direct {
1883 self.null_fill_missing();
1885 return;
1886 }
1887 let state = if state == PredicateState::Undecided {
1888 self.evaluate_against_null()
1889 } else {
1890 state
1891 };
1892 let pass = state == PredicateState::Pass;
1893 self.drain_buffered(pass);
1894 }
1895
1896 #[inline]
1897 fn wants(&self, name: &str) -> bool {
1898 self.resolve(name).is_some()
1899 }
1900
1901 #[inline]
1902 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
1903 self.plan.resolve_field(name)
1904 }
1905
1906 #[inline]
1907 fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
1908 if self.plan.filter.is_none() {
1909 self.push_field_resolved(resolved_name, value);
1910 return;
1911 }
1912 let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
1914 if !buf_worthwhile {
1915 self.push_field_resolved(resolved_name, value);
1916 return;
1917 }
1918 if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
1920 if let Some(ref frozen) = self.frozen {
1921 if frozen.is_exact()
1922 && !frozen
1923 .column_names()
1924 .iter()
1925 .any(|n| n.as_ref() == resolved_name)
1926 {
1927 if self.unknown_error.is_none() {
1928 self.unknown_error = Some(format!(
1929 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1930 resolved_name,
1931 frozen.num_columns(),
1932 frozen.is_exact()
1933 ));
1934 }
1935 return;
1936 }
1937 }
1938 }
1939 let slot = self.ensure_column_idx(resolved_name) as u32;
1940 self.record_strict_violation_at(slot as usize, &value);
1941 self.mark_predicate_slot(slot);
1942 let val_static: Value<'static> = value.into_static();
1943 let is_pred = self.is_predicate_slot(slot);
1944 #[cfg(feature = "profile")]
1945 if is_pred {
1946 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1947 } else {
1948 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1949 }
1950 if let Some(ref mut buf) = self.row_buf {
1951 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1952 buf.fields[pos].1 = val_static;
1953 } else {
1954 buf.fields.push((slot, val_static));
1955 }
1956 }
1957 if is_pred {
1958 if let Some(ref mut buf) = self.row_buf {
1960 if buf.predicate_ordinal.is_none() {
1961 buf.predicate_ordinal = Some(slot);
1962 }
1963 }
1964 let state = self.evaluate_predicate_state();
1965 if let Some(ref mut buf) = self.row_buf {
1966 buf.state = state;
1967 }
1968 }
1969 }
1970
1971 #[inline]
1975 fn put_field_at(&mut self, slot: u32, value: Value<'_>) {
1976 let idx = slot as usize;
1977 if idx >= self.columns.len() {
1978 return;
1981 }
1982 self.record_strict_violation_at(idx, &value);
1983 self.notify_put_field(idx, &value);
1984 let word = idx / 64;
1985 let bit = idx % 64;
1986 self.row_dirty[word] |= 1u64 << bit;
1987 let b = &mut self.columns[idx];
1988 let row_count = self.row_count;
1989 if b.len() > row_count {
1990 b.pop();
1991 }
1992 b.push_value(value);
1993 }
1994
1995 #[inline]
1996 fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
1997 #[cfg(feature = "profile")]
1998 RESOLVE_AND_PUT_COUNT.fetch_add(1, Ordering::Relaxed);
1999 if self.plan.filter.is_none() {
2000 if self.plan.field_map.is_empty()
2001 && self.plan.drop_fields.is_empty()
2002 && self.plan.schema_order.is_empty()
2003 {
2004 self.push_field_resolved(name, value);
2005 } else {
2006 if let Some(resolved) = self.plan.resolve_field(name) {
2007 let owned = resolved.to_owned();
2008 self.push_field_resolved(&owned, value);
2009 }
2010 }
2011 return;
2012 }
2013 if self.row_buf.as_ref().is_some_and(|b| !b.buffer_worthwhile) {
2015 if self.plan.field_map.is_empty()
2016 && self.plan.drop_fields.is_empty()
2017 && self.plan.schema_order.is_empty()
2018 {
2019 self.push_field_resolved(name, value);
2020 } else {
2021 if let Some(resolved) = self.plan.resolve_field(name) {
2022 let owned = resolved.to_owned();
2023 self.push_field_resolved(&owned, value);
2024 }
2025 }
2026 return;
2027 }
2028 if let Some(ref buf) = self.row_buf {
2030 if buf.direct {
2031 let resolved = match self.plan.resolve_field(name) {
2032 Some(r) => r.to_owned(),
2033 None => return, };
2035 if self.frozen.is_some() && !self.field_index.contains_key(resolved.as_str()) {
2036 if let Some(ref frozen) = self.frozen {
2037 if frozen.is_exact()
2038 && !frozen
2039 .column_names()
2040 .iter()
2041 .any(|n| n.as_ref() == resolved.as_str())
2042 {
2043 if self.unknown_error.is_none() {
2044 self.unknown_error = Some(format!(
2045 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
2046 resolved, frozen.num_columns(), frozen.is_exact()
2047 ));
2048 }
2049 return;
2050 }
2051 }
2052 }
2053 self.push_field_resolved(&resolved, value);
2054 return;
2055 }
2056 }
2057 if self.plan.field_map.is_empty()
2059 && self.plan.drop_fields.is_empty()
2060 && self.plan.schema_order.is_empty()
2061 {
2062 let resolved = name;
2064 if self.frozen.is_some() && !self.field_index.contains_key(resolved) {
2065 if let Some(ref frozen) = self.frozen {
2066 if frozen.is_exact()
2067 && !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
2068 {
2069 if self.unknown_error.is_none() {
2070 self.unknown_error = Some(format!(
2071 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
2072 resolved,
2073 frozen.num_columns(),
2074 frozen.is_exact()
2075 ));
2076 }
2077 return;
2078 }
2079 }
2080 }
2081 let slot = self.ensure_column_idx(resolved) as u32;
2082 self.record_strict_violation_at(slot as usize, &value);
2083 self.mark_predicate_slot(slot);
2084 let val_static: Value<'static> = value.into_static();
2085 let is_pred = self.is_predicate_slot(slot);
2086 #[cfg(feature = "profile")]
2087 if is_pred {
2088 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2089 } else {
2090 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2091 }
2092 if let Some(ref mut buf) = self.row_buf {
2093 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
2094 buf.fields[pos].1 = val_static;
2095 } else {
2096 buf.fields.push((slot, val_static));
2097 }
2098 }
2099 if is_pred {
2100 if let Some(ref mut buf) = self.row_buf {
2102 if buf.predicate_ordinal.is_none() {
2103 buf.predicate_ordinal = Some(slot);
2104 }
2105 }
2106 let state = self.evaluate_predicate_state();
2107 if let Some(ref mut buf) = self.row_buf {
2108 buf.state = state;
2109 if state == PredicateState::Pass {
2110 buf.direct = true;
2111 }
2112 }
2113 if state == PredicateState::Pass {
2114 self.drain_buffered(true);
2116 }
2117 }
2118 } else {
2119 if let Some(resolved) = self.plan.resolve_field(name) {
2120 let owned = resolved.to_owned();
2121 if self.frozen.is_some() && !self.field_index.contains_key(&owned) {
2122 if let Some(ref frozen) = self.frozen {
2123 if frozen.is_exact()
2124 && !frozen.column_names().iter().any(|n| n.as_ref() == owned)
2125 {
2126 if self.unknown_error.is_none() {
2127 self.unknown_error = Some(format!(
2128 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
2129 owned,
2130 frozen.num_columns(),
2131 frozen.is_exact()
2132 ));
2133 }
2134 return;
2135 }
2136 }
2137 }
2138 let slot = self.ensure_column_idx(&owned) as u32;
2139 self.record_strict_violation_at(slot as usize, &value);
2140 self.mark_predicate_slot(slot);
2141 let val_static: Value<'static> = value.into_static();
2142 let is_pred = self.is_predicate_slot(slot);
2143 #[cfg(feature = "profile")]
2144 if is_pred {
2145 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2146 } else {
2147 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2148 }
2149 if let Some(ref mut buf) = self.row_buf {
2150 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
2151 buf.fields[pos].1 = val_static;
2152 } else {
2153 buf.fields.push((slot, val_static));
2154 }
2155 }
2156 if is_pred {
2157 if let Some(ref mut buf) = self.row_buf {
2159 if buf.predicate_ordinal.is_none() {
2160 buf.predicate_ordinal = Some(slot);
2161 }
2162 }
2163 let state = self.evaluate_predicate_state();
2164 if let Some(ref mut buf) = self.row_buf {
2165 buf.state = state;
2166 }
2167 }
2168 }
2169 }
2170 }
2171
2172 #[inline]
2173 fn row_rejected(&self) -> bool {
2174 self.plan.filter.is_some()
2175 && self
2176 .row_buf
2177 .as_ref()
2178 .is_some_and(|b| b.state == PredicateState::Fail)
2179 }
2180
2181 #[inline]
2182 fn row_satisfied(&self) -> bool {
2183 let mask = self.wanted_mask();
2184 if mask == 0 {
2185 return false;
2186 }
2187 let dirty = self
2189 .row_dirty
2190 .iter()
2191 .copied()
2192 .reduce(|a, b| a | b)
2193 .unwrap_or(0);
2194 (dirty & mask) == mask
2195 }
2196
2197 #[inline]
2198 fn wanted_mask(&self) -> u64 {
2199 self.cached_wanted_mask
2204 }
2205
2206 #[inline]
2207 fn finish(&mut self) -> Result<RecordBatch> {
2208 if let Some(err) = self.unknown_error.take() {
2209 return Err(crate::Error::Merge(err));
2210 }
2211 if let Some(err) = self.strict_error.take() {
2212 return Err(err);
2213 }
2214 if let Some(ref obs) = self.plan.observer {
2215 obs.on_chunk_finished(
2216 self.rows_accepted + self.rows_rejected,
2217 self.rows_accepted,
2218 self.rows_rejected,
2219 );
2220 }
2221 self.normalize();
2222
2223 if self.column_order.is_empty() {
2224 let schema = Arc::new(Schema::empty());
2225 return Ok(RecordBatch::new_empty(schema));
2226 }
2227
2228 self.auto_dict_upgrade();
2229
2230 if !self.plan.schema_order.is_empty() {
2235 let order = self.plan.schema_order.clone();
2236 let mut fields = Vec::with_capacity(order.len());
2237 let mut arrays = Vec::with_capacity(order.len());
2238 for name in &order {
2239 if self.plan.drop_fields.contains(name) {
2241 continue;
2242 }
2243 let idx = match self.field_index.get(name.as_str()) {
2244 Some(&i) => i,
2245 None => {
2246 let ty = self.plan.column_type(name);
2248 let mut b = ColumnBuilder::with_capacity(self.row_count.max(1), &ty);
2249 for _ in 0..self.row_count {
2250 b.push(None);
2251 }
2252 let idx = self.columns.len();
2253 self.columns.push(b);
2254 self.field_index.insert(name.clone(), idx);
2255 idx
2256 }
2257 };
2258 let b = &mut self.columns[idx];
2259 fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
2260 arrays.push(b.to_arrow_array()?);
2261 }
2262 let schema = Arc::new(Schema::new(fields));
2263 return Ok(RecordBatch::try_new(schema, arrays)?);
2264 }
2265
2266 self.sort_columns();
2267
2268 let mut fields = Vec::with_capacity(self.column_order.len());
2269 let mut arrays = Vec::with_capacity(self.column_order.len());
2270 for name in &self.column_order {
2272 if let Some(&idx) = self.field_index.get(name.as_str()) {
2273 let b = &mut self.columns[idx];
2274 fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
2275 arrays.push(b.to_arrow_array()?);
2276 }
2277 }
2278
2279 let schema = Arc::new(Schema::new(fields));
2280 Ok(RecordBatch::try_new(schema, arrays)?)
2281 }
2282
2283 #[inline]
2284 fn expect_slot(&self, ordinal: u32) -> Option<(u32, &[u8])> {
2285 self.ordinal_expect
2286 .get(ordinal as usize)
2287 .and_then(|entry| entry.as_ref())
2288 .map(|(slot, raw)| (*slot, raw.as_slice()))
2289 }
2290
2291 #[inline]
2292 fn record_slot(&mut self, ordinal: u32, slot: u32, raw_name: &[u8]) {
2293 let idx = ordinal as usize;
2294 if idx >= self.ordinal_expect.len() {
2295 self.ordinal_expect.resize_with(idx + 1, || None);
2296 }
2297 self.ordinal_expect[idx] = Some((slot, raw_name.to_vec()));
2298 }
2299
2300 #[inline]
2301 fn layout_broken(&mut self, ordinal: u32) {
2302 if let Some(entry) = self.ordinal_expect.get_mut(ordinal as usize) {
2303 *entry = None;
2304 }
2305 }
2306
2307 #[inline]
2308 fn reset_child_ordinal(&mut self) {
2309 self.current_ordinal = 0;
2310 }
2311}
2312
2313#[cfg(test)]
2318mod tests {
2319 use super::*;
2320 use crate::columnar::ColumnBuilder;
2321 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
2322 use crate::plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
2323 use crate::value::Value;
2324 use crate::Result;
2325 use arrow::array::{Array, AsArray};
2326
2327 #[derive(Clone)]
2330 struct LineParser;
2331
2332 impl RecordParser for LineParser {
2333 fn validate(&self, bytes: &[u8]) -> Result<()> {
2334 simdutf8::basic::from_utf8(bytes)?;
2335 Ok(())
2336 }
2337
2338 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
2339 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
2340 for line in text.lines() {
2341 if line.is_empty() {
2342 continue;
2343 }
2344 sink.begin_row();
2345 for token in line.split_whitespace() {
2346 if let Some((k, v)) = token.split_once('=') {
2347 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
2348 }
2349 }
2350 sink.end_row();
2351 }
2352 Ok(())
2353 }
2354 }
2355
2356 #[derive(Clone)]
2357 struct LineSplitter;
2358
2359 impl Splitter for LineSplitter {
2360 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
2361 if from >= bytes.len() {
2362 return None;
2363 }
2364 let start = if bytes[from] == b'\n' { from + 1 } else { from };
2366 if start >= bytes.len() {
2367 return None;
2368 }
2369 memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
2370 }
2371
2372 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
2373 if max_chunks <= 1 || bytes.is_empty() {
2374 return vec![0, bytes.len()];
2375 }
2376 let mut points = vec![0usize];
2377 let mut last = 0;
2378 for (i, &b) in bytes.iter().enumerate() {
2379 if b == b'\n' {
2380 let next = i + 1;
2381 if next > last && points.len() < max_chunks {
2382 points.push(next);
2383 last = next;
2384 }
2385 }
2386 }
2387 if *points.last().unwrap() != bytes.len() {
2388 points.push(bytes.len());
2389 }
2390 points
2391 }
2392
2393 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
2394 let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
2395 (sample.len() / newline_count).max(1)
2396 }
2397 }
2398
2399 #[derive(Default)]
2401 struct RecObserver {
2402 events: std::sync::Mutex<Vec<String>>,
2403 }
2404
2405 impl RecObserver {
2406 fn log(&self, e: String) {
2407 self.events.lock().unwrap().push(e);
2408 }
2409 fn events(&self) -> Vec<String> {
2410 self.events.lock().unwrap().clone()
2411 }
2412 }
2413
2414 impl crate::RowObserver for RecObserver {
2415 fn on_begin_row(&self, row_index: usize) {
2416 self.log(format!("begin:{row_index}"));
2417 }
2418 fn on_put_field(&self, row_index: usize, name: &str, _slot: usize, _value: &Value<'_>) {
2419 self.log(format!("put:{row_index}:{name}"));
2420 }
2421 fn on_row_accepted(&self, row_index: usize) {
2422 self.log(format!("accept:{row_index}"));
2423 }
2424 fn on_row_rejected(&self, row_index: usize) {
2425 self.log(format!("reject:{row_index}"));
2426 }
2427 fn on_chunk_finished(&self, total: usize, accepted: usize, rejected: usize) {
2428 self.log(format!("fin:{total}:{accepted}:{rejected}"));
2429 }
2430 }
2431
2432 #[test]
2433 fn test_observer_lifecycle() {
2434 let obs = Arc::new(RecObserver::default());
2435 let mut plan = ExecutionPlan::new();
2436 plan.observer = Some(obs.clone());
2437 let mut engine = parse_bytes(b"X=1 Y=2\nX=3 Y=4\n", plan);
2438 engine.finish().unwrap();
2439 assert_eq!(
2440 obs.events(),
2441 vec![
2442 "begin:0",
2443 "put:0:X",
2444 "put:0:Y",
2445 "accept:0",
2446 "begin:1",
2447 "put:1:X",
2448 "put:1:Y",
2449 "accept:1",
2450 "fin:2:2:0",
2451 ]
2453 );
2454 }
2455
2456 #[test]
2457 fn test_observer_with_filter() {
2458 let obs = Arc::new(RecObserver::default());
2459 let mut plan = ExecutionPlan::new();
2460 plan.filter = Some(FilterPredicate::Equal {
2461 field: "X".to_string(),
2462 value: "keep".to_string(),
2463 });
2464 plan.observer = Some(obs.clone());
2465 let mut engine = parse_bytes(b"X=keep A=1\nX=drop A=2\n", plan);
2466 engine.finish().unwrap();
2467 let ev = obs.events();
2468 assert!(ev.contains(&"reject:1".to_string()), "events: {ev:?}");
2470 assert_eq!(
2471 ev.iter().filter(|e| e.starts_with("put:")).count(),
2472 2,
2473 "only the accepted row's fields may be observed: {ev:?}"
2474 );
2475 assert!(ev.contains(&"fin:2:1:1".to_string()), "events: {ev:?}");
2476 }
2477
2478 #[test]
2479 fn test_observer_parallel_counts_all_rows() {
2480 use std::sync::atomic::{AtomicUsize, Ordering as AOrd};
2481 #[derive(Default)]
2482 struct Counting {
2483 accepted: AtomicUsize,
2484 rejected: AtomicUsize,
2485 }
2486 impl crate::RowObserver for Counting {
2487 fn on_row_accepted(&self, _i: usize) {
2488 self.accepted.fetch_add(1, AOrd::Relaxed);
2489 }
2490 fn on_row_rejected(&self, _i: usize) {
2491 self.rejected.fetch_add(1, AOrd::Relaxed);
2492 }
2493 }
2494 let obs = Arc::new(Counting::default());
2495 let mut plan = ExecutionPlan::new();
2496 plan.filter = Some(FilterPredicate::NotEqual {
2497 field: "X".to_string(),
2498 value: "drop".to_string(),
2499 });
2500 plan.observer = Some(obs.clone());
2501 let mut data = Vec::new();
2502 for i in 0..1000 {
2503 let v = if i % 3 == 0 { "drop" } else { "keep" };
2504 data.extend_from_slice(format!("X={v} N={i}\n").as_bytes());
2505 }
2506 let pipeline = crate::Pipeline::new(LineSplitter, LineParser).with_plan(plan);
2507 let batches = pipeline.read_bytes_par(&data, 4).unwrap();
2508 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
2509 let accepted = obs.accepted.load(AOrd::Relaxed);
2510 let rejected = obs.rejected.load(AOrd::Relaxed);
2511 assert_eq!(accepted + rejected, 1000);
2512 assert_eq!(accepted, total);
2513 assert_eq!(rejected, 334);
2514 }
2515
2516 #[test]
2517 fn test_not_wraps_leaf_predicates_predicate_first() {
2518 let cases: Vec<(FilterPredicate, &[u8], Vec<&str>)> = vec![
2521 (
2522 FilterPredicate::not(FilterPredicate::Equal {
2523 field: "X".into(),
2524 value: "keep".into(),
2525 }),
2526 b"X=keep\nX=drop\n",
2527 vec!["drop"],
2528 ),
2529 (
2530 FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() }),
2531 b"X=1\nY=2\n",
2532 vec!["1"],
2533 ),
2534 (
2535 FilterPredicate::not(FilterPredicate::StartsWith {
2536 field: "X".into(),
2537 value: "ab".into(),
2538 }),
2539 b"X=abc\nX=zzz\n",
2540 vec!["zzz"],
2541 ),
2542 (
2543 FilterPredicate::not(FilterPredicate::In {
2544 field: "X".into(),
2545 values: vec!["a".into(), "b".into()],
2546 }),
2547 b"X=a\nX=b\nX=c\n",
2548 vec!["c"],
2549 ),
2550 (
2551 FilterPredicate::not(FilterPredicate::Regex {
2552 field: "X".into(),
2553 re: crate::plan::RegexSpec::new("^ERR").unwrap(),
2554 }),
2555 b"X=ERR1\nX=ok\n",
2556 vec!["ok"],
2557 ),
2558 (
2559 FilterPredicate::not(FilterPredicate::Contains {
2560 field: "X".into(),
2561 value: "zz".into(),
2562 }),
2563 b"X=azza\nX=hello\n",
2564 vec!["hello"],
2565 ),
2566 ];
2567 for (pred, data, want) in cases {
2568 let mut plan = ExecutionPlan::new();
2569 plan.filter = Some(pred);
2570 let engine = parse_bytes(data, plan);
2571 let col = engine.get_column("X").unwrap();
2572 assert_eq!(
2573 col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
2574 want,
2575 "data: {}",
2576 String::from_utf8_lossy(data)
2577 );
2578 }
2579 }
2580
2581 #[test]
2582 fn test_not_wraps_leaf_predicates_batch_check() {
2583 let data = b"X=keep N=1\nX=drop N=2\nX=keep N=3\n";
2586 let engine = parse_bytes(data, ExecutionPlan::new());
2587 let pred = FilterPredicate::not(FilterPredicate::Equal {
2588 field: "X".into(),
2589 value: "keep".into(),
2590 });
2591 let kept: Vec<usize> = (0..engine.num_rows())
2592 .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
2593 .collect();
2594 assert_eq!(kept, vec![1]);
2595
2596 let pred = FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() });
2597 let kept: Vec<usize> = (0..engine.num_rows())
2598 .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
2599 .collect();
2600 assert_eq!(kept, vec![0, 1, 2]);
2601
2602 let pred = FilterPredicate::not(FilterPredicate::Regex {
2603 field: "X".into(),
2604 re: crate::plan::RegexSpec::new("ke*p").unwrap(),
2605 });
2606 let kept: Vec<usize> = (0..engine.num_rows())
2607 .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
2608 .collect();
2609 assert_eq!(kept, vec![1]);
2610 }
2611
2612 #[test]
2613 fn test_regex_predicate() {
2614 let mut plan = ExecutionPlan::new();
2615 plan.filter = Some(FilterPredicate::Regex {
2616 field: "X".into(),
2617 re: crate::plan::RegexSpec::new("^ERR\\d+$").unwrap(),
2618 });
2619 let engine = parse_bytes(b"X=ERR12\nX=ok\nX=ERRx\nX=ERR0\n", plan);
2620 let col = engine.get_column("X").unwrap();
2621 assert_eq!(
2622 col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
2623 vec!["ERR12", "ERR0"]
2624 );
2625 let mut plan = ExecutionPlan::new();
2627 plan.filter = Some(FilterPredicate::Regex {
2628 field: "X".into(),
2629 re: crate::plan::RegexSpec::new(".*").unwrap(),
2630 });
2631 let engine = parse_bytes(b"X=a\nY=2\n", plan);
2632 assert_eq!(engine.num_rows(), 1);
2633 assert!(crate::plan::RegexSpec::new("(").is_err());
2635 assert_eq!(
2637 crate::plan::RegexSpec::new("^a").unwrap(),
2638 crate::plan::RegexSpec::new("^a").unwrap()
2639 );
2640 assert_ne!(
2641 crate::plan::RegexSpec::new("^a").unwrap(),
2642 crate::plan::RegexSpec::new("^b").unwrap()
2643 );
2644 }
2645
2646 #[test]
2647 fn test_no_observer_smoke() {
2648 let engine = parse_bytes(b"X=1\nX=2\n", ExecutionPlan::new());
2650 assert_eq!(engine.num_rows(), 2);
2651 }
2652
2653 fn parse_bytes(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
2654 let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
2655 LineParser.parse_chunk(bytes, &mut sink).unwrap();
2656 sink
2657 }
2658
2659 #[test]
2660 fn test_extend_no_duplicates() {
2661 let e1 = parse_bytes(b"A=1 B=2\n", ExecutionPlan::new());
2662 let mut merged = TableBuilder::new();
2663 merged.extend(e1).unwrap();
2664 assert_eq!(merged.num_rows(), 1);
2665 for col in &merged.columns {
2666 assert_eq!(col.len(), merged.num_rows(), "column length mismatch");
2667 }
2668 }
2669
2670 #[test]
2671 fn test_multi_chunk_same_as_single() {
2672 let data = b"A=1\nA=2\nA=3\n";
2673 let single = parse_bytes(data, ExecutionPlan::new());
2674 assert_eq!(single.num_rows(), 3);
2675
2676 let splitter = LineSplitter;
2677 let points = splitter.find_split_points(data, 2);
2678 let mut merged = TableBuilder::new();
2679 for w in points.windows(2) {
2680 let chunk = &data[w[0]..w[1]];
2681 let engine = parse_bytes(chunk, ExecutionPlan::new());
2682 merged.extend(engine).unwrap();
2683 }
2684 assert_eq!(merged.num_rows(), single.num_rows());
2685 }
2686
2687 #[test]
2688 fn test_last_write_wins_duplicate_field() {
2689 let engine = parse_bytes(b"X=10 X=20\n", ExecutionPlan::new());
2690 assert_eq!(engine.num_rows(), 1);
2691 let col = engine.get_column("X").unwrap();
2692 assert_eq!(col.as_str_vec(), vec![Some("20".into())]);
2693 }
2694
2695 #[test]
2696 fn test_build_plan_rename() {
2697 let mut plan = ExecutionPlan::new();
2698 plan.field_map.insert("X".to_string(), "Y".to_string());
2699 let engine = parse_bytes(b"X=hello\n", plan);
2700 assert_eq!(engine.num_rows(), 1);
2701 assert!(engine.get_column("Y").is_some());
2702 assert!(engine.get_column("X").is_none());
2703 assert_eq!(
2704 engine.get_column("Y").unwrap().as_str_vec(),
2705 vec![Some("hello".into())]
2706 );
2707 }
2708
2709 #[test]
2710 fn test_build_plan_drop() {
2711 let mut plan = ExecutionPlan::new();
2712 plan.drop_fields.insert("X".to_string());
2713 let engine = parse_bytes(b"X=hello Y=world\n", plan);
2714 assert_eq!(engine.num_rows(), 1);
2715 assert!(engine.get_column("X").is_none());
2716 assert!(engine.get_column("Y").is_some());
2717 }
2718
2719 #[test]
2720 fn test_build_plan_filter_ne() {
2721 let mut plan = ExecutionPlan::new();
2722 plan.filter = Some(FilterPredicate::NotEqual {
2723 field: "X".to_string(),
2724 value: "42".to_string(),
2725 });
2726 let engine = parse_bytes(b"X=10\nX=42\nX=30\n", plan);
2727 assert_eq!(engine.num_rows(), 2);
2728 let col = engine.get_column("X").unwrap();
2729 assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("30".into())]);
2730 }
2731
2732 #[test]
2733 fn test_build_plan_filter_eq() {
2734 let mut plan = ExecutionPlan::new();
2735 plan.filter = Some(FilterPredicate::Equal {
2736 field: "X".to_string(),
2737 value: "10".to_string(),
2738 });
2739 let engine = parse_bytes(b"X=10\nX=20\nX=10\n", plan);
2740 assert_eq!(engine.num_rows(), 2);
2741 let col = engine.get_column("X").unwrap();
2742 assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("10".into())]);
2743 }
2744
2745 #[test]
2746 fn test_build_plan_filter_missing_field() {
2747 let mut plan = ExecutionPlan::new();
2748 plan.filter = Some(FilterPredicate::NotEqual {
2749 field: "X".to_string(),
2750 value: "10".to_string(),
2751 });
2752 let engine = parse_bytes(b"X=10\nY=99\n", plan);
2753 assert_eq!(engine.num_rows(), 1);
2754 let col = engine.get_column("Y").unwrap();
2755 assert_eq!(col.as_str_vec(), vec![Some("99".into())]);
2756 }
2757
2758 #[test]
2759 fn test_typed_int64_column() {
2760 let mut plan = ExecutionPlan::new();
2761 plan.field_types.insert("X".to_string(), FieldType::Int64);
2762 let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
2763 assert_eq!(engine.num_rows(), 3);
2764 if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
2765 assert_eq!(v.get(0), Some(42));
2766 assert_eq!(v.get(1), None);
2767 assert_eq!(v.get(2), Some(100));
2768 } else {
2769 panic!("expected Int64 builder");
2770 }
2771 }
2772
2773 #[test]
2774 fn test_typed_float64_column() {
2775 let mut plan = ExecutionPlan::new();
2776 plan.field_types.insert("X".to_string(), FieldType::Float64);
2777 let engine = parse_bytes(b"X=1.5\n", plan);
2778 if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
2779 assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
2780 } else {
2781 panic!("expected Float64 builder");
2782 }
2783 }
2784
2785 #[test]
2786 fn test_is_null_filter() {
2787 let mut plan = ExecutionPlan::new();
2788 plan.filter = Some(FilterPredicate::IsNull {
2789 field: "X".to_string(),
2790 });
2791 let engine = parse_bytes(b"X=hello\nY=world\nX=fallback\n", plan);
2792 assert_eq!(engine.num_rows(), 1);
2793 let col = engine.get_column("Y").unwrap();
2794 assert_eq!(col.as_str_vec(), vec![Some("world".into())]);
2795 }
2796
2797 #[test]
2798 fn test_not_is_null_filter_predicate_first() {
2799 let mut plan = ExecutionPlan::new();
2800 plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
2801 field: "X".to_string(),
2802 }));
2803 let engine = parse_bytes(b"X=a\nY=only\nX=b\n", plan);
2804 assert_eq!(engine.num_rows(), 2);
2805 let col = engine.get_column("X").unwrap();
2806 assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
2807 }
2808
2809 #[test]
2810 fn test_not_is_null_filter_batch_eval() {
2811 let mut plan = ExecutionPlan::new();
2814 plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
2815 field: "X".to_string(),
2816 }));
2817 let mut sink = TableBuilder::with_plan(4, Arc::new(plan));
2818 sink.row_buf.as_mut().unwrap().buffer_worthwhile = false;
2819 LineParser
2820 .parse_chunk(b"X=a\nY=only\nX=b\n", &mut sink)
2821 .unwrap();
2822 assert_eq!(sink.num_rows(), 2);
2823 let col = sink.get_column("X").unwrap();
2824 assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
2825 }
2826
2827 #[test]
2828 fn test_strict_types_errors_on_malformed_int64() {
2829 let mut plan = ExecutionPlan::new();
2830 plan.field_types.insert("X".to_string(), FieldType::Int64);
2831 plan.strict_types = true;
2832 let mut engine = parse_bytes(b"X=42\nX=abc\n", plan);
2833 let err = engine.finish().unwrap_err();
2834 let msg = err.to_string();
2835 assert!(msg.contains("\"abc\""), "message should name value: {msg}");
2836 assert!(msg.contains("\"X\""), "message should name column: {msg}");
2837 assert!(msg.contains("Int64"), "message should name type: {msg}");
2838 assert!(msg.contains("row 1"), "message should name row: {msg}");
2839 }
2840
2841 #[test]
2842 fn test_strict_types_off_yields_null() {
2843 let mut plan = ExecutionPlan::new();
2844 plan.field_types.insert("X".to_string(), FieldType::Int64);
2845 let engine = parse_bytes(b"X=42\nX=abc\n", plan);
2846 assert!(engine.strict_error.is_none());
2847 if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
2848 assert_eq!(v.get(0), Some(42));
2849 assert_eq!(v.get(1), None);
2850 } else {
2851 panic!("expected Int64 builder");
2852 }
2853 }
2854
2855 #[test]
2856 fn test_strict_types_missing_field_stays_null() {
2857 let mut plan = ExecutionPlan::new();
2858 plan.field_types.insert("X".to_string(), FieldType::Int64);
2859 plan.strict_types = true;
2860 let mut engine = parse_bytes(b"X=42\nY=1\n", plan);
2861 let batch = engine.finish().unwrap();
2862 let idx = batch.schema().index_of("X").unwrap();
2863 let arr = batch
2864 .column(idx)
2865 .as_primitive::<arrow::datatypes::Int64Type>();
2866 assert_eq!(arr.value(0), 42);
2867 assert!(arr.is_null(1));
2868 }
2869
2870 #[test]
2871 fn test_strict_types_buffered_filter_path() {
2872 let mut plan = ExecutionPlan::new();
2875 plan.field_types.insert("X".to_string(), FieldType::Int64);
2876 plan.strict_types = true;
2877 plan.filter = Some(FilterPredicate::Equal {
2878 field: "Y".to_string(),
2879 value: "1".to_string(),
2880 });
2881 let mut engine = parse_bytes(b"Y=1 X=abc\n", plan);
2882 let err = engine.finish().unwrap_err();
2883 assert!(err.to_string().contains("strict_types"));
2884 }
2885
2886 #[test]
2887 fn test_is_type_filter_int64() {
2888 let mut plan = ExecutionPlan::new();
2889 plan.field_types.insert("X".to_string(), FieldType::Int64);
2890 plan.filter = Some(FilterPredicate::IsType {
2891 field: "X".to_string(),
2892 field_type: FieldType::Int64,
2893 });
2894 let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
2895 assert_eq!(engine.num_rows(), 2);
2896 if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
2897 assert_eq!(v.get(0), Some(42));
2898 assert_eq!(v.get(1), Some(100));
2899 } else {
2900 panic!("expected Int64 builder");
2901 }
2902 }
2903
2904 #[test]
2905 fn test_is_type_filter_float64() {
2906 let mut plan = ExecutionPlan::new();
2907 plan.field_types.insert("X".to_string(), FieldType::Float64);
2908 plan.filter = Some(FilterPredicate::IsType {
2909 field: "X".to_string(),
2910 field_type: FieldType::Float64,
2911 });
2912 let engine = parse_bytes(b"X=1.5\nX=bad\nX=2.5\n", plan);
2913 assert_eq!(engine.num_rows(), 2);
2914 if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
2915 assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
2916 assert!((v.get(1).unwrap() - 2.5).abs() < 1e-9);
2917 } else {
2918 panic!("expected Float64 builder");
2919 }
2920 }
2921
2922 #[test]
2923 fn test_dictionary_column() {
2924 let mut plan = ExecutionPlan::new();
2925 plan.dictionary_columns.insert("P".to_string());
2926 let engine = parse_bytes(b"P=Widget\nP=Gadget\nP=Widget\n", plan);
2927 assert_eq!(engine.num_rows(), 3);
2928 if let ColumnBuilder::Dictionary { codes, offsets, .. } = engine.get_column("P").unwrap() {
2929 let dict_len = offsets.len() - 1;
2930 assert_eq!(dict_len, 2);
2931 assert_eq!(
2932 codes.iter().map(|o| o.copied()).collect::<Vec<_>>(),
2933 vec![Some(0), Some(1), Some(0)]
2934 );
2935 } else {
2936 panic!("expected Dictionary builder");
2937 }
2938 }
2939
2940 #[test]
2941 fn test_ragged_late_chunk_column_debut() {
2942 let e1 = parse_bytes(b"A=1 B=2\nA=3\n", ExecutionPlan::new());
2943 let e2 = parse_bytes(b"B=4 C=5\n", ExecutionPlan::new());
2944 let mut merged = TableBuilder::new();
2945 merged.extend(e1).unwrap();
2946 merged.extend(e2).unwrap();
2947
2948 assert_eq!(
2949 merged.get_column("A").unwrap().as_str_vec(),
2950 vec![Some("1".into()), Some("3".into()), None]
2951 );
2952 assert_eq!(
2953 merged.get_column("B").unwrap().as_str_vec(),
2954 vec![Some("2".into()), None, Some("4".into())]
2955 );
2956 assert_eq!(
2957 merged.get_column("C").unwrap().as_str_vec(),
2958 vec![None, None, Some("5".into())]
2959 );
2960 }
2961
2962 #[test]
2963 fn test_auto_dict_upgrade_only_post_merge() {
2964 let mut plan = ExecutionPlan::new();
2965 plan.auto_dict = true;
2966 let a = parse_bytes(b"P=x\nP=y\n", plan.clone());
2967 let b = parse_bytes(b"P=x\nP=y\n", plan.clone());
2968 let mut merged = TableBuilder::with_plan(64, Arc::new(plan));
2969 merged.extend(a).unwrap();
2970 merged.extend(b).unwrap();
2971 merged.auto_dict_upgrade();
2972 assert_eq!(merged.num_rows(), 4);
2973 }
2974
2975 #[test]
2976 fn test_extend_string_dictionary_promotes_not_panics() {
2977 let mut e1 = parse_bytes(b"P=x\n", ExecutionPlan::new());
2978 let mut plan = ExecutionPlan::new();
2979 plan.dictionary_columns.insert("P".to_string());
2980 let e2 = parse_bytes(b"P=x\n", plan);
2981 e1.extend(e2).expect("String/Dictionary must reconcile");
2983 assert!(matches!(
2984 e1.get_column("P").unwrap(),
2985 crate::columnar::ColumnBuilder::Dictionary { .. }
2986 ));
2987 }
2988
2989 #[test]
2990 fn test_extend_irreconcilable_variants_error_not_panic() {
2991 let mut e1 = parse_bytes(b"S=x\n", ExecutionPlan::new());
2992 let mut plan = ExecutionPlan::new();
2993 plan.field_types.insert("S".to_string(), FieldType::Int64);
2994 let e2 = parse_bytes(b"S=7\n", plan);
2995 let result = e1.extend(e2);
2996 match result {
2997 Err(crate::Error::Merge(msg)) => {
2998 assert!(msg.contains("'S'"), "error should name the column: {msg}");
2999 }
3000 other => panic!("String/Int64 mismatch must return Merge error, got {other:?}"),
3001 }
3002 }
3003
3004 #[test]
3005 fn test_compare_filter_via_arrow_compute() {
3006 use crate::arrow_export::apply_compare_filter;
3007
3008 let mut plan = ExecutionPlan::new();
3009 plan.filter = Some(FilterPredicate::Compare {
3010 field_a: "A".to_string(),
3011 op: CompareOp::Gt,
3012 field_b: "B".to_string(),
3013 });
3014 let mut engine = parse_bytes(b"A=3 B=1\nA=2 B=2\nA=5 B=4\n", plan);
3015 let batch = engine.finish().unwrap();
3016 let filtered = apply_compare_filter(
3017 batch,
3018 &FilterPredicate::Compare {
3019 field_a: "A".to_string(),
3020 op: CompareOp::Gt,
3021 field_b: "B".to_string(),
3022 },
3023 )
3024 .unwrap();
3025 assert_eq!(filtered.num_rows(), 2);
3026 let a = filtered.column_by_name("A").unwrap().as_string::<i32>();
3027 assert_eq!(a.value(0), "3");
3028 assert_eq!(a.value(1), "5");
3029 }
3030
3031 #[test]
3032 fn test_row_dirty_is_full_edge_cases() {
3033 for ncols in [1, 63, 64, 65, 127, 128, 129] {
3036 let mut tb = TableBuilder::new();
3037 for i in 0..ncols {
3038 let name = format!("col{i}");
3039 tb.field_index.insert(name.clone(), i);
3040 tb.columns.push(crate::columnar::ColumnBuilder::String(
3041 crate::columnar::StrColumn::default(),
3042 ));
3043 tb.column_order.push(name);
3044 }
3045 tb.row_dirty = vec![0; ncols.div_ceil(64)];
3047 for i in 0..ncols {
3049 let word = i / 64;
3050 let bit = i % 64;
3051 tb.row_dirty[word] |= 1u64 << bit;
3052 }
3053 let full_words = ncols / 64;
3054 let rem = ncols % 64;
3055 let is_full = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
3056 && (rem == 0
3057 || tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
3058 assert!(
3059 is_full,
3060 "ncols={ncols} should be full, row_dirty={:?}",
3061 tb.row_dirty
3062 );
3063 if ncols > 0 {
3065 let word = (ncols - 1) / 64;
3066 let bit = (ncols - 1) % 64;
3067 tb.row_dirty[word] &= !(1u64 << bit);
3068 let is_full2 = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
3069 && (rem == 0
3070 || tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
3071 assert!(
3072 !is_full2,
3073 "ncols={ncols} should not be full after clearing one bit"
3074 );
3075 }
3076 }
3077 }
3078
3079 #[test]
3080 fn test_finish_twice_does_not_panic() {
3081 let mut tb = parse_bytes(b"A=1 B=2\nA=3 B=4\n", ExecutionPlan::new());
3084 let batch1 = tb.finish().unwrap();
3085 assert_eq!(batch1.num_rows(), 2);
3086 let batch2 = tb.finish().unwrap();
3088 assert_eq!(batch2.num_rows(), 0);
3089 }
3090
3091 #[test]
3092 fn test_c2_compound_reorder() {
3093 let data = b"A=1 B=2\nA=2 B=2\nA=1 B=3\nA=2 B=3\n";
3096 let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
3097 let mut plan = ExecutionPlan::new();
3098 plan.filter = Some(FilterPredicate::And(
3099 Box::new(FilterPredicate::Equal {
3100 field: a.to_string(),
3101 value: va.to_string(),
3102 }),
3103 Box::new(FilterPredicate::Equal {
3104 field: b.to_string(),
3105 value: vb.to_string(),
3106 }),
3107 ));
3108 plan
3109 };
3110 let r1 = parse_bytes(data, make_filter("A", "1", "B", "2"));
3111 let r2 = parse_bytes(data, make_filter("B", "2", "A", "1"));
3112 assert_eq!(
3113 r1.num_rows(),
3114 r2.num_rows(),
3115 "C2 reorder must produce identical results"
3116 );
3117 assert_eq!(r1.num_rows(), 1); }
3119
3120 #[test]
3121 fn test_c2_or_reorder() {
3122 let data = b"A=1 B=2\nA=2 B=2\nA=3 B=3\n";
3127 let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
3128 let mut plan = ExecutionPlan::new();
3129 plan.filter = Some(FilterPredicate::Or(
3130 Box::new(FilterPredicate::Equal {
3131 field: a.to_string(),
3132 value: va.to_string(),
3133 }),
3134 Box::new(FilterPredicate::Equal {
3135 field: b.to_string(),
3136 value: vb.to_string(),
3137 }),
3138 ));
3139 plan
3140 };
3141 let r1 = parse_bytes(data, make_filter("A", "1", "B", "3"));
3142 let r2 = parse_bytes(data, make_filter("B", "3", "A", "1"));
3143 assert_eq!(
3144 r1.num_rows(),
3145 r2.num_rows(),
3146 "Or reorder must produce identical results"
3147 );
3148 assert_eq!(r1.num_rows(), 2); }
3150
3151 #[test]
3152 fn test_predicate_slot_created_after_mask_build() {
3153 let data = b"A=1 B=ok\nA=2 B=ok\nA=3 B=ok\n";
3161 let mut plan = ExecutionPlan::new();
3162 plan.filter = Some(FilterPredicate::Equal {
3163 field: "B".to_string(),
3164 value: "reject".to_string(),
3165 });
3166 let engine = parse_bytes(data, plan);
3167 assert_eq!(
3168 engine.num_rows(),
3169 0,
3170 "all rows should be rejected (B != reject)"
3171 );
3172
3173 let data2 = b"A=1 B=ok\nA=2 B=reject\nA=3 B=ok\n";
3175 let mut plan2 = ExecutionPlan::new();
3176 plan2.filter = Some(FilterPredicate::Equal {
3177 field: "B".to_string(),
3178 value: "ok".to_string(),
3179 });
3180 let engine2 = parse_bytes(data2, plan2);
3181 assert_eq!(engine2.num_rows(), 2, "rows 1 and 3 pass (B == ok)");
3182 let col = engine2.get_column("A").unwrap();
3183 assert_eq!(col.as_str_vec(), vec![Some("1".into()), Some("3".into())]);
3184 }
3185
3186 #[test]
3187 fn test_predicate_first_via_resolve_and_put() {
3188 let mut plan = ExecutionPlan::new();
3191 plan.filter = Some(FilterPredicate::Equal {
3192 field: "B".to_string(),
3193 value: "reject".to_string(),
3194 });
3195 let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
3196 tb.begin_row();
3198 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("1")));
3199 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
3200 tb.end_row();
3201 tb.begin_row();
3203 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("2")));
3204 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("reject")));
3205 tb.end_row();
3206 tb.begin_row();
3208 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("3")));
3209 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
3210 tb.end_row();
3211 let batch = tb.finish().unwrap();
3212 assert_eq!(batch.num_rows(), 1, "only row 2 (B==reject) should pass");
3213 }
3214
3215 #[test]
3216 fn test_owned_cow_values_survive_buffered_filter() {
3217 let mut plan = ExecutionPlan::new();
3226 plan.filter = Some(FilterPredicate::Equal {
3227 field: "B".to_string(),
3228 value: "ok".to_string(),
3229 });
3230 let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
3231 for i in 0..50 {
3232 tb.begin_row();
3233 let a = Cow::Owned(format!("value-{i}-unescaped"));
3235 let b = Cow::Owned(if i % 2 == 0 {
3236 "ok".to_string()
3237 } else {
3238 "no".to_string()
3239 });
3240 tb.resolve_and_put("A", Value::Str(a));
3241 tb.resolve_and_put("B", Value::Str(b));
3242 tb.end_row();
3243 }
3244 assert_eq!(tb.num_rows(), 25, "even rows pass (B == ok)");
3245 let col = tb.get_column("A").unwrap();
3246 let vals = col.as_str_vec();
3247 assert_eq!(vals.len(), 25);
3248 assert_eq!(vals[0], Some("value-0-unescaped".to_string()));
3249 assert_eq!(vals[24], Some("value-48-unescaped".to_string()));
3250 }
3251
3252 #[test]
3253 fn test_predicate_first_skip_via_resolve_and_put() {
3254 let mut plan = ExecutionPlan::new();
3258 plan.filter = Some(FilterPredicate::Equal {
3259 field: "A".to_string(),
3260 value: "999".to_string(),
3261 });
3262 let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
3263 for _i in 0..100 {
3264 tb.begin_row();
3265 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("0")));
3266 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
3267 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
3268 tb.end_row();
3269 }
3270 let batch = tb.finish().unwrap();
3271 assert_eq!(
3272 batch.num_rows(),
3273 0,
3274 "all rows should be rejected (A=0 != 999)"
3275 );
3276 }
3277
3278 #[test]
3279 fn test_sparse_column_predicate_skip() {
3280 let mut plan = ExecutionPlan::new();
3289 plan.filter = Some(FilterPredicate::Equal {
3290 field: "A".to_string(),
3291 value: "reject".to_string(),
3292 });
3293 let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
3294 for i in 0..100 {
3295 tb.begin_row();
3296 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
3297 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
3298 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
3299 if i >= 50 {
3300 tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
3301 }
3302 tb.end_row();
3303 }
3304 let batch = tb.finish().unwrap();
3305 assert_eq!(batch.num_rows(), 0, "all rows rejected (A=ok != reject)");
3306 }
3307
3308 #[test]
3309 fn test_sparse_column_predicate_pass() {
3310 let mut plan = ExecutionPlan::new();
3313 plan.filter = Some(FilterPredicate::Equal {
3314 field: "A".to_string(),
3315 value: "ok".to_string(),
3316 });
3317 let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
3318 for _i in 0..100 {
3319 tb.begin_row();
3320 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
3321 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
3322 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
3323 if _i >= 50 {
3324 tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
3325 }
3326 tb.end_row();
3327 }
3328 let batch = tb.finish().unwrap();
3329 assert_eq!(batch.num_rows(), 100, "all rows pass (A=ok == ok)");
3330 if let Some(col) = batch.column_by_name("D") {
3332 let arr = col.as_string::<i32>();
3333 for i in 0..50 {
3334 assert!(arr.is_null(i), "row {} should have D=null", i);
3335 }
3336 for i in 50..100 {
3337 assert!(
3338 !arr.is_null(i),
3339 "row {} should not be null (D len={}, total rows={})",
3340 i,
3341 arr.len(),
3342 batch.num_rows()
3343 );
3344 assert_eq!(arr.value(i), "z", "rows 50+ should have D=z");
3345 }
3346 } else {
3347 panic!("D column should exist");
3348 }
3349 }
3350
3351 #[test]
3352 fn test_sparse_late_predicate_ordinal() {
3353 let mut plan = ExecutionPlan::new();
3357 plan.filter = Some(FilterPredicate::Equal {
3358 field: "E".to_string(),
3359 value: "999".to_string(),
3360 });
3361 let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
3362 for _i in 0..30 {
3364 tb.begin_row();
3365 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
3366 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
3367 tb.end_row();
3368 }
3369 for _i in 30..100 {
3371 tb.begin_row();
3372 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
3373 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
3374 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("c")));
3375 tb.resolve_and_put("D", Value::Str(Cow::Borrowed("d")));
3376 tb.resolve_and_put("E", Value::Str(Cow::Borrowed("e")));
3377 tb.end_row();
3378 }
3379 let batch = tb.finish().unwrap();
3380 assert_eq!(batch.num_rows(), 0, "all rows rejected (E=\"e\" != 999)");
3382 }
3383
3384 #[test]
3385 fn raw_name_helpers_apply_rename_and_drop() {
3386 use crate::decoder::ColumnarSink;
3387
3388 let plan = ExecutionPlan::new()
3389 .rename("raw", "renamed")
3390 .drop("ignored");
3391 let mut tb = TableBuilder::with_plan(2, Arc::new(plan));
3392 tb.begin_row();
3393 tb.resolve_and_put_raw(b"raw", Value::Str(Cow::Borrowed("value")));
3394 tb.resolve_and_put_raw(b"ignored", Value::Str(Cow::Borrowed("not stored")));
3395 tb.put_row(&[("other", Value::Str(Cow::Borrowed("also stored")))]);
3396 tb.end_row();
3397
3398 assert_eq!(tb.column_names(), &["renamed", "other"]);
3399 assert_eq!(tb.resolve_raw(b"raw"), Some("renamed"));
3400 }
3401
3402 struct SkippingParser;
3413
3414 impl RecordParser for SkippingParser {
3415 fn validate(&self, bytes: &[u8]) -> Result<()> {
3416 simdutf8::basic::from_utf8(bytes)?;
3417 Ok(())
3418 }
3419
3420 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
3421 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
3422 for line in text.lines() {
3423 if line.is_empty() {
3424 continue;
3425 }
3426 sink.begin_row();
3427 for token in line.split_whitespace() {
3428 if sink.row_satisfied() {
3429 break; }
3431 if let Some((k, v)) = token.split_once('=') {
3432 if sink.resolve(k).is_some() {
3433 sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
3434 }
3435 }
3436 }
3437 sink.end_row();
3438 }
3439 Ok(())
3440 }
3441 }
3442
3443 fn parse_skipping(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
3444 let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
3445 SkippingParser.parse_chunk(bytes, &mut sink).unwrap();
3446 sink
3447 }
3448
3449 fn batch_names(tb: &mut TableBuilder) -> Vec<String> {
3450 tb.finish()
3451 .unwrap()
3452 .schema()
3453 .fields()
3454 .iter()
3455 .map(|f| f.name().clone())
3456 .collect()
3457 }
3458
3459 const REPORT: &[u8] = b"Name=n1 Department=d1 Amount=1 Status=s1 Date=2024\n\
3460 Name=n2 Department=d2 Amount=2 Status=s2 Date=2024\n";
3461
3462 #[test]
3463 fn schema_projection_subset() {
3464 let mut tb = parse_skipping(
3469 REPORT,
3470 ExecutionPlan::new().schema_order(["Name", "Department"]),
3471 );
3472 assert_eq!(batch_names(&mut tb), vec!["Name", "Department"]);
3473 }
3474
3475 #[test]
3476 fn schema_projection_single_non_first_column() {
3477 let mut tb = parse_skipping(REPORT, ExecutionPlan::new().schema_order(["Department"]));
3480 assert_eq!(batch_names(&mut tb), vec!["Department"]);
3481 }
3482
3483 #[test]
3484 fn schema_projection_full_natural_order() {
3485 let mut tb = parse_skipping(
3486 REPORT,
3487 ExecutionPlan::new().schema_order(["Name", "Department", "Amount", "Status", "Date"]),
3488 );
3489 assert_eq!(
3490 batch_names(&mut tb),
3491 vec!["Name", "Department", "Amount", "Status", "Date"]
3492 );
3493 }
3494
3495 #[test]
3496 fn schema_projection_reversed_order() {
3497 let mut tb = parse_skipping(
3498 REPORT,
3499 ExecutionPlan::new().schema_order(["Date", "Status", "Amount", "Department", "Name"]),
3500 );
3501 assert_eq!(
3502 batch_names(&mut tb),
3503 vec!["Date", "Status", "Amount", "Department", "Name"]
3504 );
3505 }
3506
3507 #[test]
3508 fn schema_projection_absent_column_null_filled() {
3509 let mut tb = parse_skipping(
3511 REPORT,
3512 ExecutionPlan::new().schema_order(["Name", "Missing"]),
3513 );
3514 let batch = tb.finish().unwrap();
3515 assert_eq!(batch.num_rows(), 2);
3516 let missing = batch.column_by_name("Missing").unwrap().as_string::<i32>();
3517 assert!(missing.is_null(0) && missing.is_null(1));
3518 let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
3519 assert_eq!(name.value(0), "n1");
3520 }
3521
3522 #[test]
3523 fn schema_projection_wants_rejects_unlisted_fields() {
3524 let plan = ExecutionPlan::new().schema_order(["Name"]);
3527 let tb = TableBuilder::with_plan(4, Arc::new(plan));
3528 assert!(tb.wants("Name"));
3529 assert!(!tb.wants("Department"));
3530 assert_eq!(tb.resolve("Amount"), None);
3531 }
3532
3533 #[test]
3534 fn schema_projection_rename_then_match() {
3535 let plan = ExecutionPlan::new()
3538 .rename("Name", "Employee")
3539 .schema_order(["Employee", "Amount"]);
3540 let mut tb = parse_skipping(REPORT, plan);
3541 assert_eq!(batch_names(&mut tb), vec!["Employee", "Amount"]);
3542 }
3543
3544 #[test]
3545 fn schema_projection_field_types_apply() {
3546 let plan = ExecutionPlan::new()
3547 .type_as("Amount", FieldType::Int64)
3548 .schema_order(["Name", "Amount"]);
3549 let mut tb = parse_skipping(REPORT, plan);
3550 let batch = tb.finish().unwrap();
3551 let amount = batch.column_by_name("Amount").unwrap();
3552 assert!(matches!(
3553 amount.data_type(),
3554 arrow::datatypes::DataType::Int64
3555 ));
3556 assert_eq!(
3557 amount
3558 .as_primitive::<arrow::datatypes::Int64Type>()
3559 .value(1),
3560 2
3561 );
3562 }
3563
3564 #[test]
3565 fn schema_projection_filter_on_unlisted_field() {
3566 let plan = ExecutionPlan::new()
3569 .schema_order(["Name"])
3570 .filter_eq("Status", "s2");
3571 let mut tb = parse_skipping(REPORT, plan);
3572 let batch = tb.finish().unwrap();
3573 assert_eq!(batch.num_rows(), 1);
3574 assert_eq!(batch.schema().fields().len(), 1);
3575 let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
3576 assert_eq!(name.value(0), "n2");
3577 }
3578
3579 #[test]
3580 fn schema_projection_drop_wins_over_schema() {
3581 let plan = ExecutionPlan::new()
3585 .schema_order(["Name", "Department"])
3586 .drop("Department");
3587 let mut tb = parse_skipping(REPORT, plan);
3588 assert_eq!(batch_names(&mut tb), vec!["Name"]);
3589 }
3590
3591 #[test]
3592 fn schema_projection_no_parser_cooperation() {
3593 let mut tb = parse_bytes(
3596 REPORT,
3597 ExecutionPlan::new().schema_order(["Department", "Name"]),
3598 );
3599 let batch = tb.finish().unwrap();
3600 let names: Vec<String> = batch
3601 .schema()
3602 .fields()
3603 .iter()
3604 .map(|f| f.name().clone())
3605 .collect();
3606 assert_eq!(names, vec!["Department", "Name"]);
3607 let dept = batch
3608 .column_by_name("Department")
3609 .unwrap()
3610 .as_string::<i32>();
3611 assert_eq!(dept.value(0), "d1");
3612 }
3613}