1#[cfg(test)]
2use std::borrow::Cow;
3use std::panic::{catch_unwind, AssertUnwindSafe};
4use std::sync::Arc;
5
6#[cfg(feature = "profile")]
7use std::sync::atomic::{AtomicUsize, Ordering};
8
9#[cfg(feature = "profile")]
11pub static RESOLVE_AND_PUT_COUNT: AtomicUsize = AtomicUsize::new(0);
12#[cfg(feature = "profile")]
14pub static PREDICATE_EVALUATIONS: AtomicUsize = AtomicUsize::new(0);
15#[cfg(feature = "profile")]
17pub static PREDICATE_FAILS: AtomicUsize = AtomicUsize::new(0);
18#[cfg(feature = "profile")]
20pub static PREDICATE_UNDECIDED: AtomicUsize = AtomicUsize::new(0);
21#[cfg(feature = "profile")]
23pub static IS_PRED_TRUE: AtomicUsize = AtomicUsize::new(0);
24#[cfg(feature = "profile")]
26pub static IS_PRED_FALSE: AtomicUsize = AtomicUsize::new(0);
27
28use arrow::datatypes::{Field as ArrowField, Schema};
29use arrow::record_batch::RecordBatch;
30use rustc_hash::FxHashMap as HashMap;
31
32use smallvec::SmallVec;
33
34use crate::columnar::ColumnBuilder;
35use crate::decoder::ColumnarSink;
36use crate::engine::predicate::{PredicateState, RowBuffer};
37use crate::plan::{ExecutionPlan, FieldType, FilterPredicate};
38use crate::value::Value;
39use crate::Result;
40
41mod memory;
42mod predicates;
43
44pub struct TableBuilder {
48 pub(crate) memory_limit: Option<usize>,
49 pub(crate) columns: Vec<ColumnBuilder>,
50 pub(crate) field_index: HashMap<String, usize>,
51 pub(crate) column_order: Vec<String>,
52 pub(crate) row_count: usize,
53 pub(crate) estimated_rows: usize,
54 pub(crate) plan: Arc<ExecutionPlan>,
55 pub(crate) row_dirty: Vec<u64>,
60 pub(crate) frozen: Option<std::sync::Arc<crate::schema::FrozenSchema>>,
64 pub(crate) unknown_error: Option<String>,
65 pub(crate) row_buf: Option<Box<RowBuffer>>,
69 pub(crate) cached_wanted_mask: u64,
77 pub(crate) ordinal_expect: Vec<Option<(u32, Vec<u8>)>>,
82 pub(crate) current_ordinal: u32,
85 pub(crate) strict_error: Option<crate::Error>,
89 pub(crate) rows_accepted: usize,
92 pub(crate) rows_rejected: usize,
94}
95
96impl TableBuilder {
97 pub fn new() -> Self {
98 Self {
99 memory_limit: None,
100 columns: Vec::new(),
101 field_index: HashMap::default(),
102 column_order: Vec::new(),
103 row_count: 0,
104 estimated_rows: 0,
105 plan: Arc::new(ExecutionPlan::new()),
106 row_dirty: Vec::new(),
107 frozen: None,
108 unknown_error: None,
109 row_buf: None,
110 cached_wanted_mask: 0,
111 ordinal_expect: Vec::new(),
112 current_ordinal: 0,
113 strict_error: None,
114 rows_accepted: 0,
115 rows_rejected: 0,
116 }
117 }
118
119 pub fn with_capacity(cap: usize) -> Self {
120 Self {
121 memory_limit: None,
122 columns: Vec::new(),
123 field_index: HashMap::default(),
124 column_order: Vec::new(),
125 row_count: 0,
126 estimated_rows: cap,
127 plan: Arc::new(ExecutionPlan::new()),
128 row_dirty: Vec::new(),
129 frozen: None,
130 unknown_error: None,
131 row_buf: None,
132 cached_wanted_mask: 0,
133 ordinal_expect: Vec::new(),
134 current_ordinal: 0,
135 strict_error: None,
136 rows_accepted: 0,
137 rows_rejected: 0,
138 }
139 }
140
141 pub fn with_plan(cap: usize, plan: Arc<ExecutionPlan>) -> Self {
142 let mut b = Self {
143 memory_limit: None,
144 columns: Vec::new(),
145 field_index: HashMap::default(),
146 column_order: Vec::new(),
147 row_count: 0,
148 estimated_rows: cap,
149 plan: Arc::clone(&plan),
150 row_dirty: Vec::new(),
151 frozen: None,
152 unknown_error: None,
153 row_buf: if plan.filter.is_some() {
154 Some(Box::new(RowBuffer {
155 fields: SmallVec::new(),
156 state: PredicateState::Undecided,
157 predicate_mask: SmallVec::new(),
158 direct: false,
159 predicate_ordinal: None,
160 buffer_worthwhile: true,
161 pred_names: SmallVec::new(),
162 }))
163 } else {
164 None
165 },
166 cached_wanted_mask: 0,
167 ordinal_expect: Vec::new(),
168 current_ordinal: 0,
169 strict_error: None,
170 rows_accepted: 0,
171 rows_rejected: 0,
172 };
173 b.predeclare_schema_columns();
174 b.cached_wanted_mask = b.compute_wanted_mask();
175 b
176 }
177
178 fn predeclare_schema_columns(&mut self) {
184 if self.plan.schema_order.is_empty() {
185 return;
186 }
187 let mut names: Vec<String> = Vec::new();
188 for name in &self.plan.schema_order {
189 if !self.plan.drop_fields.contains(name) {
193 names.push(name.clone());
194 }
195 }
196 if let Some(ref filter) = self.plan.filter {
199 let mut pred_names: SmallVec<[String; 4]> = SmallVec::new();
200 Self::collect_predicate_field_names(filter, &self.plan, &mut pred_names);
201 names.extend(pred_names);
202 }
203 for name in names {
204 if self.field_index.contains_key(&name) || self.plan.drop_fields.contains(&name) {
205 continue;
206 }
207 let ty = self.plan.column_type(&name);
208 let col = ColumnBuilder::with_capacity(self.estimated_rows.max(64), &ty);
209 self.columns.push(col);
210 self.field_index
211 .insert(name.clone(), self.columns.len() - 1);
212 self.column_order.push(name);
213 }
214 let words = self.columns.len().div_ceil(64);
215 self.row_dirty.resize(words, 0);
216 }
217
218 fn compute_wanted_mask(&self) -> u64 {
225 if self.plan.schema_order.is_empty() {
226 return 0;
227 }
228 let mut mask = 0u64;
229 for &idx in self.field_index.values() {
230 if idx >= 64 {
231 return 0;
232 }
233 mask |= 1u64 << idx;
234 }
235 mask
236 }
237
238 pub fn ensure_schema(&mut self, schema: &crate::schema::FrozenSchema) -> Result<()> {
244 for (slot, name) in schema.column_names().iter().enumerate() {
245 let name_str: &str = name;
246 if self.field_index.contains_key(name_str) {
247 continue; }
249 let ty = schema.column_types()[slot].clone();
250 let col = ColumnBuilder::with_capacity(self.estimated_rows, &ty);
251 self.columns.push(col);
252 self.field_index
253 .insert(name_str.to_string(), self.columns.len() - 1);
254 self.column_order.push(name_str.to_string());
255 }
256 let words = self.columns.len().div_ceil(64);
258 self.row_dirty.resize(words, 0);
259 self.frozen = Some(std::sync::Arc::new(schema.clone()));
264 Ok(())
265 }
266
267 pub(crate) fn get_column(&self, name: &str) -> Option<&ColumnBuilder> {
269 self.field_index.get(name).map(|&i| &self.columns[i])
270 }
271
272 pub(crate) fn get_column_mut(&mut self, name: &str) -> Option<&mut ColumnBuilder> {
274 if let Some(&i) = self.field_index.get(name) {
275 Some(&mut self.columns[i])
276 } else {
277 None
278 }
279 }
280
281 pub(crate) fn bytes_used(&self) -> usize {
282 self.columns.iter().map(|c| c.bytes_used()).sum::<usize>()
283 + self.column_order.iter().map(|s| s.len()).sum::<usize>()
284 + self.row_dirty.len() * 8
285 }
286
287 pub(crate) fn split_off(&mut self, n: usize) -> Self {
293 assert!(n <= self.row_count, "split_off beyond row_count");
294 assert!(n > 0);
295 let mut other = Self {
296 memory_limit: self.memory_limit,
297 columns: Vec::with_capacity(self.columns.len()),
298 field_index: self.field_index.clone(),
299 column_order: self.column_order.clone(),
300 row_count: n,
301 estimated_rows: n,
302 plan: Arc::clone(&self.plan),
303 row_dirty: vec![0; self.columns.len().div_ceil(64)],
304 frozen: self.frozen.clone(),
305 unknown_error: None,
306 cached_wanted_mask: self.cached_wanted_mask,
307 row_buf: if self.plan.filter.is_some() {
308 Some(Box::new(RowBuffer {
309 fields: SmallVec::new(),
310 state: PredicateState::Undecided,
311 predicate_mask: self
312 .row_buf
313 .as_ref()
314 .map_or_else(SmallVec::new, |b| b.predicate_mask.clone()),
315 direct: false,
316 predicate_ordinal: None,
317 buffer_worthwhile: true,
318 pred_names: self
319 .row_buf
320 .as_ref()
321 .map_or_else(SmallVec::new, |b| b.pred_names.clone()),
322 }))
323 } else {
324 None
325 },
326 ordinal_expect: Vec::new(),
327 current_ordinal: 0,
328 strict_error: None,
329 rows_accepted: std::mem::take(&mut self.rows_accepted),
332 rows_rejected: std::mem::take(&mut self.rows_rejected),
333 };
334 for (idx, col) in self.columns.iter_mut().enumerate() {
335 let drain = col.split_off(n);
336 other.columns.push(drain);
337 let _ = idx;
339 }
340 self.row_count -= n;
341 self.row_dirty = vec![0; self.columns.len().div_ceil(64)];
343 other
345 }
346
347 pub(crate) fn take_column(&mut self, name: &str) -> Option<ColumnBuilder> {
350 let idx = self.field_index.remove(name)?;
351 let new_len = self.columns.len().div_ceil(64);
356 self.row_dirty.resize(new_len, 0);
357 self.row_dirty.fill(0);
359 let _ = idx; let last = self.columns.len() - 1;
363 let col = if idx == last {
364 self.columns.pop().unwrap()
365 } else {
366 let col = self.columns.swap_remove(idx);
367 let old_last = self.columns.len(); let moved_name = self.field_index.iter().find_map(|(k, &v)| {
372 if v == old_last {
373 Some(k.clone())
374 } else {
375 None
376 }
377 });
378 if let Some(k) = moved_name {
379 self.field_index.insert(k, idx);
380 }
381 col
382 };
383 Some(col)
384 }
385
386 pub fn num_rows(&self) -> usize {
387 self.row_count
388 }
389
390 pub fn num_columns(&self) -> usize {
391 self.column_order.len()
392 }
393
394 pub fn column_names(&self) -> &[String] {
395 &self.column_order
396 }
397
398 pub fn column_diagnostics(&self) -> Vec<(String, usize, usize)> {
400 self.column_order
401 .iter()
402 .map(|name| {
403 let idx = self.field_index[name];
404 let col = &self.columns[idx];
405 (name.clone(), col.bytes_used(), col.capacity_bytes())
406 })
407 .collect()
408 }
409
410 pub fn estimated_rows(&self) -> usize {
412 self.estimated_rows
413 }
414
415 pub fn finish(&mut self) -> Result<RecordBatch> {
419 ColumnarSink::finish(self)
420 }
421
422 pub(crate) fn prepare_finish(&mut self) -> Result<()> {
423 if let Some(err) = self.unknown_error.take() {
424 return Err(crate::Error::Merge(err));
425 }
426 if let Some(err) = self.strict_error.take() {
427 return Err(err);
428 }
429 self.check_memory_budget()?;
430 if let Some(ref observer) = self.plan.observer {
431 let result = catch_unwind(AssertUnwindSafe(|| {
432 observer.on_chunk_finished(
433 self.rows_accepted + self.rows_rejected,
434 self.rows_accepted,
435 self.rows_rejected,
436 );
437 }));
438 if let Err(payload) = result {
439 let message = payload
440 .downcast_ref::<&str>()
441 .map(|s| (*s).to_string())
442 .or_else(|| payload.downcast_ref::<String>().cloned())
443 .unwrap_or_else(|| "unknown panic".to_string());
444 return Err(crate::Error::Parser(format!(
445 "observer panicked at chunk completion: {message}"
446 )));
447 }
448 }
449 self.normalize();
450 Ok(())
451 }
452
453 pub fn reset(&mut self) {
455 self.columns.clear();
456 self.field_index.clear();
457 self.column_order.clear();
458 self.row_dirty.fill(0);
459 self.row_count = 0;
460 if let Some(ref mut buf) = self.row_buf {
461 buf.fields.clear();
462 buf.state = PredicateState::Undecided;
463 }
464 self.unknown_error = None;
466 self.strict_error = None;
467 self.rows_accepted = 0;
468 self.rows_rejected = 0;
469 }
470
471 pub fn normalize(&mut self) {
474 for b in &mut self.columns {
475 while b.len() > self.row_count {
476 b.pop();
477 }
478 }
479 self.row_dirty.fill(0);
481 }
482
483 pub fn auto_dict_upgrade(&mut self) {
486 if self.plan.auto_dict {
487 let max_ratio = self.plan.dict_threshold.unwrap_or(0.05);
488 let max_size = self.plan.dict_max_size.unwrap_or(256);
489 for b in &mut self.columns {
490 b.try_upgrade_to_dict(512, max_ratio, max_size);
491 }
492 }
493 }
494
495 pub fn sort_columns(&mut self) {
499 if self.plan.schema_order.is_empty() {
500 return;
501 }
502 let order = &self.plan.schema_order;
503 let rank = |name: &String| order.iter().position(|n| n == name).unwrap_or(usize::MAX);
504 self.column_order.sort_by_key(rank);
505 }
506
507 pub(crate) fn schema_insert_index(&self, name: &str) -> usize {
508 let order = &self.plan.schema_order;
509 if order.is_empty() {
510 return self.column_order.len();
511 }
512 let pos = order.iter().position(|n| n == name);
513 match pos {
514 Some(p) => self
515 .column_order
516 .iter()
517 .position(|existing| {
518 order
519 .iter()
520 .position(|n| n == existing)
521 .is_some_and(|ep| ep > p)
522 })
523 .unwrap_or(self.column_order.len()),
524 None => self.column_order.len(),
525 }
526 }
527
528 fn ensure_column_idx(&mut self, name: &str) -> usize {
532 if let Some(&idx) = self.field_index.get(name) {
533 return idx;
534 }
535 if self.frozen.is_some() {
542 if let Some(frozen) = &self.frozen {
543 if frozen.is_exact() {
544 if self.unknown_error.is_none() {
546 self.unknown_error = Some(format!(
547 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
548 name,
549 frozen.num_columns(),
550 frozen.is_exact()
551 ));
552 }
553 return 0;
554 }
555 }
557 }
558 let est = self.estimated_rows.max(64);
559 let col_type = self.plan.column_type(name);
560 let mut b = ColumnBuilder::with_capacity(est, &col_type);
561 let backfill = if self.row_buf.as_ref().is_some_and(|b| b.direct) {
566 self.row_count.saturating_sub(1)
567 } else {
568 self.row_count
569 };
570 for _ in 0..backfill {
571 b.push(None);
572 }
573 let idx = self.columns.len();
574 self.columns.push(b);
575 self.field_index.insert(name.to_owned(), idx);
576 let needed = self.columns.len().div_ceil(64);
578 if self.row_dirty.len() < needed {
579 self.row_dirty.resize(needed, 0);
580 }
581 let order_idx = self.schema_insert_index(name);
582 self.column_order.insert(order_idx, name.to_owned());
583 idx
584 }
585
586 fn record_strict_violation_at(&mut self, idx: usize, value: &Value<'_>) {
592 if !self.plan.strict_types || self.strict_error.is_some() {
593 return;
594 }
595 let Value::Str(s) = value else {
596 return;
597 };
598 let Some(name) = self.column_order.get(idx) else {
599 return;
600 };
601 let Some(ft) = self.plan.field_types.get(name) else {
602 return;
603 };
604 if crate::columnar::parses_as(s, ft) {
605 return;
606 }
607 let truncated: String = s.chars().take(64).collect();
608 self.strict_error = Some(crate::Error::Plan(format!(
609 "strict_types: value {truncated:?} in column {name:?} at row {} \
610 does not parse as declared type {ft:?}",
611 self.row_count
612 )));
613 }
614
615 #[inline]
618 fn notify_put_field(&self, idx: usize, value: &Value<'_>) {
619 if let Some(ref obs) = self.plan.observer {
620 if let Some(name) = self.column_order.get(idx) {
621 obs.on_put_field(self.row_count, name, idx, value);
622 }
623 }
624 }
625
626 #[inline]
629 fn push_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
630 if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
637 if let Some(frozen) = &self.frozen {
638 if frozen.is_exact() {
639 if self.unknown_error.is_none() {
641 self.unknown_error = Some(format!(
642 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
643 resolved_name,
644 frozen.num_columns(),
645 frozen.is_exact()
646 ));
647 }
648 return;
649 }
650 }
652 }
653 let idx = self.ensure_column_idx(resolved_name);
654 self.record_strict_violation_at(idx, &value);
655 self.notify_put_field(idx, &value);
656 if self.row_count == 0 {
658 let ord = self.current_ordinal as usize;
659 if ord >= self.ordinal_expect.len() {
660 self.ordinal_expect.resize_with(ord + 1, || None);
661 }
662 self.ordinal_expect[ord] = Some((idx as u32, resolved_name.as_bytes().to_vec()));
663 }
664 self.current_ordinal += 1;
665 let word = idx / 64;
666 let bit = idx % 64;
667 self.row_dirty[word] |= 1u64 << bit;
668 let b = &mut self.columns[idx];
669 let row_count = self.row_count;
670 if b.len() > row_count {
671 b.pop();
672 }
673 b.push_value(value);
674 }
675
676 #[inline]
679 fn push_field(&mut self, name: &str, value: Value<'_>) {
680 if self.plan.field_map.is_empty()
683 && self.plan.drop_fields.is_empty()
684 && self.plan.schema_order.is_empty()
685 {
686 self.push_field_resolved(name, value);
687 return;
688 }
689 if let Some(idx) = Self::resolve_and_slot(&self.plan, &self.field_index, name) {
692 self.record_strict_violation_at(idx, &value);
693 self.notify_put_field(idx, &value);
694 if self.row_count == 0 {
696 let ord = self.current_ordinal as usize;
697 if ord >= self.ordinal_expect.len() {
698 self.ordinal_expect.resize_with(ord + 1, || None);
699 }
700 let resolved = self.plan.resolve_field(name).unwrap_or(name);
702 self.ordinal_expect[ord] = Some((idx as u32, resolved.as_bytes().to_vec()));
703 }
704 self.current_ordinal += 1;
705 let word = idx / 64;
706 let bit = idx % 64;
707 self.row_dirty[word] |= 1u64 << bit;
708 let b = &mut self.columns[idx];
709 let row_count = self.row_count;
710 if b.len() > row_count {
711 b.pop();
712 }
713 b.push_value(value);
714 } else {
715 if let Some(resolved) = self.plan.resolve_field(name) {
718 let owned = resolved.to_owned();
719 self.push_field_resolved(&owned, value);
720 }
721 }
722 }
723
724 #[inline]
729 fn resolve_and_slot(
730 plan: &ExecutionPlan,
731 field_index: &HashMap<String, usize>,
732 name: &str,
733 ) -> Option<usize> {
734 let resolved = plan.resolve_field(name)?;
735 field_index.get(resolved).copied()
736 }
737
738 #[doc(hidden)]
742 pub fn advance_row(&mut self) {
743 self.row_count += 1;
744 }
745
746 fn finish_row(&mut self) {
751 let ncols = self.columns.len();
753 let full_words = ncols / 64;
755 let rem_bits = ncols % 64;
756 let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
757 && (rem_bits == 0
758 || self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
759 if is_full {
760 self.row_dirty.fill(0);
761 } else {
762 for (i, b) in self.columns.iter_mut().enumerate() {
763 let word = i / 64;
764 let bit = i % 64;
765 let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
766 if !is_set {
767 b.push(None);
768 }
769 }
770 self.row_dirty.fill(0);
771 }
772
773 if let Some(ref filter) = self.plan.filter {
774 if !filter.check(&self.columns, &self.field_index, self.row_count, &self.plan) {
775 for b in &mut self.columns {
776 b.pop();
777 }
778 self.rows_rejected += 1;
779 if let Some(ref obs) = self.plan.observer {
780 obs.on_row_rejected(self.row_count);
781 }
782 return;
783 }
784 }
785
786 self.row_count += 1;
787 self.rows_accepted += 1;
788 if let Some(ref obs) = self.plan.observer {
789 obs.on_row_accepted(self.row_count - 1);
790 }
791 }
792
793 fn null_fill_missing(&mut self) {
796 let ncols = self.columns.len();
797 let full_words = ncols / 64;
798 let rem_bits = ncols % 64;
799 let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
800 && (rem_bits == 0
801 || self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
802 if is_full {
803 self.row_dirty.fill(0);
804 } else {
805 for (i, b) in self.columns.iter_mut().enumerate() {
806 let word = i / 64;
807 let bit = i % 64;
808 let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
809 if !is_set {
810 b.push(None);
811 }
812 }
813 self.row_dirty.fill(0);
814 }
815 }
818}
819
820impl Default for TableBuilder {
821 fn default() -> Self {
822 Self::new()
823 }
824}
825
826impl ColumnarSink for TableBuilder {
827 #[inline]
828 fn begin_row(&mut self) {
829 if !self.reserve_memory(0) {
830 return;
831 }
832 if let Some(ref obs) = self.plan.observer {
833 obs.on_begin_row(self.row_count);
834 }
835 self.current_ordinal = 0;
836 if let Some(ref mut buf) = self.row_buf {
837 if buf.predicate_ordinal.is_some() && buf.buffer_worthwhile && self.row_count > 0 {
843 let ncols = self
846 .frozen
847 .as_ref()
848 .map(|f| f.num_columns() as u32)
849 .or_else(|| {
850 (!self.plan.schema_order.is_empty())
851 .then(|| self.plan.schema_order.len() as u32)
852 })
853 .unwrap_or(self.columns.len() as u32);
854 if let Some(ordinal) = buf.predicate_ordinal {
855 if ordinal >= ncols * 4 / 5 {
856 buf.buffer_worthwhile = false;
857 }
858 }
859 }
860 buf.fields.clear();
861 buf.state = PredicateState::Undecided;
862 buf.direct = false;
863 }
864 }
865
866 #[inline]
867 fn put_field(&mut self, name: &str, value: Value<'_>) {
868 if !self.reserve_field_memory(name.len(), &value) {
869 return;
870 }
871 if self.plan.filter.is_none() {
872 self.push_field(name, value);
873 return;
874 }
875 let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
877 if !buf_worthwhile {
878 self.push_field(name, value);
879 return;
880 }
881 let resolved = match self.plan.resolve_field(name) {
883 Some(r) => r.to_owned(),
884 None => return,
885 };
886 if self.frozen.is_some()
888 && !self.field_index.contains_key(&resolved)
889 && self.unknown_error.is_none()
890 {
891 if let Some(ref frozen) = self.frozen {
892 if frozen.is_exact()
893 && !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
894 {
895 self.unknown_error = Some(format!(
896 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
897 resolved,
898 frozen.num_columns(),
899 frozen.is_exact()
900 ));
901 return;
902 }
903 }
904 }
905 let slot = self.ensure_column_idx(&resolved) as u32;
906 self.record_strict_violation_at(slot as usize, &value);
907 self.mark_predicate_slot(slot);
908 let val_static: Value<'static> = value.into_static();
909 let is_pred = self.is_predicate_slot(slot);
910 #[cfg(feature = "profile")]
911 if is_pred {
912 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
913 } else {
914 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
915 }
916 if let Some(ref mut buf) = self.row_buf {
918 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
919 buf.fields[pos].1 = val_static;
920 } else {
921 buf.fields.push((slot, val_static));
922 }
923 }
924 if is_pred {
925 if let Some(ref mut buf) = self.row_buf {
927 if buf.predicate_ordinal.is_none() {
928 buf.predicate_ordinal = Some(slot);
929 }
930 }
931 let state = self.evaluate_predicate_state();
932 if let Some(ref mut buf) = self.row_buf {
933 buf.state = state;
934 }
935 }
936 }
937
938 #[inline]
939 fn end_row(&mut self) {
940 if !self.reserve_memory(0) {
941 return;
942 }
943 if self.plan.filter.is_none() {
944 self.finish_row();
945 return;
946 }
947 let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
950 if !buf_worthwhile {
951 self.finish_row();
955 return;
956 }
957 let (state, direct) = self
959 .row_buf
960 .as_ref()
961 .map_or((PredicateState::Pass, false), |b| (b.state, b.direct));
962 if direct {
963 self.null_fill_missing();
965 return;
966 }
967 let state = if state == PredicateState::Undecided {
968 self.evaluate_against_null()
969 } else {
970 state
971 };
972 let pass = state == PredicateState::Pass;
973 self.drain_buffered(pass);
974 }
975
976 #[inline]
977 fn wants(&self, name: &str) -> bool {
978 self.resolve(name).is_some()
979 }
980
981 #[inline]
982 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
983 self.plan.resolve_field(name)
984 }
985
986 #[inline]
987 fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
988 if !self.reserve_field_memory(resolved_name.len(), &value) {
989 return;
990 }
991 if self.plan.filter.is_none() {
992 self.push_field_resolved(resolved_name, value);
993 return;
994 }
995 let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
997 if !buf_worthwhile {
998 self.push_field_resolved(resolved_name, value);
999 return;
1000 }
1001 if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
1003 if let Some(ref frozen) = self.frozen {
1004 if frozen.is_exact()
1005 && !frozen
1006 .column_names()
1007 .iter()
1008 .any(|n| n.as_ref() == resolved_name)
1009 {
1010 if self.unknown_error.is_none() {
1011 self.unknown_error = Some(format!(
1012 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1013 resolved_name,
1014 frozen.num_columns(),
1015 frozen.is_exact()
1016 ));
1017 }
1018 return;
1019 }
1020 }
1021 }
1022 let slot = self.ensure_column_idx(resolved_name) as u32;
1023 self.record_strict_violation_at(slot as usize, &value);
1024 self.mark_predicate_slot(slot);
1025 let val_static: Value<'static> = value.into_static();
1026 let is_pred = self.is_predicate_slot(slot);
1027 #[cfg(feature = "profile")]
1028 if is_pred {
1029 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1030 } else {
1031 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1032 }
1033 if let Some(ref mut buf) = self.row_buf {
1034 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1035 buf.fields[pos].1 = val_static;
1036 } else {
1037 buf.fields.push((slot, val_static));
1038 }
1039 }
1040 if is_pred {
1041 if let Some(ref mut buf) = self.row_buf {
1043 if buf.predicate_ordinal.is_none() {
1044 buf.predicate_ordinal = Some(slot);
1045 }
1046 }
1047 let state = self.evaluate_predicate_state();
1048 if let Some(ref mut buf) = self.row_buf {
1049 buf.state = state;
1050 }
1051 }
1052 }
1053
1054 #[inline]
1058 fn put_field_at(&mut self, slot: u32, value: Value<'_>) {
1059 if !self.reserve_field_memory(0, &value) {
1060 return;
1061 }
1062 let idx = slot as usize;
1063 if idx >= self.columns.len() {
1064 return;
1067 }
1068 self.record_strict_violation_at(idx, &value);
1069 self.notify_put_field(idx, &value);
1070 let word = idx / 64;
1071 let bit = idx % 64;
1072 self.row_dirty[word] |= 1u64 << bit;
1073 let b = &mut self.columns[idx];
1074 let row_count = self.row_count;
1075 if b.len() > row_count {
1076 b.pop();
1077 }
1078 b.push_value(value);
1079 }
1080
1081 #[inline]
1082 fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
1083 if !self.reserve_field_memory(name.len(), &value) {
1084 return;
1085 }
1086 #[cfg(feature = "profile")]
1087 RESOLVE_AND_PUT_COUNT.fetch_add(1, Ordering::Relaxed);
1088 if self.plan.filter.is_none() {
1089 if self.plan.field_map.is_empty()
1090 && self.plan.drop_fields.is_empty()
1091 && self.plan.schema_order.is_empty()
1092 {
1093 self.push_field_resolved(name, value);
1094 } else {
1095 if let Some(resolved) = self.plan.resolve_field(name) {
1096 let owned = resolved.to_owned();
1097 self.push_field_resolved(&owned, value);
1098 }
1099 }
1100 return;
1101 }
1102 if self.row_buf.as_ref().is_some_and(|b| !b.buffer_worthwhile) {
1104 if self.plan.field_map.is_empty()
1105 && self.plan.drop_fields.is_empty()
1106 && self.plan.schema_order.is_empty()
1107 {
1108 self.push_field_resolved(name, value);
1109 } else {
1110 if let Some(resolved) = self.plan.resolve_field(name) {
1111 let owned = resolved.to_owned();
1112 self.push_field_resolved(&owned, value);
1113 }
1114 }
1115 return;
1116 }
1117 if let Some(ref buf) = self.row_buf {
1119 if buf.direct {
1120 let resolved = match self.plan.resolve_field(name) {
1121 Some(r) => r.to_owned(),
1122 None => return, };
1124 if self.frozen.is_some() && !self.field_index.contains_key(resolved.as_str()) {
1125 if let Some(ref frozen) = self.frozen {
1126 if frozen.is_exact()
1127 && !frozen
1128 .column_names()
1129 .iter()
1130 .any(|n| n.as_ref() == resolved.as_str())
1131 {
1132 if self.unknown_error.is_none() {
1133 self.unknown_error = Some(format!(
1134 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1135 resolved, frozen.num_columns(), frozen.is_exact()
1136 ));
1137 }
1138 return;
1139 }
1140 }
1141 }
1142 self.push_field_resolved(&resolved, value);
1143 return;
1144 }
1145 }
1146 if self.plan.field_map.is_empty()
1148 && self.plan.drop_fields.is_empty()
1149 && self.plan.schema_order.is_empty()
1150 {
1151 let resolved = name;
1153 if self.frozen.is_some() && !self.field_index.contains_key(resolved) {
1154 if let Some(ref frozen) = self.frozen {
1155 if frozen.is_exact()
1156 && !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
1157 {
1158 if self.unknown_error.is_none() {
1159 self.unknown_error = Some(format!(
1160 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1161 resolved,
1162 frozen.num_columns(),
1163 frozen.is_exact()
1164 ));
1165 }
1166 return;
1167 }
1168 }
1169 }
1170 let slot = self.ensure_column_idx(resolved) as u32;
1171 self.record_strict_violation_at(slot as usize, &value);
1172 self.mark_predicate_slot(slot);
1173 let val_static: Value<'static> = value.into_static();
1174 let is_pred = self.is_predicate_slot(slot);
1175 #[cfg(feature = "profile")]
1176 if is_pred {
1177 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1178 } else {
1179 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1180 }
1181 if let Some(ref mut buf) = self.row_buf {
1182 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1183 buf.fields[pos].1 = val_static;
1184 } else {
1185 buf.fields.push((slot, val_static));
1186 }
1187 }
1188 if is_pred {
1189 if let Some(ref mut buf) = self.row_buf {
1191 if buf.predicate_ordinal.is_none() {
1192 buf.predicate_ordinal = Some(slot);
1193 }
1194 }
1195 let state = self.evaluate_predicate_state();
1196 if let Some(ref mut buf) = self.row_buf {
1197 buf.state = state;
1198 if state == PredicateState::Pass {
1199 buf.direct = true;
1200 }
1201 }
1202 if state == PredicateState::Pass {
1203 self.drain_buffered(true);
1205 }
1206 }
1207 } else {
1208 if let Some(resolved) = self.plan.resolve_field(name) {
1209 let owned = resolved.to_owned();
1210 if self.frozen.is_some() && !self.field_index.contains_key(&owned) {
1211 if let Some(ref frozen) = self.frozen {
1212 if frozen.is_exact()
1213 && !frozen.column_names().iter().any(|n| n.as_ref() == owned)
1214 {
1215 if self.unknown_error.is_none() {
1216 self.unknown_error = Some(format!(
1217 "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1218 owned,
1219 frozen.num_columns(),
1220 frozen.is_exact()
1221 ));
1222 }
1223 return;
1224 }
1225 }
1226 }
1227 let slot = self.ensure_column_idx(&owned) as u32;
1228 self.record_strict_violation_at(slot as usize, &value);
1229 self.mark_predicate_slot(slot);
1230 let val_static: Value<'static> = value.into_static();
1231 let is_pred = self.is_predicate_slot(slot);
1232 #[cfg(feature = "profile")]
1233 if is_pred {
1234 crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1235 } else {
1236 crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1237 }
1238 if let Some(ref mut buf) = self.row_buf {
1239 if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1240 buf.fields[pos].1 = val_static;
1241 } else {
1242 buf.fields.push((slot, val_static));
1243 }
1244 }
1245 if is_pred {
1246 if let Some(ref mut buf) = self.row_buf {
1248 if buf.predicate_ordinal.is_none() {
1249 buf.predicate_ordinal = Some(slot);
1250 }
1251 }
1252 let state = self.evaluate_predicate_state();
1253 if let Some(ref mut buf) = self.row_buf {
1254 buf.state = state;
1255 }
1256 }
1257 }
1258 }
1259 }
1260
1261 #[inline]
1262 fn row_rejected(&self) -> bool {
1263 self.plan.filter.is_some()
1264 && self
1265 .row_buf
1266 .as_ref()
1267 .is_some_and(|b| b.state == PredicateState::Fail)
1268 }
1269
1270 #[inline]
1271 fn row_satisfied(&self) -> bool {
1272 let mask = self.wanted_mask();
1273 if mask == 0 {
1274 return false;
1275 }
1276 let dirty = self
1278 .row_dirty
1279 .iter()
1280 .copied()
1281 .reduce(|a, b| a | b)
1282 .unwrap_or(0);
1283 (dirty & mask) == mask
1284 }
1285
1286 #[inline]
1287 fn wanted_mask(&self) -> u64 {
1288 self.cached_wanted_mask
1293 }
1294
1295 #[inline]
1296 fn finish(&mut self) -> Result<RecordBatch> {
1297 self.prepare_finish()?;
1298
1299 if self.column_order.is_empty() {
1300 let schema = Arc::new(Schema::empty());
1301 return Ok(RecordBatch::new_empty(schema));
1302 }
1303
1304 self.auto_dict_upgrade();
1305
1306 if !self.plan.schema_order.is_empty() {
1311 let order = self.plan.schema_order.clone();
1312 let mut fields = Vec::with_capacity(order.len());
1313 let mut arrays = Vec::with_capacity(order.len());
1314 for name in &order {
1315 if self.plan.drop_fields.contains(name) {
1317 continue;
1318 }
1319 let idx = match self.field_index.get(name.as_str()) {
1320 Some(&i) => i,
1321 None => {
1322 let ty = self.plan.column_type(name);
1324 let mut b = ColumnBuilder::with_capacity(self.row_count.max(1), &ty);
1325 for _ in 0..self.row_count {
1326 b.push(None);
1327 }
1328 let idx = self.columns.len();
1329 self.columns.push(b);
1330 self.field_index.insert(name.clone(), idx);
1331 idx
1332 }
1333 };
1334 let b = &mut self.columns[idx];
1335 fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
1336 arrays.push(b.to_arrow_array()?);
1337 }
1338 let schema = Arc::new(Schema::new(fields));
1339 return Ok(RecordBatch::try_new(schema, arrays)?);
1340 }
1341
1342 self.sort_columns();
1343
1344 let mut fields = Vec::with_capacity(self.column_order.len());
1345 let mut arrays = Vec::with_capacity(self.column_order.len());
1346 for name in &self.column_order {
1348 if let Some(&idx) = self.field_index.get(name.as_str()) {
1349 let b = &mut self.columns[idx];
1350 fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
1351 arrays.push(b.to_arrow_array()?);
1352 }
1353 }
1354
1355 let schema = Arc::new(Schema::new(fields));
1356 Ok(RecordBatch::try_new(schema, arrays)?)
1357 }
1358
1359 #[inline]
1360 fn expect_slot(&self, ordinal: u32) -> Option<(u32, &[u8])> {
1361 self.ordinal_expect
1362 .get(ordinal as usize)
1363 .and_then(|entry| entry.as_ref())
1364 .map(|(slot, raw)| (*slot, raw.as_slice()))
1365 }
1366
1367 #[inline]
1368 fn record_slot(&mut self, ordinal: u32, slot: u32, raw_name: &[u8]) {
1369 let idx = ordinal as usize;
1370 let growth = idx
1371 .saturating_add(1)
1372 .saturating_sub(self.ordinal_expect.capacity())
1373 .saturating_mul(std::mem::size_of::<Option<(u32, Vec<u8>)>>());
1374 if !self.reserve_memory(growth.saturating_add(raw_name.len())) {
1375 return;
1376 }
1377 if idx >= self.ordinal_expect.len() {
1378 self.ordinal_expect.resize_with(idx + 1, || None);
1379 }
1380 self.ordinal_expect[idx] = Some((slot, raw_name.to_vec()));
1381 }
1382
1383 #[inline]
1384 fn layout_broken(&mut self, ordinal: u32) {
1385 if let Some(entry) = self.ordinal_expect.get_mut(ordinal as usize) {
1386 *entry = None;
1387 }
1388 }
1389
1390 #[inline]
1391 fn reset_child_ordinal(&mut self) {
1392 self.current_ordinal = 0;
1393 }
1394}
1395
1396#[cfg(test)]
1401mod tests {
1402 use super::*;
1403 use crate::columnar::ColumnBuilder;
1404 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
1405 use crate::plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
1406 use crate::value::Value;
1407 use crate::Result;
1408 use arrow::array::{Array, AsArray};
1409
1410 #[derive(Clone)]
1413 struct LineParser;
1414
1415 impl RecordParser for LineParser {
1416 fn validate(&self, bytes: &[u8]) -> Result<()> {
1417 simdutf8::basic::from_utf8(bytes)?;
1418 Ok(())
1419 }
1420
1421 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
1422 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
1423 for line in text.lines() {
1424 if line.is_empty() {
1425 continue;
1426 }
1427 sink.begin_row();
1428 for token in line.split_whitespace() {
1429 if let Some((k, v)) = token.split_once('=') {
1430 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
1431 }
1432 }
1433 sink.end_row();
1434 }
1435 Ok(())
1436 }
1437 }
1438
1439 #[derive(Clone)]
1440 struct LineSplitter;
1441
1442 impl Splitter for LineSplitter {
1443 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
1444 if from >= bytes.len() {
1445 return None;
1446 }
1447 let start = if bytes[from] == b'\n' { from + 1 } else { from };
1449 if start >= bytes.len() {
1450 return None;
1451 }
1452 memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
1453 }
1454
1455 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
1456 if max_chunks <= 1 || bytes.is_empty() {
1457 return vec![0, bytes.len()];
1458 }
1459 let mut points = vec![0usize];
1460 let mut last = 0;
1461 for (i, &b) in bytes.iter().enumerate() {
1462 if b == b'\n' {
1463 let next = i + 1;
1464 if next > last && points.len() < max_chunks {
1465 points.push(next);
1466 last = next;
1467 }
1468 }
1469 }
1470 if *points.last().unwrap() != bytes.len() {
1471 points.push(bytes.len());
1472 }
1473 points
1474 }
1475
1476 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
1477 let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
1478 (sample.len() / newline_count).max(1)
1479 }
1480 }
1481
1482 #[derive(Default)]
1484 struct RecObserver {
1485 events: std::sync::Mutex<Vec<String>>,
1486 }
1487
1488 impl RecObserver {
1489 fn log(&self, e: String) {
1490 self.events.lock().unwrap().push(e);
1491 }
1492 fn events(&self) -> Vec<String> {
1493 self.events.lock().unwrap().clone()
1494 }
1495 }
1496
1497 impl crate::RowObserver for RecObserver {
1498 fn on_begin_row(&self, row_index: usize) {
1499 self.log(format!("begin:{row_index}"));
1500 }
1501 fn on_put_field(&self, row_index: usize, name: &str, _slot: usize, _value: &Value<'_>) {
1502 self.log(format!("put:{row_index}:{name}"));
1503 }
1504 fn on_row_accepted(&self, row_index: usize) {
1505 self.log(format!("accept:{row_index}"));
1506 }
1507 fn on_row_rejected(&self, row_index: usize) {
1508 self.log(format!("reject:{row_index}"));
1509 }
1510 fn on_chunk_finished(&self, total: usize, accepted: usize, rejected: usize) {
1511 self.log(format!("fin:{total}:{accepted}:{rejected}"));
1512 }
1513 }
1514
1515 #[test]
1516 fn test_observer_lifecycle() {
1517 let obs = Arc::new(RecObserver::default());
1518 let mut plan = ExecutionPlan::new();
1519 plan.observer = Some(obs.clone());
1520 let mut engine = parse_bytes(b"X=1 Y=2\nX=3 Y=4\n", plan);
1521 engine.finish().unwrap();
1522 assert_eq!(
1523 obs.events(),
1524 vec![
1525 "begin:0",
1526 "put:0:X",
1527 "put:0:Y",
1528 "accept:0",
1529 "begin:1",
1530 "put:1:X",
1531 "put:1:Y",
1532 "accept:1",
1533 "fin:2:2:0",
1534 ]
1536 );
1537 }
1538
1539 #[test]
1540 fn test_observer_with_filter() {
1541 let obs = Arc::new(RecObserver::default());
1542 let mut plan = ExecutionPlan::new();
1543 plan.filter = Some(FilterPredicate::Equal {
1544 field: "X".to_string(),
1545 value: "keep".to_string(),
1546 });
1547 plan.observer = Some(obs.clone());
1548 let mut engine = parse_bytes(b"X=keep A=1\nX=drop A=2\n", plan);
1549 engine.finish().unwrap();
1550 let ev = obs.events();
1551 assert!(ev.contains(&"reject:1".to_string()), "events: {ev:?}");
1553 assert_eq!(
1554 ev.iter().filter(|e| e.starts_with("put:")).count(),
1555 2,
1556 "only the accepted row's fields may be observed: {ev:?}"
1557 );
1558 assert!(ev.contains(&"fin:2:1:1".to_string()), "events: {ev:?}");
1559 }
1560
1561 #[test]
1562 fn test_observer_parallel_counts_all_rows() {
1563 use std::sync::atomic::{AtomicUsize, Ordering as AOrd};
1564 #[derive(Default)]
1565 struct Counting {
1566 accepted: AtomicUsize,
1567 rejected: AtomicUsize,
1568 }
1569 impl crate::RowObserver for Counting {
1570 fn on_row_accepted(&self, _i: usize) {
1571 self.accepted.fetch_add(1, AOrd::Relaxed);
1572 }
1573 fn on_row_rejected(&self, _i: usize) {
1574 self.rejected.fetch_add(1, AOrd::Relaxed);
1575 }
1576 }
1577 let obs = Arc::new(Counting::default());
1578 let mut plan = ExecutionPlan::new();
1579 plan.filter = Some(FilterPredicate::NotEqual {
1580 field: "X".to_string(),
1581 value: "drop".to_string(),
1582 });
1583 plan.observer = Some(obs.clone());
1584 let mut data = Vec::new();
1585 for i in 0..1000 {
1586 let v = if i % 3 == 0 { "drop" } else { "keep" };
1587 data.extend_from_slice(format!("X={v} N={i}\n").as_bytes());
1588 }
1589 let pipeline = crate::Pipeline::new(LineSplitter, LineParser).with_plan(plan);
1590 let batches = pipeline.read_bytes_par(&data, 4).unwrap();
1591 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
1592 let accepted = obs.accepted.load(AOrd::Relaxed);
1593 let rejected = obs.rejected.load(AOrd::Relaxed);
1594 assert_eq!(accepted + rejected, 1000);
1595 assert_eq!(accepted, total);
1596 assert_eq!(rejected, 334);
1597 }
1598
1599 #[test]
1600 fn test_not_wraps_leaf_predicates_predicate_first() {
1601 let cases: Vec<(FilterPredicate, &[u8], Vec<&str>)> = vec![
1604 (
1605 FilterPredicate::not(FilterPredicate::Equal {
1606 field: "X".into(),
1607 value: "keep".into(),
1608 }),
1609 b"X=keep\nX=drop\n",
1610 vec!["drop"],
1611 ),
1612 (
1613 FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() }),
1614 b"X=1\nY=2\n",
1615 vec!["1"],
1616 ),
1617 (
1618 FilterPredicate::not(FilterPredicate::StartsWith {
1619 field: "X".into(),
1620 value: "ab".into(),
1621 }),
1622 b"X=abc\nX=zzz\n",
1623 vec!["zzz"],
1624 ),
1625 (
1626 FilterPredicate::not(FilterPredicate::In {
1627 field: "X".into(),
1628 values: vec!["a".into(), "b".into()],
1629 }),
1630 b"X=a\nX=b\nX=c\n",
1631 vec!["c"],
1632 ),
1633 (
1634 FilterPredicate::not(FilterPredicate::Regex {
1635 field: "X".into(),
1636 re: crate::plan::RegexSpec::new("^ERR").unwrap(),
1637 }),
1638 b"X=ERR1\nX=ok\n",
1639 vec!["ok"],
1640 ),
1641 (
1642 FilterPredicate::not(FilterPredicate::Contains {
1643 field: "X".into(),
1644 value: "zz".into(),
1645 }),
1646 b"X=azza\nX=hello\n",
1647 vec!["hello"],
1648 ),
1649 ];
1650 for (pred, data, want) in cases {
1651 let mut plan = ExecutionPlan::new();
1652 plan.filter = Some(pred);
1653 let engine = parse_bytes(data, plan);
1654 let col = engine.get_column("X").unwrap();
1655 assert_eq!(
1656 col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
1657 want,
1658 "data: {}",
1659 String::from_utf8_lossy(data)
1660 );
1661 }
1662 }
1663
1664 #[test]
1665 fn test_not_wraps_leaf_predicates_batch_check() {
1666 let data = b"X=keep N=1\nX=drop N=2\nX=keep N=3\n";
1669 let engine = parse_bytes(data, ExecutionPlan::new());
1670 let pred = FilterPredicate::not(FilterPredicate::Equal {
1671 field: "X".into(),
1672 value: "keep".into(),
1673 });
1674 let kept: Vec<usize> = (0..engine.num_rows())
1675 .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
1676 .collect();
1677 assert_eq!(kept, vec![1]);
1678
1679 let pred = FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() });
1680 let kept: Vec<usize> = (0..engine.num_rows())
1681 .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
1682 .collect();
1683 assert_eq!(kept, vec![0, 1, 2]);
1684
1685 let pred = FilterPredicate::not(FilterPredicate::Regex {
1686 field: "X".into(),
1687 re: crate::plan::RegexSpec::new("ke*p").unwrap(),
1688 });
1689 let kept: Vec<usize> = (0..engine.num_rows())
1690 .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
1691 .collect();
1692 assert_eq!(kept, vec![1]);
1693 }
1694
1695 #[test]
1696 fn test_regex_predicate() {
1697 let mut plan = ExecutionPlan::new();
1698 plan.filter = Some(FilterPredicate::Regex {
1699 field: "X".into(),
1700 re: crate::plan::RegexSpec::new("^ERR\\d+$").unwrap(),
1701 });
1702 let engine = parse_bytes(b"X=ERR12\nX=ok\nX=ERRx\nX=ERR0\n", plan);
1703 let col = engine.get_column("X").unwrap();
1704 assert_eq!(
1705 col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
1706 vec!["ERR12", "ERR0"]
1707 );
1708 let mut plan = ExecutionPlan::new();
1710 plan.filter = Some(FilterPredicate::Regex {
1711 field: "X".into(),
1712 re: crate::plan::RegexSpec::new(".*").unwrap(),
1713 });
1714 let engine = parse_bytes(b"X=a\nY=2\n", plan);
1715 assert_eq!(engine.num_rows(), 1);
1716 assert!(crate::plan::RegexSpec::new("(").is_err());
1718 assert_eq!(
1720 crate::plan::RegexSpec::new("^a").unwrap(),
1721 crate::plan::RegexSpec::new("^a").unwrap()
1722 );
1723 assert_ne!(
1724 crate::plan::RegexSpec::new("^a").unwrap(),
1725 crate::plan::RegexSpec::new("^b").unwrap()
1726 );
1727 }
1728
1729 #[test]
1730 fn test_no_observer_smoke() {
1731 let engine = parse_bytes(b"X=1\nX=2\n", ExecutionPlan::new());
1733 assert_eq!(engine.num_rows(), 2);
1734 }
1735
1736 fn parse_bytes(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
1737 let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
1738 LineParser.parse_chunk(bytes, &mut sink).unwrap();
1739 sink
1740 }
1741
1742 #[test]
1743 fn test_extend_no_duplicates() {
1744 let e1 = parse_bytes(b"A=1 B=2\n", ExecutionPlan::new());
1745 let mut merged = TableBuilder::new();
1746 merged.extend(e1).unwrap();
1747 assert_eq!(merged.num_rows(), 1);
1748 for col in &merged.columns {
1749 assert_eq!(col.len(), merged.num_rows(), "column length mismatch");
1750 }
1751 }
1752
1753 #[test]
1754 fn test_multi_chunk_same_as_single() {
1755 let data = b"A=1\nA=2\nA=3\n";
1756 let single = parse_bytes(data, ExecutionPlan::new());
1757 assert_eq!(single.num_rows(), 3);
1758
1759 let splitter = LineSplitter;
1760 let points = splitter.find_split_points(data, 2);
1761 let mut merged = TableBuilder::new();
1762 for w in points.windows(2) {
1763 let chunk = &data[w[0]..w[1]];
1764 let engine = parse_bytes(chunk, ExecutionPlan::new());
1765 merged.extend(engine).unwrap();
1766 }
1767 assert_eq!(merged.num_rows(), single.num_rows());
1768 }
1769
1770 #[test]
1771 fn test_last_write_wins_duplicate_field() {
1772 let engine = parse_bytes(b"X=10 X=20\n", ExecutionPlan::new());
1773 assert_eq!(engine.num_rows(), 1);
1774 let col = engine.get_column("X").unwrap();
1775 assert_eq!(col.as_str_vec(), vec![Some("20".into())]);
1776 }
1777
1778 #[test]
1779 fn test_build_plan_rename() {
1780 let mut plan = ExecutionPlan::new();
1781 plan.field_map.insert("X".to_string(), "Y".to_string());
1782 let engine = parse_bytes(b"X=hello\n", plan);
1783 assert_eq!(engine.num_rows(), 1);
1784 assert!(engine.get_column("Y").is_some());
1785 assert!(engine.get_column("X").is_none());
1786 assert_eq!(
1787 engine.get_column("Y").unwrap().as_str_vec(),
1788 vec![Some("hello".into())]
1789 );
1790 }
1791
1792 #[test]
1793 fn test_build_plan_drop() {
1794 let mut plan = ExecutionPlan::new();
1795 plan.drop_fields.insert("X".to_string());
1796 let engine = parse_bytes(b"X=hello Y=world\n", plan);
1797 assert_eq!(engine.num_rows(), 1);
1798 assert!(engine.get_column("X").is_none());
1799 assert!(engine.get_column("Y").is_some());
1800 }
1801
1802 #[test]
1803 fn test_build_plan_filter_ne() {
1804 let mut plan = ExecutionPlan::new();
1805 plan.filter = Some(FilterPredicate::NotEqual {
1806 field: "X".to_string(),
1807 value: "42".to_string(),
1808 });
1809 let engine = parse_bytes(b"X=10\nX=42\nX=30\n", plan);
1810 assert_eq!(engine.num_rows(), 2);
1811 let col = engine.get_column("X").unwrap();
1812 assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("30".into())]);
1813 }
1814
1815 #[test]
1816 fn test_build_plan_filter_eq() {
1817 let mut plan = ExecutionPlan::new();
1818 plan.filter = Some(FilterPredicate::Equal {
1819 field: "X".to_string(),
1820 value: "10".to_string(),
1821 });
1822 let engine = parse_bytes(b"X=10\nX=20\nX=10\n", plan);
1823 assert_eq!(engine.num_rows(), 2);
1824 let col = engine.get_column("X").unwrap();
1825 assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("10".into())]);
1826 }
1827
1828 #[test]
1829 fn test_build_plan_filter_missing_field() {
1830 let mut plan = ExecutionPlan::new();
1831 plan.filter = Some(FilterPredicate::NotEqual {
1832 field: "X".to_string(),
1833 value: "10".to_string(),
1834 });
1835 let engine = parse_bytes(b"X=10\nY=99\n", plan);
1836 assert_eq!(engine.num_rows(), 1);
1837 let col = engine.get_column("Y").unwrap();
1838 assert_eq!(col.as_str_vec(), vec![Some("99".into())]);
1839 }
1840
1841 #[test]
1842 fn test_typed_int64_column() {
1843 let mut plan = ExecutionPlan::new();
1844 plan.field_types.insert("X".to_string(), FieldType::Int64);
1845 let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
1846 assert_eq!(engine.num_rows(), 3);
1847 if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
1848 assert_eq!(v.get(0), Some(42));
1849 assert_eq!(v.get(1), None);
1850 assert_eq!(v.get(2), Some(100));
1851 } else {
1852 panic!("expected Int64 builder");
1853 }
1854 }
1855
1856 #[test]
1857 fn test_typed_float64_column() {
1858 let mut plan = ExecutionPlan::new();
1859 plan.field_types.insert("X".to_string(), FieldType::Float64);
1860 let engine = parse_bytes(b"X=1.5\n", plan);
1861 if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
1862 assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
1863 } else {
1864 panic!("expected Float64 builder");
1865 }
1866 }
1867
1868 #[test]
1869 fn test_is_null_filter() {
1870 let mut plan = ExecutionPlan::new();
1871 plan.filter = Some(FilterPredicate::IsNull {
1872 field: "X".to_string(),
1873 });
1874 let engine = parse_bytes(b"X=hello\nY=world\nX=fallback\n", plan);
1875 assert_eq!(engine.num_rows(), 1);
1876 let col = engine.get_column("Y").unwrap();
1877 assert_eq!(col.as_str_vec(), vec![Some("world".into())]);
1878 }
1879
1880 #[test]
1881 fn test_not_is_null_filter_predicate_first() {
1882 let mut plan = ExecutionPlan::new();
1883 plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
1884 field: "X".to_string(),
1885 }));
1886 let engine = parse_bytes(b"X=a\nY=only\nX=b\n", plan);
1887 assert_eq!(engine.num_rows(), 2);
1888 let col = engine.get_column("X").unwrap();
1889 assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
1890 }
1891
1892 #[test]
1893 fn test_not_is_null_filter_batch_eval() {
1894 let mut plan = ExecutionPlan::new();
1897 plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
1898 field: "X".to_string(),
1899 }));
1900 let mut sink = TableBuilder::with_plan(4, Arc::new(plan));
1901 sink.row_buf.as_mut().unwrap().buffer_worthwhile = false;
1902 LineParser
1903 .parse_chunk(b"X=a\nY=only\nX=b\n", &mut sink)
1904 .unwrap();
1905 assert_eq!(sink.num_rows(), 2);
1906 let col = sink.get_column("X").unwrap();
1907 assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
1908 }
1909
1910 #[test]
1911 fn test_strict_types_errors_on_malformed_int64() {
1912 let mut plan = ExecutionPlan::new();
1913 plan.field_types.insert("X".to_string(), FieldType::Int64);
1914 plan.strict_types = true;
1915 let mut engine = parse_bytes(b"X=42\nX=abc\n", plan);
1916 let err = engine.finish().unwrap_err();
1917 let msg = err.to_string();
1918 assert!(msg.contains("\"abc\""), "message should name value: {msg}");
1919 assert!(msg.contains("\"X\""), "message should name column: {msg}");
1920 assert!(msg.contains("Int64"), "message should name type: {msg}");
1921 assert!(msg.contains("row 1"), "message should name row: {msg}");
1922 }
1923
1924 #[test]
1925 fn test_strict_types_off_yields_null() {
1926 let mut plan = ExecutionPlan::new();
1927 plan.field_types.insert("X".to_string(), FieldType::Int64);
1928 let engine = parse_bytes(b"X=42\nX=abc\n", plan);
1929 assert!(engine.strict_error.is_none());
1930 if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
1931 assert_eq!(v.get(0), Some(42));
1932 assert_eq!(v.get(1), None);
1933 } else {
1934 panic!("expected Int64 builder");
1935 }
1936 }
1937
1938 #[test]
1939 fn test_strict_types_missing_field_stays_null() {
1940 let mut plan = ExecutionPlan::new();
1941 plan.field_types.insert("X".to_string(), FieldType::Int64);
1942 plan.strict_types = true;
1943 let mut engine = parse_bytes(b"X=42\nY=1\n", plan);
1944 let batch = engine.finish().unwrap();
1945 let idx = batch.schema().index_of("X").unwrap();
1946 let arr = batch
1947 .column(idx)
1948 .as_primitive::<arrow::datatypes::Int64Type>();
1949 assert_eq!(arr.value(0), 42);
1950 assert!(arr.is_null(1));
1951 }
1952
1953 #[test]
1954 fn test_strict_types_buffered_filter_path() {
1955 let mut plan = ExecutionPlan::new();
1958 plan.field_types.insert("X".to_string(), FieldType::Int64);
1959 plan.strict_types = true;
1960 plan.filter = Some(FilterPredicate::Equal {
1961 field: "Y".to_string(),
1962 value: "1".to_string(),
1963 });
1964 let mut engine = parse_bytes(b"Y=1 X=abc\n", plan);
1965 let err = engine.finish().unwrap_err();
1966 assert!(err.to_string().contains("strict_types"));
1967 }
1968
1969 #[test]
1970 fn test_is_type_filter_int64() {
1971 let mut plan = ExecutionPlan::new();
1972 plan.field_types.insert("X".to_string(), FieldType::Int64);
1973 plan.filter = Some(FilterPredicate::IsType {
1974 field: "X".to_string(),
1975 field_type: FieldType::Int64,
1976 });
1977 let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
1978 assert_eq!(engine.num_rows(), 2);
1979 if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
1980 assert_eq!(v.get(0), Some(42));
1981 assert_eq!(v.get(1), Some(100));
1982 } else {
1983 panic!("expected Int64 builder");
1984 }
1985 }
1986
1987 #[test]
1988 fn test_is_type_filter_float64() {
1989 let mut plan = ExecutionPlan::new();
1990 plan.field_types.insert("X".to_string(), FieldType::Float64);
1991 plan.filter = Some(FilterPredicate::IsType {
1992 field: "X".to_string(),
1993 field_type: FieldType::Float64,
1994 });
1995 let engine = parse_bytes(b"X=1.5\nX=bad\nX=2.5\n", plan);
1996 assert_eq!(engine.num_rows(), 2);
1997 if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
1998 assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
1999 assert!((v.get(1).unwrap() - 2.5).abs() < 1e-9);
2000 } else {
2001 panic!("expected Float64 builder");
2002 }
2003 }
2004
2005 #[test]
2006 fn test_dictionary_column() {
2007 let mut plan = ExecutionPlan::new();
2008 plan.dictionary_columns.insert("P".to_string());
2009 let engine = parse_bytes(b"P=Widget\nP=Gadget\nP=Widget\n", plan);
2010 assert_eq!(engine.num_rows(), 3);
2011 if let ColumnBuilder::Dictionary { codes, offsets, .. } = engine.get_column("P").unwrap() {
2012 let dict_len = offsets.len() - 1;
2013 assert_eq!(dict_len, 2);
2014 assert_eq!(
2015 codes.iter().map(|o| o.copied()).collect::<Vec<_>>(),
2016 vec![Some(0), Some(1), Some(0)]
2017 );
2018 } else {
2019 panic!("expected Dictionary builder");
2020 }
2021 }
2022
2023 #[test]
2024 fn test_ragged_late_chunk_column_debut() {
2025 let e1 = parse_bytes(b"A=1 B=2\nA=3\n", ExecutionPlan::new());
2026 let e2 = parse_bytes(b"B=4 C=5\n", ExecutionPlan::new());
2027 let mut merged = TableBuilder::new();
2028 merged.extend(e1).unwrap();
2029 merged.extend(e2).unwrap();
2030
2031 assert_eq!(
2032 merged.get_column("A").unwrap().as_str_vec(),
2033 vec![Some("1".into()), Some("3".into()), None]
2034 );
2035 assert_eq!(
2036 merged.get_column("B").unwrap().as_str_vec(),
2037 vec![Some("2".into()), None, Some("4".into())]
2038 );
2039 assert_eq!(
2040 merged.get_column("C").unwrap().as_str_vec(),
2041 vec![None, None, Some("5".into())]
2042 );
2043 }
2044
2045 #[test]
2046 fn test_auto_dict_upgrade_only_post_merge() {
2047 let mut plan = ExecutionPlan::new();
2048 plan.auto_dict = true;
2049 let a = parse_bytes(b"P=x\nP=y\n", plan.clone());
2050 let b = parse_bytes(b"P=x\nP=y\n", plan.clone());
2051 let mut merged = TableBuilder::with_plan(64, Arc::new(plan));
2052 merged.extend(a).unwrap();
2053 merged.extend(b).unwrap();
2054 merged.auto_dict_upgrade();
2055 assert_eq!(merged.num_rows(), 4);
2056 }
2057
2058 #[test]
2059 fn test_extend_string_dictionary_promotes_not_panics() {
2060 let mut e1 = parse_bytes(b"P=x\n", ExecutionPlan::new());
2061 let mut plan = ExecutionPlan::new();
2062 plan.dictionary_columns.insert("P".to_string());
2063 let e2 = parse_bytes(b"P=x\n", plan);
2064 e1.extend(e2).expect("String/Dictionary must reconcile");
2066 assert!(matches!(
2067 e1.get_column("P").unwrap(),
2068 crate::columnar::ColumnBuilder::Dictionary { .. }
2069 ));
2070 }
2071
2072 #[test]
2073 fn test_extend_irreconcilable_variants_error_not_panic() {
2074 let mut e1 = parse_bytes(b"S=x\n", ExecutionPlan::new());
2075 let mut plan = ExecutionPlan::new();
2076 plan.field_types.insert("S".to_string(), FieldType::Int64);
2077 let e2 = parse_bytes(b"S=7\n", plan);
2078 let result = e1.extend(e2);
2079 match result {
2080 Err(crate::Error::Merge(msg)) => {
2081 assert!(msg.contains("'S'"), "error should name the column: {msg}");
2082 }
2083 other => panic!("String/Int64 mismatch must return Merge error, got {other:?}"),
2084 }
2085 }
2086
2087 #[test]
2088 fn test_compare_filter_via_arrow_compute() {
2089 use crate::arrow_export::apply_compare_filter;
2090
2091 let mut plan = ExecutionPlan::new();
2092 plan.filter = Some(FilterPredicate::Compare {
2093 field_a: "A".to_string(),
2094 op: CompareOp::Gt,
2095 field_b: "B".to_string(),
2096 });
2097 let mut engine = parse_bytes(b"A=3 B=1\nA=2 B=2\nA=5 B=4\n", plan);
2098 let batch = engine.finish().unwrap();
2099 let filtered = apply_compare_filter(
2100 batch,
2101 &FilterPredicate::Compare {
2102 field_a: "A".to_string(),
2103 op: CompareOp::Gt,
2104 field_b: "B".to_string(),
2105 },
2106 )
2107 .unwrap();
2108 assert_eq!(filtered.num_rows(), 2);
2109 let a = filtered.column_by_name("A").unwrap().as_string::<i32>();
2110 assert_eq!(a.value(0), "3");
2111 assert_eq!(a.value(1), "5");
2112 }
2113
2114 #[test]
2115 fn test_row_dirty_is_full_edge_cases() {
2116 for ncols in [1, 63, 64, 65, 127, 128, 129] {
2119 let mut tb = TableBuilder::new();
2120 for i in 0..ncols {
2121 let name = format!("col{i}");
2122 tb.field_index.insert(name.clone(), i);
2123 tb.columns.push(crate::columnar::ColumnBuilder::String(
2124 crate::columnar::StrColumn::default(),
2125 ));
2126 tb.column_order.push(name);
2127 }
2128 tb.row_dirty = vec![0; ncols.div_ceil(64)];
2130 for i in 0..ncols {
2132 let word = i / 64;
2133 let bit = i % 64;
2134 tb.row_dirty[word] |= 1u64 << bit;
2135 }
2136 let full_words = ncols / 64;
2137 let rem = ncols % 64;
2138 let is_full = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
2139 && (rem == 0
2140 || tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
2141 assert!(
2142 is_full,
2143 "ncols={ncols} should be full, row_dirty={:?}",
2144 tb.row_dirty
2145 );
2146 if ncols > 0 {
2148 let word = (ncols - 1) / 64;
2149 let bit = (ncols - 1) % 64;
2150 tb.row_dirty[word] &= !(1u64 << bit);
2151 let is_full2 = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
2152 && (rem == 0
2153 || tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
2154 assert!(
2155 !is_full2,
2156 "ncols={ncols} should not be full after clearing one bit"
2157 );
2158 }
2159 }
2160 }
2161
2162 #[test]
2163 fn test_finish_twice_does_not_panic() {
2164 let mut tb = parse_bytes(b"A=1 B=2\nA=3 B=4\n", ExecutionPlan::new());
2167 let batch1 = tb.finish().unwrap();
2168 assert_eq!(batch1.num_rows(), 2);
2169 let batch2 = tb.finish().unwrap();
2171 assert_eq!(batch2.num_rows(), 0);
2172 }
2173
2174 #[test]
2175 fn test_c2_compound_reorder() {
2176 let data = b"A=1 B=2\nA=2 B=2\nA=1 B=3\nA=2 B=3\n";
2179 let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
2180 let mut plan = ExecutionPlan::new();
2181 plan.filter = Some(FilterPredicate::And(
2182 Box::new(FilterPredicate::Equal {
2183 field: a.to_string(),
2184 value: va.to_string(),
2185 }),
2186 Box::new(FilterPredicate::Equal {
2187 field: b.to_string(),
2188 value: vb.to_string(),
2189 }),
2190 ));
2191 plan
2192 };
2193 let r1 = parse_bytes(data, make_filter("A", "1", "B", "2"));
2194 let r2 = parse_bytes(data, make_filter("B", "2", "A", "1"));
2195 assert_eq!(
2196 r1.num_rows(),
2197 r2.num_rows(),
2198 "C2 reorder must produce identical results"
2199 );
2200 assert_eq!(r1.num_rows(), 1); }
2202
2203 #[test]
2204 fn test_c2_or_reorder() {
2205 let data = b"A=1 B=2\nA=2 B=2\nA=3 B=3\n";
2210 let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
2211 let mut plan = ExecutionPlan::new();
2212 plan.filter = Some(FilterPredicate::Or(
2213 Box::new(FilterPredicate::Equal {
2214 field: a.to_string(),
2215 value: va.to_string(),
2216 }),
2217 Box::new(FilterPredicate::Equal {
2218 field: b.to_string(),
2219 value: vb.to_string(),
2220 }),
2221 ));
2222 plan
2223 };
2224 let r1 = parse_bytes(data, make_filter("A", "1", "B", "3"));
2225 let r2 = parse_bytes(data, make_filter("B", "3", "A", "1"));
2226 assert_eq!(
2227 r1.num_rows(),
2228 r2.num_rows(),
2229 "Or reorder must produce identical results"
2230 );
2231 assert_eq!(r1.num_rows(), 2); }
2233
2234 #[test]
2235 fn test_predicate_slot_created_after_mask_build() {
2236 let data = b"A=1 B=ok\nA=2 B=ok\nA=3 B=ok\n";
2244 let mut plan = ExecutionPlan::new();
2245 plan.filter = Some(FilterPredicate::Equal {
2246 field: "B".to_string(),
2247 value: "reject".to_string(),
2248 });
2249 let engine = parse_bytes(data, plan);
2250 assert_eq!(
2251 engine.num_rows(),
2252 0,
2253 "all rows should be rejected (B != reject)"
2254 );
2255
2256 let data2 = b"A=1 B=ok\nA=2 B=reject\nA=3 B=ok\n";
2258 let mut plan2 = ExecutionPlan::new();
2259 plan2.filter = Some(FilterPredicate::Equal {
2260 field: "B".to_string(),
2261 value: "ok".to_string(),
2262 });
2263 let engine2 = parse_bytes(data2, plan2);
2264 assert_eq!(engine2.num_rows(), 2, "rows 1 and 3 pass (B == ok)");
2265 let col = engine2.get_column("A").unwrap();
2266 assert_eq!(col.as_str_vec(), vec![Some("1".into()), Some("3".into())]);
2267 }
2268
2269 #[test]
2270 fn test_predicate_first_via_resolve_and_put() {
2271 let mut plan = ExecutionPlan::new();
2274 plan.filter = Some(FilterPredicate::Equal {
2275 field: "B".to_string(),
2276 value: "reject".to_string(),
2277 });
2278 let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
2279 tb.begin_row();
2281 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("1")));
2282 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
2283 tb.end_row();
2284 tb.begin_row();
2286 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("2")));
2287 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("reject")));
2288 tb.end_row();
2289 tb.begin_row();
2291 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("3")));
2292 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
2293 tb.end_row();
2294 let batch = tb.finish().unwrap();
2295 assert_eq!(batch.num_rows(), 1, "only row 2 (B==reject) should pass");
2296 }
2297
2298 #[test]
2299 fn test_owned_cow_values_survive_buffered_filter() {
2300 let mut plan = ExecutionPlan::new();
2309 plan.filter = Some(FilterPredicate::Equal {
2310 field: "B".to_string(),
2311 value: "ok".to_string(),
2312 });
2313 let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
2314 for i in 0..50 {
2315 tb.begin_row();
2316 let a = Cow::Owned(format!("value-{i}-unescaped"));
2318 let b = Cow::Owned(if i % 2 == 0 {
2319 "ok".to_string()
2320 } else {
2321 "no".to_string()
2322 });
2323 tb.resolve_and_put("A", Value::Str(a));
2324 tb.resolve_and_put("B", Value::Str(b));
2325 tb.end_row();
2326 }
2327 assert_eq!(tb.num_rows(), 25, "even rows pass (B == ok)");
2328 let col = tb.get_column("A").unwrap();
2329 let vals = col.as_str_vec();
2330 assert_eq!(vals.len(), 25);
2331 assert_eq!(vals[0], Some("value-0-unescaped".to_string()));
2332 assert_eq!(vals[24], Some("value-48-unescaped".to_string()));
2333 }
2334
2335 #[test]
2336 fn test_predicate_first_skip_via_resolve_and_put() {
2337 let mut plan = ExecutionPlan::new();
2341 plan.filter = Some(FilterPredicate::Equal {
2342 field: "A".to_string(),
2343 value: "999".to_string(),
2344 });
2345 let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
2346 for _i in 0..100 {
2347 tb.begin_row();
2348 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("0")));
2349 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
2350 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
2351 tb.end_row();
2352 }
2353 let batch = tb.finish().unwrap();
2354 assert_eq!(
2355 batch.num_rows(),
2356 0,
2357 "all rows should be rejected (A=0 != 999)"
2358 );
2359 }
2360
2361 #[test]
2362 fn test_sparse_column_predicate_skip() {
2363 let mut plan = ExecutionPlan::new();
2372 plan.filter = Some(FilterPredicate::Equal {
2373 field: "A".to_string(),
2374 value: "reject".to_string(),
2375 });
2376 let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
2377 for i in 0..100 {
2378 tb.begin_row();
2379 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
2380 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
2381 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
2382 if i >= 50 {
2383 tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
2384 }
2385 tb.end_row();
2386 }
2387 let batch = tb.finish().unwrap();
2388 assert_eq!(batch.num_rows(), 0, "all rows rejected (A=ok != reject)");
2389 }
2390
2391 #[test]
2392 fn test_sparse_column_predicate_pass() {
2393 let mut plan = ExecutionPlan::new();
2396 plan.filter = Some(FilterPredicate::Equal {
2397 field: "A".to_string(),
2398 value: "ok".to_string(),
2399 });
2400 let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
2401 for _i in 0..100 {
2402 tb.begin_row();
2403 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
2404 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
2405 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
2406 if _i >= 50 {
2407 tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
2408 }
2409 tb.end_row();
2410 }
2411 let batch = tb.finish().unwrap();
2412 assert_eq!(batch.num_rows(), 100, "all rows pass (A=ok == ok)");
2413 if let Some(col) = batch.column_by_name("D") {
2415 let arr = col.as_string::<i32>();
2416 for i in 0..50 {
2417 assert!(arr.is_null(i), "row {} should have D=null", i);
2418 }
2419 for i in 50..100 {
2420 assert!(
2421 !arr.is_null(i),
2422 "row {} should not be null (D len={}, total rows={})",
2423 i,
2424 arr.len(),
2425 batch.num_rows()
2426 );
2427 assert_eq!(arr.value(i), "z", "rows 50+ should have D=z");
2428 }
2429 } else {
2430 panic!("D column should exist");
2431 }
2432 }
2433
2434 #[test]
2435 fn test_sparse_late_predicate_ordinal() {
2436 let mut plan = ExecutionPlan::new();
2440 plan.filter = Some(FilterPredicate::Equal {
2441 field: "E".to_string(),
2442 value: "999".to_string(),
2443 });
2444 let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
2445 for _i in 0..30 {
2447 tb.begin_row();
2448 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
2449 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
2450 tb.end_row();
2451 }
2452 for _i in 30..100 {
2454 tb.begin_row();
2455 tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
2456 tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
2457 tb.resolve_and_put("C", Value::Str(Cow::Borrowed("c")));
2458 tb.resolve_and_put("D", Value::Str(Cow::Borrowed("d")));
2459 tb.resolve_and_put("E", Value::Str(Cow::Borrowed("e")));
2460 tb.end_row();
2461 }
2462 let batch = tb.finish().unwrap();
2463 assert_eq!(batch.num_rows(), 0, "all rows rejected (E=\"e\" != 999)");
2465 }
2466
2467 #[test]
2468 fn raw_name_helpers_apply_rename_and_drop() {
2469 use crate::decoder::ColumnarSink;
2470
2471 let plan = ExecutionPlan::new()
2472 .rename("raw", "renamed")
2473 .drop("ignored");
2474 let mut tb = TableBuilder::with_plan(2, Arc::new(plan));
2475 tb.begin_row();
2476 tb.resolve_and_put_raw(b"raw", Value::Str(Cow::Borrowed("value")));
2477 tb.resolve_and_put_raw(b"ignored", Value::Str(Cow::Borrowed("not stored")));
2478 tb.put_row(&[("other", Value::Str(Cow::Borrowed("also stored")))]);
2479 tb.end_row();
2480
2481 assert_eq!(tb.column_names(), &["renamed", "other"]);
2482 assert_eq!(tb.resolve_raw(b"raw"), Some("renamed"));
2483 }
2484
2485 struct SkippingParser;
2496
2497 impl RecordParser for SkippingParser {
2498 fn validate(&self, bytes: &[u8]) -> Result<()> {
2499 simdutf8::basic::from_utf8(bytes)?;
2500 Ok(())
2501 }
2502
2503 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
2504 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
2505 for line in text.lines() {
2506 if line.is_empty() {
2507 continue;
2508 }
2509 sink.begin_row();
2510 for token in line.split_whitespace() {
2511 if sink.row_satisfied() {
2512 break; }
2514 if let Some((k, v)) = token.split_once('=') {
2515 if sink.resolve(k).is_some() {
2516 sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
2517 }
2518 }
2519 }
2520 sink.end_row();
2521 }
2522 Ok(())
2523 }
2524 }
2525
2526 fn parse_skipping(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
2527 let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
2528 SkippingParser.parse_chunk(bytes, &mut sink).unwrap();
2529 sink
2530 }
2531
2532 fn batch_names(tb: &mut TableBuilder) -> Vec<String> {
2533 tb.finish()
2534 .unwrap()
2535 .schema()
2536 .fields()
2537 .iter()
2538 .map(|f| f.name().clone())
2539 .collect()
2540 }
2541
2542 const REPORT: &[u8] = b"Name=n1 Department=d1 Amount=1 Status=s1 Date=2024\n\
2543 Name=n2 Department=d2 Amount=2 Status=s2 Date=2024\n";
2544
2545 #[test]
2546 fn schema_projection_subset() {
2547 let mut tb = parse_skipping(
2552 REPORT,
2553 ExecutionPlan::new().schema_order(["Name", "Department"]),
2554 );
2555 assert_eq!(batch_names(&mut tb), vec!["Name", "Department"]);
2556 }
2557
2558 #[test]
2559 fn schema_projection_single_non_first_column() {
2560 let mut tb = parse_skipping(REPORT, ExecutionPlan::new().schema_order(["Department"]));
2563 assert_eq!(batch_names(&mut tb), vec!["Department"]);
2564 }
2565
2566 #[test]
2567 fn schema_projection_full_natural_order() {
2568 let mut tb = parse_skipping(
2569 REPORT,
2570 ExecutionPlan::new().schema_order(["Name", "Department", "Amount", "Status", "Date"]),
2571 );
2572 assert_eq!(
2573 batch_names(&mut tb),
2574 vec!["Name", "Department", "Amount", "Status", "Date"]
2575 );
2576 }
2577
2578 #[test]
2579 fn schema_projection_reversed_order() {
2580 let mut tb = parse_skipping(
2581 REPORT,
2582 ExecutionPlan::new().schema_order(["Date", "Status", "Amount", "Department", "Name"]),
2583 );
2584 assert_eq!(
2585 batch_names(&mut tb),
2586 vec!["Date", "Status", "Amount", "Department", "Name"]
2587 );
2588 }
2589
2590 #[test]
2591 fn schema_projection_absent_column_null_filled() {
2592 let mut tb = parse_skipping(
2594 REPORT,
2595 ExecutionPlan::new().schema_order(["Name", "Missing"]),
2596 );
2597 let batch = tb.finish().unwrap();
2598 assert_eq!(batch.num_rows(), 2);
2599 let missing = batch.column_by_name("Missing").unwrap().as_string::<i32>();
2600 assert!(missing.is_null(0) && missing.is_null(1));
2601 let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
2602 assert_eq!(name.value(0), "n1");
2603 }
2604
2605 #[test]
2606 fn schema_projection_wants_rejects_unlisted_fields() {
2607 let plan = ExecutionPlan::new().schema_order(["Name"]);
2610 let tb = TableBuilder::with_plan(4, Arc::new(plan));
2611 assert!(tb.wants("Name"));
2612 assert!(!tb.wants("Department"));
2613 assert_eq!(tb.resolve("Amount"), None);
2614 }
2615
2616 #[test]
2617 fn schema_projection_rename_then_match() {
2618 let plan = ExecutionPlan::new()
2621 .rename("Name", "Employee")
2622 .schema_order(["Employee", "Amount"]);
2623 let mut tb = parse_skipping(REPORT, plan);
2624 assert_eq!(batch_names(&mut tb), vec!["Employee", "Amount"]);
2625 }
2626
2627 #[test]
2628 fn schema_projection_field_types_apply() {
2629 let plan = ExecutionPlan::new()
2630 .type_as("Amount", FieldType::Int64)
2631 .schema_order(["Name", "Amount"]);
2632 let mut tb = parse_skipping(REPORT, plan);
2633 let batch = tb.finish().unwrap();
2634 let amount = batch.column_by_name("Amount").unwrap();
2635 assert!(matches!(
2636 amount.data_type(),
2637 arrow::datatypes::DataType::Int64
2638 ));
2639 assert_eq!(
2640 amount
2641 .as_primitive::<arrow::datatypes::Int64Type>()
2642 .value(1),
2643 2
2644 );
2645 }
2646
2647 #[test]
2648 fn schema_projection_filter_on_unlisted_field() {
2649 let plan = ExecutionPlan::new()
2652 .schema_order(["Name"])
2653 .filter_eq("Status", "s2");
2654 let mut tb = parse_skipping(REPORT, plan);
2655 let batch = tb.finish().unwrap();
2656 assert_eq!(batch.num_rows(), 1);
2657 assert_eq!(batch.schema().fields().len(), 1);
2658 let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
2659 assert_eq!(name.value(0), "n2");
2660 }
2661
2662 #[test]
2663 fn schema_projection_drop_wins_over_schema() {
2664 let plan = ExecutionPlan::new()
2668 .schema_order(["Name", "Department"])
2669 .drop("Department");
2670 let mut tb = parse_skipping(REPORT, plan);
2671 assert_eq!(batch_names(&mut tb), vec!["Name"]);
2672 }
2673
2674 #[test]
2675 fn schema_projection_no_parser_cooperation() {
2676 let mut tb = parse_bytes(
2679 REPORT,
2680 ExecutionPlan::new().schema_order(["Department", "Name"]),
2681 );
2682 let batch = tb.finish().unwrap();
2683 let names: Vec<String> = batch
2684 .schema()
2685 .fields()
2686 .iter()
2687 .map(|f| f.name().clone())
2688 .collect();
2689 assert_eq!(names, vec!["Department", "Name"]);
2690 let dept = batch
2691 .column_by_name("Department")
2692 .unwrap()
2693 .as_string::<i32>();
2694 assert_eq!(dept.value(0), "d1");
2695 }
2696}