#[cfg(test)]
use std::borrow::Cow;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::Arc;
#[cfg(feature = "profile")]
use std::sync::atomic::{AtomicUsize, Ordering};
#[cfg(feature = "profile")]
pub static RESOLVE_AND_PUT_COUNT: AtomicUsize = AtomicUsize::new(0);
#[cfg(feature = "profile")]
pub static PREDICATE_EVALUATIONS: AtomicUsize = AtomicUsize::new(0);
#[cfg(feature = "profile")]
pub static PREDICATE_FAILS: AtomicUsize = AtomicUsize::new(0);
#[cfg(feature = "profile")]
pub static PREDICATE_UNDECIDED: AtomicUsize = AtomicUsize::new(0);
#[cfg(feature = "profile")]
pub static IS_PRED_TRUE: AtomicUsize = AtomicUsize::new(0);
#[cfg(feature = "profile")]
pub static IS_PRED_FALSE: AtomicUsize = AtomicUsize::new(0);
use arrow::datatypes::{Field as ArrowField, Schema};
use arrow::record_batch::RecordBatch;
use rustc_hash::FxHashMap as HashMap;
use smallvec::SmallVec;
use crate::columnar::ColumnBuilder;
use crate::decoder::ColumnarSink;
use crate::engine::predicate::{PredicateState, RowBuffer};
use crate::plan::{ExecutionPlan, FieldType, FilterPredicate};
use crate::value::Value;
use crate::Result;
mod memory;
mod predicates;
pub struct TableBuilder {
pub(crate) memory_limit: Option<usize>,
pub(crate) columns: Vec<ColumnBuilder>,
pub(crate) field_index: HashMap<String, usize>,
pub(crate) column_order: Vec<String>,
pub(crate) row_count: usize,
pub(crate) estimated_rows: usize,
pub(crate) plan: Arc<ExecutionPlan>,
pub(crate) row_dirty: Vec<u64>,
pub(crate) frozen: Option<std::sync::Arc<crate::schema::FrozenSchema>>,
pub(crate) unknown_error: Option<String>,
pub(crate) row_buf: Option<Box<RowBuffer>>,
pub(crate) cached_wanted_mask: u64,
pub(crate) ordinal_expect: Vec<Option<(u32, Vec<u8>)>>,
pub(crate) current_ordinal: u32,
pub(crate) strict_error: Option<crate::Error>,
pub(crate) rows_accepted: usize,
pub(crate) rows_rejected: usize,
}
impl TableBuilder {
pub fn new() -> Self {
Self {
memory_limit: None,
columns: Vec::new(),
field_index: HashMap::default(),
column_order: Vec::new(),
row_count: 0,
estimated_rows: 0,
plan: Arc::new(ExecutionPlan::new()),
row_dirty: Vec::new(),
frozen: None,
unknown_error: None,
row_buf: None,
cached_wanted_mask: 0,
ordinal_expect: Vec::new(),
current_ordinal: 0,
strict_error: None,
rows_accepted: 0,
rows_rejected: 0,
}
}
pub fn with_capacity(cap: usize) -> Self {
Self {
memory_limit: None,
columns: Vec::new(),
field_index: HashMap::default(),
column_order: Vec::new(),
row_count: 0,
estimated_rows: cap,
plan: Arc::new(ExecutionPlan::new()),
row_dirty: Vec::new(),
frozen: None,
unknown_error: None,
row_buf: None,
cached_wanted_mask: 0,
ordinal_expect: Vec::new(),
current_ordinal: 0,
strict_error: None,
rows_accepted: 0,
rows_rejected: 0,
}
}
pub fn with_plan(cap: usize, plan: Arc<ExecutionPlan>) -> Self {
let mut b = Self {
memory_limit: None,
columns: Vec::new(),
field_index: HashMap::default(),
column_order: Vec::new(),
row_count: 0,
estimated_rows: cap,
plan: Arc::clone(&plan),
row_dirty: Vec::new(),
frozen: None,
unknown_error: None,
row_buf: if plan.filter.is_some() {
Some(Box::new(RowBuffer {
fields: SmallVec::new(),
state: PredicateState::Undecided,
predicate_mask: SmallVec::new(),
direct: false,
predicate_ordinal: None,
buffer_worthwhile: true,
pred_names: SmallVec::new(),
}))
} else {
None
},
cached_wanted_mask: 0,
ordinal_expect: Vec::new(),
current_ordinal: 0,
strict_error: None,
rows_accepted: 0,
rows_rejected: 0,
};
b.predeclare_schema_columns();
b.cached_wanted_mask = b.compute_wanted_mask();
b
}
fn predeclare_schema_columns(&mut self) {
if self.plan.schema_order.is_empty() {
return;
}
let mut names: Vec<String> = Vec::new();
for name in &self.plan.schema_order {
if !self.plan.drop_fields.contains(name) {
names.push(name.clone());
}
}
if let Some(ref filter) = self.plan.filter {
let mut pred_names: SmallVec<[String; 4]> = SmallVec::new();
Self::collect_predicate_field_names(filter, &self.plan, &mut pred_names);
names.extend(pred_names);
}
for name in names {
if self.field_index.contains_key(&name) || self.plan.drop_fields.contains(&name) {
continue;
}
let ty = self.plan.column_type(&name);
let col = ColumnBuilder::with_capacity(self.estimated_rows.max(64), &ty);
self.columns.push(col);
self.field_index
.insert(name.clone(), self.columns.len() - 1);
self.column_order.push(name);
}
let words = self.columns.len().div_ceil(64);
self.row_dirty.resize(words, 0);
}
fn compute_wanted_mask(&self) -> u64 {
if self.plan.schema_order.is_empty() {
return 0;
}
let mut mask = 0u64;
for &idx in self.field_index.values() {
if idx >= 64 {
return 0;
}
mask |= 1u64 << idx;
}
mask
}
pub fn ensure_schema(&mut self, schema: &crate::schema::FrozenSchema) -> Result<()> {
for (slot, name) in schema.column_names().iter().enumerate() {
let name_str: &str = name;
if self.field_index.contains_key(name_str) {
continue; }
let ty = schema.column_types()[slot].clone();
let col = ColumnBuilder::with_capacity(self.estimated_rows, &ty);
self.columns.push(col);
self.field_index
.insert(name_str.to_string(), self.columns.len() - 1);
self.column_order.push(name_str.to_string());
}
let words = self.columns.len().div_ceil(64);
self.row_dirty.resize(words, 0);
self.frozen = Some(std::sync::Arc::new(schema.clone()));
Ok(())
}
pub(crate) fn get_column(&self, name: &str) -> Option<&ColumnBuilder> {
self.field_index.get(name).map(|&i| &self.columns[i])
}
pub(crate) fn get_column_mut(&mut self, name: &str) -> Option<&mut ColumnBuilder> {
if let Some(&i) = self.field_index.get(name) {
Some(&mut self.columns[i])
} else {
None
}
}
#[allow(dead_code)]
pub(crate) fn bytes_used(&self) -> usize {
self.columns.iter().map(|c| c.bytes_used()).sum::<usize>()
+ self.column_order.iter().map(|s| s.len()).sum::<usize>()
+ self.row_dirty.len() * 8
}
pub(crate) fn split_off(&mut self, n: usize) -> Self {
assert!(n <= self.row_count, "split_off beyond row_count");
assert!(n > 0);
let mut other = Self {
memory_limit: self.memory_limit,
columns: Vec::with_capacity(self.columns.len()),
field_index: self.field_index.clone(),
column_order: self.column_order.clone(),
row_count: n,
estimated_rows: n,
plan: Arc::clone(&self.plan),
row_dirty: vec![0; self.columns.len().div_ceil(64)],
frozen: self.frozen.clone(),
unknown_error: None,
cached_wanted_mask: self.cached_wanted_mask,
row_buf: if self.plan.filter.is_some() {
Some(Box::new(RowBuffer {
fields: SmallVec::new(),
state: PredicateState::Undecided,
predicate_mask: self
.row_buf
.as_ref()
.map_or_else(SmallVec::new, |b| b.predicate_mask.clone()),
direct: false,
predicate_ordinal: None,
buffer_worthwhile: true,
pred_names: self
.row_buf
.as_ref()
.map_or_else(SmallVec::new, |b| b.pred_names.clone()),
}))
} else {
None
},
ordinal_expect: Vec::new(),
current_ordinal: 0,
strict_error: None,
rows_accepted: std::mem::take(&mut self.rows_accepted),
rows_rejected: std::mem::take(&mut self.rows_rejected),
};
for (idx, col) in self.columns.iter_mut().enumerate() {
let drain = col.split_off(n);
other.columns.push(drain);
let _ = idx;
}
self.row_count -= n;
self.row_dirty = vec![0; self.columns.len().div_ceil(64)];
other
}
pub(crate) fn take_column(&mut self, name: &str) -> Option<ColumnBuilder> {
let idx = self.field_index.remove(name)?;
let new_len = self.columns.len().div_ceil(64);
self.row_dirty.resize(new_len, 0);
self.row_dirty.fill(0);
let _ = idx; let last = self.columns.len() - 1;
let col = if idx == last {
self.columns.pop().unwrap()
} else {
let col = self.columns.swap_remove(idx);
let old_last = self.columns.len(); let moved_name = self.field_index.iter().find_map(|(k, &v)| {
if v == old_last {
Some(k.clone())
} else {
None
}
});
if let Some(k) = moved_name {
self.field_index.insert(k, idx);
}
col
};
Some(col)
}
pub fn num_rows(&self) -> usize {
self.row_count
}
pub fn num_columns(&self) -> usize {
self.column_order.len()
}
pub fn column_names(&self) -> &[String] {
&self.column_order
}
pub fn column_diagnostics(&self) -> Vec<(String, usize, usize)> {
self.column_order
.iter()
.map(|name| {
let idx = self.field_index[name];
let col = &self.columns[idx];
(name.clone(), col.bytes_used(), col.capacity_bytes())
})
.collect()
}
pub fn estimated_rows(&self) -> usize {
self.estimated_rows
}
pub fn finish(&mut self) -> Result<RecordBatch> {
ColumnarSink::finish(self)
}
pub(crate) fn prepare_finish(&mut self) -> Result<()> {
if let Some(err) = self.unknown_error.take() {
return Err(crate::Error::Merge(err));
}
if let Some(err) = self.strict_error.take() {
return Err(err);
}
self.check_memory_budget()?;
if let Some(ref observer) = self.plan.observer {
let result = catch_unwind(AssertUnwindSafe(|| {
observer.on_chunk_finished(
self.rows_accepted + self.rows_rejected,
self.rows_accepted,
self.rows_rejected,
);
}));
if let Err(payload) = result {
let message = payload
.downcast_ref::<&str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "unknown panic".to_string());
return Err(crate::Error::Parser(format!(
"observer panicked at chunk completion: {message}"
)));
}
}
self.normalize();
Ok(())
}
pub fn reset(&mut self) {
self.columns.clear();
self.field_index.clear();
self.column_order.clear();
self.row_dirty.fill(0);
self.row_count = 0;
if let Some(ref mut buf) = self.row_buf {
buf.fields.clear();
buf.state = PredicateState::Undecided;
}
self.unknown_error = None;
self.strict_error = None;
self.rows_accepted = 0;
self.rows_rejected = 0;
}
pub fn normalize(&mut self) {
for b in &mut self.columns {
while b.len() > self.row_count {
b.pop();
}
}
self.row_dirty.fill(0);
}
pub fn auto_dict_upgrade(&mut self) {
if self.plan.auto_dict {
let max_ratio = self.plan.dict_threshold.unwrap_or(0.05);
let max_size = self.plan.dict_max_size.unwrap_or(256);
for b in &mut self.columns {
b.try_upgrade_to_dict(512, max_ratio, max_size);
}
}
}
pub fn sort_columns(&mut self) {
if self.plan.schema_order.is_empty() {
return;
}
let order = &self.plan.schema_order;
let rank = |name: &String| order.iter().position(|n| n == name).unwrap_or(usize::MAX);
self.column_order.sort_by_key(rank);
}
pub(crate) fn schema_insert_index(&self, name: &str) -> usize {
let order = &self.plan.schema_order;
if order.is_empty() {
return self.column_order.len();
}
let pos = order.iter().position(|n| n == name);
match pos {
Some(p) => self
.column_order
.iter()
.position(|existing| {
order
.iter()
.position(|n| n == existing)
.is_some_and(|ep| ep > p)
})
.unwrap_or(self.column_order.len()),
None => self.column_order.len(),
}
}
fn ensure_column_idx(&mut self, name: &str) -> usize {
if let Some(&idx) = self.field_index.get(name) {
return idx;
}
if self.frozen.is_some() {
if let Some(frozen) = &self.frozen {
if frozen.is_exact() {
if self.unknown_error.is_none() {
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
name,
frozen.num_columns(),
frozen.is_exact()
));
}
return 0;
}
}
}
let est = self.estimated_rows.max(64);
let col_type = self.plan.column_type(name);
let mut b = ColumnBuilder::with_capacity(est, &col_type);
let backfill = if self.row_buf.as_ref().is_some_and(|b| b.direct) {
self.row_count.saturating_sub(1)
} else {
self.row_count
};
for _ in 0..backfill {
b.push(None);
}
let idx = self.columns.len();
self.columns.push(b);
self.field_index.insert(name.to_owned(), idx);
let needed = self.columns.len().div_ceil(64);
if self.row_dirty.len() < needed {
self.row_dirty.resize(needed, 0);
}
let order_idx = self.schema_insert_index(name);
self.column_order.insert(order_idx, name.to_owned());
idx
}
#[inline]
fn record_strict_violation_at(&mut self, idx: usize, value: &Value<'_>) {
if !self.plan.strict_types || self.strict_error.is_some() {
return;
}
let Value::Str(s) = value else {
return;
};
let Some(name) = self.column_order.get(idx) else {
return;
};
let Some(ft) = self.plan.field_types.get(name) else {
return;
};
if crate::columnar::parses_as(s, ft) {
return;
}
let truncated: String = s.chars().take(64).collect();
self.strict_error = Some(crate::Error::Plan(format!(
"strict_types: value {truncated:?} in column {name:?} at row {} \
does not parse as declared type {ft:?}",
self.row_count
)));
}
#[inline]
fn notify_put_field(&self, idx: usize, value: &Value<'_>) {
if let Some(ref obs) = self.plan.observer {
if let Some(name) = self.column_order.get(idx) {
obs.on_put_field(self.row_count, name, idx, value);
}
}
}
#[inline]
fn push_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
if let Some(frozen) = &self.frozen {
if frozen.is_exact() {
if self.unknown_error.is_none() {
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
resolved_name,
frozen.num_columns(),
frozen.is_exact()
));
}
return;
}
}
}
let idx = self.ensure_column_idx(resolved_name);
self.record_strict_violation_at(idx, &value);
self.notify_put_field(idx, &value);
if self.row_count == 0 {
let ord = self.current_ordinal as usize;
if ord >= self.ordinal_expect.len() {
self.ordinal_expect.resize_with(ord + 1, || None);
}
self.ordinal_expect[ord] = Some((idx as u32, resolved_name.as_bytes().to_vec()));
}
self.current_ordinal += 1;
let word = idx / 64;
let bit = idx % 64;
self.row_dirty[word] |= 1u64 << bit;
let b = &mut self.columns[idx];
let row_count = self.row_count;
if b.len() > row_count {
b.pop();
}
b.push_value(value);
}
#[inline]
fn push_field(&mut self, name: &str, value: Value<'_>) {
if self.plan.field_map.is_empty()
&& self.plan.drop_fields.is_empty()
&& self.plan.schema_order.is_empty()
{
self.push_field_resolved(name, value);
return;
}
if let Some(idx) = Self::resolve_and_slot(&self.plan, &self.field_index, name) {
self.record_strict_violation_at(idx, &value);
self.notify_put_field(idx, &value);
if self.row_count == 0 {
let ord = self.current_ordinal as usize;
if ord >= self.ordinal_expect.len() {
self.ordinal_expect.resize_with(ord + 1, || None);
}
let resolved = self.plan.resolve_field(name).unwrap_or(name);
self.ordinal_expect[ord] = Some((idx as u32, resolved.as_bytes().to_vec()));
}
self.current_ordinal += 1;
let word = idx / 64;
let bit = idx % 64;
self.row_dirty[word] |= 1u64 << bit;
let b = &mut self.columns[idx];
let row_count = self.row_count;
if b.len() > row_count {
b.pop();
}
b.push_value(value);
} else {
if let Some(resolved) = self.plan.resolve_field(name) {
let owned = resolved.to_owned();
self.push_field_resolved(&owned, value);
}
}
}
#[inline]
fn resolve_and_slot(
plan: &ExecutionPlan,
field_index: &HashMap<String, usize>,
name: &str,
) -> Option<usize> {
let resolved = plan.resolve_field(name)?;
field_index.get(resolved).copied()
}
#[doc(hidden)]
pub fn advance_row(&mut self) {
self.row_count += 1;
}
fn finish_row(&mut self) {
let ncols = self.columns.len();
let full_words = ncols / 64;
let rem_bits = ncols % 64;
let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
&& (rem_bits == 0
|| self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
if is_full {
self.row_dirty.fill(0);
} else {
for (i, b) in self.columns.iter_mut().enumerate() {
let word = i / 64;
let bit = i % 64;
let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
if !is_set {
b.push(None);
}
}
self.row_dirty.fill(0);
}
if let Some(ref filter) = self.plan.filter {
if !filter.check(&self.columns, &self.field_index, self.row_count, &self.plan) {
for b in &mut self.columns {
b.pop();
}
self.rows_rejected += 1;
if let Some(ref obs) = self.plan.observer {
obs.on_row_rejected(self.row_count);
}
return;
}
}
self.row_count += 1;
self.rows_accepted += 1;
if let Some(ref obs) = self.plan.observer {
obs.on_row_accepted(self.row_count - 1);
}
}
fn null_fill_missing(&mut self) {
let ncols = self.columns.len();
let full_words = ncols / 64;
let rem_bits = ncols % 64;
let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
&& (rem_bits == 0
|| self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
if is_full {
self.row_dirty.fill(0);
} else {
for (i, b) in self.columns.iter_mut().enumerate() {
let word = i / 64;
let bit = i % 64;
let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
if !is_set {
b.push(None);
}
}
self.row_dirty.fill(0);
}
}
}
impl Default for TableBuilder {
fn default() -> Self {
Self::new()
}
}
impl ColumnarSink for TableBuilder {
#[inline]
fn begin_row(&mut self) {
if !self.reserve_memory(0) {
return;
}
if let Some(ref obs) = self.plan.observer {
obs.on_begin_row(self.row_count);
}
self.current_ordinal = 0;
if let Some(ref mut buf) = self.row_buf {
if buf.predicate_ordinal.is_some() && buf.buffer_worthwhile && self.row_count > 0 {
let ncols = self
.frozen
.as_ref()
.map(|f| f.num_columns() as u32)
.or_else(|| {
(!self.plan.schema_order.is_empty())
.then(|| self.plan.schema_order.len() as u32)
})
.unwrap_or(self.columns.len() as u32);
if let Some(ordinal) = buf.predicate_ordinal {
if ordinal >= ncols * 4 / 5 {
buf.buffer_worthwhile = false;
}
}
}
buf.fields.clear();
buf.state = PredicateState::Undecided;
buf.direct = false;
}
}
#[inline]
fn put_field(&mut self, name: &str, value: Value<'_>) {
if !self.reserve_field_memory(name.len(), &value) {
return;
}
if self.plan.filter.is_none() {
self.push_field(name, value);
return;
}
let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
if !buf_worthwhile {
self.push_field(name, value);
return;
}
let resolved = match self.plan.resolve_field(name) {
Some(r) => r.to_owned(),
None => return,
};
if self.frozen.is_some()
&& !self.field_index.contains_key(&resolved)
&& self.unknown_error.is_none()
{
if let Some(ref frozen) = self.frozen {
if frozen.is_exact()
&& !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
{
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
resolved,
frozen.num_columns(),
frozen.is_exact()
));
return;
}
}
}
let slot = self.ensure_column_idx(&resolved) as u32;
self.record_strict_violation_at(slot as usize, &value);
self.mark_predicate_slot(slot);
let val_static: Value<'static> = value.into_static();
let is_pred = self.is_predicate_slot(slot);
#[cfg(feature = "profile")]
if is_pred {
crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
} else {
crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
if let Some(ref mut buf) = self.row_buf {
if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
buf.fields[pos].1 = val_static;
} else {
buf.fields.push((slot, val_static));
}
}
if is_pred {
if let Some(ref mut buf) = self.row_buf {
if buf.predicate_ordinal.is_none() {
buf.predicate_ordinal = Some(slot);
}
}
let state = self.evaluate_predicate_state();
if let Some(ref mut buf) = self.row_buf {
buf.state = state;
}
}
}
#[inline]
fn end_row(&mut self) {
if !self.reserve_memory(0) {
return;
}
if self.plan.filter.is_none() {
self.finish_row();
return;
}
let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
if !buf_worthwhile {
self.finish_row();
return;
}
let (state, direct) = self
.row_buf
.as_ref()
.map_or((PredicateState::Pass, false), |b| (b.state, b.direct));
if direct {
self.null_fill_missing();
return;
}
let state = if state == PredicateState::Undecided {
self.evaluate_against_null()
} else {
state
};
let pass = state == PredicateState::Pass;
self.drain_buffered(pass);
}
#[inline]
fn wants(&self, name: &str) -> bool {
self.resolve(name).is_some()
}
#[inline]
fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
self.plan.resolve_field(name)
}
#[inline]
fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
if !self.reserve_field_memory(resolved_name.len(), &value) {
return;
}
if self.plan.filter.is_none() {
self.push_field_resolved(resolved_name, value);
return;
}
let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
if !buf_worthwhile {
self.push_field_resolved(resolved_name, value);
return;
}
if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
if let Some(ref frozen) = self.frozen {
if frozen.is_exact()
&& !frozen
.column_names()
.iter()
.any(|n| n.as_ref() == resolved_name)
{
if self.unknown_error.is_none() {
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
resolved_name,
frozen.num_columns(),
frozen.is_exact()
));
}
return;
}
}
}
let slot = self.ensure_column_idx(resolved_name) as u32;
self.record_strict_violation_at(slot as usize, &value);
self.mark_predicate_slot(slot);
let val_static: Value<'static> = value.into_static();
let is_pred = self.is_predicate_slot(slot);
#[cfg(feature = "profile")]
if is_pred {
crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
} else {
crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
if let Some(ref mut buf) = self.row_buf {
if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
buf.fields[pos].1 = val_static;
} else {
buf.fields.push((slot, val_static));
}
}
if is_pred {
if let Some(ref mut buf) = self.row_buf {
if buf.predicate_ordinal.is_none() {
buf.predicate_ordinal = Some(slot);
}
}
let state = self.evaluate_predicate_state();
if let Some(ref mut buf) = self.row_buf {
buf.state = state;
}
}
}
#[inline]
fn put_field_at(&mut self, slot: u32, value: Value<'_>) {
if !self.reserve_field_memory(0, &value) {
return;
}
let idx = slot as usize;
if idx >= self.columns.len() {
return;
}
self.record_strict_violation_at(idx, &value);
self.notify_put_field(idx, &value);
let word = idx / 64;
let bit = idx % 64;
self.row_dirty[word] |= 1u64 << bit;
let b = &mut self.columns[idx];
let row_count = self.row_count;
if b.len() > row_count {
b.pop();
}
b.push_value(value);
}
#[inline]
fn put_str_at(&mut self, slot: u32, value: &str) {
if self.memory_limit.is_some() || self.row_buf.is_some() {
self.put_field_at(slot, Value::Str(std::borrow::Cow::Borrowed(value)));
return;
}
let idx = slot as usize;
if idx >= self.columns.len() {
return;
}
if self.plan.strict_types {
self.record_strict_violation_at(idx, &Value::Str(std::borrow::Cow::Borrowed(value)));
}
if self.plan.observer.is_some() {
self.notify_put_field(idx, &Value::Str(std::borrow::Cow::Borrowed(value)));
}
let word = idx / 64;
let bit = idx % 64;
self.row_dirty[word] |= 1u64 << bit;
let b = &mut self.columns[idx];
let row_count = self.row_count;
if b.len() > row_count {
b.pop();
}
b.push_str(Some(value));
}
#[inline]
fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
if !self.reserve_field_memory(name.len(), &value) {
return;
}
#[cfg(feature = "profile")]
RESOLVE_AND_PUT_COUNT.fetch_add(1, Ordering::Relaxed);
if self.plan.filter.is_none() {
if self.plan.field_map.is_empty()
&& self.plan.drop_fields.is_empty()
&& self.plan.schema_order.is_empty()
{
self.push_field_resolved(name, value);
} else {
if let Some(resolved) = self.plan.resolve_field(name) {
let owned = resolved.to_owned();
self.push_field_resolved(&owned, value);
}
}
return;
}
if self.row_buf.as_ref().is_some_and(|b| !b.buffer_worthwhile) {
if self.plan.field_map.is_empty()
&& self.plan.drop_fields.is_empty()
&& self.plan.schema_order.is_empty()
{
self.push_field_resolved(name, value);
} else {
if let Some(resolved) = self.plan.resolve_field(name) {
let owned = resolved.to_owned();
self.push_field_resolved(&owned, value);
}
}
return;
}
if let Some(ref buf) = self.row_buf {
if buf.direct {
let resolved = match self.plan.resolve_field(name) {
Some(r) => r.to_owned(),
None => return, };
if self.frozen.is_some() && !self.field_index.contains_key(resolved.as_str()) {
if let Some(ref frozen) = self.frozen {
if frozen.is_exact()
&& !frozen
.column_names()
.iter()
.any(|n| n.as_ref() == resolved.as_str())
{
if self.unknown_error.is_none() {
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
resolved, frozen.num_columns(), frozen.is_exact()
));
}
return;
}
}
}
self.push_field_resolved(&resolved, value);
return;
}
}
if self.plan.field_map.is_empty()
&& self.plan.drop_fields.is_empty()
&& self.plan.schema_order.is_empty()
{
let resolved = name;
if self.frozen.is_some() && !self.field_index.contains_key(resolved) {
if let Some(ref frozen) = self.frozen {
if frozen.is_exact()
&& !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
{
if self.unknown_error.is_none() {
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
resolved,
frozen.num_columns(),
frozen.is_exact()
));
}
return;
}
}
}
let slot = self.ensure_column_idx(resolved) as u32;
self.record_strict_violation_at(slot as usize, &value);
self.mark_predicate_slot(slot);
let val_static: Value<'static> = value.into_static();
let is_pred = self.is_predicate_slot(slot);
#[cfg(feature = "profile")]
if is_pred {
crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
} else {
crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
if let Some(ref mut buf) = self.row_buf {
if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
buf.fields[pos].1 = val_static;
} else {
buf.fields.push((slot, val_static));
}
}
if is_pred {
if let Some(ref mut buf) = self.row_buf {
if buf.predicate_ordinal.is_none() {
buf.predicate_ordinal = Some(slot);
}
}
let state = self.evaluate_predicate_state();
if let Some(ref mut buf) = self.row_buf {
buf.state = state;
if state == PredicateState::Pass {
buf.direct = true;
}
}
if state == PredicateState::Pass {
self.drain_buffered(true);
}
}
} else {
if let Some(resolved) = self.plan.resolve_field(name) {
let owned = resolved.to_owned();
if self.frozen.is_some() && !self.field_index.contains_key(&owned) {
if let Some(ref frozen) = self.frozen {
if frozen.is_exact()
&& !frozen.column_names().iter().any(|n| n.as_ref() == owned)
{
if self.unknown_error.is_none() {
self.unknown_error = Some(format!(
"unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
owned,
frozen.num_columns(),
frozen.is_exact()
));
}
return;
}
}
}
let slot = self.ensure_column_idx(&owned) as u32;
self.record_strict_violation_at(slot as usize, &value);
self.mark_predicate_slot(slot);
let val_static: Value<'static> = value.into_static();
let is_pred = self.is_predicate_slot(slot);
#[cfg(feature = "profile")]
if is_pred {
crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
} else {
crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
if let Some(ref mut buf) = self.row_buf {
if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
buf.fields[pos].1 = val_static;
} else {
buf.fields.push((slot, val_static));
}
}
if is_pred {
if let Some(ref mut buf) = self.row_buf {
if buf.predicate_ordinal.is_none() {
buf.predicate_ordinal = Some(slot);
}
}
let state = self.evaluate_predicate_state();
if let Some(ref mut buf) = self.row_buf {
buf.state = state;
}
}
}
}
}
#[inline]
fn row_rejected(&self) -> bool {
self.plan.filter.is_some()
&& self
.row_buf
.as_ref()
.is_some_and(|b| b.state == PredicateState::Fail)
}
#[inline]
fn row_satisfied(&self) -> bool {
let mask = self.wanted_mask();
if mask == 0 {
return false;
}
let dirty = self
.row_dirty
.iter()
.copied()
.reduce(|a, b| a | b)
.unwrap_or(0);
(dirty & mask) == mask
}
#[inline]
fn wanted_mask(&self) -> u64 {
self.cached_wanted_mask
}
#[inline]
fn finish(&mut self) -> Result<RecordBatch> {
self.prepare_finish()?;
if self.column_order.is_empty() {
let schema = Arc::new(Schema::empty());
return Ok(RecordBatch::new_empty(schema));
}
self.auto_dict_upgrade();
if !self.plan.schema_order.is_empty() {
let order = self.plan.schema_order.clone();
let mut fields = Vec::with_capacity(order.len());
let mut arrays = Vec::with_capacity(order.len());
for name in &order {
if self.plan.drop_fields.contains(name) {
continue;
}
let idx = match self.field_index.get(name.as_str()) {
Some(&i) => i,
None => {
let ty = self.plan.column_type(name);
let mut b = ColumnBuilder::with_capacity(self.row_count.max(1), &ty);
for _ in 0..self.row_count {
b.push(None);
}
let idx = self.columns.len();
self.columns.push(b);
self.field_index.insert(name.clone(), idx);
idx
}
};
let b = &mut self.columns[idx];
fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
arrays.push(b.to_arrow_array()?);
}
let schema = Arc::new(Schema::new(fields));
return Ok(RecordBatch::try_new(schema, arrays)?);
}
self.sort_columns();
let mut fields = Vec::with_capacity(self.column_order.len());
let mut arrays = Vec::with_capacity(self.column_order.len());
for name in &self.column_order {
if let Some(&idx) = self.field_index.get(name.as_str()) {
let b = &mut self.columns[idx];
fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
arrays.push(b.to_arrow_array()?);
}
}
let schema = Arc::new(Schema::new(fields));
Ok(RecordBatch::try_new(schema, arrays)?)
}
#[inline]
fn expect_slot(&self, ordinal: u32) -> Option<(u32, &[u8])> {
self.ordinal_expect
.get(ordinal as usize)
.and_then(|entry| entry.as_ref())
.map(|(slot, raw)| (*slot, raw.as_slice()))
}
#[inline]
fn record_slot(&mut self, ordinal: u32, slot: u32, raw_name: &[u8]) {
let idx = ordinal as usize;
let growth = idx
.saturating_add(1)
.saturating_sub(self.ordinal_expect.capacity())
.saturating_mul(std::mem::size_of::<Option<(u32, Vec<u8>)>>());
if !self.reserve_memory(growth.saturating_add(raw_name.len())) {
return;
}
if idx >= self.ordinal_expect.len() {
self.ordinal_expect.resize_with(idx + 1, || None);
}
self.ordinal_expect[idx] = Some((slot, raw_name.to_vec()));
}
#[inline]
fn layout_broken(&mut self, ordinal: u32) {
if let Some(entry) = self.ordinal_expect.get_mut(ordinal as usize) {
*entry = None;
}
}
#[inline]
fn reset_child_ordinal(&mut self) {
self.current_ordinal = 0;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::columnar::ColumnBuilder;
use crate::decoder::{ColumnarSink, RecordParser, Splitter};
use crate::plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
use crate::value::Value;
use crate::Result;
use arrow::array::{Array, AsArray};
#[derive(Clone)]
struct LineParser;
impl RecordParser for LineParser {
fn validate(&self, bytes: &[u8]) -> Result<()> {
simdutf8::basic::from_utf8(bytes)?;
Ok(())
}
fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
for line in text.lines() {
if line.is_empty() {
continue;
}
sink.begin_row();
for token in line.split_whitespace() {
if let Some((k, v)) = token.split_once('=') {
sink.put_field(k, Value::Str(Cow::Borrowed(v)));
}
}
sink.end_row();
}
Ok(())
}
}
#[derive(Clone)]
struct LineSplitter;
impl Splitter for LineSplitter {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
if from >= bytes.len() {
return None;
}
let start = if bytes[from] == b'\n' { from + 1 } else { from };
if start >= bytes.len() {
return None;
}
memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
}
fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
if max_chunks <= 1 || bytes.is_empty() {
return vec![0, bytes.len()];
}
let mut points = vec![0usize];
let mut last = 0;
for (i, &b) in bytes.iter().enumerate() {
if b == b'\n' {
let next = i + 1;
if next > last && points.len() < max_chunks {
points.push(next);
last = next;
}
}
}
if *points.last().unwrap() != bytes.len() {
points.push(bytes.len());
}
points
}
fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
(sample.len() / newline_count).max(1)
}
}
#[derive(Default)]
struct RecObserver {
events: std::sync::Mutex<Vec<String>>,
}
impl RecObserver {
fn log(&self, e: String) {
self.events.lock().unwrap().push(e);
}
fn events(&self) -> Vec<String> {
self.events.lock().unwrap().clone()
}
}
impl crate::RowObserver for RecObserver {
fn on_begin_row(&self, row_index: usize) {
self.log(format!("begin:{row_index}"));
}
fn on_put_field(&self, row_index: usize, name: &str, _slot: usize, _value: &Value<'_>) {
self.log(format!("put:{row_index}:{name}"));
}
fn on_row_accepted(&self, row_index: usize) {
self.log(format!("accept:{row_index}"));
}
fn on_row_rejected(&self, row_index: usize) {
self.log(format!("reject:{row_index}"));
}
fn on_chunk_finished(&self, total: usize, accepted: usize, rejected: usize) {
self.log(format!("fin:{total}:{accepted}:{rejected}"));
}
}
#[test]
fn test_observer_lifecycle() {
let obs = Arc::new(RecObserver::default());
let mut plan = ExecutionPlan::new();
plan.observer = Some(obs.clone());
let mut engine = parse_bytes(b"X=1 Y=2\nX=3 Y=4\n", plan);
engine.finish().unwrap();
assert_eq!(
obs.events(),
vec![
"begin:0",
"put:0:X",
"put:0:Y",
"accept:0",
"begin:1",
"put:1:X",
"put:1:Y",
"accept:1",
"fin:2:2:0",
]
);
}
#[test]
fn test_observer_with_filter() {
let obs = Arc::new(RecObserver::default());
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "X".to_string(),
value: "keep".to_string(),
});
plan.observer = Some(obs.clone());
let mut engine = parse_bytes(b"X=keep A=1\nX=drop A=2\n", plan);
engine.finish().unwrap();
let ev = obs.events();
assert!(ev.contains(&"reject:1".to_string()), "events: {ev:?}");
assert_eq!(
ev.iter().filter(|e| e.starts_with("put:")).count(),
2,
"only the accepted row's fields may be observed: {ev:?}"
);
assert!(ev.contains(&"fin:2:1:1".to_string()), "events: {ev:?}");
}
#[test]
fn test_observer_parallel_counts_all_rows() {
use std::sync::atomic::{AtomicUsize, Ordering as AOrd};
#[derive(Default)]
struct Counting {
accepted: AtomicUsize,
rejected: AtomicUsize,
}
impl crate::RowObserver for Counting {
fn on_row_accepted(&self, _i: usize) {
self.accepted.fetch_add(1, AOrd::Relaxed);
}
fn on_row_rejected(&self, _i: usize) {
self.rejected.fetch_add(1, AOrd::Relaxed);
}
}
let obs = Arc::new(Counting::default());
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::NotEqual {
field: "X".to_string(),
value: "drop".to_string(),
});
plan.observer = Some(obs.clone());
let mut data = Vec::new();
for i in 0..1000 {
let v = if i % 3 == 0 { "drop" } else { "keep" };
data.extend_from_slice(format!("X={v} N={i}\n").as_bytes());
}
let pipeline = crate::Pipeline::new(LineSplitter, LineParser).with_plan(plan);
let batches = pipeline.read_bytes_par(&data, 4).unwrap();
let total: usize = batches.iter().map(|b| b.num_rows()).sum();
let accepted = obs.accepted.load(AOrd::Relaxed);
let rejected = obs.rejected.load(AOrd::Relaxed);
assert_eq!(accepted + rejected, 1000);
assert_eq!(accepted, total);
assert_eq!(rejected, 334);
}
#[test]
fn test_not_wraps_leaf_predicates_predicate_first() {
let cases: Vec<(FilterPredicate, &[u8], Vec<&str>)> = vec![
(
FilterPredicate::not(FilterPredicate::Equal {
field: "X".into(),
value: "keep".into(),
}),
b"X=keep\nX=drop\n",
vec!["drop"],
),
(
FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() }),
b"X=1\nY=2\n",
vec!["1"],
),
(
FilterPredicate::not(FilterPredicate::StartsWith {
field: "X".into(),
value: "ab".into(),
}),
b"X=abc\nX=zzz\n",
vec!["zzz"],
),
(
FilterPredicate::not(FilterPredicate::In {
field: "X".into(),
values: vec!["a".into(), "b".into()],
}),
b"X=a\nX=b\nX=c\n",
vec!["c"],
),
(
FilterPredicate::not(FilterPredicate::Regex {
field: "X".into(),
re: crate::plan::RegexSpec::new("^ERR").unwrap(),
}),
b"X=ERR1\nX=ok\n",
vec!["ok"],
),
(
FilterPredicate::not(FilterPredicate::Contains {
field: "X".into(),
value: "zz".into(),
}),
b"X=azza\nX=hello\n",
vec!["hello"],
),
];
for (pred, data, want) in cases {
let mut plan = ExecutionPlan::new();
plan.filter = Some(pred);
let engine = parse_bytes(data, plan);
let col = engine.get_column("X").unwrap();
assert_eq!(
col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
want,
"data: {}",
String::from_utf8_lossy(data)
);
}
}
#[test]
fn test_not_wraps_leaf_predicates_batch_check() {
let data = b"X=keep N=1\nX=drop N=2\nX=keep N=3\n";
let engine = parse_bytes(data, ExecutionPlan::new());
let pred = FilterPredicate::not(FilterPredicate::Equal {
field: "X".into(),
value: "keep".into(),
});
let kept: Vec<usize> = (0..engine.num_rows())
.filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
.collect();
assert_eq!(kept, vec![1]);
let pred = FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() });
let kept: Vec<usize> = (0..engine.num_rows())
.filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
.collect();
assert_eq!(kept, vec![0, 1, 2]);
let pred = FilterPredicate::not(FilterPredicate::Regex {
field: "X".into(),
re: crate::plan::RegexSpec::new("ke*p").unwrap(),
});
let kept: Vec<usize> = (0..engine.num_rows())
.filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
.collect();
assert_eq!(kept, vec![1]);
}
#[test]
fn test_regex_predicate() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Regex {
field: "X".into(),
re: crate::plan::RegexSpec::new("^ERR\\d+$").unwrap(),
});
let engine = parse_bytes(b"X=ERR12\nX=ok\nX=ERRx\nX=ERR0\n", plan);
let col = engine.get_column("X").unwrap();
assert_eq!(
col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
vec!["ERR12", "ERR0"]
);
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Regex {
field: "X".into(),
re: crate::plan::RegexSpec::new(".*").unwrap(),
});
let engine = parse_bytes(b"X=a\nY=2\n", plan);
assert_eq!(engine.num_rows(), 1);
assert!(crate::plan::RegexSpec::new("(").is_err());
assert_eq!(
crate::plan::RegexSpec::new("^a").unwrap(),
crate::plan::RegexSpec::new("^a").unwrap()
);
assert_ne!(
crate::plan::RegexSpec::new("^a").unwrap(),
crate::plan::RegexSpec::new("^b").unwrap()
);
}
#[test]
fn test_no_observer_smoke() {
let engine = parse_bytes(b"X=1\nX=2\n", ExecutionPlan::new());
assert_eq!(engine.num_rows(), 2);
}
fn parse_bytes(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
LineParser.parse_chunk(bytes, &mut sink).unwrap();
sink
}
#[test]
fn test_extend_no_duplicates() {
let e1 = parse_bytes(b"A=1 B=2\n", ExecutionPlan::new());
let mut merged = TableBuilder::new();
merged.extend(e1).unwrap();
assert_eq!(merged.num_rows(), 1);
for col in &merged.columns {
assert_eq!(col.len(), merged.num_rows(), "column length mismatch");
}
}
#[test]
fn test_multi_chunk_same_as_single() {
let data = b"A=1\nA=2\nA=3\n";
let single = parse_bytes(data, ExecutionPlan::new());
assert_eq!(single.num_rows(), 3);
let splitter = LineSplitter;
let points = splitter.find_split_points(data, 2);
let mut merged = TableBuilder::new();
for w in points.windows(2) {
let chunk = &data[w[0]..w[1]];
let engine = parse_bytes(chunk, ExecutionPlan::new());
merged.extend(engine).unwrap();
}
assert_eq!(merged.num_rows(), single.num_rows());
}
#[test]
fn test_last_write_wins_duplicate_field() {
let engine = parse_bytes(b"X=10 X=20\n", ExecutionPlan::new());
assert_eq!(engine.num_rows(), 1);
let col = engine.get_column("X").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("20".into())]);
}
#[test]
fn test_build_plan_rename() {
let mut plan = ExecutionPlan::new();
plan.field_map.insert("X".to_string(), "Y".to_string());
let engine = parse_bytes(b"X=hello\n", plan);
assert_eq!(engine.num_rows(), 1);
assert!(engine.get_column("Y").is_some());
assert!(engine.get_column("X").is_none());
assert_eq!(
engine.get_column("Y").unwrap().as_str_vec(),
vec![Some("hello".into())]
);
}
#[test]
fn test_build_plan_drop() {
let mut plan = ExecutionPlan::new();
plan.drop_fields.insert("X".to_string());
let engine = parse_bytes(b"X=hello Y=world\n", plan);
assert_eq!(engine.num_rows(), 1);
assert!(engine.get_column("X").is_none());
assert!(engine.get_column("Y").is_some());
}
#[test]
fn test_build_plan_filter_ne() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::NotEqual {
field: "X".to_string(),
value: "42".to_string(),
});
let engine = parse_bytes(b"X=10\nX=42\nX=30\n", plan);
assert_eq!(engine.num_rows(), 2);
let col = engine.get_column("X").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("30".into())]);
}
#[test]
fn test_build_plan_filter_eq() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "X".to_string(),
value: "10".to_string(),
});
let engine = parse_bytes(b"X=10\nX=20\nX=10\n", plan);
assert_eq!(engine.num_rows(), 2);
let col = engine.get_column("X").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("10".into())]);
}
#[test]
fn test_build_plan_filter_missing_field() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::NotEqual {
field: "X".to_string(),
value: "10".to_string(),
});
let engine = parse_bytes(b"X=10\nY=99\n", plan);
assert_eq!(engine.num_rows(), 1);
let col = engine.get_column("Y").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("99".into())]);
}
#[test]
fn test_typed_int64_column() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Int64);
let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
assert_eq!(engine.num_rows(), 3);
if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
assert_eq!(v.get(0), Some(42));
assert_eq!(v.get(1), None);
assert_eq!(v.get(2), Some(100));
} else {
panic!("expected Int64 builder");
}
}
#[test]
fn test_typed_float64_column() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Float64);
let engine = parse_bytes(b"X=1.5\n", plan);
if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
} else {
panic!("expected Float64 builder");
}
}
#[test]
fn test_is_null_filter() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::IsNull {
field: "X".to_string(),
});
let engine = parse_bytes(b"X=hello\nY=world\nX=fallback\n", plan);
assert_eq!(engine.num_rows(), 1);
let col = engine.get_column("Y").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("world".into())]);
}
#[test]
fn test_not_is_null_filter_predicate_first() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
field: "X".to_string(),
}));
let engine = parse_bytes(b"X=a\nY=only\nX=b\n", plan);
assert_eq!(engine.num_rows(), 2);
let col = engine.get_column("X").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
}
#[test]
fn test_not_is_null_filter_batch_eval() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
field: "X".to_string(),
}));
let mut sink = TableBuilder::with_plan(4, Arc::new(plan));
sink.row_buf.as_mut().unwrap().buffer_worthwhile = false;
LineParser
.parse_chunk(b"X=a\nY=only\nX=b\n", &mut sink)
.unwrap();
assert_eq!(sink.num_rows(), 2);
let col = sink.get_column("X").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
}
#[test]
fn test_strict_types_errors_on_malformed_int64() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Int64);
plan.strict_types = true;
let mut engine = parse_bytes(b"X=42\nX=abc\n", plan);
let err = engine.finish().unwrap_err();
let msg = err.to_string();
assert!(msg.contains("\"abc\""), "message should name value: {msg}");
assert!(msg.contains("\"X\""), "message should name column: {msg}");
assert!(msg.contains("Int64"), "message should name type: {msg}");
assert!(msg.contains("row 1"), "message should name row: {msg}");
}
#[test]
fn test_strict_types_off_yields_null() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Int64);
let engine = parse_bytes(b"X=42\nX=abc\n", plan);
assert!(engine.strict_error.is_none());
if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
assert_eq!(v.get(0), Some(42));
assert_eq!(v.get(1), None);
} else {
panic!("expected Int64 builder");
}
}
#[test]
fn test_strict_types_missing_field_stays_null() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Int64);
plan.strict_types = true;
let mut engine = parse_bytes(b"X=42\nY=1\n", plan);
let batch = engine.finish().unwrap();
let idx = batch.schema().index_of("X").unwrap();
let arr = batch
.column(idx)
.as_primitive::<arrow::datatypes::Int64Type>();
assert_eq!(arr.value(0), 42);
assert!(arr.is_null(1));
}
#[test]
fn test_strict_types_buffered_filter_path() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Int64);
plan.strict_types = true;
plan.filter = Some(FilterPredicate::Equal {
field: "Y".to_string(),
value: "1".to_string(),
});
let mut engine = parse_bytes(b"Y=1 X=abc\n", plan);
let err = engine.finish().unwrap_err();
assert!(err.to_string().contains("strict_types"));
}
#[test]
fn test_is_type_filter_int64() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Int64);
plan.filter = Some(FilterPredicate::IsType {
field: "X".to_string(),
field_type: FieldType::Int64,
});
let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
assert_eq!(engine.num_rows(), 2);
if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
assert_eq!(v.get(0), Some(42));
assert_eq!(v.get(1), Some(100));
} else {
panic!("expected Int64 builder");
}
}
#[test]
fn test_is_type_filter_float64() {
let mut plan = ExecutionPlan::new();
plan.field_types.insert("X".to_string(), FieldType::Float64);
plan.filter = Some(FilterPredicate::IsType {
field: "X".to_string(),
field_type: FieldType::Float64,
});
let engine = parse_bytes(b"X=1.5\nX=bad\nX=2.5\n", plan);
assert_eq!(engine.num_rows(), 2);
if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
assert!((v.get(1).unwrap() - 2.5).abs() < 1e-9);
} else {
panic!("expected Float64 builder");
}
}
#[test]
fn test_dictionary_column() {
let mut plan = ExecutionPlan::new();
plan.dictionary_columns.insert("P".to_string());
let engine = parse_bytes(b"P=Widget\nP=Gadget\nP=Widget\n", plan);
assert_eq!(engine.num_rows(), 3);
if let ColumnBuilder::Dictionary { codes, offsets, .. } = engine.get_column("P").unwrap() {
let dict_len = offsets.len() - 1;
assert_eq!(dict_len, 2);
assert_eq!(
codes.iter().map(|o| o.copied()).collect::<Vec<_>>(),
vec![Some(0), Some(1), Some(0)]
);
} else {
panic!("expected Dictionary builder");
}
}
#[test]
fn test_ragged_late_chunk_column_debut() {
let e1 = parse_bytes(b"A=1 B=2\nA=3\n", ExecutionPlan::new());
let e2 = parse_bytes(b"B=4 C=5\n", ExecutionPlan::new());
let mut merged = TableBuilder::new();
merged.extend(e1).unwrap();
merged.extend(e2).unwrap();
assert_eq!(
merged.get_column("A").unwrap().as_str_vec(),
vec![Some("1".into()), Some("3".into()), None]
);
assert_eq!(
merged.get_column("B").unwrap().as_str_vec(),
vec![Some("2".into()), None, Some("4".into())]
);
assert_eq!(
merged.get_column("C").unwrap().as_str_vec(),
vec![None, None, Some("5".into())]
);
}
#[test]
fn test_auto_dict_upgrade_only_post_merge() {
let mut plan = ExecutionPlan::new();
plan.auto_dict = true;
let a = parse_bytes(b"P=x\nP=y\n", plan.clone());
let b = parse_bytes(b"P=x\nP=y\n", plan.clone());
let mut merged = TableBuilder::with_plan(64, Arc::new(plan));
merged.extend(a).unwrap();
merged.extend(b).unwrap();
merged.auto_dict_upgrade();
assert_eq!(merged.num_rows(), 4);
}
#[test]
fn test_extend_string_dictionary_promotes_not_panics() {
let mut e1 = parse_bytes(b"P=x\n", ExecutionPlan::new());
let mut plan = ExecutionPlan::new();
plan.dictionary_columns.insert("P".to_string());
let e2 = parse_bytes(b"P=x\n", plan);
e1.extend(e2).expect("String/Dictionary must reconcile");
assert!(matches!(
e1.get_column("P").unwrap(),
crate::columnar::ColumnBuilder::Dictionary { .. }
));
}
#[test]
fn test_extend_irreconcilable_variants_error_not_panic() {
let mut e1 = parse_bytes(b"S=x\n", ExecutionPlan::new());
let mut plan = ExecutionPlan::new();
plan.field_types.insert("S".to_string(), FieldType::Int64);
let e2 = parse_bytes(b"S=7\n", plan);
let result = e1.extend(e2);
match result {
Err(crate::Error::Merge(msg)) => {
assert!(msg.contains("'S'"), "error should name the column: {msg}");
}
other => panic!("String/Int64 mismatch must return Merge error, got {other:?}"),
}
}
#[test]
fn test_compare_filter_via_arrow_compute() {
use crate::arrow_export::apply_compare_filter;
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Compare {
field_a: "A".to_string(),
op: CompareOp::Gt,
field_b: "B".to_string(),
});
let mut engine = parse_bytes(b"A=3 B=1\nA=2 B=2\nA=5 B=4\n", plan);
let batch = engine.finish().unwrap();
let filtered = apply_compare_filter(
batch,
&FilterPredicate::Compare {
field_a: "A".to_string(),
op: CompareOp::Gt,
field_b: "B".to_string(),
},
)
.unwrap();
assert_eq!(filtered.num_rows(), 2);
let a = filtered.column_by_name("A").unwrap().as_string::<i32>();
assert_eq!(a.value(0), "3");
assert_eq!(a.value(1), "5");
}
#[test]
fn test_row_dirty_is_full_edge_cases() {
for ncols in [1, 63, 64, 65, 127, 128, 129] {
let mut tb = TableBuilder::new();
for i in 0..ncols {
let name = format!("col{i}");
tb.field_index.insert(name.clone(), i);
tb.columns.push(crate::columnar::ColumnBuilder::String(
crate::columnar::StrColumn::default(),
));
tb.column_order.push(name);
}
tb.row_dirty = vec![0; ncols.div_ceil(64)];
for i in 0..ncols {
let word = i / 64;
let bit = i % 64;
tb.row_dirty[word] |= 1u64 << bit;
}
let full_words = ncols / 64;
let rem = ncols % 64;
let is_full = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
&& (rem == 0
|| tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
assert!(
is_full,
"ncols={ncols} should be full, row_dirty={:?}",
tb.row_dirty
);
if ncols > 0 {
let word = (ncols - 1) / 64;
let bit = (ncols - 1) % 64;
tb.row_dirty[word] &= !(1u64 << bit);
let is_full2 = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
&& (rem == 0
|| tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
assert!(
!is_full2,
"ncols={ncols} should not be full after clearing one bit"
);
}
}
}
#[test]
fn test_finish_twice_does_not_panic() {
let mut tb = parse_bytes(b"A=1 B=2\nA=3 B=4\n", ExecutionPlan::new());
let batch1 = tb.finish().unwrap();
assert_eq!(batch1.num_rows(), 2);
let batch2 = tb.finish().unwrap();
assert_eq!(batch2.num_rows(), 0);
}
#[test]
fn test_c2_compound_reorder() {
let data = b"A=1 B=2\nA=2 B=2\nA=1 B=3\nA=2 B=3\n";
let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::And(
Box::new(FilterPredicate::Equal {
field: a.to_string(),
value: va.to_string(),
}),
Box::new(FilterPredicate::Equal {
field: b.to_string(),
value: vb.to_string(),
}),
));
plan
};
let r1 = parse_bytes(data, make_filter("A", "1", "B", "2"));
let r2 = parse_bytes(data, make_filter("B", "2", "A", "1"));
assert_eq!(
r1.num_rows(),
r2.num_rows(),
"C2 reorder must produce identical results"
);
assert_eq!(r1.num_rows(), 1); }
#[test]
fn test_c2_or_reorder() {
let data = b"A=1 B=2\nA=2 B=2\nA=3 B=3\n";
let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Or(
Box::new(FilterPredicate::Equal {
field: a.to_string(),
value: va.to_string(),
}),
Box::new(FilterPredicate::Equal {
field: b.to_string(),
value: vb.to_string(),
}),
));
plan
};
let r1 = parse_bytes(data, make_filter("A", "1", "B", "3"));
let r2 = parse_bytes(data, make_filter("B", "3", "A", "1"));
assert_eq!(
r1.num_rows(),
r2.num_rows(),
"Or reorder must produce identical results"
);
assert_eq!(r1.num_rows(), 2); }
#[test]
fn test_predicate_slot_created_after_mask_build() {
let data = b"A=1 B=ok\nA=2 B=ok\nA=3 B=ok\n";
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "B".to_string(),
value: "reject".to_string(),
});
let engine = parse_bytes(data, plan);
assert_eq!(
engine.num_rows(),
0,
"all rows should be rejected (B != reject)"
);
let data2 = b"A=1 B=ok\nA=2 B=reject\nA=3 B=ok\n";
let mut plan2 = ExecutionPlan::new();
plan2.filter = Some(FilterPredicate::Equal {
field: "B".to_string(),
value: "ok".to_string(),
});
let engine2 = parse_bytes(data2, plan2);
assert_eq!(engine2.num_rows(), 2, "rows 1 and 3 pass (B == ok)");
let col = engine2.get_column("A").unwrap();
assert_eq!(col.as_str_vec(), vec![Some("1".into()), Some("3".into())]);
}
#[test]
fn test_predicate_first_via_resolve_and_put() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "B".to_string(),
value: "reject".to_string(),
});
let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("1")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
tb.end_row();
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("2")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("reject")));
tb.end_row();
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("3")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
tb.end_row();
let batch = tb.finish().unwrap();
assert_eq!(batch.num_rows(), 1, "only row 2 (B==reject) should pass");
}
#[test]
fn test_owned_cow_values_survive_buffered_filter() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "B".to_string(),
value: "ok".to_string(),
});
let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
for i in 0..50 {
tb.begin_row();
let a = Cow::Owned(format!("value-{i}-unescaped"));
let b = Cow::Owned(if i % 2 == 0 {
"ok".to_string()
} else {
"no".to_string()
});
tb.resolve_and_put("A", Value::Str(a));
tb.resolve_and_put("B", Value::Str(b));
tb.end_row();
}
assert_eq!(tb.num_rows(), 25, "even rows pass (B == ok)");
let col = tb.get_column("A").unwrap();
let vals = col.as_str_vec();
assert_eq!(vals.len(), 25);
assert_eq!(vals[0], Some("value-0-unescaped".to_string()));
assert_eq!(vals[24], Some("value-48-unescaped".to_string()));
}
#[test]
fn test_predicate_first_skip_via_resolve_and_put() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "A".to_string(),
value: "999".to_string(),
});
let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
for _i in 0..100 {
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("0")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
tb.end_row();
}
let batch = tb.finish().unwrap();
assert_eq!(
batch.num_rows(),
0,
"all rows should be rejected (A=0 != 999)"
);
}
#[test]
fn test_sparse_column_predicate_skip() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "A".to_string(),
value: "reject".to_string(),
});
let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
for i in 0..100 {
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
if i >= 50 {
tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
}
tb.end_row();
}
let batch = tb.finish().unwrap();
assert_eq!(batch.num_rows(), 0, "all rows rejected (A=ok != reject)");
}
#[test]
fn test_sparse_column_predicate_pass() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "A".to_string(),
value: "ok".to_string(),
});
let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
for _i in 0..100 {
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
if _i >= 50 {
tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
}
tb.end_row();
}
let batch = tb.finish().unwrap();
assert_eq!(batch.num_rows(), 100, "all rows pass (A=ok == ok)");
if let Some(col) = batch.column_by_name("D") {
let arr = col.as_string::<i32>();
for i in 0..50 {
assert!(arr.is_null(i), "row {} should have D=null", i);
}
for i in 50..100 {
assert!(
!arr.is_null(i),
"row {} should not be null (D len={}, total rows={})",
i,
arr.len(),
batch.num_rows()
);
assert_eq!(arr.value(i), "z", "rows 50+ should have D=z");
}
} else {
panic!("D column should exist");
}
}
#[test]
fn test_sparse_late_predicate_ordinal() {
let mut plan = ExecutionPlan::new();
plan.filter = Some(FilterPredicate::Equal {
field: "E".to_string(),
value: "999".to_string(),
});
let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
for _i in 0..30 {
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
tb.end_row();
}
for _i in 30..100 {
tb.begin_row();
tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
tb.resolve_and_put("C", Value::Str(Cow::Borrowed("c")));
tb.resolve_and_put("D", Value::Str(Cow::Borrowed("d")));
tb.resolve_and_put("E", Value::Str(Cow::Borrowed("e")));
tb.end_row();
}
let batch = tb.finish().unwrap();
assert_eq!(batch.num_rows(), 0, "all rows rejected (E=\"e\" != 999)");
}
#[test]
fn raw_name_helpers_apply_rename_and_drop() {
use crate::decoder::ColumnarSink;
let plan = ExecutionPlan::new()
.rename("raw", "renamed")
.drop("ignored");
let mut tb = TableBuilder::with_plan(2, Arc::new(plan));
tb.begin_row();
tb.resolve_and_put_raw(b"raw", Value::Str(Cow::Borrowed("value")));
tb.resolve_and_put_raw(b"ignored", Value::Str(Cow::Borrowed("not stored")));
tb.put_row(&[("other", Value::Str(Cow::Borrowed("also stored")))]);
tb.end_row();
assert_eq!(tb.column_names(), &["renamed", "other"]);
assert_eq!(tb.resolve_raw(b"raw"), Some("renamed"));
}
struct SkippingParser;
impl RecordParser for SkippingParser {
fn validate(&self, bytes: &[u8]) -> Result<()> {
simdutf8::basic::from_utf8(bytes)?;
Ok(())
}
fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
for line in text.lines() {
if line.is_empty() {
continue;
}
sink.begin_row();
for token in line.split_whitespace() {
if sink.row_satisfied() {
break; }
if let Some((k, v)) = token.split_once('=') {
if sink.resolve(k).is_some() {
sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
}
}
}
sink.end_row();
}
Ok(())
}
}
fn parse_skipping(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
SkippingParser.parse_chunk(bytes, &mut sink).unwrap();
sink
}
fn batch_names(tb: &mut TableBuilder) -> Vec<String> {
tb.finish()
.unwrap()
.schema()
.fields()
.iter()
.map(|f| f.name().clone())
.collect()
}
const REPORT: &[u8] = b"Name=n1 Department=d1 Amount=1 Status=s1 Date=2024\n\
Name=n2 Department=d2 Amount=2 Status=s2 Date=2024\n";
#[test]
fn schema_projection_subset() {
let mut tb = parse_skipping(
REPORT,
ExecutionPlan::new().schema_order(["Name", "Department"]),
);
assert_eq!(batch_names(&mut tb), vec!["Name", "Department"]);
}
#[test]
fn schema_projection_single_non_first_column() {
let mut tb = parse_skipping(REPORT, ExecutionPlan::new().schema_order(["Department"]));
assert_eq!(batch_names(&mut tb), vec!["Department"]);
}
#[test]
fn schema_projection_full_natural_order() {
let mut tb = parse_skipping(
REPORT,
ExecutionPlan::new().schema_order(["Name", "Department", "Amount", "Status", "Date"]),
);
assert_eq!(
batch_names(&mut tb),
vec!["Name", "Department", "Amount", "Status", "Date"]
);
}
#[test]
fn schema_projection_reversed_order() {
let mut tb = parse_skipping(
REPORT,
ExecutionPlan::new().schema_order(["Date", "Status", "Amount", "Department", "Name"]),
);
assert_eq!(
batch_names(&mut tb),
vec!["Date", "Status", "Amount", "Department", "Name"]
);
}
#[test]
fn schema_projection_absent_column_null_filled() {
let mut tb = parse_skipping(
REPORT,
ExecutionPlan::new().schema_order(["Name", "Missing"]),
);
let batch = tb.finish().unwrap();
assert_eq!(batch.num_rows(), 2);
let missing = batch.column_by_name("Missing").unwrap().as_string::<i32>();
assert!(missing.is_null(0) && missing.is_null(1));
let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
assert_eq!(name.value(0), "n1");
}
#[test]
fn schema_projection_wants_rejects_unlisted_fields() {
let plan = ExecutionPlan::new().schema_order(["Name"]);
let tb = TableBuilder::with_plan(4, Arc::new(plan));
assert!(tb.wants("Name"));
assert!(!tb.wants("Department"));
assert_eq!(tb.resolve("Amount"), None);
}
#[test]
fn schema_projection_rename_then_match() {
let plan = ExecutionPlan::new()
.rename("Name", "Employee")
.schema_order(["Employee", "Amount"]);
let mut tb = parse_skipping(REPORT, plan);
assert_eq!(batch_names(&mut tb), vec!["Employee", "Amount"]);
}
#[test]
fn schema_projection_field_types_apply() {
let plan = ExecutionPlan::new()
.type_as("Amount", FieldType::Int64)
.schema_order(["Name", "Amount"]);
let mut tb = parse_skipping(REPORT, plan);
let batch = tb.finish().unwrap();
let amount = batch.column_by_name("Amount").unwrap();
assert!(matches!(
amount.data_type(),
arrow::datatypes::DataType::Int64
));
assert_eq!(
amount
.as_primitive::<arrow::datatypes::Int64Type>()
.value(1),
2
);
}
#[test]
fn schema_projection_filter_on_unlisted_field() {
let plan = ExecutionPlan::new()
.schema_order(["Name"])
.filter_eq("Status", "s2");
let mut tb = parse_skipping(REPORT, plan);
let batch = tb.finish().unwrap();
assert_eq!(batch.num_rows(), 1);
assert_eq!(batch.schema().fields().len(), 1);
let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
assert_eq!(name.value(0), "n2");
}
#[test]
fn schema_projection_drop_wins_over_schema() {
let plan = ExecutionPlan::new()
.schema_order(["Name", "Department"])
.drop("Department");
let mut tb = parse_skipping(REPORT, plan);
assert_eq!(batch_names(&mut tb), vec!["Name"]);
}
#[test]
fn schema_projection_no_parser_cooperation() {
let mut tb = parse_bytes(
REPORT,
ExecutionPlan::new().schema_order(["Department", "Name"]),
);
let batch = tb.finish().unwrap();
let names: Vec<String> = batch
.schema()
.fields()
.iter()
.map(|f| f.name().clone())
.collect();
assert_eq!(names, vec!["Department", "Name"]);
let dept = batch
.column_by_name("Department")
.unwrap()
.as_string::<i32>();
assert_eq!(dept.value(0), "d1");
}
}