Skip to main content

rypipe_core/engine/
table_builder.rs

1#[cfg(test)]
2use std::borrow::Cow;
3use std::sync::Arc;
4
5#[cfg(feature = "profile")]
6use std::sync::atomic::{AtomicUsize, Ordering};
7
8/// Diagnostic: counts resolve_and_put calls across all TableBuilder instances.
9#[cfg(feature = "profile")]
10pub static RESOLVE_AND_PUT_COUNT: AtomicUsize = AtomicUsize::new(0);
11/// Diagnostic: counts predicate evaluations across all TableBuilder instances.
12#[cfg(feature = "profile")]
13pub static PREDICATE_EVALUATIONS: AtomicUsize = AtomicUsize::new(0);
14/// Diagnostic: counts how many times evaluate_predicate_state returns Fail.
15#[cfg(feature = "profile")]
16pub static PREDICATE_FAILS: AtomicUsize = AtomicUsize::new(0);
17/// Diagnostic: counts how many times evaluate_predicate_state returns Undecided.
18#[cfg(feature = "profile")]
19pub static PREDICATE_UNDECIDED: AtomicUsize = AtomicUsize::new(0);
20/// Diagnostic: counts how many times is_predicate_slot returns true.
21#[cfg(feature = "profile")]
22pub static IS_PRED_TRUE: AtomicUsize = AtomicUsize::new(0);
23/// Diagnostic: counts how many times is_predicate_slot returns false.
24#[cfg(feature = "profile")]
25pub static IS_PRED_FALSE: AtomicUsize = AtomicUsize::new(0);
26
27use arrow::datatypes::{Field as ArrowField, Schema};
28use arrow::record_batch::RecordBatch;
29use rustc_hash::FxHashMap as HashMap;
30
31use smallvec::SmallVec;
32
33use crate::columnar::ColumnBuilder;
34use crate::decoder::ColumnarSink;
35use crate::engine::predicate::{PredicateState, RowBuffer};
36use crate::plan::{ExecutionPlan, FieldType, FilterPredicate};
37use crate::value::Value;
38use crate::Result;
39
40/// Generic columnar table builder.  Implements `ColumnarSink` so any decoder
41/// can feed it field/value events; at the end it produces an Arrow
42/// `RecordBatch`.
43pub struct TableBuilder {
44    pub(crate) columns: Vec<ColumnBuilder>,
45    pub(crate) field_index: HashMap<String, usize>,
46    pub(crate) column_order: Vec<String>,
47    pub(crate) row_count: usize,
48    pub(crate) estimated_rows: usize,
49    pub(crate) plan: Arc<ExecutionPlan>,
50    /// Dirty mask for the current row: bit `i` set iff column `i` received a
51    /// value in this row. `Vec<u64>` word array so >64 columns work (e.g.,
52    /// Crystal Reports exports with >64 fields). One compare `mask != full`
53    /// replaces `for col in 0..ncols` loop when every field is present (dense).
54    pub(crate) row_dirty: Vec<u64>,
55    /// Frozen schema for parallel streaming. When Some, any field not in the
56    /// schema is an unknown field and hard-errors on `finish()` (data loss
57    /// would otherwise be silent). Discovery samples 16×2 MiB for >128 MiB.
58    pub(crate) frozen: Option<std::sync::Arc<crate::schema::FrozenSchema>>,
59    pub(crate) unknown_error: Option<String>,
60    /// Heap-allocated row buffer for predicate-first deferred materialization.
61    /// `None` when `plan.filter` is `None`; unfiltered parses carry zero
62    /// overhead from predicate machinery.
63    pub(crate) row_buf: Option<Box<RowBuffer>>,
64    /// Cached projection mask for `wanted_mask()`/`row_satisfied()`: bit `i`
65    /// set iff column `i` is wanted by the schema projection (declared schema
66    /// columns plus any extra fields the row filter needs).  Computed once in
67    /// `with_plan` after schema columns are pre-declared, so it is complete
68    /// from the first row — unlike a mask derived from `field_index` at call
69    /// time, which only covers columns already seen in the data.  0 when no
70    /// schema projection is active (short-circuit disabled).
71    pub(crate) cached_wanted_mask: u64,
72    /// Per-ordinal layout expectation: after row 1, each ordinal maps to a
73    /// (slot_index, raw_name) pair.  The adapter can memcmp the raw bytes
74    /// in-place instead of running the full attribute scan → UTF-8 decode →
75    /// hash → lookup path.  `None` after `layout_broken` or before row 1.
76    pub(crate) ordinal_expect: Vec<Option<(u32, Vec<u8>)>>,
77    /// Current ordinal within the row (incremented per put_field call).
78    /// Used to populate `ordinal_expect` on the first row.
79    pub(crate) current_ordinal: u32,
80    /// First strict-types violation (column, value, row). Recorded during
81    /// puts (the `ColumnarSink` trait is infallible) and surfaced by
82    /// `finish()` as an error.
83    pub(crate) strict_error: Option<crate::Error>,
84    /// Rows committed since the last `finish()`/`reset()` (for observer
85    /// `on_chunk_finished`). Tracked unconditionally: two usize increments.
86    pub(crate) rows_accepted: usize,
87    /// Rows rejected by the filter since the last `finish()`/`reset()`.
88    pub(crate) rows_rejected: usize,
89}
90
91impl TableBuilder {
92    pub fn new() -> Self {
93        Self {
94            columns: Vec::new(),
95            field_index: HashMap::default(),
96            column_order: Vec::new(),
97            row_count: 0,
98            estimated_rows: 0,
99            plan: Arc::new(ExecutionPlan::new()),
100            row_dirty: Vec::new(),
101            frozen: None,
102            unknown_error: None,
103            row_buf: None,
104            cached_wanted_mask: 0,
105            ordinal_expect: Vec::new(),
106            current_ordinal: 0,
107            strict_error: None,
108            rows_accepted: 0,
109            rows_rejected: 0,
110        }
111    }
112
113    pub fn with_capacity(cap: usize) -> Self {
114        Self {
115            columns: Vec::new(),
116            field_index: HashMap::default(),
117            column_order: Vec::new(),
118            row_count: 0,
119            estimated_rows: cap,
120            plan: Arc::new(ExecutionPlan::new()),
121            row_dirty: Vec::new(),
122            frozen: None,
123            unknown_error: None,
124            row_buf: None,
125            cached_wanted_mask: 0,
126            ordinal_expect: Vec::new(),
127            current_ordinal: 0,
128            strict_error: None,
129            rows_accepted: 0,
130            rows_rejected: 0,
131        }
132    }
133
134    pub fn with_plan(cap: usize, plan: Arc<ExecutionPlan>) -> Self {
135        let mut b = Self {
136            columns: Vec::new(),
137            field_index: HashMap::default(),
138            column_order: Vec::new(),
139            row_count: 0,
140            estimated_rows: cap,
141            plan: Arc::clone(&plan),
142            row_dirty: Vec::new(),
143            frozen: None,
144            unknown_error: None,
145            row_buf: if plan.filter.is_some() {
146                Some(Box::new(RowBuffer {
147                    fields: SmallVec::new(),
148                    state: PredicateState::Undecided,
149                    predicate_mask: SmallVec::new(),
150                    direct: false,
151                    predicate_ordinal: None,
152                    buffer_worthwhile: true,
153                    pred_names: SmallVec::new(),
154                }))
155            } else {
156                None
157            },
158            cached_wanted_mask: 0,
159            ordinal_expect: Vec::new(),
160            current_ordinal: 0,
161            strict_error: None,
162            rows_accepted: 0,
163            rows_rejected: 0,
164        };
165        b.predeclare_schema_columns();
166        b.cached_wanted_mask = b.compute_wanted_mask();
167        b
168    }
169
170    /// Pre-create the declared schema columns (plus any extra columns the row
171    /// filter needs) when `schema_order` is a projection.  This makes the
172    /// output layout, per-row null-filling, and `cached_wanted_mask` stable
173    /// from the first row, regardless of which fields appear in the data or
174    /// in what order.  No-op when no schema projection is declared.
175    fn predeclare_schema_columns(&mut self) {
176        if self.plan.schema_order.is_empty() {
177            return;
178        }
179        let mut names: Vec<String> = Vec::new();
180        for name in &self.plan.schema_order {
181            // drop_fields wins over schema: a column listed in both is dropped
182            // (dropping is an explicit exclusion; silently null-filling it
183            // would resurrect a column the user asked to remove).
184            if !self.plan.drop_fields.contains(name) {
185                names.push(name.clone());
186            }
187        }
188        // Filter-referenced fields are materialized even when not in the
189        // schema so predicates evaluate correctly; finish() projects them out.
190        if let Some(ref filter) = self.plan.filter {
191            let mut pred_names: SmallVec<[String; 4]> = SmallVec::new();
192            Self::collect_predicate_field_names(filter, &self.plan, &mut pred_names);
193            names.extend(pred_names);
194        }
195        for name in names {
196            if self.field_index.contains_key(&name) || self.plan.drop_fields.contains(&name) {
197                continue;
198            }
199            let ty = self.plan.column_type(&name);
200            let col = ColumnBuilder::with_capacity(self.estimated_rows.max(64), &ty);
201            self.columns.push(col);
202            self.field_index
203                .insert(name.clone(), self.columns.len() - 1);
204            self.column_order.push(name);
205        }
206        let words = self.columns.len().div_ceil(64);
207        self.row_dirty.resize(words, 0);
208    }
209
210    /// Bitmask of wanted columns for the projection short-circuit.  All
211    /// wanted columns are pre-declared by `predeclare_schema_columns`, and
212    /// non-wanted fields are rejected by `plan.resolve_field`, so the set of
213    /// columns existing at construction time is exactly the wanted set.
214    /// Returns 0 (short-circuit disabled) when no projection is declared or
215    /// when more than 64 columns are wanted.
216    fn compute_wanted_mask(&self) -> u64 {
217        if self.plan.schema_order.is_empty() {
218            return 0;
219        }
220        let mut mask = 0u64;
221        for &idx in self.field_index.values() {
222            if idx >= 64 {
223                return 0;
224            }
225            mask |= 1u64 << idx;
226        }
227        mask
228    }
229
230    /// Pre-size all columns from a `FrozenSchema`.
231    ///
232    /// Called by parallel streaming workers so that every chunk's
233    /// `TableBuilder` has the full column set from construction,
234    /// regardless of which fields appear in that chunk.
235    pub fn ensure_schema(&mut self, schema: &crate::schema::FrozenSchema) -> Result<()> {
236        for (slot, name) in schema.column_names().iter().enumerate() {
237            let name_str: &str = name;
238            if self.field_index.contains_key(name_str) {
239                continue; // already present
240            }
241            let ty = schema.column_types()[slot].clone();
242            let col = ColumnBuilder::with_capacity(self.estimated_rows, &ty);
243            self.columns.push(col);
244            self.field_index
245                .insert(name_str.to_string(), self.columns.len() - 1);
246            self.column_order.push(name_str.to_string());
247        }
248        // Resize row_dirty bitmask to cover all columns.
249        let words = self.columns.len().div_ceil(64);
250        self.row_dirty.resize(words, 0);
251        // Freeze: any later field not in this schema is an unknown field and
252        // must hard-error (otherwise data loss would be silent on a default
253        // path). Sampling 16×2 MiB covers ~6% of a 533 MB file, so 1% Text21
254        // is expected ~280 hits, but 0.05% would be missed.
255        self.frozen = Some(std::sync::Arc::new(schema.clone()));
256        Ok(())
257    }
258
259    /// Lookup a column by resolved name.
260    pub(crate) fn get_column(&self, name: &str) -> Option<&ColumnBuilder> {
261        self.field_index.get(name).map(|&i| &self.columns[i])
262    }
263
264    /// Mutable lookup by resolved name.
265    pub(crate) fn get_column_mut(&mut self, name: &str) -> Option<&mut ColumnBuilder> {
266        if let Some(&i) = self.field_index.get(name) {
267            Some(&mut self.columns[i])
268        } else {
269            None
270        }
271    }
272
273    pub(crate) fn bytes_used(&self) -> usize {
274        self.columns.iter().map(|c| c.bytes_used()).sum::<usize>()
275            + self.column_order.iter().map(|s| s.len()).sum::<usize>()
276            + self.row_dirty.len() * 8
277    }
278
279    /// Split off the first `n` rows into a new `TableBuilder`.
280    ///
281    /// Leaves `n` rows in `self`'s remainder as `self - n`. Used for
282    /// 64KB streaming where a single file chunk may contain many more rows
283    /// than `rows_per_batch`.
284    pub(crate) fn split_off(&mut self, n: usize) -> Self {
285        assert!(n <= self.row_count, "split_off beyond row_count");
286        assert!(n > 0);
287        let mut other = Self {
288            columns: Vec::with_capacity(self.columns.len()),
289            field_index: self.field_index.clone(),
290            column_order: self.column_order.clone(),
291            row_count: n,
292            estimated_rows: n,
293            plan: Arc::clone(&self.plan),
294            row_dirty: vec![0; self.columns.len().div_ceil(64)],
295            frozen: self.frozen.clone(),
296            unknown_error: None,
297            cached_wanted_mask: self.cached_wanted_mask,
298            row_buf: if self.plan.filter.is_some() {
299                Some(Box::new(RowBuffer {
300                    fields: SmallVec::new(),
301                    state: PredicateState::Undecided,
302                    predicate_mask: self
303                        .row_buf
304                        .as_ref()
305                        .map_or_else(SmallVec::new, |b| b.predicate_mask.clone()),
306                    direct: false,
307                    predicate_ordinal: None,
308                    buffer_worthwhile: true,
309                    pred_names: self
310                        .row_buf
311                        .as_ref()
312                        .map_or_else(SmallVec::new, |b| b.pred_names.clone()),
313                }))
314            } else {
315                None
316            },
317            ordinal_expect: Vec::new(),
318            current_ordinal: 0,
319            strict_error: None,
320            // The split-off batch reports the rows accumulated since the
321            // previous split; the remainder starts a fresh delta.
322            rows_accepted: std::mem::take(&mut self.rows_accepted),
323            rows_rejected: std::mem::take(&mut self.rows_rejected),
324        };
325        for (idx, col) in self.columns.iter_mut().enumerate() {
326            let drain = col.split_off(n);
327            other.columns.push(drain);
328            // Remainder stays in self.columns[idx]
329            let _ = idx;
330        }
331        self.row_count -= n;
332        // row_dirty for self should be all false (no dirty in remainder yet)
333        self.row_dirty = vec![0; self.columns.len().div_ceil(64)];
334        // row_dirty for other is also false (just finished batch)
335        other
336    }
337
338    /// Remove and return a column by name, fixing the Vec index map.
339    /// Used by `merge::extend` to move builders out of the `other` table.
340    pub(crate) fn take_column(&mut self, name: &str) -> Option<ColumnBuilder> {
341        let idx = self.field_index.remove(name)?;
342        // Keep row_dirty in sync: Vec<u64> bitmask, need to handle bit removal
343        // For simplicity, just rebuild row_dirty as all zeros after column removal
344        // (row_dirty is per-row, not per-column persistent, so clearing is fine)
345        // Rebuild to correct length
346        let new_len = self.columns.len().div_ceil(64);
347        self.row_dirty.resize(new_len, 0);
348        // Clear all bits (no dirty in remainder for take_column case)
349        self.row_dirty.fill(0);
350        let _ = idx; // suppress unused warning
351                     // After removal, columns.len() == old_len; last index = old_len - 1.
352                     // swap_remove will move the last element into idx (if not already last).
353        let last = self.columns.len() - 1;
354        let col = if idx == last {
355            self.columns.pop().unwrap()
356        } else {
357            let col = self.columns.swap_remove(idx);
358            // The element that was at `last` is now at `idx`; fix its map entry.
359            let old_last = self.columns.len(); // == last, new len after pop/swap
360                                               // Find the key that pointed to old_last and repoint it to idx.
361                                               // We must not borrow field_index mutably while iterating, so clone the key first.
362            let moved_name = self.field_index.iter().find_map(|(k, &v)| {
363                if v == old_last {
364                    Some(k.clone())
365                } else {
366                    None
367                }
368            });
369            if let Some(k) = moved_name {
370                self.field_index.insert(k, idx);
371            }
372            col
373        };
374        Some(col)
375    }
376
377    pub fn num_rows(&self) -> usize {
378        self.row_count
379    }
380
381    pub fn num_columns(&self) -> usize {
382        self.column_order.len()
383    }
384
385    pub fn column_names(&self) -> &[String] {
386        &self.column_order
387    }
388
389    /// Diagnostic: (name, bytes_used, bytes_capacity) for each column.
390    pub fn column_diagnostics(&self) -> Vec<(String, usize, usize)> {
391        self.column_order
392            .iter()
393            .map(|name| {
394                let idx = self.field_index[name];
395                let col = &self.columns[idx];
396                (name.clone(), col.bytes_used(), col.capacity_bytes())
397            })
398            .collect()
399    }
400
401    /// The estimated_rows capacity hint passed to with_plan.
402    pub fn estimated_rows(&self) -> usize {
403        self.estimated_rows
404    }
405
406    /// Finalize the builder into an Arrow `RecordBatch`.
407    ///
408    /// This is also available as the `ColumnarSink::finish` trait method.
409    pub fn finish(&mut self) -> Result<RecordBatch> {
410        ColumnarSink::finish(self)
411    }
412
413    /// Reset all data while preserving the plan and estimated rows.
414    pub fn reset(&mut self) {
415        self.columns.clear();
416        self.field_index.clear();
417        self.column_order.clear();
418        self.row_dirty.fill(0);
419        self.row_count = 0;
420        if let Some(ref mut buf) = self.row_buf {
421            buf.fields.clear();
422            buf.state = PredicateState::Undecided;
423        }
424        // Keep frozen/row_buf (per builder, not per row)
425        self.unknown_error = None;
426        self.strict_error = None;
427        self.rows_accepted = 0;
428        self.rows_rejected = 0;
429    }
430
431    /// Truncate every column back to `row_count`, dropping any partial-row
432    /// values from a mid-field EOF.  Idempotent.
433    pub fn normalize(&mut self) {
434        for b in &mut self.columns {
435            while b.len() > self.row_count {
436                b.pop();
437            }
438        }
439        // Discard any dirty state for the partial row.
440        self.row_dirty.fill(0);
441    }
442
443    /// If `auto_dict` is set, upgrade low-cardinality string columns using the
444    /// plan's threshold/max-size tuning (defaults: 5% ratio, max size 256).
445    pub fn auto_dict_upgrade(&mut self) {
446        if self.plan.auto_dict {
447            let max_ratio = self.plan.dict_threshold.unwrap_or(0.05);
448            let max_size = self.plan.dict_max_size.unwrap_or(256);
449            for b in &mut self.columns {
450                b.try_upgrade_to_dict(512, max_ratio, max_size);
451            }
452        }
453    }
454
455    /// Sort columns according to `schema_order`.  Columns named in
456    /// `schema_order` appear in that order; any other columns keep their
457    /// relative first-appearance order after the ordered ones.
458    pub fn sort_columns(&mut self) {
459        if self.plan.schema_order.is_empty() {
460            return;
461        }
462        let order = &self.plan.schema_order;
463        let rank = |name: &String| order.iter().position(|n| n == name).unwrap_or(usize::MAX);
464        self.column_order.sort_by_key(rank);
465    }
466
467    pub(crate) fn schema_insert_index(&self, name: &str) -> usize {
468        let order = &self.plan.schema_order;
469        if order.is_empty() {
470            return self.column_order.len();
471        }
472        let pos = order.iter().position(|n| n == name);
473        match pos {
474            Some(p) => self
475                .column_order
476                .iter()
477                .position(|existing| {
478                    order
479                        .iter()
480                        .position(|n| n == existing)
481                        .is_some_and(|ep| ep > p)
482                })
483                .unwrap_or(self.column_order.len()),
484            None => self.column_order.len(),
485        }
486    }
487
488    /// Ensure the column exists and return its Vec index.
489    /// Single hash lookup for the hot path; new columns are created and
490    /// inserted into `column_order` via `schema_insert_index`.
491    fn ensure_column_idx(&mut self, name: &str) -> usize {
492        if let Some(&idx) = self.field_index.get(name) {
493            return idx;
494        }
495        // Frozen schema: any new column not pre-sized by ensure_schema is an
496        // unknown field (sampling miss or file has column absent during
497        // discovery). Don't silently create it; hard-error on finish().
498        // We still return a dummy idx to keep the hot path inlinable, but
499        // push_field_resolved will have already early-returned after setting
500        // unknown_error, so this path is only for non-frozen builders.
501        if self.frozen.is_some() {
502            if let Some(frozen) = &self.frozen {
503                if frozen.is_exact() {
504                    // Exact schema: error on unknown fields
505                    if self.unknown_error.is_none() {
506                        self.unknown_error = Some(format!(
507                            "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
508                            name,
509                            frozen.num_columns(),
510                            frozen.is_exact()
511                        ));
512                    }
513                    return 0;
514                }
515                // Non-exact (partial schema): fall through to create column dynamically
516            }
517        }
518        let est = self.estimated_rows.max(64);
519        let col_type = self.plan.column_type(name);
520        let mut b = ColumnBuilder::with_capacity(est, &col_type);
521        // When the predicate passes mid-row, drain_buffered increments
522        // row_count before all fields are pushed. New columns created after
523        // that must backfill row_count-1 Nones (for the already-committed
524        // rows), not row_count, because the current row's value follows.
525        let backfill = if self.row_buf.as_ref().is_some_and(|b| b.direct) {
526            self.row_count.saturating_sub(1)
527        } else {
528            self.row_count
529        };
530        for _ in 0..backfill {
531            b.push(None);
532        }
533        let idx = self.columns.len();
534        self.columns.push(b);
535        self.field_index.insert(name.to_owned(), idx);
536        // Ensure row_dirty has enough words for new column
537        let needed = self.columns.len().div_ceil(64);
538        if self.row_dirty.len() < needed {
539            self.row_dirty.resize(needed, 0);
540        }
541        let order_idx = self.schema_insert_index(name);
542        self.column_order.insert(order_idx, name.to_owned());
543        idx
544    }
545
546    /// Strict-types check for the value about to land in column `idx`: when
547    /// `plan.strict_types` is set, a non-null string that does not parse into
548    /// the declared `field_types` type is recorded as the first strict
549    /// violation (surfaced by `finish()`). Nulls and non-string values never
550    /// trigger it.
551    fn record_strict_violation_at(&mut self, idx: usize, value: &Value<'_>) {
552        if !self.plan.strict_types || self.strict_error.is_some() {
553            return;
554        }
555        let Value::Str(s) = value else {
556            return;
557        };
558        let Some(name) = self.column_order.get(idx) else {
559            return;
560        };
561        let Some(ft) = self.plan.field_types.get(name) else {
562            return;
563        };
564        if crate::columnar::parses_as(s, ft) {
565            return;
566        }
567        let truncated: String = s.chars().take(64).collect();
568        self.strict_error = Some(crate::Error::Plan(format!(
569            "strict_types: value {truncated:?} in column {name:?} at row {} \
570             does not parse as declared type {ft:?}",
571            self.row_count
572        )));
573    }
574
575    /// Notify the plan's observer (if any) that `value` landed in column
576    /// `idx`. Not called for buffered values; those fire on drain instead.
577    #[inline]
578    fn notify_put_field(&self, idx: usize, value: &Value<'_>) {
579        if let Some(ref obs) = self.plan.observer {
580            if let Some(name) = self.column_order.get(idx) {
581                obs.on_put_field(self.row_count, name, idx, value);
582            }
583        }
584    }
585
586    /// Push a field value without resolving renames/drops.
587    /// Caller must have already resolved `resolved_name` (or know it is kept).
588    #[inline]
589    fn push_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
590        // #[inline]: small hot path, called per-field. ensure_column_idx is not
591        // inlined (too large); push_value is the real work and benefits from
592        // being in the same compilation unit as its caller.
593        // Frozen check: hard error on unknown field when exact schema (data loss
594        // would be silent on a default path). Non-exact schemas allow unknown
595        // fields to be created dynamically.
596        if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
597            if let Some(frozen) = &self.frozen {
598                if frozen.is_exact() {
599                    // Exact schema: error on unknown fields
600                    if self.unknown_error.is_none() {
601                        self.unknown_error = Some(format!(
602                            "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...] with full column list or use full-scan discovery",
603                            resolved_name,
604                            frozen.num_columns(),
605                            frozen.is_exact()
606                        ));
607                    }
608                    return;
609                }
610                // Non-exact (partial schema): fall through to ensure_column_idx
611            }
612        }
613        let idx = self.ensure_column_idx(resolved_name);
614        self.record_strict_violation_at(idx, &value);
615        self.notify_put_field(idx, &value);
616        // Track ordinal→slot mapping on first row for expect_slot fast path.
617        if self.row_count == 0 {
618            let ord = self.current_ordinal as usize;
619            if ord >= self.ordinal_expect.len() {
620                self.ordinal_expect.resize_with(ord + 1, || None);
621            }
622            self.ordinal_expect[ord] = Some((idx as u32, resolved_name.as_bytes().to_vec()));
623        }
624        self.current_ordinal += 1;
625        let word = idx / 64;
626        let bit = idx % 64;
627        self.row_dirty[word] |= 1u64 << bit;
628        let b = &mut self.columns[idx];
629        let row_count = self.row_count;
630        if b.len() > row_count {
631            b.pop();
632        }
633        b.push_value(value);
634    }
635
636    /// Push a field value, resolving renames/drops and applying last-write-wins
637    /// within the current uncommitted row.
638    #[inline]
639    fn push_field(&mut self, name: &str, value: Value<'_>) {
640        // #[inline]: fast-path (no rename/drop) is a single branch + delegate.
641        // Fast path: no rename/drop/projection configured; zero allocation.
642        if self.plan.field_map.is_empty()
643            && self.plan.drop_fields.is_empty()
644            && self.plan.schema_order.is_empty()
645        {
646            self.push_field_resolved(name, value);
647            return;
648        }
649        // Resolve name via plan's field_map (borrows &self.plan).
650        // Try the zero-allocation fast path: column already exists.
651        if let Some(idx) = Self::resolve_and_slot(&self.plan, &self.field_index, name) {
652            self.record_strict_violation_at(idx, &value);
653            self.notify_put_field(idx, &value);
654            // Track ordinal→slot mapping on first row for expect_slot fast path.
655            if self.row_count == 0 {
656                let ord = self.current_ordinal as usize;
657                if ord >= self.ordinal_expect.len() {
658                    self.ordinal_expect.resize_with(ord + 1, || None);
659                }
660                // Store the resolved name (what plan.resolve_field would return).
661                let resolved = self.plan.resolve_field(name).unwrap_or(name);
662                self.ordinal_expect[ord] = Some((idx as u32, resolved.as_bytes().to_vec()));
663            }
664            self.current_ordinal += 1;
665            let word = idx / 64;
666            let bit = idx % 64;
667            self.row_dirty[word] |= 1u64 << bit;
668            let b = &mut self.columns[idx];
669            let row_count = self.row_count;
670            if b.len() > row_count {
671                b.pop();
672            }
673            b.push_value(value);
674        } else {
675            // Column doesn't exist yet (first row with rename) or field was
676            // dropped. Only allocate when the field is kept.
677            if let Some(resolved) = self.plan.resolve_field(name) {
678                let owned = resolved.to_owned();
679                self.push_field_resolved(&owned, value);
680            }
681        }
682    }
683
684    /// Resolve a field name through the plan's field_map and look up its slot
685    /// index. Returns `Some(slot)` only if the column already exists in
686    /// field_index. Zero allocation; borrows plan and field_index as
687    /// separate fields.
688    #[inline]
689    fn resolve_and_slot(
690        plan: &ExecutionPlan,
691        field_index: &HashMap<String, usize>,
692        name: &str,
693    ) -> Option<usize> {
694        let resolved = plan.resolve_field(name)?;
695        field_index.get(resolved).copied()
696    }
697
698    /// Advance the row counter without null-fill, filter, or dirty-mask clear.
699    /// For benchmarking only; separates per-field push cost from per-row
700    /// finalization.
701    #[doc(hidden)]
702    pub fn advance_row(&mut self) {
703        self.row_count += 1;
704    }
705
706    /// Null-fill any column missing this row, then apply the per-row filter.
707    /// If the filter rejects the row, undo it by popping values.
708    /// Uses the dirty bitmask so only missing columns are touched; fast path
709    /// when every column was set (dense data, 10 cols) skips the loop.
710    fn finish_row(&mut self) {
711        // No #[inline]: 30+ lines with loops; inlining causes code bloat.
712        let ncols = self.columns.len();
713        // Fast path: check if all bits set
714        let full_words = ncols / 64;
715        let rem_bits = ncols % 64;
716        let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
717            && (rem_bits == 0
718                || self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
719        if is_full {
720            self.row_dirty.fill(0);
721        } else {
722            for (i, b) in self.columns.iter_mut().enumerate() {
723                let word = i / 64;
724                let bit = i % 64;
725                let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
726                if !is_set {
727                    b.push(None);
728                }
729            }
730            self.row_dirty.fill(0);
731        }
732
733        if let Some(ref filter) = self.plan.filter {
734            if !filter.check(&self.columns, &self.field_index, self.row_count, &self.plan) {
735                for b in &mut self.columns {
736                    b.pop();
737                }
738                self.rows_rejected += 1;
739                if let Some(ref obs) = self.plan.observer {
740                    obs.on_row_rejected(self.row_count);
741                }
742                return;
743            }
744        }
745
746        self.row_count += 1;
747        self.rows_accepted += 1;
748        if let Some(ref obs) = self.plan.observer {
749            obs.on_row_accepted(self.row_count - 1);
750        }
751    }
752
753    // Predicate-first helpers
754    /// Build the predicate bitmask from field_index. Called lazily on first
755    /// `is_predicate_slot` call when the mask is still empty.
756    fn build_predicate_mask(&mut self) {
757        let buf = match self.row_buf {
758            Some(ref mut b) => b,
759            None => return,
760        };
761        if !buf.pred_names.is_empty() {
762            return; // already built (pred_names populated)
763        }
764        // Collect predicate field names (one-time clone of the filter).
765        let mut names: SmallVec<[String; 4]> = SmallVec::new();
766        if let Some(ref f) = self.plan.filter.clone() {
767            Self::collect_predicate_field_names(f, &self.plan, &mut names);
768        }
769        // Cache the names so future calls can check without cloning.
770        buf.pred_names.clone_from(&names);
771        // Grow mask to cover all columns.
772        let ncols = self.columns.len();
773        let words = ncols.div_ceil(64);
774        buf.predicate_mask.resize(words, 0);
775        // Mark slots for predicate fields that already exist.
776        for name in &names {
777            if let Some(&slot) = self.field_index.get(name.as_str()) {
778                let word = slot / 64;
779                let bit = slot % 64;
780                if word < buf.predicate_mask.len() {
781                    buf.predicate_mask[word] |= 1u64 << bit;
782                }
783            }
784        }
785    }
786
787    /// Mark a slot as a predicate column by checking cached predicate names.
788    /// Called once per new column to handle fields created after the initial
789    /// mask build (e.g., the predicate column appears in the data after
790    /// `build_predicate_mask` already ran for earlier columns).
791    #[inline]
792    fn mark_predicate_slot(&mut self, slot: u32) {
793        if self.plan.filter.is_none() {
794            return;
795        }
796        // Phase 1: check fast path (already marked) via immutable borrow,
797        // then check if pred_names are populated.  Avoid holding &mut buf
798        // across build_predicate_mask (which needs &mut self).
799        let needs_build = self.row_buf.as_ref().is_some_and(|buf| {
800            let word = slot as usize / 64;
801            let bit = slot as usize % 64;
802            if word < buf.predicate_mask.len() && (buf.predicate_mask[word] >> bit) & 1 == 1 {
803                return false; // already marked; no-op
804            }
805            buf.pred_names.is_empty() // need to build the mask
806        });
807        if needs_build {
808            self.build_predicate_mask();
809        }
810        // Phase 2: check if this slot is now marked after build.
811        if let Some(ref buf) = self.row_buf {
812            let word = slot as usize / 64;
813            let bit = slot as usize % 64;
814            if word < buf.predicate_mask.len() && (buf.predicate_mask[word] >> bit) & 1 == 1 {
815                return; // marked during build or was already marked
816            }
817        }
818        // Phase 3: mask is built but this slot isn't in it. Check if the
819        // column name matches any cached predicate name (no filter clone).
820        if let Some(col_name) = self.column_order.get(slot as usize) {
821            let is_pred = self
822                .row_buf
823                .as_ref()
824                .is_some_and(|buf| buf.pred_names.iter().any(|n| n == col_name));
825            if is_pred {
826                let buf = self.row_buf.as_mut().unwrap();
827                let word = slot as usize / 64;
828                let bit = slot as usize % 64;
829                if word >= buf.predicate_mask.len() {
830                    buf.predicate_mask.resize(word + 1, 0);
831                }
832                buf.predicate_mask[word] |= 1u64 << bit;
833            }
834        }
835    }
836
837    fn collect_predicate_field_names(
838        pred: &FilterPredicate,
839        plan: &ExecutionPlan,
840        names: &mut SmallVec<[String; 4]>,
841    ) {
842        match pred {
843            FilterPredicate::Equal { field, .. }
844            | FilterPredicate::NotEqual { field, .. }
845            | FilterPredicate::CompareLiteral { field, .. }
846            | FilterPredicate::StartsWith { field, .. }
847            | FilterPredicate::EndsWith { field, .. }
848            | FilterPredicate::In { field, .. }
849            | FilterPredicate::NotIn { field, .. }
850            | FilterPredicate::NotField { field, .. }
851            | FilterPredicate::ArithmeticCompare { field, .. }
852            | FilterPredicate::Strip { field, .. }
853            | FilterPredicate::Lower { field, .. }
854            | FilterPredicate::Upper { field, .. }
855            | FilterPredicate::Replace { field, .. }
856            | FilterPredicate::Length { field, .. }
857            | FilterPredicate::Contains { field, .. }
858            | FilterPredicate::IsNull { field, .. }
859            | FilterPredicate::IsType { field, .. }
860            | FilterPredicate::Regex { field, .. } => {
861                let resolved = plan.resolve_field(field).unwrap_or(field);
862                names.push(resolved.to_string());
863            }
864            FilterPredicate::Always(_) => {}
865            FilterPredicate::Compare {
866                field_a, field_b, ..
867            } => {
868                for f in [field_a, field_b] {
869                    let resolved = plan.resolve_field(f).unwrap_or(f);
870                    names.push(resolved.to_string());
871                }
872            }
873            FilterPredicate::And(a, b) | FilterPredicate::Or(a, b) => {
874                Self::collect_predicate_field_names(a, plan, names);
875                Self::collect_predicate_field_names(b, plan, names);
876            }
877            FilterPredicate::Not(inner) => Self::collect_predicate_field_names(inner, plan, names),
878        }
879    }
880
881    #[inline]
882    fn is_predicate_slot(&self, slot: u32) -> bool {
883        self.row_buf.as_ref().is_some_and(|b| {
884            let word = slot as usize / 64;
885            let bit = slot as usize % 64;
886            word < b.predicate_mask.len() && (b.predicate_mask[word] >> bit) & 1 == 1
887        })
888    }
889
890    fn get_buffered_value(&self, field: &str) -> Option<&Value<'static>> {
891        let buf = self.row_buf.as_ref()?;
892        let resolved = self.plan.resolve_field(field)?;
893        let slot = *self.field_index.get(resolved)? as u32;
894        for (s, v) in buf.fields.iter().rev() {
895            if *s == slot {
896                return Some(v);
897            }
898        }
899        None
900    }
901
902    fn get_buffered_str(&self, field: &str) -> Option<String> {
903        match self.get_buffered_value(field) {
904            Some(Value::Str(s)) => Some(s.to_string()),
905            Some(Value::Int64(i)) => Some(i.to_string()),
906            Some(Value::Float64(f)) => Some(f.to_string()),
907            Some(Value::Bool(b)) => Some(b.to_string()),
908            Some(v) => Some(format!("{:?}", v)),
909            None => None,
910        }
911    }
912
913    fn evaluate_predicate_state(&self) -> PredicateState {
914        let Some(ref pred) = self.plan.filter else {
915            return PredicateState::Pass;
916        };
917        #[cfg(feature = "profile")]
918        {
919            crate::engine::PREDICATE_EVALUATIONS.fetch_add(1, Ordering::Relaxed);
920        }
921        let result = Self::eval_predicate(pred, self);
922        #[cfg(feature = "profile")]
923        match result {
924            PredicateState::Fail => {
925                crate::engine::PREDICATE_FAILS.fetch_add(1, Ordering::Relaxed);
926            }
927            PredicateState::Undecided => {
928                crate::engine::PREDICATE_UNDECIDED.fetch_add(1, Ordering::Relaxed);
929            }
930            _ => {}
931        }
932        result
933    }
934
935    fn eval_predicate(pred: &FilterPredicate, tb: &TableBuilder) -> PredicateState {
936        match pred {
937            FilterPredicate::Equal { field, value } => match tb.get_buffered_str(field) {
938                Some(actual) => {
939                    if actual == *value {
940                        PredicateState::Pass
941                    } else {
942                        PredicateState::Fail
943                    }
944                }
945                None => PredicateState::Undecided,
946            },
947            FilterPredicate::NotEqual { field, value } => match tb.get_buffered_str(field) {
948                Some(actual) => {
949                    if actual != *value {
950                        PredicateState::Pass
951                    } else {
952                        PredicateState::Fail
953                    }
954                }
955                None => PredicateState::Undecided,
956            },
957            FilterPredicate::Compare {
958                field_a,
959                op,
960                field_b,
961            } => {
962                let normalize = |field: &str, value: &Value<'static>| {
963                    let field = tb.plan.resolve_field(field).unwrap_or(field);
964                    match tb.plan.column_type(field) {
965                        FieldType::Int64 => match value.as_str() {
966                            Some(s) => lexical::parse::<i64, _>(s.as_bytes())
967                                .ok()
968                                .map(Value::Int64),
969                            _ => Some(value.clone()),
970                        },
971                        FieldType::Float64 => match value.as_str() {
972                            Some(s) => lexical::parse::<f64, _>(s.as_bytes())
973                                .ok()
974                                .map(Value::Float64),
975                            _ => Some(value.clone()),
976                        },
977                        FieldType::Boolean => match value.as_str() {
978                            Some(s) => s.parse::<bool>().ok().map(Value::Bool),
979                            _ => Some(value.clone()),
980                        },
981                        FieldType::Date32 => match value.as_str() {
982                            Some(s) => crate::columnar::parse_date32(s).map(Value::Date32),
983                            _ => Some(value.clone()),
984                        },
985                        FieldType::Timestamp(unit, fmt) => match value.as_str() {
986                            Some(s) => crate::columnar::parse_timestamp(s, unit, fmt.as_deref())
987                                .map(Value::Timestamp),
988                            _ => Some(value.clone()),
989                        },
990                        FieldType::String | FieldType::Dictionary => Some(value.clone()),
991                        FieldType::Decimal128(_) => value.as_str().and_then(|s| {
992                            s.trim()
993                                .parse::<i128>()
994                                .ok()
995                                .map(|n| Value::Int64(n as i64))
996                        }),
997                    }
998                };
999                let va = tb
1000                    .get_buffered_value(field_a)
1001                    .and_then(|value| normalize(field_a, value));
1002                let vb = tb
1003                    .get_buffered_value(field_b)
1004                    .and_then(|value| normalize(field_b, value));
1005                match (va, vb) {
1006                    (Some(ref a), Some(ref b)) => {
1007                        let ord = match (a, b) {
1008                            (crate::value::Value::Int64(ai), crate::value::Value::Int64(bi)) => {
1009                                Some(ai.cmp(bi))
1010                            }
1011                            (
1012                                crate::value::Value::Float64(af),
1013                                crate::value::Value::Float64(bf),
1014                            ) => af.partial_cmp(bf),
1015                            (crate::value::Value::Int64(ai), crate::value::Value::Float64(bf)) => {
1016                                (*ai as f64).partial_cmp(bf)
1017                            }
1018                            (crate::value::Value::Float64(af), crate::value::Value::Int64(bi)) => {
1019                                af.partial_cmp(&(*bi as f64))
1020                            }
1021                            (crate::value::Value::Str(a), crate::value::Value::Str(b)) => {
1022                                let resolved_a =
1023                                    tb.plan.resolve_field(field_a).unwrap_or(field_a.as_str());
1024                                let resolved_b =
1025                                    tb.plan.resolve_field(field_b).unwrap_or(field_b.as_str());
1026                                let type_a = tb.plan.field_types.get(resolved_a);
1027                                let type_b = tb.plan.field_types.get(resolved_b);
1028                                match (type_a, type_b) {
1029                                    (
1030                                        Some(crate::plan::FieldType::Int64),
1031                                        Some(crate::plan::FieldType::Int64),
1032                                    ) => {
1033                                        let ai: i64 =
1034                                            lexical::parse(a.as_bytes()).ok().unwrap_or(0);
1035                                        let bi: i64 =
1036                                            lexical::parse(b.as_bytes()).ok().unwrap_or(0);
1037                                        Some(ai.cmp(&bi))
1038                                    }
1039                                    (
1040                                        Some(crate::plan::FieldType::Float64),
1041                                        Some(crate::plan::FieldType::Float64),
1042                                    ) => {
1043                                        let af: f64 =
1044                                            lexical::parse(a.as_bytes()).ok().unwrap_or(0.0);
1045                                        let bf: f64 =
1046                                            lexical::parse(b.as_bytes()).ok().unwrap_or(0.0);
1047                                        af.partial_cmp(&bf)
1048                                    }
1049                                    (
1050                                        Some(crate::plan::FieldType::Int64),
1051                                        Some(crate::plan::FieldType::Float64),
1052                                    ) => {
1053                                        let ai: f64 = lexical::parse::<i64, _>(a.as_bytes())
1054                                            .ok()
1055                                            .unwrap_or(0)
1056                                            as f64;
1057                                        let bf: f64 =
1058                                            lexical::parse(b.as_bytes()).ok().unwrap_or(0.0);
1059                                        ai.partial_cmp(&bf)
1060                                    }
1061                                    (
1062                                        Some(crate::plan::FieldType::Float64),
1063                                        Some(crate::plan::FieldType::Int64),
1064                                    ) => {
1065                                        let af: f64 =
1066                                            lexical::parse(a.as_bytes()).ok().unwrap_or(0.0);
1067                                        let bi: f64 = lexical::parse::<i64, _>(b.as_bytes())
1068                                            .ok()
1069                                            .unwrap_or(0)
1070                                            as f64;
1071                                        af.partial_cmp(&bi)
1072                                    }
1073                                    // Mixed typed/untyped or String vs non-numeric type:
1074                                    // type mismatch; fail the comparison.
1075                                    (Some(_), None) | (None, Some(_)) => None,
1076                                    // Both untyped (String): fall back to lexicographic.
1077                                    (None, None) => Some(a.cmp(b)),
1078                                    // Both Timestamp: parse and compare as i64.
1079                                    (
1080                                        Some(crate::plan::FieldType::Timestamp(ua, fa)),
1081                                        Some(crate::plan::FieldType::Timestamp(ub, fb)),
1082                                    ) => {
1083                                        let ta = crate::columnar::parse_timestamp(
1084                                            a.as_ref(),
1085                                            *ua,
1086                                            fa.as_deref(),
1087                                        );
1088                                        let tb_val = crate::columnar::parse_timestamp(
1089                                            b.as_ref(),
1090                                            *ub,
1091                                            fb.as_deref(),
1092                                        );
1093                                        match (ta, tb_val) {
1094                                            (Some(ai), Some(bi)) => Some(ai.cmp(&bi)),
1095                                            _ => None,
1096                                        }
1097                                    }
1098                                    // Both Date32: parse and compare as i32.
1099                                    (
1100                                        Some(crate::plan::FieldType::Date32),
1101                                        Some(crate::plan::FieldType::Date32),
1102                                    ) => {
1103                                        let da = crate::columnar::parse_date32(a.as_ref());
1104                                        let db = crate::columnar::parse_date32(b.as_ref());
1105                                        match (da, db) {
1106                                            (Some(ai), Some(bi)) => Some(ai.cmp(&bi)),
1107                                            _ => None,
1108                                        }
1109                                    }
1110                                    // Different non-numeric types (e.g. String vs Bool): fail.
1111                                    _ => None,
1112                                }
1113                            }
1114                            (crate::value::Value::Bool(a), crate::value::Value::Bool(b)) => {
1115                                Some(a.cmp(b))
1116                            }
1117                            _ => {
1118                                let av = format!("{:?}", a);
1119                                let bv = format!("{:?}", b);
1120                                Some(av.cmp(&bv))
1121                            }
1122                        };
1123                        let pass = ord.is_some_and(|ord| match op {
1124                            crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1125                            crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1126                            crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1127                            crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1128                            crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1129                            crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1130                        });
1131                        if pass {
1132                            PredicateState::Pass
1133                        } else {
1134                            PredicateState::Fail
1135                        }
1136                    }
1137                    _ => PredicateState::Undecided,
1138                }
1139            }
1140            FilterPredicate::CompareLiteral { field, op, value } => {
1141                let va = tb.get_buffered_value(field).and_then(|v| {
1142                    let resolved = tb.plan.resolve_field(field).unwrap_or(field);
1143                    match tb.plan.column_type(resolved) {
1144                        crate::plan::FieldType::Int64 => match v.as_str() {
1145                            Some(s) => lexical::parse::<i64, _>(s.as_bytes())
1146                                .ok()
1147                                .map(crate::value::Value::Int64),
1148                            _ => Some(v.clone()),
1149                        },
1150                        crate::plan::FieldType::Float64 => match v.as_str() {
1151                            Some(s) => lexical::parse::<f64, _>(s.as_bytes())
1152                                .ok()
1153                                .map(crate::value::Value::Float64),
1154                            _ => Some(v.clone()),
1155                        },
1156                        crate::plan::FieldType::Boolean => match v.as_str() {
1157                            Some(s) => s.parse::<bool>().ok().map(crate::value::Value::Bool),
1158                            _ => Some(v.clone()),
1159                        },
1160                        _ => Some(v.clone()),
1161                    }
1162                });
1163                let vb = match tb
1164                    .plan
1165                    .column_type(tb.plan.resolve_field(field).unwrap_or(field))
1166                {
1167                    crate::plan::FieldType::Int64 => lexical::parse::<i64, _>(value.as_bytes())
1168                        .ok()
1169                        .map(crate::value::Value::Int64),
1170                    crate::plan::FieldType::Float64 => lexical::parse::<f64, _>(value.as_bytes())
1171                        .ok()
1172                        .map(crate::value::Value::Float64),
1173                    crate::plan::FieldType::Boolean => {
1174                        value.parse::<bool>().ok().map(crate::value::Value::Bool)
1175                    }
1176                    _ => Some(crate::value::Value::Str(std::borrow::Cow::Borrowed(value))),
1177                };
1178                match (va, vb) {
1179                    (Some(ref a), Some(ref b)) => {
1180                        let ord = match (a, b) {
1181                            (crate::value::Value::Int64(ai), crate::value::Value::Int64(bi)) => {
1182                                Some(ai.cmp(bi))
1183                            }
1184                            (
1185                                crate::value::Value::Float64(af),
1186                                crate::value::Value::Float64(bf),
1187                            ) => af.partial_cmp(bf),
1188                            (crate::value::Value::Int64(ai), crate::value::Value::Float64(bf)) => {
1189                                (*ai as f64).partial_cmp(bf)
1190                            }
1191                            (crate::value::Value::Float64(af), crate::value::Value::Int64(bi)) => {
1192                                af.partial_cmp(&(*bi as f64))
1193                            }
1194                            (crate::value::Value::Str(a), crate::value::Value::Str(b)) => {
1195                                Some(a.cmp(b))
1196                            }
1197                            (crate::value::Value::Bool(a), crate::value::Value::Bool(b)) => {
1198                                Some(a.cmp(b))
1199                            }
1200                            _ => None,
1201                        };
1202                        let pass = ord.is_some_and(|ord| match op {
1203                            crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1204                            crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1205                            crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1206                            crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1207                            crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1208                            crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1209                        });
1210                        if pass {
1211                            PredicateState::Pass
1212                        } else {
1213                            PredicateState::Fail
1214                        }
1215                    }
1216                    _ => PredicateState::Undecided,
1217                }
1218            }
1219            FilterPredicate::StartsWith { field, value } => match tb.get_buffered_str(field) {
1220                Some(actual) => {
1221                    if actual.starts_with(value.as_str()) {
1222                        PredicateState::Pass
1223                    } else {
1224                        PredicateState::Fail
1225                    }
1226                }
1227                None => PredicateState::Undecided,
1228            },
1229            FilterPredicate::EndsWith { field, value } => match tb.get_buffered_str(field) {
1230                Some(actual) => {
1231                    if actual.ends_with(value.as_str()) {
1232                        PredicateState::Pass
1233                    } else {
1234                        PredicateState::Fail
1235                    }
1236                }
1237                None => PredicateState::Undecided,
1238            },
1239            FilterPredicate::In { field, values } => match tb.get_buffered_str(field) {
1240                Some(actual) => {
1241                    if values.contains(&actual) {
1242                        PredicateState::Pass
1243                    } else {
1244                        PredicateState::Fail
1245                    }
1246                }
1247                None => PredicateState::Fail,
1248            },
1249            FilterPredicate::NotIn { field, values } => match tb.get_buffered_str(field) {
1250                Some(actual) => {
1251                    if !values.contains(&actual) {
1252                        PredicateState::Pass
1253                    } else {
1254                        PredicateState::Fail
1255                    }
1256                }
1257                None => PredicateState::Pass,
1258            },
1259            FilterPredicate::Always(keep) => {
1260                if *keep {
1261                    PredicateState::Pass
1262                } else {
1263                    PredicateState::Fail
1264                }
1265            }
1266            FilterPredicate::NotField { field } => match tb.get_buffered_str(field) {
1267                Some(actual) => {
1268                    if actual.is_empty() {
1269                        PredicateState::Pass
1270                    } else {
1271                        PredicateState::Fail
1272                    }
1273                }
1274                None => PredicateState::Pass,
1275            },
1276            FilterPredicate::ArithmeticCompare {
1277                field,
1278                arith_op,
1279                arith_value,
1280                cmp_op,
1281                cmp_value,
1282            } => match tb.get_buffered_str(field) {
1283                Some(actual) => {
1284                    let field_f64 = actual.parse::<f64>().unwrap_or(0.0);
1285                    let result = match arith_op {
1286                        crate::plan::ArithOp::Add => field_f64 + arith_value,
1287                        crate::plan::ArithOp::Sub => field_f64 - arith_value,
1288                        crate::plan::ArithOp::Mul => field_f64 * arith_value,
1289                        crate::plan::ArithOp::Div => field_f64 / arith_value,
1290                    };
1291                    let cmp_f64 = cmp_value.parse::<f64>().unwrap_or(0.0);
1292                    match result.partial_cmp(&cmp_f64) {
1293                        Some(ord) => {
1294                            let pass = match cmp_op {
1295                                crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1296                                crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1297                                crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1298                                crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1299                                crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1300                                crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1301                            };
1302                            if pass {
1303                                PredicateState::Pass
1304                            } else {
1305                                PredicateState::Fail
1306                            }
1307                        }
1308                        None => PredicateState::Fail,
1309                    }
1310                }
1311                None => PredicateState::Undecided,
1312            },
1313            FilterPredicate::Strip { field, op, value } => match tb.get_buffered_str(field) {
1314                Some(actual) => {
1315                    let transformed = actual.trim().to_string();
1316                    match transformed.as_str().partial_cmp(value.as_str()) {
1317                        Some(ord) => {
1318                            let pass = match op {
1319                                crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1320                                crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1321                                crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1322                                crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1323                                crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1324                                crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1325                            };
1326                            if pass {
1327                                PredicateState::Pass
1328                            } else {
1329                                PredicateState::Fail
1330                            }
1331                        }
1332                        None => PredicateState::Fail,
1333                    }
1334                }
1335                None => PredicateState::Undecided,
1336            },
1337            FilterPredicate::Lower { field, op, value } => match tb.get_buffered_str(field) {
1338                Some(actual) => {
1339                    let transformed = actual.to_lowercase();
1340                    match transformed.as_str().partial_cmp(value.as_str()) {
1341                        Some(ord) => {
1342                            let pass = match op {
1343                                crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1344                                crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1345                                crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1346                                crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1347                                crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1348                                crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1349                            };
1350                            if pass {
1351                                PredicateState::Pass
1352                            } else {
1353                                PredicateState::Fail
1354                            }
1355                        }
1356                        None => PredicateState::Fail,
1357                    }
1358                }
1359                None => PredicateState::Undecided,
1360            },
1361            FilterPredicate::Upper { field, op, value } => match tb.get_buffered_str(field) {
1362                Some(actual) => {
1363                    let transformed = actual.to_uppercase();
1364                    match transformed.as_str().partial_cmp(value.as_str()) {
1365                        Some(ord) => {
1366                            let pass = match op {
1367                                crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1368                                crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1369                                crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1370                                crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1371                                crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1372                                crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1373                            };
1374                            if pass {
1375                                PredicateState::Pass
1376                            } else {
1377                                PredicateState::Fail
1378                            }
1379                        }
1380                        None => PredicateState::Fail,
1381                    }
1382                }
1383                None => PredicateState::Undecided,
1384            },
1385            FilterPredicate::Replace {
1386                field,
1387                old,
1388                new,
1389                op,
1390                value,
1391            } => match tb.get_buffered_str(field) {
1392                Some(actual) => {
1393                    let transformed = actual.replace(old.as_str(), new.as_str());
1394                    match transformed.as_str().partial_cmp(value.as_str()) {
1395                        Some(ord) => {
1396                            let pass = match op {
1397                                crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1398                                crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1399                                crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1400                                crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1401                                crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1402                                crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1403                            };
1404                            if pass {
1405                                PredicateState::Pass
1406                            } else {
1407                                PredicateState::Fail
1408                            }
1409                        }
1410                        None => PredicateState::Fail,
1411                    }
1412                }
1413                None => PredicateState::Undecided,
1414            },
1415            FilterPredicate::Length { field, op, value } => match tb.get_buffered_str(field) {
1416                Some(actual) => {
1417                    let len = actual.len() as f64;
1418                    let cmp_val = value.parse::<f64>().unwrap_or(0.0);
1419                    match len.partial_cmp(&cmp_val) {
1420                        Some(ord) => {
1421                            let pass = match op {
1422                                crate::plan::CompareOp::Gt => ord == std::cmp::Ordering::Greater,
1423                                crate::plan::CompareOp::Lt => ord == std::cmp::Ordering::Less,
1424                                crate::plan::CompareOp::Ge => ord != std::cmp::Ordering::Less,
1425                                crate::plan::CompareOp::Le => ord != std::cmp::Ordering::Greater,
1426                                crate::plan::CompareOp::Eq => ord == std::cmp::Ordering::Equal,
1427                                crate::plan::CompareOp::Ne => ord != std::cmp::Ordering::Equal,
1428                            };
1429                            if pass {
1430                                PredicateState::Pass
1431                            } else {
1432                                PredicateState::Fail
1433                            }
1434                        }
1435                        None => PredicateState::Fail,
1436                    }
1437                }
1438                None => PredicateState::Undecided,
1439            },
1440            FilterPredicate::Contains { field, value } => match tb.get_buffered_str(field) {
1441                Some(actual) => {
1442                    if actual.contains(value.as_str()) {
1443                        PredicateState::Pass
1444                    } else {
1445                        PredicateState::Fail
1446                    }
1447                }
1448                None => PredicateState::Undecided,
1449            },
1450            FilterPredicate::Regex { field, re } => match tb.get_buffered_str(field) {
1451                Some(actual) => {
1452                    if re.compiled.is_match(actual.as_ref()) {
1453                        PredicateState::Pass
1454                    } else {
1455                        PredicateState::Fail
1456                    }
1457                }
1458                None => PredicateState::Undecided,
1459            },
1460            FilterPredicate::IsNull { field } => match tb.get_buffered_str(field) {
1461                Some(_) => PredicateState::Fail,
1462                None => PredicateState::Pass,
1463            },
1464            FilterPredicate::IsType { field, field_type } => {
1465                // Check the plan's declared type first (fast path)
1466                let resolved = tb.plan.resolve_field(field).unwrap_or(field);
1467                let declared = tb.plan.column_type(resolved);
1468                if declared == *field_type {
1469                    // Declared type matches: pass if value exists
1470                    if tb.get_buffered_value(field).is_some() {
1471                        return PredicateState::Pass;
1472                    }
1473                }
1474                // For String columns, check if the buffered value can be parsed
1475                if let Some(val) = tb.get_buffered_value(field) {
1476                    match field_type {
1477                        crate::plan::FieldType::Int64 => {
1478                            if let Some(s) = val.as_str() {
1479                                if s.parse::<i64>().is_ok() {
1480                                    return PredicateState::Pass;
1481                                }
1482                            }
1483                        }
1484                        crate::plan::FieldType::Float64 => {
1485                            if let Some(s) = val.as_str() {
1486                                if s.parse::<f64>().is_ok() {
1487                                    return PredicateState::Pass;
1488                                }
1489                            }
1490                        }
1491                        crate::plan::FieldType::Boolean => {
1492                            if let Some(s) = val.as_str() {
1493                                if matches!(
1494                                    s.to_lowercase().as_str(),
1495                                    "true" | "false" | "1" | "0" | "yes" | "no"
1496                                ) {
1497                                    return PredicateState::Pass;
1498                                }
1499                            }
1500                        }
1501                        crate::plan::FieldType::String | crate::plan::FieldType::Dictionary => {
1502                            return PredicateState::Pass;
1503                        }
1504                        _ => {}
1505                    }
1506                }
1507                PredicateState::Fail
1508            }
1509            FilterPredicate::And(a, b) => {
1510                let sa = Self::eval_predicate(a, tb);
1511                let sb = Self::eval_predicate(b, tb);
1512                match (sa, sb) {
1513                    (PredicateState::Fail, _) | (_, PredicateState::Fail) => PredicateState::Fail,
1514                    (PredicateState::Pass, PredicateState::Pass) => PredicateState::Pass,
1515                    _ => PredicateState::Undecided,
1516                }
1517            }
1518            FilterPredicate::Or(a, b) => {
1519                let sa = Self::eval_predicate(a, tb);
1520                let sb = Self::eval_predicate(b, tb);
1521                match (sa, sb) {
1522                    (PredicateState::Pass, _) | (_, PredicateState::Pass) => PredicateState::Pass,
1523                    (PredicateState::Fail, PredicateState::Fail) => PredicateState::Fail,
1524                    _ => PredicateState::Undecided,
1525                }
1526            }
1527            FilterPredicate::Not(inner) => match Self::eval_predicate(inner, tb) {
1528                PredicateState::Pass => PredicateState::Fail,
1529                PredicateState::Fail => PredicateState::Pass,
1530                PredicateState::Undecided => PredicateState::Undecided,
1531            },
1532        }
1533    }
1534
1535    fn drain_buffered(&mut self, pass: bool) {
1536        let observer = self.plan.observer.clone();
1537        let buf = match self.row_buf {
1538            Some(ref mut b) => b,
1539            None => return,
1540        };
1541        if pass {
1542            // Deduplicate last-write-wins by slot index (u32).
1543            // O(n) reverse scan with a u64 bitmask; first hit in reverse is
1544            // the last write in forward order (last-write-wins).
1545            let ncols = self.columns.len();
1546            let words = ncols.div_ceil(64);
1547            if self.row_dirty.len() < words {
1548                self.row_dirty.resize(words, 0);
1549            }
1550            let mut seen: u64 = 0;
1551            let mut seen_extra: SmallVec<[u64; 1]> = SmallVec::new();
1552            if words > 1 {
1553                seen_extra.resize(words - 1, 0);
1554            }
1555            for &(slot, ref val) in buf.fields.iter().rev() {
1556                let word = slot as usize / 64;
1557                let bit = slot as usize % 64;
1558                let mask = 1u64 << bit;
1559                let already = if word == 0 {
1560                    seen & mask != 0
1561                } else {
1562                    seen_extra.get(word - 1).is_some_and(|w| w & mask != 0)
1563                };
1564                if already {
1565                    continue;
1566                }
1567                if word == 0 {
1568                    seen |= mask;
1569                } else if let Some(w) = seen_extra.get_mut(word - 1) {
1570                    *w |= mask;
1571                }
1572                self.row_dirty[word] |= mask;
1573                let b = &mut self.columns[slot as usize];
1574                if b.len() > self.row_count {
1575                    b.pop();
1576                }
1577                b.push_value(val.clone());
1578                // Buffered values are observable only once the row is
1579                // accepted (drained), never while merely buffered.
1580                if let Some(ref obs) = observer {
1581                    if let Some(name) = self.column_order.get(slot as usize) {
1582                        obs.on_put_field(self.row_count, name, slot as usize, val);
1583                    }
1584                }
1585            }
1586            // Null-fill missing columns and handle filter/dirty, then increment row_count.
1587            // Skip null-fill when called mid-row (buf.direct = true): the remaining
1588            // fields will be pushed via the direct path, and end_row's
1589            // null_fill_missing handles any columns that don't appear.
1590            if !buf.direct {
1591                let ncols = self.columns.len();
1592                let full_words = ncols / 64;
1593                let rem_bits = ncols % 64;
1594                let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
1595                    && (rem_bits == 0
1596                        || self.row_dirty.get(full_words).copied().unwrap_or(0)
1597                            == (1u64 << rem_bits) - 1);
1598                if is_full {
1599                    self.row_dirty.fill(0);
1600                } else {
1601                    for (i, b) in self.columns.iter_mut().enumerate() {
1602                        let word = i / 64;
1603                        let bit = i % 64;
1604                        let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
1605                        if !is_set {
1606                            b.push(None);
1607                        }
1608                    }
1609                    self.row_dirty.fill(0);
1610                }
1611            } else {
1612                // Mid-row: keep dirty bits set so null_fill_missing in end_row
1613                // can take the is_full fast path when all columns were pushed.
1614            }
1615            self.row_count += 1;
1616            self.rows_accepted += 1;
1617            if let Some(ref obs) = observer {
1618                obs.on_row_accepted(self.row_count - 1);
1619            }
1620        } else {
1621            // Fail: discard buffered fields, no row increment, clear dirty
1622            buf.fields.clear();
1623            self.row_dirty.fill(0);
1624            self.rows_rejected += 1;
1625            if let Some(ref obs) = observer {
1626                obs.on_row_rejected(self.row_count);
1627            }
1628        }
1629        // Ensure fields are cleared for next row
1630        if !buf.fields.is_empty() {
1631            buf.fields.clear();
1632        }
1633        buf.state = PredicateState::Undecided;
1634    }
1635
1636    fn evaluate_against_null(&self) -> PredicateState {
1637        // At end_row, any predicate field still missing is NULL.
1638        // For Equal with missing => Fail, NotEqual with missing => Pass (as per old finish_row check where get_value returns None)
1639        // For Compare with missing => Fail.
1640        // We can reuse eval_predicate which returns Undecided for missing, then map Undecided to Pass/Fail per leaf semantics.
1641        let Some(ref pred) = self.plan.filter else {
1642            return PredicateState::Pass;
1643        };
1644        Self::eval_predicate_with_null(pred, self)
1645    }
1646
1647    fn eval_predicate_with_null(pred: &FilterPredicate, tb: &TableBuilder) -> PredicateState {
1648        match pred {
1649            FilterPredicate::Equal { field, value } => match tb.get_buffered_str(field) {
1650                Some(actual) => {
1651                    if actual == *value {
1652                        PredicateState::Pass
1653                    } else {
1654                        PredicateState::Fail
1655                    }
1656                }
1657                None => PredicateState::Fail, // missing => None != Some(value) => Fail for Equal
1658            },
1659            FilterPredicate::NotEqual { field, value } => match tb.get_buffered_str(field) {
1660                Some(actual) => {
1661                    if actual != *value {
1662                        PredicateState::Pass
1663                    } else {
1664                        PredicateState::Fail
1665                    }
1666                }
1667                None => PredicateState::Pass, // missing => None != Some(value) => Pass
1668            },
1669            FilterPredicate::Compare { .. }
1670            | FilterPredicate::CompareLiteral { .. }
1671            | FilterPredicate::StartsWith { .. }
1672            | FilterPredicate::EndsWith { .. }
1673            | FilterPredicate::In { .. }
1674            | FilterPredicate::NotIn { .. }
1675            | FilterPredicate::NotField { .. }
1676            | FilterPredicate::ArithmeticCompare { .. }
1677            | FilterPredicate::Strip { .. }
1678            | FilterPredicate::Lower { .. }
1679            | FilterPredicate::Upper { .. }
1680            | FilterPredicate::Replace { .. }
1681            | FilterPredicate::Length { .. }
1682            | FilterPredicate::Contains { .. }
1683            | FilterPredicate::IsNull { .. }
1684            | FilterPredicate::IsType { .. }
1685            | FilterPredicate::Regex { .. } => match Self::eval_predicate(pred, tb) {
1686                PredicateState::Undecided => PredicateState::Fail,
1687                other => other,
1688            },
1689            FilterPredicate::Always(keep) => {
1690                if *keep {
1691                    PredicateState::Pass
1692                } else {
1693                    PredicateState::Fail
1694                }
1695            }
1696            FilterPredicate::And(a, b) => {
1697                let sa = Self::eval_predicate_with_null(a, tb);
1698                let sb = Self::eval_predicate_with_null(b, tb);
1699                match (sa, sb) {
1700                    (PredicateState::Fail, _) | (_, PredicateState::Fail) => PredicateState::Fail,
1701                    (PredicateState::Pass, PredicateState::Pass) => PredicateState::Pass,
1702                    _ => PredicateState::Undecided, // Should not happen after null mapping, but treat as Fail?
1703                }
1704            }
1705            FilterPredicate::Or(a, b) => {
1706                let sa = Self::eval_predicate_with_null(a, tb);
1707                let sb = Self::eval_predicate_with_null(b, tb);
1708                match (sa, sb) {
1709                    (PredicateState::Pass, _) | (_, PredicateState::Pass) => PredicateState::Pass,
1710                    (PredicateState::Fail, PredicateState::Fail) => PredicateState::Fail,
1711                    _ => PredicateState::Fail,
1712                }
1713            }
1714            FilterPredicate::Not(inner) => match Self::eval_predicate_with_null(inner, tb) {
1715                PredicateState::Pass => PredicateState::Fail,
1716                PredicateState::Fail => PredicateState::Pass,
1717                PredicateState::Undecided => PredicateState::Fail, // Not Undecided -> Fail?
1718            },
1719        }
1720    }
1721
1722    /// Null-fill any column missing the current row, then clear row_dirty.
1723    /// Called when the predicate resolved to Pass mid-row (direct mode).
1724    fn null_fill_missing(&mut self) {
1725        let ncols = self.columns.len();
1726        let full_words = ncols / 64;
1727        let rem_bits = ncols % 64;
1728        let is_full = (0..full_words).all(|w| self.row_dirty[w] == u64::MAX)
1729            && (rem_bits == 0
1730                || self.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem_bits) - 1);
1731        if is_full {
1732            self.row_dirty.fill(0);
1733        } else {
1734            for (i, b) in self.columns.iter_mut().enumerate() {
1735                let word = i / 64;
1736                let bit = i % 64;
1737                let is_set = (self.row_dirty[word] >> bit) & 1 == 1;
1738                if !is_set {
1739                    b.push(None);
1740                }
1741            }
1742            self.row_dirty.fill(0);
1743        }
1744        // Note: row_count was already incremented by drain_buffered(true)
1745        // when the predicate resolved to Pass mid-row. Do NOT increment here.
1746    }
1747}
1748
1749impl Default for TableBuilder {
1750    fn default() -> Self {
1751        Self::new()
1752    }
1753}
1754
1755impl ColumnarSink for TableBuilder {
1756    #[inline]
1757    fn begin_row(&mut self) {
1758        if let Some(ref obs) = self.plan.observer {
1759            obs.on_begin_row(self.row_count);
1760        }
1761        self.current_ordinal = 0;
1762        if let Some(ref mut buf) = self.row_buf {
1763            // Adaptive strategy: after at least one committed row, decide
1764            // whether buffering is worthwhile based on the predicate column's
1765            // ordinal relative to the total number of columns.  We must wait
1766            // until row_count > 0 because early-rejected rows don't create
1767            // all columns, making ncols unreliable for the ordinal comparison.
1768            if buf.predicate_ordinal.is_some() && buf.buffer_worthwhile && self.row_count > 0 {
1769                // Use the most accurate column count available. self.columns.len()
1770                // is unreliable for sparse files where not all columns exist yet.
1771                let ncols = self
1772                    .frozen
1773                    .as_ref()
1774                    .map(|f| f.num_columns() as u32)
1775                    .or_else(|| {
1776                        (!self.plan.schema_order.is_empty())
1777                            .then(|| self.plan.schema_order.len() as u32)
1778                    })
1779                    .unwrap_or(self.columns.len() as u32);
1780                if let Some(ordinal) = buf.predicate_ordinal {
1781                    if ordinal >= ncols * 4 / 5 {
1782                        buf.buffer_worthwhile = false;
1783                    }
1784                }
1785            }
1786            buf.fields.clear();
1787            buf.state = PredicateState::Undecided;
1788            buf.direct = false;
1789        }
1790    }
1791
1792    #[inline]
1793    fn put_field(&mut self, name: &str, value: Value<'_>) {
1794        if self.plan.filter.is_none() {
1795            self.push_field(name, value);
1796            return;
1797        }
1798        // Adaptive: late predicate → push directly, pop-on-reject at end_row.
1799        let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
1800        if !buf_worthwhile {
1801            self.push_field(name, value);
1802            return;
1803        }
1804        // Buffered path: resolve name → slot, buffer (slot, value).
1805        let resolved = match self.plan.resolve_field(name) {
1806            Some(r) => r.to_owned(),
1807            None => return,
1808        };
1809        // Check frozen unknown (as in push_field_resolved)
1810        if self.frozen.is_some()
1811            && !self.field_index.contains_key(&resolved)
1812            && self.unknown_error.is_none()
1813        {
1814            if let Some(ref frozen) = self.frozen {
1815                if frozen.is_exact()
1816                    && !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
1817                {
1818                    self.unknown_error = Some(format!(
1819                            "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1820                            resolved,
1821                            frozen.num_columns(),
1822                            frozen.is_exact()
1823                        ));
1824                    return;
1825                }
1826            }
1827        }
1828        let slot = self.ensure_column_idx(&resolved) as u32;
1829        self.record_strict_violation_at(slot as usize, &value);
1830        self.mark_predicate_slot(slot);
1831        let val_static: Value<'static> = value.into_static();
1832        let is_pred = self.is_predicate_slot(slot);
1833        #[cfg(feature = "profile")]
1834        if is_pred {
1835            crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1836        } else {
1837            crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1838        }
1839        // Push value to buffer BEFORE evaluating predicate (value must be findable).
1840        if let Some(ref mut buf) = self.row_buf {
1841            if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1842                buf.fields[pos].1 = val_static;
1843            } else {
1844                buf.fields.push((slot, val_static));
1845            }
1846        }
1847        if is_pred {
1848            // Record predicate ordinal for adaptive strategy (once)
1849            if let Some(ref mut buf) = self.row_buf {
1850                if buf.predicate_ordinal.is_none() {
1851                    buf.predicate_ordinal = Some(slot);
1852                }
1853            }
1854            let state = self.evaluate_predicate_state();
1855            if let Some(ref mut buf) = self.row_buf {
1856                buf.state = state;
1857            }
1858        }
1859    }
1860
1861    #[inline]
1862    fn end_row(&mut self) {
1863        if self.plan.filter.is_none() {
1864            self.finish_row();
1865            return;
1866        }
1867        // Check adaptive strategy: if buffering is not worthwhile (late
1868        // predicate), use direct push + pop-on-reject (finish_row).
1869        let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
1870        if !buf_worthwhile {
1871            // Late predicate: values were pushed directly to columns.
1872            // finish_row null-fills missing, evaluates filter against columns,
1873            // and pops on reject; exactly the pre-buffering behavior.
1874            self.finish_row();
1875            return;
1876        }
1877        // Buffered path
1878        let (state, direct) = self
1879            .row_buf
1880            .as_ref()
1881            .map_or((PredicateState::Pass, false), |b| (b.state, b.direct));
1882        if direct {
1883            // Already flushed to columns; just null-fill missing columns.
1884            self.null_fill_missing();
1885            return;
1886        }
1887        let state = if state == PredicateState::Undecided {
1888            self.evaluate_against_null()
1889        } else {
1890            state
1891        };
1892        let pass = state == PredicateState::Pass;
1893        self.drain_buffered(pass);
1894    }
1895
1896    #[inline]
1897    fn wants(&self, name: &str) -> bool {
1898        self.resolve(name).is_some()
1899    }
1900
1901    #[inline]
1902    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
1903        self.plan.resolve_field(name)
1904    }
1905
1906    #[inline]
1907    fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
1908        if self.plan.filter.is_none() {
1909            self.push_field_resolved(resolved_name, value);
1910            return;
1911        }
1912        // Adaptive: late predicate → push directly, pop-on-reject at end_row.
1913        let buf_worthwhile = self.row_buf.as_ref().is_none_or(|b| b.buffer_worthwhile);
1914        if !buf_worthwhile {
1915            self.push_field_resolved(resolved_name, value);
1916            return;
1917        }
1918        // Buffered path for already-resolved name (no rename lookup)
1919        if self.frozen.is_some() && !self.field_index.contains_key(resolved_name) {
1920            if let Some(ref frozen) = self.frozen {
1921                if frozen.is_exact()
1922                    && !frozen
1923                        .column_names()
1924                        .iter()
1925                        .any(|n| n.as_ref() == resolved_name)
1926                {
1927                    if self.unknown_error.is_none() {
1928                        self.unknown_error = Some(format!(
1929                            "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
1930                            resolved_name,
1931                            frozen.num_columns(),
1932                            frozen.is_exact()
1933                        ));
1934                    }
1935                    return;
1936                }
1937            }
1938        }
1939        let slot = self.ensure_column_idx(resolved_name) as u32;
1940        self.record_strict_violation_at(slot as usize, &value);
1941        self.mark_predicate_slot(slot);
1942        let val_static: Value<'static> = value.into_static();
1943        let is_pred = self.is_predicate_slot(slot);
1944        #[cfg(feature = "profile")]
1945        if is_pred {
1946            crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1947        } else {
1948            crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
1949        }
1950        if let Some(ref mut buf) = self.row_buf {
1951            if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
1952                buf.fields[pos].1 = val_static;
1953            } else {
1954                buf.fields.push((slot, val_static));
1955            }
1956        }
1957        if is_pred {
1958            // Record predicate ordinal for adaptive strategy (once)
1959            if let Some(ref mut buf) = self.row_buf {
1960                if buf.predicate_ordinal.is_none() {
1961                    buf.predicate_ordinal = Some(slot);
1962                }
1963            }
1964            let state = self.evaluate_predicate_state();
1965            if let Some(ref mut buf) = self.row_buf {
1966                buf.state = state;
1967            }
1968        }
1969    }
1970
1971    /// Push a value directly to a known slot index, bypassing all name
1972    /// resolution and hash lookups.  Used by adapters that verified the
1973    /// field identity via `expect_slot` + memcmp on raw bytes.
1974    #[inline]
1975    fn put_field_at(&mut self, slot: u32, value: Value<'_>) {
1976        let idx = slot as usize;
1977        if idx >= self.columns.len() {
1978            // Slot doesn't exist yet (sparse row). Fall back to ensure.
1979            // This shouldn't happen after row 1 with a stable layout.
1980            return;
1981        }
1982        self.record_strict_violation_at(idx, &value);
1983        self.notify_put_field(idx, &value);
1984        let word = idx / 64;
1985        let bit = idx % 64;
1986        self.row_dirty[word] |= 1u64 << bit;
1987        let b = &mut self.columns[idx];
1988        let row_count = self.row_count;
1989        if b.len() > row_count {
1990            b.pop();
1991        }
1992        b.push_value(value);
1993    }
1994
1995    #[inline]
1996    fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
1997        #[cfg(feature = "profile")]
1998        RESOLVE_AND_PUT_COUNT.fetch_add(1, Ordering::Relaxed);
1999        if self.plan.filter.is_none() {
2000            if self.plan.field_map.is_empty()
2001                && self.plan.drop_fields.is_empty()
2002                && self.plan.schema_order.is_empty()
2003            {
2004                self.push_field_resolved(name, value);
2005            } else {
2006                if let Some(resolved) = self.plan.resolve_field(name) {
2007                    let owned = resolved.to_owned();
2008                    self.push_field_resolved(&owned, value);
2009                }
2010            }
2011            return;
2012        }
2013        // Adaptive: late predicate → push directly, pop-on-reject at end_row.
2014        if self.row_buf.as_ref().is_some_and(|b| !b.buffer_worthwhile) {
2015            if self.plan.field_map.is_empty()
2016                && self.plan.drop_fields.is_empty()
2017                && self.plan.schema_order.is_empty()
2018            {
2019                self.push_field_resolved(name, value);
2020            } else {
2021                if let Some(resolved) = self.plan.resolve_field(name) {
2022                    let owned = resolved.to_owned();
2023                    self.push_field_resolved(&owned, value);
2024                }
2025            }
2026            return;
2027        }
2028        // Direct mode: predicate already passed, push directly to columns.
2029        if let Some(ref buf) = self.row_buf {
2030            if buf.direct {
2031                let resolved = match self.plan.resolve_field(name) {
2032                    Some(r) => r.to_owned(),
2033                    None => return, // field is dropped
2034                };
2035                if self.frozen.is_some() && !self.field_index.contains_key(resolved.as_str()) {
2036                    if let Some(ref frozen) = self.frozen {
2037                        if frozen.is_exact()
2038                            && !frozen
2039                                .column_names()
2040                                .iter()
2041                                .any(|n| n.as_ref() == resolved.as_str())
2042                        {
2043                            if self.unknown_error.is_none() {
2044                                self.unknown_error = Some(format!(
2045                                    "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
2046                                    resolved, frozen.num_columns(), frozen.is_exact()
2047                                ));
2048                            }
2049                            return;
2050                        }
2051                    }
2052                }
2053                self.push_field_resolved(&resolved, value);
2054                return;
2055            }
2056        }
2057        // Buffered path: resolve → slot, buffer (slot, value).
2058        if self.plan.field_map.is_empty()
2059            && self.plan.drop_fields.is_empty()
2060            && self.plan.schema_order.is_empty()
2061        {
2062            // No rename/drop, raw == resolved
2063            let resolved = name;
2064            if self.frozen.is_some() && !self.field_index.contains_key(resolved) {
2065                if let Some(ref frozen) = self.frozen {
2066                    if frozen.is_exact()
2067                        && !frozen.column_names().iter().any(|n| n.as_ref() == resolved)
2068                    {
2069                        if self.unknown_error.is_none() {
2070                            self.unknown_error = Some(format!(
2071                                "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
2072                                resolved,
2073                                frozen.num_columns(),
2074                                frozen.is_exact()
2075                            ));
2076                        }
2077                        return;
2078                    }
2079                }
2080            }
2081            let slot = self.ensure_column_idx(resolved) as u32;
2082            self.record_strict_violation_at(slot as usize, &value);
2083            self.mark_predicate_slot(slot);
2084            let val_static: Value<'static> = value.into_static();
2085            let is_pred = self.is_predicate_slot(slot);
2086            #[cfg(feature = "profile")]
2087            if is_pred {
2088                crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2089            } else {
2090                crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2091            }
2092            if let Some(ref mut buf) = self.row_buf {
2093                if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
2094                    buf.fields[pos].1 = val_static;
2095                } else {
2096                    buf.fields.push((slot, val_static));
2097                }
2098            }
2099            if is_pred {
2100                // Record predicate ordinal for adaptive strategy (once)
2101                if let Some(ref mut buf) = self.row_buf {
2102                    if buf.predicate_ordinal.is_none() {
2103                        buf.predicate_ordinal = Some(slot);
2104                    }
2105                }
2106                let state = self.evaluate_predicate_state();
2107                if let Some(ref mut buf) = self.row_buf {
2108                    buf.state = state;
2109                    if state == PredicateState::Pass {
2110                        buf.direct = true;
2111                    }
2112                }
2113                if state == PredicateState::Pass {
2114                    // Predicate passed: drain buffered fields, switch to direct mode.
2115                    self.drain_buffered(true);
2116                }
2117            }
2118        } else {
2119            if let Some(resolved) = self.plan.resolve_field(name) {
2120                let owned = resolved.to_owned();
2121                if self.frozen.is_some() && !self.field_index.contains_key(&owned) {
2122                    if let Some(ref frozen) = self.frozen {
2123                        if frozen.is_exact()
2124                            && !frozen.column_names().iter().any(|n| n.as_ref() == owned)
2125                        {
2126                            if self.unknown_error.is_none() {
2127                                self.unknown_error = Some(format!(
2128                                    "unknown field {:?} not in frozen schema ({} columns, exact={}); pass schema=[...]",
2129                                    owned,
2130                                    frozen.num_columns(),
2131                                    frozen.is_exact()
2132                                ));
2133                            }
2134                            return;
2135                        }
2136                    }
2137                }
2138                let slot = self.ensure_column_idx(&owned) as u32;
2139                self.record_strict_violation_at(slot as usize, &value);
2140                self.mark_predicate_slot(slot);
2141                let val_static: Value<'static> = value.into_static();
2142                let is_pred = self.is_predicate_slot(slot);
2143                #[cfg(feature = "profile")]
2144                if is_pred {
2145                    crate::engine::IS_PRED_TRUE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2146                } else {
2147                    crate::engine::IS_PRED_FALSE.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2148                }
2149                if let Some(ref mut buf) = self.row_buf {
2150                    if let Some(pos) = buf.fields.iter().position(|(s, _)| *s == slot) {
2151                        buf.fields[pos].1 = val_static;
2152                    } else {
2153                        buf.fields.push((slot, val_static));
2154                    }
2155                }
2156                if is_pred {
2157                    // Record predicate ordinal for adaptive strategy (once)
2158                    if let Some(ref mut buf) = self.row_buf {
2159                        if buf.predicate_ordinal.is_none() {
2160                            buf.predicate_ordinal = Some(slot);
2161                        }
2162                    }
2163                    let state = self.evaluate_predicate_state();
2164                    if let Some(ref mut buf) = self.row_buf {
2165                        buf.state = state;
2166                    }
2167                }
2168            }
2169        }
2170    }
2171
2172    #[inline]
2173    fn row_rejected(&self) -> bool {
2174        self.plan.filter.is_some()
2175            && self
2176                .row_buf
2177                .as_ref()
2178                .is_some_and(|b| b.state == PredicateState::Fail)
2179    }
2180
2181    #[inline]
2182    fn row_satisfied(&self) -> bool {
2183        let mask = self.wanted_mask();
2184        if mask == 0 {
2185            return false;
2186        }
2187        // Check that every wanted bit is set in row_dirty.
2188        let dirty = self
2189            .row_dirty
2190            .iter()
2191            .copied()
2192            .reduce(|a, b| a | b)
2193            .unwrap_or(0);
2194        (dirty & mask) == mask
2195    }
2196
2197    #[inline]
2198    fn wanted_mask(&self) -> u64 {
2199        // Precomputed in with_plan after schema columns are pre-declared.
2200        // Complete from the first row; 0 when no projection is active (the
2201        // short-circuit stays disabled unless the user explicitly declared
2202        // which columns they want via schema_order).
2203        self.cached_wanted_mask
2204    }
2205
2206    #[inline]
2207    fn finish(&mut self) -> Result<RecordBatch> {
2208        if let Some(err) = self.unknown_error.take() {
2209            return Err(crate::Error::Merge(err));
2210        }
2211        if let Some(err) = self.strict_error.take() {
2212            return Err(err);
2213        }
2214        if let Some(ref obs) = self.plan.observer {
2215            obs.on_chunk_finished(
2216                self.rows_accepted + self.rows_rejected,
2217                self.rows_accepted,
2218                self.rows_rejected,
2219            );
2220        }
2221        self.normalize();
2222
2223        if self.column_order.is_empty() {
2224            let schema = Arc::new(Schema::empty());
2225            return Ok(RecordBatch::new_empty(schema));
2226        }
2227
2228        self.auto_dict_upgrade();
2229
2230        // Schema projection: emit exactly the declared columns, in the
2231        // declared order.  Columns declared but absent from the data are
2232        // null-filled; materialized non-schema columns (e.g. filter-only
2233        // fields kept for predicate evaluation) are projected out.
2234        if !self.plan.schema_order.is_empty() {
2235            let order = self.plan.schema_order.clone();
2236            let mut fields = Vec::with_capacity(order.len());
2237            let mut arrays = Vec::with_capacity(order.len());
2238            for name in &order {
2239                // drop_fields wins over schema (see predeclare_schema_columns).
2240                if self.plan.drop_fields.contains(name) {
2241                    continue;
2242                }
2243                let idx = match self.field_index.get(name.as_str()) {
2244                    Some(&i) => i,
2245                    None => {
2246                        // Declared but never seen: all-null column.
2247                        let ty = self.plan.column_type(name);
2248                        let mut b = ColumnBuilder::with_capacity(self.row_count.max(1), &ty);
2249                        for _ in 0..self.row_count {
2250                            b.push(None);
2251                        }
2252                        let idx = self.columns.len();
2253                        self.columns.push(b);
2254                        self.field_index.insert(name.clone(), idx);
2255                        idx
2256                    }
2257                };
2258                let b = &mut self.columns[idx];
2259                fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
2260                arrays.push(b.to_arrow_array()?);
2261            }
2262            let schema = Arc::new(Schema::new(fields));
2263            return Ok(RecordBatch::try_new(schema, arrays)?);
2264        }
2265
2266        self.sort_columns();
2267
2268        let mut fields = Vec::with_capacity(self.column_order.len());
2269        let mut arrays = Vec::with_capacity(self.column_order.len());
2270        // Iterate by index so we can borrow columns mutably for zero-copy export.
2271        for name in &self.column_order {
2272            if let Some(&idx) = self.field_index.get(name.as_str()) {
2273                let b = &mut self.columns[idx];
2274                fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
2275                arrays.push(b.to_arrow_array()?);
2276            }
2277        }
2278
2279        let schema = Arc::new(Schema::new(fields));
2280        Ok(RecordBatch::try_new(schema, arrays)?)
2281    }
2282
2283    #[inline]
2284    fn expect_slot(&self, ordinal: u32) -> Option<(u32, &[u8])> {
2285        self.ordinal_expect
2286            .get(ordinal as usize)
2287            .and_then(|entry| entry.as_ref())
2288            .map(|(slot, raw)| (*slot, raw.as_slice()))
2289    }
2290
2291    #[inline]
2292    fn record_slot(&mut self, ordinal: u32, slot: u32, raw_name: &[u8]) {
2293        let idx = ordinal as usize;
2294        if idx >= self.ordinal_expect.len() {
2295            self.ordinal_expect.resize_with(idx + 1, || None);
2296        }
2297        self.ordinal_expect[idx] = Some((slot, raw_name.to_vec()));
2298    }
2299
2300    #[inline]
2301    fn layout_broken(&mut self, ordinal: u32) {
2302        if let Some(entry) = self.ordinal_expect.get_mut(ordinal as usize) {
2303            *entry = None;
2304        }
2305    }
2306
2307    #[inline]
2308    fn reset_child_ordinal(&mut self) {
2309        self.current_ordinal = 0;
2310    }
2311}
2312
2313// ---------------------------------------------------------------------------
2314// LocateOnly sink: walk rows, resolve field names, decode nothing.
2315// ---------------------------------------------------------------------------
2316
2317#[cfg(test)]
2318mod tests {
2319    use super::*;
2320    use crate::columnar::ColumnBuilder;
2321    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
2322    use crate::plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
2323    use crate::value::Value;
2324    use crate::Result;
2325    use arrow::array::{Array, AsArray};
2326
2327    /// Simple newline-delimited parser for test data.
2328    /// Each line is a row; fields are `key=value` separated by spaces.
2329    #[derive(Clone)]
2330    struct LineParser;
2331
2332    impl RecordParser for LineParser {
2333        fn validate(&self, bytes: &[u8]) -> Result<()> {
2334            simdutf8::basic::from_utf8(bytes)?;
2335            Ok(())
2336        }
2337
2338        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
2339            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
2340            for line in text.lines() {
2341                if line.is_empty() {
2342                    continue;
2343                }
2344                sink.begin_row();
2345                for token in line.split_whitespace() {
2346                    if let Some((k, v)) = token.split_once('=') {
2347                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
2348                    }
2349                }
2350                sink.end_row();
2351            }
2352            Ok(())
2353        }
2354    }
2355
2356    #[derive(Clone)]
2357    struct LineSplitter;
2358
2359    impl Splitter for LineSplitter {
2360        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
2361            if from >= bytes.len() {
2362                return None;
2363            }
2364            // If we're at a newline, advance past it.
2365            let start = if bytes[from] == b'\n' { from + 1 } else { from };
2366            if start >= bytes.len() {
2367                return None;
2368            }
2369            memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
2370        }
2371
2372        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
2373            if max_chunks <= 1 || bytes.is_empty() {
2374                return vec![0, bytes.len()];
2375            }
2376            let mut points = vec![0usize];
2377            let mut last = 0;
2378            for (i, &b) in bytes.iter().enumerate() {
2379                if b == b'\n' {
2380                    let next = i + 1;
2381                    if next > last && points.len() < max_chunks {
2382                        points.push(next);
2383                        last = next;
2384                    }
2385                }
2386            }
2387            if *points.last().unwrap() != bytes.len() {
2388                points.push(bytes.len());
2389            }
2390            points
2391        }
2392
2393        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
2394            let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
2395            (sample.len() / newline_count).max(1)
2396        }
2397    }
2398
2399    /// Records every hook call into a shared log, in call order.
2400    #[derive(Default)]
2401    struct RecObserver {
2402        events: std::sync::Mutex<Vec<String>>,
2403    }
2404
2405    impl RecObserver {
2406        fn log(&self, e: String) {
2407            self.events.lock().unwrap().push(e);
2408        }
2409        fn events(&self) -> Vec<String> {
2410            self.events.lock().unwrap().clone()
2411        }
2412    }
2413
2414    impl crate::RowObserver for RecObserver {
2415        fn on_begin_row(&self, row_index: usize) {
2416            self.log(format!("begin:{row_index}"));
2417        }
2418        fn on_put_field(&self, row_index: usize, name: &str, _slot: usize, _value: &Value<'_>) {
2419            self.log(format!("put:{row_index}:{name}"));
2420        }
2421        fn on_row_accepted(&self, row_index: usize) {
2422            self.log(format!("accept:{row_index}"));
2423        }
2424        fn on_row_rejected(&self, row_index: usize) {
2425            self.log(format!("reject:{row_index}"));
2426        }
2427        fn on_chunk_finished(&self, total: usize, accepted: usize, rejected: usize) {
2428            self.log(format!("fin:{total}:{accepted}:{rejected}"));
2429        }
2430    }
2431
2432    #[test]
2433    fn test_observer_lifecycle() {
2434        let obs = Arc::new(RecObserver::default());
2435        let mut plan = ExecutionPlan::new();
2436        plan.observer = Some(obs.clone());
2437        let mut engine = parse_bytes(b"X=1 Y=2\nX=3 Y=4\n", plan);
2438        engine.finish().unwrap();
2439        assert_eq!(
2440            obs.events(),
2441            vec![
2442                "begin:0",
2443                "put:0:X",
2444                "put:0:Y",
2445                "accept:0",
2446                "begin:1",
2447                "put:1:X",
2448                "put:1:Y",
2449                "accept:1",
2450                "fin:2:2:0",
2451                // finish() a second time re-fires with the same counters.
2452            ]
2453        );
2454    }
2455
2456    #[test]
2457    fn test_observer_with_filter() {
2458        let obs = Arc::new(RecObserver::default());
2459        let mut plan = ExecutionPlan::new();
2460        plan.filter = Some(FilterPredicate::Equal {
2461            field: "X".to_string(),
2462            value: "keep".to_string(),
2463        });
2464        plan.observer = Some(obs.clone());
2465        let mut engine = parse_bytes(b"X=keep A=1\nX=drop A=2\n", plan);
2466        engine.finish().unwrap();
2467        let ev = obs.events();
2468        // Rejected row: begin + reject, but its buffered fields never fire put.
2469        assert!(ev.contains(&"reject:1".to_string()), "events: {ev:?}");
2470        assert_eq!(
2471            ev.iter().filter(|e| e.starts_with("put:")).count(),
2472            2,
2473            "only the accepted row's fields may be observed: {ev:?}"
2474        );
2475        assert!(ev.contains(&"fin:2:1:1".to_string()), "events: {ev:?}");
2476    }
2477
2478    #[test]
2479    fn test_observer_parallel_counts_all_rows() {
2480        use std::sync::atomic::{AtomicUsize, Ordering as AOrd};
2481        #[derive(Default)]
2482        struct Counting {
2483            accepted: AtomicUsize,
2484            rejected: AtomicUsize,
2485        }
2486        impl crate::RowObserver for Counting {
2487            fn on_row_accepted(&self, _i: usize) {
2488                self.accepted.fetch_add(1, AOrd::Relaxed);
2489            }
2490            fn on_row_rejected(&self, _i: usize) {
2491                self.rejected.fetch_add(1, AOrd::Relaxed);
2492            }
2493        }
2494        let obs = Arc::new(Counting::default());
2495        let mut plan = ExecutionPlan::new();
2496        plan.filter = Some(FilterPredicate::NotEqual {
2497            field: "X".to_string(),
2498            value: "drop".to_string(),
2499        });
2500        plan.observer = Some(obs.clone());
2501        let mut data = Vec::new();
2502        for i in 0..1000 {
2503            let v = if i % 3 == 0 { "drop" } else { "keep" };
2504            data.extend_from_slice(format!("X={v} N={i}\n").as_bytes());
2505        }
2506        let pipeline = crate::Pipeline::new(LineSplitter, LineParser).with_plan(plan);
2507        let batches = pipeline.read_bytes_par(&data, 4).unwrap();
2508        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
2509        let accepted = obs.accepted.load(AOrd::Relaxed);
2510        let rejected = obs.rejected.load(AOrd::Relaxed);
2511        assert_eq!(accepted + rejected, 1000);
2512        assert_eq!(accepted, total);
2513        assert_eq!(rejected, 334);
2514    }
2515
2516    #[test]
2517    fn test_not_wraps_leaf_predicates_predicate_first() {
2518        // Every leaf under Not must invert its result in the predicate-first
2519        // (buffered) path used when plan.filter is set.
2520        let cases: Vec<(FilterPredicate, &[u8], Vec<&str>)> = vec![
2521            (
2522                FilterPredicate::not(FilterPredicate::Equal {
2523                    field: "X".into(),
2524                    value: "keep".into(),
2525                }),
2526                b"X=keep\nX=drop\n",
2527                vec!["drop"],
2528            ),
2529            (
2530                FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() }),
2531                b"X=1\nY=2\n",
2532                vec!["1"],
2533            ),
2534            (
2535                FilterPredicate::not(FilterPredicate::StartsWith {
2536                    field: "X".into(),
2537                    value: "ab".into(),
2538                }),
2539                b"X=abc\nX=zzz\n",
2540                vec!["zzz"],
2541            ),
2542            (
2543                FilterPredicate::not(FilterPredicate::In {
2544                    field: "X".into(),
2545                    values: vec!["a".into(), "b".into()],
2546                }),
2547                b"X=a\nX=b\nX=c\n",
2548                vec!["c"],
2549            ),
2550            (
2551                FilterPredicate::not(FilterPredicate::Regex {
2552                    field: "X".into(),
2553                    re: crate::plan::RegexSpec::new("^ERR").unwrap(),
2554                }),
2555                b"X=ERR1\nX=ok\n",
2556                vec!["ok"],
2557            ),
2558            (
2559                FilterPredicate::not(FilterPredicate::Contains {
2560                    field: "X".into(),
2561                    value: "zz".into(),
2562                }),
2563                b"X=azza\nX=hello\n",
2564                vec!["hello"],
2565            ),
2566        ];
2567        for (pred, data, want) in cases {
2568            let mut plan = ExecutionPlan::new();
2569            plan.filter = Some(pred);
2570            let engine = parse_bytes(data, plan);
2571            let col = engine.get_column("X").unwrap();
2572            assert_eq!(
2573                col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
2574                want,
2575                "data: {}",
2576                String::from_utf8_lossy(data)
2577            );
2578        }
2579    }
2580
2581    #[test]
2582    fn test_not_wraps_leaf_predicates_batch_check() {
2583        // The batch path (FilterPredicate::check over finished columns) must
2584        // agree with the predicate-first path for Not-wrapped leaves.
2585        let data = b"X=keep N=1\nX=drop N=2\nX=keep N=3\n";
2586        let engine = parse_bytes(data, ExecutionPlan::new());
2587        let pred = FilterPredicate::not(FilterPredicate::Equal {
2588            field: "X".into(),
2589            value: "keep".into(),
2590        });
2591        let kept: Vec<usize> = (0..engine.num_rows())
2592            .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
2593            .collect();
2594        assert_eq!(kept, vec![1]);
2595
2596        let pred = FilterPredicate::not(FilterPredicate::IsNull { field: "X".into() });
2597        let kept: Vec<usize> = (0..engine.num_rows())
2598            .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
2599            .collect();
2600        assert_eq!(kept, vec![0, 1, 2]);
2601
2602        let pred = FilterPredicate::not(FilterPredicate::Regex {
2603            field: "X".into(),
2604            re: crate::plan::RegexSpec::new("ke*p").unwrap(),
2605        });
2606        let kept: Vec<usize> = (0..engine.num_rows())
2607            .filter(|&r| pred.check(&engine.columns, &engine.field_index, r, &engine.plan))
2608            .collect();
2609        assert_eq!(kept, vec![1]);
2610    }
2611
2612    #[test]
2613    fn test_regex_predicate() {
2614        let mut plan = ExecutionPlan::new();
2615        plan.filter = Some(FilterPredicate::Regex {
2616            field: "X".into(),
2617            re: crate::plan::RegexSpec::new("^ERR\\d+$").unwrap(),
2618        });
2619        let engine = parse_bytes(b"X=ERR12\nX=ok\nX=ERRx\nX=ERR0\n", plan);
2620        let col = engine.get_column("X").unwrap();
2621        assert_eq!(
2622            col.as_str_vec().iter().flatten().collect::<Vec<_>>(),
2623            vec!["ERR12", "ERR0"]
2624        );
2625        // Missing field never matches.
2626        let mut plan = ExecutionPlan::new();
2627        plan.filter = Some(FilterPredicate::Regex {
2628            field: "X".into(),
2629            re: crate::plan::RegexSpec::new(".*").unwrap(),
2630        });
2631        let engine = parse_bytes(b"X=a\nY=2\n", plan);
2632        assert_eq!(engine.num_rows(), 1);
2633        // Invalid pattern is rejected at construction.
2634        assert!(crate::plan::RegexSpec::new("(").is_err());
2635        // Equality follows the pattern string.
2636        assert_eq!(
2637            crate::plan::RegexSpec::new("^a").unwrap(),
2638            crate::plan::RegexSpec::new("^a").unwrap()
2639        );
2640        assert_ne!(
2641            crate::plan::RegexSpec::new("^a").unwrap(),
2642            crate::plan::RegexSpec::new("^b").unwrap()
2643        );
2644    }
2645
2646    #[test]
2647    fn test_no_observer_smoke() {
2648        // plan.observer = None: hooks compile to a skipped branch.
2649        let engine = parse_bytes(b"X=1\nX=2\n", ExecutionPlan::new());
2650        assert_eq!(engine.num_rows(), 2);
2651    }
2652
2653    fn parse_bytes(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
2654        let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
2655        LineParser.parse_chunk(bytes, &mut sink).unwrap();
2656        sink
2657    }
2658
2659    #[test]
2660    fn test_extend_no_duplicates() {
2661        let e1 = parse_bytes(b"A=1 B=2\n", ExecutionPlan::new());
2662        let mut merged = TableBuilder::new();
2663        merged.extend(e1).unwrap();
2664        assert_eq!(merged.num_rows(), 1);
2665        for col in &merged.columns {
2666            assert_eq!(col.len(), merged.num_rows(), "column length mismatch");
2667        }
2668    }
2669
2670    #[test]
2671    fn test_multi_chunk_same_as_single() {
2672        let data = b"A=1\nA=2\nA=3\n";
2673        let single = parse_bytes(data, ExecutionPlan::new());
2674        assert_eq!(single.num_rows(), 3);
2675
2676        let splitter = LineSplitter;
2677        let points = splitter.find_split_points(data, 2);
2678        let mut merged = TableBuilder::new();
2679        for w in points.windows(2) {
2680            let chunk = &data[w[0]..w[1]];
2681            let engine = parse_bytes(chunk, ExecutionPlan::new());
2682            merged.extend(engine).unwrap();
2683        }
2684        assert_eq!(merged.num_rows(), single.num_rows());
2685    }
2686
2687    #[test]
2688    fn test_last_write_wins_duplicate_field() {
2689        let engine = parse_bytes(b"X=10 X=20\n", ExecutionPlan::new());
2690        assert_eq!(engine.num_rows(), 1);
2691        let col = engine.get_column("X").unwrap();
2692        assert_eq!(col.as_str_vec(), vec![Some("20".into())]);
2693    }
2694
2695    #[test]
2696    fn test_build_plan_rename() {
2697        let mut plan = ExecutionPlan::new();
2698        plan.field_map.insert("X".to_string(), "Y".to_string());
2699        let engine = parse_bytes(b"X=hello\n", plan);
2700        assert_eq!(engine.num_rows(), 1);
2701        assert!(engine.get_column("Y").is_some());
2702        assert!(engine.get_column("X").is_none());
2703        assert_eq!(
2704            engine.get_column("Y").unwrap().as_str_vec(),
2705            vec![Some("hello".into())]
2706        );
2707    }
2708
2709    #[test]
2710    fn test_build_plan_drop() {
2711        let mut plan = ExecutionPlan::new();
2712        plan.drop_fields.insert("X".to_string());
2713        let engine = parse_bytes(b"X=hello Y=world\n", plan);
2714        assert_eq!(engine.num_rows(), 1);
2715        assert!(engine.get_column("X").is_none());
2716        assert!(engine.get_column("Y").is_some());
2717    }
2718
2719    #[test]
2720    fn test_build_plan_filter_ne() {
2721        let mut plan = ExecutionPlan::new();
2722        plan.filter = Some(FilterPredicate::NotEqual {
2723            field: "X".to_string(),
2724            value: "42".to_string(),
2725        });
2726        let engine = parse_bytes(b"X=10\nX=42\nX=30\n", plan);
2727        assert_eq!(engine.num_rows(), 2);
2728        let col = engine.get_column("X").unwrap();
2729        assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("30".into())]);
2730    }
2731
2732    #[test]
2733    fn test_build_plan_filter_eq() {
2734        let mut plan = ExecutionPlan::new();
2735        plan.filter = Some(FilterPredicate::Equal {
2736            field: "X".to_string(),
2737            value: "10".to_string(),
2738        });
2739        let engine = parse_bytes(b"X=10\nX=20\nX=10\n", plan);
2740        assert_eq!(engine.num_rows(), 2);
2741        let col = engine.get_column("X").unwrap();
2742        assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("10".into())]);
2743    }
2744
2745    #[test]
2746    fn test_build_plan_filter_missing_field() {
2747        let mut plan = ExecutionPlan::new();
2748        plan.filter = Some(FilterPredicate::NotEqual {
2749            field: "X".to_string(),
2750            value: "10".to_string(),
2751        });
2752        let engine = parse_bytes(b"X=10\nY=99\n", plan);
2753        assert_eq!(engine.num_rows(), 1);
2754        let col = engine.get_column("Y").unwrap();
2755        assert_eq!(col.as_str_vec(), vec![Some("99".into())]);
2756    }
2757
2758    #[test]
2759    fn test_typed_int64_column() {
2760        let mut plan = ExecutionPlan::new();
2761        plan.field_types.insert("X".to_string(), FieldType::Int64);
2762        let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
2763        assert_eq!(engine.num_rows(), 3);
2764        if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
2765            assert_eq!(v.get(0), Some(42));
2766            assert_eq!(v.get(1), None);
2767            assert_eq!(v.get(2), Some(100));
2768        } else {
2769            panic!("expected Int64 builder");
2770        }
2771    }
2772
2773    #[test]
2774    fn test_typed_float64_column() {
2775        let mut plan = ExecutionPlan::new();
2776        plan.field_types.insert("X".to_string(), FieldType::Float64);
2777        let engine = parse_bytes(b"X=1.5\n", plan);
2778        if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
2779            assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
2780        } else {
2781            panic!("expected Float64 builder");
2782        }
2783    }
2784
2785    #[test]
2786    fn test_is_null_filter() {
2787        let mut plan = ExecutionPlan::new();
2788        plan.filter = Some(FilterPredicate::IsNull {
2789            field: "X".to_string(),
2790        });
2791        let engine = parse_bytes(b"X=hello\nY=world\nX=fallback\n", plan);
2792        assert_eq!(engine.num_rows(), 1);
2793        let col = engine.get_column("Y").unwrap();
2794        assert_eq!(col.as_str_vec(), vec![Some("world".into())]);
2795    }
2796
2797    #[test]
2798    fn test_not_is_null_filter_predicate_first() {
2799        let mut plan = ExecutionPlan::new();
2800        plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
2801            field: "X".to_string(),
2802        }));
2803        let engine = parse_bytes(b"X=a\nY=only\nX=b\n", plan);
2804        assert_eq!(engine.num_rows(), 2);
2805        let col = engine.get_column("X").unwrap();
2806        assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
2807    }
2808
2809    #[test]
2810    fn test_not_is_null_filter_batch_eval() {
2811        // Force the late-predicate path (direct push + pop-on-reject at
2812        // end_row, evaluated via FilterPredicate::check in plan.rs).
2813        let mut plan = ExecutionPlan::new();
2814        plan.filter = Some(FilterPredicate::not(FilterPredicate::IsNull {
2815            field: "X".to_string(),
2816        }));
2817        let mut sink = TableBuilder::with_plan(4, Arc::new(plan));
2818        sink.row_buf.as_mut().unwrap().buffer_worthwhile = false;
2819        LineParser
2820            .parse_chunk(b"X=a\nY=only\nX=b\n", &mut sink)
2821            .unwrap();
2822        assert_eq!(sink.num_rows(), 2);
2823        let col = sink.get_column("X").unwrap();
2824        assert_eq!(col.as_str_vec(), vec![Some("a".into()), Some("b".into())]);
2825    }
2826
2827    #[test]
2828    fn test_strict_types_errors_on_malformed_int64() {
2829        let mut plan = ExecutionPlan::new();
2830        plan.field_types.insert("X".to_string(), FieldType::Int64);
2831        plan.strict_types = true;
2832        let mut engine = parse_bytes(b"X=42\nX=abc\n", plan);
2833        let err = engine.finish().unwrap_err();
2834        let msg = err.to_string();
2835        assert!(msg.contains("\"abc\""), "message should name value: {msg}");
2836        assert!(msg.contains("\"X\""), "message should name column: {msg}");
2837        assert!(msg.contains("Int64"), "message should name type: {msg}");
2838        assert!(msg.contains("row 1"), "message should name row: {msg}");
2839    }
2840
2841    #[test]
2842    fn test_strict_types_off_yields_null() {
2843        let mut plan = ExecutionPlan::new();
2844        plan.field_types.insert("X".to_string(), FieldType::Int64);
2845        let engine = parse_bytes(b"X=42\nX=abc\n", plan);
2846        assert!(engine.strict_error.is_none());
2847        if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
2848            assert_eq!(v.get(0), Some(42));
2849            assert_eq!(v.get(1), None);
2850        } else {
2851            panic!("expected Int64 builder");
2852        }
2853    }
2854
2855    #[test]
2856    fn test_strict_types_missing_field_stays_null() {
2857        let mut plan = ExecutionPlan::new();
2858        plan.field_types.insert("X".to_string(), FieldType::Int64);
2859        plan.strict_types = true;
2860        let mut engine = parse_bytes(b"X=42\nY=1\n", plan);
2861        let batch = engine.finish().unwrap();
2862        let idx = batch.schema().index_of("X").unwrap();
2863        let arr = batch
2864            .column(idx)
2865            .as_primitive::<arrow::datatypes::Int64Type>();
2866        assert_eq!(arr.value(0), 42);
2867        assert!(arr.is_null(1));
2868    }
2869
2870    #[test]
2871    fn test_strict_types_buffered_filter_path() {
2872        // With a filter, values travel through the predicate row buffer; the
2873        // strict check must fire there too.
2874        let mut plan = ExecutionPlan::new();
2875        plan.field_types.insert("X".to_string(), FieldType::Int64);
2876        plan.strict_types = true;
2877        plan.filter = Some(FilterPredicate::Equal {
2878            field: "Y".to_string(),
2879            value: "1".to_string(),
2880        });
2881        let mut engine = parse_bytes(b"Y=1 X=abc\n", plan);
2882        let err = engine.finish().unwrap_err();
2883        assert!(err.to_string().contains("strict_types"));
2884    }
2885
2886    #[test]
2887    fn test_is_type_filter_int64() {
2888        let mut plan = ExecutionPlan::new();
2889        plan.field_types.insert("X".to_string(), FieldType::Int64);
2890        plan.filter = Some(FilterPredicate::IsType {
2891            field: "X".to_string(),
2892            field_type: FieldType::Int64,
2893        });
2894        let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
2895        assert_eq!(engine.num_rows(), 2);
2896        if let ColumnBuilder::Int64(v) = engine.get_column("X").unwrap() {
2897            assert_eq!(v.get(0), Some(42));
2898            assert_eq!(v.get(1), Some(100));
2899        } else {
2900            panic!("expected Int64 builder");
2901        }
2902    }
2903
2904    #[test]
2905    fn test_is_type_filter_float64() {
2906        let mut plan = ExecutionPlan::new();
2907        plan.field_types.insert("X".to_string(), FieldType::Float64);
2908        plan.filter = Some(FilterPredicate::IsType {
2909            field: "X".to_string(),
2910            field_type: FieldType::Float64,
2911        });
2912        let engine = parse_bytes(b"X=1.5\nX=bad\nX=2.5\n", plan);
2913        assert_eq!(engine.num_rows(), 2);
2914        if let ColumnBuilder::Float64(v) = engine.get_column("X").unwrap() {
2915            assert!((v.get(0).unwrap() - 1.5).abs() < 1e-9);
2916            assert!((v.get(1).unwrap() - 2.5).abs() < 1e-9);
2917        } else {
2918            panic!("expected Float64 builder");
2919        }
2920    }
2921
2922    #[test]
2923    fn test_dictionary_column() {
2924        let mut plan = ExecutionPlan::new();
2925        plan.dictionary_columns.insert("P".to_string());
2926        let engine = parse_bytes(b"P=Widget\nP=Gadget\nP=Widget\n", plan);
2927        assert_eq!(engine.num_rows(), 3);
2928        if let ColumnBuilder::Dictionary { codes, offsets, .. } = engine.get_column("P").unwrap() {
2929            let dict_len = offsets.len() - 1;
2930            assert_eq!(dict_len, 2);
2931            assert_eq!(
2932                codes.iter().map(|o| o.copied()).collect::<Vec<_>>(),
2933                vec![Some(0), Some(1), Some(0)]
2934            );
2935        } else {
2936            panic!("expected Dictionary builder");
2937        }
2938    }
2939
2940    #[test]
2941    fn test_ragged_late_chunk_column_debut() {
2942        let e1 = parse_bytes(b"A=1 B=2\nA=3\n", ExecutionPlan::new());
2943        let e2 = parse_bytes(b"B=4 C=5\n", ExecutionPlan::new());
2944        let mut merged = TableBuilder::new();
2945        merged.extend(e1).unwrap();
2946        merged.extend(e2).unwrap();
2947
2948        assert_eq!(
2949            merged.get_column("A").unwrap().as_str_vec(),
2950            vec![Some("1".into()), Some("3".into()), None]
2951        );
2952        assert_eq!(
2953            merged.get_column("B").unwrap().as_str_vec(),
2954            vec![Some("2".into()), None, Some("4".into())]
2955        );
2956        assert_eq!(
2957            merged.get_column("C").unwrap().as_str_vec(),
2958            vec![None, None, Some("5".into())]
2959        );
2960    }
2961
2962    #[test]
2963    fn test_auto_dict_upgrade_only_post_merge() {
2964        let mut plan = ExecutionPlan::new();
2965        plan.auto_dict = true;
2966        let a = parse_bytes(b"P=x\nP=y\n", plan.clone());
2967        let b = parse_bytes(b"P=x\nP=y\n", plan.clone());
2968        let mut merged = TableBuilder::with_plan(64, Arc::new(plan));
2969        merged.extend(a).unwrap();
2970        merged.extend(b).unwrap();
2971        merged.auto_dict_upgrade();
2972        assert_eq!(merged.num_rows(), 4);
2973    }
2974
2975    #[test]
2976    fn test_extend_string_dictionary_promotes_not_panics() {
2977        let mut e1 = parse_bytes(b"P=x\n", ExecutionPlan::new());
2978        let mut plan = ExecutionPlan::new();
2979        plan.dictionary_columns.insert("P".to_string());
2980        let e2 = parse_bytes(b"P=x\n", plan);
2981        // Safe promotion: string + dictionary reconcile to dictionary.
2982        e1.extend(e2).expect("String/Dictionary must reconcile");
2983        assert!(matches!(
2984            e1.get_column("P").unwrap(),
2985            crate::columnar::ColumnBuilder::Dictionary { .. }
2986        ));
2987    }
2988
2989    #[test]
2990    fn test_extend_irreconcilable_variants_error_not_panic() {
2991        let mut e1 = parse_bytes(b"S=x\n", ExecutionPlan::new());
2992        let mut plan = ExecutionPlan::new();
2993        plan.field_types.insert("S".to_string(), FieldType::Int64);
2994        let e2 = parse_bytes(b"S=7\n", plan);
2995        let result = e1.extend(e2);
2996        match result {
2997            Err(crate::Error::Merge(msg)) => {
2998                assert!(msg.contains("'S'"), "error should name the column: {msg}");
2999            }
3000            other => panic!("String/Int64 mismatch must return Merge error, got {other:?}"),
3001        }
3002    }
3003
3004    #[test]
3005    fn test_compare_filter_via_arrow_compute() {
3006        use crate::arrow_export::apply_compare_filter;
3007
3008        let mut plan = ExecutionPlan::new();
3009        plan.filter = Some(FilterPredicate::Compare {
3010            field_a: "A".to_string(),
3011            op: CompareOp::Gt,
3012            field_b: "B".to_string(),
3013        });
3014        let mut engine = parse_bytes(b"A=3 B=1\nA=2 B=2\nA=5 B=4\n", plan);
3015        let batch = engine.finish().unwrap();
3016        let filtered = apply_compare_filter(
3017            batch,
3018            &FilterPredicate::Compare {
3019                field_a: "A".to_string(),
3020                op: CompareOp::Gt,
3021                field_b: "B".to_string(),
3022            },
3023        )
3024        .unwrap();
3025        assert_eq!(filtered.num_rows(), 2);
3026        let a = filtered.column_by_name("A").unwrap().as_string::<i32>();
3027        assert_eq!(a.value(0), "3");
3028        assert_eq!(a.value(1), "5");
3029    }
3030
3031    #[test]
3032    fn test_row_dirty_is_full_edge_cases() {
3033        // Directly test the is_full logic for ncols = 1,63,64,65,127,128,129
3034        // We do this via TableBuilder with varying column counts
3035        for ncols in [1, 63, 64, 65, 127, 128, 129] {
3036            let mut tb = TableBuilder::new();
3037            for i in 0..ncols {
3038                let name = format!("col{i}");
3039                tb.field_index.insert(name.clone(), i);
3040                tb.columns.push(crate::columnar::ColumnBuilder::String(
3041                    crate::columnar::StrColumn::default(),
3042                ));
3043                tb.column_order.push(name);
3044            }
3045            // row_dirty should be vec![0; (ncols+63)/64]
3046            tb.row_dirty = vec![0; ncols.div_ceil(64)];
3047            // Test is_full when all bits set
3048            for i in 0..ncols {
3049                let word = i / 64;
3050                let bit = i % 64;
3051                tb.row_dirty[word] |= 1u64 << bit;
3052            }
3053            let full_words = ncols / 64;
3054            let rem = ncols % 64;
3055            let is_full = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
3056                && (rem == 0
3057                    || tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
3058            assert!(
3059                is_full,
3060                "ncols={ncols} should be full, row_dirty={:?}",
3061                tb.row_dirty
3062            );
3063            // Clear one bit and check not full
3064            if ncols > 0 {
3065                let word = (ncols - 1) / 64;
3066                let bit = (ncols - 1) % 64;
3067                tb.row_dirty[word] &= !(1u64 << bit);
3068                let is_full2 = (0..full_words).all(|w| tb.row_dirty[w] == u64::MAX)
3069                    && (rem == 0
3070                        || tb.row_dirty.get(full_words).copied().unwrap_or(0) == (1u64 << rem) - 1);
3071                assert!(
3072                    !is_full2,
3073                    "ncols={ncols} should not be full after clearing one bit"
3074                );
3075            }
3076        }
3077    }
3078
3079    #[test]
3080    fn test_finish_twice_does_not_panic() {
3081        // L7: zero-copy export (mem::take) leaves builders empty. Calling
3082        // finish() twice should return an empty batch, not panic.
3083        let mut tb = parse_bytes(b"A=1 B=2\nA=3 B=4\n", ExecutionPlan::new());
3084        let batch1 = tb.finish().unwrap();
3085        assert_eq!(batch1.num_rows(), 2);
3086        // Second call: columns are empty after mem::take.
3087        let batch2 = tb.finish().unwrap();
3088        assert_eq!(batch2.num_rows(), 0);
3089    }
3090
3091    #[test]
3092    fn test_c2_compound_reorder() {
3093        // L7: C2 reorder: Field2 == x AND Field1 == y must give the same
3094        // result as Field1 == y AND Field2 == x.
3095        let data = b"A=1 B=2\nA=2 B=2\nA=1 B=3\nA=2 B=3\n";
3096        let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
3097            let mut plan = ExecutionPlan::new();
3098            plan.filter = Some(FilterPredicate::And(
3099                Box::new(FilterPredicate::Equal {
3100                    field: a.to_string(),
3101                    value: va.to_string(),
3102                }),
3103                Box::new(FilterPredicate::Equal {
3104                    field: b.to_string(),
3105                    value: vb.to_string(),
3106                }),
3107            ));
3108            plan
3109        };
3110        let r1 = parse_bytes(data, make_filter("A", "1", "B", "2"));
3111        let r2 = parse_bytes(data, make_filter("B", "2", "A", "1"));
3112        assert_eq!(
3113            r1.num_rows(),
3114            r2.num_rows(),
3115            "C2 reorder must produce identical results"
3116        );
3117        assert_eq!(r1.num_rows(), 1); // A=1 B=2
3118    }
3119
3120    #[test]
3121    fn test_c2_or_reorder() {
3122        // L7: Or reorder: same result regardless of operand order.
3123        // Row 1: A=1 B=2 → A==1 → Pass
3124        // Row 2: A=2 B=2 → A!=1, B!=3 → Fail
3125        // Row 3: A=3 B=3 → B==3 → Pass
3126        let data = b"A=1 B=2\nA=2 B=2\nA=3 B=3\n";
3127        let make_filter = |a: &str, va: &str, b: &str, vb: &str| -> ExecutionPlan {
3128            let mut plan = ExecutionPlan::new();
3129            plan.filter = Some(FilterPredicate::Or(
3130                Box::new(FilterPredicate::Equal {
3131                    field: a.to_string(),
3132                    value: va.to_string(),
3133                }),
3134                Box::new(FilterPredicate::Equal {
3135                    field: b.to_string(),
3136                    value: vb.to_string(),
3137                }),
3138            ));
3139            plan
3140        };
3141        let r1 = parse_bytes(data, make_filter("A", "1", "B", "3"));
3142        let r2 = parse_bytes(data, make_filter("B", "3", "A", "1"));
3143        assert_eq!(
3144            r1.num_rows(),
3145            r2.num_rows(),
3146            "Or reorder must produce identical results"
3147        );
3148        assert_eq!(r1.num_rows(), 2); // rows 1 and 3
3149    }
3150
3151    #[test]
3152    fn test_predicate_slot_created_after_mask_build() {
3153        // Regression: when the predicate column appears AFTER other columns
3154        // in document order, build_predicate_mask runs before the predicate
3155        // column exists in field_index, so its bit is never set. The old
3156        // early-return optimization prevented it from being added later.
3157        //
3158        // Data: A comes first, B is the predicate column (comes second).
3159        // Filter: B == "reject" → all rows should be rejected.
3160        let data = b"A=1 B=ok\nA=2 B=ok\nA=3 B=ok\n";
3161        let mut plan = ExecutionPlan::new();
3162        plan.filter = Some(FilterPredicate::Equal {
3163            field: "B".to_string(),
3164            value: "reject".to_string(),
3165        });
3166        let engine = parse_bytes(data, plan);
3167        assert_eq!(
3168            engine.num_rows(),
3169            0,
3170            "all rows should be rejected (B != reject)"
3171        );
3172
3173        // Positive case: some rows pass
3174        let data2 = b"A=1 B=ok\nA=2 B=reject\nA=3 B=ok\n";
3175        let mut plan2 = ExecutionPlan::new();
3176        plan2.filter = Some(FilterPredicate::Equal {
3177            field: "B".to_string(),
3178            value: "ok".to_string(),
3179        });
3180        let engine2 = parse_bytes(data2, plan2);
3181        assert_eq!(engine2.num_rows(), 2, "rows 1 and 3 pass (B == ok)");
3182        let col = engine2.get_column("A").unwrap();
3183        assert_eq!(col.as_str_vec(), vec![Some("1".into()), Some("3".into())]);
3184    }
3185
3186    #[test]
3187    fn test_predicate_first_via_resolve_and_put() {
3188        // Uses resolve_and_put (scanner path) instead of put_field (LineParser path)
3189        // to verify the predicate machinery works through the scanner's code path.
3190        let mut plan = ExecutionPlan::new();
3191        plan.filter = Some(FilterPredicate::Equal {
3192            field: "B".to_string(),
3193            value: "reject".to_string(),
3194        });
3195        let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
3196        // Row 1: B=ok → should pass
3197        tb.begin_row();
3198        tb.resolve_and_put("A", Value::Str(Cow::Borrowed("1")));
3199        tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
3200        tb.end_row();
3201        // Row 2: B=reject → should fail
3202        tb.begin_row();
3203        tb.resolve_and_put("A", Value::Str(Cow::Borrowed("2")));
3204        tb.resolve_and_put("B", Value::Str(Cow::Borrowed("reject")));
3205        tb.end_row();
3206        // Row 3: B=ok → should pass
3207        tb.begin_row();
3208        tb.resolve_and_put("A", Value::Str(Cow::Borrowed("3")));
3209        tb.resolve_and_put("B", Value::Str(Cow::Borrowed("ok")));
3210        tb.end_row();
3211        let batch = tb.finish().unwrap();
3212        assert_eq!(batch.num_rows(), 1, "only row 2 (B==reject) should pass");
3213    }
3214
3215    #[test]
3216    fn test_owned_cow_values_survive_buffered_filter() {
3217        // Regression for the use-after-free behind this change: Value::Str
3218        // used to be `&'a str`, and the buffered filtered path transmutes
3219        // `Value<'a>` to `Value<'static>` to hold values until the predicate
3220        // is decided. Adapters that unescape entities produce owned Strings;
3221        // the owned buffer was dropped at the end of the scan function and
3222        // the transmuted reference dangled, so heap reuse corrupted accepted
3223        // rows after enough rejections. With `Str(Cow<'a, str>)` the owned
3224        // buffer moves into the row buffer.
3225        let mut plan = ExecutionPlan::new();
3226        plan.filter = Some(FilterPredicate::Equal {
3227            field: "B".to_string(),
3228            value: "ok".to_string(),
3229        });
3230        let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
3231        for i in 0..50 {
3232            tb.begin_row();
3233            // Owned values, as produced by entity unescaping in an adapter.
3234            let a = Cow::Owned(format!("value-{i}-unescaped"));
3235            let b = Cow::Owned(if i % 2 == 0 {
3236                "ok".to_string()
3237            } else {
3238                "no".to_string()
3239            });
3240            tb.resolve_and_put("A", Value::Str(a));
3241            tb.resolve_and_put("B", Value::Str(b));
3242            tb.end_row();
3243        }
3244        assert_eq!(tb.num_rows(), 25, "even rows pass (B == ok)");
3245        let col = tb.get_column("A").unwrap();
3246        let vals = col.as_str_vec();
3247        assert_eq!(vals.len(), 25);
3248        assert_eq!(vals[0], Some("value-0-unescaped".to_string()));
3249        assert_eq!(vals[24], Some("value-48-unescaped".to_string()));
3250    }
3251
3252    #[test]
3253    fn test_predicate_first_skip_via_resolve_and_put() {
3254        // Test the skip path: predicate on first field, all rows rejected.
3255        // This exercises the mark_predicate_slot + row_rejected + end_row path
3256        // that the scanner uses via resolve_and_put.
3257        let mut plan = ExecutionPlan::new();
3258        plan.filter = Some(FilterPredicate::Equal {
3259            field: "A".to_string(),
3260            value: "999".to_string(),
3261        });
3262        let mut tb = TableBuilder::with_plan(16, Arc::new(plan));
3263        for _i in 0..100 {
3264            tb.begin_row();
3265            tb.resolve_and_put("A", Value::Str(Cow::Borrowed("0")));
3266            tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
3267            tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
3268            tb.end_row();
3269        }
3270        let batch = tb.finish().unwrap();
3271        assert_eq!(
3272            batch.num_rows(),
3273            0,
3274            "all rows should be rejected (A=0 != 999)"
3275        );
3276    }
3277
3278    #[test]
3279    fn test_sparse_column_predicate_skip() {
3280        // Regression test for ncols gate: sparse column appears only after
3281        // row 50, predicate on early field (A). The adaptive strategy must
3282        // not disable buffering because ncols is computed from the incomplete
3283        // column set before the sparse column appears.
3284        //
3285        // Data: 100 rows. Rows 1-50 have A, B, C. Rows 51-100 also have D.
3286        // Filter: A == "reject" → all rows rejected.
3287        // The skip must fire for all rows, not just the first few.
3288        let mut plan = ExecutionPlan::new();
3289        plan.filter = Some(FilterPredicate::Equal {
3290            field: "A".to_string(),
3291            value: "reject".to_string(),
3292        });
3293        let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
3294        for i in 0..100 {
3295            tb.begin_row();
3296            tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
3297            tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
3298            tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
3299            if i >= 50 {
3300                tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
3301            }
3302            tb.end_row();
3303        }
3304        let batch = tb.finish().unwrap();
3305        assert_eq!(batch.num_rows(), 0, "all rows rejected (A=ok != reject)");
3306    }
3307
3308    #[test]
3309    fn test_sparse_column_predicate_pass() {
3310        // Like above, but predicate passes. The sparse column D should appear
3311        // only for rows 51-100; rows 1-50 should have D = null.
3312        let mut plan = ExecutionPlan::new();
3313        plan.filter = Some(FilterPredicate::Equal {
3314            field: "A".to_string(),
3315            value: "ok".to_string(),
3316        });
3317        let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
3318        for _i in 0..100 {
3319            tb.begin_row();
3320            tb.resolve_and_put("A", Value::Str(Cow::Borrowed("ok")));
3321            tb.resolve_and_put("B", Value::Str(Cow::Borrowed("x")));
3322            tb.resolve_and_put("C", Value::Str(Cow::Borrowed("y")));
3323            if _i >= 50 {
3324                tb.resolve_and_put("D", Value::Str(Cow::Borrowed("z")));
3325            }
3326            tb.end_row();
3327        }
3328        let batch = tb.finish().unwrap();
3329        assert_eq!(batch.num_rows(), 100, "all rows pass (A=ok == ok)");
3330        // D column: null for first 50, "z" for last 50
3331        if let Some(col) = batch.column_by_name("D") {
3332            let arr = col.as_string::<i32>();
3333            for i in 0..50 {
3334                assert!(arr.is_null(i), "row {} should have D=null", i);
3335            }
3336            for i in 50..100 {
3337                assert!(
3338                    !arr.is_null(i),
3339                    "row {} should not be null (D len={}, total rows={})",
3340                    i,
3341                    arr.len(),
3342                    batch.num_rows()
3343                );
3344                assert_eq!(arr.value(i), "z", "rows 50+ should have D=z");
3345            }
3346        } else {
3347            panic!("D column should exist");
3348        }
3349    }
3350
3351    #[test]
3352    fn test_sparse_late_predicate_ordinal() {
3353        // Regression: predicate on column E that only appears after row 30.
3354        // ncols gate must use schema/column count that includes E, not just
3355        // the columns seen so far.
3356        let mut plan = ExecutionPlan::new();
3357        plan.filter = Some(FilterPredicate::Equal {
3358            field: "E".to_string(),
3359            value: "999".to_string(),
3360        });
3361        let mut tb = TableBuilder::with_plan(128, Arc::new(plan));
3362        // First 30 rows: only A, B
3363        for _i in 0..30 {
3364            tb.begin_row();
3365            tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
3366            tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
3367            tb.end_row();
3368        }
3369        // Rows 31-100: A, B, C, D, E
3370        for _i in 30..100 {
3371            tb.begin_row();
3372            tb.resolve_and_put("A", Value::Str(Cow::Borrowed("x")));
3373            tb.resolve_and_put("B", Value::Str(Cow::Borrowed("y")));
3374            tb.resolve_and_put("C", Value::Str(Cow::Borrowed("c")));
3375            tb.resolve_and_put("D", Value::Str(Cow::Borrowed("d")));
3376            tb.resolve_and_put("E", Value::Str(Cow::Borrowed("e")));
3377            tb.end_row();
3378        }
3379        let batch = tb.finish().unwrap();
3380        // E="e" != "999" → all rows rejected
3381        assert_eq!(batch.num_rows(), 0, "all rows rejected (E=\"e\" != 999)");
3382    }
3383
3384    #[test]
3385    fn raw_name_helpers_apply_rename_and_drop() {
3386        use crate::decoder::ColumnarSink;
3387
3388        let plan = ExecutionPlan::new()
3389            .rename("raw", "renamed")
3390            .drop("ignored");
3391        let mut tb = TableBuilder::with_plan(2, Arc::new(plan));
3392        tb.begin_row();
3393        tb.resolve_and_put_raw(b"raw", Value::Str(Cow::Borrowed("value")));
3394        tb.resolve_and_put_raw(b"ignored", Value::Str(Cow::Borrowed("not stored")));
3395        tb.put_row(&[("other", Value::Str(Cow::Borrowed("also stored")))]);
3396        tb.end_row();
3397
3398        assert_eq!(tb.column_names(), &["renamed", "other"]);
3399        assert_eq!(tb.resolve_raw(b"raw"), Some("renamed"));
3400    }
3401
3402    // -----------------------------------------------------------------------
3403    // Schema projection semantics: schema=[...] declares exactly which
3404    // columns are extracted, in order.  Unlisted fields are skipped during
3405    // parsing (wants()/resolve() reject them); listed columns absent from
3406    // the data are null-filled.
3407    // -----------------------------------------------------------------------
3408
3409    /// Parser that mimics adapter projection behavior (e.g. crxml's scanner):
3410    /// skips fields the sink rejects via `resolve()` and byte-jumps to the
3411    /// end of the row once `row_satisfied()` reports all wanted columns seen.
3412    struct SkippingParser;
3413
3414    impl RecordParser for SkippingParser {
3415        fn validate(&self, bytes: &[u8]) -> Result<()> {
3416            simdutf8::basic::from_utf8(bytes)?;
3417            Ok(())
3418        }
3419
3420        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
3421            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
3422            for line in text.lines() {
3423                if line.is_empty() {
3424                    continue;
3425                }
3426                sink.begin_row();
3427                for token in line.split_whitespace() {
3428                    if sink.row_satisfied() {
3429                        break; // byte-jump to row close: remaining fields skipped
3430                    }
3431                    if let Some((k, v)) = token.split_once('=') {
3432                        if sink.resolve(k).is_some() {
3433                            sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
3434                        }
3435                    }
3436                }
3437                sink.end_row();
3438            }
3439            Ok(())
3440        }
3441    }
3442
3443    fn parse_skipping(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
3444        let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), Arc::new(plan));
3445        SkippingParser.parse_chunk(bytes, &mut sink).unwrap();
3446        sink
3447    }
3448
3449    fn batch_names(tb: &mut TableBuilder) -> Vec<String> {
3450        tb.finish()
3451            .unwrap()
3452            .schema()
3453            .fields()
3454            .iter()
3455            .map(|f| f.name().clone())
3456            .collect()
3457    }
3458
3459    const REPORT: &[u8] = b"Name=n1 Department=d1 Amount=1 Status=s1 Date=2024\n\
3460                            Name=n2 Department=d2 Amount=2 Status=s2 Date=2024\n";
3461
3462    #[test]
3463    fn schema_projection_subset() {
3464        // schema=["Name","Department"] must yield exactly those two columns,
3465        // in that order.  Regression: the old wanted_mask was derived from
3466        // field_index mid-row, so row_satisfied fired after "Name" on row 1
3467        // and "Department" was never materialized (output was ["Name"]).
3468        let mut tb = parse_skipping(
3469            REPORT,
3470            ExecutionPlan::new().schema_order(["Name", "Department"]),
3471        );
3472        assert_eq!(batch_names(&mut tb), vec!["Name", "Department"]);
3473    }
3474
3475    #[test]
3476    fn schema_projection_single_non_first_column() {
3477        // Regression: the old code emitted ["Department", "Name"] here —
3478        // "Name" was materialized because resolve_field did no projection.
3479        let mut tb = parse_skipping(REPORT, ExecutionPlan::new().schema_order(["Department"]));
3480        assert_eq!(batch_names(&mut tb), vec!["Department"]);
3481    }
3482
3483    #[test]
3484    fn schema_projection_full_natural_order() {
3485        let mut tb = parse_skipping(
3486            REPORT,
3487            ExecutionPlan::new().schema_order(["Name", "Department", "Amount", "Status", "Date"]),
3488        );
3489        assert_eq!(
3490            batch_names(&mut tb),
3491            vec!["Name", "Department", "Amount", "Status", "Date"]
3492        );
3493    }
3494
3495    #[test]
3496    fn schema_projection_reversed_order() {
3497        let mut tb = parse_skipping(
3498            REPORT,
3499            ExecutionPlan::new().schema_order(["Date", "Status", "Amount", "Department", "Name"]),
3500        );
3501        assert_eq!(
3502            batch_names(&mut tb),
3503            vec!["Date", "Status", "Amount", "Department", "Name"]
3504        );
3505    }
3506
3507    #[test]
3508    fn schema_projection_absent_column_null_filled() {
3509        // Schema is a superset of the data: the absent column is null-filled.
3510        let mut tb = parse_skipping(
3511            REPORT,
3512            ExecutionPlan::new().schema_order(["Name", "Missing"]),
3513        );
3514        let batch = tb.finish().unwrap();
3515        assert_eq!(batch.num_rows(), 2);
3516        let missing = batch.column_by_name("Missing").unwrap().as_string::<i32>();
3517        assert!(missing.is_null(0) && missing.is_null(1));
3518        let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
3519        assert_eq!(name.value(0), "n1");
3520    }
3521
3522    #[test]
3523    fn schema_projection_wants_rejects_unlisted_fields() {
3524        // The perf contract: unlisted fields are rejected by wants()/resolve()
3525        // so adapters skip extracting them entirely.
3526        let plan = ExecutionPlan::new().schema_order(["Name"]);
3527        let tb = TableBuilder::with_plan(4, Arc::new(plan));
3528        assert!(tb.wants("Name"));
3529        assert!(!tb.wants("Department"));
3530        assert_eq!(tb.resolve("Amount"), None);
3531    }
3532
3533    #[test]
3534    fn schema_projection_rename_then_match() {
3535        // field_mapping applies before schema matching: the schema lists the
3536        // renamed (output) column name.
3537        let plan = ExecutionPlan::new()
3538            .rename("Name", "Employee")
3539            .schema_order(["Employee", "Amount"]);
3540        let mut tb = parse_skipping(REPORT, plan);
3541        assert_eq!(batch_names(&mut tb), vec!["Employee", "Amount"]);
3542    }
3543
3544    #[test]
3545    fn schema_projection_field_types_apply() {
3546        let plan = ExecutionPlan::new()
3547            .type_as("Amount", FieldType::Int64)
3548            .schema_order(["Name", "Amount"]);
3549        let mut tb = parse_skipping(REPORT, plan);
3550        let batch = tb.finish().unwrap();
3551        let amount = batch.column_by_name("Amount").unwrap();
3552        assert!(matches!(
3553            amount.data_type(),
3554            arrow::datatypes::DataType::Int64
3555        ));
3556        assert_eq!(
3557            amount
3558                .as_primitive::<arrow::datatypes::Int64Type>()
3559                .value(1),
3560            2
3561        );
3562    }
3563
3564    #[test]
3565    fn schema_projection_filter_on_unlisted_field() {
3566        // Filter fields are materialized for predicate evaluation even when
3567        // not in the schema, but are projected out of the final batch.
3568        let plan = ExecutionPlan::new()
3569            .schema_order(["Name"])
3570            .filter_eq("Status", "s2");
3571        let mut tb = parse_skipping(REPORT, plan);
3572        let batch = tb.finish().unwrap();
3573        assert_eq!(batch.num_rows(), 1);
3574        assert_eq!(batch.schema().fields().len(), 1);
3575        let name = batch.column_by_name("Name").unwrap().as_string::<i32>();
3576        assert_eq!(name.value(0), "n2");
3577    }
3578
3579    #[test]
3580    fn schema_projection_drop_wins_over_schema() {
3581        // A column listed in both schema and drop_fields is dropped:
3582        // dropping is an explicit exclusion and would otherwise be
3583        // resurrected as a null-filled column.
3584        let plan = ExecutionPlan::new()
3585            .schema_order(["Name", "Department"])
3586            .drop("Department");
3587        let mut tb = parse_skipping(REPORT, plan);
3588        assert_eq!(batch_names(&mut tb), vec!["Name"]);
3589    }
3590
3591    #[test]
3592    fn schema_projection_no_parser_cooperation() {
3593        // Even when the adapter pushes every field (no wants/resolve checks),
3594        // the projection still holds: unlisted fields are never materialized.
3595        let mut tb = parse_bytes(
3596            REPORT,
3597            ExecutionPlan::new().schema_order(["Department", "Name"]),
3598        );
3599        let batch = tb.finish().unwrap();
3600        let names: Vec<String> = batch
3601            .schema()
3602            .fields()
3603            .iter()
3604            .map(|f| f.name().clone())
3605            .collect();
3606        assert_eq!(names, vec!["Department", "Name"]);
3607        let dept = batch
3608            .column_by_name("Department")
3609            .unwrap()
3610            .as_string::<i32>();
3611        assert_eq!(dept.value(0), "d1");
3612    }
3613}