Skip to main content

rypipe_core/
engine.rs

1use std::sync::Arc;
2
3use arrow::datatypes::{Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rustc_hash::FxHashMap as HashMap;
6
7use crate::columnar::ColumnBuilder;
8use crate::decoder::ColumnarSink;
9use crate::plan::ExecutionPlan;
10use crate::value::Value;
11use crate::Result;
12
13/// Generic columnar table builder.  Implements `ColumnarSink` so any decoder
14/// can feed it field/value events; at the end it produces an Arrow
15/// `RecordBatch`.
16pub struct TableBuilder {
17    pub(crate) columns: HashMap<String, ColumnBuilder>,
18    pub(crate) column_order: Vec<String>,
19    pub(crate) row_count: usize,
20    pub(crate) estimated_rows: usize,
21    pub(crate) plan: ExecutionPlan,
22}
23
24impl TableBuilder {
25    pub fn new() -> Self {
26        Self {
27            columns: HashMap::default(),
28            column_order: Vec::new(),
29            row_count: 0,
30            estimated_rows: 0,
31            plan: ExecutionPlan::new(),
32        }
33    }
34
35    pub fn with_capacity(cap: usize) -> Self {
36        Self {
37            columns: HashMap::default(),
38            column_order: Vec::new(),
39            row_count: 0,
40            estimated_rows: cap,
41            plan: ExecutionPlan::new(),
42        }
43    }
44
45    pub fn with_plan(cap: usize, plan: ExecutionPlan) -> Self {
46        Self {
47            columns: HashMap::default(),
48            column_order: Vec::new(),
49            row_count: 0,
50            estimated_rows: cap,
51            plan,
52        }
53    }
54
55    pub fn num_rows(&self) -> usize {
56        self.row_count
57    }
58
59    pub fn num_columns(&self) -> usize {
60        self.column_order.len()
61    }
62
63    pub fn column_names(&self) -> &[String] {
64        &self.column_order
65    }
66
67    /// Finalize the builder into an Arrow `RecordBatch`.
68    ///
69    /// This is also available as the `ColumnarSink::finish` trait method.
70    pub fn finish(&mut self) -> Result<RecordBatch> {
71        ColumnarSink::finish(self)
72    }
73
74    /// Reset all data while preserving the plan and estimated rows.
75    pub fn reset(&mut self) {
76        self.columns.clear();
77        self.column_order.clear();
78        self.row_count = 0;
79    }
80
81    /// Truncate every column back to `row_count`, dropping any partial-row
82    /// values from a mid-field EOF.  Idempotent.
83    pub fn normalize(&mut self) {
84        for b in self.columns.values_mut() {
85            while b.len() > self.row_count {
86                b.pop();
87            }
88        }
89    }
90
91    /// If `auto_dict` is set, upgrade low-cardinality string columns.
92    pub fn auto_dict_upgrade(&mut self) {
93        if self.plan.auto_dict {
94            for b in self.columns.values_mut() {
95                b.try_upgrade_to_dict(512);
96            }
97        }
98    }
99
100    /// Sort columns according to `schema_order`.  Columns named in
101    /// `schema_order` appear in that order; any other columns keep their
102    /// relative first-appearance order after the ordered ones.
103    pub fn sort_columns(&mut self) {
104        if self.plan.schema_order.is_empty() {
105            return;
106        }
107        let order = &self.plan.schema_order;
108        let rank = |name: &String| order.iter().position(|n| n == name).unwrap_or(usize::MAX);
109        self.column_order.sort_by_key(rank);
110    }
111
112    pub(crate) fn schema_insert_index(&self, name: &str) -> usize {
113        let order = &self.plan.schema_order;
114        if order.is_empty() {
115            return self.column_order.len();
116        }
117        let pos = order.iter().position(|n| n == name);
118        match pos {
119            Some(p) => self
120                .column_order
121                .iter()
122                .position(|existing| {
123                    order
124                        .iter()
125                        .position(|n| n == existing)
126                        .is_some_and(|ep| ep > p)
127                })
128                .unwrap_or(self.column_order.len()),
129            None => self.column_order.len(),
130        }
131    }
132
133    fn ensure_column(&mut self, name: &str) {
134        if !self.columns.contains_key(name) {
135            let est = self.estimated_rows.max(64);
136            let col_type = self.plan.column_type(name);
137            let mut b = ColumnBuilder::with_capacity(est, &col_type);
138            for _ in 0..self.row_count {
139                b.push(None);
140            }
141            self.columns.insert(name.to_owned(), b);
142            let idx = self.schema_insert_index(name);
143            self.column_order.insert(idx, name.to_owned());
144        }
145    }
146
147    /// Push a field value, resolving renames/drops and applying last-write-wins
148    /// within the current uncommitted row.
149    fn push_field(&mut self, name: &str, value: Value<'_>) {
150        // Fast path: no rename/drop configured.
151        let owned;
152        let resolved: &str = if self.plan.field_map.is_empty() && self.plan.drop_fields.is_empty() {
153            name
154        } else {
155            match self.plan.resolve_field(name) {
156                Some(n) => {
157                    owned = n.to_owned();
158                    &owned
159                }
160                None => return,
161            }
162        };
163
164        self.ensure_column(resolved);
165        let row_count = self.row_count;
166        if let Some(b) = self.columns.get_mut(resolved) {
167            if b.len() > row_count {
168                b.pop();
169            }
170            b.push_value(value);
171        }
172    }
173
174    /// Null-fill any column missing this row, then apply the per-row filter.
175    /// If the filter rejects the row, undo it by popping values.
176    fn finish_row(&mut self) {
177        let target = self.row_count + 1;
178        for b in self.columns.values_mut() {
179            while b.len() < target {
180                b.push(None);
181            }
182        }
183
184        if let Some(ref filter) = self.plan.filter {
185            if !filter.check(&self.columns, self.row_count, &self.plan) {
186                for b in self.columns.values_mut() {
187                    b.pop();
188                }
189                return;
190            }
191        }
192
193        self.row_count += 1;
194    }
195}
196
197impl Default for TableBuilder {
198    fn default() -> Self {
199        Self::new()
200    }
201}
202
203impl ColumnarSink for TableBuilder {
204    fn begin_row(&mut self) {
205        // Row boundaries are tracked by `row_count`; no state to set up.
206    }
207
208    fn put_field(&mut self, name: &str, value: Value<'_>) {
209        self.push_field(name, value);
210    }
211
212    fn end_row(&mut self) {
213        self.finish_row();
214    }
215
216    fn wants(&self, name: &str) -> bool {
217        self.plan.resolve_field(name).is_some()
218    }
219
220    fn finish(&mut self) -> Result<RecordBatch> {
221        self.normalize();
222
223        if self.column_order.is_empty() {
224            let schema = Arc::new(Schema::empty());
225            return Ok(RecordBatch::try_new(schema, Vec::new())?);
226        }
227
228        self.auto_dict_upgrade();
229        self.sort_columns();
230
231        let mut fields = Vec::with_capacity(self.column_order.len());
232        let mut arrays = Vec::with_capacity(self.column_order.len());
233        for name in &self.column_order {
234            if let Some(b) = self.columns.get(name) {
235                fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
236                arrays.push(b.to_arrow_array()?);
237            }
238        }
239
240        let schema = Arc::new(Schema::new(fields));
241        Ok(RecordBatch::try_new(schema, arrays)?)
242    }
243}
244
245#[cfg(test)]
246mod tests {
247    use super::*;
248    use crate::columnar::ColumnBuilder;
249    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
250    use crate::plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
251    use crate::value::Value;
252    use crate::Result;
253    use arrow::array::AsArray;
254
255    /// Simple newline-delimited parser for test data.
256    /// Each line is a row; fields are `key=value` separated by spaces.
257    struct LineParser;
258
259    impl RecordParser for LineParser {
260        fn validate(&self, bytes: &[u8]) -> Result<()> {
261            simdutf8::basic::from_utf8(bytes)?;
262            Ok(())
263        }
264
265        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
266            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
267            for line in text.lines() {
268                if line.is_empty() {
269                    continue;
270                }
271                sink.begin_row();
272                for token in line.split_whitespace() {
273                    if let Some((k, v)) = token.split_once('=') {
274                        sink.put_field(k, Value::Str(v));
275                    }
276                }
277                sink.end_row();
278            }
279            Ok(())
280        }
281    }
282
283    struct LineSplitter;
284
285    impl Splitter for LineSplitter {
286        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
287            if max_chunks <= 1 || bytes.is_empty() {
288                return vec![0, bytes.len()];
289            }
290            let mut points = vec![0usize];
291            let mut last = 0;
292            for (i, &b) in bytes.iter().enumerate() {
293                if b == b'\n' {
294                    let next = i + 1;
295                    if next > last && points.len() < max_chunks {
296                        points.push(next);
297                        last = next;
298                    }
299                }
300            }
301            if *points.last().unwrap() != bytes.len() {
302                points.push(bytes.len());
303            }
304            points
305        }
306
307        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
308            let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
309            (sample.len() / newline_count).max(1)
310        }
311    }
312
313    fn parse_bytes(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
314        let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), plan);
315        LineParser.parse_chunk(bytes, &mut sink).unwrap();
316        sink
317    }
318
319    #[test]
320    fn test_extend_no_duplicates() {
321        let e1 = parse_bytes(b"A=1 B=2\n", ExecutionPlan::new());
322        let mut merged = TableBuilder::new();
323        merged.extend(e1).unwrap();
324        assert_eq!(merged.num_rows(), 1);
325        for (name, col) in &merged.columns {
326            assert_eq!(
327                col.len(),
328                merged.num_rows(),
329                "column {} length mismatch",
330                name
331            );
332        }
333    }
334
335    #[test]
336    fn test_multi_chunk_same_as_single() {
337        let data = b"A=1\nA=2\nA=3\n";
338        let single = parse_bytes(data, ExecutionPlan::new());
339        assert_eq!(single.num_rows(), 3);
340
341        let splitter = LineSplitter;
342        let points = splitter.find_split_points(data, 2);
343        let mut merged = TableBuilder::new();
344        for w in points.windows(2) {
345            let chunk = &data[w[0]..w[1]];
346            let engine = parse_bytes(chunk, ExecutionPlan::new());
347            merged.extend(engine).unwrap();
348        }
349        assert_eq!(merged.num_rows(), single.num_rows());
350    }
351
352    #[test]
353    fn test_last_write_wins_duplicate_field() {
354        let engine = parse_bytes(b"X=10 X=20\n", ExecutionPlan::new());
355        assert_eq!(engine.num_rows(), 1);
356        let col = engine.columns.get("X").unwrap();
357        assert_eq!(col.as_str_vec(), vec![Some("20".into())]);
358    }
359
360    #[test]
361    fn test_build_plan_rename() {
362        let mut plan = ExecutionPlan::new();
363        plan.field_map.insert("X".to_string(), "Y".to_string());
364        let engine = parse_bytes(b"X=hello\n", plan);
365        assert_eq!(engine.num_rows(), 1);
366        assert!(engine.columns.contains_key("Y"));
367        assert!(!engine.columns.contains_key("X"));
368        assert_eq!(
369            engine.columns.get("Y").unwrap().as_str_vec(),
370            vec![Some("hello".into())]
371        );
372    }
373
374    #[test]
375    fn test_build_plan_drop() {
376        let mut plan = ExecutionPlan::new();
377        plan.drop_fields.insert("X".to_string());
378        let engine = parse_bytes(b"X=hello Y=world\n", plan);
379        assert_eq!(engine.num_rows(), 1);
380        assert!(!engine.columns.contains_key("X"));
381        assert!(engine.columns.contains_key("Y"));
382    }
383
384    #[test]
385    fn test_build_plan_filter_ne() {
386        let mut plan = ExecutionPlan::new();
387        plan.filter = Some(FilterPredicate::NotEqual {
388            field: "X".to_string(),
389            value: "42".to_string(),
390        });
391        let engine = parse_bytes(b"X=10\nX=42\nX=30\n", plan);
392        assert_eq!(engine.num_rows(), 2);
393        let col = engine.columns.get("X").unwrap();
394        assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("30".into())]);
395    }
396
397    #[test]
398    fn test_build_plan_filter_eq() {
399        let mut plan = ExecutionPlan::new();
400        plan.filter = Some(FilterPredicate::Equal {
401            field: "X".to_string(),
402            value: "10".to_string(),
403        });
404        let engine = parse_bytes(b"X=10\nX=20\nX=10\n", plan);
405        assert_eq!(engine.num_rows(), 2);
406        let col = engine.columns.get("X").unwrap();
407        assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("10".into())]);
408    }
409
410    #[test]
411    fn test_build_plan_filter_missing_field() {
412        let mut plan = ExecutionPlan::new();
413        plan.filter = Some(FilterPredicate::NotEqual {
414            field: "X".to_string(),
415            value: "10".to_string(),
416        });
417        let engine = parse_bytes(b"X=10\nY=99\n", plan);
418        assert_eq!(engine.num_rows(), 1);
419        let col = engine.columns.get("Y").unwrap();
420        assert_eq!(col.as_str_vec(), vec![Some("99".into())]);
421    }
422
423    #[test]
424    fn test_typed_int64_column() {
425        let mut plan = ExecutionPlan::new();
426        plan.field_types.insert("X".to_string(), FieldType::Int64);
427        let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
428        assert_eq!(engine.num_rows(), 3);
429        if let ColumnBuilder::Int64(v) = &engine.columns["X"] {
430            assert_eq!(v, &vec![Some(42), None, Some(100)]);
431        } else {
432            panic!("expected Int64 builder");
433        }
434    }
435
436    #[test]
437    fn test_typed_float64_column() {
438        let mut plan = ExecutionPlan::new();
439        plan.field_types.insert("X".to_string(), FieldType::Float64);
440        let engine = parse_bytes(b"X=1.5\n", plan);
441        if let ColumnBuilder::Float64(v) = &engine.columns["X"] {
442            assert!((v[0].unwrap() - 1.5).abs() < 1e-9);
443        } else {
444            panic!("expected Float64 builder");
445        }
446    }
447
448    #[test]
449    fn test_dictionary_column() {
450        let mut plan = ExecutionPlan::new();
451        plan.dictionary_columns.insert("P".to_string());
452        let engine = parse_bytes(b"P=Widget\nP=Gadget\nP=Widget\n", plan);
453        assert_eq!(engine.num_rows(), 3);
454        if let ColumnBuilder::Dictionary { codes, dict, .. } = &engine.columns["P"] {
455            assert_eq!(dict.len(), 2);
456            assert_eq!(codes, &vec![Some(0), Some(1), Some(0)]);
457        } else {
458            panic!("expected Dictionary builder");
459        }
460    }
461
462    #[test]
463    fn test_ragged_late_chunk_column_debut() {
464        let e1 = parse_bytes(b"A=1 B=2\nA=3\n", ExecutionPlan::new());
465        let e2 = parse_bytes(b"B=4 C=5\n", ExecutionPlan::new());
466        let mut merged = TableBuilder::new();
467        merged.extend(e1).unwrap();
468        merged.extend(e2).unwrap();
469
470        assert_eq!(
471            merged.columns["A"].as_str_vec(),
472            vec![Some("1".into()), Some("3".into()), None]
473        );
474        assert_eq!(
475            merged.columns["B"].as_str_vec(),
476            vec![Some("2".into()), None, Some("4".into())]
477        );
478        assert_eq!(
479            merged.columns["C"].as_str_vec(),
480            vec![None, None, Some("5".into())]
481        );
482    }
483
484    #[test]
485    fn test_auto_dict_upgrade_only_post_merge() {
486        let mut plan = ExecutionPlan::new();
487        plan.auto_dict = true;
488        let a = parse_bytes(b"P=x\nP=y\n", plan.clone());
489        let b = parse_bytes(b"P=x\nP=y\n", plan.clone());
490        let mut merged = TableBuilder::with_plan(64, plan);
491        merged.extend(a).unwrap();
492        merged.extend(b).unwrap();
493        merged.auto_dict_upgrade();
494        assert_eq!(merged.num_rows(), 4);
495    }
496
497    #[test]
498    fn test_extend_variant_mismatch_errors_not_panics() {
499        let mut e1 = parse_bytes(b"P=x\n", ExecutionPlan::new());
500        let mut plan = ExecutionPlan::new();
501        plan.dictionary_columns.insert("P".to_string());
502        let e2 = parse_bytes(b"P=x\n", plan);
503        let result = e1.extend(e2);
504        assert!(
505            result.is_err(),
506            "String/Dictionary mismatch must return Err"
507        );
508    }
509
510    #[test]
511    fn test_compare_filter_via_arrow_compute() {
512        use crate::arrow_export::apply_compare_filter;
513
514        let mut plan = ExecutionPlan::new();
515        plan.filter = Some(FilterPredicate::Compare {
516            field_a: "A".to_string(),
517            op: CompareOp::Gt,
518            field_b: "B".to_string(),
519        });
520        let mut engine = parse_bytes(b"A=3 B=1\nA=2 B=2\nA=5 B=4\n", plan);
521        let batch = engine.finish().unwrap();
522        let filtered = apply_compare_filter(
523            batch,
524            &FilterPredicate::Compare {
525                field_a: "A".to_string(),
526                op: CompareOp::Gt,
527                field_b: "B".to_string(),
528            },
529        )
530        .unwrap();
531        assert_eq!(filtered.num_rows(), 2);
532        let a = filtered.column_by_name("A").unwrap().as_string::<i32>();
533        assert_eq!(a.value(0), "3");
534        assert_eq!(a.value(1), "5");
535    }
536}