Skip to main content

rypipe_core/
merge.rs

1use std::sync::Arc;
2
3use arrow::datatypes::{Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::engine::TableBuilder;
10use crate::plan::ExecutionPlan;
11use crate::Result;
12
13impl TableBuilder {
14    /// Merge another builder's data into this one, consuming `other`.
15    ///
16    /// New columns discovered in `other` are null-padded for the existing rows
17    /// in `self`.  Columns present in `self` but absent from `other` are
18    /// null-padded for `other`'s rows.  Column order follows first-appearance
19    /// order across both builders.
20    pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
21        let self_rows = self.row_count;
22        let other_rows = other.row_count;
23
24        // 1. Create columns from other that self doesn't have yet.
25        let have: std::collections::HashSet<String> = self.column_order.iter().cloned().collect();
26        for name in &other.column_order {
27            if !have.contains(name) {
28                let est = self_rows + other.estimated_rows.max(64);
29                let col_type = other.plan.column_type(name);
30                let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
31                for _ in 0..self_rows {
32                    builder.push(None);
33                }
34                self.columns.insert(name.clone(), builder);
35                let idx = self.schema_insert_index(name);
36                self.column_order.insert(idx, name.clone());
37            }
38        }
39
40        // 2. Append other's values to all columns, null-pad missing ones.
41        for name in &self.column_order {
42            if let Some(self_b) = self.columns.get_mut(name) {
43                if let Some(other_b) = other.columns.remove(name) {
44                    self_b.extend_owned(other_b)?;
45                } else {
46                    for _ in 0..other_rows {
47                        self_b.push(None);
48                    }
49                }
50            }
51        }
52
53        self.row_count = self_rows + other_rows;
54        Ok(())
55    }
56}
57
58/// Export per-chunk builders as `RecordBatch`es without merging them in
59/// columnar form.  Each builder becomes a batch; the resulting vector has
60/// chunked columns.  This skips the serial merge-then-re-copy that `extend`
61/// would do.
62///
63/// The `Compare` filter, if present, is applied to each batch.
64pub fn engines_to_record_batches(
65    mut engines: Vec<TableBuilder>,
66    plan: &ExecutionPlan,
67) -> Result<Vec<RecordBatch>> {
68    for e in engines.iter_mut() {
69        e.normalize();
70    }
71    engines.retain(|e| e.row_count > 0);
72
73    // Unified column order + datatypes across chunks. Types are deterministic
74    // from the plan, so first sighting wins.
75    let mut order: Vec<String> = Vec::new();
76    let mut types: HashMap<String, arrow::datatypes::DataType> = HashMap::default();
77    for e in &engines {
78        for name in &e.column_order {
79            if !types.contains_key(name) {
80                if let Some(b) = e.columns.get(name) {
81                    types.insert(name.clone(), b.arrow_datatype());
82                    order.push(name.clone());
83                }
84            }
85        }
86    }
87
88    if order.is_empty() {
89        return Ok(Vec::new());
90    }
91
92    let fields: Vec<ArrowField> = order
93        .iter()
94        .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
95        .collect();
96    let schema = Arc::new(Schema::new(fields));
97
98    let batches: Result<Vec<RecordBatch>> = engines
99        .par_iter()
100        .map(|e| {
101            let mut arrays = Vec::with_capacity(order.len());
102            for name in &order {
103                match e.columns.get(name) {
104                    Some(b) => arrays.push(b.to_arrow_array()?),
105                    None => arrays.push(null_array(&types[name], e.row_count)),
106                }
107            }
108            RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
109        })
110        .collect();
111    let mut batches = batches?;
112
113    if let Some(ref filter) = plan.filter {
114        for batch in &mut batches {
115            *batch = apply_compare_filter(
116                std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
117                filter,
118            )?;
119        }
120    }
121
122    Ok(batches)
123}