Skip to main content

rypipe_core/
merge.rs

1use std::sync::Arc;
2
3use arrow::datatypes::{DataType, Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::columnar::unify_variants;
10use crate::engine::TableBuilder;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14impl TableBuilder {
15    /// Merge another builder's data into this one, consuming `other`.
16    ///
17    /// New columns discovered in `other` are null-padded for the existing rows
18    /// in `self`.  Columns present in `self` but absent from `other` are
19    /// null-padded for `other`'s rows.  Column order follows first-appearance
20    /// order across both builders.
21    ///
22    /// Columns that disagree on storage variant are reconciled with safe
23    /// promotions (`int64`→`float64`, `string`→`dictionary`); irreconcilable
24    /// conflicts return [`crate::Error::Merge`].
25    pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
26        self.normalize();
27        other.normalize();
28        // A strict-types violation recorded while building `other` must not be
29        // silently dropped by the merge.
30        if self.strict_error.is_none() {
31            self.strict_error = other.strict_error.take();
32        }
33        if self.unknown_error.is_none() {
34            self.unknown_error = other.unknown_error.take();
35        }
36        // Per-row hooks already fired in the chunk's thread; only the
37        // accepted/rejected totals carry over for `on_chunk_finished`.
38        self.rows_accepted += other.rows_accepted;
39        self.rows_rejected += other.rows_rejected;
40        let self_rows = self.row_count;
41        let other_rows = other.row_count;
42
43        // 1. Create columns from other that self doesn't have yet.
44        for name in &other.column_order.clone() {
45            if !self.field_index.contains_key(name) {
46                let est = self_rows + other.estimated_rows.max(64);
47                let col_type = other.plan.column_type(name);
48                let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
49                for _ in 0..self_rows {
50                    builder.push(None);
51                }
52                let idx = self.columns.len();
53                self.columns.push(builder);
54                self.field_index.insert(name.clone(), idx);
55                // Ensure row_dirty has enough words for new column
56                let needed = self.columns.len().div_ceil(64);
57                if self.row_dirty.len() < needed {
58                    self.row_dirty.push(0);
59                }
60                let order_idx = self.schema_insert_index(name);
61                self.column_order.insert(order_idx, name.clone());
62            }
63        }
64
65        // 2. Append other's values to all columns, null-pad missing ones.
66        // Clone column_order to avoid borrowing self while mutably borrowing columns.
67        let order_snapshot = self.column_order.clone();
68        for name in &order_snapshot {
69            if let Some(self_idx) = self.field_index.get(name).copied() {
70                let self_b = &mut self.columns[self_idx];
71                if let Some(mut other_b) = other.take_column(name) {
72                    let skey = self_b.variant_key();
73                    let okey = other_b.variant_key();
74                    if skey != okey {
75                        let target = unify_variants(skey, okey).ok_or_else(|| {
76                            crate::Error::Merge(format!(
77                                "column '{name}' has conflicting types across chunks \
78                                 ({skey} vs {okey}); provide explicit field_types"
79                            ))
80                        })?;
81                        self_b.promote_to_variant(target)?;
82                        other_b.promote_to_variant(target)?;
83                    }
84                    self_b.extend_owned(other_b)?;
85                } else {
86                    for _ in 0..other_rows {
87                        self_b.push(None);
88                    }
89                }
90            }
91        }
92
93        self.row_count = self_rows + other_rows;
94        Ok(())
95    }
96}
97
98/// Export per-chunk builders as `RecordBatch`es without merging them in
99/// columnar form.  Each builder becomes a batch; the resulting vector has
100/// chunked columns.  This skips the serial merge-then-re-copy that `extend`
101/// would do.
102///
103/// All batches share a single unified schema: column order follows
104/// first appearance, chunks missing a column are null-filled, and columns
105/// whose storage variants differ across chunks are promoted to their common
106/// type where safe (`int64`+`float64` → `float64`,
107/// `string`+`dictionary` → `dictionary`). Irreconcilable conflicts raise
108/// [`crate::Error::Merge`].
109///
110/// The `Compare` filter, if present, is applied to each batch.
111pub fn engines_to_record_batches(
112    mut engines: Vec<TableBuilder>,
113    plan: &ExecutionPlan,
114) -> Result<Vec<RecordBatch>> {
115    for e in engines.iter_mut() {
116        e.prepare_finish()?;
117    }
118    engines.retain(|e| e.row_count > 0);
119
120    // Unified column order + storage variant across chunks. Order follows
121    // first appearance; conflicting variants are folded through
122    // `unify_variants` instead of silently keeping the first sighting.
123    let mut order: Vec<String> = Vec::new();
124    let mut targets: HashMap<String, &'static str> = HashMap::default();
125    for e in &engines {
126        for name in &e.column_order {
127            let Some(b) = e.get_column(name.as_str()) else {
128                continue;
129            };
130            match targets.get_mut(name) {
131                None => {
132                    order.push(name.clone());
133                    targets.insert(name.clone(), b.variant_key());
134                }
135                Some(t) => {
136                    let key = b.variant_key();
137                    *t = unify_variants(t, key).ok_or_else(|| {
138                        crate::Error::Merge(format!(
139                            "column '{name}' has conflicting types across chunks \
140                             ({t} vs {key}); provide explicit field_types"
141                        ))
142                    })?;
143                }
144            }
145        }
146    }
147
148    if order.is_empty() {
149        return Ok(Vec::new());
150    }
151
152    // Promote every builder to the unified variant so all exported arrays
153    // share one Arrow type per column.
154    for e in engines.iter_mut() {
155        for name in &e.column_order.clone() {
156            if let Some(target) = targets.get(name).copied() {
157                if let Some(b) = e.get_column_mut(name) {
158                    b.promote_to_variant(target)?;
159                }
160            }
161        }
162    }
163
164    // With promotion complete, each column's Arrow datatype is deterministic
165    // from its unified variant; first sighting suffices.
166    let mut types: HashMap<String, DataType> = HashMap::default();
167    for e in &engines {
168        for name in &e.column_order {
169            types.entry(name.clone()).or_insert_with(|| {
170                e.get_column(name.as_str())
171                    .expect("column must exist after promotion")
172                    .arrow_datatype()
173            });
174        }
175    }
176
177    let fields: Vec<ArrowField> = order
178        .iter()
179        .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
180        .collect();
181    let schema = Arc::new(Schema::new(fields));
182
183    let batches: Result<Vec<RecordBatch>> = engines
184        .into_par_iter()
185        .map(|mut e| {
186            let mut arrays = Vec::with_capacity(order.len());
187            for name in &order {
188                match e.get_column_mut(name) {
189                    Some(b) => arrays.push(b.to_arrow_array()?),
190                    None => arrays.push(null_array(&types[name], e.row_count)),
191                }
192            }
193            RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
194        })
195        .collect();
196    let mut batches = batches?;
197
198    if let Some(ref filter) = plan.filter {
199        for batch in &mut batches {
200            *batch = apply_compare_filter(
201                std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
202                filter,
203            )?;
204        }
205    }
206
207    Ok(batches)
208}
209
210#[cfg(test)]
211mod tests {
212    use super::*;
213    use crate::{ColumnarSink, FieldType, Value};
214    use std::borrow::Cow;
215
216    /// Builder whose second row violates `strict_types` for `N: Int64`.
217    fn strict_builder_with_bad_value() -> TableBuilder {
218        let mut plan = ExecutionPlan::new();
219        plan.field_types.insert("N".to_string(), FieldType::Int64);
220        plan.strict_types = true;
221        let mut b = TableBuilder::with_plan(4, Arc::new(plan));
222        ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("1")));
223        ColumnarSink::end_row(&mut b);
224        ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("abc")));
225        ColumnarSink::end_row(&mut b);
226        b
227    }
228
229    #[test]
230    fn test_extend_propagates_strict_error() {
231        let mut merged = TableBuilder::with_plan(4, Arc::new(ExecutionPlan::new()));
232        merged.extend(strict_builder_with_bad_value()).unwrap();
233        let err = merged.finish().unwrap_err();
234        assert!(
235            err.to_string().contains("strict_types"),
236            "extend must preserve the chunk's strict violation: {err}"
237        );
238    }
239
240    #[test]
241    fn test_engines_to_record_batches_surfaces_strict_error() {
242        let err =
243            engines_to_record_batches(vec![strict_builder_with_bad_value()], &ExecutionPlan::new())
244                .unwrap_err();
245        assert!(
246            err.to_string().contains("strict_types"),
247            "fast-path export must surface the strict violation: {err}"
248        );
249    }
250}