Skip to main content

rypipe_core/
merge.rs

1use std::sync::Arc;
2
3use arrow::datatypes::{DataType, Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::columnar::unify_variants;
10use crate::engine::TableBuilder;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14impl TableBuilder {
15    /// Merge another builder's data into this one, consuming `other`.
16    ///
17    /// New columns discovered in `other` are null-padded for the existing rows
18    /// in `self`.  Columns present in `self` but absent from `other` are
19    /// null-padded for `other`'s rows.  Column order follows first-appearance
20    /// order across both builders.
21    ///
22    /// Columns that disagree on storage variant are reconciled with safe
23    /// promotions (`int64`→`float64`, `string`→`dictionary`); irreconcilable
24    /// conflicts return [`crate::Error::Merge`].
25    pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
26        // A strict-types violation recorded while building `other` must not be
27        // silently dropped by the merge.
28        if self.strict_error.is_none() {
29            self.strict_error = other.strict_error.take();
30        }
31        // Per-row hooks already fired in the chunk's thread; only the
32        // accepted/rejected totals carry over for `on_chunk_finished`.
33        self.rows_accepted += other.rows_accepted;
34        self.rows_rejected += other.rows_rejected;
35        let self_rows = self.row_count;
36        let other_rows = other.row_count;
37
38        // 1. Create columns from other that self doesn't have yet.
39        for name in &other.column_order.clone() {
40            if !self.field_index.contains_key(name) {
41                let est = self_rows + other.estimated_rows.max(64);
42                let col_type = other.plan.column_type(name);
43                let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
44                for _ in 0..self_rows {
45                    builder.push(None);
46                }
47                let idx = self.columns.len();
48                self.columns.push(builder);
49                self.field_index.insert(name.clone(), idx);
50                // Ensure row_dirty has enough words for new column
51                let needed = self.columns.len().div_ceil(64);
52                if self.row_dirty.len() < needed {
53                    self.row_dirty.push(0);
54                }
55                let order_idx = self.schema_insert_index(name);
56                self.column_order.insert(order_idx, name.clone());
57            }
58        }
59
60        // 2. Append other's values to all columns, null-pad missing ones.
61        // Clone column_order to avoid borrowing self while mutably borrowing columns.
62        let order_snapshot = self.column_order.clone();
63        for name in &order_snapshot {
64            if let Some(self_idx) = self.field_index.get(name).copied() {
65                let self_b = &mut self.columns[self_idx];
66                if let Some(mut other_b) = other.take_column(name) {
67                    let skey = self_b.variant_key();
68                    let okey = other_b.variant_key();
69                    if skey != okey {
70                        let target = unify_variants(skey, okey).ok_or_else(|| {
71                            crate::Error::Merge(format!(
72                                "column '{name}' has conflicting types across chunks \
73                                 ({skey} vs {okey}); provide explicit field_types"
74                            ))
75                        })?;
76                        self_b.promote_to_variant(target)?;
77                        other_b.promote_to_variant(target)?;
78                    }
79                    self_b.extend_owned(other_b)?;
80                } else {
81                    for _ in 0..other_rows {
82                        self_b.push(None);
83                    }
84                }
85            }
86        }
87
88        self.row_count = self_rows + other_rows;
89        Ok(())
90    }
91}
92
93/// Export per-chunk builders as `RecordBatch`es without merging them in
94/// columnar form.  Each builder becomes a batch; the resulting vector has
95/// chunked columns.  This skips the serial merge-then-re-copy that `extend`
96/// would do.
97///
98/// All batches share a single unified schema: column order follows
99/// first appearance, chunks missing a column are null-filled, and columns
100/// whose storage variants differ across chunks are promoted to their common
101/// type where safe (`int64`+`float64` → `float64`,
102/// `string`+`dictionary` → `dictionary`). Irreconcilable conflicts raise
103/// [`crate::Error::Merge`].
104///
105/// The `Compare` filter, if present, is applied to each batch.
106pub fn engines_to_record_batches(
107    mut engines: Vec<TableBuilder>,
108    plan: &ExecutionPlan,
109) -> Result<Vec<RecordBatch>> {
110    for e in engines.iter_mut() {
111        // `finish()` is bypassed here, so surface deferred per-chunk errors
112        // (strict-types violations, unknown-field errors) explicitly.
113        if let Some(err) = e.unknown_error.take() {
114            return Err(crate::Error::Merge(err));
115        }
116        if let Some(err) = e.strict_error.take() {
117            return Err(err);
118        }
119        e.normalize();
120    }
121    engines.retain(|e| e.row_count > 0);
122
123    // Unified column order + storage variant across chunks. Order follows
124    // first appearance; conflicting variants are folded through
125    // `unify_variants` instead of silently keeping the first sighting.
126    let mut order: Vec<String> = Vec::new();
127    let mut targets: HashMap<String, &'static str> = HashMap::default();
128    for e in &engines {
129        for name in &e.column_order {
130            let Some(b) = e.get_column(name.as_str()) else {
131                continue;
132            };
133            match targets.get_mut(name) {
134                None => {
135                    order.push(name.clone());
136                    targets.insert(name.clone(), b.variant_key());
137                }
138                Some(t) => {
139                    let key = b.variant_key();
140                    *t = unify_variants(t, key).ok_or_else(|| {
141                        crate::Error::Merge(format!(
142                            "column '{name}' has conflicting types across chunks \
143                             ({t} vs {key}); provide explicit field_types"
144                        ))
145                    })?;
146                }
147            }
148        }
149    }
150
151    if order.is_empty() {
152        return Ok(Vec::new());
153    }
154
155    // Promote every builder to the unified variant so all exported arrays
156    // share one Arrow type per column.
157    for e in engines.iter_mut() {
158        for name in &e.column_order.clone() {
159            if let Some(target) = targets.get(name).copied() {
160                if let Some(b) = e.get_column_mut(name) {
161                    b.promote_to_variant(target)?;
162                }
163            }
164        }
165    }
166
167    // With promotion complete, each column's Arrow datatype is deterministic
168    // from its unified variant; first sighting suffices.
169    let mut types: HashMap<String, DataType> = HashMap::default();
170    for e in &engines {
171        for name in &e.column_order {
172            types.entry(name.clone()).or_insert_with(|| {
173                e.get_column(name.as_str())
174                    .expect("column must exist after promotion")
175                    .arrow_datatype()
176            });
177        }
178    }
179
180    let fields: Vec<ArrowField> = order
181        .iter()
182        .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
183        .collect();
184    let schema = Arc::new(Schema::new(fields));
185
186    let batches: Result<Vec<RecordBatch>> = engines
187        .into_par_iter()
188        .map(|mut e| {
189            let mut arrays = Vec::with_capacity(order.len());
190            for name in &order {
191                match e.get_column_mut(name) {
192                    Some(b) => arrays.push(b.to_arrow_array()?),
193                    None => arrays.push(null_array(&types[name], e.row_count)),
194                }
195            }
196            RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
197        })
198        .collect();
199    let mut batches = batches?;
200
201    if let Some(ref filter) = plan.filter {
202        for batch in &mut batches {
203            *batch = apply_compare_filter(
204                std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
205                filter,
206            )?;
207        }
208    }
209
210    Ok(batches)
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216    use crate::{ColumnarSink, FieldType, Value};
217    use std::borrow::Cow;
218
219    /// Builder whose second row violates `strict_types` for `N: Int64`.
220    fn strict_builder_with_bad_value() -> TableBuilder {
221        let mut plan = ExecutionPlan::new();
222        plan.field_types.insert("N".to_string(), FieldType::Int64);
223        plan.strict_types = true;
224        let mut b = TableBuilder::with_plan(4, Arc::new(plan));
225        ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("1")));
226        ColumnarSink::end_row(&mut b);
227        ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("abc")));
228        ColumnarSink::end_row(&mut b);
229        b
230    }
231
232    #[test]
233    fn test_extend_propagates_strict_error() {
234        let mut merged = TableBuilder::with_plan(4, Arc::new(ExecutionPlan::new()));
235        merged.extend(strict_builder_with_bad_value()).unwrap();
236        let err = merged.finish().unwrap_err();
237        assert!(
238            err.to_string().contains("strict_types"),
239            "extend must preserve the chunk's strict violation: {err}"
240        );
241    }
242
243    #[test]
244    fn test_engines_to_record_batches_surfaces_strict_error() {
245        let err =
246            engines_to_record_batches(vec![strict_builder_with_bad_value()], &ExecutionPlan::new())
247                .unwrap_err();
248        assert!(
249            err.to_string().contains("strict_types"),
250            "fast-path export must surface the strict violation: {err}"
251        );
252    }
253}