Skip to main content

rypipe_core/
merge.rs

1use std::sync::Arc;
2
3use arrow::datatypes::{DataType, Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::columnar::unify_variants;
10use crate::engine::TableBuilder;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14impl TableBuilder {
15    /// Merge another builder's data into this one, consuming `other`.
16    ///
17    /// New columns discovered in `other` are null-padded for the existing rows
18    /// in `self`.  Columns present in `self` but absent from `other` are
19    /// null-padded for `other`'s rows.  Column order follows first-appearance
20    /// order across both builders.
21    ///
22    /// Columns that disagree on storage variant are reconciled with safe
23    /// promotions (`int64`→`float64`, `string`→`dictionary`); irreconcilable
24    /// conflicts return [`crate::Error::Merge`].
25    pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
26        let self_rows = self.row_count;
27        let other_rows = other.row_count;
28
29        // 1. Create columns from other that self doesn't have yet.
30        for name in &other.column_order.clone() {
31            if !self.field_index.contains_key(name) {
32                let est = self_rows + other.estimated_rows.max(64);
33                let col_type = other.plan.column_type(name);
34                let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
35                for _ in 0..self_rows {
36                    builder.push(None);
37                }
38                let idx = self.columns.len();
39                self.columns.push(builder);
40                self.field_index.insert(name.clone(), idx);
41                // Ensure row_dirty has enough words for new column
42                let needed = self.columns.len().div_ceil(64);
43                if self.row_dirty.len() < needed {
44                    self.row_dirty.push(0);
45                }
46                let order_idx = self.schema_insert_index(name);
47                self.column_order.insert(order_idx, name.clone());
48            }
49        }
50
51        // 2. Append other's values to all columns, null-pad missing ones.
52        // Clone column_order to avoid borrowing self while mutably borrowing columns.
53        let order_snapshot = self.column_order.clone();
54        for name in &order_snapshot {
55            if let Some(self_idx) = self.field_index.get(name).copied() {
56                let self_b = &mut self.columns[self_idx];
57                if let Some(mut other_b) = other.take_column(name) {
58                    let skey = self_b.variant_key();
59                    let okey = other_b.variant_key();
60                    if skey != okey {
61                        let target = unify_variants(skey, okey).ok_or_else(|| {
62                            crate::Error::Merge(format!(
63                                "column '{name}' has conflicting types across chunks \
64                                 ({skey} vs {okey}); provide explicit field_types"
65                            ))
66                        })?;
67                        self_b.promote_to_variant(target)?;
68                        other_b.promote_to_variant(target)?;
69                    }
70                    self_b.extend_owned(other_b)?;
71                } else {
72                    for _ in 0..other_rows {
73                        self_b.push(None);
74                    }
75                }
76            }
77        }
78
79        self.row_count = self_rows + other_rows;
80        Ok(())
81    }
82}
83
84/// Export per-chunk builders as `RecordBatch`es without merging them in
85/// columnar form.  Each builder becomes a batch; the resulting vector has
86/// chunked columns.  This skips the serial merge-then-re-copy that `extend`
87/// would do.
88///
89/// All batches share a single unified schema: column order follows
90/// first appearance, chunks missing a column are null-filled, and columns
91/// whose storage variants differ across chunks are promoted to their common
92/// type where safe (`int64`+`float64` → `float64`,
93/// `string`+`dictionary` → `dictionary`). Irreconcilable conflicts raise
94/// [`crate::Error::Merge`].
95///
96/// The `Compare` filter, if present, is applied to each batch.
97pub fn engines_to_record_batches(
98    mut engines: Vec<TableBuilder>,
99    plan: &ExecutionPlan,
100) -> Result<Vec<RecordBatch>> {
101    for e in engines.iter_mut() {
102        e.normalize();
103    }
104    engines.retain(|e| e.row_count > 0);
105
106    // Unified column order + storage variant across chunks. Order follows
107    // first appearance; conflicting variants are folded through
108    // `unify_variants` instead of silently keeping the first sighting.
109    let mut order: Vec<String> = Vec::new();
110    let mut targets: HashMap<String, &'static str> = HashMap::default();
111    for e in &engines {
112        for name in &e.column_order {
113            let Some(b) = e.get_column(name.as_str()) else {
114                continue;
115            };
116            match targets.get_mut(name) {
117                None => {
118                    order.push(name.clone());
119                    targets.insert(name.clone(), b.variant_key());
120                }
121                Some(t) => {
122                    let key = b.variant_key();
123                    *t = unify_variants(t, key).ok_or_else(|| {
124                        crate::Error::Merge(format!(
125                            "column '{name}' has conflicting types across chunks \
126                             ({t} vs {key}); provide explicit field_types"
127                        ))
128                    })?;
129                }
130            }
131        }
132    }
133
134    if order.is_empty() {
135        return Ok(Vec::new());
136    }
137
138    // Promote every builder to the unified variant so all exported arrays
139    // share one Arrow type per column.
140    for e in engines.iter_mut() {
141        for name in &e.column_order.clone() {
142            if let Some(target) = targets.get(name).copied() {
143                if let Some(b) = e.get_column_mut(name) {
144                    b.promote_to_variant(target)?;
145                }
146            }
147        }
148    }
149
150    // With promotion complete, each column's Arrow datatype is deterministic
151    // from its unified variant; first sighting suffices.
152    let mut types: HashMap<String, DataType> = HashMap::default();
153    for e in &engines {
154        for name in &e.column_order {
155            types.entry(name.clone()).or_insert_with(|| {
156                e.get_column(name.as_str())
157                    .expect("column must exist after promotion")
158                    .arrow_datatype()
159            });
160        }
161    }
162
163    let fields: Vec<ArrowField> = order
164        .iter()
165        .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
166        .collect();
167    let schema = Arc::new(Schema::new(fields));
168
169    let batches: Result<Vec<RecordBatch>> = engines
170        .into_par_iter()
171        .map(|mut e| {
172            let mut arrays = Vec::with_capacity(order.len());
173            for name in &order {
174                match e.get_column_mut(name) {
175                    Some(b) => arrays.push(b.to_arrow_array()?),
176                    None => arrays.push(null_array(&types[name], e.row_count)),
177                }
178            }
179            RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
180        })
181        .collect();
182    let mut batches = batches?;
183
184    if let Some(ref filter) = plan.filter {
185        for batch in &mut batches {
186            *batch = apply_compare_filter(
187                std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
188                filter,
189            )?;
190        }
191    }
192
193    Ok(batches)
194}