1use std::sync::Arc;
2
3use arrow::datatypes::{DataType, Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::columnar::unify_variants;
10use crate::engine::TableBuilder;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14impl TableBuilder {
15 pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
26 let self_rows = self.row_count;
27 let other_rows = other.row_count;
28
29 for name in &other.column_order.clone() {
31 if !self.field_index.contains_key(name) {
32 let est = self_rows + other.estimated_rows.max(64);
33 let col_type = other.plan.column_type(name);
34 let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
35 for _ in 0..self_rows {
36 builder.push(None);
37 }
38 let idx = self.columns.len();
39 self.columns.push(builder);
40 self.field_index.insert(name.clone(), idx);
41 let needed = self.columns.len().div_ceil(64);
43 if self.row_dirty.len() < needed {
44 self.row_dirty.push(0);
45 }
46 let order_idx = self.schema_insert_index(name);
47 self.column_order.insert(order_idx, name.clone());
48 }
49 }
50
51 let order_snapshot = self.column_order.clone();
54 for name in &order_snapshot {
55 if let Some(self_idx) = self.field_index.get(name).copied() {
56 let self_b = &mut self.columns[self_idx];
57 if let Some(mut other_b) = other.take_column(name) {
58 let skey = self_b.variant_key();
59 let okey = other_b.variant_key();
60 if skey != okey {
61 let target = unify_variants(skey, okey).ok_or_else(|| {
62 crate::Error::Merge(format!(
63 "column '{name}' has conflicting types across chunks \
64 ({skey} vs {okey}); provide explicit field_types"
65 ))
66 })?;
67 self_b.promote_to_variant(target)?;
68 other_b.promote_to_variant(target)?;
69 }
70 self_b.extend_owned(other_b)?;
71 } else {
72 for _ in 0..other_rows {
73 self_b.push(None);
74 }
75 }
76 }
77 }
78
79 self.row_count = self_rows + other_rows;
80 Ok(())
81 }
82}
83
84pub fn engines_to_record_batches(
98 mut engines: Vec<TableBuilder>,
99 plan: &ExecutionPlan,
100) -> Result<Vec<RecordBatch>> {
101 for e in engines.iter_mut() {
102 e.normalize();
103 }
104 engines.retain(|e| e.row_count > 0);
105
106 let mut order: Vec<String> = Vec::new();
110 let mut targets: HashMap<String, &'static str> = HashMap::default();
111 for e in &engines {
112 for name in &e.column_order {
113 let Some(b) = e.get_column(name.as_str()) else {
114 continue;
115 };
116 match targets.get_mut(name) {
117 None => {
118 order.push(name.clone());
119 targets.insert(name.clone(), b.variant_key());
120 }
121 Some(t) => {
122 let key = b.variant_key();
123 *t = unify_variants(t, key).ok_or_else(|| {
124 crate::Error::Merge(format!(
125 "column '{name}' has conflicting types across chunks \
126 ({t} vs {key}); provide explicit field_types"
127 ))
128 })?;
129 }
130 }
131 }
132 }
133
134 if order.is_empty() {
135 return Ok(Vec::new());
136 }
137
138 for e in engines.iter_mut() {
141 for name in &e.column_order.clone() {
142 if let Some(target) = targets.get(name).copied() {
143 if let Some(b) = e.get_column_mut(name) {
144 b.promote_to_variant(target)?;
145 }
146 }
147 }
148 }
149
150 let mut types: HashMap<String, DataType> = HashMap::default();
153 for e in &engines {
154 for name in &e.column_order {
155 types.entry(name.clone()).or_insert_with(|| {
156 e.get_column(name.as_str())
157 .expect("column must exist after promotion")
158 .arrow_datatype()
159 });
160 }
161 }
162
163 let fields: Vec<ArrowField> = order
164 .iter()
165 .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
166 .collect();
167 let schema = Arc::new(Schema::new(fields));
168
169 let batches: Result<Vec<RecordBatch>> = engines
170 .into_par_iter()
171 .map(|mut e| {
172 let mut arrays = Vec::with_capacity(order.len());
173 for name in &order {
174 match e.get_column_mut(name) {
175 Some(b) => arrays.push(b.to_arrow_array()?),
176 None => arrays.push(null_array(&types[name], e.row_count)),
177 }
178 }
179 RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
180 })
181 .collect();
182 let mut batches = batches?;
183
184 if let Some(ref filter) = plan.filter {
185 for batch in &mut batches {
186 *batch = apply_compare_filter(
187 std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
188 filter,
189 )?;
190 }
191 }
192
193 Ok(batches)
194}