1use std::sync::Arc;
2
3use arrow::datatypes::{DataType, Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::columnar::unify_variants;
10use crate::engine::TableBuilder;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14impl TableBuilder {
15 pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
26 self.normalize();
27 other.normalize();
28 if self.strict_error.is_none() {
31 self.strict_error = other.strict_error.take();
32 }
33 if self.unknown_error.is_none() {
34 self.unknown_error = other.unknown_error.take();
35 }
36 self.rows_accepted += other.rows_accepted;
39 self.rows_rejected += other.rows_rejected;
40 let self_rows = self.row_count;
41 let other_rows = other.row_count;
42
43 for name in &other.column_order.clone() {
45 if !self.field_index.contains_key(name) {
46 let est = self_rows + other.estimated_rows.max(64);
47 let col_type = other.plan.column_type(name);
48 let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
49 for _ in 0..self_rows {
50 builder.push(None);
51 }
52 let idx = self.columns.len();
53 self.columns.push(builder);
54 self.field_index.insert(name.clone(), idx);
55 let needed = self.columns.len().div_ceil(64);
57 if self.row_dirty.len() < needed {
58 self.row_dirty.push(0);
59 }
60 let order_idx = self.schema_insert_index(name);
61 self.column_order.insert(order_idx, name.clone());
62 }
63 }
64
65 let order_snapshot = self.column_order.clone();
68 for name in &order_snapshot {
69 if let Some(self_idx) = self.field_index.get(name).copied() {
70 let self_b = &mut self.columns[self_idx];
71 if let Some(mut other_b) = other.take_column(name) {
72 let skey = self_b.variant_key();
73 let okey = other_b.variant_key();
74 if skey != okey {
75 let target = unify_variants(skey, okey).ok_or_else(|| {
76 crate::Error::Merge(format!(
77 "column '{name}' has conflicting types across chunks \
78 ({skey} vs {okey}); provide explicit field_types"
79 ))
80 })?;
81 self_b.promote_to_variant(target)?;
82 other_b.promote_to_variant(target)?;
83 }
84 self_b.extend_owned(other_b)?;
85 } else {
86 for _ in 0..other_rows {
87 self_b.push(None);
88 }
89 }
90 }
91 }
92
93 self.row_count = self_rows + other_rows;
94 Ok(())
95 }
96}
97
98pub fn engines_to_record_batches(
112 mut engines: Vec<TableBuilder>,
113 plan: &ExecutionPlan,
114) -> Result<Vec<RecordBatch>> {
115 for e in engines.iter_mut() {
116 e.prepare_finish()?;
117 }
118 engines.retain(|e| e.row_count > 0);
119
120 let mut order: Vec<String> = Vec::new();
124 let mut targets: HashMap<String, &'static str> = HashMap::default();
125 for e in &engines {
126 for name in &e.column_order {
127 let Some(b) = e.get_column(name.as_str()) else {
128 continue;
129 };
130 match targets.get_mut(name) {
131 None => {
132 order.push(name.clone());
133 targets.insert(name.clone(), b.variant_key());
134 }
135 Some(t) => {
136 let key = b.variant_key();
137 *t = unify_variants(t, key).ok_or_else(|| {
138 crate::Error::Merge(format!(
139 "column '{name}' has conflicting types across chunks \
140 ({t} vs {key}); provide explicit field_types"
141 ))
142 })?;
143 }
144 }
145 }
146 }
147
148 if order.is_empty() {
149 return Ok(Vec::new());
150 }
151
152 for e in engines.iter_mut() {
155 for name in &e.column_order.clone() {
156 if let Some(target) = targets.get(name).copied() {
157 if let Some(b) = e.get_column_mut(name) {
158 b.promote_to_variant(target)?;
159 }
160 }
161 }
162 }
163
164 let mut types: HashMap<String, DataType> = HashMap::default();
167 for e in &engines {
168 for name in &e.column_order {
169 types.entry(name.clone()).or_insert_with(|| {
170 e.get_column(name.as_str())
171 .expect("column must exist after promotion")
172 .arrow_datatype()
173 });
174 }
175 }
176
177 let fields: Vec<ArrowField> = order
178 .iter()
179 .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
180 .collect();
181 let schema = Arc::new(Schema::new(fields));
182
183 let batches: Result<Vec<RecordBatch>> = engines
184 .into_par_iter()
185 .map(|mut e| {
186 let mut arrays = Vec::with_capacity(order.len());
187 for name in &order {
188 match e.get_column_mut(name) {
189 Some(b) => arrays.push(b.to_arrow_array()?),
190 None => arrays.push(null_array(&types[name], e.row_count)),
191 }
192 }
193 RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
194 })
195 .collect();
196 let mut batches = batches?;
197
198 if let Some(ref filter) = plan.filter {
199 for batch in &mut batches {
200 *batch = apply_compare_filter(
201 std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
202 filter,
203 )?;
204 }
205 }
206
207 Ok(batches)
208}
209
210#[cfg(test)]
211mod tests {
212 use super::*;
213 use crate::{ColumnarSink, FieldType, Value};
214 use std::borrow::Cow;
215
216 fn strict_builder_with_bad_value() -> TableBuilder {
218 let mut plan = ExecutionPlan::new();
219 plan.field_types.insert("N".to_string(), FieldType::Int64);
220 plan.strict_types = true;
221 let mut b = TableBuilder::with_plan(4, Arc::new(plan));
222 ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("1")));
223 ColumnarSink::end_row(&mut b);
224 ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("abc")));
225 ColumnarSink::end_row(&mut b);
226 b
227 }
228
229 #[test]
230 fn test_extend_propagates_strict_error() {
231 let mut merged = TableBuilder::with_plan(4, Arc::new(ExecutionPlan::new()));
232 merged.extend(strict_builder_with_bad_value()).unwrap();
233 let err = merged.finish().unwrap_err();
234 assert!(
235 err.to_string().contains("strict_types"),
236 "extend must preserve the chunk's strict violation: {err}"
237 );
238 }
239
240 #[test]
241 fn test_engines_to_record_batches_surfaces_strict_error() {
242 let err =
243 engines_to_record_batches(vec![strict_builder_with_bad_value()], &ExecutionPlan::new())
244 .unwrap_err();
245 assert!(
246 err.to_string().contains("strict_types"),
247 "fast-path export must surface the strict violation: {err}"
248 );
249 }
250}