1use std::sync::Arc;
2
3use arrow::datatypes::{DataType, Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::columnar::unify_variants;
10use crate::engine::TableBuilder;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14impl TableBuilder {
15 pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
26 if self.strict_error.is_none() {
29 self.strict_error = other.strict_error.take();
30 }
31 self.rows_accepted += other.rows_accepted;
34 self.rows_rejected += other.rows_rejected;
35 let self_rows = self.row_count;
36 let other_rows = other.row_count;
37
38 for name in &other.column_order.clone() {
40 if !self.field_index.contains_key(name) {
41 let est = self_rows + other.estimated_rows.max(64);
42 let col_type = other.plan.column_type(name);
43 let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
44 for _ in 0..self_rows {
45 builder.push(None);
46 }
47 let idx = self.columns.len();
48 self.columns.push(builder);
49 self.field_index.insert(name.clone(), idx);
50 let needed = self.columns.len().div_ceil(64);
52 if self.row_dirty.len() < needed {
53 self.row_dirty.push(0);
54 }
55 let order_idx = self.schema_insert_index(name);
56 self.column_order.insert(order_idx, name.clone());
57 }
58 }
59
60 let order_snapshot = self.column_order.clone();
63 for name in &order_snapshot {
64 if let Some(self_idx) = self.field_index.get(name).copied() {
65 let self_b = &mut self.columns[self_idx];
66 if let Some(mut other_b) = other.take_column(name) {
67 let skey = self_b.variant_key();
68 let okey = other_b.variant_key();
69 if skey != okey {
70 let target = unify_variants(skey, okey).ok_or_else(|| {
71 crate::Error::Merge(format!(
72 "column '{name}' has conflicting types across chunks \
73 ({skey} vs {okey}); provide explicit field_types"
74 ))
75 })?;
76 self_b.promote_to_variant(target)?;
77 other_b.promote_to_variant(target)?;
78 }
79 self_b.extend_owned(other_b)?;
80 } else {
81 for _ in 0..other_rows {
82 self_b.push(None);
83 }
84 }
85 }
86 }
87
88 self.row_count = self_rows + other_rows;
89 Ok(())
90 }
91}
92
93pub fn engines_to_record_batches(
107 mut engines: Vec<TableBuilder>,
108 plan: &ExecutionPlan,
109) -> Result<Vec<RecordBatch>> {
110 for e in engines.iter_mut() {
111 if let Some(err) = e.unknown_error.take() {
114 return Err(crate::Error::Merge(err));
115 }
116 if let Some(err) = e.strict_error.take() {
117 return Err(err);
118 }
119 e.normalize();
120 }
121 engines.retain(|e| e.row_count > 0);
122
123 let mut order: Vec<String> = Vec::new();
127 let mut targets: HashMap<String, &'static str> = HashMap::default();
128 for e in &engines {
129 for name in &e.column_order {
130 let Some(b) = e.get_column(name.as_str()) else {
131 continue;
132 };
133 match targets.get_mut(name) {
134 None => {
135 order.push(name.clone());
136 targets.insert(name.clone(), b.variant_key());
137 }
138 Some(t) => {
139 let key = b.variant_key();
140 *t = unify_variants(t, key).ok_or_else(|| {
141 crate::Error::Merge(format!(
142 "column '{name}' has conflicting types across chunks \
143 ({t} vs {key}); provide explicit field_types"
144 ))
145 })?;
146 }
147 }
148 }
149 }
150
151 if order.is_empty() {
152 return Ok(Vec::new());
153 }
154
155 for e in engines.iter_mut() {
158 for name in &e.column_order.clone() {
159 if let Some(target) = targets.get(name).copied() {
160 if let Some(b) = e.get_column_mut(name) {
161 b.promote_to_variant(target)?;
162 }
163 }
164 }
165 }
166
167 let mut types: HashMap<String, DataType> = HashMap::default();
170 for e in &engines {
171 for name in &e.column_order {
172 types.entry(name.clone()).or_insert_with(|| {
173 e.get_column(name.as_str())
174 .expect("column must exist after promotion")
175 .arrow_datatype()
176 });
177 }
178 }
179
180 let fields: Vec<ArrowField> = order
181 .iter()
182 .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
183 .collect();
184 let schema = Arc::new(Schema::new(fields));
185
186 let batches: Result<Vec<RecordBatch>> = engines
187 .into_par_iter()
188 .map(|mut e| {
189 let mut arrays = Vec::with_capacity(order.len());
190 for name in &order {
191 match e.get_column_mut(name) {
192 Some(b) => arrays.push(b.to_arrow_array()?),
193 None => arrays.push(null_array(&types[name], e.row_count)),
194 }
195 }
196 RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
197 })
198 .collect();
199 let mut batches = batches?;
200
201 if let Some(ref filter) = plan.filter {
202 for batch in &mut batches {
203 *batch = apply_compare_filter(
204 std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
205 filter,
206 )?;
207 }
208 }
209
210 Ok(batches)
211}
212
213#[cfg(test)]
214mod tests {
215 use super::*;
216 use crate::{ColumnarSink, FieldType, Value};
217 use std::borrow::Cow;
218
219 fn strict_builder_with_bad_value() -> TableBuilder {
221 let mut plan = ExecutionPlan::new();
222 plan.field_types.insert("N".to_string(), FieldType::Int64);
223 plan.strict_types = true;
224 let mut b = TableBuilder::with_plan(4, Arc::new(plan));
225 ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("1")));
226 ColumnarSink::end_row(&mut b);
227 ColumnarSink::put_field(&mut b, "N", Value::Str(Cow::Borrowed("abc")));
228 ColumnarSink::end_row(&mut b);
229 b
230 }
231
232 #[test]
233 fn test_extend_propagates_strict_error() {
234 let mut merged = TableBuilder::with_plan(4, Arc::new(ExecutionPlan::new()));
235 merged.extend(strict_builder_with_bad_value()).unwrap();
236 let err = merged.finish().unwrap_err();
237 assert!(
238 err.to_string().contains("strict_types"),
239 "extend must preserve the chunk's strict violation: {err}"
240 );
241 }
242
243 #[test]
244 fn test_engines_to_record_batches_surfaces_strict_error() {
245 let err =
246 engines_to_record_batches(vec![strict_builder_with_bad_value()], &ExecutionPlan::new())
247 .unwrap_err();
248 assert!(
249 err.to_string().contains("strict_types"),
250 "fast-path export must surface the strict violation: {err}"
251 );
252 }
253}