1use std::sync::Arc;
2
3use arrow::datatypes::{Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rayon::prelude::*;
6use rustc_hash::FxHashMap as HashMap;
7
8use crate::arrow_export::{apply_compare_filter, null_array};
9use crate::engine::TableBuilder;
10use crate::plan::ExecutionPlan;
11use crate::Result;
12
13impl TableBuilder {
14 pub fn extend(&mut self, mut other: TableBuilder) -> Result<()> {
21 let self_rows = self.row_count;
22 let other_rows = other.row_count;
23
24 let have: std::collections::HashSet<String> = self.column_order.iter().cloned().collect();
26 for name in &other.column_order {
27 if !have.contains(name) {
28 let est = self_rows + other.estimated_rows.max(64);
29 let col_type = other.plan.column_type(name);
30 let mut builder = crate::columnar::ColumnBuilder::with_capacity(est, &col_type);
31 for _ in 0..self_rows {
32 builder.push(None);
33 }
34 self.columns.insert(name.clone(), builder);
35 let idx = self.schema_insert_index(name);
36 self.column_order.insert(idx, name.clone());
37 }
38 }
39
40 for name in &self.column_order {
42 if let Some(self_b) = self.columns.get_mut(name) {
43 if let Some(other_b) = other.columns.remove(name) {
44 self_b.extend_owned(other_b)?;
45 } else {
46 for _ in 0..other_rows {
47 self_b.push(None);
48 }
49 }
50 }
51 }
52
53 self.row_count = self_rows + other_rows;
54 Ok(())
55 }
56}
57
58pub fn engines_to_record_batches(
65 mut engines: Vec<TableBuilder>,
66 plan: &ExecutionPlan,
67) -> Result<Vec<RecordBatch>> {
68 for e in engines.iter_mut() {
69 e.normalize();
70 }
71 engines.retain(|e| e.row_count > 0);
72
73 let mut order: Vec<String> = Vec::new();
76 let mut types: HashMap<String, arrow::datatypes::DataType> = HashMap::default();
77 for e in &engines {
78 for name in &e.column_order {
79 if !types.contains_key(name) {
80 if let Some(b) = e.columns.get(name) {
81 types.insert(name.clone(), b.arrow_datatype());
82 order.push(name.clone());
83 }
84 }
85 }
86 }
87
88 if order.is_empty() {
89 return Ok(Vec::new());
90 }
91
92 let fields: Vec<ArrowField> = order
93 .iter()
94 .map(|n| ArrowField::new(n.as_str(), types[n].clone(), true))
95 .collect();
96 let schema = Arc::new(Schema::new(fields));
97
98 let batches: Result<Vec<RecordBatch>> = engines
99 .par_iter()
100 .map(|e| {
101 let mut arrays = Vec::with_capacity(order.len());
102 for name in &order {
103 match e.columns.get(name) {
104 Some(b) => arrays.push(b.to_arrow_array()?),
105 None => arrays.push(null_array(&types[name], e.row_count)),
106 }
107 }
108 RecordBatch::try_new(schema.clone(), arrays).map_err(crate::Error::Arrow)
109 })
110 .collect();
111 let mut batches = batches?;
112
113 if let Some(ref filter) = plan.filter {
114 for batch in &mut batches {
115 *batch = apply_compare_filter(
116 std::mem::replace(batch, RecordBatch::new_empty(schema.clone())),
117 filter,
118 )?;
119 }
120 }
121
122 Ok(batches)
123}