1use arrow::datatypes::{DataType, Schema};
2use arrow::record_batch::RecordBatch;
3
4use crate::Result;
5
6pub trait BatchConsumer {
12 fn consume(&mut self, batch: RecordBatch) -> Result<()>;
13}
14
15pub struct CollectingConsumer(pub Vec<RecordBatch>);
19
20impl CollectingConsumer {
21 pub fn finish(self) -> Result<Vec<RecordBatch>> {
23 let Some(first) = self.0.first() else {
24 return Ok(self.0);
25 };
26 if self.0.iter().all(|batch| batch.schema() == first.schema()) {
27 return Ok(self.0);
28 }
29 let mut encodings = std::collections::HashMap::<String, u8>::new();
30 for batch in &self.0 {
31 for field in batch.schema().fields() {
32 let encoding = match field.data_type() {
33 DataType::Utf8 => 1,
34 DataType::Dictionary(_, value) if **value == DataType::Utf8 => 2,
35 _ => continue,
36 };
37 *encodings.entry(field.name().clone()).or_default() |= encoding;
38 }
39 }
40 let schema = std::sync::Arc::new(Schema::try_merge(self.0.iter().map(|batch| {
41 let schema = batch.schema();
42 let fields = schema
43 .fields()
44 .iter()
45 .map(|field| {
46 if encodings.get(field.name()) == Some(&3) {
47 field.as_ref().clone().with_data_type(DataType::Utf8)
48 } else {
49 field.as_ref().clone()
50 }
51 })
52 .collect::<Vec<_>>();
53 Schema::new_with_metadata(fields, schema.metadata().clone())
54 }))?);
55 self.0
56 .into_iter()
57 .map(|batch| {
58 let columns = schema
59 .fields()
60 .iter()
61 .map(|field| -> Result<_> {
62 match batch.column_by_name(field.name()) {
63 Some(column) if column.data_type() != field.data_type() => {
64 Ok(arrow::compute::cast(column, field.data_type())?)
65 }
66 Some(column) => Ok(column.clone()),
67 None => Ok(crate::arrow_export::null_array(
68 field.data_type(),
69 batch.num_rows(),
70 )),
71 }
72 })
73 .collect::<Result<Vec<_>>>()?;
74 Ok(RecordBatch::try_new_with_options(
75 schema.clone(),
76 columns,
77 &arrow::record_batch::RecordBatchOptions::new()
78 .with_row_count(Some(batch.num_rows())),
79 )?)
80 })
81 .collect()
82 }
83}
84
85impl BatchConsumer for CollectingConsumer {
86 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
87 self.0.push(batch);
88 Ok(())
89 }
90}
91
92pub struct DiscardingConsumer;
96
97impl BatchConsumer for DiscardingConsumer {
98 fn consume(&mut self, _batch: RecordBatch) -> Result<()> {
99 Ok(())
100 }
101}