Skip to main content

rypipe_core/
consumer.rs

1use arrow::datatypes::{DataType, Schema};
2use arrow::record_batch::RecordBatch;
3
4use crate::Result;
5
6/// Consumer for streaming `RecordBatch` output.
7///
8/// The engine transfers ownership of each batch to `consume`. Consumers that
9/// drop batches avoid accumulating output; input and parser buffers still
10/// contribute to peak memory. See `BoundedExecutor::run_stream`.
11pub trait BatchConsumer {
12    fn consume(&mut self, batch: RecordBatch) -> Result<()>;
13}
14
15/// Collecting consumer that accumulates batches into a `Vec`.
16///
17/// Used to implement the legacy `run` / `run_bytes` methods via `run_stream`.
18pub struct CollectingConsumer(pub Vec<RecordBatch>);
19
20impl CollectingConsumer {
21    /// Align collected batches when columns first appear in later chunks.
22    pub fn finish(self) -> Result<Vec<RecordBatch>> {
23        let Some(first) = self.0.first() else {
24            return Ok(self.0);
25        };
26        if self.0.iter().all(|batch| batch.schema() == first.schema()) {
27            return Ok(self.0);
28        }
29        let mut encodings = std::collections::HashMap::<String, u8>::new();
30        for batch in &self.0 {
31            for field in batch.schema().fields() {
32                let encoding = match field.data_type() {
33                    DataType::Utf8 => 1,
34                    DataType::Dictionary(_, value) if **value == DataType::Utf8 => 2,
35                    _ => continue,
36                };
37                *encodings.entry(field.name().clone()).or_default() |= encoding;
38            }
39        }
40        let schema = std::sync::Arc::new(Schema::try_merge(self.0.iter().map(|batch| {
41            let schema = batch.schema();
42            let fields = schema
43                .fields()
44                .iter()
45                .map(|field| {
46                    if encodings.get(field.name()) == Some(&3) {
47                        field.as_ref().clone().with_data_type(DataType::Utf8)
48                    } else {
49                        field.as_ref().clone()
50                    }
51                })
52                .collect::<Vec<_>>();
53            Schema::new_with_metadata(fields, schema.metadata().clone())
54        }))?);
55        self.0
56            .into_iter()
57            .map(|batch| {
58                let columns = schema
59                    .fields()
60                    .iter()
61                    .map(|field| -> Result<_> {
62                        match batch.column_by_name(field.name()) {
63                            Some(column) if column.data_type() != field.data_type() => {
64                                Ok(arrow::compute::cast(column, field.data_type())?)
65                            }
66                            Some(column) => Ok(column.clone()),
67                            None => Ok(crate::arrow_export::null_array(
68                                field.data_type(),
69                                batch.num_rows(),
70                            )),
71                        }
72                    })
73                    .collect::<Result<Vec<_>>>()?;
74                Ok(RecordBatch::try_new_with_options(
75                    schema.clone(),
76                    columns,
77                    &arrow::record_batch::RecordBatchOptions::new()
78                        .with_row_count(Some(batch.num_rows())),
79                )?)
80            })
81            .collect()
82    }
83}
84
85impl BatchConsumer for CollectingConsumer {
86    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
87        self.0.push(batch);
88        Ok(())
89    }
90}
91
92/// No-op consumer that drops batches immediately.
93///
94/// Useful for throughput benchmarks with constant memory.
95pub struct DiscardingConsumer;
96
97impl BatchConsumer for DiscardingConsumer {
98    fn consume(&mut self, _batch: RecordBatch) -> Result<()> {
99        Ok(())
100    }
101}