Skip to main content

rypipe_core/
consumer.rs

1use arrow::record_batch::RecordBatch;
2
3use crate::Result;
4
5/// Consumer for streaming `RecordBatch` output.
6///
7/// The engine calls `consume` once per batch and immediately drops the batch
8/// afterward. Peak memory is therefore `budget + batch` rather than
9/// `sum(all batches)`. See `BoundedExecutor::run_stream`.
10pub trait BatchConsumer {
11    fn consume(&mut self, batch: RecordBatch) -> Result<()>;
12}
13
14/// Collecting consumer that accumulates batches into a `Vec`.
15///
16/// Used to implement the legacy `run` / `run_bytes` methods via `run_stream`.
17pub struct CollectingConsumer(pub Vec<RecordBatch>);
18
19impl BatchConsumer for CollectingConsumer {
20    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
21        self.0.push(batch);
22        Ok(())
23    }
24}
25
26/// No-op consumer that drops batches immediately.
27///
28/// Useful for throughput benchmarks with constant memory.
29pub struct DiscardingConsumer;
30
31impl BatchConsumer for DiscardingConsumer {
32    fn consume(&mut self, _batch: RecordBatch) -> Result<()> {
33        Ok(())
34    }
35}