use arrow::record_batch::RecordBatch;
use crate::bounded::MemoryBudget;
use crate::budget::{BudgetLedger, StreamStats};
use crate::consumer::BatchConsumer;
use crate::decoder::ColumnarSink;
use crate::engine::TableBuilder;
use crate::value::Value;
use crate::Result;
pub struct SpillSink<'a, C: BatchConsumer> {
engine: TableBuilder,
consumer: &'a mut C,
rows_per_batch: usize,
batch_target_bytes: usize,
soft_cap_bytes: usize,
oversize: bool,
budget: MemoryBudget,
ledger: &'a mut BudgetLedger,
stats: &'a mut StreamStats,
error: Option<crate::Error>,
}
impl<'a, C: BatchConsumer> SpillSink<'a, C> {
#[allow(clippy::too_many_arguments)]
pub fn new(
engine: TableBuilder,
consumer: &'a mut C,
rows_per_batch: usize,
batch_target_bytes: usize,
soft_cap_bytes: usize,
oversize: bool,
budget: MemoryBudget,
ledger: &'a mut BudgetLedger,
stats: &'a mut StreamStats,
) -> Self {
Self {
engine,
consumer,
rows_per_batch,
batch_target_bytes,
soft_cap_bytes,
oversize,
budget,
ledger,
stats,
error: None,
}
}
#[inline]
fn should_spill(&self) -> bool {
self.error.is_none()
&& (self.engine.num_rows() >= self.rows_per_batch
|| (!self.oversize && self.engine.capacity_bytes() >= self.soft_cap_bytes))
}
fn spill(&mut self) {
if self.error.is_some() || self.engine.num_rows() == 0 {
return;
}
if let Err(err) = self.spill_inner() {
self.error = Some(err);
}
}
fn spill_inner(&mut self) -> Result<()> {
let rows = self.engine.num_rows();
let mut part = self.engine.split_off(rows);
let batch = part.finish()?;
self.ledger.set_builders(self.engine.capacity_bytes());
let batch_bytes = batch.get_array_memory_size();
self.budget.share(4).check(batch_bytes)?;
if rows == 1 && batch_bytes > self.batch_target_bytes {
self.stats.oversize_batches += 1;
}
self.ledger.charge_in_flight(batch_bytes);
let res = self.consumer.consume(batch);
self.ledger.release_in_flight(batch_bytes);
res?;
self.stats.batches += 1;
self.stats.rows += rows;
Ok(())
}
pub fn finish_run(&mut self) -> Result<()> {
self.spill();
if let Some(err) = self.error.take() {
return Err(err);
}
if let Some(err) = self.engine.unknown_error.take() {
return Err(crate::Error::Merge(err));
}
if let Some(err) = self.engine.strict_error.take() {
return Err(err);
}
self.engine.check_memory_budget()?;
self.ledger.set_builders(self.engine.capacity_bytes());
Ok(())
}
}
impl<C: BatchConsumer> ColumnarSink for SpillSink<'_, C> {
#[inline]
fn begin_row(&mut self) {
self.engine.begin_row();
}
#[inline]
fn put_field(&mut self, name: &str, value: Value<'_>) {
self.engine.put_field(name, value);
}
#[inline]
fn end_row(&mut self) {
self.engine.end_row();
if self.should_spill() {
self.spill();
}
}
#[inline]
fn wants(&self, name: &str) -> bool {
self.engine.wants(name)
}
#[inline]
fn resolve<'r>(&'r self, name: &'r str) -> Option<&'r str> {
self.engine.resolve(name)
}
#[inline]
fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
self.engine.put_field_resolved(resolved_name, value);
}
#[inline]
fn put_field_at(&mut self, slot: u32, value: Value<'_>) {
self.engine.put_field_at(slot, value);
}
#[inline]
fn put_str_at(&mut self, slot: u32, value: &str) {
self.engine.put_str_at(slot, value);
}
#[inline]
fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
self.engine.resolve_and_put(name, value);
}
#[inline]
fn row_rejected(&self) -> bool {
self.engine.row_rejected()
}
#[inline]
fn should_stop(&self) -> bool {
self.error.is_some()
}
#[inline]
fn row_satisfied(&self) -> bool {
self.engine.row_satisfied()
}
#[inline]
fn wanted_mask(&self) -> u64 {
self.engine.wanted_mask()
}
#[inline]
fn expect_slot(&self, ordinal: u32) -> Option<(u32, &[u8])> {
self.engine.expect_slot(ordinal)
}
#[inline]
fn record_slot(&mut self, ordinal: u32, slot: u32, raw_name: &[u8]) {
self.engine.record_slot(ordinal, slot, raw_name);
}
#[inline]
fn layout_broken(&mut self, ordinal: u32) {
self.engine.layout_broken(ordinal);
}
#[inline]
fn reset_child_ordinal(&mut self) {
self.engine.reset_child_ordinal();
}
fn finish(&mut self) -> Result<RecordBatch> {
self.engine.finish()
}
}