#[cfg(feature = "mmap")]
use std::fs::File;
#[cfg(feature = "mmap")]
use std::io::{Read, Seek, SeekFrom};
use std::ops::Range;
use std::path::Path;
use std::sync::Arc;
use arrow::record_batch::RecordBatch;
use crate::arrow_export::apply_compare_filter;
use crate::consumer::CollectingConsumer;
use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
use crate::engine::TableBuilder;
use crate::input::InputBuffer;
use crate::plan::ExecutionPlan;
use crate::Result;
#[derive(Clone, Copy, Debug)]
pub struct MemoryBudget {
bytes: usize,
}
impl MemoryBudget {
pub fn new(bytes: usize) -> Self {
Self { bytes }
}
pub fn bytes(&self) -> usize {
self.bytes
}
}
const MAX_SPLIT_CHUNKS: usize = 100_000;
pub struct BoundedExecutor {
budget: MemoryBudget,
}
impl BoundedExecutor {
pub fn new(budget: MemoryBudget) -> Self {
Self { budget }
}
fn plan_chunks(
&self,
bytes: &[u8],
splitter: &dyn Splitter,
) -> (Vec<Range<usize>>, usize, usize) {
let bytes_per_row = splitter.estimate_bytes_per_row(bytes).max(1);
let total_rows_est = bytes.len() / bytes_per_row;
let rows_per_batch = (self.budget.bytes() / bytes_per_row)
.max(1)
.min(total_rows_est.max(1));
let num_batches = (total_rows_est / rows_per_batch).max(1);
let capped = num_batches.min(MAX_SPLIT_CHUNKS);
let split_points = splitter.find_split_points(bytes, capped);
let chunks = split_points_to_ranges(&split_points, bytes.len());
(chunks, rows_per_batch, bytes_per_row)
}
pub fn run_bytes_stream<P, C>(
&self,
bytes: &[u8],
splitter: &dyn Splitter,
parser: P,
plan: Arc<ExecutionPlan>,
consumer: &mut C,
) -> Result<()>
where
P: RecordParser + Clone + Send + Sync,
C: crate::consumer::BatchConsumer,
{
if bytes.is_empty() {
return Ok(());
}
let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
let mut rows_in_batch = 0usize;
for chunk in &chunks {
let chunk_bytes = &bytes[chunk.start..chunk.end];
let mut chunk_engine = TableBuilder::with_plan(
(chunk.len() / bytes_per_row.max(512)).max(64),
Arc::clone(&plan),
);
parser.validate(chunk_bytes)?;
parser.parse_chunk_generic(chunk_bytes, &mut chunk_engine)?;
let chunk_rows = chunk_engine.num_rows();
batch_engine.extend(chunk_engine)?;
rows_in_batch += chunk_rows;
while rows_in_batch >= rows_per_batch
|| batch_engine.bytes_used() >= self.budget.bytes()
{
if batch_engine.num_rows() == 0 {
break;
}
let n = rows_per_batch.min(batch_engine.num_rows());
let n = if batch_engine.bytes_used() >= self.budget.bytes()
&& batch_engine.num_rows() > 1
{
let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
/ batch_engine.bytes_used() as f64) as usize;
est.clamp(1, n)
} else {
n
};
let mut to_consume = batch_engine.split_off(n);
let mut batch = to_consume.finish()?;
if let Some(ref filter) = plan.filter {
batch = apply_compare_filter(batch, filter)?;
}
consumer.consume(batch)?;
rows_in_batch = rows_in_batch.saturating_sub(n);
}
}
if batch_engine.num_rows() > 0 {
let mut batch = batch_engine.finish()?;
if let Some(ref filter) = plan.filter {
batch = apply_compare_filter(batch, filter)?;
}
consumer.consume(batch)?;
}
Ok(())
}
pub fn run_bytes<P>(
&self,
bytes: &[u8],
splitter: &dyn Splitter,
parser: P,
plan: Arc<ExecutionPlan>,
) -> Result<Vec<RecordBatch>>
where
P: RecordParser + Clone + Send + Sync,
{
let batches = Vec::new();
let mut consumer = CollectingConsumer(batches);
self.run_bytes_stream(bytes, splitter, parser, plan, &mut consumer)?;
Ok(consumer.0)
}
pub fn run_stream<P, C>(
&self,
path: &Path,
splitter: &dyn Splitter,
parser: P,
plan: Arc<ExecutionPlan>,
prefault: bool,
consumer: &mut C,
) -> Result<()>
where
P: RecordParser + Clone + Send + Sync,
C: crate::consumer::BatchConsumer,
{
let use_mmap = cfg!(feature = "mmap");
let input = InputBuffer::open(path, use_mmap, prefault)?;
#[cfg(feature = "mmap")]
if matches!(input, InputBuffer::Mmap(_)) {
return self.run_mapped_stream(path, input, splitter, parser, plan, consumer);
}
self.run_bytes_stream(input.as_slice(), splitter, parser, plan, consumer)
}
pub fn run<P>(
&self,
path: &Path,
splitter: &dyn Splitter,
parser: P,
plan: Arc<ExecutionPlan>,
prefault: bool,
) -> Result<Vec<RecordBatch>>
where
P: RecordParser + Clone + Send + Sync,
{
let batches = Vec::new();
let mut consumer = CollectingConsumer(batches);
self.run_stream(path, splitter, parser, plan, prefault, &mut consumer)?;
Ok(consumer.0)
}
#[cfg(feature = "mmap")]
fn run_mapped_stream<P, C>(
&self,
path: &Path,
input: InputBuffer,
splitter: &dyn Splitter,
parser: P,
plan: Arc<ExecutionPlan>,
consumer: &mut C,
) -> Result<()>
where
P: RecordParser + Clone + Send + Sync,
C: crate::consumer::BatchConsumer,
{
let bytes = input.as_slice();
if bytes.is_empty() {
return Ok(());
}
let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
drop(input);
let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
let mut rows_in_batch = 0usize;
let mut file = File::open(path)?;
let max_chunk = chunks.iter().map(|r| r.len()).max().unwrap_or(0);
let mut chunk_buf = Vec::with_capacity(max_chunk);
for chunk in &chunks {
let chunk_len = chunk.len();
chunk_buf.resize(chunk_len, 0);
file.seek(SeekFrom::Start(chunk.start as u64))?;
file.read_exact(&mut chunk_buf)?;
let mut chunk_engine = TableBuilder::with_plan(
(chunk.len() / bytes_per_row.max(512)).max(64),
Arc::clone(&plan),
);
parser.validate(&chunk_buf)?;
parser.parse_chunk_generic(&chunk_buf, &mut chunk_engine)?;
let chunk_rows = chunk_engine.num_rows();
batch_engine.extend(chunk_engine)?;
rows_in_batch += chunk_rows;
while rows_in_batch >= rows_per_batch
|| batch_engine.bytes_used() >= self.budget.bytes()
{
if batch_engine.num_rows() == 0 {
break;
}
let n = rows_per_batch.min(batch_engine.num_rows());
let n = if batch_engine.bytes_used() >= self.budget.bytes()
&& batch_engine.num_rows() > 1
{
let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
/ batch_engine.bytes_used() as f64) as usize;
est.clamp(1, n)
} else {
n
};
let mut to_consume = batch_engine.split_off(n);
let mut batch = to_consume.finish()?;
if let Some(ref filter) = plan.filter {
batch = apply_compare_filter(batch, filter)?;
}
consumer.consume(batch)?;
rows_in_batch = rows_in_batch.saturating_sub(n);
}
}
if batch_engine.num_rows() > 0 {
let mut batch = batch_engine.finish()?;
if let Some(ref filter) = plan.filter {
batch = apply_compare_filter(batch, filter)?;
}
consumer.consume(batch)?;
}
Ok(())
}
#[cfg(feature = "mmap")]
#[expect(dead_code, reason = "legacy path kept for reference")]
fn run_mapped<P>(
&self,
path: &Path,
input: InputBuffer,
splitter: &dyn Splitter,
parser: P,
plan: Arc<ExecutionPlan>,
) -> Result<Vec<RecordBatch>>
where
P: RecordParser + Clone + Send + Sync,
{
let batches = Vec::new();
let mut consumer = CollectingConsumer(batches);
self.run_mapped_stream(path, input, splitter, parser, plan, &mut consumer)?;
Ok(consumer.0)
}
}
#[expect(dead_code, reason = "legacy path kept for reference")]
fn apply_plan_filter(batches: &mut Vec<RecordBatch>, plan: &ExecutionPlan) -> Result<()> {
if let Some(ref filter) = plan.filter {
for batch in batches {
*batch = apply_compare_filter(
std::mem::replace(batch, RecordBatch::new_empty(batch.schema())),
filter,
)?;
}
}
Ok(())
}