#[cfg(test)]
use std::borrow::Cow;
use std::path::PathBuf;
use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
use std::sync::Arc;
use std::thread::{self, JoinHandle};
use arrow::record_batch::RecordBatch;
use crate::bounded::{BoundedExecutor, MemoryBudget};
use crate::consumer::BatchConsumer;
use crate::decoder::{RecordParser, Splitter};
use crate::plan::ExecutionPlan;
use crate::Result;
struct ChannelConsumer {
sender: SyncSender<Result<RecordBatch>>,
}
impl BatchConsumer for ChannelConsumer {
fn consume(&mut self, batch: RecordBatch) -> Result<()> {
let _ = self.sender.send(Ok(batch));
Ok(())
}
}
pub struct StreamingBatchIterator {
receiver: Receiver<Result<RecordBatch>>,
handle: Option<JoinHandle<Result<()>>>,
done: bool,
}
impl StreamingBatchIterator {
pub fn new<P, S>(
path: PathBuf,
splitter: S,
parser: P,
plan: Arc<ExecutionPlan>,
budget: MemoryBudget,
prefault: bool,
) -> Self
where
P: RecordParser + Clone + Send + Sync + 'static,
S: Splitter + Clone + Send + Sync + 'static,
{
let (sender, receiver) = sync_channel(1);
let handle = thread::spawn(move || {
let executor = BoundedExecutor::new(budget);
let mut consumer = ChannelConsumer {
sender: sender.clone(),
};
let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
if let Err(e) = res {
let _ = sender.send(Err(e));
}
Ok(())
});
Self {
receiver,
handle: Some(handle),
done: false,
}
}
pub fn new_bytes<P, S>(
bytes: Vec<u8>,
splitter: S,
parser: P,
plan: Arc<ExecutionPlan>,
budget: MemoryBudget,
) -> Self
where
P: RecordParser + Clone + Send + Sync + 'static,
S: Splitter + Clone + Send + Sync + 'static,
{
let (sender, receiver) = sync_channel(1);
let handle = thread::spawn(move || {
let executor = BoundedExecutor::new(budget);
let mut consumer = ChannelConsumer {
sender: sender.clone(),
};
let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
if let Err(e) = res {
let _ = sender.send(Err(e));
}
Ok(())
});
Self {
receiver,
handle: Some(handle),
done: false,
}
}
}
impl Iterator for StreamingBatchIterator {
type Item = Result<RecordBatch>;
fn next(&mut self) -> Option<Self::Item> {
if self.done {
return None;
}
match self.receiver.recv() {
Ok(res) => Some(res),
Err(_) => {
self.done = true;
if let Some(handle) = self.handle.take() {
if let Err(payload) = handle.join() {
let msg = if let Some(s) = payload.downcast_ref::<&str>() {
(*s).to_string()
} else if let Some(s) = payload.downcast_ref::<String>() {
s.clone()
} else {
"worker panicked".to_string()
};
return Some(Err(crate::Error::Merge(format!(
"streaming worker panicked: {msg}"
))));
}
}
None
}
}
}
}
impl Drop for StreamingBatchIterator {
fn drop(&mut self) {
self.done = true;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::decoder::{ColumnarSink, RecordParser, Splitter};
use crate::plan::ExecutionPlan;
use crate::value::Value;
use crate::Result;
#[derive(Clone, Debug, Default)]
struct LineSplitter;
impl Splitter for LineSplitter {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
if from >= bytes.len() {
return None;
}
let start = if bytes[from] == b'\n' { from + 1 } else { from };
if start >= bytes.len() {
return None;
}
memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
}
fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
if max_chunks <= 1 || bytes.is_empty() {
return vec![0, bytes.len()];
}
let mut points = vec![0usize];
for (i, &b) in bytes.iter().enumerate() {
if b == b'\n' {
let next = i + 1;
if next > 0 && points.len() < max_chunks {
points.push(next);
}
}
}
if *points.last().unwrap() != bytes.len() {
points.push(bytes.len());
}
points
}
fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
(sample.len() / n).max(1)
}
}
#[derive(Clone, Debug, Default)]
struct LineParser;
impl RecordParser for LineParser {
fn validate(&self, bytes: &[u8]) -> Result<()> {
simdutf8::basic::from_utf8(bytes)?;
Ok(())
}
fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
let text = std::str::from_utf8(bytes).unwrap();
for line in text.lines() {
if line.is_empty() {
continue;
}
sink.begin_row();
for token in line.split_whitespace() {
if let Some((k, v)) = token.split_once('=') {
sink.put_field(k, Value::Str(Cow::Borrowed(v)));
}
}
sink.end_row();
}
Ok(())
}
}
#[test]
fn test_streaming_bytes_single_row_batches() {
let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
let budget = MemoryBudget::new(6);
let iter = StreamingBatchIterator::new_bytes(
data.to_vec(),
LineSplitter,
LineParser,
Arc::new(ExecutionPlan::new()),
budget,
);
let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(rows, 3);
assert!(batches.len() >= 2);
}
#[test]
fn test_streaming_bytes_vs_bounded() {
let data = b"A=1 B=2\nA=3 B=4\n";
let budget = MemoryBudget::new(1024);
let expected = BoundedExecutor::new(budget)
.run_bytes(
data,
&LineSplitter,
LineParser,
Arc::new(ExecutionPlan::new()),
)
.unwrap();
let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
let iter = StreamingBatchIterator::new_bytes(
data.to_vec(),
LineSplitter,
LineParser,
Arc::new(ExecutionPlan::new()),
budget,
);
let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
assert_eq!(streamed_rows, expected_rows);
}
}