rypipe-core 0.6.1

Format-agnostic ingestion framework with Arrow export
Documentation
#[cfg(test)]
use std::borrow::Cow;
use std::path::PathBuf;
use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
use std::sync::Arc;
use std::thread::{self, JoinHandle};

use arrow::record_batch::RecordBatch;

use crate::bounded::{BoundedExecutor, MemoryBudget};
use crate::consumer::BatchConsumer;
use crate::decoder::{RecordParser, Splitter};
use crate::plan::ExecutionPlan;
use crate::Result;

/// Channel-based consumer for the streaming iterator.
///
/// Sends `Result<RecordBatch>` via a bounded channel (capacity 1) to provide
/// backpressure; the producer blocks until the consumer calls `next()`.
struct ChannelConsumer {
    sender: SyncSender<Result<RecordBatch>>,
}

impl BatchConsumer for ChannelConsumer {
    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
        // If the receiver is dropped (consumer gone), we stop.
        self.sender
            .send(Ok(batch))
            .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
    }
}

/// Streaming iterator that yields `RecordBatch`es through a bounded queue.
/// The budget guides batch sizing; it does not cap process memory.
///
/// Created by `BoundedExecutor::stream` / `Pipeline::stream_batches`.
/// Backed by a worker thread running `BoundedExecutor::run_stream` and a
/// `sync_channel(1)` for backpressure; no accumulation into `Vec`.
pub struct StreamingBatchIterator {
    receiver: Receiver<Result<RecordBatch>>,
    handle: Option<JoinHandle<Result<()>>>,
    done: bool,
}

impl StreamingBatchIterator {
    /// Create a streaming iterator for a file path.
    ///
    /// Spawns a worker thread that runs `BoundedExecutor::run_stream` with a
    /// `ChannelConsumer`. The iterator yields batches as they are produced.
    pub fn new<P, S>(
        path: PathBuf,
        splitter: S,
        parser: P,
        plan: Arc<ExecutionPlan>,
        budget: MemoryBudget,
        prefault: bool,
    ) -> Self
    where
        P: RecordParser + Clone + Send + Sync + 'static,
        S: Splitter + Clone + Send + Sync + 'static,
    {
        let (sender, receiver) = sync_channel(1);
        let handle = thread::spawn(move || {
            let executor = BoundedExecutor::new(budget.share(2));
            let mut consumer = ChannelConsumer {
                sender: sender.clone(),
            };
            let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
            // If run_stream fails, send the error.
            if let Err(e) = res {
                let _ = sender.send(Err(e));
            }
            Ok(())
        });

        Self {
            receiver,
            handle: Some(handle),
            done: false,
        }
    }

    /// Create a streaming iterator for in-memory bytes.
    pub fn new_bytes<P, S>(
        bytes: Vec<u8>,
        splitter: S,
        parser: P,
        plan: Arc<ExecutionPlan>,
        budget: MemoryBudget,
    ) -> Self
    where
        P: RecordParser + Clone + Send + Sync + 'static,
        S: Splitter + Clone + Send + Sync + 'static,
    {
        let (sender, receiver) = sync_channel(1);
        let handle = thread::spawn(move || {
            let executor = BoundedExecutor::new(budget.share(2));
            let mut consumer = ChannelConsumer {
                sender: sender.clone(),
            };
            let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
            if let Err(e) = res {
                let _ = sender.send(Err(e));
            }
            Ok(())
        });

        Self {
            receiver,
            handle: Some(handle),
            done: false,
        }
    }
}

impl Iterator for StreamingBatchIterator {
    type Item = Result<RecordBatch>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.done {
            return None;
        }
        match self.receiver.recv() {
            Ok(res) => Some(res),
            Err(_) => {
                // Channel closed; worker finished. Check for join errors.
                self.done = true;
                if let Some(handle) = self.handle.take() {
                    // Propagate panics as errors.
                    if let Err(payload) = handle.join() {
                        let msg = if let Some(s) = payload.downcast_ref::<&str>() {
                            (*s).to_string()
                        } else if let Some(s) = payload.downcast_ref::<String>() {
                            s.clone()
                        } else {
                            "worker panicked".to_string()
                        };
                        return Some(Err(crate::Error::Parser(format!(
                            "streaming worker panicked: {msg}. \
                             This usually indicates a bug in the parser (e.g., returning \
                             Cow::Borrowed that outlives the input chunk, or an unwrap() \
                             on None/Err during parsing). Check your RecordParser::parse_chunk \
                             implementation for incorrect lifetime handling or missing error checks."
                        ))));
                    }
                }
                None
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
    use crate::plan::ExecutionPlan;
    use crate::value::Value;
    use crate::Result;
    use std::sync::atomic::{AtomicUsize, Ordering};

    #[test]
    fn channel_consumer_reports_disconnect() {
        let (sender, receiver) = sync_channel(1);
        drop(receiver);
        let mut consumer = ChannelConsumer { sender };
        let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
        assert!(consumer.consume(batch).is_err());
    }

    #[derive(Clone)]
    struct CountingParser(Arc<AtomicUsize>);
    impl RecordParser for CountingParser {
        fn validate(&self, bytes: &[u8]) -> Result<()> {
            LineParser.validate(bytes)
        }
        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
            // Count a row only once it has actually been parsed, so an early
            // `should_stop` (consumer disconnected) is observable.
            let text = std::str::from_utf8(bytes).unwrap();
            for line in text.lines() {
                if sink.should_stop() {
                    break;
                }
                if line.is_empty() {
                    continue;
                }
                self.0.fetch_add(1, Ordering::SeqCst);
                sink.begin_row();
                for token in line.split_whitespace() {
                    if let Some((k, v)) = token.split_once('=') {
                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
                    }
                }
                sink.end_row();
            }
            Ok(())
        }
    }

    #[test]
    fn dropping_iterator_stops_worker_after_disconnect() {
        let seen = Arc::new(AtomicUsize::new(0));
        let data = b"A=1\n".repeat(500);
        let mut iter = StreamingBatchIterator::new_bytes(
            data,
            LineSplitter,
            CountingParser(Arc::clone(&seen)),
            Arc::new(ExecutionPlan::new()),
            MemoryBudget::new(64),
        );
        let worker = iter.handle.take().unwrap();
        drop(iter);
        worker.join().unwrap().unwrap();
        assert!(seen.load(Ordering::SeqCst) < 500);
    }

    #[derive(Clone, Debug, Default)]
    struct LineSplitter;
    impl Splitter for LineSplitter {
        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
            if from >= bytes.len() {
                return None;
            }
            // If we're at a newline, advance past it.
            let start = if bytes[from] == b'\n' { from + 1 } else { from };
            if start >= bytes.len() {
                return None;
            }
            memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
        }

        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
            if max_chunks <= 1 || bytes.is_empty() {
                return vec![0, bytes.len()];
            }
            let mut points = vec![0usize];
            for (i, &b) in bytes.iter().enumerate() {
                if b == b'\n' {
                    let next = i + 1;
                    if next > 0 && points.len() < max_chunks {
                        points.push(next);
                    }
                }
            }
            if *points.last().unwrap() != bytes.len() {
                points.push(bytes.len());
            }
            points
        }
        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
            (sample.len() / n).max(1)
        }
    }

    #[derive(Clone, Debug, Default)]
    struct LineParser;
    impl RecordParser for LineParser {
        fn validate(&self, bytes: &[u8]) -> Result<()> {
            simdutf8::basic::from_utf8(bytes)?;
            Ok(())
        }
        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
            let text = std::str::from_utf8(bytes).unwrap();
            for line in text.lines() {
                if sink.should_stop() {
                    break;
                }
                if line.is_empty() {
                    continue;
                }
                sink.begin_row();
                for token in line.split_whitespace() {
                    if let Some((k, v)) = token.split_once('=') {
                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
                    }
                }
                sink.end_row();
            }
            Ok(())
        }
    }

    #[test]
    fn test_streaming_bytes_single_row_batches() {
        let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
        let budget = MemoryBudget::new(6);
        let iter = StreamingBatchIterator::new_bytes(
            data.to_vec(),
            LineSplitter,
            LineParser,
            Arc::new(ExecutionPlan::new()),
            budget,
        );
        let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
        assert_eq!(rows, 3);
        // Budget (6B) cannot hold a row, so every batch is oversize;
        // oversize rows are batched together (MIN_ROWS_OVERSIZE_BATCH)
        // instead of emitting one batch per row.
        assert!(
            !batches.is_empty() && batches.len() <= 2,
            "got {} batches",
            batches.len()
        );
    }

    #[test]
    fn test_streaming_bytes_vs_bounded() {
        let data = b"A=1 B=2\nA=3 B=4\n";
        let budget = MemoryBudget::new(1024);
        let expected = BoundedExecutor::new(budget)
            .run_bytes(
                data,
                &LineSplitter,
                LineParser,
                Arc::new(ExecutionPlan::new()),
            )
            .unwrap();
        let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();

        let iter = StreamingBatchIterator::new_bytes(
            data.to_vec(),
            LineSplitter,
            LineParser,
            Arc::new(ExecutionPlan::new()),
            budget,
        );
        let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
        let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
        assert_eq!(streamed_rows, expected_rows);
    }

    #[test]
    fn test_run_bytes_stream_with_stats_counts_batches_rows_and_peak() {
        let data = b"A=1 B=2\nA=3 B=4\n".repeat(200);
        let budget = MemoryBudget::new(64 * 1024);
        let executor = BoundedExecutor::new(budget);
        let stats = executor
            .run_bytes_stream_with_stats(
                &data,
                &LineSplitter,
                LineParser,
                Arc::new(ExecutionPlan::new()),
                &mut crate::consumer::DiscardingConsumer,
            )
            .unwrap();
        assert_eq!(stats.rows, 400);
        assert!(stats.batches >= 2, "expected several batches: {stats:?}");
        assert!(stats.peak_tracked_bytes > 0);
        assert_eq!(stats.oversize_batches, 0);
    }

    #[test]
    fn test_run_bytes_stream_with_stats_flags_oversize_single_record() {
        // One record far larger than the batch target forces a lone
        // oversize batch; the small records around it are not flagged.
        let big = format!("K={}\n", "x".repeat(64 * 1024));
        let data = format!("A=1\n{big}B=2\n");
        let budget = MemoryBudget::new(1024 * 1024);
        let executor = BoundedExecutor::new(budget);
        let stats = executor
            .run_bytes_stream_with_stats(
                data.as_bytes(),
                &LineSplitter,
                LineParser,
                Arc::new(ExecutionPlan::new()),
                &mut crate::consumer::DiscardingConsumer,
            )
            .unwrap();
        assert_eq!(stats.rows, 3);
        assert_eq!(stats.oversize_batches, 1, "{stats:?}");
    }
}