Skip to main content

rypipe_core/
streaming.rs

1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16/// Channel-based consumer for the streaming iterator.
17///
18/// Sends `Result<RecordBatch>` via a bounded channel (capacity 1) to provide
19/// backpressure; the producer blocks until the consumer calls `next()`.
20struct ChannelConsumer {
21    sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26        // If the receiver is dropped (consumer gone), we stop.
27        let _ = self.sender.send(Ok(batch));
28        Ok(())
29    }
30}
31
32/// Streaming iterator that yields `RecordBatch`es one at a time, with
33/// constant memory bounded by `budget + batch`.
34///
35/// Created by `BoundedExecutor::stream` / `Pipeline::stream_batches`.
36/// Backed by a worker thread running `BoundedExecutor::run_stream` and a
37/// `sync_channel(1)` for backpressure; no accumulation into `Vec`.
38pub struct StreamingBatchIterator {
39    receiver: Receiver<Result<RecordBatch>>,
40    handle: Option<JoinHandle<Result<()>>>,
41    done: bool,
42}
43
44impl StreamingBatchIterator {
45    /// Create a streaming iterator for a file path.
46    ///
47    /// Spawns a worker thread that runs `BoundedExecutor::run_stream` with a
48    /// `ChannelConsumer`. The iterator yields batches as they are produced.
49    pub fn new<P, S>(
50        path: PathBuf,
51        splitter: S,
52        parser: P,
53        plan: Arc<ExecutionPlan>,
54        budget: MemoryBudget,
55        prefault: bool,
56    ) -> Self
57    where
58        P: RecordParser + Clone + Send + Sync + 'static,
59        S: Splitter + Clone + Send + Sync + 'static,
60    {
61        let (sender, receiver) = sync_channel(1);
62        let handle = thread::spawn(move || {
63            let executor = BoundedExecutor::new(budget);
64            let mut consumer = ChannelConsumer {
65                sender: sender.clone(),
66            };
67            let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
68            // If run_stream fails, send the error.
69            if let Err(e) = res {
70                let _ = sender.send(Err(e));
71            }
72            Ok(())
73        });
74
75        Self {
76            receiver,
77            handle: Some(handle),
78            done: false,
79        }
80    }
81
82    /// Create a streaming iterator for in-memory bytes.
83    pub fn new_bytes<P, S>(
84        bytes: Vec<u8>,
85        splitter: S,
86        parser: P,
87        plan: Arc<ExecutionPlan>,
88        budget: MemoryBudget,
89    ) -> Self
90    where
91        P: RecordParser + Clone + Send + Sync + 'static,
92        S: Splitter + Clone + Send + Sync + 'static,
93    {
94        let (sender, receiver) = sync_channel(1);
95        let handle = thread::spawn(move || {
96            let executor = BoundedExecutor::new(budget);
97            let mut consumer = ChannelConsumer {
98                sender: sender.clone(),
99            };
100            let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
101            if let Err(e) = res {
102                let _ = sender.send(Err(e));
103            }
104            Ok(())
105        });
106
107        Self {
108            receiver,
109            handle: Some(handle),
110            done: false,
111        }
112    }
113}
114
115impl Iterator for StreamingBatchIterator {
116    type Item = Result<RecordBatch>;
117
118    fn next(&mut self) -> Option<Self::Item> {
119        if self.done {
120            return None;
121        }
122        match self.receiver.recv() {
123            Ok(res) => Some(res),
124            Err(_) => {
125                // Channel closed; worker finished. Check for join errors.
126                self.done = true;
127                if let Some(handle) = self.handle.take() {
128                    // Propagate panics as errors.
129                    if let Err(payload) = handle.join() {
130                        let msg = if let Some(s) = payload.downcast_ref::<&str>() {
131                            (*s).to_string()
132                        } else if let Some(s) = payload.downcast_ref::<String>() {
133                            s.clone()
134                        } else {
135                            "worker panicked".to_string()
136                        };
137                        return Some(Err(crate::Error::Parser(format!(
138                            "streaming worker panicked: {msg}. \
139                             This usually indicates a bug in the parser (e.g., returning \
140                             Cow::Borrowed that outlives the input chunk, or an unwrap() \
141                             on None/Err during parsing). Check your RecordParser::parse_chunk \
142                             implementation for incorrect lifetime handling or missing error checks."
143                        ))));
144                    }
145                }
146                None
147            }
148        }
149    }
150}
151
152impl Drop for StreamingBatchIterator {
153    fn drop(&mut self) {
154        // Drop the receiver to unblock the worker if it's waiting on send.
155        // The worker will then exit when it tries to send.
156        self.done = true;
157        // Receiver is dropped here; handle's thread will be joined on drop
158        // via the Option<JoinHandle>; we don't block in Drop, just let it
159        // detach. The OS will clean up.
160    }
161}
162
163#[cfg(test)]
164mod tests {
165    use super::*;
166    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
167    use crate::plan::ExecutionPlan;
168    use crate::value::Value;
169    use crate::Result;
170
171    #[derive(Clone, Debug, Default)]
172    struct LineSplitter;
173    impl Splitter for LineSplitter {
174        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
175            if from >= bytes.len() {
176                return None;
177            }
178            // If we're at a newline, advance past it.
179            let start = if bytes[from] == b'\n' { from + 1 } else { from };
180            if start >= bytes.len() {
181                return None;
182            }
183            memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
184        }
185
186        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
187            if max_chunks <= 1 || bytes.is_empty() {
188                return vec![0, bytes.len()];
189            }
190            let mut points = vec![0usize];
191            for (i, &b) in bytes.iter().enumerate() {
192                if b == b'\n' {
193                    let next = i + 1;
194                    if next > 0 && points.len() < max_chunks {
195                        points.push(next);
196                    }
197                }
198            }
199            if *points.last().unwrap() != bytes.len() {
200                points.push(bytes.len());
201            }
202            points
203        }
204        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
205            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
206            (sample.len() / n).max(1)
207        }
208    }
209
210    #[derive(Clone, Debug, Default)]
211    struct LineParser;
212    impl RecordParser for LineParser {
213        fn validate(&self, bytes: &[u8]) -> Result<()> {
214            simdutf8::basic::from_utf8(bytes)?;
215            Ok(())
216        }
217        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
218            let text = std::str::from_utf8(bytes).unwrap();
219            for line in text.lines() {
220                if line.is_empty() {
221                    continue;
222                }
223                sink.begin_row();
224                for token in line.split_whitespace() {
225                    if let Some((k, v)) = token.split_once('=') {
226                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
227                    }
228                }
229                sink.end_row();
230            }
231            Ok(())
232        }
233    }
234
235    #[test]
236    fn test_streaming_bytes_single_row_batches() {
237        let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
238        let budget = MemoryBudget::new(6);
239        let iter = StreamingBatchIterator::new_bytes(
240            data.to_vec(),
241            LineSplitter,
242            LineParser,
243            Arc::new(ExecutionPlan::new()),
244            budget,
245        );
246        let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
247        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
248        assert_eq!(rows, 3);
249        // With 6B budget and ~6B/row, we expect 1 row per batch
250        assert!(batches.len() >= 2);
251    }
252
253    #[test]
254    fn test_streaming_bytes_vs_bounded() {
255        let data = b"A=1 B=2\nA=3 B=4\n";
256        let budget = MemoryBudget::new(1024);
257        let expected = BoundedExecutor::new(budget)
258            .run_bytes(
259                data,
260                &LineSplitter,
261                LineParser,
262                Arc::new(ExecutionPlan::new()),
263            )
264            .unwrap();
265        let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
266
267        let iter = StreamingBatchIterator::new_bytes(
268            data.to_vec(),
269            LineSplitter,
270            LineParser,
271            Arc::new(ExecutionPlan::new()),
272            budget,
273        );
274        let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
275        let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
276        assert_eq!(streamed_rows, expected_rows);
277    }
278}