Skip to main content

rypipe_core/
streaming.rs

1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16/// Channel-based consumer for the streaming iterator.
17///
18/// Sends `Result<RecordBatch>` via a bounded channel (capacity 1) to provide
19/// backpressure; the producer blocks until the consumer calls `next()`.
20struct ChannelConsumer {
21    sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26        // If the receiver is dropped (consumer gone), we stop.
27        self.sender
28            .send(Ok(batch))
29            .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
30    }
31}
32
33/// Streaming iterator that yields `RecordBatch`es through a bounded queue.
34/// The budget guides batch sizing; it does not cap process memory.
35///
36/// Created by `BoundedExecutor::stream` / `Pipeline::stream_batches`.
37/// Backed by a worker thread running `BoundedExecutor::run_stream` and a
38/// `sync_channel(1)` for backpressure; no accumulation into `Vec`.
39pub struct StreamingBatchIterator {
40    receiver: Receiver<Result<RecordBatch>>,
41    handle: Option<JoinHandle<Result<()>>>,
42    done: bool,
43}
44
45impl StreamingBatchIterator {
46    /// Create a streaming iterator for a file path.
47    ///
48    /// Spawns a worker thread that runs `BoundedExecutor::run_stream` with a
49    /// `ChannelConsumer`. The iterator yields batches as they are produced.
50    pub fn new<P, S>(
51        path: PathBuf,
52        splitter: S,
53        parser: P,
54        plan: Arc<ExecutionPlan>,
55        budget: MemoryBudget,
56        prefault: bool,
57    ) -> Self
58    where
59        P: RecordParser + Clone + Send + Sync + 'static,
60        S: Splitter + Clone + Send + Sync + 'static,
61    {
62        let (sender, receiver) = sync_channel(1);
63        let handle = thread::spawn(move || {
64            let executor = BoundedExecutor::new(budget.share(2));
65            let mut consumer = ChannelConsumer {
66                sender: sender.clone(),
67            };
68            let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
69            // If run_stream fails, send the error.
70            if let Err(e) = res {
71                let _ = sender.send(Err(e));
72            }
73            Ok(())
74        });
75
76        Self {
77            receiver,
78            handle: Some(handle),
79            done: false,
80        }
81    }
82
83    /// Create a streaming iterator for in-memory bytes.
84    pub fn new_bytes<P, S>(
85        bytes: Vec<u8>,
86        splitter: S,
87        parser: P,
88        plan: Arc<ExecutionPlan>,
89        budget: MemoryBudget,
90    ) -> Self
91    where
92        P: RecordParser + Clone + Send + Sync + 'static,
93        S: Splitter + Clone + Send + Sync + 'static,
94    {
95        let (sender, receiver) = sync_channel(1);
96        let handle = thread::spawn(move || {
97            let executor = BoundedExecutor::new(budget.share(2));
98            let mut consumer = ChannelConsumer {
99                sender: sender.clone(),
100            };
101            let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
102            if let Err(e) = res {
103                let _ = sender.send(Err(e));
104            }
105            Ok(())
106        });
107
108        Self {
109            receiver,
110            handle: Some(handle),
111            done: false,
112        }
113    }
114}
115
116impl Iterator for StreamingBatchIterator {
117    type Item = Result<RecordBatch>;
118
119    fn next(&mut self) -> Option<Self::Item> {
120        if self.done {
121            return None;
122        }
123        match self.receiver.recv() {
124            Ok(res) => Some(res),
125            Err(_) => {
126                // Channel closed; worker finished. Check for join errors.
127                self.done = true;
128                if let Some(handle) = self.handle.take() {
129                    // Propagate panics as errors.
130                    if let Err(payload) = handle.join() {
131                        let msg = if let Some(s) = payload.downcast_ref::<&str>() {
132                            (*s).to_string()
133                        } else if let Some(s) = payload.downcast_ref::<String>() {
134                            s.clone()
135                        } else {
136                            "worker panicked".to_string()
137                        };
138                        return Some(Err(crate::Error::Parser(format!(
139                            "streaming worker panicked: {msg}. \
140                             This usually indicates a bug in the parser (e.g., returning \
141                             Cow::Borrowed that outlives the input chunk, or an unwrap() \
142                             on None/Err during parsing). Check your RecordParser::parse_chunk \
143                             implementation for incorrect lifetime handling or missing error checks."
144                        ))));
145                    }
146                }
147                None
148            }
149        }
150    }
151}
152
153#[cfg(test)]
154mod tests {
155    use super::*;
156    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
157    use crate::plan::ExecutionPlan;
158    use crate::value::Value;
159    use crate::Result;
160    use std::sync::atomic::{AtomicUsize, Ordering};
161
162    #[test]
163    fn channel_consumer_reports_disconnect() {
164        let (sender, receiver) = sync_channel(1);
165        drop(receiver);
166        let mut consumer = ChannelConsumer { sender };
167        let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
168        assert!(consumer.consume(batch).is_err());
169    }
170
171    #[derive(Clone)]
172    struct CountingParser(Arc<AtomicUsize>);
173    impl RecordParser for CountingParser {
174        fn validate(&self, bytes: &[u8]) -> Result<()> {
175            LineParser.validate(bytes)
176        }
177        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
178            self.0.fetch_add(
179                bytes.iter().filter(|&&b| b == b'\n').count(),
180                Ordering::SeqCst,
181            );
182            LineParser.parse_chunk(bytes, sink)
183        }
184    }
185
186    #[test]
187    fn dropping_iterator_stops_worker_after_disconnect() {
188        let seen = Arc::new(AtomicUsize::new(0));
189        let data = b"A=1\n".repeat(500);
190        let mut iter = StreamingBatchIterator::new_bytes(
191            data,
192            LineSplitter,
193            CountingParser(Arc::clone(&seen)),
194            Arc::new(ExecutionPlan::new()),
195            MemoryBudget::new(64),
196        );
197        let worker = iter.handle.take().unwrap();
198        drop(iter);
199        worker.join().unwrap().unwrap();
200        assert!(seen.load(Ordering::SeqCst) < 500);
201    }
202
203    #[derive(Clone, Debug, Default)]
204    struct LineSplitter;
205    impl Splitter for LineSplitter {
206        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
207            if from >= bytes.len() {
208                return None;
209            }
210            // If we're at a newline, advance past it.
211            let start = if bytes[from] == b'\n' { from + 1 } else { from };
212            if start >= bytes.len() {
213                return None;
214            }
215            memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
216        }
217
218        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
219            if max_chunks <= 1 || bytes.is_empty() {
220                return vec![0, bytes.len()];
221            }
222            let mut points = vec![0usize];
223            for (i, &b) in bytes.iter().enumerate() {
224                if b == b'\n' {
225                    let next = i + 1;
226                    if next > 0 && points.len() < max_chunks {
227                        points.push(next);
228                    }
229                }
230            }
231            if *points.last().unwrap() != bytes.len() {
232                points.push(bytes.len());
233            }
234            points
235        }
236        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
237            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
238            (sample.len() / n).max(1)
239        }
240    }
241
242    #[derive(Clone, Debug, Default)]
243    struct LineParser;
244    impl RecordParser for LineParser {
245        fn validate(&self, bytes: &[u8]) -> Result<()> {
246            simdutf8::basic::from_utf8(bytes)?;
247            Ok(())
248        }
249        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
250            let text = std::str::from_utf8(bytes).unwrap();
251            for line in text.lines() {
252                if line.is_empty() {
253                    continue;
254                }
255                sink.begin_row();
256                for token in line.split_whitespace() {
257                    if let Some((k, v)) = token.split_once('=') {
258                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
259                    }
260                }
261                sink.end_row();
262            }
263            Ok(())
264        }
265    }
266
267    #[test]
268    fn test_streaming_bytes_single_row_batches() {
269        let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
270        let budget = MemoryBudget::new(6);
271        let iter = StreamingBatchIterator::new_bytes(
272            data.to_vec(),
273            LineSplitter,
274            LineParser,
275            Arc::new(ExecutionPlan::new()),
276            budget,
277        );
278        let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
279        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
280        assert_eq!(rows, 3);
281        // With 6B budget and ~6B/row, we expect 1 row per batch
282        assert!(batches.len() >= 2);
283    }
284
285    #[test]
286    fn test_streaming_bytes_vs_bounded() {
287        let data = b"A=1 B=2\nA=3 B=4\n";
288        let budget = MemoryBudget::new(1024);
289        let expected = BoundedExecutor::new(budget)
290            .run_bytes(
291                data,
292                &LineSplitter,
293                LineParser,
294                Arc::new(ExecutionPlan::new()),
295            )
296            .unwrap();
297        let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
298
299        let iter = StreamingBatchIterator::new_bytes(
300            data.to_vec(),
301            LineSplitter,
302            LineParser,
303            Arc::new(ExecutionPlan::new()),
304            budget,
305        );
306        let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
307        let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
308        assert_eq!(streamed_rows, expected_rows);
309    }
310
311    #[test]
312    fn test_run_bytes_stream_with_stats_counts_batches_rows_and_peak() {
313        let data = b"A=1 B=2\nA=3 B=4\n".repeat(200);
314        let budget = MemoryBudget::new(64 * 1024);
315        let executor = BoundedExecutor::new(budget);
316        let stats = executor
317            .run_bytes_stream_with_stats(
318                &data,
319                &LineSplitter,
320                LineParser,
321                Arc::new(ExecutionPlan::new()),
322                &mut crate::consumer::DiscardingConsumer,
323            )
324            .unwrap();
325        assert_eq!(stats.rows, 400);
326        assert!(stats.batches >= 2, "expected several batches: {stats:?}");
327        assert!(stats.peak_tracked_bytes > 0);
328        assert_eq!(stats.oversize_batches, 0);
329    }
330
331    #[test]
332    fn test_run_bytes_stream_with_stats_flags_oversize_single_record() {
333        // One record far larger than the batch target forces a lone
334        // oversize batch; the small records around it are not flagged.
335        let big = format!("K={}\n", "x".repeat(64 * 1024));
336        let data = format!("A=1\n{big}B=2\n");
337        let budget = MemoryBudget::new(1024 * 1024);
338        let executor = BoundedExecutor::new(budget);
339        let stats = executor
340            .run_bytes_stream_with_stats(
341                data.as_bytes(),
342                &LineSplitter,
343                LineParser,
344                Arc::new(ExecutionPlan::new()),
345                &mut crate::consumer::DiscardingConsumer,
346            )
347            .unwrap();
348        assert_eq!(stats.rows, 3);
349        assert_eq!(stats.oversize_batches, 1, "{stats:?}");
350    }
351}