Skip to main content

rypipe_core/
streaming.rs

1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16/// Channel-based consumer for the streaming iterator.
17///
18/// Sends `Result<RecordBatch>` via a bounded channel (capacity 1) to provide
19/// backpressure; the producer blocks until the consumer calls `next()`.
20struct ChannelConsumer {
21    sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26        // If the receiver is dropped (consumer gone), we stop.
27        self.sender
28            .send(Ok(batch))
29            .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
30    }
31}
32
33/// Streaming iterator that yields `RecordBatch`es through a bounded queue.
34/// The budget guides batch sizing; it does not cap process memory.
35///
36/// Created by `BoundedExecutor::stream` / `Pipeline::stream_batches`.
37/// Backed by a worker thread running `BoundedExecutor::run_stream` and a
38/// `sync_channel(1)` for backpressure; no accumulation into `Vec`.
39pub struct StreamingBatchIterator {
40    receiver: Receiver<Result<RecordBatch>>,
41    handle: Option<JoinHandle<Result<()>>>,
42    done: bool,
43}
44
45impl StreamingBatchIterator {
46    /// Create a streaming iterator for a file path.
47    ///
48    /// Spawns a worker thread that runs `BoundedExecutor::run_stream` with a
49    /// `ChannelConsumer`. The iterator yields batches as they are produced.
50    pub fn new<P, S>(
51        path: PathBuf,
52        splitter: S,
53        parser: P,
54        plan: Arc<ExecutionPlan>,
55        budget: MemoryBudget,
56        prefault: bool,
57    ) -> Self
58    where
59        P: RecordParser + Clone + Send + Sync + 'static,
60        S: Splitter + Clone + Send + Sync + 'static,
61    {
62        let (sender, receiver) = sync_channel(1);
63        let handle = thread::spawn(move || {
64            let executor = BoundedExecutor::new(budget.share(2));
65            let mut consumer = ChannelConsumer {
66                sender: sender.clone(),
67            };
68            let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
69            // If run_stream fails, send the error.
70            if let Err(e) = res {
71                let _ = sender.send(Err(e));
72            }
73            Ok(())
74        });
75
76        Self {
77            receiver,
78            handle: Some(handle),
79            done: false,
80        }
81    }
82
83    /// Create a streaming iterator for in-memory bytes.
84    pub fn new_bytes<P, S>(
85        bytes: Vec<u8>,
86        splitter: S,
87        parser: P,
88        plan: Arc<ExecutionPlan>,
89        budget: MemoryBudget,
90    ) -> Self
91    where
92        P: RecordParser + Clone + Send + Sync + 'static,
93        S: Splitter + Clone + Send + Sync + 'static,
94    {
95        let (sender, receiver) = sync_channel(1);
96        let handle = thread::spawn(move || {
97            let executor = BoundedExecutor::new(budget.share(2));
98            let mut consumer = ChannelConsumer {
99                sender: sender.clone(),
100            };
101            let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
102            if let Err(e) = res {
103                let _ = sender.send(Err(e));
104            }
105            Ok(())
106        });
107
108        Self {
109            receiver,
110            handle: Some(handle),
111            done: false,
112        }
113    }
114}
115
116impl Iterator for StreamingBatchIterator {
117    type Item = Result<RecordBatch>;
118
119    fn next(&mut self) -> Option<Self::Item> {
120        if self.done {
121            return None;
122        }
123        match self.receiver.recv() {
124            Ok(res) => Some(res),
125            Err(_) => {
126                // Channel closed; worker finished. Check for join errors.
127                self.done = true;
128                if let Some(handle) = self.handle.take() {
129                    // Propagate panics as errors.
130                    if let Err(payload) = handle.join() {
131                        let msg = if let Some(s) = payload.downcast_ref::<&str>() {
132                            (*s).to_string()
133                        } else if let Some(s) = payload.downcast_ref::<String>() {
134                            s.clone()
135                        } else {
136                            "worker panicked".to_string()
137                        };
138                        return Some(Err(crate::Error::Parser(format!(
139                            "streaming worker panicked: {msg}. \
140                             This usually indicates a bug in the parser (e.g., returning \
141                             Cow::Borrowed that outlives the input chunk, or an unwrap() \
142                             on None/Err during parsing). Check your RecordParser::parse_chunk \
143                             implementation for incorrect lifetime handling or missing error checks."
144                        ))));
145                    }
146                }
147                None
148            }
149        }
150    }
151}
152
153#[cfg(test)]
154mod tests {
155    use super::*;
156    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
157    use crate::plan::ExecutionPlan;
158    use crate::value::Value;
159    use crate::Result;
160    use std::sync::atomic::{AtomicUsize, Ordering};
161
162    #[test]
163    fn channel_consumer_reports_disconnect() {
164        let (sender, receiver) = sync_channel(1);
165        drop(receiver);
166        let mut consumer = ChannelConsumer { sender };
167        let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
168        assert!(consumer.consume(batch).is_err());
169    }
170
171    #[derive(Clone)]
172    struct CountingParser(Arc<AtomicUsize>);
173    impl RecordParser for CountingParser {
174        fn validate(&self, bytes: &[u8]) -> Result<()> {
175            LineParser.validate(bytes)
176        }
177        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
178            // Count a row only once it has actually been parsed, so an early
179            // `should_stop` (consumer disconnected) is observable.
180            let text = std::str::from_utf8(bytes).unwrap();
181            for line in text.lines() {
182                if sink.should_stop() {
183                    break;
184                }
185                if line.is_empty() {
186                    continue;
187                }
188                self.0.fetch_add(1, Ordering::SeqCst);
189                sink.begin_row();
190                for token in line.split_whitespace() {
191                    if let Some((k, v)) = token.split_once('=') {
192                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
193                    }
194                }
195                sink.end_row();
196            }
197            Ok(())
198        }
199    }
200
201    #[test]
202    fn dropping_iterator_stops_worker_after_disconnect() {
203        let seen = Arc::new(AtomicUsize::new(0));
204        let data = b"A=1\n".repeat(500);
205        let mut iter = StreamingBatchIterator::new_bytes(
206            data,
207            LineSplitter,
208            CountingParser(Arc::clone(&seen)),
209            Arc::new(ExecutionPlan::new()),
210            MemoryBudget::new(64),
211        );
212        let worker = iter.handle.take().unwrap();
213        drop(iter);
214        worker.join().unwrap().unwrap();
215        assert!(seen.load(Ordering::SeqCst) < 500);
216    }
217
218    #[derive(Clone, Debug, Default)]
219    struct LineSplitter;
220    impl Splitter for LineSplitter {
221        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
222            if from >= bytes.len() {
223                return None;
224            }
225            // If we're at a newline, advance past it.
226            let start = if bytes[from] == b'\n' { from + 1 } else { from };
227            if start >= bytes.len() {
228                return None;
229            }
230            memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
231        }
232
233        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
234            if max_chunks <= 1 || bytes.is_empty() {
235                return vec![0, bytes.len()];
236            }
237            let mut points = vec![0usize];
238            for (i, &b) in bytes.iter().enumerate() {
239                if b == b'\n' {
240                    let next = i + 1;
241                    if next > 0 && points.len() < max_chunks {
242                        points.push(next);
243                    }
244                }
245            }
246            if *points.last().unwrap() != bytes.len() {
247                points.push(bytes.len());
248            }
249            points
250        }
251        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
252            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
253            (sample.len() / n).max(1)
254        }
255    }
256
257    #[derive(Clone, Debug, Default)]
258    struct LineParser;
259    impl RecordParser for LineParser {
260        fn validate(&self, bytes: &[u8]) -> Result<()> {
261            simdutf8::basic::from_utf8(bytes)?;
262            Ok(())
263        }
264        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
265            let text = std::str::from_utf8(bytes).unwrap();
266            for line in text.lines() {
267                if sink.should_stop() {
268                    break;
269                }
270                if line.is_empty() {
271                    continue;
272                }
273                sink.begin_row();
274                for token in line.split_whitespace() {
275                    if let Some((k, v)) = token.split_once('=') {
276                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
277                    }
278                }
279                sink.end_row();
280            }
281            Ok(())
282        }
283    }
284
285    #[test]
286    fn test_streaming_bytes_single_row_batches() {
287        let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
288        let budget = MemoryBudget::new(6);
289        let iter = StreamingBatchIterator::new_bytes(
290            data.to_vec(),
291            LineSplitter,
292            LineParser,
293            Arc::new(ExecutionPlan::new()),
294            budget,
295        );
296        let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
297        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
298        assert_eq!(rows, 3);
299        // Budget (6B) cannot hold a row, so every batch is oversize;
300        // oversize rows are batched together (MIN_ROWS_OVERSIZE_BATCH)
301        // instead of emitting one batch per row.
302        assert!(
303            !batches.is_empty() && batches.len() <= 2,
304            "got {} batches",
305            batches.len()
306        );
307    }
308
309    #[test]
310    fn test_streaming_bytes_vs_bounded() {
311        let data = b"A=1 B=2\nA=3 B=4\n";
312        let budget = MemoryBudget::new(1024);
313        let expected = BoundedExecutor::new(budget)
314            .run_bytes(
315                data,
316                &LineSplitter,
317                LineParser,
318                Arc::new(ExecutionPlan::new()),
319            )
320            .unwrap();
321        let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
322
323        let iter = StreamingBatchIterator::new_bytes(
324            data.to_vec(),
325            LineSplitter,
326            LineParser,
327            Arc::new(ExecutionPlan::new()),
328            budget,
329        );
330        let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
331        let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
332        assert_eq!(streamed_rows, expected_rows);
333    }
334
335    #[test]
336    fn test_run_bytes_stream_with_stats_counts_batches_rows_and_peak() {
337        let data = b"A=1 B=2\nA=3 B=4\n".repeat(200);
338        let budget = MemoryBudget::new(64 * 1024);
339        let executor = BoundedExecutor::new(budget);
340        let stats = executor
341            .run_bytes_stream_with_stats(
342                &data,
343                &LineSplitter,
344                LineParser,
345                Arc::new(ExecutionPlan::new()),
346                &mut crate::consumer::DiscardingConsumer,
347            )
348            .unwrap();
349        assert_eq!(stats.rows, 400);
350        assert!(stats.batches >= 2, "expected several batches: {stats:?}");
351        assert!(stats.peak_tracked_bytes > 0);
352        assert_eq!(stats.oversize_batches, 0);
353    }
354
355    #[test]
356    fn test_run_bytes_stream_with_stats_flags_oversize_single_record() {
357        // One record far larger than the batch target forces a lone
358        // oversize batch; the small records around it are not flagged.
359        let big = format!("K={}\n", "x".repeat(64 * 1024));
360        let data = format!("A=1\n{big}B=2\n");
361        let budget = MemoryBudget::new(1024 * 1024);
362        let executor = BoundedExecutor::new(budget);
363        let stats = executor
364            .run_bytes_stream_with_stats(
365                data.as_bytes(),
366                &LineSplitter,
367                LineParser,
368                Arc::new(ExecutionPlan::new()),
369                &mut crate::consumer::DiscardingConsumer,
370            )
371            .unwrap();
372        assert_eq!(stats.rows, 3);
373        assert_eq!(stats.oversize_batches, 1, "{stats:?}");
374    }
375}