Skip to main content

rypipe_core/
streaming.rs

1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16/// Channel-based consumer for the streaming iterator.
17///
18/// Sends `Result<RecordBatch>` via a bounded channel (capacity 1) to provide
19/// backpressure; the producer blocks until the consumer calls `next()`.
20struct ChannelConsumer {
21    sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26        // If the receiver is dropped (consumer gone), we stop.
27        let _ = self.sender.send(Ok(batch));
28        Ok(())
29    }
30}
31
32/// Streaming iterator that yields `RecordBatch`es one at a time, with
33/// constant memory bounded by `budget + batch`.
34///
35/// Created by `BoundedExecutor::stream` / `Pipeline::stream_batches`.
36/// Backed by a worker thread running `BoundedExecutor::run_stream` and a
37/// `sync_channel(1)` for backpressure; no accumulation into `Vec`.
38pub struct StreamingBatchIterator {
39    receiver: Receiver<Result<RecordBatch>>,
40    handle: Option<JoinHandle<Result<()>>>,
41    done: bool,
42}
43
44impl StreamingBatchIterator {
45    /// Create a streaming iterator for a file path.
46    ///
47    /// Spawns a worker thread that runs `BoundedExecutor::run_stream` with a
48    /// `ChannelConsumer`. The iterator yields batches as they are produced.
49    pub fn new<P, S>(
50        path: PathBuf,
51        splitter: S,
52        parser: P,
53        plan: Arc<ExecutionPlan>,
54        budget: MemoryBudget,
55        prefault: bool,
56    ) -> Self
57    where
58        P: RecordParser + Clone + Send + Sync + 'static,
59        S: Splitter + Clone + Send + Sync + 'static,
60    {
61        let (sender, receiver) = sync_channel(1);
62        let handle = thread::spawn(move || {
63            let executor = BoundedExecutor::new(budget);
64            let mut consumer = ChannelConsumer {
65                sender: sender.clone(),
66            };
67            let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
68            // If run_stream fails, send the error.
69            if let Err(e) = res {
70                let _ = sender.send(Err(e));
71            }
72            Ok(())
73        });
74
75        Self {
76            receiver,
77            handle: Some(handle),
78            done: false,
79        }
80    }
81
82    /// Create a streaming iterator for in-memory bytes.
83    pub fn new_bytes<P, S>(
84        bytes: Vec<u8>,
85        splitter: S,
86        parser: P,
87        plan: Arc<ExecutionPlan>,
88        budget: MemoryBudget,
89    ) -> Self
90    where
91        P: RecordParser + Clone + Send + Sync + 'static,
92        S: Splitter + Clone + Send + Sync + 'static,
93    {
94        let (sender, receiver) = sync_channel(1);
95        let handle = thread::spawn(move || {
96            let executor = BoundedExecutor::new(budget);
97            let mut consumer = ChannelConsumer {
98                sender: sender.clone(),
99            };
100            let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
101            if let Err(e) = res {
102                let _ = sender.send(Err(e));
103            }
104            Ok(())
105        });
106
107        Self {
108            receiver,
109            handle: Some(handle),
110            done: false,
111        }
112    }
113}
114
115impl Iterator for StreamingBatchIterator {
116    type Item = Result<RecordBatch>;
117
118    fn next(&mut self) -> Option<Self::Item> {
119        if self.done {
120            return None;
121        }
122        match self.receiver.recv() {
123            Ok(res) => Some(res),
124            Err(_) => {
125                // Channel closed; worker finished. Check for join errors.
126                self.done = true;
127                if let Some(handle) = self.handle.take() {
128                    // Propagate panics as errors.
129                    if let Err(payload) = handle.join() {
130                        let msg = if let Some(s) = payload.downcast_ref::<&str>() {
131                            (*s).to_string()
132                        } else if let Some(s) = payload.downcast_ref::<String>() {
133                            s.clone()
134                        } else {
135                            "worker panicked".to_string()
136                        };
137                        return Some(Err(crate::Error::Merge(format!(
138                            "streaming worker panicked: {msg}"
139                        ))));
140                    }
141                }
142                None
143            }
144        }
145    }
146}
147
148impl Drop for StreamingBatchIterator {
149    fn drop(&mut self) {
150        // Drop the receiver to unblock the worker if it's waiting on send.
151        // The worker will then exit when it tries to send.
152        self.done = true;
153        // Receiver is dropped here; handle's thread will be joined on drop
154        // via the Option<JoinHandle>; we don't block in Drop, just let it
155        // detach. The OS will clean up.
156    }
157}
158
159#[cfg(test)]
160mod tests {
161    use super::*;
162    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
163    use crate::plan::ExecutionPlan;
164    use crate::value::Value;
165    use crate::Result;
166
167    #[derive(Clone, Debug, Default)]
168    struct LineSplitter;
169    impl Splitter for LineSplitter {
170        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
171            if from >= bytes.len() {
172                return None;
173            }
174            // If we're at a newline, advance past it.
175            let start = if bytes[from] == b'\n' { from + 1 } else { from };
176            if start >= bytes.len() {
177                return None;
178            }
179            memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
180        }
181
182        fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
183            if max_chunks <= 1 || bytes.is_empty() {
184                return vec![0, bytes.len()];
185            }
186            let mut points = vec![0usize];
187            for (i, &b) in bytes.iter().enumerate() {
188                if b == b'\n' {
189                    let next = i + 1;
190                    if next > 0 && points.len() < max_chunks {
191                        points.push(next);
192                    }
193                }
194            }
195            if *points.last().unwrap() != bytes.len() {
196                points.push(bytes.len());
197            }
198            points
199        }
200        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
201            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
202            (sample.len() / n).max(1)
203        }
204    }
205
206    #[derive(Clone, Debug, Default)]
207    struct LineParser;
208    impl RecordParser for LineParser {
209        fn validate(&self, bytes: &[u8]) -> Result<()> {
210            simdutf8::basic::from_utf8(bytes)?;
211            Ok(())
212        }
213        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
214            let text = std::str::from_utf8(bytes).unwrap();
215            for line in text.lines() {
216                if line.is_empty() {
217                    continue;
218                }
219                sink.begin_row();
220                for token in line.split_whitespace() {
221                    if let Some((k, v)) = token.split_once('=') {
222                        sink.put_field(k, Value::Str(Cow::Borrowed(v)));
223                    }
224                }
225                sink.end_row();
226            }
227            Ok(())
228        }
229    }
230
231    #[test]
232    fn test_streaming_bytes_single_row_batches() {
233        let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
234        let budget = MemoryBudget::new(6);
235        let iter = StreamingBatchIterator::new_bytes(
236            data.to_vec(),
237            LineSplitter,
238            LineParser,
239            Arc::new(ExecutionPlan::new()),
240            budget,
241        );
242        let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
243        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
244        assert_eq!(rows, 3);
245        // With 6B budget and ~6B/row, we expect 1 row per batch
246        assert!(batches.len() >= 2);
247    }
248
249    #[test]
250    fn test_streaming_bytes_vs_bounded() {
251        let data = b"A=1 B=2\nA=3 B=4\n";
252        let budget = MemoryBudget::new(1024);
253        let expected = BoundedExecutor::new(budget)
254            .run_bytes(
255                data,
256                &LineSplitter,
257                LineParser,
258                Arc::new(ExecutionPlan::new()),
259            )
260            .unwrap();
261        let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
262
263        let iter = StreamingBatchIterator::new_bytes(
264            data.to_vec(),
265            LineSplitter,
266            LineParser,
267            Arc::new(ExecutionPlan::new()),
268            budget,
269        );
270        let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
271        let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
272        assert_eq!(streamed_rows, expected_rows);
273    }
274}