1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16struct ChannelConsumer {
21 sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26 let _ = self.sender.send(Ok(batch));
28 Ok(())
29 }
30}
31
32pub struct StreamingBatchIterator {
39 receiver: Receiver<Result<RecordBatch>>,
40 handle: Option<JoinHandle<Result<()>>>,
41 done: bool,
42}
43
44impl StreamingBatchIterator {
45 pub fn new<P, S>(
50 path: PathBuf,
51 splitter: S,
52 parser: P,
53 plan: Arc<ExecutionPlan>,
54 budget: MemoryBudget,
55 prefault: bool,
56 ) -> Self
57 where
58 P: RecordParser + Clone + Send + Sync + 'static,
59 S: Splitter + Clone + Send + Sync + 'static,
60 {
61 let (sender, receiver) = sync_channel(1);
62 let handle = thread::spawn(move || {
63 let executor = BoundedExecutor::new(budget);
64 let mut consumer = ChannelConsumer {
65 sender: sender.clone(),
66 };
67 let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
68 if let Err(e) = res {
70 let _ = sender.send(Err(e));
71 }
72 Ok(())
73 });
74
75 Self {
76 receiver,
77 handle: Some(handle),
78 done: false,
79 }
80 }
81
82 pub fn new_bytes<P, S>(
84 bytes: Vec<u8>,
85 splitter: S,
86 parser: P,
87 plan: Arc<ExecutionPlan>,
88 budget: MemoryBudget,
89 ) -> Self
90 where
91 P: RecordParser + Clone + Send + Sync + 'static,
92 S: Splitter + Clone + Send + Sync + 'static,
93 {
94 let (sender, receiver) = sync_channel(1);
95 let handle = thread::spawn(move || {
96 let executor = BoundedExecutor::new(budget);
97 let mut consumer = ChannelConsumer {
98 sender: sender.clone(),
99 };
100 let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
101 if let Err(e) = res {
102 let _ = sender.send(Err(e));
103 }
104 Ok(())
105 });
106
107 Self {
108 receiver,
109 handle: Some(handle),
110 done: false,
111 }
112 }
113}
114
115impl Iterator for StreamingBatchIterator {
116 type Item = Result<RecordBatch>;
117
118 fn next(&mut self) -> Option<Self::Item> {
119 if self.done {
120 return None;
121 }
122 match self.receiver.recv() {
123 Ok(res) => Some(res),
124 Err(_) => {
125 self.done = true;
127 if let Some(handle) = self.handle.take() {
128 if let Err(payload) = handle.join() {
130 let msg = if let Some(s) = payload.downcast_ref::<&str>() {
131 (*s).to_string()
132 } else if let Some(s) = payload.downcast_ref::<String>() {
133 s.clone()
134 } else {
135 "worker panicked".to_string()
136 };
137 return Some(Err(crate::Error::Merge(format!(
138 "streaming worker panicked: {msg}"
139 ))));
140 }
141 }
142 None
143 }
144 }
145 }
146}
147
148impl Drop for StreamingBatchIterator {
149 fn drop(&mut self) {
150 self.done = true;
153 }
157}
158
159#[cfg(test)]
160mod tests {
161 use super::*;
162 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
163 use crate::plan::ExecutionPlan;
164 use crate::value::Value;
165 use crate::Result;
166
167 #[derive(Clone, Debug, Default)]
168 struct LineSplitter;
169 impl Splitter for LineSplitter {
170 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
171 if from >= bytes.len() {
172 return None;
173 }
174 let start = if bytes[from] == b'\n' { from + 1 } else { from };
176 if start >= bytes.len() {
177 return None;
178 }
179 memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
180 }
181
182 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
183 if max_chunks <= 1 || bytes.is_empty() {
184 return vec![0, bytes.len()];
185 }
186 let mut points = vec![0usize];
187 for (i, &b) in bytes.iter().enumerate() {
188 if b == b'\n' {
189 let next = i + 1;
190 if next > 0 && points.len() < max_chunks {
191 points.push(next);
192 }
193 }
194 }
195 if *points.last().unwrap() != bytes.len() {
196 points.push(bytes.len());
197 }
198 points
199 }
200 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
201 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
202 (sample.len() / n).max(1)
203 }
204 }
205
206 #[derive(Clone, Debug, Default)]
207 struct LineParser;
208 impl RecordParser for LineParser {
209 fn validate(&self, bytes: &[u8]) -> Result<()> {
210 simdutf8::basic::from_utf8(bytes)?;
211 Ok(())
212 }
213 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
214 let text = std::str::from_utf8(bytes).unwrap();
215 for line in text.lines() {
216 if line.is_empty() {
217 continue;
218 }
219 sink.begin_row();
220 for token in line.split_whitespace() {
221 if let Some((k, v)) = token.split_once('=') {
222 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
223 }
224 }
225 sink.end_row();
226 }
227 Ok(())
228 }
229 }
230
231 #[test]
232 fn test_streaming_bytes_single_row_batches() {
233 let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
234 let budget = MemoryBudget::new(6);
235 let iter = StreamingBatchIterator::new_bytes(
236 data.to_vec(),
237 LineSplitter,
238 LineParser,
239 Arc::new(ExecutionPlan::new()),
240 budget,
241 );
242 let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
243 let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
244 assert_eq!(rows, 3);
245 assert!(batches.len() >= 2);
247 }
248
249 #[test]
250 fn test_streaming_bytes_vs_bounded() {
251 let data = b"A=1 B=2\nA=3 B=4\n";
252 let budget = MemoryBudget::new(1024);
253 let expected = BoundedExecutor::new(budget)
254 .run_bytes(
255 data,
256 &LineSplitter,
257 LineParser,
258 Arc::new(ExecutionPlan::new()),
259 )
260 .unwrap();
261 let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
262
263 let iter = StreamingBatchIterator::new_bytes(
264 data.to_vec(),
265 LineSplitter,
266 LineParser,
267 Arc::new(ExecutionPlan::new()),
268 budget,
269 );
270 let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
271 let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
272 assert_eq!(streamed_rows, expected_rows);
273 }
274}