1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16struct ChannelConsumer {
21 sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26 self.sender
28 .send(Ok(batch))
29 .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
30 }
31}
32
33pub struct StreamingBatchIterator {
40 receiver: Receiver<Result<RecordBatch>>,
41 handle: Option<JoinHandle<Result<()>>>,
42 done: bool,
43}
44
45impl StreamingBatchIterator {
46 pub fn new<P, S>(
51 path: PathBuf,
52 splitter: S,
53 parser: P,
54 plan: Arc<ExecutionPlan>,
55 budget: MemoryBudget,
56 prefault: bool,
57 ) -> Self
58 where
59 P: RecordParser + Clone + Send + Sync + 'static,
60 S: Splitter + Clone + Send + Sync + 'static,
61 {
62 let (sender, receiver) = sync_channel(1);
63 let handle = thread::spawn(move || {
64 let executor = BoundedExecutor::new(budget.share(2));
65 let mut consumer = ChannelConsumer {
66 sender: sender.clone(),
67 };
68 let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
69 if let Err(e) = res {
71 let _ = sender.send(Err(e));
72 }
73 Ok(())
74 });
75
76 Self {
77 receiver,
78 handle: Some(handle),
79 done: false,
80 }
81 }
82
83 pub fn new_bytes<P, S>(
85 bytes: Vec<u8>,
86 splitter: S,
87 parser: P,
88 plan: Arc<ExecutionPlan>,
89 budget: MemoryBudget,
90 ) -> Self
91 where
92 P: RecordParser + Clone + Send + Sync + 'static,
93 S: Splitter + Clone + Send + Sync + 'static,
94 {
95 let (sender, receiver) = sync_channel(1);
96 let handle = thread::spawn(move || {
97 let executor = BoundedExecutor::new(budget.share(2));
98 let mut consumer = ChannelConsumer {
99 sender: sender.clone(),
100 };
101 let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
102 if let Err(e) = res {
103 let _ = sender.send(Err(e));
104 }
105 Ok(())
106 });
107
108 Self {
109 receiver,
110 handle: Some(handle),
111 done: false,
112 }
113 }
114}
115
116impl Iterator for StreamingBatchIterator {
117 type Item = Result<RecordBatch>;
118
119 fn next(&mut self) -> Option<Self::Item> {
120 if self.done {
121 return None;
122 }
123 match self.receiver.recv() {
124 Ok(res) => Some(res),
125 Err(_) => {
126 self.done = true;
128 if let Some(handle) = self.handle.take() {
129 if let Err(payload) = handle.join() {
131 let msg = if let Some(s) = payload.downcast_ref::<&str>() {
132 (*s).to_string()
133 } else if let Some(s) = payload.downcast_ref::<String>() {
134 s.clone()
135 } else {
136 "worker panicked".to_string()
137 };
138 return Some(Err(crate::Error::Parser(format!(
139 "streaming worker panicked: {msg}. \
140 This usually indicates a bug in the parser (e.g., returning \
141 Cow::Borrowed that outlives the input chunk, or an unwrap() \
142 on None/Err during parsing). Check your RecordParser::parse_chunk \
143 implementation for incorrect lifetime handling or missing error checks."
144 ))));
145 }
146 }
147 None
148 }
149 }
150 }
151}
152
153#[cfg(test)]
154mod tests {
155 use super::*;
156 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
157 use crate::plan::ExecutionPlan;
158 use crate::value::Value;
159 use crate::Result;
160 use std::sync::atomic::{AtomicUsize, Ordering};
161
162 #[test]
163 fn channel_consumer_reports_disconnect() {
164 let (sender, receiver) = sync_channel(1);
165 drop(receiver);
166 let mut consumer = ChannelConsumer { sender };
167 let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
168 assert!(consumer.consume(batch).is_err());
169 }
170
171 #[derive(Clone)]
172 struct CountingParser(Arc<AtomicUsize>);
173 impl RecordParser for CountingParser {
174 fn validate(&self, bytes: &[u8]) -> Result<()> {
175 LineParser.validate(bytes)
176 }
177 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
178 let text = std::str::from_utf8(bytes).unwrap();
181 for line in text.lines() {
182 if sink.should_stop() {
183 break;
184 }
185 if line.is_empty() {
186 continue;
187 }
188 self.0.fetch_add(1, Ordering::SeqCst);
189 sink.begin_row();
190 for token in line.split_whitespace() {
191 if let Some((k, v)) = token.split_once('=') {
192 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
193 }
194 }
195 sink.end_row();
196 }
197 Ok(())
198 }
199 }
200
201 #[test]
202 fn dropping_iterator_stops_worker_after_disconnect() {
203 let seen = Arc::new(AtomicUsize::new(0));
204 let data = b"A=1\n".repeat(500);
205 let mut iter = StreamingBatchIterator::new_bytes(
206 data,
207 LineSplitter,
208 CountingParser(Arc::clone(&seen)),
209 Arc::new(ExecutionPlan::new()),
210 MemoryBudget::new(64),
211 );
212 let worker = iter.handle.take().unwrap();
213 drop(iter);
214 worker.join().unwrap().unwrap();
215 assert!(seen.load(Ordering::SeqCst) < 500);
216 }
217
218 #[derive(Clone, Debug, Default)]
219 struct LineSplitter;
220 impl Splitter for LineSplitter {
221 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
222 if from >= bytes.len() {
223 return None;
224 }
225 let start = if bytes[from] == b'\n' { from + 1 } else { from };
227 if start >= bytes.len() {
228 return None;
229 }
230 memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
231 }
232
233 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
234 if max_chunks <= 1 || bytes.is_empty() {
235 return vec![0, bytes.len()];
236 }
237 let mut points = vec![0usize];
238 for (i, &b) in bytes.iter().enumerate() {
239 if b == b'\n' {
240 let next = i + 1;
241 if next > 0 && points.len() < max_chunks {
242 points.push(next);
243 }
244 }
245 }
246 if *points.last().unwrap() != bytes.len() {
247 points.push(bytes.len());
248 }
249 points
250 }
251 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
252 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
253 (sample.len() / n).max(1)
254 }
255 }
256
257 #[derive(Clone, Debug, Default)]
258 struct LineParser;
259 impl RecordParser for LineParser {
260 fn validate(&self, bytes: &[u8]) -> Result<()> {
261 simdutf8::basic::from_utf8(bytes)?;
262 Ok(())
263 }
264 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
265 let text = std::str::from_utf8(bytes).unwrap();
266 for line in text.lines() {
267 if sink.should_stop() {
268 break;
269 }
270 if line.is_empty() {
271 continue;
272 }
273 sink.begin_row();
274 for token in line.split_whitespace() {
275 if let Some((k, v)) = token.split_once('=') {
276 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
277 }
278 }
279 sink.end_row();
280 }
281 Ok(())
282 }
283 }
284
285 #[test]
286 fn test_streaming_bytes_single_row_batches() {
287 let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
288 let budget = MemoryBudget::new(6);
289 let iter = StreamingBatchIterator::new_bytes(
290 data.to_vec(),
291 LineSplitter,
292 LineParser,
293 Arc::new(ExecutionPlan::new()),
294 budget,
295 );
296 let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
297 let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
298 assert_eq!(rows, 3);
299 assert!(
303 !batches.is_empty() && batches.len() <= 2,
304 "got {} batches",
305 batches.len()
306 );
307 }
308
309 #[test]
310 fn test_streaming_bytes_vs_bounded() {
311 let data = b"A=1 B=2\nA=3 B=4\n";
312 let budget = MemoryBudget::new(1024);
313 let expected = BoundedExecutor::new(budget)
314 .run_bytes(
315 data,
316 &LineSplitter,
317 LineParser,
318 Arc::new(ExecutionPlan::new()),
319 )
320 .unwrap();
321 let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
322
323 let iter = StreamingBatchIterator::new_bytes(
324 data.to_vec(),
325 LineSplitter,
326 LineParser,
327 Arc::new(ExecutionPlan::new()),
328 budget,
329 );
330 let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
331 let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
332 assert_eq!(streamed_rows, expected_rows);
333 }
334
335 #[test]
336 fn test_run_bytes_stream_with_stats_counts_batches_rows_and_peak() {
337 let data = b"A=1 B=2\nA=3 B=4\n".repeat(200);
338 let budget = MemoryBudget::new(64 * 1024);
339 let executor = BoundedExecutor::new(budget);
340 let stats = executor
341 .run_bytes_stream_with_stats(
342 &data,
343 &LineSplitter,
344 LineParser,
345 Arc::new(ExecutionPlan::new()),
346 &mut crate::consumer::DiscardingConsumer,
347 )
348 .unwrap();
349 assert_eq!(stats.rows, 400);
350 assert!(stats.batches >= 2, "expected several batches: {stats:?}");
351 assert!(stats.peak_tracked_bytes > 0);
352 assert_eq!(stats.oversize_batches, 0);
353 }
354
355 #[test]
356 fn test_run_bytes_stream_with_stats_flags_oversize_single_record() {
357 let big = format!("K={}\n", "x".repeat(64 * 1024));
360 let data = format!("A=1\n{big}B=2\n");
361 let budget = MemoryBudget::new(1024 * 1024);
362 let executor = BoundedExecutor::new(budget);
363 let stats = executor
364 .run_bytes_stream_with_stats(
365 data.as_bytes(),
366 &LineSplitter,
367 LineParser,
368 Arc::new(ExecutionPlan::new()),
369 &mut crate::consumer::DiscardingConsumer,
370 )
371 .unwrap();
372 assert_eq!(stats.rows, 3);
373 assert_eq!(stats.oversize_batches, 1, "{stats:?}");
374 }
375}