1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16struct ChannelConsumer {
21 sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26 self.sender
28 .send(Ok(batch))
29 .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
30 }
31}
32
33pub struct StreamingBatchIterator {
40 receiver: Receiver<Result<RecordBatch>>,
41 handle: Option<JoinHandle<Result<()>>>,
42 done: bool,
43}
44
45impl StreamingBatchIterator {
46 pub fn new<P, S>(
51 path: PathBuf,
52 splitter: S,
53 parser: P,
54 plan: Arc<ExecutionPlan>,
55 budget: MemoryBudget,
56 prefault: bool,
57 ) -> Self
58 where
59 P: RecordParser + Clone + Send + Sync + 'static,
60 S: Splitter + Clone + Send + Sync + 'static,
61 {
62 let (sender, receiver) = sync_channel(1);
63 let handle = thread::spawn(move || {
64 let executor = BoundedExecutor::new(budget.share(2));
65 let mut consumer = ChannelConsumer {
66 sender: sender.clone(),
67 };
68 let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
69 if let Err(e) = res {
71 let _ = sender.send(Err(e));
72 }
73 Ok(())
74 });
75
76 Self {
77 receiver,
78 handle: Some(handle),
79 done: false,
80 }
81 }
82
83 pub fn new_bytes<P, S>(
85 bytes: Vec<u8>,
86 splitter: S,
87 parser: P,
88 plan: Arc<ExecutionPlan>,
89 budget: MemoryBudget,
90 ) -> Self
91 where
92 P: RecordParser + Clone + Send + Sync + 'static,
93 S: Splitter + Clone + Send + Sync + 'static,
94 {
95 let (sender, receiver) = sync_channel(1);
96 let handle = thread::spawn(move || {
97 let executor = BoundedExecutor::new(budget.share(2));
98 let mut consumer = ChannelConsumer {
99 sender: sender.clone(),
100 };
101 let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
102 if let Err(e) = res {
103 let _ = sender.send(Err(e));
104 }
105 Ok(())
106 });
107
108 Self {
109 receiver,
110 handle: Some(handle),
111 done: false,
112 }
113 }
114}
115
116impl Iterator for StreamingBatchIterator {
117 type Item = Result<RecordBatch>;
118
119 fn next(&mut self) -> Option<Self::Item> {
120 if self.done {
121 return None;
122 }
123 match self.receiver.recv() {
124 Ok(res) => Some(res),
125 Err(_) => {
126 self.done = true;
128 if let Some(handle) = self.handle.take() {
129 if let Err(payload) = handle.join() {
131 let msg = if let Some(s) = payload.downcast_ref::<&str>() {
132 (*s).to_string()
133 } else if let Some(s) = payload.downcast_ref::<String>() {
134 s.clone()
135 } else {
136 "worker panicked".to_string()
137 };
138 return Some(Err(crate::Error::Parser(format!(
139 "streaming worker panicked: {msg}. \
140 This usually indicates a bug in the parser (e.g., returning \
141 Cow::Borrowed that outlives the input chunk, or an unwrap() \
142 on None/Err during parsing). Check your RecordParser::parse_chunk \
143 implementation for incorrect lifetime handling or missing error checks."
144 ))));
145 }
146 }
147 None
148 }
149 }
150 }
151}
152
153#[cfg(test)]
154mod tests {
155 use super::*;
156 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
157 use crate::plan::ExecutionPlan;
158 use crate::value::Value;
159 use crate::Result;
160 use std::sync::atomic::{AtomicUsize, Ordering};
161
162 #[test]
163 fn channel_consumer_reports_disconnect() {
164 let (sender, receiver) = sync_channel(1);
165 drop(receiver);
166 let mut consumer = ChannelConsumer { sender };
167 let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
168 assert!(consumer.consume(batch).is_err());
169 }
170
171 #[derive(Clone)]
172 struct CountingParser(Arc<AtomicUsize>);
173 impl RecordParser for CountingParser {
174 fn validate(&self, bytes: &[u8]) -> Result<()> {
175 LineParser.validate(bytes)
176 }
177 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
178 self.0.fetch_add(
179 bytes.iter().filter(|&&b| b == b'\n').count(),
180 Ordering::SeqCst,
181 );
182 LineParser.parse_chunk(bytes, sink)
183 }
184 }
185
186 #[test]
187 fn dropping_iterator_stops_worker_after_disconnect() {
188 let seen = Arc::new(AtomicUsize::new(0));
189 let data = b"A=1\n".repeat(500);
190 let mut iter = StreamingBatchIterator::new_bytes(
191 data,
192 LineSplitter,
193 CountingParser(Arc::clone(&seen)),
194 Arc::new(ExecutionPlan::new()),
195 MemoryBudget::new(64),
196 );
197 let worker = iter.handle.take().unwrap();
198 drop(iter);
199 worker.join().unwrap().unwrap();
200 assert!(seen.load(Ordering::SeqCst) < 500);
201 }
202
203 #[derive(Clone, Debug, Default)]
204 struct LineSplitter;
205 impl Splitter for LineSplitter {
206 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
207 if from >= bytes.len() {
208 return None;
209 }
210 let start = if bytes[from] == b'\n' { from + 1 } else { from };
212 if start >= bytes.len() {
213 return None;
214 }
215 memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
216 }
217
218 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
219 if max_chunks <= 1 || bytes.is_empty() {
220 return vec![0, bytes.len()];
221 }
222 let mut points = vec![0usize];
223 for (i, &b) in bytes.iter().enumerate() {
224 if b == b'\n' {
225 let next = i + 1;
226 if next > 0 && points.len() < max_chunks {
227 points.push(next);
228 }
229 }
230 }
231 if *points.last().unwrap() != bytes.len() {
232 points.push(bytes.len());
233 }
234 points
235 }
236 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
237 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
238 (sample.len() / n).max(1)
239 }
240 }
241
242 #[derive(Clone, Debug, Default)]
243 struct LineParser;
244 impl RecordParser for LineParser {
245 fn validate(&self, bytes: &[u8]) -> Result<()> {
246 simdutf8::basic::from_utf8(bytes)?;
247 Ok(())
248 }
249 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
250 let text = std::str::from_utf8(bytes).unwrap();
251 for line in text.lines() {
252 if line.is_empty() {
253 continue;
254 }
255 sink.begin_row();
256 for token in line.split_whitespace() {
257 if let Some((k, v)) = token.split_once('=') {
258 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
259 }
260 }
261 sink.end_row();
262 }
263 Ok(())
264 }
265 }
266
267 #[test]
268 fn test_streaming_bytes_single_row_batches() {
269 let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
270 let budget = MemoryBudget::new(6);
271 let iter = StreamingBatchIterator::new_bytes(
272 data.to_vec(),
273 LineSplitter,
274 LineParser,
275 Arc::new(ExecutionPlan::new()),
276 budget,
277 );
278 let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
279 let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
280 assert_eq!(rows, 3);
281 assert!(batches.len() >= 2);
283 }
284
285 #[test]
286 fn test_streaming_bytes_vs_bounded() {
287 let data = b"A=1 B=2\nA=3 B=4\n";
288 let budget = MemoryBudget::new(1024);
289 let expected = BoundedExecutor::new(budget)
290 .run_bytes(
291 data,
292 &LineSplitter,
293 LineParser,
294 Arc::new(ExecutionPlan::new()),
295 )
296 .unwrap();
297 let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
298
299 let iter = StreamingBatchIterator::new_bytes(
300 data.to_vec(),
301 LineSplitter,
302 LineParser,
303 Arc::new(ExecutionPlan::new()),
304 budget,
305 );
306 let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
307 let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
308 assert_eq!(streamed_rows, expected_rows);
309 }
310
311 #[test]
312 fn test_run_bytes_stream_with_stats_counts_batches_rows_and_peak() {
313 let data = b"A=1 B=2\nA=3 B=4\n".repeat(200);
314 let budget = MemoryBudget::new(64 * 1024);
315 let executor = BoundedExecutor::new(budget);
316 let stats = executor
317 .run_bytes_stream_with_stats(
318 &data,
319 &LineSplitter,
320 LineParser,
321 Arc::new(ExecutionPlan::new()),
322 &mut crate::consumer::DiscardingConsumer,
323 )
324 .unwrap();
325 assert_eq!(stats.rows, 400);
326 assert!(stats.batches >= 2, "expected several batches: {stats:?}");
327 assert!(stats.peak_tracked_bytes > 0);
328 assert_eq!(stats.oversize_batches, 0);
329 }
330
331 #[test]
332 fn test_run_bytes_stream_with_stats_flags_oversize_single_record() {
333 let big = format!("K={}\n", "x".repeat(64 * 1024));
336 let data = format!("A=1\n{big}B=2\n");
337 let budget = MemoryBudget::new(1024 * 1024);
338 let executor = BoundedExecutor::new(budget);
339 let stats = executor
340 .run_bytes_stream_with_stats(
341 data.as_bytes(),
342 &LineSplitter,
343 LineParser,
344 Arc::new(ExecutionPlan::new()),
345 &mut crate::consumer::DiscardingConsumer,
346 )
347 .unwrap();
348 assert_eq!(stats.rows, 3);
349 assert_eq!(stats.oversize_batches, 1, "{stats:?}");
350 }
351}