1#[cfg(test)]
2use std::borrow::Cow;
3use std::path::PathBuf;
4use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
5use std::sync::Arc;
6use std::thread::{self, JoinHandle};
7
8use arrow::record_batch::RecordBatch;
9
10use crate::bounded::{BoundedExecutor, MemoryBudget};
11use crate::consumer::BatchConsumer;
12use crate::decoder::{RecordParser, Splitter};
13use crate::plan::ExecutionPlan;
14use crate::Result;
15
16struct ChannelConsumer {
21 sender: SyncSender<Result<RecordBatch>>,
22}
23
24impl BatchConsumer for ChannelConsumer {
25 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
26 let _ = self.sender.send(Ok(batch));
28 Ok(())
29 }
30}
31
32pub struct StreamingBatchIterator {
39 receiver: Receiver<Result<RecordBatch>>,
40 handle: Option<JoinHandle<Result<()>>>,
41 done: bool,
42}
43
44impl StreamingBatchIterator {
45 pub fn new<P, S>(
50 path: PathBuf,
51 splitter: S,
52 parser: P,
53 plan: Arc<ExecutionPlan>,
54 budget: MemoryBudget,
55 prefault: bool,
56 ) -> Self
57 where
58 P: RecordParser + Clone + Send + Sync + 'static,
59 S: Splitter + Clone + Send + Sync + 'static,
60 {
61 let (sender, receiver) = sync_channel(1);
62 let handle = thread::spawn(move || {
63 let executor = BoundedExecutor::new(budget);
64 let mut consumer = ChannelConsumer {
65 sender: sender.clone(),
66 };
67 let res = executor.run_stream(&path, &splitter, parser, plan, prefault, &mut consumer);
68 if let Err(e) = res {
70 let _ = sender.send(Err(e));
71 }
72 Ok(())
73 });
74
75 Self {
76 receiver,
77 handle: Some(handle),
78 done: false,
79 }
80 }
81
82 pub fn new_bytes<P, S>(
84 bytes: Vec<u8>,
85 splitter: S,
86 parser: P,
87 plan: Arc<ExecutionPlan>,
88 budget: MemoryBudget,
89 ) -> Self
90 where
91 P: RecordParser + Clone + Send + Sync + 'static,
92 S: Splitter + Clone + Send + Sync + 'static,
93 {
94 let (sender, receiver) = sync_channel(1);
95 let handle = thread::spawn(move || {
96 let executor = BoundedExecutor::new(budget);
97 let mut consumer = ChannelConsumer {
98 sender: sender.clone(),
99 };
100 let res = executor.run_bytes_stream(&bytes, &splitter, parser, plan, &mut consumer);
101 if let Err(e) = res {
102 let _ = sender.send(Err(e));
103 }
104 Ok(())
105 });
106
107 Self {
108 receiver,
109 handle: Some(handle),
110 done: false,
111 }
112 }
113}
114
115impl Iterator for StreamingBatchIterator {
116 type Item = Result<RecordBatch>;
117
118 fn next(&mut self) -> Option<Self::Item> {
119 if self.done {
120 return None;
121 }
122 match self.receiver.recv() {
123 Ok(res) => Some(res),
124 Err(_) => {
125 self.done = true;
127 if let Some(handle) = self.handle.take() {
128 if let Err(payload) = handle.join() {
130 let msg = if let Some(s) = payload.downcast_ref::<&str>() {
131 (*s).to_string()
132 } else if let Some(s) = payload.downcast_ref::<String>() {
133 s.clone()
134 } else {
135 "worker panicked".to_string()
136 };
137 return Some(Err(crate::Error::Parser(format!(
138 "streaming worker panicked: {msg}. \
139 This usually indicates a bug in the parser (e.g., returning \
140 Cow::Borrowed that outlives the input chunk, or an unwrap() \
141 on None/Err during parsing). Check your RecordParser::parse_chunk \
142 implementation for incorrect lifetime handling or missing error checks."
143 ))));
144 }
145 }
146 None
147 }
148 }
149 }
150}
151
152impl Drop for StreamingBatchIterator {
153 fn drop(&mut self) {
154 self.done = true;
157 }
161}
162
163#[cfg(test)]
164mod tests {
165 use super::*;
166 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
167 use crate::plan::ExecutionPlan;
168 use crate::value::Value;
169 use crate::Result;
170
171 #[derive(Clone, Debug, Default)]
172 struct LineSplitter;
173 impl Splitter for LineSplitter {
174 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
175 if from >= bytes.len() {
176 return None;
177 }
178 let start = if bytes[from] == b'\n' { from + 1 } else { from };
180 if start >= bytes.len() {
181 return None;
182 }
183 memchr::memchr(b'\n', &bytes[start..]).map(|rel| start + rel + 1)
184 }
185
186 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
187 if max_chunks <= 1 || bytes.is_empty() {
188 return vec![0, bytes.len()];
189 }
190 let mut points = vec![0usize];
191 for (i, &b) in bytes.iter().enumerate() {
192 if b == b'\n' {
193 let next = i + 1;
194 if next > 0 && points.len() < max_chunks {
195 points.push(next);
196 }
197 }
198 }
199 if *points.last().unwrap() != bytes.len() {
200 points.push(bytes.len());
201 }
202 points
203 }
204 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
205 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
206 (sample.len() / n).max(1)
207 }
208 }
209
210 #[derive(Clone, Debug, Default)]
211 struct LineParser;
212 impl RecordParser for LineParser {
213 fn validate(&self, bytes: &[u8]) -> Result<()> {
214 simdutf8::basic::from_utf8(bytes)?;
215 Ok(())
216 }
217 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
218 let text = std::str::from_utf8(bytes).unwrap();
219 for line in text.lines() {
220 if line.is_empty() {
221 continue;
222 }
223 sink.begin_row();
224 for token in line.split_whitespace() {
225 if let Some((k, v)) = token.split_once('=') {
226 sink.put_field(k, Value::Str(Cow::Borrowed(v)));
227 }
228 }
229 sink.end_row();
230 }
231 Ok(())
232 }
233 }
234
235 #[test]
236 fn test_streaming_bytes_single_row_batches() {
237 let data = b"A=1 B=2\nA=3 B=4\nA=5 B=6\n";
238 let budget = MemoryBudget::new(6);
239 let iter = StreamingBatchIterator::new_bytes(
240 data.to_vec(),
241 LineSplitter,
242 LineParser,
243 Arc::new(ExecutionPlan::new()),
244 budget,
245 );
246 let batches: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
247 let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
248 assert_eq!(rows, 3);
249 assert!(batches.len() >= 2);
251 }
252
253 #[test]
254 fn test_streaming_bytes_vs_bounded() {
255 let data = b"A=1 B=2\nA=3 B=4\n";
256 let budget = MemoryBudget::new(1024);
257 let expected = BoundedExecutor::new(budget)
258 .run_bytes(
259 data,
260 &LineSplitter,
261 LineParser,
262 Arc::new(ExecutionPlan::new()),
263 )
264 .unwrap();
265 let expected_rows: usize = expected.iter().map(|b| b.num_rows()).sum();
266
267 let iter = StreamingBatchIterator::new_bytes(
268 data.to_vec(),
269 LineSplitter,
270 LineParser,
271 Arc::new(ExecutionPlan::new()),
272 budget,
273 );
274 let streamed: Vec<_> = iter.collect::<Result<Vec<_>>>().unwrap();
275 let streamed_rows: usize = streamed.iter().map(|b| b.num_rows()).sum();
276 assert_eq!(streamed_rows, expected_rows);
277 }
278}