1use std::collections::BTreeMap;
4use std::io::{Read, Seek, SeekFrom};
5use std::path::{Path, PathBuf};
6use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
7use std::sync::{Arc, Condvar, Mutex};
8use std::thread::{self, JoinHandle};
9
10use arrow::record_batch::RecordBatch;
11
12use std::sync::atomic::{AtomicU64, Ordering};
13
14use crate::bounded::MemoryBudget;
15use crate::consumer::BatchConsumer;
16use crate::decoder::{RecordParser, Splitter};
17use crate::engine::TableBuilder;
18use crate::input::InputBuffer;
19use crate::plan::ExecutionPlan;
20use crate::schema::{DiscoveryOpts, FrozenSchema};
21use crate::Result;
22
23mod discovery;
24
25pub(crate) use discovery::discover_schema;
26pub use discovery::{
27 discover_schema_for_bytes, discovery_profile, reset_discovery_profile, ParallelStreamOpts,
28};
29
30pub struct ParallelStreamingExecutor {
32 budget: MemoryBudget,
33 max_in_flight: usize,
34}
35
36type DispatchState = Arc<(Mutex<Option<usize>>, Condvar)>;
37
38struct DispatchGuard(DispatchState);
39
40impl Drop for DispatchGuard {
41 fn drop(&mut self) {
42 *self.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = None;
43 self.0 .1.notify_all();
44 }
45}
46
47impl ParallelStreamingExecutor {
48 pub fn new(budget: MemoryBudget, max_in_flight: usize) -> Self {
49 Self {
50 budget,
51 max_in_flight: max_in_flight.max(1),
52 }
53 }
54
55 #[allow(clippy::too_many_arguments)]
57 pub fn run_stream<P, C>(
58 &self,
59 path: &Path,
60 splitter: &dyn Splitter,
61 parser: P,
62 plan: Arc<ExecutionPlan>,
63 prefault: bool,
64 opts: ParallelStreamOpts,
65 consumer: &mut C,
66 ) -> Result<()>
67 where
68 P: RecordParser + Clone + Send + Sync + 'static,
69 C: BatchConsumer,
70 {
71 let input = InputBuffer::open(path, cfg!(feature = "mmap"), prefault)?;
72 if input.is_empty() {
73 return Ok(());
74 }
75 #[cfg(not(feature = "mmap"))]
76 self.budget.share(4).check(input.len())?;
77 #[cfg(feature = "mmap")]
78 let opts = if let InputBuffer::Mmap(handle) = &input {
79 let mut opts = opts;
80 if opts.schema.is_none() && plan.schema_order.is_empty() {
81 opts.schema = Some(discovery::discover_schema_for_file(
82 handle,
83 splitter,
84 &parser,
85 &plan,
86 self.budget,
87 )?);
88 }
89 opts
90 } else {
91 self.budget.share(4).check(input.len())?;
92 opts
93 };
94 let shared = std::sync::Arc::new(input);
97 self.run_bytes_stream_shared(shared, splitter, parser, plan, opts, consumer)
98 }
99
100 fn run_bytes_stream_shared<P, C>(
102 &self,
103 input: std::sync::Arc<InputBuffer>,
104 splitter: &dyn Splitter,
105 parser: P,
106 plan: Arc<ExecutionPlan>,
107 opts: ParallelStreamOpts,
108 consumer: &mut C,
109 ) -> Result<()>
110 where
111 P: RecordParser + Clone + Send + Sync + 'static,
112 C: BatchConsumer,
113 {
114 let dummy: &[u8] = &[];
118 self.run_bytes_stream_core(dummy, Some(input), splitter, parser, plan, opts, consumer)
119 }
120
121 pub fn run_bytes_stream<P, C>(
123 &self,
124 bytes: &[u8],
125 splitter: &dyn Splitter,
126 parser: P,
127 plan: Arc<ExecutionPlan>,
128 opts: ParallelStreamOpts,
129 consumer: &mut C,
130 ) -> Result<()>
131 where
132 P: RecordParser + Clone + Send + Sync + 'static,
133 C: BatchConsumer,
134 {
135 self.run_bytes_stream_core(bytes, None, splitter, parser, plan, opts, consumer)
136 }
137
138 #[allow(clippy::too_many_arguments)]
140 fn run_bytes_stream_core<P, C>(
141 &self,
142 bytes: &[u8],
143 input: Option<std::sync::Arc<InputBuffer>>,
144 splitter: &dyn Splitter,
145 parser: P,
146 plan: Arc<ExecutionPlan>,
147 opts: ParallelStreamOpts,
148 consumer: &mut C,
149 ) -> Result<()>
150 where
151 P: RecordParser + Clone + Send + Sync + 'static,
152 C: BatchConsumer,
153 {
154 let actual_bytes = match input {
156 Some(ref inp) => inp.as_slice(),
157 None => bytes,
158 };
159 if actual_bytes.is_empty() {
160 return Ok(());
161 }
162 let n = opts.threads.max(1);
163 let max_reorder = if opts.max_reorder > 0 {
164 opts.max_reorder
165 } else {
166 n
167 };
168 let schema: Option<FrozenSchema> = match opts.schema {
175 Some(s) => Some(s),
176 None => {
177 if !plan.schema_order.is_empty() {
178 let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
179 Some(FrozenSchema::from_partial_plan(&names, &plan))
180 } else {
181 let opts = DiscoveryOpts::default();
182 let sig = crate::schema::layout_signature(actual_bytes, &opts);
183 let cached = crate::schema::SCHEMA_CACHE
184 .read()
185 .unwrap_or_else(|e| e.into_inner())
186 .get(&sig)
187 .cloned();
188 let (schema, order) = if let Some(order) = cached {
189 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
190 (FrozenSchema::from_discovered(&order, &plan), order)
191 } else {
192 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
193 let (schema, order) =
194 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
195 discover_schema(actual_bytes, &parser, &plan, splitter)
196 }))
197 .map_err(|_| {
198 crate::Error::Parser(
199 "parser panicked during schema discovery".into(),
200 )
201 })?;
202 (schema, Arc::new(order))
203 };
204 if !order.is_empty() {
205 crate::schema::insert_schema_cache(sig, order);
206 }
207 Some(schema)
208 }
209 }
210 };
211
212 let bytes_per_row = splitter
213 .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
214 .max(1);
215 let slots = n
216 .saturating_mul(3)
217 .saturating_add(self.max_in_flight.saturating_mul(2))
218 .saturating_add(max_reorder)
219 .saturating_add(1);
220 let allowance = self.budget.share(slots);
221 let chunk_size = (self.budget.bytes() / n.saturating_mul(32))
222 .max(bytes_per_row.saturating_mul(10))
223 .max(if self.budget.is_strict() {
224 1
225 } else {
226 64 * 1024
227 });
228 let num_chunks = actual_bytes.len().div_ceil(chunk_size).max(n).min(10000);
229 let split_points = splitter.find_split_points(actual_bytes, num_chunks);
230 let mut ranges = crate::decoder::split_points_to_ranges(&split_points, actual_bytes.len());
231 if ranges.is_empty() {
232 ranges.push(0..actual_bytes.len());
233 }
234
235 let chunks_with_seq: Vec<(usize, std::ops::Range<usize>)> =
236 ranges.into_iter().enumerate().rev().collect();
237 allowance.check(
238 chunks_with_seq.capacity() * std::mem::size_of::<(usize, std::ops::Range<usize>)>(),
239 )?;
240
241 let (sender, receiver) = sync_channel(self.max_in_flight);
242 let mut handles: Vec<JoinHandle<Result<()>>> = Vec::with_capacity(n);
243 let chunk_queue = std::sync::Arc::new(std::sync::Mutex::new(chunks_with_seq));
244 let plan_arc = plan;
245 let schema_arc = schema.map(std::sync::Arc::new);
246 let dispatch = DispatchGuard(Arc::new((Mutex::new(Some(0)), Condvar::new())));
247 let dispatch_window = max_reorder.saturating_add(1);
248 let ordered = opts.ordered;
249 let est_row = splitter
250 .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
251 .max(512);
252
253 let bytes_fallback = if input.is_none() {
254 allowance.check(actual_bytes.len())?;
255 Some(Arc::<[u8]>::from(actual_bytes))
256 } else {
257 None
258 };
259 let (input, file) = match input {
260 #[cfg(feature = "mmap")]
261 Some(input) if matches!(&*input, InputBuffer::Mmap(_)) => {
262 let input = Arc::try_unwrap(input).map_err(|_| {
263 crate::Error::Plan("stream input still shared during planning".into())
264 })?;
265 let InputBuffer::Mmap(handle) = input else {
266 unreachable!()
267 };
268 drop(handle.mmap);
269 (None, Some(Arc::new(std::sync::Mutex::new(handle.file))))
270 }
271 input => (input, None::<Arc<std::sync::Mutex<std::fs::File>>>),
272 };
273 for _ in 0..n {
274 let queue = std::sync::Arc::clone(&chunk_queue);
275 let sender_clone: SyncSender<(usize, Result<RecordBatch>)> = sender.clone();
276 let parser_clone = parser.clone();
277 let plan_clone = std::sync::Arc::clone(&plan_arc);
278 let schema_clone = schema_arc.clone();
279 let input_clone = input.clone();
280 let fallback_clone = bytes_fallback.clone();
281 let file_clone = file.clone();
282 let dispatch_clone = dispatch.0.clone();
283 let handle = thread::spawn(move || -> Result<()> {
284 let bytes_ref: &[u8] = match input_clone {
285 Some(ref inp) => inp.as_slice(),
286 None => fallback_clone.as_deref().unwrap_or(&[]),
287 };
288 let mut chunk_buffer = Vec::new();
289 loop {
290 let next = {
291 let mut q = queue.lock().unwrap_or_else(|e| e.into_inner());
292 q.pop()
293 };
294 let Some((seq, range)) = next else { break };
295 if ordered {
296 let (next, ready) = &*dispatch_clone;
297 let next = ready
298 .wait_while(next.lock().unwrap_or_else(|e| e.into_inner()), |next| {
299 next.is_some_and(|next| seq >= next.saturating_add(dispatch_window))
300 })
301 .unwrap_or_else(|e| e.into_inner());
302 if next.is_none() {
303 break;
304 }
305 }
306 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(
307 || -> Result<RecordBatch> {
308 allowance.check(range.len())?;
309 let chunk_bytes = if let Some(file) = &file_clone {
310 chunk_buffer.resize(range.len(), 0);
311 allowance.check(chunk_buffer.capacity())?;
312 let mut file = file.lock().unwrap_or_else(|e| e.into_inner());
313 file.seek(SeekFrom::Start(range.start as u64))?;
314 file.read_exact(&mut chunk_buffer)?;
315 chunk_buffer.as_slice()
316 } else {
317 &bytes_ref[range.start..range.end]
318 };
319 let mut builder = TableBuilder::with_plan(
320 (chunk_bytes.len() / est_row).max(64),
321 plan_clone.clone(),
322 );
323 builder.set_memory_budget(allowance)?;
324 if let Some(ref schema) = schema_clone {
326 builder.ensure_schema(schema)?;
327 builder.check_memory_budget()?;
328 }
329 parser_clone.validate(chunk_bytes)?;
330 parser_clone.parse_chunk_generic(chunk_bytes, &mut builder)?;
331 let batch = builder.finish()?;
332 allowance.check(batch.get_array_memory_size())?;
333 Ok(batch)
334 },
335 ))
336 .unwrap_or_else(|_| {
337 Err(crate::Error::Parser(
338 "worker panicked during parallel parse".into(),
339 ))
340 });
341 let failed = result.is_err();
342 if sender_clone.send((seq, result)).is_err() || failed {
343 break;
344 }
345 }
346 Ok(())
347 });
348 handles.push(handle);
349 }
350 drop(sender);
351
352 let mut pending: BTreeMap<usize, RecordBatch> = BTreeMap::new();
356 let mut next_seq = 0usize;
357 let mut reorder_bytes: usize = 0;
358 let reorder_limit = max_reorder.saturating_mul(allowance.bytes());
359 let mut result = (|| -> Result<()> {
360 for (seq, res) in receiver {
361 let batch = res?;
362 if opts.ordered {
363 if seq == next_seq {
364 consumer.consume(batch)?;
365 next_seq += 1;
366 while let Some(b) = pending.remove(&next_seq) {
367 reorder_bytes -= b.get_array_memory_size();
368 consumer.consume(b)?;
369 next_seq += 1;
370 }
371 *dispatch.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = Some(next_seq);
372 dispatch.0 .1.notify_all();
373 } else {
374 reorder_bytes = reorder_bytes.saturating_add(batch.get_array_memory_size());
375 if self.budget.is_strict() && reorder_bytes > reorder_limit {
376 return Err(crate::Error::Memory {
377 used: reorder_bytes,
378 limit: reorder_limit,
379 });
380 }
381 pending.insert(seq, batch);
382 }
383 } else {
384 consumer.consume(batch)?;
385 }
386 }
387 Ok(())
388 })();
389 drop(dispatch);
390 for h in handles {
391 let joined = h
392 .join()
393 .map_err(|_| {
394 crate::Error::Parser(
395 "worker panicked during parallel parse. \
396 This usually indicates a bug in the parser (e.g., returning \
397 Cow::Borrowed that outlives the input chunk, or an unwrap() \
398 on None/Err during parsing). Check your RecordParser::parse_chunk \
399 implementation for incorrect lifetime handling or missing error checks."
400 .to_string(),
401 )
402 })
403 .and_then(|r| r);
404 if result.is_ok() {
405 result = joined;
406 }
407 }
408 result
409 }
410}
411
412pub struct ParallelStreamingBatchIterator {
414 receiver: Receiver<Result<RecordBatch>>,
415 handle: Option<JoinHandle<Result<()>>>,
416 done: bool,
417}
418
419impl ParallelStreamingBatchIterator {
420 pub fn new<P, S>(
421 path: PathBuf,
422 splitter: S,
423 parser: P,
424 plan: Arc<ExecutionPlan>,
425 budget: MemoryBudget,
426 prefault: bool,
427 opts: ParallelStreamOpts,
428 ) -> Self
429 where
430 P: RecordParser + Clone + Send + Sync + 'static,
431 S: Splitter + Clone + Send + Sync + 'static,
432 {
433 let num_threads = opts.threads;
434 let max_in_flight = 2 * num_threads;
435 let (sender, receiver) = sync_channel(max_in_flight);
436 let handle = thread::spawn(move || {
437 let exec = ParallelStreamingExecutor::new(budget, max_in_flight);
438 let mut consumer = ChannelConsumer {
439 sender: sender.clone(),
440 };
441 let res = exec.run_stream(
442 &path,
443 &splitter,
444 parser,
445 plan,
446 prefault,
447 opts,
448 &mut consumer,
449 );
450 if let Err(e) = res {
451 let _ = sender.send(Err(e));
452 }
453 Ok(())
454 });
455 Self {
456 receiver,
457 handle: Some(handle),
458 done: false,
459 }
460 }
461}
462
463struct ChannelConsumer {
464 sender: SyncSender<Result<RecordBatch>>,
465}
466impl BatchConsumer for ChannelConsumer {
467 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
468 self.sender
469 .send(Ok(batch))
470 .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
471 }
472}
473
474impl Iterator for ParallelStreamingBatchIterator {
475 type Item = Result<RecordBatch>;
476 fn next(&mut self) -> Option<Self::Item> {
477 if self.done {
478 return None;
479 }
480 match self.receiver.recv() {
481 Ok(Ok(batch)) => Some(Ok(batch)),
482 Ok(Err(e)) => {
483 self.done = true;
484 Some(Err(e))
485 }
486 Err(_) => {
487 self.done = true;
488 if let Some(h) = self.handle.take() {
489 if let Err(p) = h.join() {
490 let msg = if let Some(s) = p.downcast_ref::<&str>() {
491 (*s).to_string()
492 } else if let Some(s) = p.downcast_ref::<String>() {
493 s.clone()
494 } else {
495 "worker panicked".to_string()
496 };
497 return Some(Err(crate::Error::Parser(format!(
498 "parallel streaming worker panicked: {msg}. \
499 This usually indicates a bug in the parser (e.g., returning \
500 Cow::Borrowed that outlives the input chunk, or an unwrap() \
501 on None/Err during parsing). Check your RecordParser::parse_chunk \
502 implementation for incorrect lifetime handling or missing error checks."
503 ))));
504 }
505 }
506 None
507 }
508 }
509 }
510}
511
512#[cfg(test)]
513mod tests {
514 use std::borrow::Cow;
515
516 use super::*;
517
518 #[test]
519 fn channel_consumer_reports_disconnect() {
520 let (sender, receiver) = sync_channel(1);
521 drop(receiver);
522 let mut consumer = ChannelConsumer { sender };
523 let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
524 assert!(consumer.consume(batch).is_err());
525 }
526 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
527 use crate::plan::ExecutionPlan;
528 use crate::schema::clear_schema_cache;
529 use crate::value::Value;
530 use arrow::array::{Array, AsArray};
531
532 #[derive(Clone, Debug, Default)]
534 struct NameValueParser;
535
536 impl RecordParser for NameValueParser {
537 fn validate(&self, _bytes: &[u8]) -> Result<()> {
538 Ok(())
539 }
540
541 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
542 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
543 for line in text.lines() {
544 if line.is_empty() {
545 continue;
546 }
547 sink.begin_row();
548 for token in line.split('\t') {
549 if let Some((k, v)) = token.split_once('=') {
550 sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
551 }
552 }
553 sink.end_row();
554 }
555 Ok(())
556 }
557 }
558
559 struct NullSplitter;
561
562 impl Splitter for NullSplitter {
563 fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
564 None
565 }
566
567 fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
568 vec![0, bytes.len()]
569 }
570
571 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
572 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
573 sample.len() / n
574 }
575 }
576
577 struct TwoChunkSplitter;
578 impl Splitter for TwoChunkSplitter {
579 fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
580 None
581 }
582 fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
583 vec![0, 4.min(bytes.len()), bytes.len()]
584 }
585 fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
586 5
587 }
588 }
589
590 struct VecConsumer(Vec<RecordBatch>);
591 impl BatchConsumer for VecConsumer {
592 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
593 self.0.push(batch);
594 Ok(())
595 }
596 }
597
598 #[derive(Clone)]
599 struct DelayedParser;
600 impl RecordParser for DelayedParser {
601 fn validate(&self, bytes: &[u8]) -> Result<()> {
602 NameValueParser.validate(bytes)
603 }
604 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
605 if bytes.starts_with(b"a=1") {
606 std::thread::sleep(std::time::Duration::from_millis(30));
607 }
608 NameValueParser.parse_chunk(bytes, sink)
609 }
610 }
611
612 #[test]
613 fn ordered_soft_stream_tolerates_oversized_pending_batch() {
614 let exec = ParallelStreamingExecutor::new(MemoryBudget::new(1), 2);
615 let mut consumer = VecConsumer(Vec::new());
616 let opts = ParallelStreamOpts {
617 threads: 2,
618 max_reorder: 1,
619 ..Default::default()
620 };
621 let result = exec.run_bytes_stream(
622 b"a=1\nb=2\n",
623 &TwoChunkSplitter,
624 DelayedParser,
625 Arc::new(ExecutionPlan::new()),
626 opts,
627 &mut consumer,
628 );
629 result.unwrap();
630 assert_eq!(
631 consumer.0.iter().map(RecordBatch::num_rows).sum::<usize>(),
632 2
633 );
634 assert_eq!(
635 consumer.0[0]
636 .column_by_name("a")
637 .unwrap()
638 .as_string::<i32>()
639 .value(0),
640 "1"
641 );
642 assert_eq!(
643 consumer.0[1]
644 .column_by_name("b")
645 .unwrap()
646 .as_string::<i32>()
647 .value(0),
648 "2"
649 );
650 }
651
652 fn names(schema: &FrozenSchema) -> Vec<&str> {
653 schema.column_names().iter().map(|s| s.as_ref()).collect()
654 }
655
656 #[test]
657 fn schema_cache_hits_misses_and_plan_changes() {
658 clear_schema_cache();
659 let splitter = NullSplitter;
660 let parser = NameValueParser;
661
662 let bytes = b"a=1\tb=2\na=3\tb=4\n";
664 let plan = ExecutionPlan::new();
665 let s1 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
666 assert_eq!(names(&s1), vec!["a", "b"]);
667
668 let s2 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
670 assert_eq!(names(&s2), vec!["a", "b"]);
671
672 let bytes2 = b"c=1\td=2\n";
674 let s3 = discover_schema_for_bytes(bytes2, &splitter, &parser, &plan);
675 assert_eq!(names(&s3), vec!["c", "d"]);
676
677 let mut renamed = ExecutionPlan::new();
680 renamed.field_map.insert("a".to_string(), "x".to_string());
681 let s4 = discover_schema_for_bytes(bytes, &splitter, &parser, &renamed);
682 assert_eq!(names(&s4), vec!["x", "b"]);
683
684 clear_schema_cache();
685 }
686
687 #[test]
688 fn tail_scan_catches_late_columns() {
689 let mut data = Vec::new();
693 for i in 0..900 {
695 data.extend_from_slice(format!("a={i}\tb=val{i}\n").as_bytes());
696 }
697 for i in 900..1000 {
699 data.extend_from_slice(format!("a={i}\tb=val{i}\tlate=new{i}\n").as_bytes());
700 }
701
702 let splitter = NullSplitter;
703 let parser = NameValueParser;
704 let plan = ExecutionPlan::new();
705 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
706
707 let cols = names(&schema);
709 assert!(cols.contains(&"a"), "column 'a' missing");
710 assert!(cols.contains(&"b"), "column 'b' missing");
711 assert!(
712 cols.contains(&"late"),
713 "column 'late' missing from tail scan"
714 );
715
716 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
718 builder.ensure_schema(&schema).unwrap();
719 parser.parse_chunk(&data, &mut builder).unwrap();
720 let batch = builder.finish().unwrap();
721
722 assert_eq!(batch.num_rows(), 1000);
723 assert_eq!(batch.num_columns(), 3);
724
725 let late_col = batch.column_by_name("late").expect("late column missing");
727 let late_arr = late_col
728 .as_any()
729 .downcast_ref::<arrow::array::StringArray>()
730 .expect("late column not string array");
731 for i in 0..900 {
733 assert!(
734 late_arr.is_null(i),
735 "row {i}: expected null for 'late', got {:?}",
736 late_arr.value(i)
737 );
738 }
739 for i in 900..1000 {
741 assert!(
742 !late_arr.is_null(i),
743 "row {i}: expected value for 'late', got null"
744 );
745 assert!(
746 late_arr.value(i).starts_with("new"),
747 "row {i}: expected 'new{i}', got {:?}",
748 late_arr.value(i)
749 );
750 }
751 }
752
753 #[test]
754 fn all_columns_discovered_no_data_loss() {
755 let mut data = Vec::new();
757 let rows: Vec<&[u8]> = vec![
759 b"a=1\tb=2\tc=3\n",
760 b"b=5\td=6\n",
761 b"a=7\tc=8\te=9\n",
762 b"f=10\n",
763 b"a=11\tb=12\tc=13\td=14\te=15\tf=16\n",
764 ];
765 for row in &rows {
766 data.extend_from_slice(row);
767 }
768
769 let splitter = NullSplitter;
770 let parser = NameValueParser;
771 let plan = ExecutionPlan::new();
772 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
773
774 let cols = names(&schema);
776 assert_eq!(cols.len(), 6, "expected 6 columns, got {:?}", cols);
777 assert!(cols.contains(&"a"));
778 assert!(cols.contains(&"b"));
779 assert!(cols.contains(&"c"));
780 assert!(cols.contains(&"d"));
781 assert!(cols.contains(&"e"));
782 assert!(cols.contains(&"f"));
783
784 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
786 builder.ensure_schema(&schema).unwrap();
787 parser.parse_chunk(&data, &mut builder).unwrap();
788 let batch = builder.finish().unwrap();
789
790 assert_eq!(batch.num_rows(), 5);
791 assert_eq!(batch.num_columns(), 6);
792
793 let a_col = batch.column_by_name("a").unwrap();
795 let a_arr = a_col
796 .as_any()
797 .downcast_ref::<arrow::array::StringArray>()
798 .unwrap();
799 assert_eq!(a_arr.value(0), "1");
800 assert_eq!(a_arr.value(2), "7");
801 assert_eq!(a_arr.value(4), "11");
802 }
803
804 #[test]
805 fn dynamic_sizing_discovers_same_columns() {
806 let mut data = Vec::new();
809 for i in 0..1000 {
811 if i < 100 {
812 data.extend_from_slice(format!("a={i}\tb={i}\n").as_bytes());
814 } else if i < 500 {
815 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\n").as_bytes());
817 } else if i < 900 {
818 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\n").as_bytes());
820 } else {
821 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\te={i}\n").as_bytes());
823 }
824 }
825
826 let splitter = NullSplitter;
827 let parser = NameValueParser;
828 let plan = ExecutionPlan::new();
829 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
830
831 let cols = names(&schema);
833 assert_eq!(cols.len(), 5, "expected 5 columns, got {:?}", cols);
834 assert!(cols.contains(&"a"));
835 assert!(cols.contains(&"b"));
836 assert!(cols.contains(&"c"));
837 assert!(cols.contains(&"d"));
838 assert!(cols.contains(&"e"));
839
840 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
842 builder.ensure_schema(&schema).unwrap();
843 parser.parse_chunk(&data, &mut builder).unwrap();
844 let batch = builder.finish().unwrap();
845
846 assert_eq!(batch.num_rows(), 1000);
847 assert_eq!(batch.num_columns(), 5);
848 }
849
850 #[test]
851 fn column_order_consistent_across_file_sizes() {
852 let make_data = |n: usize| -> Vec<u8> {
855 let mut data = Vec::new();
856 for i in 0..n {
857 data.extend_from_slice(format!("x={i}\ty={i}\tz={i}\n").as_bytes());
858 }
859 data
860 };
861
862 let splitter = NullSplitter;
863 let parser = NameValueParser;
864 let plan = ExecutionPlan::new();
865
866 let sizes = vec![10, 100, 1000];
868 let mut first_order: Option<Vec<String>> = None;
869
870 for size in sizes {
871 let data = make_data(size);
872 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
873 let cols: Vec<String> = schema
874 .column_names()
875 .iter()
876 .map(|s| s.to_string())
877 .collect();
878
879 if let Some(ref order) = first_order {
880 assert_eq!(
881 &cols, order,
882 "column order changed for size {size}: {cols:?} vs {order:?}"
883 );
884 } else {
885 first_order = Some(cols);
886 }
887 }
888 }
889
890 #[test]
891 fn esoteric_sparse_schema_one_column_per_row() {
892 let mut data = Vec::new();
894 for i in 0..100 {
895 data.extend_from_slice(format!("col_{i}={i}\n").as_bytes());
896 }
897
898 let splitter = NullSplitter;
899 let parser = NameValueParser;
900 let plan = ExecutionPlan::new();
901 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
902
903 let cols = names(&schema);
905 assert_eq!(
906 cols.len(),
907 100,
908 "expected 100 columns, got {:?}",
909 cols.len()
910 );
911 }
912
913 #[test]
914 fn esoteric_empty_rows() {
915 let mut data = Vec::new();
917 for i in 0..50 {
918 data.extend_from_slice(format!("a={i}\n").as_bytes());
919 data.extend_from_slice(b"\n"); data.extend_from_slice(b"\n"); data.extend_from_slice(format!("b={i}\n").as_bytes());
922 }
923
924 let splitter = NullSplitter;
925 let parser = NameValueParser;
926 let plan = ExecutionPlan::new();
927 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
928
929 let cols = names(&schema);
930 assert!(cols.contains(&"a"), "column 'a' missing");
931 assert!(cols.contains(&"b"), "column 'b' missing");
932 }
933
934 #[test]
935 fn esoteric_single_character_columns() {
936 let data = b"a=1\tb=2\tc=3\nd=4\te=5\n";
938
939 let splitter = NullSplitter;
940 let parser = NameValueParser;
941 let plan = ExecutionPlan::new();
942 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
943
944 let cols = names(&schema);
945 assert_eq!(cols.len(), 5);
946 assert!(cols.contains(&"a"));
947 assert!(cols.contains(&"e"));
948 }
949
950 #[test]
951 fn esoteric_unicode_column_names() {
952 let data = "名前=Alice\t年齢=30\n名前=Bob\t年齢=25\n".as_bytes();
954
955 let splitter = NullSplitter;
956 let parser = NameValueParser;
957 let plan = ExecutionPlan::new();
958 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
959
960 let cols = names(&schema);
961 assert_eq!(cols.len(), 2);
962 assert!(cols.contains(&"名前"));
963 assert!(cols.contains(&"年齢"));
964 }
965
966 #[test]
967 fn esoteric_very_long_column_names() {
968 let long_name = "a".repeat(200);
970 let data = format!("{long_name}=1\tb=2\n{long_name}=3\tc=4\n");
971
972 let splitter = NullSplitter;
973 let parser = NameValueParser;
974 let plan = ExecutionPlan::new();
975 let schema = discover_schema_for_bytes(data.as_bytes(), &splitter, &parser, &plan);
976
977 let cols = names(&schema);
978 assert_eq!(cols.len(), 3);
979 assert!(cols.contains(&long_name.as_str()));
980 }
981
982 #[test]
983 fn esoteric_duplicate_columns_in_row() {
984 let data = b"a=1\ta=2\ta=3\tb=4\n";
986
987 let splitter = NullSplitter;
988 let parser = NameValueParser;
989 let plan = ExecutionPlan::new();
990 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
991
992 let cols = names(&schema);
993 assert_eq!(cols.len(), 2, "expected 2 unique columns, got {:?}", cols);
994 assert!(cols.contains(&"a"));
995 assert!(cols.contains(&"b"));
996 }
997}