1use std::collections::BTreeMap;
4use std::io::{Read, Seek, SeekFrom};
5use std::path::{Path, PathBuf};
6use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
7use std::sync::{Arc, Condvar, Mutex};
8use std::thread::{self, JoinHandle};
9
10use arrow::record_batch::RecordBatch;
11
12use std::sync::atomic::{AtomicU64, Ordering};
13
14use crate::bounded::MemoryBudget;
15use crate::consumer::BatchConsumer;
16use crate::decoder::{RecordParser, Splitter};
17use crate::engine::TableBuilder;
18use crate::input::InputBuffer;
19use crate::plan::ExecutionPlan;
20use crate::schema::{DiscoveryOpts, FrozenSchema};
21use crate::Result;
22
23mod discovery;
24
25pub(crate) use discovery::discover_schema;
26pub use discovery::{
27 discover_schema_for_bytes, discovery_profile, reset_discovery_profile, ParallelStreamOpts,
28};
29
30pub struct ParallelStreamingExecutor {
32 budget: MemoryBudget,
33 max_in_flight: usize,
34}
35
36type DispatchState = Arc<(Mutex<Option<usize>>, Condvar)>;
37
38struct DispatchGuard(DispatchState);
39
40impl Drop for DispatchGuard {
41 fn drop(&mut self) {
42 *self.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = None;
43 self.0 .1.notify_all();
44 }
45}
46
47impl ParallelStreamingExecutor {
48 pub fn new(budget: MemoryBudget, max_in_flight: usize) -> Self {
49 Self {
50 budget,
51 max_in_flight: max_in_flight.max(1),
52 }
53 }
54
55 #[allow(clippy::too_many_arguments)]
57 pub fn run_stream<P, C>(
58 &self,
59 path: &Path,
60 splitter: &dyn Splitter,
61 parser: P,
62 plan: Arc<ExecutionPlan>,
63 prefault: bool,
64 opts: ParallelStreamOpts,
65 consumer: &mut C,
66 ) -> Result<()>
67 where
68 P: RecordParser + Clone + Send + Sync + 'static,
69 C: BatchConsumer,
70 {
71 let input = InputBuffer::open(path, cfg!(feature = "mmap"), prefault)?;
72 if input.is_empty() {
73 return Ok(());
74 }
75 #[cfg(not(feature = "mmap"))]
76 self.budget.share(4).check(input.len())?;
77 #[cfg(feature = "mmap")]
78 let opts = if let InputBuffer::Mmap(handle) = &input {
79 let mut opts = opts;
80 if opts.schema.is_none() && plan.schema_order.is_empty() {
81 opts.schema = Some(discovery::discover_schema_for_file(
82 handle,
83 splitter,
84 &parser,
85 &plan,
86 self.budget,
87 )?);
88 }
89 opts
90 } else {
91 self.budget.share(4).check(input.len())?;
92 opts
93 };
94 let shared = std::sync::Arc::new(input);
97 self.run_bytes_stream_shared(shared, splitter, parser, plan, opts, consumer)
98 }
99
100 fn run_bytes_stream_shared<P, C>(
102 &self,
103 input: std::sync::Arc<InputBuffer>,
104 splitter: &dyn Splitter,
105 parser: P,
106 plan: Arc<ExecutionPlan>,
107 opts: ParallelStreamOpts,
108 consumer: &mut C,
109 ) -> Result<()>
110 where
111 P: RecordParser + Clone + Send + Sync + 'static,
112 C: BatchConsumer,
113 {
114 let dummy: &[u8] = &[];
118 self.run_bytes_stream_core(dummy, Some(input), splitter, parser, plan, opts, consumer)
119 }
120
121 pub fn run_bytes_stream<P, C>(
123 &self,
124 bytes: &[u8],
125 splitter: &dyn Splitter,
126 parser: P,
127 plan: Arc<ExecutionPlan>,
128 opts: ParallelStreamOpts,
129 consumer: &mut C,
130 ) -> Result<()>
131 where
132 P: RecordParser + Clone + Send + Sync + 'static,
133 C: BatchConsumer,
134 {
135 self.run_bytes_stream_core(bytes, None, splitter, parser, plan, opts, consumer)
136 }
137
138 #[allow(clippy::too_many_arguments)]
140 fn run_bytes_stream_core<P, C>(
141 &self,
142 bytes: &[u8],
143 input: Option<std::sync::Arc<InputBuffer>>,
144 splitter: &dyn Splitter,
145 parser: P,
146 plan: Arc<ExecutionPlan>,
147 opts: ParallelStreamOpts,
148 consumer: &mut C,
149 ) -> Result<()>
150 where
151 P: RecordParser + Clone + Send + Sync + 'static,
152 C: BatchConsumer,
153 {
154 let actual_bytes = match input {
156 Some(ref inp) => inp.as_slice(),
157 None => bytes,
158 };
159 if actual_bytes.is_empty() {
160 return Ok(());
161 }
162 let n = opts.threads.max(1);
163 let max_reorder = if opts.max_reorder > 0 {
164 opts.max_reorder
165 } else {
166 n
167 };
168 let schema: Option<FrozenSchema> = match opts.schema {
175 Some(s) => Some(s),
176 None => {
177 if !plan.schema_order.is_empty() {
178 let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
179 Some(FrozenSchema::from_partial_plan(&names, &plan))
180 } else {
181 let opts = DiscoveryOpts::default();
182 let sig = crate::schema::layout_signature(actual_bytes, &opts);
183 let cached = crate::schema::SCHEMA_CACHE
184 .read()
185 .unwrap_or_else(|e| e.into_inner())
186 .get(&sig)
187 .cloned();
188 let (schema, order) = if let Some(order) = cached {
189 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
190 (FrozenSchema::from_discovered(&order, &plan), order)
191 } else {
192 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
193 let (schema, order) =
194 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
195 discover_schema(actual_bytes, &parser, &plan, splitter)
196 }))
197 .map_err(|_| {
198 crate::Error::Parser(
199 "parser panicked during schema discovery".into(),
200 )
201 })?;
202 (schema, Arc::new(order))
203 };
204 if !order.is_empty() {
205 crate::schema::insert_schema_cache(sig, order);
206 }
207 Some(schema)
208 }
209 }
210 };
211
212 let bytes_per_row = splitter
213 .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
214 .max(1);
215 let slots = n
216 .saturating_mul(3)
217 .saturating_add(self.max_in_flight.saturating_mul(2))
218 .saturating_add(max_reorder)
219 .saturating_add(1);
220 let allowance = self.budget.share(slots);
221 let chunk_size = (self.budget.bytes() / n.saturating_mul(32))
222 .max(bytes_per_row.saturating_mul(10))
223 .max(if self.budget.is_strict() {
224 1
225 } else {
226 64 * 1024
227 });
228 let num_chunks = actual_bytes.len().div_ceil(chunk_size).max(n).min(10000);
229 let split_points = splitter.find_split_points_with(
230 actual_bytes,
231 num_chunks,
232 plan.min_chunk_bytes
233 .unwrap_or(crate::decoder::MIN_CHUNK_BYTES),
234 );
235 let mut ranges = crate::decoder::split_points_to_ranges(&split_points, actual_bytes.len());
236 if ranges.is_empty() {
237 ranges.push(0..actual_bytes.len());
238 }
239
240 let chunks_with_seq: Vec<(usize, std::ops::Range<usize>)> =
241 ranges.into_iter().enumerate().rev().collect();
242 allowance.check(
243 chunks_with_seq.capacity() * std::mem::size_of::<(usize, std::ops::Range<usize>)>(),
244 )?;
245
246 let (sender, receiver) = sync_channel(self.max_in_flight);
247 let mut handles: Vec<JoinHandle<Result<()>>> = Vec::with_capacity(n);
248 let chunk_queue = std::sync::Arc::new(std::sync::Mutex::new(chunks_with_seq));
249 let plan_arc = plan;
250 let schema_arc = schema.map(std::sync::Arc::new);
251 let dispatch = DispatchGuard(Arc::new((Mutex::new(Some(0)), Condvar::new())));
252 let dispatch_window = max_reorder.saturating_add(1);
253 let ordered = opts.ordered;
254 let est_row = splitter
255 .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
256 .max(512);
257
258 let bytes_fallback = if input.is_none() {
259 allowance.check(actual_bytes.len())?;
260 Some(Arc::<[u8]>::from(actual_bytes))
261 } else {
262 None
263 };
264 let (input, file) = match input {
265 #[cfg(feature = "mmap")]
266 Some(input) if matches!(&*input, InputBuffer::Mmap(_)) => {
267 let input = Arc::try_unwrap(input).map_err(|_| {
268 crate::Error::Plan("stream input still shared during planning".into())
269 })?;
270 let InputBuffer::Mmap(handle) = input else {
271 unreachable!()
272 };
273 drop(handle.mmap);
274 (None, Some(Arc::new(std::sync::Mutex::new(handle.file))))
275 }
276 input => (input, None::<Arc<std::sync::Mutex<std::fs::File>>>),
277 };
278 for _ in 0..n {
279 let queue = std::sync::Arc::clone(&chunk_queue);
280 let sender_clone: SyncSender<(usize, Result<RecordBatch>)> = sender.clone();
281 let parser_clone = parser.clone();
282 let plan_clone = std::sync::Arc::clone(&plan_arc);
283 let schema_clone = schema_arc.clone();
284 let input_clone = input.clone();
285 let fallback_clone = bytes_fallback.clone();
286 let file_clone = file.clone();
287 let dispatch_clone = dispatch.0.clone();
288 let handle = thread::spawn(move || -> Result<()> {
289 let bytes_ref: &[u8] = match input_clone {
290 Some(ref inp) => inp.as_slice(),
291 None => fallback_clone.as_deref().unwrap_or(&[]),
292 };
293 let mut chunk_buffer = Vec::new();
294 loop {
295 let next = {
296 let mut q = queue.lock().unwrap_or_else(|e| e.into_inner());
297 q.pop()
298 };
299 let Some((seq, range)) = next else { break };
300 if ordered {
301 let (next, ready) = &*dispatch_clone;
302 let next = ready
303 .wait_while(next.lock().unwrap_or_else(|e| e.into_inner()), |next| {
304 next.is_some_and(|next| seq >= next.saturating_add(dispatch_window))
305 })
306 .unwrap_or_else(|e| e.into_inner());
307 if next.is_none() {
308 break;
309 }
310 }
311 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(
312 || -> Result<RecordBatch> {
313 allowance.check(range.len())?;
314 let chunk_bytes = if let Some(file) = &file_clone {
315 chunk_buffer.resize(range.len(), 0);
316 allowance.check(chunk_buffer.capacity())?;
317 let mut file = file.lock().unwrap_or_else(|e| e.into_inner());
318 file.seek(SeekFrom::Start(range.start as u64))?;
319 file.read_exact(&mut chunk_buffer)?;
320 chunk_buffer.as_slice()
321 } else {
322 &bytes_ref[range.start..range.end]
323 };
324 let mut builder = TableBuilder::with_plan(
325 (chunk_bytes.len() / est_row).max(64),
326 plan_clone.clone(),
327 );
328 builder.set_memory_budget(allowance)?;
329 if let Some(ref schema) = schema_clone {
331 builder.ensure_schema(schema)?;
332 builder.check_memory_budget()?;
333 }
334 parser_clone.validate(chunk_bytes)?;
335 parser_clone.parse_chunk_generic(chunk_bytes, &mut builder)?;
336 let batch = builder.finish()?;
337 allowance.check(batch.get_array_memory_size())?;
338 Ok(batch)
339 },
340 ))
341 .unwrap_or_else(|_| {
342 Err(crate::Error::Parser(
343 "worker panicked during parallel parse".into(),
344 ))
345 });
346 let failed = result.is_err();
347 if sender_clone.send((seq, result)).is_err() || failed {
348 break;
349 }
350 }
351 Ok(())
352 });
353 handles.push(handle);
354 }
355 drop(sender);
356
357 let mut pending: BTreeMap<usize, RecordBatch> = BTreeMap::new();
361 let mut next_seq = 0usize;
362 let mut reorder_bytes: usize = 0;
363 let reorder_limit = max_reorder.saturating_mul(allowance.bytes());
364 let mut result = (|| -> Result<()> {
365 for (seq, res) in receiver {
366 let batch = res?;
367 if opts.ordered {
368 if seq == next_seq {
369 consumer.consume(batch)?;
370 next_seq += 1;
371 while let Some(b) = pending.remove(&next_seq) {
372 reorder_bytes -= b.get_array_memory_size();
373 consumer.consume(b)?;
374 next_seq += 1;
375 }
376 *dispatch.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = Some(next_seq);
377 dispatch.0 .1.notify_all();
378 } else {
379 reorder_bytes = reorder_bytes.saturating_add(batch.get_array_memory_size());
380 if self.budget.is_strict() && reorder_bytes > reorder_limit {
381 return Err(crate::Error::Memory {
382 used: reorder_bytes,
383 limit: reorder_limit,
384 });
385 }
386 pending.insert(seq, batch);
387 }
388 } else {
389 consumer.consume(batch)?;
390 }
391 }
392 Ok(())
393 })();
394 drop(dispatch);
395 for h in handles {
396 let joined = h
397 .join()
398 .map_err(|_| {
399 crate::Error::Parser(
400 "worker panicked during parallel parse. \
401 This usually indicates a bug in the parser (e.g., returning \
402 Cow::Borrowed that outlives the input chunk, or an unwrap() \
403 on None/Err during parsing). Check your RecordParser::parse_chunk \
404 implementation for incorrect lifetime handling or missing error checks."
405 .to_string(),
406 )
407 })
408 .and_then(|r| r);
409 if result.is_ok() {
410 result = joined;
411 }
412 }
413 result
414 }
415}
416
417pub struct ParallelStreamingBatchIterator {
419 receiver: Receiver<Result<RecordBatch>>,
420 handle: Option<JoinHandle<Result<()>>>,
421 done: bool,
422}
423
424impl ParallelStreamingBatchIterator {
425 pub fn new<P, S>(
426 path: PathBuf,
427 splitter: S,
428 parser: P,
429 plan: Arc<ExecutionPlan>,
430 budget: MemoryBudget,
431 prefault: bool,
432 opts: ParallelStreamOpts,
433 ) -> Self
434 where
435 P: RecordParser + Clone + Send + Sync + 'static,
436 S: Splitter + Clone + Send + Sync + 'static,
437 {
438 let num_threads = opts.threads;
439 let max_in_flight = 2 * num_threads;
440 let (sender, receiver) = sync_channel(max_in_flight);
441 let handle = thread::spawn(move || {
442 let exec = ParallelStreamingExecutor::new(budget, max_in_flight);
443 let mut consumer = ChannelConsumer {
444 sender: sender.clone(),
445 };
446 let res = exec.run_stream(
447 &path,
448 &splitter,
449 parser,
450 plan,
451 prefault,
452 opts,
453 &mut consumer,
454 );
455 if let Err(e) = res {
456 let _ = sender.send(Err(e));
457 }
458 Ok(())
459 });
460 Self {
461 receiver,
462 handle: Some(handle),
463 done: false,
464 }
465 }
466}
467
468struct ChannelConsumer {
469 sender: SyncSender<Result<RecordBatch>>,
470}
471impl BatchConsumer for ChannelConsumer {
472 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
473 self.sender
474 .send(Ok(batch))
475 .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
476 }
477}
478
479impl Iterator for ParallelStreamingBatchIterator {
480 type Item = Result<RecordBatch>;
481 fn next(&mut self) -> Option<Self::Item> {
482 if self.done {
483 return None;
484 }
485 match self.receiver.recv() {
486 Ok(Ok(batch)) => Some(Ok(batch)),
487 Ok(Err(e)) => {
488 self.done = true;
489 Some(Err(e))
490 }
491 Err(_) => {
492 self.done = true;
493 if let Some(h) = self.handle.take() {
494 if let Err(p) = h.join() {
495 let msg = if let Some(s) = p.downcast_ref::<&str>() {
496 (*s).to_string()
497 } else if let Some(s) = p.downcast_ref::<String>() {
498 s.clone()
499 } else {
500 "worker panicked".to_string()
501 };
502 return Some(Err(crate::Error::Parser(format!(
503 "parallel streaming worker panicked: {msg}. \
504 This usually indicates a bug in the parser (e.g., returning \
505 Cow::Borrowed that outlives the input chunk, or an unwrap() \
506 on None/Err during parsing). Check your RecordParser::parse_chunk \
507 implementation for incorrect lifetime handling or missing error checks."
508 ))));
509 }
510 }
511 None
512 }
513 }
514 }
515}
516
517#[cfg(test)]
518mod tests {
519 use std::borrow::Cow;
520
521 use super::*;
522
523 #[test]
524 fn channel_consumer_reports_disconnect() {
525 let (sender, receiver) = sync_channel(1);
526 drop(receiver);
527 let mut consumer = ChannelConsumer { sender };
528 let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
529 assert!(consumer.consume(batch).is_err());
530 }
531 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
532 use crate::plan::ExecutionPlan;
533 use crate::schema::clear_schema_cache;
534 use crate::value::Value;
535 use arrow::array::{Array, AsArray};
536
537 #[derive(Clone, Debug, Default)]
539 struct NameValueParser;
540
541 impl RecordParser for NameValueParser {
542 fn validate(&self, _bytes: &[u8]) -> Result<()> {
543 Ok(())
544 }
545
546 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
547 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
548 for line in text.lines() {
549 if line.is_empty() {
550 continue;
551 }
552 sink.begin_row();
553 for token in line.split('\t') {
554 if let Some((k, v)) = token.split_once('=') {
555 sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
556 }
557 }
558 sink.end_row();
559 }
560 Ok(())
561 }
562 }
563
564 struct NullSplitter;
566
567 impl Splitter for NullSplitter {
568 fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
569 None
570 }
571
572 fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
573 vec![0, bytes.len()]
574 }
575
576 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
577 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
578 sample.len() / n
579 }
580 }
581
582 struct TwoChunkSplitter;
583 impl Splitter for TwoChunkSplitter {
584 fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
585 None
586 }
587 fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
588 vec![0, 4.min(bytes.len()), bytes.len()]
589 }
590 fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
591 5
592 }
593 }
594
595 struct VecConsumer(Vec<RecordBatch>);
596 impl BatchConsumer for VecConsumer {
597 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
598 self.0.push(batch);
599 Ok(())
600 }
601 }
602
603 #[derive(Clone)]
604 struct DelayedParser;
605 impl RecordParser for DelayedParser {
606 fn validate(&self, bytes: &[u8]) -> Result<()> {
607 NameValueParser.validate(bytes)
608 }
609 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
610 if bytes.starts_with(b"a=1") {
611 std::thread::sleep(std::time::Duration::from_millis(30));
612 }
613 NameValueParser.parse_chunk(bytes, sink)
614 }
615 }
616
617 #[test]
618 fn ordered_soft_stream_tolerates_oversized_pending_batch() {
619 let exec = ParallelStreamingExecutor::new(MemoryBudget::new(1), 2);
620 let mut consumer = VecConsumer(Vec::new());
621 let opts = ParallelStreamOpts {
622 threads: 2,
623 max_reorder: 1,
624 ..Default::default()
625 };
626 let result = exec.run_bytes_stream(
627 b"a=1\nb=2\n",
628 &TwoChunkSplitter,
629 DelayedParser,
630 Arc::new(ExecutionPlan::new()),
631 opts,
632 &mut consumer,
633 );
634 result.unwrap();
635 assert_eq!(
636 consumer.0.iter().map(RecordBatch::num_rows).sum::<usize>(),
637 2
638 );
639 assert_eq!(
640 consumer.0[0]
641 .column_by_name("a")
642 .unwrap()
643 .as_string::<i32>()
644 .value(0),
645 "1"
646 );
647 assert_eq!(
648 consumer.0[1]
649 .column_by_name("b")
650 .unwrap()
651 .as_string::<i32>()
652 .value(0),
653 "2"
654 );
655 }
656
657 fn names(schema: &FrozenSchema) -> Vec<&str> {
658 schema.column_names().iter().map(|s| s.as_ref()).collect()
659 }
660
661 #[test]
662 fn schema_cache_hits_misses_and_plan_changes() {
663 clear_schema_cache();
664 let splitter = NullSplitter;
665 let parser = NameValueParser;
666
667 let bytes = b"a=1\tb=2\na=3\tb=4\n";
669 let plan = ExecutionPlan::new();
670 let s1 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
671 assert_eq!(names(&s1), vec!["a", "b"]);
672
673 let s2 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
675 assert_eq!(names(&s2), vec!["a", "b"]);
676
677 let bytes2 = b"c=1\td=2\n";
679 let s3 = discover_schema_for_bytes(bytes2, &splitter, &parser, &plan);
680 assert_eq!(names(&s3), vec!["c", "d"]);
681
682 let mut renamed = ExecutionPlan::new();
685 renamed.field_map.insert("a".to_string(), "x".to_string());
686 let s4 = discover_schema_for_bytes(bytes, &splitter, &parser, &renamed);
687 assert_eq!(names(&s4), vec!["x", "b"]);
688
689 clear_schema_cache();
690 }
691
692 #[test]
693 fn tail_scan_catches_late_columns() {
694 let mut data = Vec::new();
698 for i in 0..900 {
700 data.extend_from_slice(format!("a={i}\tb=val{i}\n").as_bytes());
701 }
702 for i in 900..1000 {
704 data.extend_from_slice(format!("a={i}\tb=val{i}\tlate=new{i}\n").as_bytes());
705 }
706
707 let splitter = NullSplitter;
708 let parser = NameValueParser;
709 let plan = ExecutionPlan::new();
710 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
711
712 let cols = names(&schema);
714 assert!(cols.contains(&"a"), "column 'a' missing");
715 assert!(cols.contains(&"b"), "column 'b' missing");
716 assert!(
717 cols.contains(&"late"),
718 "column 'late' missing from tail scan"
719 );
720
721 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
723 builder.ensure_schema(&schema).unwrap();
724 parser.parse_chunk(&data, &mut builder).unwrap();
725 let batch = builder.finish().unwrap();
726
727 assert_eq!(batch.num_rows(), 1000);
728 assert_eq!(batch.num_columns(), 3);
729
730 let late_col = batch.column_by_name("late").expect("late column missing");
732 let late_arr = late_col
733 .as_any()
734 .downcast_ref::<arrow::array::StringArray>()
735 .expect("late column not string array");
736 for i in 0..900 {
738 assert!(
739 late_arr.is_null(i),
740 "row {i}: expected null for 'late', got {:?}",
741 late_arr.value(i)
742 );
743 }
744 for i in 900..1000 {
746 assert!(
747 !late_arr.is_null(i),
748 "row {i}: expected value for 'late', got null"
749 );
750 assert!(
751 late_arr.value(i).starts_with("new"),
752 "row {i}: expected 'new{i}', got {:?}",
753 late_arr.value(i)
754 );
755 }
756 }
757
758 #[test]
759 fn all_columns_discovered_no_data_loss() {
760 let mut data = Vec::new();
762 let rows: Vec<&[u8]> = vec![
764 b"a=1\tb=2\tc=3\n",
765 b"b=5\td=6\n",
766 b"a=7\tc=8\te=9\n",
767 b"f=10\n",
768 b"a=11\tb=12\tc=13\td=14\te=15\tf=16\n",
769 ];
770 for row in &rows {
771 data.extend_from_slice(row);
772 }
773
774 let splitter = NullSplitter;
775 let parser = NameValueParser;
776 let plan = ExecutionPlan::new();
777 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
778
779 let cols = names(&schema);
781 assert_eq!(cols.len(), 6, "expected 6 columns, got {:?}", cols);
782 assert!(cols.contains(&"a"));
783 assert!(cols.contains(&"b"));
784 assert!(cols.contains(&"c"));
785 assert!(cols.contains(&"d"));
786 assert!(cols.contains(&"e"));
787 assert!(cols.contains(&"f"));
788
789 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
791 builder.ensure_schema(&schema).unwrap();
792 parser.parse_chunk(&data, &mut builder).unwrap();
793 let batch = builder.finish().unwrap();
794
795 assert_eq!(batch.num_rows(), 5);
796 assert_eq!(batch.num_columns(), 6);
797
798 let a_col = batch.column_by_name("a").unwrap();
800 let a_arr = a_col
801 .as_any()
802 .downcast_ref::<arrow::array::StringArray>()
803 .unwrap();
804 assert_eq!(a_arr.value(0), "1");
805 assert_eq!(a_arr.value(2), "7");
806 assert_eq!(a_arr.value(4), "11");
807 }
808
809 #[test]
810 fn dynamic_sizing_discovers_same_columns() {
811 let mut data = Vec::new();
814 for i in 0..1000 {
816 if i < 100 {
817 data.extend_from_slice(format!("a={i}\tb={i}\n").as_bytes());
819 } else if i < 500 {
820 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\n").as_bytes());
822 } else if i < 900 {
823 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\n").as_bytes());
825 } else {
826 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\te={i}\n").as_bytes());
828 }
829 }
830
831 let splitter = NullSplitter;
832 let parser = NameValueParser;
833 let plan = ExecutionPlan::new();
834 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
835
836 let cols = names(&schema);
838 assert_eq!(cols.len(), 5, "expected 5 columns, got {:?}", cols);
839 assert!(cols.contains(&"a"));
840 assert!(cols.contains(&"b"));
841 assert!(cols.contains(&"c"));
842 assert!(cols.contains(&"d"));
843 assert!(cols.contains(&"e"));
844
845 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
847 builder.ensure_schema(&schema).unwrap();
848 parser.parse_chunk(&data, &mut builder).unwrap();
849 let batch = builder.finish().unwrap();
850
851 assert_eq!(batch.num_rows(), 1000);
852 assert_eq!(batch.num_columns(), 5);
853 }
854
855 #[test]
856 fn column_order_consistent_across_file_sizes() {
857 let make_data = |n: usize| -> Vec<u8> {
860 let mut data = Vec::new();
861 for i in 0..n {
862 data.extend_from_slice(format!("x={i}\ty={i}\tz={i}\n").as_bytes());
863 }
864 data
865 };
866
867 let splitter = NullSplitter;
868 let parser = NameValueParser;
869 let plan = ExecutionPlan::new();
870
871 let sizes = vec![10, 100, 1000];
873 let mut first_order: Option<Vec<String>> = None;
874
875 for size in sizes {
876 let data = make_data(size);
877 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
878 let cols: Vec<String> = schema
879 .column_names()
880 .iter()
881 .map(|s| s.to_string())
882 .collect();
883
884 if let Some(ref order) = first_order {
885 assert_eq!(
886 &cols, order,
887 "column order changed for size {size}: {cols:?} vs {order:?}"
888 );
889 } else {
890 first_order = Some(cols);
891 }
892 }
893 }
894
895 #[test]
896 fn esoteric_sparse_schema_one_column_per_row() {
897 let mut data = Vec::new();
899 for i in 0..100 {
900 data.extend_from_slice(format!("col_{i}={i}\n").as_bytes());
901 }
902
903 let splitter = NullSplitter;
904 let parser = NameValueParser;
905 let plan = ExecutionPlan::new();
906 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
907
908 let cols = names(&schema);
910 assert_eq!(
911 cols.len(),
912 100,
913 "expected 100 columns, got {:?}",
914 cols.len()
915 );
916 }
917
918 #[test]
919 fn esoteric_empty_rows() {
920 let mut data = Vec::new();
922 for i in 0..50 {
923 data.extend_from_slice(format!("a={i}\n").as_bytes());
924 data.extend_from_slice(b"\n"); data.extend_from_slice(b"\n"); data.extend_from_slice(format!("b={i}\n").as_bytes());
927 }
928
929 let splitter = NullSplitter;
930 let parser = NameValueParser;
931 let plan = ExecutionPlan::new();
932 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
933
934 let cols = names(&schema);
935 assert!(cols.contains(&"a"), "column 'a' missing");
936 assert!(cols.contains(&"b"), "column 'b' missing");
937 }
938
939 #[test]
940 fn esoteric_single_character_columns() {
941 let data = b"a=1\tb=2\tc=3\nd=4\te=5\n";
943
944 let splitter = NullSplitter;
945 let parser = NameValueParser;
946 let plan = ExecutionPlan::new();
947 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
948
949 let cols = names(&schema);
950 assert_eq!(cols.len(), 5);
951 assert!(cols.contains(&"a"));
952 assert!(cols.contains(&"e"));
953 }
954
955 #[test]
956 fn esoteric_unicode_column_names() {
957 let data = "名前=Alice\t年齢=30\n名前=Bob\t年齢=25\n".as_bytes();
959
960 let splitter = NullSplitter;
961 let parser = NameValueParser;
962 let plan = ExecutionPlan::new();
963 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
964
965 let cols = names(&schema);
966 assert_eq!(cols.len(), 2);
967 assert!(cols.contains(&"名前"));
968 assert!(cols.contains(&"年齢"));
969 }
970
971 #[test]
972 fn esoteric_very_long_column_names() {
973 let long_name = "a".repeat(200);
975 let data = format!("{long_name}=1\tb=2\n{long_name}=3\tc=4\n");
976
977 let splitter = NullSplitter;
978 let parser = NameValueParser;
979 let plan = ExecutionPlan::new();
980 let schema = discover_schema_for_bytes(data.as_bytes(), &splitter, &parser, &plan);
981
982 let cols = names(&schema);
983 assert_eq!(cols.len(), 3);
984 assert!(cols.contains(&long_name.as_str()));
985 }
986
987 #[test]
988 fn esoteric_duplicate_columns_in_row() {
989 let data = b"a=1\ta=2\ta=3\tb=4\n";
991
992 let splitter = NullSplitter;
993 let parser = NameValueParser;
994 let plan = ExecutionPlan::new();
995 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
996
997 let cols = names(&schema);
998 assert_eq!(cols.len(), 2, "expected 2 unique columns, got {:?}", cols);
999 assert!(cols.contains(&"a"));
1000 assert!(cols.contains(&"b"));
1001 }
1002}