1use std::collections::BTreeMap;
4use std::path::{Path, PathBuf};
5use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
6use std::sync::Arc;
7use std::thread::{self, JoinHandle};
8
9use arrow::record_batch::RecordBatch;
10
11use std::sync::atomic::{AtomicU64, Ordering};
12
13use crate::bounded::MemoryBudget;
14use crate::consumer::BatchConsumer;
15use crate::decoder::{RecordParser, Splitter};
16use crate::engine::TableBuilder;
17use crate::input::InputBuffer;
18use crate::plan::ExecutionPlan;
19use crate::schema::{DiscoveryOpts, FrozenSchema};
20use crate::Result;
21
22static DISCOVERY_NS: AtomicU64 = AtomicU64::new(0);
23
24pub fn discovery_profile() -> u64 {
26 DISCOVERY_NS.load(Ordering::Relaxed)
27}
28pub fn reset_discovery_profile() {
30 DISCOVERY_NS.store(0, Ordering::Relaxed);
31}
32
33pub struct ParallelStreamOpts {
35 pub threads: usize,
37 pub ordered: bool,
39 pub max_reorder: usize,
41 pub schema: Option<FrozenSchema>,
44}
45
46impl Default for ParallelStreamOpts {
47 fn default() -> Self {
48 Self {
49 threads: std::thread::available_parallelism()
50 .map(|n| n.get())
51 .unwrap_or(1),
52 ordered: true,
53 max_reorder: 0, schema: None,
55 }
56 }
57}
58
59struct DiscoverySink {
64 seen: std::cell::RefCell<rustc_hash::FxHashSet<Box<str>>>,
65 order: std::cell::RefCell<Vec<String>>,
66}
67
68impl DiscoverySink {
69 fn new() -> Self {
70 Self {
71 seen: std::cell::RefCell::new(rustc_hash::FxHashSet::default()),
72 order: std::cell::RefCell::new(Vec::new()),
73 }
74 }
75 fn into_order(self) -> Vec<String> {
76 self.order.into_inner()
77 }
78}
79
80impl crate::decoder::ColumnarSink for DiscoverySink {
81 #[inline]
82 fn begin_row(&mut self) {}
83 #[inline]
84 fn put_field(&mut self, name: &str, _value: crate::value::Value<'_>) {
85 let mut seen = self.seen.borrow_mut();
87 if seen.insert(Box::from(name)) {
88 self.order.borrow_mut().push(name.to_string());
89 }
90 }
91 #[inline]
92 fn end_row(&mut self) {}
93 #[inline]
94 fn wants(&self, _name: &str) -> bool {
95 true
96 }
97 #[inline]
98 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
99 let mut seen = self.seen.borrow_mut();
101 if seen.insert(Box::from(name)) {
102 self.order.borrow_mut().push(name.to_string());
103 }
104 Some(name)
105 }
106 #[inline]
107 fn needs_value(&self) -> bool {
108 false
109 }
110 fn finish(&mut self) -> crate::Result<arrow::record_batch::RecordBatch> {
112 Ok(arrow::record_batch::RecordBatch::new_empty(
113 std::sync::Arc::new(arrow::datatypes::Schema::empty()),
114 ))
115 }
116}
117
118fn discover_schema<P: crate::decoder::RecordParser>(
119 bytes: &[u8],
120 parser: &P,
121 plan: &crate::plan::ExecutionPlan,
122 splitter: &dyn crate::decoder::Splitter,
123) -> (FrozenSchema, Vec<String>) {
124 let t0 = std::time::Instant::now();
125 let opts = DiscoveryOpts::default();
127
128 let file_size = bytes.len() as u64;
129
130 let n = crate::schema::dynamic_window_count(file_size);
132 let wbytes = crate::schema::dynamic_window_size(file_size);
133
134 let order: Vec<String> = if file_size < opts.full_scan_threshold || n == 0 {
135 let mut sink = DiscoverySink::new();
137 let _ = parser.parse_chunk_generic(bytes, &mut sink);
138 sink.into_order()
139 } else {
140 use rayon::prelude::*;
141 let per_window: Vec<Vec<String>> = (0..n)
143 .into_par_iter()
144 .map(|i| {
145 let start = (file_size * i as u64 / n as u64) as usize;
146 let end = (start + wbytes).min(bytes.len());
147 if start >= end {
148 return Vec::new();
149 }
150 let mut sink = DiscoverySink::new();
151 let slice = &bytes[start..end];
152 let _ = parser.parse_chunk_generic(slice, &mut sink);
153 sink.into_order()
154 })
155 .collect();
156 let mut seen = rustc_hash::FxHashSet::<String>::default();
158 let mut merged = Vec::new();
159 for mut v in per_window {
160 for name in v.drain(..) {
161 if seen.insert(name.clone()) {
162 merged.push(name);
163 }
164 }
165 }
166 if opts.always_scan_tail {
168 let tail_start = bytes.len().saturating_sub(wbytes);
169 let mut tail_sink = DiscoverySink::new();
170 let _ = parser.parse_chunk_generic(&bytes[tail_start..], &mut tail_sink);
171 for name in tail_sink.into_order() {
172 if seen.insert(name.clone()) {
173 merged.push(name);
174 }
175 }
176 }
177 if merged.is_empty() && !bytes.is_empty() {
178 let mut sink = DiscoverySink::new();
179 let _ = parser.parse_chunk_generic(bytes, &mut sink);
180 sink.into_order()
181 } else {
182 merged
183 }
184 };
185 let elapsed = t0.elapsed().as_nanos() as u64;
186 DISCOVERY_NS.store(elapsed, Ordering::Relaxed);
187 let _ = splitter; let schema = if order.is_empty() {
189 FrozenSchema::from_plan(&[], plan)
190 } else {
191 FrozenSchema::from_discovered(&order, plan)
192 };
193 (schema, order)
194}
195
196pub fn discover_schema_for_bytes<P: crate::decoder::RecordParser>(
203 bytes: &[u8],
204 splitter: &dyn crate::decoder::Splitter,
205 parser: &P,
206 plan: &crate::plan::ExecutionPlan,
207) -> FrozenSchema {
208 if !plan.schema_order.is_empty() {
209 let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
210 return FrozenSchema::from_partial_plan(&names, plan);
211 }
212 let opts = DiscoveryOpts::default();
213 let sig = crate::schema::layout_signature(bytes, &opts);
214 {
215 let cache = crate::schema::SCHEMA_CACHE.read().unwrap();
216 if let Some(order) = cache.get(&sig) {
217 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
218 return FrozenSchema::from_discovered(order, plan);
219 }
220 }
221 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
222 let (schema, order) = discover_schema(bytes, parser, plan, splitter);
223 if !order.is_empty() {
224 crate::schema::insert_schema_cache(sig, Arc::new(order));
225 }
226 schema
227}
228
229pub struct ParallelStreamingExecutor {
231 budget: MemoryBudget,
232 max_in_flight: usize,
233}
234
235impl ParallelStreamingExecutor {
236 pub fn new(budget: MemoryBudget, max_in_flight: usize) -> Self {
237 Self {
238 budget,
239 max_in_flight: max_in_flight.max(1),
240 }
241 }
242
243 #[allow(clippy::too_many_arguments)]
245 pub fn run_stream<P, C>(
246 &self,
247 path: &Path,
248 splitter: &dyn Splitter,
249 parser: P,
250 plan: Arc<ExecutionPlan>,
251 prefault: bool,
252 opts: ParallelStreamOpts,
253 consumer: &mut C,
254 ) -> Result<()>
255 where
256 P: RecordParser + Clone + Send + Sync + 'static,
257 C: BatchConsumer,
258 {
259 let input = InputBuffer::open(path, cfg!(feature = "mmap"), prefault)?;
260 if input.is_empty() {
261 return Ok(());
262 }
263 let shared = std::sync::Arc::new(input);
266 self.run_bytes_stream_shared(shared, splitter, parser, plan, opts, consumer)
267 }
268
269 fn run_bytes_stream_shared<P, C>(
271 &self,
272 input: std::sync::Arc<InputBuffer>,
273 splitter: &dyn Splitter,
274 parser: P,
275 plan: Arc<ExecutionPlan>,
276 opts: ParallelStreamOpts,
277 consumer: &mut C,
278 ) -> Result<()>
279 where
280 P: RecordParser + Clone + Send + Sync + 'static,
281 C: BatchConsumer,
282 {
283 let dummy: &[u8] = &[];
287 self.run_bytes_stream_core(dummy, Some(input), splitter, parser, plan, opts, consumer)
288 }
289
290 pub fn run_bytes_stream<P, C>(
292 &self,
293 bytes: &[u8],
294 splitter: &dyn Splitter,
295 parser: P,
296 plan: Arc<ExecutionPlan>,
297 opts: ParallelStreamOpts,
298 consumer: &mut C,
299 ) -> Result<()>
300 where
301 P: RecordParser + Clone + Send + Sync + 'static,
302 C: BatchConsumer,
303 {
304 self.run_bytes_stream_core(bytes, None, splitter, parser, plan, opts, consumer)
305 }
306
307 #[allow(clippy::too_many_arguments)]
309 fn run_bytes_stream_core<P, C>(
310 &self,
311 bytes: &[u8],
312 input: Option<std::sync::Arc<InputBuffer>>,
313 splitter: &dyn Splitter,
314 parser: P,
315 plan: Arc<ExecutionPlan>,
316 opts: ParallelStreamOpts,
317 consumer: &mut C,
318 ) -> Result<()>
319 where
320 P: RecordParser + Clone + Send + Sync + 'static,
321 C: BatchConsumer,
322 {
323 let actual_bytes = match input {
325 Some(ref inp) => inp.as_slice(),
326 None => bytes,
327 };
328 let n = opts.threads.max(1);
329 let max_reorder = if opts.max_reorder > 0 {
330 opts.max_reorder
331 } else {
332 n
333 };
334 let schema: Option<FrozenSchema> = match opts.schema {
341 Some(s) => Some(s),
342 None => {
343 if !plan.schema_order.is_empty() {
344 let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
345 Some(FrozenSchema::from_partial_plan(&names, &plan))
346 } else {
347 let opts = DiscoveryOpts::default();
348 let sig = crate::schema::layout_signature(actual_bytes, &opts);
349 let cached = crate::schema::SCHEMA_CACHE
350 .read()
351 .unwrap()
352 .get(&sig)
353 .cloned();
354 let (schema, order) = if let Some(order) = cached {
355 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
356 (FrozenSchema::from_discovered(&order, &plan), order)
357 } else {
358 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
359 let (schema, order) =
360 discover_schema(actual_bytes, &parser, &plan, splitter);
361 (schema, Arc::new(order))
362 };
363 if !order.is_empty() {
364 crate::schema::insert_schema_cache(sig, order);
365 }
366 Some(schema)
367 }
368 }
369 };
370
371 let bytes_per_row = splitter
372 .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
373 .max(1);
374 let chunk_size = (self.budget.bytes() / (n * 2))
375 .max(bytes_per_row * 10)
376 .max(64 * 1024);
377 let num_chunks = (actual_bytes.len() / chunk_size).max(n).min(10000);
378 let split_points = splitter.find_split_points(actual_bytes, num_chunks);
379 let mut ranges = crate::decoder::split_points_to_ranges(&split_points, actual_bytes.len());
380 if ranges.is_empty() {
381 ranges.push(0..bytes.len());
382 }
383
384 let chunks_with_seq: Vec<(usize, std::ops::Range<usize>)> =
385 ranges.into_iter().enumerate().collect();
386
387 let (sender, receiver) = sync_channel(self.max_in_flight);
388 let mut handles: Vec<JoinHandle<Result<()>>> = Vec::with_capacity(n);
389 let chunk_queue = std::sync::Arc::new(std::sync::Mutex::new(chunks_with_seq));
390 let plan_arc = plan;
391 let schema_arc = schema.map(std::sync::Arc::new);
392 let est_row = splitter
393 .estimate_bytes_per_row(&bytes[..bytes.len().min(65536)])
394 .max(512);
395
396 let bytes_fallback = if input.is_none() {
398 Some(actual_bytes.to_vec())
399 } else {
400 None
401 };
402 for _ in 0..n {
403 let queue = std::sync::Arc::clone(&chunk_queue);
404 let sender_clone: SyncSender<(usize, Result<RecordBatch>)> = sender.clone();
405 let parser_clone = parser.clone();
406 let plan_clone = std::sync::Arc::clone(&plan_arc);
407 let schema_clone = schema_arc.clone();
408 let input_clone = input.clone();
409 let fallback_clone = bytes_fallback.clone();
410 let handle = thread::spawn(move || -> Result<()> {
411 let bytes_ref: &[u8] = match input_clone {
414 Some(ref inp) => inp.as_slice(),
415 None => fallback_clone.as_deref().unwrap_or(&[]),
416 };
417 loop {
418 let next = {
419 let mut q = queue.lock().unwrap();
420 q.pop()
421 };
422 let Some((seq, range)) = next else { break };
423 let chunk_bytes = &bytes_ref[range.start..range.end];
424 let mut builder = TableBuilder::with_plan(
425 (chunk_bytes.len() / est_row).max(64),
426 plan_clone.clone(),
427 );
428 if let Some(ref schema) = schema_clone {
430 if let Err(e) = builder.ensure_schema(schema) {
431 let _ = sender_clone.send((seq, Err(e)));
432 break;
433 }
434 }
435 if let Err(e) = parser_clone.validate(chunk_bytes) {
436 let _ = sender_clone.send((seq, Err(e)));
437 break;
438 }
439 if let Err(e) = parser_clone.parse_chunk_generic(chunk_bytes, &mut builder) {
440 let _ = sender_clone.send((seq, Err(e)));
441 break;
442 }
443 let res = builder.finish();
444 if sender_clone.send((seq, res)).is_err() {
445 break;
446 }
447 }
448 Ok(())
449 });
450 handles.push(handle);
451 }
452 drop(sender);
453
454 let mut pending: BTreeMap<usize, RecordBatch> = BTreeMap::new();
459 let mut next_seq = 0usize;
460 let mut reorder_bytes: usize = 0;
461 let mut fallback_unordered = false;
462 let mut first_chunk_error: Option<crate::Error> = None;
463 let reorder_limit = max_reorder * self.budget.bytes();
464 for (seq, res) in receiver {
465 match res {
466 Ok(batch) => {
467 if opts.ordered && !fallback_unordered {
468 if seq == next_seq {
469 consumer.consume(batch)?;
470 next_seq += 1;
471 while let Some(b) = pending.remove(&next_seq) {
472 consumer.consume(b)?;
473 next_seq += 1;
474 }
475 } else {
476 reorder_bytes += batch.get_array_memory_size();
477 if reorder_bytes > reorder_limit {
478 for (_, b) in std::mem::take(&mut pending) {
483 consumer.consume(b)?;
484 }
485 consumer.consume(batch)?;
486 reorder_bytes = 0;
487 fallback_unordered = true;
488 } else {
489 pending.insert(seq, batch);
490 }
491 }
492 } else {
493 consumer.consume(batch)?;
495 }
496 }
497 Err(e) => {
498 if first_chunk_error.is_none() {
501 first_chunk_error = Some(e);
502 }
503 }
504 }
505 }
506 if let Some(e) = first_chunk_error {
507 return Err(e);
508 }
509 if opts.ordered && !fallback_unordered {
511 while let Some(b) = pending.remove(&next_seq) {
512 consumer.consume(b)?;
513 next_seq += 1;
514 }
515 }
516 for h in handles {
517 h.join().map_err(|_| {
518 crate::Error::Parser(
519 "worker panicked during parallel parse. \
520 This usually indicates a bug in the parser (e.g., returning \
521 Cow::Borrowed that outlives the input chunk, or an unwrap() \
522 on None/Err during parsing). Check your RecordParser::parse_chunk \
523 implementation for incorrect lifetime handling or missing error checks."
524 .to_string(),
525 )
526 })??;
527 }
528 Ok(())
529 }
530}
531
532pub struct ParallelStreamingBatchIterator {
534 receiver: Receiver<Result<RecordBatch>>,
535 handle: Option<JoinHandle<Result<()>>>,
536 done: bool,
537}
538
539impl ParallelStreamingBatchIterator {
540 pub fn new<P, S>(
541 path: PathBuf,
542 splitter: S,
543 parser: P,
544 plan: Arc<ExecutionPlan>,
545 budget: MemoryBudget,
546 prefault: bool,
547 opts: ParallelStreamOpts,
548 ) -> Self
549 where
550 P: RecordParser + Clone + Send + Sync + 'static,
551 S: Splitter + Clone + Send + Sync + 'static,
552 {
553 let num_threads = opts.threads;
554 let max_in_flight = 2 * num_threads;
555 let (sender, receiver) = sync_channel(max_in_flight);
556 let handle = thread::spawn(move || {
557 let exec = ParallelStreamingExecutor::new(budget, max_in_flight);
558 let mut consumer = ChannelConsumer {
559 sender: sender.clone(),
560 };
561 let res = exec.run_stream(
562 &path,
563 &splitter,
564 parser,
565 plan,
566 prefault,
567 opts,
568 &mut consumer,
569 );
570 if let Err(e) = res {
571 let _ = sender.send(Err(e));
572 }
573 Ok(())
574 });
575 Self {
576 receiver,
577 handle: Some(handle),
578 done: false,
579 }
580 }
581}
582
583struct ChannelConsumer {
584 sender: SyncSender<Result<RecordBatch>>,
585}
586impl BatchConsumer for ChannelConsumer {
587 fn consume(&mut self, batch: RecordBatch) -> Result<()> {
588 let _ = self.sender.send(Ok(batch));
589 Ok(())
590 }
591}
592
593impl Iterator for ParallelStreamingBatchIterator {
594 type Item = Result<RecordBatch>;
595 fn next(&mut self) -> Option<Self::Item> {
596 if self.done {
597 return None;
598 }
599 match self.receiver.recv() {
603 Ok(Ok(batch)) => Some(Ok(batch)),
604 Ok(Err(e)) => {
605 self.done = true;
606 Some(Err(e))
607 }
608 Err(_) => {
609 self.done = true;
610 if let Some(h) = self.handle.take() {
611 if let Err(p) = h.join() {
612 let msg = if let Some(s) = p.downcast_ref::<&str>() {
613 (*s).to_string()
614 } else if let Some(s) = p.downcast_ref::<String>() {
615 s.clone()
616 } else {
617 "worker panicked".to_string()
618 };
619 return Some(Err(crate::Error::Parser(format!(
620 "parallel streaming worker panicked: {msg}. \
621 This usually indicates a bug in the parser (e.g., returning \
622 Cow::Borrowed that outlives the input chunk, or an unwrap() \
623 on None/Err during parsing). Check your RecordParser::parse_chunk \
624 implementation for incorrect lifetime handling or missing error checks."
625 ))));
626 }
627 }
628 None
629 }
630 }
631 }
632}
633
634#[cfg(test)]
635mod tests {
636 use std::borrow::Cow;
637
638 use super::*;
639 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
640 use crate::plan::ExecutionPlan;
641 use crate::schema::clear_schema_cache;
642 use crate::value::Value;
643 use arrow::array::Array;
644
645 #[derive(Clone, Debug, Default)]
647 struct NameValueParser;
648
649 impl RecordParser for NameValueParser {
650 fn validate(&self, _bytes: &[u8]) -> Result<()> {
651 Ok(())
652 }
653
654 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
655 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
656 for line in text.lines() {
657 if line.is_empty() {
658 continue;
659 }
660 sink.begin_row();
661 for token in line.split('\t') {
662 if let Some((k, v)) = token.split_once('=') {
663 sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
664 }
665 }
666 sink.end_row();
667 }
668 Ok(())
669 }
670 }
671
672 struct NullSplitter;
674
675 impl Splitter for NullSplitter {
676 fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
677 None
678 }
679
680 fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
681 vec![0, bytes.len()]
682 }
683
684 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
685 let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
686 sample.len() / n
687 }
688 }
689
690 fn names(schema: &FrozenSchema) -> Vec<&str> {
691 schema.column_names().iter().map(|s| s.as_ref()).collect()
692 }
693
694 #[test]
695 fn schema_cache_hits_misses_and_plan_changes() {
696 clear_schema_cache();
697 let splitter = NullSplitter;
698 let parser = NameValueParser;
699
700 let bytes = b"a=1\tb=2\na=3\tb=4\n";
702 let plan = ExecutionPlan::new();
703 let s1 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
704 assert_eq!(names(&s1), vec!["a", "b"]);
705
706 let s2 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
708 assert_eq!(names(&s2), vec!["a", "b"]);
709
710 let bytes2 = b"c=1\td=2\n";
712 let s3 = discover_schema_for_bytes(bytes2, &splitter, &parser, &plan);
713 assert_eq!(names(&s3), vec!["c", "d"]);
714
715 let mut renamed = ExecutionPlan::new();
718 renamed.field_map.insert("a".to_string(), "x".to_string());
719 let s4 = discover_schema_for_bytes(bytes, &splitter, &parser, &renamed);
720 assert_eq!(names(&s4), vec!["x", "b"]);
721
722 clear_schema_cache();
723 }
724
725 #[test]
726 fn tail_scan_catches_late_columns() {
727 let mut data = Vec::new();
731 for i in 0..900 {
733 data.extend_from_slice(format!("a={i}\tb=val{i}\n").as_bytes());
734 }
735 for i in 900..1000 {
737 data.extend_from_slice(format!("a={i}\tb=val{i}\tlate=new{i}\n").as_bytes());
738 }
739
740 let splitter = NullSplitter;
741 let parser = NameValueParser;
742 let plan = ExecutionPlan::new();
743 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
744
745 let cols = names(&schema);
747 assert!(cols.contains(&"a"), "column 'a' missing");
748 assert!(cols.contains(&"b"), "column 'b' missing");
749 assert!(
750 cols.contains(&"late"),
751 "column 'late' missing from tail scan"
752 );
753
754 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
756 builder.ensure_schema(&schema).unwrap();
757 parser.parse_chunk(&data, &mut builder).unwrap();
758 let batch = builder.finish().unwrap();
759
760 assert_eq!(batch.num_rows(), 1000);
761 assert_eq!(batch.num_columns(), 3);
762
763 let late_col = batch.column_by_name("late").expect("late column missing");
765 let late_arr = late_col
766 .as_any()
767 .downcast_ref::<arrow::array::StringArray>()
768 .expect("late column not string array");
769 for i in 0..900 {
771 assert!(
772 late_arr.is_null(i),
773 "row {i}: expected null for 'late', got {:?}",
774 late_arr.value(i)
775 );
776 }
777 for i in 900..1000 {
779 assert!(
780 !late_arr.is_null(i),
781 "row {i}: expected value for 'late', got null"
782 );
783 assert!(
784 late_arr.value(i).starts_with("new"),
785 "row {i}: expected 'new{i}', got {:?}",
786 late_arr.value(i)
787 );
788 }
789 }
790
791 #[test]
792 fn all_columns_discovered_no_data_loss() {
793 let mut data = Vec::new();
795 let rows: Vec<&[u8]> = vec![
797 b"a=1\tb=2\tc=3\n",
798 b"b=5\td=6\n",
799 b"a=7\tc=8\te=9\n",
800 b"f=10\n",
801 b"a=11\tb=12\tc=13\td=14\te=15\tf=16\n",
802 ];
803 for row in &rows {
804 data.extend_from_slice(row);
805 }
806
807 let splitter = NullSplitter;
808 let parser = NameValueParser;
809 let plan = ExecutionPlan::new();
810 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
811
812 let cols = names(&schema);
814 assert_eq!(cols.len(), 6, "expected 6 columns, got {:?}", cols);
815 assert!(cols.contains(&"a"));
816 assert!(cols.contains(&"b"));
817 assert!(cols.contains(&"c"));
818 assert!(cols.contains(&"d"));
819 assert!(cols.contains(&"e"));
820 assert!(cols.contains(&"f"));
821
822 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
824 builder.ensure_schema(&schema).unwrap();
825 parser.parse_chunk(&data, &mut builder).unwrap();
826 let batch = builder.finish().unwrap();
827
828 assert_eq!(batch.num_rows(), 5);
829 assert_eq!(batch.num_columns(), 6);
830
831 let a_col = batch.column_by_name("a").unwrap();
833 let a_arr = a_col
834 .as_any()
835 .downcast_ref::<arrow::array::StringArray>()
836 .unwrap();
837 assert_eq!(a_arr.value(0), "1");
838 assert_eq!(a_arr.value(2), "7");
839 assert_eq!(a_arr.value(4), "11");
840 }
841
842 #[test]
843 fn dynamic_sizing_discovers_same_columns() {
844 let mut data = Vec::new();
847 for i in 0..1000 {
849 if i < 100 {
850 data.extend_from_slice(format!("a={i}\tb={i}\n").as_bytes());
852 } else if i < 500 {
853 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\n").as_bytes());
855 } else if i < 900 {
856 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\n").as_bytes());
858 } else {
859 data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\te={i}\n").as_bytes());
861 }
862 }
863
864 let splitter = NullSplitter;
865 let parser = NameValueParser;
866 let plan = ExecutionPlan::new();
867 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
868
869 let cols = names(&schema);
871 assert_eq!(cols.len(), 5, "expected 5 columns, got {:?}", cols);
872 assert!(cols.contains(&"a"));
873 assert!(cols.contains(&"b"));
874 assert!(cols.contains(&"c"));
875 assert!(cols.contains(&"d"));
876 assert!(cols.contains(&"e"));
877
878 let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
880 builder.ensure_schema(&schema).unwrap();
881 parser.parse_chunk(&data, &mut builder).unwrap();
882 let batch = builder.finish().unwrap();
883
884 assert_eq!(batch.num_rows(), 1000);
885 assert_eq!(batch.num_columns(), 5);
886 }
887
888 #[test]
889 fn column_order_consistent_across_file_sizes() {
890 let make_data = |n: usize| -> Vec<u8> {
893 let mut data = Vec::new();
894 for i in 0..n {
895 data.extend_from_slice(format!("x={i}\ty={i}\tz={i}\n").as_bytes());
896 }
897 data
898 };
899
900 let splitter = NullSplitter;
901 let parser = NameValueParser;
902 let plan = ExecutionPlan::new();
903
904 let sizes = vec![10, 100, 1000];
906 let mut first_order: Option<Vec<String>> = None;
907
908 for size in sizes {
909 let data = make_data(size);
910 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
911 let cols: Vec<String> = schema
912 .column_names()
913 .iter()
914 .map(|s| s.to_string())
915 .collect();
916
917 if let Some(ref order) = first_order {
918 assert_eq!(
919 &cols, order,
920 "column order changed for size {size}: {cols:?} vs {order:?}"
921 );
922 } else {
923 first_order = Some(cols);
924 }
925 }
926 }
927
928 #[test]
929 fn esoteric_sparse_schema_one_column_per_row() {
930 let mut data = Vec::new();
932 for i in 0..100 {
933 data.extend_from_slice(format!("col_{i}={i}\n").as_bytes());
934 }
935
936 let splitter = NullSplitter;
937 let parser = NameValueParser;
938 let plan = ExecutionPlan::new();
939 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
940
941 let cols = names(&schema);
943 assert_eq!(
944 cols.len(),
945 100,
946 "expected 100 columns, got {:?}",
947 cols.len()
948 );
949 }
950
951 #[test]
952 fn esoteric_empty_rows() {
953 let mut data = Vec::new();
955 for i in 0..50 {
956 data.extend_from_slice(format!("a={i}\n").as_bytes());
957 data.extend_from_slice(b"\n"); data.extend_from_slice(b"\n"); data.extend_from_slice(format!("b={i}\n").as_bytes());
960 }
961
962 let splitter = NullSplitter;
963 let parser = NameValueParser;
964 let plan = ExecutionPlan::new();
965 let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
966
967 let cols = names(&schema);
968 assert!(cols.contains(&"a"), "column 'a' missing");
969 assert!(cols.contains(&"b"), "column 'b' missing");
970 }
971
972 #[test]
973 fn esoteric_single_character_columns() {
974 let data = b"a=1\tb=2\tc=3\nd=4\te=5\n";
976
977 let splitter = NullSplitter;
978 let parser = NameValueParser;
979 let plan = ExecutionPlan::new();
980 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
981
982 let cols = names(&schema);
983 assert_eq!(cols.len(), 5);
984 assert!(cols.contains(&"a"));
985 assert!(cols.contains(&"e"));
986 }
987
988 #[test]
989 fn esoteric_unicode_column_names() {
990 let data = "名前=Alice\t年齢=30\n名前=Bob\t年齢=25\n".as_bytes();
992
993 let splitter = NullSplitter;
994 let parser = NameValueParser;
995 let plan = ExecutionPlan::new();
996 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
997
998 let cols = names(&schema);
999 assert_eq!(cols.len(), 2);
1000 assert!(cols.contains(&"名前"));
1001 assert!(cols.contains(&"年齢"));
1002 }
1003
1004 #[test]
1005 fn esoteric_very_long_column_names() {
1006 let long_name = "a".repeat(200);
1008 let data = format!("{long_name}=1\tb=2\n{long_name}=3\tc=4\n");
1009
1010 let splitter = NullSplitter;
1011 let parser = NameValueParser;
1012 let plan = ExecutionPlan::new();
1013 let schema = discover_schema_for_bytes(data.as_bytes(), &splitter, &parser, &plan);
1014
1015 let cols = names(&schema);
1016 assert_eq!(cols.len(), 3);
1017 assert!(cols.contains(&long_name.as_str()));
1018 }
1019
1020 #[test]
1021 fn esoteric_duplicate_columns_in_row() {
1022 let data = b"a=1\ta=2\ta=3\tb=4\n";
1024
1025 let splitter = NullSplitter;
1026 let parser = NameValueParser;
1027 let plan = ExecutionPlan::new();
1028 let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
1029
1030 let cols = names(&schema);
1031 assert_eq!(cols.len(), 2, "expected 2 unique columns, got {:?}", cols);
1032 assert!(cols.contains(&"a"));
1033 assert!(cols.contains(&"b"));
1034 }
1035}