Skip to main content

rypipe_core/
parallel_stream.rs

1//! Parallel streaming executor: multi-core parsing with bounded memory.
2
3use std::collections::BTreeMap;
4use std::io::{Read, Seek, SeekFrom};
5use std::path::{Path, PathBuf};
6use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
7use std::sync::{Arc, Condvar, Mutex};
8use std::thread::{self, JoinHandle};
9
10use arrow::record_batch::RecordBatch;
11
12use std::sync::atomic::{AtomicU64, Ordering};
13
14use crate::bounded::MemoryBudget;
15use crate::consumer::BatchConsumer;
16use crate::decoder::{RecordParser, Splitter};
17use crate::engine::TableBuilder;
18use crate::input::InputBuffer;
19use crate::plan::ExecutionPlan;
20use crate::schema::{DiscoveryOpts, FrozenSchema};
21use crate::Result;
22
23mod discovery;
24
25pub(crate) use discovery::discover_schema;
26pub use discovery::{
27    discover_schema_for_bytes, discovery_profile, reset_discovery_profile, ParallelStreamOpts,
28};
29
30/// Parallel streaming executor: parses chunks concurrently with bounded memory.
31pub struct ParallelStreamingExecutor {
32    budget: MemoryBudget,
33    max_in_flight: usize,
34}
35
36type DispatchState = Arc<(Mutex<Option<usize>>, Condvar)>;
37
38struct DispatchGuard(DispatchState);
39
40impl Drop for DispatchGuard {
41    fn drop(&mut self) {
42        *self.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = None;
43        self.0 .1.notify_all();
44    }
45}
46
47impl ParallelStreamingExecutor {
48    pub fn new(budget: MemoryBudget, max_in_flight: usize) -> Self {
49        Self {
50            budget,
51            max_in_flight: max_in_flight.max(1),
52        }
53    }
54
55    /// Stream a file in parallel, calling `consumer` per batch in order.
56    #[allow(clippy::too_many_arguments)]
57    pub fn run_stream<P, C>(
58        &self,
59        path: &Path,
60        splitter: &dyn Splitter,
61        parser: P,
62        plan: Arc<ExecutionPlan>,
63        prefault: bool,
64        opts: ParallelStreamOpts,
65        consumer: &mut C,
66    ) -> Result<()>
67    where
68        P: RecordParser + Clone + Send + Sync + 'static,
69        C: BatchConsumer,
70    {
71        let input = InputBuffer::open(path, cfg!(feature = "mmap"), prefault)?;
72        if input.is_empty() {
73            return Ok(());
74        }
75        #[cfg(not(feature = "mmap"))]
76        self.budget.share(4).check(input.len())?;
77        #[cfg(feature = "mmap")]
78        let opts = if let InputBuffer::Mmap(handle) = &input {
79            let mut opts = opts;
80            if opts.schema.is_none() && plan.schema_order.is_empty() {
81                opts.schema = Some(discovery::discover_schema_for_file(
82                    handle,
83                    splitter,
84                    &parser,
85                    &plan,
86                    self.budget,
87                )?);
88            }
89            opts
90        } else {
91            self.budget.share(4).check(input.len())?;
92            opts
93        };
94        // Share the InputBuffer across workers via Arc to avoid
95        // the O(file_size × threads) to_vec() clone.
96        let shared = std::sync::Arc::new(input);
97        self.run_bytes_stream_shared(shared, splitter, parser, plan, opts, consumer)
98    }
99
100    /// Plan from shared input, then release file mappings before parsing.
101    fn run_bytes_stream_shared<P, C>(
102        &self,
103        input: std::sync::Arc<InputBuffer>,
104        splitter: &dyn Splitter,
105        parser: P,
106        plan: Arc<ExecutionPlan>,
107        opts: ParallelStreamOpts,
108        consumer: &mut C,
109    ) -> Result<()>
110    where
111        P: RecordParser + Clone + Send + Sync + 'static,
112        C: BatchConsumer,
113    {
114        // Safety: the Arc keeps the InputBuffer alive for the duration.
115        // We pass a dummy &[u8] to satisfy the signature; run_bytes_stream_core
116        // ignores it when input is Some and uses input.as_slice() instead.
117        let dummy: &[u8] = &[];
118        self.run_bytes_stream_core(dummy, Some(input), splitter, parser, plan, opts, consumer)
119    }
120
121    /// Stream bytes in parallel.
122    pub fn run_bytes_stream<P, C>(
123        &self,
124        bytes: &[u8],
125        splitter: &dyn Splitter,
126        parser: P,
127        plan: Arc<ExecutionPlan>,
128        opts: ParallelStreamOpts,
129        consumer: &mut C,
130    ) -> Result<()>
131    where
132        P: RecordParser + Clone + Send + Sync + 'static,
133        C: BatchConsumer,
134    {
135        self.run_bytes_stream_core(bytes, None, splitter, parser, plan, opts, consumer)
136    }
137
138    /// Core implementation shared by `run_bytes_stream` and `run_bytes_stream_shared`.
139    #[allow(clippy::too_many_arguments)]
140    fn run_bytes_stream_core<P, C>(
141        &self,
142        bytes: &[u8],
143        input: Option<std::sync::Arc<InputBuffer>>,
144        splitter: &dyn Splitter,
145        parser: P,
146        plan: Arc<ExecutionPlan>,
147        opts: ParallelStreamOpts,
148        consumer: &mut C,
149    ) -> Result<()>
150    where
151        P: RecordParser + Clone + Send + Sync + 'static,
152        C: BatchConsumer,
153    {
154        // Prefer the shared InputBuffer's slice over the passed-in bytes.
155        let actual_bytes = match input {
156            Some(ref inp) => inp.as_slice(),
157            None => bytes,
158        };
159        if actual_bytes.is_empty() {
160            return Ok(());
161        }
162        let n = opts.threads.max(1);
163        let max_reorder = if opts.max_reorder > 0 {
164            opts.max_reorder
165        } else {
166            n
167        };
168        // Frozen schema: gates correct ParquetWriter / StreamWriter usage.
169        // Without it, batch 2 can have different column order (FieldG vs Text20
170        // last) even when the set is identical, breaking `write_batch`.
171        // If opts.schema is None, auto-discover:
172        //  - explicit plan.schema_order → from_partial_plan (allows unknown columns)
173        //  - else sampled discovery (16×2 MiB windows for >128 MiB, else full)
174        let schema: Option<FrozenSchema> = match opts.schema {
175            Some(s) => Some(s),
176            None => {
177                if !plan.schema_order.is_empty() {
178                    let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
179                    Some(FrozenSchema::from_partial_plan(&names, &plan))
180                } else {
181                    let opts = DiscoveryOpts::default();
182                    let sig = crate::schema::layout_signature(actual_bytes, &opts);
183                    let cached = crate::schema::SCHEMA_CACHE
184                        .read()
185                        .unwrap_or_else(|e| e.into_inner())
186                        .get(&sig)
187                        .cloned();
188                    let (schema, order) = if let Some(order) = cached {
189                        crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
190                        (FrozenSchema::from_discovered(&order, &plan), order)
191                    } else {
192                        crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
193                        let (schema, order) =
194                            std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
195                                discover_schema(actual_bytes, &parser, &plan, splitter)
196                            }))
197                            .map_err(|_| {
198                                crate::Error::Parser(
199                                    "parser panicked during schema discovery".into(),
200                                )
201                            })?;
202                        (schema, Arc::new(order))
203                    };
204                    if !order.is_empty() {
205                        crate::schema::insert_schema_cache(sig, order);
206                    }
207                    Some(schema)
208                }
209            }
210        };
211
212        let bytes_per_row = splitter
213            .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
214            .max(1);
215        let slots = n
216            .saturating_mul(3)
217            .saturating_add(self.max_in_flight.saturating_mul(2))
218            .saturating_add(max_reorder)
219            .saturating_add(1);
220        let allowance = self.budget.share(slots);
221        let chunk_size = (self.budget.bytes() / n.saturating_mul(32))
222            .max(bytes_per_row.saturating_mul(10))
223            .max(if self.budget.is_strict() {
224                1
225            } else {
226                64 * 1024
227            });
228        let num_chunks = actual_bytes.len().div_ceil(chunk_size).max(n).min(10000);
229        let split_points = splitter.find_split_points(actual_bytes, num_chunks);
230        let mut ranges = crate::decoder::split_points_to_ranges(&split_points, actual_bytes.len());
231        if ranges.is_empty() {
232            ranges.push(0..actual_bytes.len());
233        }
234
235        let chunks_with_seq: Vec<(usize, std::ops::Range<usize>)> =
236            ranges.into_iter().enumerate().rev().collect();
237        allowance.check(
238            chunks_with_seq.capacity() * std::mem::size_of::<(usize, std::ops::Range<usize>)>(),
239        )?;
240
241        let (sender, receiver) = sync_channel(self.max_in_flight);
242        let mut handles: Vec<JoinHandle<Result<()>>> = Vec::with_capacity(n);
243        let chunk_queue = std::sync::Arc::new(std::sync::Mutex::new(chunks_with_seq));
244        let plan_arc = plan;
245        let schema_arc = schema.map(std::sync::Arc::new);
246        let dispatch = DispatchGuard(Arc::new((Mutex::new(Some(0)), Condvar::new())));
247        let dispatch_window = max_reorder.saturating_add(1);
248        let ordered = opts.ordered;
249        let est_row = splitter
250            .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
251            .max(512);
252
253        let bytes_fallback = if input.is_none() {
254            allowance.check(actual_bytes.len())?;
255            Some(Arc::<[u8]>::from(actual_bytes))
256        } else {
257            None
258        };
259        let (input, file) = match input {
260            #[cfg(feature = "mmap")]
261            Some(input) if matches!(&*input, InputBuffer::Mmap(_)) => {
262                let input = Arc::try_unwrap(input).map_err(|_| {
263                    crate::Error::Plan("stream input still shared during planning".into())
264                })?;
265                let InputBuffer::Mmap(handle) = input else {
266                    unreachable!()
267                };
268                drop(handle.mmap);
269                (None, Some(Arc::new(std::sync::Mutex::new(handle.file))))
270            }
271            input => (input, None::<Arc<std::sync::Mutex<std::fs::File>>>),
272        };
273        for _ in 0..n {
274            let queue = std::sync::Arc::clone(&chunk_queue);
275            let sender_clone: SyncSender<(usize, Result<RecordBatch>)> = sender.clone();
276            let parser_clone = parser.clone();
277            let plan_clone = std::sync::Arc::clone(&plan_arc);
278            let schema_clone = schema_arc.clone();
279            let input_clone = input.clone();
280            let fallback_clone = bytes_fallback.clone();
281            let file_clone = file.clone();
282            let dispatch_clone = dispatch.0.clone();
283            let handle = thread::spawn(move || -> Result<()> {
284                let bytes_ref: &[u8] = match input_clone {
285                    Some(ref inp) => inp.as_slice(),
286                    None => fallback_clone.as_deref().unwrap_or(&[]),
287                };
288                let mut chunk_buffer = Vec::new();
289                loop {
290                    let next = {
291                        let mut q = queue.lock().unwrap_or_else(|e| e.into_inner());
292                        q.pop()
293                    };
294                    let Some((seq, range)) = next else { break };
295                    if ordered {
296                        let (next, ready) = &*dispatch_clone;
297                        let next = ready
298                            .wait_while(next.lock().unwrap_or_else(|e| e.into_inner()), |next| {
299                                next.is_some_and(|next| seq >= next.saturating_add(dispatch_window))
300                            })
301                            .unwrap_or_else(|e| e.into_inner());
302                        if next.is_none() {
303                            break;
304                        }
305                    }
306                    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(
307                        || -> Result<RecordBatch> {
308                            allowance.check(range.len())?;
309                            let chunk_bytes = if let Some(file) = &file_clone {
310                                chunk_buffer.resize(range.len(), 0);
311                                allowance.check(chunk_buffer.capacity())?;
312                                let mut file = file.lock().unwrap_or_else(|e| e.into_inner());
313                                file.seek(SeekFrom::Start(range.start as u64))?;
314                                file.read_exact(&mut chunk_buffer)?;
315                                chunk_buffer.as_slice()
316                            } else {
317                                &bytes_ref[range.start..range.end]
318                            };
319                            let mut builder = TableBuilder::with_plan(
320                                (chunk_bytes.len() / est_row).max(64),
321                                plan_clone.clone(),
322                            );
323                            builder.set_memory_budget(allowance)?;
324                            // If schema is provided, pre-size columns from it.
325                            if let Some(ref schema) = schema_clone {
326                                builder.ensure_schema(schema)?;
327                                builder.check_memory_budget()?;
328                            }
329                            parser_clone.validate(chunk_bytes)?;
330                            parser_clone.parse_chunk_generic(chunk_bytes, &mut builder)?;
331                            let batch = builder.finish()?;
332                            allowance.check(batch.get_array_memory_size())?;
333                            Ok(batch)
334                        },
335                    ))
336                    .unwrap_or_else(|_| {
337                        Err(crate::Error::Parser(
338                            "worker panicked during parallel parse".into(),
339                        ))
340                    });
341                    let failed = result.is_err();
342                    if sender_clone.send((seq, result)).is_err() || failed {
343                        break;
344                    }
345                }
346                Ok(())
347            });
348            handles.push(handle);
349        }
350        drop(sender);
351
352        // Coordinator: order by seq and deliver.
353        // If `ordered`, buffer out-of-order batches until the next-in-sequence arrives.
354        // If unordered, deliver immediately.
355        let mut pending: BTreeMap<usize, RecordBatch> = BTreeMap::new();
356        let mut next_seq = 0usize;
357        let mut reorder_bytes: usize = 0;
358        let reorder_limit = max_reorder.saturating_mul(allowance.bytes());
359        let mut result = (|| -> Result<()> {
360            for (seq, res) in receiver {
361                let batch = res?;
362                if opts.ordered {
363                    if seq == next_seq {
364                        consumer.consume(batch)?;
365                        next_seq += 1;
366                        while let Some(b) = pending.remove(&next_seq) {
367                            reorder_bytes -= b.get_array_memory_size();
368                            consumer.consume(b)?;
369                            next_seq += 1;
370                        }
371                        *dispatch.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = Some(next_seq);
372                        dispatch.0 .1.notify_all();
373                    } else {
374                        reorder_bytes = reorder_bytes.saturating_add(batch.get_array_memory_size());
375                        if self.budget.is_strict() && reorder_bytes > reorder_limit {
376                            return Err(crate::Error::Memory {
377                                used: reorder_bytes,
378                                limit: reorder_limit,
379                            });
380                        }
381                        pending.insert(seq, batch);
382                    }
383                } else {
384                    consumer.consume(batch)?;
385                }
386            }
387            Ok(())
388        })();
389        drop(dispatch);
390        for h in handles {
391            let joined = h
392                .join()
393                .map_err(|_| {
394                    crate::Error::Parser(
395                        "worker panicked during parallel parse. \
396                     This usually indicates a bug in the parser (e.g., returning \
397                     Cow::Borrowed that outlives the input chunk, or an unwrap() \
398                     on None/Err during parsing). Check your RecordParser::parse_chunk \
399                     implementation for incorrect lifetime handling or missing error checks."
400                            .to_string(),
401                    )
402                })
403                .and_then(|r| r);
404            if result.is_ok() {
405                result = joined;
406            }
407        }
408        result
409    }
410}
411
412/// Iterator wrapper for parallel streaming (pull-based).
413pub struct ParallelStreamingBatchIterator {
414    receiver: Receiver<Result<RecordBatch>>,
415    handle: Option<JoinHandle<Result<()>>>,
416    done: bool,
417}
418
419impl ParallelStreamingBatchIterator {
420    pub fn new<P, S>(
421        path: PathBuf,
422        splitter: S,
423        parser: P,
424        plan: Arc<ExecutionPlan>,
425        budget: MemoryBudget,
426        prefault: bool,
427        opts: ParallelStreamOpts,
428    ) -> Self
429    where
430        P: RecordParser + Clone + Send + Sync + 'static,
431        S: Splitter + Clone + Send + Sync + 'static,
432    {
433        let num_threads = opts.threads;
434        let max_in_flight = 2 * num_threads;
435        let (sender, receiver) = sync_channel(max_in_flight);
436        let handle = thread::spawn(move || {
437            let exec = ParallelStreamingExecutor::new(budget, max_in_flight);
438            let mut consumer = ChannelConsumer {
439                sender: sender.clone(),
440            };
441            let res = exec.run_stream(
442                &path,
443                &splitter,
444                parser,
445                plan,
446                prefault,
447                opts,
448                &mut consumer,
449            );
450            if let Err(e) = res {
451                let _ = sender.send(Err(e));
452            }
453            Ok(())
454        });
455        Self {
456            receiver,
457            handle: Some(handle),
458            done: false,
459        }
460    }
461}
462
463struct ChannelConsumer {
464    sender: SyncSender<Result<RecordBatch>>,
465}
466impl BatchConsumer for ChannelConsumer {
467    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
468        self.sender
469            .send(Ok(batch))
470            .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
471    }
472}
473
474impl Iterator for ParallelStreamingBatchIterator {
475    type Item = Result<RecordBatch>;
476    fn next(&mut self) -> Option<Self::Item> {
477        if self.done {
478            return None;
479        }
480        match self.receiver.recv() {
481            Ok(Ok(batch)) => Some(Ok(batch)),
482            Ok(Err(e)) => {
483                self.done = true;
484                Some(Err(e))
485            }
486            Err(_) => {
487                self.done = true;
488                if let Some(h) = self.handle.take() {
489                    if let Err(p) = h.join() {
490                        let msg = if let Some(s) = p.downcast_ref::<&str>() {
491                            (*s).to_string()
492                        } else if let Some(s) = p.downcast_ref::<String>() {
493                            s.clone()
494                        } else {
495                            "worker panicked".to_string()
496                        };
497                        return Some(Err(crate::Error::Parser(format!(
498                            "parallel streaming worker panicked: {msg}. \
499                             This usually indicates a bug in the parser (e.g., returning \
500                             Cow::Borrowed that outlives the input chunk, or an unwrap() \
501                             on None/Err during parsing). Check your RecordParser::parse_chunk \
502                             implementation for incorrect lifetime handling or missing error checks."
503                        ))));
504                    }
505                }
506                None
507            }
508        }
509    }
510}
511
512#[cfg(test)]
513mod tests {
514    use std::borrow::Cow;
515
516    use super::*;
517
518    #[test]
519    fn channel_consumer_reports_disconnect() {
520        let (sender, receiver) = sync_channel(1);
521        drop(receiver);
522        let mut consumer = ChannelConsumer { sender };
523        let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
524        assert!(consumer.consume(batch).is_err());
525    }
526    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
527    use crate::plan::ExecutionPlan;
528    use crate::schema::clear_schema_cache;
529    use crate::value::Value;
530    use arrow::array::{Array, AsArray};
531
532    /// Minimal parser: newline-separated rows of `key=value\t...` tokens.
533    #[derive(Clone, Debug, Default)]
534    struct NameValueParser;
535
536    impl RecordParser for NameValueParser {
537        fn validate(&self, _bytes: &[u8]) -> Result<()> {
538            Ok(())
539        }
540
541        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
542            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
543            for line in text.lines() {
544                if line.is_empty() {
545                    continue;
546                }
547                sink.begin_row();
548                for token in line.split('\t') {
549                    if let Some((k, v)) = token.split_once('=') {
550                        sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
551                    }
552                }
553                sink.end_row();
554            }
555            Ok(())
556        }
557    }
558
559    /// Trivial splitter: one chunk covering the whole input.
560    struct NullSplitter;
561
562    impl Splitter for NullSplitter {
563        fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
564            None
565        }
566
567        fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
568            vec![0, bytes.len()]
569        }
570
571        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
572            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
573            sample.len() / n
574        }
575    }
576
577    struct TwoChunkSplitter;
578    impl Splitter for TwoChunkSplitter {
579        fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
580            None
581        }
582        fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
583            vec![0, 4.min(bytes.len()), bytes.len()]
584        }
585        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
586            5
587        }
588    }
589
590    struct VecConsumer(Vec<RecordBatch>);
591    impl BatchConsumer for VecConsumer {
592        fn consume(&mut self, batch: RecordBatch) -> Result<()> {
593            self.0.push(batch);
594            Ok(())
595        }
596    }
597
598    #[derive(Clone)]
599    struct DelayedParser;
600    impl RecordParser for DelayedParser {
601        fn validate(&self, bytes: &[u8]) -> Result<()> {
602            NameValueParser.validate(bytes)
603        }
604        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
605            if bytes.starts_with(b"a=1") {
606                std::thread::sleep(std::time::Duration::from_millis(30));
607            }
608            NameValueParser.parse_chunk(bytes, sink)
609        }
610    }
611
612    #[test]
613    fn ordered_soft_stream_tolerates_oversized_pending_batch() {
614        let exec = ParallelStreamingExecutor::new(MemoryBudget::new(1), 2);
615        let mut consumer = VecConsumer(Vec::new());
616        let opts = ParallelStreamOpts {
617            threads: 2,
618            max_reorder: 1,
619            ..Default::default()
620        };
621        let result = exec.run_bytes_stream(
622            b"a=1\nb=2\n",
623            &TwoChunkSplitter,
624            DelayedParser,
625            Arc::new(ExecutionPlan::new()),
626            opts,
627            &mut consumer,
628        );
629        result.unwrap();
630        assert_eq!(
631            consumer.0.iter().map(RecordBatch::num_rows).sum::<usize>(),
632            2
633        );
634        assert_eq!(
635            consumer.0[0]
636                .column_by_name("a")
637                .unwrap()
638                .as_string::<i32>()
639                .value(0),
640            "1"
641        );
642        assert_eq!(
643            consumer.0[1]
644                .column_by_name("b")
645                .unwrap()
646                .as_string::<i32>()
647                .value(0),
648            "2"
649        );
650    }
651
652    fn names(schema: &FrozenSchema) -> Vec<&str> {
653        schema.column_names().iter().map(|s| s.as_ref()).collect()
654    }
655
656    #[test]
657    fn schema_cache_hits_misses_and_plan_changes() {
658        clear_schema_cache();
659        let splitter = NullSplitter;
660        let parser = NameValueParser;
661
662        // First layout: miss.
663        let bytes = b"a=1\tb=2\na=3\tb=4\n";
664        let plan = ExecutionPlan::new();
665        let s1 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
666        assert_eq!(names(&s1), vec!["a", "b"]);
667
668        // Same layout: hit (cached).
669        let s2 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
670        assert_eq!(names(&s2), vec!["a", "b"]);
671
672        // Different layout: new discovery.
673        let bytes2 = b"c=1\td=2\n";
674        let s3 = discover_schema_for_bytes(bytes2, &splitter, &parser, &plan);
675        assert_eq!(names(&s3), vec!["c", "d"]);
676
677        // Same layout as the first parse but with a different plan:
678        // cache hit, but the applied schema differs.
679        let mut renamed = ExecutionPlan::new();
680        renamed.field_map.insert("a".to_string(), "x".to_string());
681        let s4 = discover_schema_for_bytes(bytes, &splitter, &parser, &renamed);
682        assert_eq!(names(&s4), vec!["x", "b"]);
683
684        clear_schema_cache();
685    }
686
687    #[test]
688    fn tail_scan_catches_late_columns() {
689        // Create data where column "late" only appears in the last few rows.
690        // With 16 windows of 2 MiB each, the last window starts at ~94%.
691        // Column "late" appears only in the last ~5% of the data.
692        let mut data = Vec::new();
693        // First 90%: only columns "a" and "b"
694        for i in 0..900 {
695            data.extend_from_slice(format!("a={i}\tb=val{i}\n").as_bytes());
696        }
697        // Last 10%: introduce column "late"
698        for i in 900..1000 {
699            data.extend_from_slice(format!("a={i}\tb=val{i}\tlate=new{i}\n").as_bytes());
700        }
701
702        let splitter = NullSplitter;
703        let parser = NameValueParser;
704        let plan = ExecutionPlan::new();
705        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
706
707        // Column "late" must be discovered
708        let cols = names(&schema);
709        assert!(cols.contains(&"a"), "column 'a' missing");
710        assert!(cols.contains(&"b"), "column 'b' missing");
711        assert!(
712            cols.contains(&"late"),
713            "column 'late' missing from tail scan"
714        );
715
716        // Verify all values are preserved by parsing the full data
717        let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
718        builder.ensure_schema(&schema).unwrap();
719        parser.parse_chunk(&data, &mut builder).unwrap();
720        let batch = builder.finish().unwrap();
721
722        assert_eq!(batch.num_rows(), 1000);
723        assert_eq!(batch.num_columns(), 3);
724
725        // Check that the "late" column has values in the last 100 rows
726        let late_col = batch.column_by_name("late").expect("late column missing");
727        let late_arr = late_col
728            .as_any()
729            .downcast_ref::<arrow::array::StringArray>()
730            .expect("late column not string array");
731        // First 900 rows should be null for "late"
732        for i in 0..900 {
733            assert!(
734                late_arr.is_null(i),
735                "row {i}: expected null for 'late', got {:?}",
736                late_arr.value(i)
737            );
738        }
739        // Last 100 rows should have values
740        for i in 900..1000 {
741            assert!(
742                !late_arr.is_null(i),
743                "row {i}: expected value for 'late', got null"
744            );
745            assert!(
746                late_arr.value(i).starts_with("new"),
747                "row {i}: expected 'new{i}', got {:?}",
748                late_arr.value(i)
749            );
750        }
751    }
752
753    #[test]
754    fn all_columns_discovered_no_data_loss() {
755        // Test that all columns from a file are discovered, even with sparse data
756        let mut data = Vec::new();
757        // Mix of columns across rows
758        let rows: Vec<&[u8]> = vec![
759            b"a=1\tb=2\tc=3\n",
760            b"b=5\td=6\n",
761            b"a=7\tc=8\te=9\n",
762            b"f=10\n",
763            b"a=11\tb=12\tc=13\td=14\te=15\tf=16\n",
764        ];
765        for row in &rows {
766            data.extend_from_slice(row);
767        }
768
769        let splitter = NullSplitter;
770        let parser = NameValueParser;
771        let plan = ExecutionPlan::new();
772        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
773
774        // All 6 columns must be discovered
775        let cols = names(&schema);
776        assert_eq!(cols.len(), 6, "expected 6 columns, got {:?}", cols);
777        assert!(cols.contains(&"a"));
778        assert!(cols.contains(&"b"));
779        assert!(cols.contains(&"c"));
780        assert!(cols.contains(&"d"));
781        assert!(cols.contains(&"e"));
782        assert!(cols.contains(&"f"));
783
784        // Parse all data and verify values
785        let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
786        builder.ensure_schema(&schema).unwrap();
787        parser.parse_chunk(&data, &mut builder).unwrap();
788        let batch = builder.finish().unwrap();
789
790        assert_eq!(batch.num_rows(), 5);
791        assert_eq!(batch.num_columns(), 6);
792
793        // Verify specific values
794        let a_col = batch.column_by_name("a").unwrap();
795        let a_arr = a_col
796            .as_any()
797            .downcast_ref::<arrow::array::StringArray>()
798            .unwrap();
799        assert_eq!(a_arr.value(0), "1");
800        assert_eq!(a_arr.value(2), "7");
801        assert_eq!(a_arr.value(4), "11");
802    }
803
804    #[test]
805    fn dynamic_sizing_discovers_same_columns() {
806        // Verify that dynamic window sizing discovers the same columns
807        // as the fixed 16-window approach
808        let mut data = Vec::new();
809        // Create data with columns spread across the file
810        for i in 0..1000 {
811            if i < 100 {
812                // First 10%: columns a, b
813                data.extend_from_slice(format!("a={i}\tb={i}\n").as_bytes());
814            } else if i < 500 {
815                // Middle 40%: columns a, b, c
816                data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\n").as_bytes());
817            } else if i < 900 {
818                // Next 40%: columns a, b, c, d
819                data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\n").as_bytes());
820            } else {
821                // Last 10%: columns a, b, c, d, e (late column)
822                data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\te={i}\n").as_bytes());
823            }
824        }
825
826        let splitter = NullSplitter;
827        let parser = NameValueParser;
828        let plan = ExecutionPlan::new();
829        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
830
831        // All 5 columns must be discovered
832        let cols = names(&schema);
833        assert_eq!(cols.len(), 5, "expected 5 columns, got {:?}", cols);
834        assert!(cols.contains(&"a"));
835        assert!(cols.contains(&"b"));
836        assert!(cols.contains(&"c"));
837        assert!(cols.contains(&"d"));
838        assert!(cols.contains(&"e"));
839
840        // Parse and verify
841        let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
842        builder.ensure_schema(&schema).unwrap();
843        parser.parse_chunk(&data, &mut builder).unwrap();
844        let batch = builder.finish().unwrap();
845
846        assert_eq!(batch.num_rows(), 1000);
847        assert_eq!(batch.num_columns(), 5);
848    }
849
850    #[test]
851    fn column_order_consistent_across_file_sizes() {
852        // Verify that column order is consistent regardless of file size
853        // This ensures no data reordering issues
854        let make_data = |n: usize| -> Vec<u8> {
855            let mut data = Vec::new();
856            for i in 0..n {
857                data.extend_from_slice(format!("x={i}\ty={i}\tz={i}\n").as_bytes());
858            }
859            data
860        };
861
862        let splitter = NullSplitter;
863        let parser = NameValueParser;
864        let plan = ExecutionPlan::new();
865
866        // Test with different file sizes
867        let sizes = vec![10, 100, 1000];
868        let mut first_order: Option<Vec<String>> = None;
869
870        for size in sizes {
871            let data = make_data(size);
872            let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
873            let cols: Vec<String> = schema
874                .column_names()
875                .iter()
876                .map(|s| s.to_string())
877                .collect();
878
879            if let Some(ref order) = first_order {
880                assert_eq!(
881                    &cols, order,
882                    "column order changed for size {size}: {cols:?} vs {order:?}"
883                );
884            } else {
885                first_order = Some(cols);
886            }
887        }
888    }
889
890    #[test]
891    fn esoteric_sparse_schema_one_column_per_row() {
892        // Each row has a unique column name - extremely sparse
893        let mut data = Vec::new();
894        for i in 0..100 {
895            data.extend_from_slice(format!("col_{i}={i}\n").as_bytes());
896        }
897
898        let splitter = NullSplitter;
899        let parser = NameValueParser;
900        let plan = ExecutionPlan::new();
901        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
902
903        // All 100 columns should be discovered
904        let cols = names(&schema);
905        assert_eq!(
906            cols.len(),
907            100,
908            "expected 100 columns, got {:?}",
909            cols.len()
910        );
911    }
912
913    #[test]
914    fn esoteric_empty_rows() {
915        // File with many empty rows interspersed
916        let mut data = Vec::new();
917        for i in 0..50 {
918            data.extend_from_slice(format!("a={i}\n").as_bytes());
919            data.extend_from_slice(b"\n"); // empty row
920            data.extend_from_slice(b"\n"); // another empty row
921            data.extend_from_slice(format!("b={i}\n").as_bytes());
922        }
923
924        let splitter = NullSplitter;
925        let parser = NameValueParser;
926        let plan = ExecutionPlan::new();
927        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
928
929        let cols = names(&schema);
930        assert!(cols.contains(&"a"), "column 'a' missing");
931        assert!(cols.contains(&"b"), "column 'b' missing");
932    }
933
934    #[test]
935    fn esoteric_single_character_columns() {
936        // Very short column names
937        let data = b"a=1\tb=2\tc=3\nd=4\te=5\n";
938
939        let splitter = NullSplitter;
940        let parser = NameValueParser;
941        let plan = ExecutionPlan::new();
942        let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
943
944        let cols = names(&schema);
945        assert_eq!(cols.len(), 5);
946        assert!(cols.contains(&"a"));
947        assert!(cols.contains(&"e"));
948    }
949
950    #[test]
951    fn esoteric_unicode_column_names() {
952        // Unicode characters in column names
953        let data = "名前=Alice\t年齢=30\n名前=Bob\t年齢=25\n".as_bytes();
954
955        let splitter = NullSplitter;
956        let parser = NameValueParser;
957        let plan = ExecutionPlan::new();
958        let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
959
960        let cols = names(&schema);
961        assert_eq!(cols.len(), 2);
962        assert!(cols.contains(&"名前"));
963        assert!(cols.contains(&"年齢"));
964    }
965
966    #[test]
967    fn esoteric_very_long_column_names() {
968        // Very long column names (100+ chars)
969        let long_name = "a".repeat(200);
970        let data = format!("{long_name}=1\tb=2\n{long_name}=3\tc=4\n");
971
972        let splitter = NullSplitter;
973        let parser = NameValueParser;
974        let plan = ExecutionPlan::new();
975        let schema = discover_schema_for_bytes(data.as_bytes(), &splitter, &parser, &plan);
976
977        let cols = names(&schema);
978        assert_eq!(cols.len(), 3);
979        assert!(cols.contains(&long_name.as_str()));
980    }
981
982    #[test]
983    fn esoteric_duplicate_columns_in_row() {
984        // Same column appears multiple times in a row
985        let data = b"a=1\ta=2\ta=3\tb=4\n";
986
987        let splitter = NullSplitter;
988        let parser = NameValueParser;
989        let plan = ExecutionPlan::new();
990        let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
991
992        let cols = names(&schema);
993        assert_eq!(cols.len(), 2, "expected 2 unique columns, got {:?}", cols);
994        assert!(cols.contains(&"a"));
995        assert!(cols.contains(&"b"));
996    }
997}