Skip to main content

rypipe_core/
parallel_stream.rs

1//! Parallel streaming executor: multi-core parsing with bounded memory.
2
3use std::collections::BTreeMap;
4use std::io::{Read, Seek, SeekFrom};
5use std::path::{Path, PathBuf};
6use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
7use std::sync::{Arc, Condvar, Mutex};
8use std::thread::{self, JoinHandle};
9
10use arrow::record_batch::RecordBatch;
11
12use std::sync::atomic::{AtomicU64, Ordering};
13
14use crate::bounded::MemoryBudget;
15use crate::consumer::BatchConsumer;
16use crate::decoder::{RecordParser, Splitter};
17use crate::engine::TableBuilder;
18use crate::input::InputBuffer;
19use crate::plan::ExecutionPlan;
20use crate::schema::{DiscoveryOpts, FrozenSchema};
21use crate::Result;
22
23mod discovery;
24
25pub(crate) use discovery::discover_schema;
26pub use discovery::{
27    discover_schema_for_bytes, discovery_profile, reset_discovery_profile, ParallelStreamOpts,
28};
29
30/// Parallel streaming executor: parses chunks concurrently with bounded memory.
31pub struct ParallelStreamingExecutor {
32    budget: MemoryBudget,
33    max_in_flight: usize,
34}
35
36type DispatchState = Arc<(Mutex<Option<usize>>, Condvar)>;
37
38struct DispatchGuard(DispatchState);
39
40impl Drop for DispatchGuard {
41    fn drop(&mut self) {
42        *self.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = None;
43        self.0 .1.notify_all();
44    }
45}
46
47impl ParallelStreamingExecutor {
48    pub fn new(budget: MemoryBudget, max_in_flight: usize) -> Self {
49        Self {
50            budget,
51            max_in_flight: max_in_flight.max(1),
52        }
53    }
54
55    /// Stream a file in parallel, calling `consumer` per batch in order.
56    #[allow(clippy::too_many_arguments)]
57    pub fn run_stream<P, C>(
58        &self,
59        path: &Path,
60        splitter: &dyn Splitter,
61        parser: P,
62        plan: Arc<ExecutionPlan>,
63        prefault: bool,
64        opts: ParallelStreamOpts,
65        consumer: &mut C,
66    ) -> Result<()>
67    where
68        P: RecordParser + Clone + Send + Sync + 'static,
69        C: BatchConsumer,
70    {
71        let input = InputBuffer::open(path, cfg!(feature = "mmap"), prefault)?;
72        if input.is_empty() {
73            return Ok(());
74        }
75        #[cfg(not(feature = "mmap"))]
76        self.budget.share(4).check(input.len())?;
77        #[cfg(feature = "mmap")]
78        let opts = if let InputBuffer::Mmap(handle) = &input {
79            let mut opts = opts;
80            if opts.schema.is_none() && plan.schema_order.is_empty() {
81                opts.schema = Some(discovery::discover_schema_for_file(
82                    handle,
83                    splitter,
84                    &parser,
85                    &plan,
86                    self.budget,
87                )?);
88            }
89            opts
90        } else {
91            self.budget.share(4).check(input.len())?;
92            opts
93        };
94        // Share the InputBuffer across workers via Arc to avoid
95        // the O(file_size × threads) to_vec() clone.
96        let shared = std::sync::Arc::new(input);
97        self.run_bytes_stream_shared(shared, splitter, parser, plan, opts, consumer)
98    }
99
100    /// Plan from shared input, then release file mappings before parsing.
101    fn run_bytes_stream_shared<P, C>(
102        &self,
103        input: std::sync::Arc<InputBuffer>,
104        splitter: &dyn Splitter,
105        parser: P,
106        plan: Arc<ExecutionPlan>,
107        opts: ParallelStreamOpts,
108        consumer: &mut C,
109    ) -> Result<()>
110    where
111        P: RecordParser + Clone + Send + Sync + 'static,
112        C: BatchConsumer,
113    {
114        // Safety: the Arc keeps the InputBuffer alive for the duration.
115        // We pass a dummy &[u8] to satisfy the signature; run_bytes_stream_core
116        // ignores it when input is Some and uses input.as_slice() instead.
117        let dummy: &[u8] = &[];
118        self.run_bytes_stream_core(dummy, Some(input), splitter, parser, plan, opts, consumer)
119    }
120
121    /// Stream bytes in parallel.
122    pub fn run_bytes_stream<P, C>(
123        &self,
124        bytes: &[u8],
125        splitter: &dyn Splitter,
126        parser: P,
127        plan: Arc<ExecutionPlan>,
128        opts: ParallelStreamOpts,
129        consumer: &mut C,
130    ) -> Result<()>
131    where
132        P: RecordParser + Clone + Send + Sync + 'static,
133        C: BatchConsumer,
134    {
135        self.run_bytes_stream_core(bytes, None, splitter, parser, plan, opts, consumer)
136    }
137
138    /// Core implementation shared by `run_bytes_stream` and `run_bytes_stream_shared`.
139    #[allow(clippy::too_many_arguments)]
140    fn run_bytes_stream_core<P, C>(
141        &self,
142        bytes: &[u8],
143        input: Option<std::sync::Arc<InputBuffer>>,
144        splitter: &dyn Splitter,
145        parser: P,
146        plan: Arc<ExecutionPlan>,
147        opts: ParallelStreamOpts,
148        consumer: &mut C,
149    ) -> Result<()>
150    where
151        P: RecordParser + Clone + Send + Sync + 'static,
152        C: BatchConsumer,
153    {
154        // Prefer the shared InputBuffer's slice over the passed-in bytes.
155        let actual_bytes = match input {
156            Some(ref inp) => inp.as_slice(),
157            None => bytes,
158        };
159        if actual_bytes.is_empty() {
160            return Ok(());
161        }
162        let n = opts.threads.max(1);
163        let max_reorder = if opts.max_reorder > 0 {
164            opts.max_reorder
165        } else {
166            n
167        };
168        // Frozen schema: gates correct ParquetWriter / StreamWriter usage.
169        // Without it, batch 2 can have different column order (FieldG vs Text20
170        // last) even when the set is identical, breaking `write_batch`.
171        // If opts.schema is None, auto-discover:
172        //  - explicit plan.schema_order → from_partial_plan (allows unknown columns)
173        //  - else sampled discovery (16×2 MiB windows for >128 MiB, else full)
174        let schema: Option<FrozenSchema> = match opts.schema {
175            Some(s) => Some(s),
176            None => {
177                if !plan.schema_order.is_empty() {
178                    let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
179                    Some(FrozenSchema::from_partial_plan(&names, &plan))
180                } else {
181                    let opts = DiscoveryOpts::default();
182                    let sig = crate::schema::layout_signature(actual_bytes, &opts);
183                    let cached = crate::schema::SCHEMA_CACHE
184                        .read()
185                        .unwrap_or_else(|e| e.into_inner())
186                        .get(&sig)
187                        .cloned();
188                    let (schema, order) = if let Some(order) = cached {
189                        crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
190                        (FrozenSchema::from_discovered(&order, &plan), order)
191                    } else {
192                        crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
193                        let (schema, order) =
194                            std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
195                                discover_schema(actual_bytes, &parser, &plan, splitter)
196                            }))
197                            .map_err(|_| {
198                                crate::Error::Parser(
199                                    "parser panicked during schema discovery".into(),
200                                )
201                            })?;
202                        (schema, Arc::new(order))
203                    };
204                    if !order.is_empty() {
205                        crate::schema::insert_schema_cache(sig, order);
206                    }
207                    Some(schema)
208                }
209            }
210        };
211
212        let bytes_per_row = splitter
213            .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
214            .max(1);
215        let slots = n
216            .saturating_mul(3)
217            .saturating_add(self.max_in_flight.saturating_mul(2))
218            .saturating_add(max_reorder)
219            .saturating_add(1);
220        let allowance = self.budget.share(slots);
221        let chunk_size = (self.budget.bytes() / n.saturating_mul(32))
222            .max(bytes_per_row.saturating_mul(10))
223            .max(if self.budget.is_strict() {
224                1
225            } else {
226                64 * 1024
227            });
228        let num_chunks = actual_bytes.len().div_ceil(chunk_size).max(n).min(10000);
229        let split_points = splitter.find_split_points_with(
230            actual_bytes,
231            num_chunks,
232            plan.min_chunk_bytes
233                .unwrap_or(crate::decoder::MIN_CHUNK_BYTES),
234        );
235        let mut ranges = crate::decoder::split_points_to_ranges(&split_points, actual_bytes.len());
236        if ranges.is_empty() {
237            ranges.push(0..actual_bytes.len());
238        }
239
240        let chunks_with_seq: Vec<(usize, std::ops::Range<usize>)> =
241            ranges.into_iter().enumerate().rev().collect();
242        allowance.check(
243            chunks_with_seq.capacity() * std::mem::size_of::<(usize, std::ops::Range<usize>)>(),
244        )?;
245
246        let (sender, receiver) = sync_channel(self.max_in_flight);
247        let mut handles: Vec<JoinHandle<Result<()>>> = Vec::with_capacity(n);
248        let chunk_queue = std::sync::Arc::new(std::sync::Mutex::new(chunks_with_seq));
249        let plan_arc = plan;
250        let schema_arc = schema.map(std::sync::Arc::new);
251        let dispatch = DispatchGuard(Arc::new((Mutex::new(Some(0)), Condvar::new())));
252        let dispatch_window = max_reorder.saturating_add(1);
253        let ordered = opts.ordered;
254        let est_row = splitter
255            .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
256            .max(512);
257
258        let bytes_fallback = if input.is_none() {
259            allowance.check(actual_bytes.len())?;
260            Some(Arc::<[u8]>::from(actual_bytes))
261        } else {
262            None
263        };
264        let (input, file) = match input {
265            #[cfg(feature = "mmap")]
266            Some(input) if matches!(&*input, InputBuffer::Mmap(_)) => {
267                let input = Arc::try_unwrap(input).map_err(|_| {
268                    crate::Error::Plan("stream input still shared during planning".into())
269                })?;
270                let InputBuffer::Mmap(handle) = input else {
271                    unreachable!()
272                };
273                drop(handle.mmap);
274                (None, Some(Arc::new(std::sync::Mutex::new(handle.file))))
275            }
276            input => (input, None::<Arc<std::sync::Mutex<std::fs::File>>>),
277        };
278        for _ in 0..n {
279            let queue = std::sync::Arc::clone(&chunk_queue);
280            let sender_clone: SyncSender<(usize, Result<RecordBatch>)> = sender.clone();
281            let parser_clone = parser.clone();
282            let plan_clone = std::sync::Arc::clone(&plan_arc);
283            let schema_clone = schema_arc.clone();
284            let input_clone = input.clone();
285            let fallback_clone = bytes_fallback.clone();
286            let file_clone = file.clone();
287            let dispatch_clone = dispatch.0.clone();
288            let handle = thread::spawn(move || -> Result<()> {
289                let bytes_ref: &[u8] = match input_clone {
290                    Some(ref inp) => inp.as_slice(),
291                    None => fallback_clone.as_deref().unwrap_or(&[]),
292                };
293                let mut chunk_buffer = Vec::new();
294                loop {
295                    let next = {
296                        let mut q = queue.lock().unwrap_or_else(|e| e.into_inner());
297                        q.pop()
298                    };
299                    let Some((seq, range)) = next else { break };
300                    if ordered {
301                        let (next, ready) = &*dispatch_clone;
302                        let next = ready
303                            .wait_while(next.lock().unwrap_or_else(|e| e.into_inner()), |next| {
304                                next.is_some_and(|next| seq >= next.saturating_add(dispatch_window))
305                            })
306                            .unwrap_or_else(|e| e.into_inner());
307                        if next.is_none() {
308                            break;
309                        }
310                    }
311                    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(
312                        || -> Result<RecordBatch> {
313                            allowance.check(range.len())?;
314                            let chunk_bytes = if let Some(file) = &file_clone {
315                                chunk_buffer.resize(range.len(), 0);
316                                allowance.check(chunk_buffer.capacity())?;
317                                let mut file = file.lock().unwrap_or_else(|e| e.into_inner());
318                                file.seek(SeekFrom::Start(range.start as u64))?;
319                                file.read_exact(&mut chunk_buffer)?;
320                                chunk_buffer.as_slice()
321                            } else {
322                                &bytes_ref[range.start..range.end]
323                            };
324                            let mut builder = TableBuilder::with_plan(
325                                (chunk_bytes.len() / est_row).max(64),
326                                plan_clone.clone(),
327                            );
328                            builder.set_memory_budget(allowance)?;
329                            // If schema is provided, pre-size columns from it.
330                            if let Some(ref schema) = schema_clone {
331                                builder.ensure_schema(schema)?;
332                                builder.check_memory_budget()?;
333                            }
334                            parser_clone.validate(chunk_bytes)?;
335                            parser_clone.parse_chunk_generic(chunk_bytes, &mut builder)?;
336                            let batch = builder.finish()?;
337                            allowance.check(batch.get_array_memory_size())?;
338                            Ok(batch)
339                        },
340                    ))
341                    .unwrap_or_else(|_| {
342                        Err(crate::Error::Parser(
343                            "worker panicked during parallel parse".into(),
344                        ))
345                    });
346                    let failed = result.is_err();
347                    if sender_clone.send((seq, result)).is_err() || failed {
348                        break;
349                    }
350                }
351                Ok(())
352            });
353            handles.push(handle);
354        }
355        drop(sender);
356
357        // Coordinator: order by seq and deliver.
358        // If `ordered`, buffer out-of-order batches until the next-in-sequence arrives.
359        // If unordered, deliver immediately.
360        let mut pending: BTreeMap<usize, RecordBatch> = BTreeMap::new();
361        let mut next_seq = 0usize;
362        let mut reorder_bytes: usize = 0;
363        let reorder_limit = max_reorder.saturating_mul(allowance.bytes());
364        let mut result = (|| -> Result<()> {
365            for (seq, res) in receiver {
366                let batch = res?;
367                if opts.ordered {
368                    if seq == next_seq {
369                        consumer.consume(batch)?;
370                        next_seq += 1;
371                        while let Some(b) = pending.remove(&next_seq) {
372                            reorder_bytes -= b.get_array_memory_size();
373                            consumer.consume(b)?;
374                            next_seq += 1;
375                        }
376                        *dispatch.0 .0.lock().unwrap_or_else(|e| e.into_inner()) = Some(next_seq);
377                        dispatch.0 .1.notify_all();
378                    } else {
379                        reorder_bytes = reorder_bytes.saturating_add(batch.get_array_memory_size());
380                        if self.budget.is_strict() && reorder_bytes > reorder_limit {
381                            return Err(crate::Error::Memory {
382                                used: reorder_bytes,
383                                limit: reorder_limit,
384                            });
385                        }
386                        pending.insert(seq, batch);
387                    }
388                } else {
389                    consumer.consume(batch)?;
390                }
391            }
392            Ok(())
393        })();
394        drop(dispatch);
395        for h in handles {
396            let joined = h
397                .join()
398                .map_err(|_| {
399                    crate::Error::Parser(
400                        "worker panicked during parallel parse. \
401                     This usually indicates a bug in the parser (e.g., returning \
402                     Cow::Borrowed that outlives the input chunk, or an unwrap() \
403                     on None/Err during parsing). Check your RecordParser::parse_chunk \
404                     implementation for incorrect lifetime handling or missing error checks."
405                            .to_string(),
406                    )
407                })
408                .and_then(|r| r);
409            if result.is_ok() {
410                result = joined;
411            }
412        }
413        result
414    }
415}
416
417/// Iterator wrapper for parallel streaming (pull-based).
418pub struct ParallelStreamingBatchIterator {
419    receiver: Receiver<Result<RecordBatch>>,
420    handle: Option<JoinHandle<Result<()>>>,
421    done: bool,
422}
423
424impl ParallelStreamingBatchIterator {
425    pub fn new<P, S>(
426        path: PathBuf,
427        splitter: S,
428        parser: P,
429        plan: Arc<ExecutionPlan>,
430        budget: MemoryBudget,
431        prefault: bool,
432        opts: ParallelStreamOpts,
433    ) -> Self
434    where
435        P: RecordParser + Clone + Send + Sync + 'static,
436        S: Splitter + Clone + Send + Sync + 'static,
437    {
438        let num_threads = opts.threads;
439        let max_in_flight = 2 * num_threads;
440        let (sender, receiver) = sync_channel(max_in_flight);
441        let handle = thread::spawn(move || {
442            let exec = ParallelStreamingExecutor::new(budget, max_in_flight);
443            let mut consumer = ChannelConsumer {
444                sender: sender.clone(),
445            };
446            let res = exec.run_stream(
447                &path,
448                &splitter,
449                parser,
450                plan,
451                prefault,
452                opts,
453                &mut consumer,
454            );
455            if let Err(e) = res {
456                let _ = sender.send(Err(e));
457            }
458            Ok(())
459        });
460        Self {
461            receiver,
462            handle: Some(handle),
463            done: false,
464        }
465    }
466}
467
468struct ChannelConsumer {
469    sender: SyncSender<Result<RecordBatch>>,
470}
471impl BatchConsumer for ChannelConsumer {
472    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
473        self.sender
474            .send(Ok(batch))
475            .map_err(|_| crate::Error::Parser("stream consumer disconnected".into()))
476    }
477}
478
479impl Iterator for ParallelStreamingBatchIterator {
480    type Item = Result<RecordBatch>;
481    fn next(&mut self) -> Option<Self::Item> {
482        if self.done {
483            return None;
484        }
485        match self.receiver.recv() {
486            Ok(Ok(batch)) => Some(Ok(batch)),
487            Ok(Err(e)) => {
488                self.done = true;
489                Some(Err(e))
490            }
491            Err(_) => {
492                self.done = true;
493                if let Some(h) = self.handle.take() {
494                    if let Err(p) = h.join() {
495                        let msg = if let Some(s) = p.downcast_ref::<&str>() {
496                            (*s).to_string()
497                        } else if let Some(s) = p.downcast_ref::<String>() {
498                            s.clone()
499                        } else {
500                            "worker panicked".to_string()
501                        };
502                        return Some(Err(crate::Error::Parser(format!(
503                            "parallel streaming worker panicked: {msg}. \
504                             This usually indicates a bug in the parser (e.g., returning \
505                             Cow::Borrowed that outlives the input chunk, or an unwrap() \
506                             on None/Err during parsing). Check your RecordParser::parse_chunk \
507                             implementation for incorrect lifetime handling or missing error checks."
508                        ))));
509                    }
510                }
511                None
512            }
513        }
514    }
515}
516
517#[cfg(test)]
518mod tests {
519    use std::borrow::Cow;
520
521    use super::*;
522
523    #[test]
524    fn channel_consumer_reports_disconnect() {
525        let (sender, receiver) = sync_channel(1);
526        drop(receiver);
527        let mut consumer = ChannelConsumer { sender };
528        let batch = RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty()));
529        assert!(consumer.consume(batch).is_err());
530    }
531    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
532    use crate::plan::ExecutionPlan;
533    use crate::schema::clear_schema_cache;
534    use crate::value::Value;
535    use arrow::array::{Array, AsArray};
536
537    /// Minimal parser: newline-separated rows of `key=value\t...` tokens.
538    #[derive(Clone, Debug, Default)]
539    struct NameValueParser;
540
541    impl RecordParser for NameValueParser {
542        fn validate(&self, _bytes: &[u8]) -> Result<()> {
543            Ok(())
544        }
545
546        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
547            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
548            for line in text.lines() {
549                if line.is_empty() {
550                    continue;
551                }
552                sink.begin_row();
553                for token in line.split('\t') {
554                    if let Some((k, v)) = token.split_once('=') {
555                        sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
556                    }
557                }
558                sink.end_row();
559            }
560            Ok(())
561        }
562    }
563
564    /// Trivial splitter: one chunk covering the whole input.
565    struct NullSplitter;
566
567    impl Splitter for NullSplitter {
568        fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
569            None
570        }
571
572        fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
573            vec![0, bytes.len()]
574        }
575
576        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
577            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
578            sample.len() / n
579        }
580    }
581
582    struct TwoChunkSplitter;
583    impl Splitter for TwoChunkSplitter {
584        fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
585            None
586        }
587        fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
588            vec![0, 4.min(bytes.len()), bytes.len()]
589        }
590        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
591            5
592        }
593    }
594
595    struct VecConsumer(Vec<RecordBatch>);
596    impl BatchConsumer for VecConsumer {
597        fn consume(&mut self, batch: RecordBatch) -> Result<()> {
598            self.0.push(batch);
599            Ok(())
600        }
601    }
602
603    #[derive(Clone)]
604    struct DelayedParser;
605    impl RecordParser for DelayedParser {
606        fn validate(&self, bytes: &[u8]) -> Result<()> {
607            NameValueParser.validate(bytes)
608        }
609        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
610            if bytes.starts_with(b"a=1") {
611                std::thread::sleep(std::time::Duration::from_millis(30));
612            }
613            NameValueParser.parse_chunk(bytes, sink)
614        }
615    }
616
617    #[test]
618    fn ordered_soft_stream_tolerates_oversized_pending_batch() {
619        let exec = ParallelStreamingExecutor::new(MemoryBudget::new(1), 2);
620        let mut consumer = VecConsumer(Vec::new());
621        let opts = ParallelStreamOpts {
622            threads: 2,
623            max_reorder: 1,
624            ..Default::default()
625        };
626        let result = exec.run_bytes_stream(
627            b"a=1\nb=2\n",
628            &TwoChunkSplitter,
629            DelayedParser,
630            Arc::new(ExecutionPlan::new()),
631            opts,
632            &mut consumer,
633        );
634        result.unwrap();
635        assert_eq!(
636            consumer.0.iter().map(RecordBatch::num_rows).sum::<usize>(),
637            2
638        );
639        assert_eq!(
640            consumer.0[0]
641                .column_by_name("a")
642                .unwrap()
643                .as_string::<i32>()
644                .value(0),
645            "1"
646        );
647        assert_eq!(
648            consumer.0[1]
649                .column_by_name("b")
650                .unwrap()
651                .as_string::<i32>()
652                .value(0),
653            "2"
654        );
655    }
656
657    fn names(schema: &FrozenSchema) -> Vec<&str> {
658        schema.column_names().iter().map(|s| s.as_ref()).collect()
659    }
660
661    #[test]
662    fn schema_cache_hits_misses_and_plan_changes() {
663        clear_schema_cache();
664        let splitter = NullSplitter;
665        let parser = NameValueParser;
666
667        // First layout: miss.
668        let bytes = b"a=1\tb=2\na=3\tb=4\n";
669        let plan = ExecutionPlan::new();
670        let s1 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
671        assert_eq!(names(&s1), vec!["a", "b"]);
672
673        // Same layout: hit (cached).
674        let s2 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
675        assert_eq!(names(&s2), vec!["a", "b"]);
676
677        // Different layout: new discovery.
678        let bytes2 = b"c=1\td=2\n";
679        let s3 = discover_schema_for_bytes(bytes2, &splitter, &parser, &plan);
680        assert_eq!(names(&s3), vec!["c", "d"]);
681
682        // Same layout as the first parse but with a different plan:
683        // cache hit, but the applied schema differs.
684        let mut renamed = ExecutionPlan::new();
685        renamed.field_map.insert("a".to_string(), "x".to_string());
686        let s4 = discover_schema_for_bytes(bytes, &splitter, &parser, &renamed);
687        assert_eq!(names(&s4), vec!["x", "b"]);
688
689        clear_schema_cache();
690    }
691
692    #[test]
693    fn tail_scan_catches_late_columns() {
694        // Create data where column "late" only appears in the last few rows.
695        // With 16 windows of 2 MiB each, the last window starts at ~94%.
696        // Column "late" appears only in the last ~5% of the data.
697        let mut data = Vec::new();
698        // First 90%: only columns "a" and "b"
699        for i in 0..900 {
700            data.extend_from_slice(format!("a={i}\tb=val{i}\n").as_bytes());
701        }
702        // Last 10%: introduce column "late"
703        for i in 900..1000 {
704            data.extend_from_slice(format!("a={i}\tb=val{i}\tlate=new{i}\n").as_bytes());
705        }
706
707        let splitter = NullSplitter;
708        let parser = NameValueParser;
709        let plan = ExecutionPlan::new();
710        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
711
712        // Column "late" must be discovered
713        let cols = names(&schema);
714        assert!(cols.contains(&"a"), "column 'a' missing");
715        assert!(cols.contains(&"b"), "column 'b' missing");
716        assert!(
717            cols.contains(&"late"),
718            "column 'late' missing from tail scan"
719        );
720
721        // Verify all values are preserved by parsing the full data
722        let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
723        builder.ensure_schema(&schema).unwrap();
724        parser.parse_chunk(&data, &mut builder).unwrap();
725        let batch = builder.finish().unwrap();
726
727        assert_eq!(batch.num_rows(), 1000);
728        assert_eq!(batch.num_columns(), 3);
729
730        // Check that the "late" column has values in the last 100 rows
731        let late_col = batch.column_by_name("late").expect("late column missing");
732        let late_arr = late_col
733            .as_any()
734            .downcast_ref::<arrow::array::StringArray>()
735            .expect("late column not string array");
736        // First 900 rows should be null for "late"
737        for i in 0..900 {
738            assert!(
739                late_arr.is_null(i),
740                "row {i}: expected null for 'late', got {:?}",
741                late_arr.value(i)
742            );
743        }
744        // Last 100 rows should have values
745        for i in 900..1000 {
746            assert!(
747                !late_arr.is_null(i),
748                "row {i}: expected value for 'late', got null"
749            );
750            assert!(
751                late_arr.value(i).starts_with("new"),
752                "row {i}: expected 'new{i}', got {:?}",
753                late_arr.value(i)
754            );
755        }
756    }
757
758    #[test]
759    fn all_columns_discovered_no_data_loss() {
760        // Test that all columns from a file are discovered, even with sparse data
761        let mut data = Vec::new();
762        // Mix of columns across rows
763        let rows: Vec<&[u8]> = vec![
764            b"a=1\tb=2\tc=3\n",
765            b"b=5\td=6\n",
766            b"a=7\tc=8\te=9\n",
767            b"f=10\n",
768            b"a=11\tb=12\tc=13\td=14\te=15\tf=16\n",
769        ];
770        for row in &rows {
771            data.extend_from_slice(row);
772        }
773
774        let splitter = NullSplitter;
775        let parser = NameValueParser;
776        let plan = ExecutionPlan::new();
777        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
778
779        // All 6 columns must be discovered
780        let cols = names(&schema);
781        assert_eq!(cols.len(), 6, "expected 6 columns, got {:?}", cols);
782        assert!(cols.contains(&"a"));
783        assert!(cols.contains(&"b"));
784        assert!(cols.contains(&"c"));
785        assert!(cols.contains(&"d"));
786        assert!(cols.contains(&"e"));
787        assert!(cols.contains(&"f"));
788
789        // Parse all data and verify values
790        let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
791        builder.ensure_schema(&schema).unwrap();
792        parser.parse_chunk(&data, &mut builder).unwrap();
793        let batch = builder.finish().unwrap();
794
795        assert_eq!(batch.num_rows(), 5);
796        assert_eq!(batch.num_columns(), 6);
797
798        // Verify specific values
799        let a_col = batch.column_by_name("a").unwrap();
800        let a_arr = a_col
801            .as_any()
802            .downcast_ref::<arrow::array::StringArray>()
803            .unwrap();
804        assert_eq!(a_arr.value(0), "1");
805        assert_eq!(a_arr.value(2), "7");
806        assert_eq!(a_arr.value(4), "11");
807    }
808
809    #[test]
810    fn dynamic_sizing_discovers_same_columns() {
811        // Verify that dynamic window sizing discovers the same columns
812        // as the fixed 16-window approach
813        let mut data = Vec::new();
814        // Create data with columns spread across the file
815        for i in 0..1000 {
816            if i < 100 {
817                // First 10%: columns a, b
818                data.extend_from_slice(format!("a={i}\tb={i}\n").as_bytes());
819            } else if i < 500 {
820                // Middle 40%: columns a, b, c
821                data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\n").as_bytes());
822            } else if i < 900 {
823                // Next 40%: columns a, b, c, d
824                data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\n").as_bytes());
825            } else {
826                // Last 10%: columns a, b, c, d, e (late column)
827                data.extend_from_slice(format!("a={i}\tb={i}\tc={i}\td={i}\te={i}\n").as_bytes());
828            }
829        }
830
831        let splitter = NullSplitter;
832        let parser = NameValueParser;
833        let plan = ExecutionPlan::new();
834        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
835
836        // All 5 columns must be discovered
837        let cols = names(&schema);
838        assert_eq!(cols.len(), 5, "expected 5 columns, got {:?}", cols);
839        assert!(cols.contains(&"a"));
840        assert!(cols.contains(&"b"));
841        assert!(cols.contains(&"c"));
842        assert!(cols.contains(&"d"));
843        assert!(cols.contains(&"e"));
844
845        // Parse and verify
846        let mut builder = crate::engine::TableBuilder::with_plan(1024, Arc::new(plan.clone()));
847        builder.ensure_schema(&schema).unwrap();
848        parser.parse_chunk(&data, &mut builder).unwrap();
849        let batch = builder.finish().unwrap();
850
851        assert_eq!(batch.num_rows(), 1000);
852        assert_eq!(batch.num_columns(), 5);
853    }
854
855    #[test]
856    fn column_order_consistent_across_file_sizes() {
857        // Verify that column order is consistent regardless of file size
858        // This ensures no data reordering issues
859        let make_data = |n: usize| -> Vec<u8> {
860            let mut data = Vec::new();
861            for i in 0..n {
862                data.extend_from_slice(format!("x={i}\ty={i}\tz={i}\n").as_bytes());
863            }
864            data
865        };
866
867        let splitter = NullSplitter;
868        let parser = NameValueParser;
869        let plan = ExecutionPlan::new();
870
871        // Test with different file sizes
872        let sizes = vec![10, 100, 1000];
873        let mut first_order: Option<Vec<String>> = None;
874
875        for size in sizes {
876            let data = make_data(size);
877            let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
878            let cols: Vec<String> = schema
879                .column_names()
880                .iter()
881                .map(|s| s.to_string())
882                .collect();
883
884            if let Some(ref order) = first_order {
885                assert_eq!(
886                    &cols, order,
887                    "column order changed for size {size}: {cols:?} vs {order:?}"
888                );
889            } else {
890                first_order = Some(cols);
891            }
892        }
893    }
894
895    #[test]
896    fn esoteric_sparse_schema_one_column_per_row() {
897        // Each row has a unique column name - extremely sparse
898        let mut data = Vec::new();
899        for i in 0..100 {
900            data.extend_from_slice(format!("col_{i}={i}\n").as_bytes());
901        }
902
903        let splitter = NullSplitter;
904        let parser = NameValueParser;
905        let plan = ExecutionPlan::new();
906        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
907
908        // All 100 columns should be discovered
909        let cols = names(&schema);
910        assert_eq!(
911            cols.len(),
912            100,
913            "expected 100 columns, got {:?}",
914            cols.len()
915        );
916    }
917
918    #[test]
919    fn esoteric_empty_rows() {
920        // File with many empty rows interspersed
921        let mut data = Vec::new();
922        for i in 0..50 {
923            data.extend_from_slice(format!("a={i}\n").as_bytes());
924            data.extend_from_slice(b"\n"); // empty row
925            data.extend_from_slice(b"\n"); // another empty row
926            data.extend_from_slice(format!("b={i}\n").as_bytes());
927        }
928
929        let splitter = NullSplitter;
930        let parser = NameValueParser;
931        let plan = ExecutionPlan::new();
932        let schema = discover_schema_for_bytes(&data, &splitter, &parser, &plan);
933
934        let cols = names(&schema);
935        assert!(cols.contains(&"a"), "column 'a' missing");
936        assert!(cols.contains(&"b"), "column 'b' missing");
937    }
938
939    #[test]
940    fn esoteric_single_character_columns() {
941        // Very short column names
942        let data = b"a=1\tb=2\tc=3\nd=4\te=5\n";
943
944        let splitter = NullSplitter;
945        let parser = NameValueParser;
946        let plan = ExecutionPlan::new();
947        let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
948
949        let cols = names(&schema);
950        assert_eq!(cols.len(), 5);
951        assert!(cols.contains(&"a"));
952        assert!(cols.contains(&"e"));
953    }
954
955    #[test]
956    fn esoteric_unicode_column_names() {
957        // Unicode characters in column names
958        let data = "名前=Alice\t年齢=30\n名前=Bob\t年齢=25\n".as_bytes();
959
960        let splitter = NullSplitter;
961        let parser = NameValueParser;
962        let plan = ExecutionPlan::new();
963        let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
964
965        let cols = names(&schema);
966        assert_eq!(cols.len(), 2);
967        assert!(cols.contains(&"名前"));
968        assert!(cols.contains(&"年齢"));
969    }
970
971    #[test]
972    fn esoteric_very_long_column_names() {
973        // Very long column names (100+ chars)
974        let long_name = "a".repeat(200);
975        let data = format!("{long_name}=1\tb=2\n{long_name}=3\tc=4\n");
976
977        let splitter = NullSplitter;
978        let parser = NameValueParser;
979        let plan = ExecutionPlan::new();
980        let schema = discover_schema_for_bytes(data.as_bytes(), &splitter, &parser, &plan);
981
982        let cols = names(&schema);
983        assert_eq!(cols.len(), 3);
984        assert!(cols.contains(&long_name.as_str()));
985    }
986
987    #[test]
988    fn esoteric_duplicate_columns_in_row() {
989        // Same column appears multiple times in a row
990        let data = b"a=1\ta=2\ta=3\tb=4\n";
991
992        let splitter = NullSplitter;
993        let parser = NameValueParser;
994        let plan = ExecutionPlan::new();
995        let schema = discover_schema_for_bytes(data, &splitter, &parser, &plan);
996
997        let cols = names(&schema);
998        assert_eq!(cols.len(), 2, "expected 2 unique columns, got {:?}", cols);
999        assert!(cols.contains(&"a"));
1000        assert!(cols.contains(&"b"));
1001    }
1002}