Skip to main content

rypipe_core/
parallel_stream.rs

1//! Parallel streaming executor: multi-core parsing with bounded memory.
2
3use std::collections::BTreeMap;
4use std::path::{Path, PathBuf};
5use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
6use std::sync::Arc;
7use std::thread::{self, JoinHandle};
8
9use arrow::record_batch::RecordBatch;
10
11use std::sync::atomic::{AtomicU64, Ordering};
12
13use crate::bounded::MemoryBudget;
14use crate::consumer::BatchConsumer;
15use crate::decoder::{RecordParser, Splitter};
16use crate::engine::TableBuilder;
17use crate::input::InputBuffer;
18use crate::plan::ExecutionPlan;
19use crate::schema::{DiscoveryOpts, FrozenSchema};
20use crate::Result;
21
22static DISCOVERY_NS: AtomicU64 = AtomicU64::new(0);
23
24/// Return the elapsed time (in nanoseconds) spent in schema discovery.
25pub fn discovery_profile() -> u64 {
26    DISCOVERY_NS.load(Ordering::Relaxed)
27}
28/// Reset the discovery timer to zero.
29pub fn reset_discovery_profile() {
30    DISCOVERY_NS.store(0, Ordering::Relaxed);
31}
32
33/// Options for parallel streaming.
34pub struct ParallelStreamOpts {
35    /// Number of worker threads.
36    pub threads: usize,
37    /// Whether to preserve row order (default: true).
38    pub ordered: bool,
39    /// Maximum reorder buffer size (default: = threads).
40    pub max_reorder: usize,
41    /// Explicit schema.  If `Some`, no discovery pass is needed.
42    /// Workers pre-size all columns from construction.
43    pub schema: Option<FrozenSchema>,
44}
45
46impl Default for ParallelStreamOpts {
47    fn default() -> Self {
48        Self {
49            threads: std::thread::available_parallelism()
50                .map(|n| n.get())
51                .unwrap_or(1),
52            ordered: true,
53            max_reorder: 0, // 0 = use threads value
54            schema: None,
55        }
56    }
57}
58
59/// Discovery sink that collects raw field names in encounter order.
60/// `needs_value=false` keeps the scanner in locate-only mode (no value
61/// extraction), but we still need `resolve()` calls, so `needs_resolve`
62/// stays true. Capture happens in `resolve()` via interior mutability.
63struct DiscoverySink {
64    seen: std::cell::RefCell<rustc_hash::FxHashSet<Box<str>>>,
65    order: std::cell::RefCell<Vec<String>>,
66}
67
68impl DiscoverySink {
69    fn new() -> Self {
70        Self {
71            seen: std::cell::RefCell::new(rustc_hash::FxHashSet::default()),
72            order: std::cell::RefCell::new(Vec::new()),
73        }
74    }
75    fn into_order(self) -> Vec<String> {
76        self.order.into_inner()
77    }
78}
79
80impl crate::decoder::ColumnarSink for DiscoverySink {
81    #[inline]
82    fn begin_row(&mut self) {}
83    #[inline]
84    fn put_field(&mut self, name: &str, _value: crate::value::Value<'_>) {
85        // Full-value fallback (if needs_value were true); keep for completeness.
86        let mut seen = self.seen.borrow_mut();
87        if seen.insert(Box::from(name)) {
88            self.order.borrow_mut().push(name.to_string());
89        }
90    }
91    #[inline]
92    fn end_row(&mut self) {}
93    #[inline]
94    fn wants(&self, _name: &str) -> bool {
95        true
96    }
97    #[inline]
98    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
99        // Locate-only path calls `resolve` without `put_field`; capture here.
100        let mut seen = self.seen.borrow_mut();
101        if seen.insert(Box::from(name)) {
102            self.order.borrow_mut().push(name.to_string());
103        }
104        Some(name)
105    }
106    #[inline]
107    fn needs_value(&self) -> bool {
108        false
109    }
110    // needs_resolve defaults to true → scanner calls resolve() for each field.
111    fn finish(&mut self) -> crate::Result<arrow::record_batch::RecordBatch> {
112        Ok(arrow::record_batch::RecordBatch::new_empty(
113            std::sync::Arc::new(arrow::datatypes::Schema::empty()),
114        ))
115    }
116}
117
118fn discover_schema<P: crate::decoder::RecordParser>(
119    bytes: &[u8],
120    parser: &P,
121    plan: &crate::plan::ExecutionPlan,
122    splitter: &dyn crate::decoder::Splitter,
123) -> (FrozenSchema, Vec<String>) {
124    let t0 = std::time::Instant::now();
125    // Explicit schema already handled by caller; this is auto-discovery.
126    let opts = DiscoveryOpts::default();
127    let order: Vec<String> = if (bytes.len() as u64) < opts.full_scan_threshold {
128        let mut sink = DiscoverySink::new();
129        let _ = parser.parse_chunk_generic(bytes, &mut sink);
130        sink.into_order()
131    } else {
132        use rayon::prelude::*;
133        let n = opts.windows;
134        let wbytes = opts.window_bytes;
135        // Parallelise windows: 16×2 MiB independent parses (~19 ms serial → ~2 ms on 16t)
136        let per_window: Vec<Vec<String>> = (0..n)
137            .into_par_iter()
138            .map(|i| {
139                let start = (bytes.len() as u64 * i as u64 / n as u64) as usize;
140                let end = (start + wbytes).min(bytes.len());
141                if start >= end {
142                    return Vec::new();
143                }
144                let mut sink = DiscoverySink::new();
145                let slice = &bytes[start..end];
146                let _ = parser.parse_chunk_generic(slice, &mut sink);
147                sink.into_order()
148            })
149            .collect();
150        // Merge in file order, deduplicating, so global order approximates file order.
151        let mut seen = rustc_hash::FxHashSet::<String>::default();
152        let mut merged = Vec::new();
153        for mut v in per_window {
154            for name in v.drain(..) {
155                if seen.insert(name.clone()) {
156                    merged.push(name);
157                }
158            }
159        }
160        if merged.is_empty() && !bytes.is_empty() {
161            let mut sink = DiscoverySink::new();
162            let _ = parser.parse_chunk_generic(bytes, &mut sink);
163            sink.into_order()
164        } else {
165            merged
166        }
167    };
168    let elapsed = t0.elapsed().as_nanos() as u64;
169    DISCOVERY_NS.store(elapsed, Ordering::Relaxed);
170    let _ = splitter; // keep splitter in signature for future alignment
171    let schema = if order.is_empty() {
172        FrozenSchema::from_plan(&[], plan)
173    } else {
174        FrozenSchema::from_discovered(&order, plan)
175    };
176    (schema, order)
177}
178
179/// Public helper for batch workloads: discover the schema once and reuse.
180/// Example:
181/// ```ignore
182/// let schema = discover_schema_for_path(path, &splitter, &parser, &plan);
183/// for f in files { ParallelStreamingBatchIterator::new(..., schema.clone(), ...) }
184/// ```
185pub fn discover_schema_for_bytes<P: crate::decoder::RecordParser>(
186    bytes: &[u8],
187    splitter: &dyn crate::decoder::Splitter,
188    parser: &P,
189    plan: &crate::plan::ExecutionPlan,
190) -> FrozenSchema {
191    if !plan.schema_order.is_empty() {
192        let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
193        return FrozenSchema::from_plan(&names, plan);
194    }
195    let opts = DiscoveryOpts::default();
196    let sig = crate::schema::layout_signature(bytes, &opts);
197    {
198        let cache = crate::schema::SCHEMA_CACHE.read().unwrap();
199        if let Some(order) = cache.get(&sig) {
200            crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
201            return FrozenSchema::from_discovered(order, plan);
202        }
203    }
204    crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
205    let (schema, order) = discover_schema(bytes, parser, plan, splitter);
206    if !order.is_empty() {
207        crate::schema::insert_schema_cache(sig, Arc::new(order));
208    }
209    schema
210}
211
212/// Parallel streaming executor: parses chunks concurrently with bounded memory.
213pub struct ParallelStreamingExecutor {
214    budget: MemoryBudget,
215    max_in_flight: usize,
216}
217
218impl ParallelStreamingExecutor {
219    pub fn new(budget: MemoryBudget, max_in_flight: usize) -> Self {
220        Self {
221            budget,
222            max_in_flight: max_in_flight.max(1),
223        }
224    }
225
226    /// Stream a file in parallel, calling `consumer` per batch in order.
227    #[allow(clippy::too_many_arguments)]
228    pub fn run_stream<P, C>(
229        &self,
230        path: &Path,
231        splitter: &dyn Splitter,
232        parser: P,
233        plan: Arc<ExecutionPlan>,
234        prefault: bool,
235        opts: ParallelStreamOpts,
236        consumer: &mut C,
237    ) -> Result<()>
238    where
239        P: RecordParser + Clone + Send + Sync + 'static,
240        C: BatchConsumer,
241    {
242        let input = InputBuffer::open(path, cfg!(feature = "mmap"), prefault)?;
243        if input.is_empty() {
244            return Ok(());
245        }
246        // Share the InputBuffer across workers via Arc to avoid
247        // the O(file_size × threads) to_vec() clone.
248        let shared = std::sync::Arc::new(input);
249        self.run_bytes_stream_shared(shared, splitter, parser, plan, opts, consumer)
250    }
251
252    /// Stream from a shared InputBuffer (zero-copy on mmap).
253    fn run_bytes_stream_shared<P, C>(
254        &self,
255        input: std::sync::Arc<InputBuffer>,
256        splitter: &dyn Splitter,
257        parser: P,
258        plan: Arc<ExecutionPlan>,
259        opts: ParallelStreamOpts,
260        consumer: &mut C,
261    ) -> Result<()>
262    where
263        P: RecordParser + Clone + Send + Sync + 'static,
264        C: BatchConsumer,
265    {
266        // Safety: the Arc keeps the InputBuffer alive for the duration.
267        // We pass a dummy &[u8] to satisfy the signature; run_bytes_stream_core
268        // ignores it when input is Some and uses input.as_slice() instead.
269        let dummy: &[u8] = &[];
270        self.run_bytes_stream_core(dummy, Some(input), splitter, parser, plan, opts, consumer)
271    }
272
273    /// Stream bytes in parallel.
274    pub fn run_bytes_stream<P, C>(
275        &self,
276        bytes: &[u8],
277        splitter: &dyn Splitter,
278        parser: P,
279        plan: Arc<ExecutionPlan>,
280        opts: ParallelStreamOpts,
281        consumer: &mut C,
282    ) -> Result<()>
283    where
284        P: RecordParser + Clone + Send + Sync + 'static,
285        C: BatchConsumer,
286    {
287        self.run_bytes_stream_core(bytes, None, splitter, parser, plan, opts, consumer)
288    }
289
290    /// Core implementation shared by `run_bytes_stream` and `run_bytes_stream_shared`.
291    #[allow(clippy::too_many_arguments)]
292    fn run_bytes_stream_core<P, C>(
293        &self,
294        bytes: &[u8],
295        input: Option<std::sync::Arc<InputBuffer>>,
296        splitter: &dyn Splitter,
297        parser: P,
298        plan: Arc<ExecutionPlan>,
299        opts: ParallelStreamOpts,
300        consumer: &mut C,
301    ) -> Result<()>
302    where
303        P: RecordParser + Clone + Send + Sync + 'static,
304        C: BatchConsumer,
305    {
306        // Prefer the shared InputBuffer's slice over the passed-in bytes.
307        let actual_bytes = match input {
308            Some(ref inp) => inp.as_slice(),
309            None => bytes,
310        };
311        let n = opts.threads.max(1);
312        let max_reorder = if opts.max_reorder > 0 {
313            opts.max_reorder
314        } else {
315            n
316        };
317        // Frozen schema: gates correct ParquetWriter / StreamWriter usage.
318        // Without it, batch 2 can have different column order (FieldG vs Text20
319        // last) even when the set is identical, breaking `write_batch`.
320        // If opts.schema is None, auto-discover:
321        //  - explicit plan.schema_order → from_plan (exact)
322        //  - else sampled discovery (16×2 MiB windows for >128 MiB, else full)
323        let schema: Option<FrozenSchema> = match opts.schema {
324            Some(s) => Some(s),
325            None => {
326                if !plan.schema_order.is_empty() {
327                    let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
328                    Some(FrozenSchema::from_plan(&names, &plan))
329                } else {
330                    let opts = DiscoveryOpts::default();
331                    let sig = crate::schema::layout_signature(actual_bytes, &opts);
332                    let cached = crate::schema::SCHEMA_CACHE
333                        .read()
334                        .unwrap()
335                        .get(&sig)
336                        .cloned();
337                    let (schema, order) = if let Some(order) = cached {
338                        crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
339                        (FrozenSchema::from_discovered(&order, &plan), order)
340                    } else {
341                        crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
342                        let (schema, order) =
343                            discover_schema(actual_bytes, &parser, &plan, splitter);
344                        (schema, Arc::new(order))
345                    };
346                    if !order.is_empty() {
347                        crate::schema::insert_schema_cache(sig, order);
348                    }
349                    Some(schema)
350                }
351            }
352        };
353
354        let bytes_per_row = splitter
355            .estimate_bytes_per_row(&actual_bytes[..actual_bytes.len().min(65536)])
356            .max(1);
357        let chunk_size = (self.budget.bytes() / (n * 2))
358            .max(bytes_per_row * 10)
359            .max(64 * 1024);
360        let num_chunks = (actual_bytes.len() / chunk_size).max(n).min(10000);
361        let split_points = splitter.find_split_points(actual_bytes, num_chunks);
362        let mut ranges = crate::decoder::split_points_to_ranges(&split_points, actual_bytes.len());
363        if ranges.is_empty() {
364            ranges.push(0..bytes.len());
365        }
366
367        let chunks_with_seq: Vec<(usize, std::ops::Range<usize>)> =
368            ranges.into_iter().enumerate().collect();
369
370        let (sender, receiver) = sync_channel(self.max_in_flight);
371        let mut handles: Vec<JoinHandle<Result<()>>> = Vec::with_capacity(n);
372        let chunk_queue = std::sync::Arc::new(std::sync::Mutex::new(chunks_with_seq));
373        let plan_arc = plan;
374        let schema_arc = schema.map(std::sync::Arc::new);
375        let est_row = splitter
376            .estimate_bytes_per_row(&bytes[..bytes.len().min(65536)])
377            .max(512);
378
379        // Pre-clone bytes for the fallback path (when input is None).
380        let bytes_fallback = if input.is_none() {
381            Some(actual_bytes.to_vec())
382        } else {
383            None
384        };
385        for _ in 0..n {
386            let queue = std::sync::Arc::clone(&chunk_queue);
387            let sender_clone: SyncSender<(usize, Result<RecordBatch>)> = sender.clone();
388            let parser_clone = parser.clone();
389            let plan_clone = std::sync::Arc::clone(&plan_arc);
390            let schema_clone = schema_arc.clone();
391            let input_clone = input.clone();
392            let fallback_clone = bytes_fallback.clone();
393            let handle = thread::spawn(move || -> Result<()> {
394                // Use shared InputBuffer when available (zero-copy on mmap);
395                // fall back to pre-cloned bytes for external callers.
396                let bytes_ref: &[u8] = match input_clone {
397                    Some(ref inp) => inp.as_slice(),
398                    None => fallback_clone.as_deref().unwrap_or(&[]),
399                };
400                loop {
401                    let next = {
402                        let mut q = queue.lock().unwrap();
403                        q.pop()
404                    };
405                    let Some((seq, range)) = next else { break };
406                    let chunk_bytes = &bytes_ref[range.start..range.end];
407                    let mut builder = TableBuilder::with_plan(
408                        (chunk_bytes.len() / est_row).max(64),
409                        plan_clone.clone(),
410                    );
411                    // If schema is provided, pre-size columns from it.
412                    if let Some(ref schema) = schema_clone {
413                        if let Err(e) = builder.ensure_schema(schema) {
414                            let _ = sender_clone.send((seq, Err(e)));
415                            break;
416                        }
417                    }
418                    if let Err(e) = parser_clone.validate(chunk_bytes) {
419                        let _ = sender_clone.send((seq, Err(e)));
420                        break;
421                    }
422                    if let Err(e) = parser_clone.parse_chunk_generic(chunk_bytes, &mut builder) {
423                        let _ = sender_clone.send((seq, Err(e)));
424                        break;
425                    }
426                    let res = builder.finish();
427                    if sender_clone.send((seq, res)).is_err() {
428                        break;
429                    }
430                }
431                Ok(())
432            });
433            handles.push(handle);
434        }
435        drop(sender);
436
437        // Coordinator: order by seq and deliver.
438        // If `ordered`, buffer out-of-order batches until the next-in-sequence arrives.
439        // If unordered, deliver immediately.
440        // When the reorder buffer overflows, fall back to unordered delivery.
441        let mut pending: BTreeMap<usize, RecordBatch> = BTreeMap::new();
442        let mut next_seq = 0usize;
443        let mut reorder_bytes: usize = 0;
444        let mut fallback_unordered = false;
445        let reorder_limit = max_reorder * self.budget.bytes();
446        for (seq, res) in receiver {
447            match res {
448                Ok(batch) => {
449                    if opts.ordered && !fallback_unordered {
450                        if seq == next_seq {
451                            consumer.consume(batch)?;
452                            next_seq += 1;
453                            while let Some(b) = pending.remove(&next_seq) {
454                                consumer.consume(b)?;
455                                next_seq += 1;
456                            }
457                        } else {
458                            reorder_bytes += batch.get_array_memory_size();
459                            if reorder_bytes > reorder_limit {
460                                // Buffer overflow: drain pending in arrival order
461                                // (closest to correct), then deliver all remaining
462                                // batches unordered.  This avoids a hard error when
463                                // the memory budget is too small for the chunk count.
464                                for (_, b) in std::mem::take(&mut pending) {
465                                    consumer.consume(b)?;
466                                }
467                                consumer.consume(batch)?;
468                                reorder_bytes = 0;
469                                fallback_unordered = true;
470                            } else {
471                                pending.insert(seq, batch);
472                            }
473                        }
474                    } else {
475                        // Unordered (either opted out or fell back): deliver immediately.
476                        consumer.consume(batch)?;
477                    }
478                }
479                Err(_) => {
480                    // Skip failed chunks (logged upstream).
481                }
482            }
483        }
484        // Drain remaining.
485        if opts.ordered && !fallback_unordered {
486            while let Some(b) = pending.remove(&next_seq) {
487                consumer.consume(b)?;
488                next_seq += 1;
489            }
490        }
491        for h in handles {
492            h.join()
493                .map_err(|_| crate::Error::Merge("worker panicked".into()))??;
494        }
495        Ok(())
496    }
497}
498
499/// Iterator wrapper for parallel streaming (pull-based).
500pub struct ParallelStreamingBatchIterator {
501    receiver: Receiver<Result<RecordBatch>>,
502    handle: Option<JoinHandle<Result<()>>>,
503    done: bool,
504}
505
506impl ParallelStreamingBatchIterator {
507    pub fn new<P, S>(
508        path: PathBuf,
509        splitter: S,
510        parser: P,
511        plan: Arc<ExecutionPlan>,
512        budget: MemoryBudget,
513        prefault: bool,
514        opts: ParallelStreamOpts,
515    ) -> Self
516    where
517        P: RecordParser + Clone + Send + Sync + 'static,
518        S: Splitter + Clone + Send + Sync + 'static,
519    {
520        let num_threads = opts.threads;
521        let max_in_flight = 2 * num_threads;
522        let (sender, receiver) = sync_channel(max_in_flight);
523        let handle = thread::spawn(move || {
524            let exec = ParallelStreamingExecutor::new(budget, max_in_flight);
525            let mut consumer = ChannelConsumer {
526                sender: sender.clone(),
527            };
528            let res = exec.run_stream(
529                &path,
530                &splitter,
531                parser,
532                plan,
533                prefault,
534                opts,
535                &mut consumer,
536            );
537            if let Err(e) = res {
538                let _ = sender.send(Err(e));
539            }
540            Ok(())
541        });
542        Self {
543            receiver,
544            handle: Some(handle),
545            done: false,
546        }
547    }
548}
549
550struct ChannelConsumer {
551    sender: SyncSender<Result<RecordBatch>>,
552}
553impl BatchConsumer for ChannelConsumer {
554    fn consume(&mut self, batch: RecordBatch) -> Result<()> {
555        let _ = self.sender.send(Ok(batch));
556        Ok(())
557    }
558}
559
560impl Iterator for ParallelStreamingBatchIterator {
561    type Item = Result<RecordBatch>;
562    fn next(&mut self) -> Option<Self::Item> {
563        if self.done {
564            return None;
565        }
566        // This is a simplified version that just receives in order as sent,
567        // not handling seq ordering for now. For true ordered, need BTreeMap logic as in run_stream.
568        // For first version, we will just receive and yield.
569        match self.receiver.recv() {
570            Ok(Ok(batch)) => Some(Ok(batch)),
571            Ok(Err(e)) => {
572                self.done = true;
573                Some(Err(e))
574            }
575            Err(_) => {
576                self.done = true;
577                if let Some(h) = self.handle.take() {
578                    if let Err(p) = h.join() {
579                        let msg = if let Some(s) = p.downcast_ref::<&str>() {
580                            (*s).to_string()
581                        } else if let Some(s) = p.downcast_ref::<String>() {
582                            s.clone()
583                        } else {
584                            "worker panicked".to_string()
585                        };
586                        return Some(Err(crate::Error::Merge(format!(
587                            "parallel streaming worker panicked: {msg}"
588                        ))));
589                    }
590                }
591                None
592            }
593        }
594    }
595}
596
597#[cfg(test)]
598mod tests {
599    use std::borrow::Cow;
600
601    use super::*;
602    use crate::decoder::{ColumnarSink, RecordParser, Splitter};
603    use crate::plan::ExecutionPlan;
604    use crate::schema::{clear_schema_cache, schema_cache_stats};
605    use crate::value::Value;
606
607    /// Minimal parser: newline-separated rows of `key=value\t...` tokens.
608    #[derive(Clone, Debug, Default)]
609    struct NameValueParser;
610
611    impl RecordParser for NameValueParser {
612        fn validate(&self, _bytes: &[u8]) -> Result<()> {
613            Ok(())
614        }
615
616        fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
617            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
618            for line in text.lines() {
619                if line.is_empty() {
620                    continue;
621                }
622                sink.begin_row();
623                for token in line.split('\t') {
624                    if let Some((k, v)) = token.split_once('=') {
625                        sink.resolve_and_put(k, Value::Str(Cow::Borrowed(v)));
626                    }
627                }
628                sink.end_row();
629            }
630            Ok(())
631        }
632    }
633
634    /// Trivial splitter: one chunk covering the whole input.
635    struct NullSplitter;
636
637    impl Splitter for NullSplitter {
638        fn next_record_start(&self, _bytes: &[u8], _from: usize) -> Option<usize> {
639            None
640        }
641
642        fn find_split_points(&self, bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
643            vec![0, bytes.len()]
644        }
645
646        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
647            let n = sample.iter().filter(|&&b| b == b'\n').count().max(1);
648            sample.len() / n
649        }
650    }
651
652    fn names(schema: &FrozenSchema) -> Vec<&str> {
653        schema.column_names().iter().map(|s| s.as_ref()).collect()
654    }
655
656    #[test]
657    fn schema_cache_hits_misses_and_plan_changes() {
658        clear_schema_cache();
659        let splitter = NullSplitter;
660        let parser = NameValueParser;
661
662        // First layout: miss.
663        let bytes = b"a=1\tb=2\na=3\tb=4\n";
664        let plan = ExecutionPlan::new();
665        let s1 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
666        assert_eq!(names(&s1), vec!["a", "b"]);
667        assert_eq!(schema_cache_stats(), (0, 1));
668
669        // Same layout: hit.
670        let s2 = discover_schema_for_bytes(bytes, &splitter, &parser, &plan);
671        assert_eq!(names(&s2), vec!["a", "b"]);
672        assert_eq!(schema_cache_stats(), (1, 1));
673
674        // Different layout: new miss.
675        let bytes2 = b"c=1\td=2\n";
676        let s3 = discover_schema_for_bytes(bytes2, &splitter, &parser, &plan);
677        assert_eq!(names(&s3), vec!["c", "d"]);
678        assert_eq!(schema_cache_stats(), (1, 2));
679
680        // Same layout as the first parse but with a different plan:
681        // cache hit, but the applied schema differs.
682        let mut renamed = ExecutionPlan::new();
683        renamed.field_map.insert("a".to_string(), "x".to_string());
684        let s4 = discover_schema_for_bytes(bytes, &splitter, &parser, &renamed);
685        assert_eq!(names(&s4), vec!["x", "b"]);
686        assert_eq!(schema_cache_stats(), (2, 2));
687    }
688}