Skip to main content

rypipe_core/
decoder.rs

1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8// ---------------------------------------------------------------------------
9// Chunk-size floor
10// ---------------------------------------------------------------------------
11
12/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
13/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
14/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
15pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
16
17/// Maximum number of split chunks. Above this, scheduling overhead dominates.
18pub const MAX_SPLIT_CHUNKS: usize = 1024;
19
20/// Mode for chunk planning: parallel and streaming have different optima.
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub enum SplitMode {
23    /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
24    Parallel,
25    /// Streaming: peak at 2 MB chunks (measured with bounded memory).
26    Streaming,
27}
28
29/// Plan the number of chunks given input size, thread count, and mode.
30/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a floor of
31/// `min_chunk_bytes` (default [`MIN_CHUNK_BYTES`] via [`plan_chunk_count`]).
32pub fn plan_chunk_count_with(
33    bytes: usize,
34    threads: usize,
35    mode: SplitMode,
36    min_chunk_bytes: usize,
37) -> usize {
38    let by_size = bytes / min_chunk_bytes.max(1);
39    let cap = match mode {
40        SplitMode::Parallel => 16usize.saturating_mul(threads),
41        SplitMode::Streaming => 8usize.saturating_mul(threads),
42    };
43    by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
44}
45
46/// [`plan_chunk_count_with`] using the default [`MIN_CHUNK_BYTES`] floor.
47pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
48    plan_chunk_count_with(bytes, threads, mode, MIN_CHUNK_BYTES)
49}
50
51// ---------------------------------------------------------------------------
52// SkipRegionFinder: byte ranges where a candidate boundary is invalid
53// ---------------------------------------------------------------------------
54
55/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
56/// a candidate record boundary must be rejected.  The engine calls this
57/// during split-point discovery to avoid splitting inside comments or strings.
58pub trait SkipRegionFinder: Send + Sync {
59    /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
60    fn openers(&self) -> &[&'static [u8]];
61
62    /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
63    fn closer_for(&self, opener: &[u8]) -> &'static [u8];
64
65    /// Maximum backward scan window in bytes.  Default 64 KiB: sufficient
66    /// for most XML/CSV comments and JSON string literals.
67    fn window(&self) -> usize {
68        64 * 1024
69    }
70}
71
72/// Check whether byte position `at` falls inside a skip region by scanning
73/// backward up to `finder.window()` bytes, looking for an unclosed opener.
74///
75/// Returns `false` immediately when the file contains none of the openers;
76/// check that once per chunk, not per candidate.
77pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
78    if at > bytes.len() {
79        return false;
80    }
81    let openers = finder.openers();
82    if openers.is_empty() {
83        return false;
84    }
85    let scan_start = at.saturating_sub(finder.window());
86    let scan = &bytes[scan_start..at];
87    for opener in openers {
88        let closer = finder.closer_for(opener);
89        // Use memchr::memmem::find directly (no Finder::new preprocessing).
90        let mut search = 0;
91        while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
92            let open_pos = scan_start + search + rel;
93            // Check if there's a closer between the opener and `at`.
94            let closer_start = open_pos + opener.len();
95            if closer_start >= at {
96                // Opener extends past `at`; treat as unclosed (inside skip region).
97                return true;
98            }
99            let between = &bytes[closer_start..at];
100            if memchr::memmem::find(between, closer).is_none() {
101                // Unclosed opener; `at` is inside this skip region.
102                return true;
103            }
104            search += rel + 1;
105        }
106    }
107    false
108}
109
110// ---------------------------------------------------------------------------
111// Splitter trait with default
112// ---------------------------------------------------------------------------
113
114/// How records are separated in the input.
115#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
116pub enum RecordBoundary {
117    /// Split on every newline (default). Each `\n` is a record boundary.
118    #[default]
119    Line,
120    /// Split on blank lines. Records span multiple lines; a blank line
121    /// (`\n\n`) separates records.
122    BlankLine,
123}
124
125/// Format-specific splitter: decides where it is safe to divide an input byte
126/// stream into independent chunks.
127///
128/// The only required method is `next_record_start`; adapters provide the
129/// record boundary logic, and the engine handles chunk planning, skip-region
130/// rejection, and deduplication.
131///
132/// For multi-line formats, implement the declarative methods
133/// ([`record_boundary`](Self::record_boundary),
134/// [`continuation_char`](Self::continuation_char),
135/// [`comment_prefixes`](Self::comment_prefixes)) and call
136/// [`find_next_record_boundary`] from your `next_record_start`:
137///
138/// ```rust
139/// use rypipe_core::{find_next_record_boundary, RecordBoundary, Splitter};
140/// struct Lines;
141/// impl Splitter for Lines {
142///     fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
143///         find_next_record_boundary(bytes, from, self.continuation_char(), self.comment_prefixes(), self.record_boundary() == RecordBoundary::BlankLine)
144///     }
145///     fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize { sample.len().max(1) }
146/// }
147/// assert_eq!(Lines.next_record_start(b"one\ntwo\n", 0), Some(4));
148/// ```
149pub trait Splitter: Send + Sync {
150    /// Return the next record boundary at or after `from`.  Must return a
151    /// position where a record starts, or `None` if no more records exist.
152    fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
153
154    /// Estimate the average bytes per record from a sample of the input.
155    fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
156
157    /// How records are separated. Default: [`RecordBoundary::Line`] (every
158    /// newline is a boundary). Override to [`RecordBoundary::BlankLine`] for
159    /// multi-line formats where blank lines separate records.
160    fn record_boundary(&self) -> RecordBoundary {
161        RecordBoundary::Line
162    }
163
164    /// Character that continues a record across lines (e.g. `b'\\'` for
165    /// properties files). When set, newlines preceded by this character are
166    /// NOT record boundaries. Default: `None` (no continuations).
167    fn continuation_char(&self) -> Option<u8> {
168        None
169    }
170
171    /// Line prefixes to skip (e.g. `&[b"#", b"!"]` for properties files).
172    /// Lines starting with any of these are ignored during splitting.
173    /// Default: empty (no comments).
174    fn comment_prefixes(&self) -> &[&[u8]] {
175        &[]
176    }
177
178    /// Optional: byte ranges where a candidate boundary must be rejected.
179    /// See `in_skip_region`.
180    fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
181        None
182    }
183
184    /// Find split points.  **Do not override** without a measured reason.
185    ///
186    /// Returns a sorted list of byte offsets where it is safe to split the
187    /// input into independent chunks. The return value must satisfy:
188    /// - Sorted ascending, no duplicates.
189    /// - First element is `0`, last element is `bytes.len()`.
190    /// - All values are `<= bytes.len()`.
191    ///
192    /// Default implementation:
193    /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
194    /// 2. Scan each nominal offset with `next_record_start`.
195    /// 3. Reject candidates inside skip regions via `in_skip_region`.
196    /// 4. Dedup, sort, prepend 0.
197    /// 5. Apply chunk floor via `plan_chunk_count`.
198    ///
199    /// This is the default-floor entry point. Adapters that override it keep
200    /// working; engines call [`Splitter::find_split_points_with`] instead, and
201    /// that defers to this override at the default floor.
202    fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
203        default_find_split_points(self, bytes, max_chunks, MIN_CHUNK_BYTES)
204    }
205
206    /// Find split points with an explicit chunk-size floor.
207    ///
208    /// The floor is opt-in: lowering it yields more, smaller chunks (better
209    /// load balancing on many or small cores, at more scheduling overhead);
210    /// raising it yields fewer, larger chunks. Default is [`MIN_CHUNK_BYTES`].
211    ///
212    /// At the default floor this calls [`Splitter::find_split_points`], so an
213    /// adapter's override is honored; a non-default floor uses the engine's
214    /// default planning (override `find_split_points_with` to customize it).
215    fn find_split_points_with(
216        &self,
217        bytes: &[u8],
218        max_chunks: usize,
219        min_chunk_bytes: usize,
220    ) -> Vec<usize> {
221        if min_chunk_bytes == MIN_CHUNK_BYTES {
222            return self.find_split_points(bytes, max_chunks);
223        }
224        default_find_split_points(self, bytes, max_chunks, min_chunk_bytes)
225    }
226}
227
228/// The engine's default chunk planning, shared by the `Splitter` methods.
229fn default_find_split_points<S: Splitter + ?Sized>(
230    splitter: &S,
231    bytes: &[u8],
232    max_chunks: usize,
233    min_chunk_bytes: usize,
234) -> Vec<usize> {
235    if max_chunks <= 1 || bytes.is_empty() {
236        return vec![0, bytes.len()];
237    }
238    let n = plan_chunk_count_with(
239        bytes.len(),
240        max_chunks,
241        SplitMode::Parallel,
242        min_chunk_bytes,
243    );
244    let skip = splitter.skip_regions();
245    // Planning must not start a global worker pool for bounded readers.
246    let mut points: Vec<usize> = (1..n)
247        .filter_map(|i| {
248            let approx = bytes.len() / n * i;
249            let pos = splitter.next_record_start(bytes, approx)?;
250            // Reject candidates inside skip regions.
251            if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
252                return None;
253            }
254            Some(pos)
255        })
256        .collect();
257
258    points.sort_unstable();
259    points.dedup();
260    // Drop candidates at 0: the prepend below already supplies the
261    // leading split point, and keeping them would duplicate it (this
262    // happens when bytes.len() < n, where bytes.len() / n == 0 puts
263    // every nominal offset at 0).
264    points.retain(|&p| p > 0);
265    points.insert(0, 0);
266    if *points.last().unwrap_or(&0) != bytes.len() {
267        points.push(bytes.len());
268    }
269    points
270}
271
272// ---------------------------------------------------------------------------
273// Declarative splitting helpers
274// ---------------------------------------------------------------------------
275
276/// Check whether the newline at `pos` is preceded by a continuation character.
277///
278/// Used by Splitters that need to skip continued newlines. For example, in
279/// properties files, `key = value \` continues on the next line: the `\n`
280/// after `\` is NOT a record boundary.
281///
282/// ```rust
283/// use rypipe_core::is_continued_newline;
284/// assert!(is_continued_newline(b"key=value\\\nnext\n", 10));
285/// ```
286#[inline]
287pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
288    if pos == 0 || bytes.get(pos) != Some(&b'\n') {
289        return false;
290    }
291    let mut run = 0;
292    let mut cursor = pos;
293    if bytes[cursor - 1] == b'\r' {
294        cursor -= 1;
295    }
296    while cursor > 0 && bytes[cursor - 1] == b'\\' {
297        run += 1;
298        cursor -= 1;
299    }
300    run % 2 == 1
301}
302
303/// Scan forward from `from` to find the next record boundary.
304///
305/// This is the declarative counterpart to manual byte scanning. It handles:
306/// - **Continuations**: skips `\n` preceded by `continuation` (e.g. `b'\\'`)
307/// - **Comments**: skips lines starting with any prefix in `comment_prefixes`
308/// - **Blank-line mode**: splits after an empty LF or CRLF line
309///
310/// Returns the byte position after the boundary newline, or `None` if no
311/// more boundaries exist.
312///
313/// # Example
314///
315/// ```rust
316/// use rypipe_core::find_next_record_boundary;
317/// let input = b"# comment\nfirst\nsecond\n";
318/// assert_eq!(find_next_record_boundary(input, 0, None, &[b"#"], false), Some(16));
319/// ```
320#[inline]
321pub fn find_next_record_boundary(
322    bytes: &[u8],
323    from: usize,
324    continuation: Option<u8>,
325    comment_prefixes: &[&[u8]],
326    blank_line_mode: bool,
327) -> Option<usize> {
328    if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
329        return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
330    }
331    let mut pos = from;
332    while pos < bytes.len() {
333        let rel = memchr::memchr(b'\n', &bytes[pos..])?;
334        let nl_pos = pos + rel;
335
336        // Skip continued newlines (preceded by continuation character)
337        let continued = |newline: usize| {
338            continuation.is_some_and(|character| {
339                let end = if newline > 0 && bytes[newline - 1] == b'\r' {
340                    newline - 1
341                } else {
342                    newline
343                };
344                if character == b'\\' {
345                    is_continued_newline(bytes, newline)
346                } else {
347                    end > 0 && bytes[end - 1] == character
348                }
349            })
350        };
351        if continued(nl_pos) {
352            pos = nl_pos + 1;
353            continue;
354        }
355
356        if !blank_line_mode && comment_prefixes.is_empty() {
357            return Some(nl_pos + 1);
358        }
359
360        if blank_line_mode {
361            let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
362            if end == 0 || bytes[end - 1] != b'\n' {
363                pos = nl_pos + 1;
364                continue;
365            }
366            if comment_prefixes.is_empty() && !continued(end - 1) {
367                return Some(nl_pos + 1);
368            }
369        }
370
371        // Compute line start for comment check
372        let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
373
374        // Skip comment lines
375        if !comment_prefixes.is_empty() {
376            let line_bytes = &bytes[line_start..nl_pos];
377            let is_comment = comment_prefixes
378                .iter()
379                .any(|prefix| line_bytes.starts_with(prefix));
380            if is_comment {
381                pos = nl_pos + 1;
382                continue;
383            }
384        }
385
386        // Blank-line mode: split on empty lines
387        if blank_line_mode {
388            let line = &bytes[line_start..nl_pos];
389            if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
390                let previous_start =
391                    memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
392                if !comment_prefixes
393                    .iter()
394                    .any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
395                {
396                    return Some(nl_pos + 1);
397                }
398            }
399            pos = nl_pos + 1;
400            continue;
401        }
402
403        // Line mode: every newline is a boundary
404        return Some(nl_pos + 1);
405    }
406    None
407}
408
409/// Format-specific parser: turns a byte chunk into field/value events sent to
410/// a `ColumnarSink`.
411pub trait RecordParser: Send + Sync {
412    /// Validate that the whole byte slice is well-formed for this format.
413    fn validate(&self, bytes: &[u8]) -> Result<()>;
414
415    /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
416    fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
417
418    /// Parse one chunk generically (monomorphized, inlinable).
419    ///
420    /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
421    /// have not yet migrated. Adapters that implement this directly get
422    /// devirtualized `sink` calls and cross-crate inlining.
423    #[inline]
424    fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
425    where
426        Self: Sized,
427    {
428        // Default impl delegates to the object-safe path. Adapters that override
429        // this method get devirtualized sink calls and cross-crate inlining.
430        self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
431    }
432}
433
434/// Sink for decoder events.  The decoder calls `begin_row` / `put_field` /
435/// `end_row` for each record.
436///
437/// Implementations must be `Send + Sync` when used with the parallel or
438/// parallel_streaming executors. `TableBuilder` satisfies this.
439pub trait ColumnarSink {
440    /// Begin a new row. Must be paired with exactly one `end_row` per row.
441    fn begin_row(&mut self);
442    fn put_field(&mut self, name: &str, value: Value<'_>);
443
444    /// Push a complete row.
445    ///
446    /// The default preserves event-oriented semantics. Sinks may override this
447    /// to optimize row handling while preserving filtering and duplicate-field
448    /// behavior.
449    #[inline]
450    fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
451        for (name, value) in fields {
452            self.put_field(name, value.clone());
453        }
454    }
455
456    /// Resolve a valid raw name without allocating a UTF-8 `String`.
457    #[inline]
458    fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
459        std::str::from_utf8(raw_name)
460            .ok()
461            .and_then(|name| self.resolve(name))
462    }
463
464    /// Push a value for a field whose name is still in its raw byte form.
465    #[inline]
466    fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
467        if let Ok(name) = std::str::from_utf8(raw_name) {
468            self.resolve_and_put(name, value);
469        }
470    }
471
472    /// End the current row. Must be paired with exactly one `begin_row` per row.
473    fn end_row(&mut self);
474
475    /// Return `false` to signal that the engine will drop this field.
476    #[inline]
477    fn wants(&self, _name: &str) -> bool {
478        true
479    }
480
481    /// Resolve a raw field name to its output column name, or `None` if dropped.
482    /// Default keeps the name as-is. Adapters that do expensive extraction can
483    /// call this once and then `put_field_resolved` to avoid a second
484    /// `resolve_field` inside `put_field`.
485    #[inline]
486    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
487        Some(name)
488    }
489
490    /// Push a field that has already been resolved via `resolve`.
491    /// Default delegates to `put_field` (which will resolve again). Overrides
492    /// should bypass the rename/drop lookup.
493    #[inline]
494    fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
495        self.put_field(resolved_name, value);
496    }
497
498    /// Push a value directly to a known slot index, bypassing all name
499    /// resolution, hash lookups, and attribute scanning.  Used by adapters
500    /// that have verified the field identity via `expect_slot` + memcmp.
501    ///
502    /// Default: discards the value. Engines MUST override this to push
503    /// directly to columns by slot index; the default silently drops data.
504    #[inline]
505    fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
506        let _ = value;
507    }
508
509    /// Resolve a field name and push it in one call, avoiding the double
510    /// `resolve_field` that `wants` + `put_field` would perform.
511    /// Default: resolve then put_field_resolved (with owned clone to satisfy
512    /// the borrow checker). Overrides should avoid the allocation.
513    #[inline]
514    fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
515        if let Some(resolved) = self.resolve(name) {
516            let owned = resolved.to_owned();
517            self.put_field_resolved(&owned, value);
518        }
519    }
520
521    /// Whether this sink needs decoded values from the parser.
522    ///
523    /// Return `false` for locate-only sinks that walk rows and resolve field
524    /// names but skip value extraction entirely.  The parser can use this to
525    /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
526    #[inline]
527    fn needs_value(&self) -> bool {
528        true
529    }
530
531    /// Whether this sink needs field-name resolution (`wants` + `resolve`).
532    ///
533    /// Return `false` for traversal-only sinks that walk rows and find field
534    /// extents but don't need to resolve names or check `wants`.  When false,
535    /// the parser skips `wants()` and `resolve()` calls entirely; it only
536    /// locates the byte extents of each field within a row.
537    ///
538    /// **Note:** this method is primarily for benchmarking/profiling harnesses.
539    /// A sink returning `false` may receive `put_field` calls with unresolved
540    /// names, or none at all, depending on the adapter.  Consider this
541    /// experimental until a non-benchmark consumer exists.
542    #[doc(hidden)]
543    #[inline]
544    fn needs_resolve(&self) -> bool {
545        true
546    }
547
548    /// Whether the current row has been rejected by the filter and the scanner
549    /// should skip the remainder of the row. Used for predicate-first
550    /// deferred materialization: once `Fail`, the scanner can byte-jump to
551    /// `</Details>` without further field scanning. Default false.
552    #[inline]
553    fn row_rejected(&self) -> bool {
554        false
555    }
556
557    /// Whether every wanted column for the current row already has a value.
558    /// The adapter can byte-jump to the row close tag when this returns true,
559    /// skipping remaining fields.  Composes with `row_rejected` (a row can
560    /// be satisfied OR rejected, both short-circuit).
561    ///
562    /// Default: false (never short-circuit).
563    #[inline]
564    fn row_satisfied(&self) -> bool {
565        false
566    }
567
568    /// Bitmask of columns that the projection wants.  Bit `i` is set iff
569    /// column `i` is in the output schema.  The adapter can test membership
570    /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
571    /// field.  Returns empty mask (all zeros) when no projection is active.
572    #[inline]
573    fn wanted_mask(&self) -> u64 {
574        0
575    }
576
577    /// After the layout is learned, the expected (slot, raw name bytes) at
578    /// this ordinal position in the record.  The adapter can compare the
579    /// raw bytes in-place (one memcmp) instead of running the full
580    /// attribute scan → UTF-8 decode → hash → lookup path.
581    ///
582    /// Return `None` to fall back to the generic path.  Default: `None`.
583    #[inline]
584    fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
585        None
586    }
587
588    /// Adapter reports the slot it resolved for `ordinal` so the engine
589    /// can cache it for subsequent rows.  Called after the adapter resolves
590    /// a field via the generic path (wants + resolve + put_field_resolved).
591    #[inline]
592    fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
593
594    /// Adapter reports that the layout expected at `ordinal` did not match.
595    /// The engine should invalidate its cached expectation and fall back to
596    /// generic resolution for subsequent rows.
597    #[inline]
598    fn layout_broken(&mut self, _ordinal: u32) {}
599
600    /// Reset the ordinal counter for child-element tracking.  Called after
601    /// row-tag attributes are emitted so child ordinals start at 0.
602    #[inline]
603    fn reset_child_ordinal(&mut self) {}
604
605    /// Finalize the sink into an Arrow `RecordBatch`.
606    fn finish(&mut self) -> Result<RecordBatch>;
607}
608
609/// Convert split-point offsets returned by a `Splitter` into non-empty
610/// `Range<usize>` chunks.
611pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
612    if points.len() < 2 {
613        return std::iter::once(0..len).collect();
614    }
615    points
616        .windows(2)
617        .filter_map(|w| {
618            let (start, end) = (w[0], w[1]);
619            if start < end {
620                Some(start..end)
621            } else {
622                None
623            }
624        })
625        .collect()
626}
627
628#[cfg(test)]
629mod tests {
630    use super::*;
631
632    /// Records start at every multiple of 4, including 0.
633    struct FixedWidthSplitter;
634
635    impl Splitter for FixedWidthSplitter {
636        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
637            (from..bytes.len()).find(|&p| p % 4 == 0)
638        }
639
640        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
641            4
642        }
643    }
644
645    #[test]
646    fn find_split_points_no_duplicate_leading_zero() {
647        // Input shorter than the planned chunk count: bytes.len() / n == 0
648        // puts every nominal offset at 0, so without the guard the prepended
649        // 0 duplicates the candidate at 0.
650        let bytes = vec![0u8; 8];
651        let splitter = FixedWidthSplitter;
652        for max_chunks in [2, 8, 64, 1024] {
653            let points = splitter.find_split_points(&bytes, max_chunks);
654            assert_eq!(points.first(), Some(&0), "must start at 0");
655            assert_eq!(points.last(), Some(&bytes.len()), "must end at len");
656            assert!(
657                points.windows(2).all(|w| w[0] < w[1]),
658                "duplicate or unsorted points for max_chunks={max_chunks}: {points:?}"
659            );
660        }
661    }
662}
663
664#[cfg(test)]
665mod chunk_floor_tests {
666    use super::*;
667
668    /// A splitter that overrides the documented entry point with a sentinel.
669    struct FixedSplitter;
670    impl Splitter for FixedSplitter {
671        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
672            (from..bytes.len())
673                .find(|&p| bytes[p] == b'\n')
674                .map(|p| p + 1)
675        }
676        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
677            4
678        }
679        fn find_split_points(&self, _bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
680            vec![0, 7]
681        }
682    }
683
684    #[test]
685    fn adapter_override_is_honored_at_the_default_floor() {
686        // Engines call find_split_points_with; at the default floor it must
687        // defer to an adapter's find_split_points override.
688        assert_eq!(
689            FixedSplitter.find_split_points_with(b"ignored", 8, MIN_CHUNK_BYTES),
690            vec![0, 7]
691        );
692    }
693
694    #[test]
695    fn a_lower_floor_yields_more_chunks() {
696        let bytes = 8 << 20; // 8 MiB
697        let default = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, MIN_CHUNK_BYTES);
698        let small = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, 256 * 1024);
699        assert!(small > default, "default={default} small={small}");
700    }
701}