Skip to main content

rypipe_core/
decoder.rs

1use std::borrow::Cow;
2use std::ops::Range;
3
4use arrow::record_batch::RecordBatch;
5
6use crate::value::Value;
7use crate::Result;
8
9// ---------------------------------------------------------------------------
10// Chunk-size floor
11// ---------------------------------------------------------------------------
12
13/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
14/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
15/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
16pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
17
18/// Maximum number of split chunks. Above this, scheduling overhead dominates.
19pub const MAX_SPLIT_CHUNKS: usize = 1024;
20
21/// Mode for chunk planning: parallel and streaming have different optima.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum SplitMode {
24    /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
25    Parallel,
26    /// Streaming: peak at 2 MB chunks (measured with bounded memory).
27    Streaming,
28}
29
30/// Plan the number of chunks given input size, thread count, and mode.
31/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a floor of
32/// `min_chunk_bytes` (default [`MIN_CHUNK_BYTES`] via [`plan_chunk_count`]).
33pub fn plan_chunk_count_with(
34    bytes: usize,
35    threads: usize,
36    mode: SplitMode,
37    min_chunk_bytes: usize,
38) -> usize {
39    let by_size = bytes / min_chunk_bytes.max(1);
40    let cap = match mode {
41        SplitMode::Parallel => 16usize.saturating_mul(threads),
42        SplitMode::Streaming => 8usize.saturating_mul(threads),
43    };
44    by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
45}
46
47/// [`plan_chunk_count_with`] using the default [`MIN_CHUNK_BYTES`] floor.
48pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
49    plan_chunk_count_with(bytes, threads, mode, MIN_CHUNK_BYTES)
50}
51
52// ---------------------------------------------------------------------------
53// SkipRegionFinder: byte ranges where a candidate boundary is invalid
54// ---------------------------------------------------------------------------
55
56/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
57/// a candidate record boundary must be rejected.  The engine calls this
58/// during split-point discovery to avoid splitting inside comments or strings.
59pub trait SkipRegionFinder: Send + Sync {
60    /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
61    fn openers(&self) -> &[&'static [u8]];
62
63    /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
64    fn closer_for(&self, opener: &[u8]) -> &'static [u8];
65
66    /// Maximum backward scan window in bytes.  Default 64 KiB: sufficient
67    /// for most XML/CSV comments and JSON string literals.
68    fn window(&self) -> usize {
69        64 * 1024
70    }
71}
72
73/// Check whether byte position `at` falls inside a skip region by scanning
74/// backward up to `finder.window()` bytes, looking for an unclosed opener.
75///
76/// Returns `false` immediately when the file contains none of the openers;
77/// check that once per chunk, not per candidate.
78pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
79    if at > bytes.len() {
80        return false;
81    }
82    let openers = finder.openers();
83    if openers.is_empty() {
84        return false;
85    }
86    let scan_start = at.saturating_sub(finder.window());
87    let scan = &bytes[scan_start..at];
88    for opener in openers {
89        let closer = finder.closer_for(opener);
90        // Use memchr::memmem::find directly (no Finder::new preprocessing).
91        let mut search = 0;
92        while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
93            let open_pos = scan_start + search + rel;
94            // Check if there's a closer between the opener and `at`.
95            let closer_start = open_pos + opener.len();
96            if closer_start >= at {
97                // Opener extends past `at`; treat as unclosed (inside skip region).
98                return true;
99            }
100            let between = &bytes[closer_start..at];
101            if memchr::memmem::find(between, closer).is_none() {
102                // Unclosed opener; `at` is inside this skip region.
103                return true;
104            }
105            search += rel + 1;
106        }
107    }
108    false
109}
110
111// ---------------------------------------------------------------------------
112// Splitter trait with default
113// ---------------------------------------------------------------------------
114
115/// How records are separated in the input.
116#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
117pub enum RecordBoundary {
118    /// Split on every newline (default). Each `\n` is a record boundary.
119    #[default]
120    Line,
121    /// Split on blank lines. Records span multiple lines; a blank line
122    /// (`\n\n`) separates records.
123    BlankLine,
124}
125
126/// Format-specific splitter: decides where it is safe to divide an input byte
127/// stream into independent chunks.
128///
129/// The only required method is `next_record_start`; adapters provide the
130/// record boundary logic, and the engine handles chunk planning, skip-region
131/// rejection, and deduplication.
132///
133/// For multi-line formats, implement the declarative methods
134/// ([`record_boundary`](Self::record_boundary),
135/// [`continuation_char`](Self::continuation_char),
136/// [`comment_prefixes`](Self::comment_prefixes)) and call
137/// [`find_next_record_boundary`] from your `next_record_start`:
138///
139/// ```rust
140/// use rypipe_core::{find_next_record_boundary, RecordBoundary, Splitter};
141/// struct Lines;
142/// impl Splitter for Lines {
143///     fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
144///         find_next_record_boundary(bytes, from, self.continuation_char(), self.comment_prefixes(), self.record_boundary() == RecordBoundary::BlankLine)
145///     }
146///     fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize { sample.len().max(1) }
147/// }
148/// assert_eq!(Lines.next_record_start(b"one\ntwo\n", 0), Some(4));
149/// ```
150pub trait Splitter: Send + Sync {
151    /// Return the next record boundary at or after `from`.  Must return a
152    /// position where a record starts, or `None` if no more records exist.
153    fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
154
155    /// Estimate the average bytes per record from a sample of the input.
156    fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
157
158    /// How records are separated. Default: [`RecordBoundary::Line`] (every
159    /// newline is a boundary). Override to [`RecordBoundary::BlankLine`] for
160    /// multi-line formats where blank lines separate records.
161    fn record_boundary(&self) -> RecordBoundary {
162        RecordBoundary::Line
163    }
164
165    /// Character that continues a record across lines (e.g. `b'\\'` for
166    /// properties files). When set, newlines preceded by this character are
167    /// NOT record boundaries. Default: `None` (no continuations).
168    fn continuation_char(&self) -> Option<u8> {
169        None
170    }
171
172    /// Line prefixes to skip (e.g. `&[b"#", b"!"]` for properties files).
173    /// Lines starting with any of these are ignored during splitting.
174    /// Default: empty (no comments).
175    fn comment_prefixes(&self) -> &[&[u8]] {
176        &[]
177    }
178
179    /// Optional: byte ranges where a candidate boundary must be rejected.
180    /// See `in_skip_region`.
181    fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
182        None
183    }
184
185    /// Find split points.  **Do not override** without a measured reason.
186    ///
187    /// Returns a sorted list of byte offsets where it is safe to split the
188    /// input into independent chunks. The return value must satisfy:
189    /// - Sorted ascending, no duplicates.
190    /// - First element is `0`, last element is `bytes.len()`.
191    /// - All values are `<= bytes.len()`.
192    ///
193    /// Default implementation:
194    /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
195    /// 2. Scan each nominal offset with `next_record_start`.
196    /// 3. Reject candidates inside skip regions via `in_skip_region`.
197    /// 4. Dedup, sort, prepend 0.
198    /// 5. Apply chunk floor via `plan_chunk_count`.
199    ///
200    /// This is the default-floor entry point. Adapters that override it keep
201    /// working; engines call [`Splitter::find_split_points_with`] instead, and
202    /// that defers to this override at the default floor.
203    fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
204        default_find_split_points(self, bytes, max_chunks, MIN_CHUNK_BYTES)
205    }
206
207    /// Find split points with an explicit chunk-size floor.
208    ///
209    /// The floor is opt-in: lowering it yields more, smaller chunks (better
210    /// load balancing on many or small cores, at more scheduling overhead);
211    /// raising it yields fewer, larger chunks. Default is [`MIN_CHUNK_BYTES`].
212    ///
213    /// At the default floor this calls [`Splitter::find_split_points`], so an
214    /// adapter's override is honored; a non-default floor uses the engine's
215    /// default planning (override `find_split_points_with` to customize it).
216    fn find_split_points_with(
217        &self,
218        bytes: &[u8],
219        max_chunks: usize,
220        min_chunk_bytes: usize,
221    ) -> Vec<usize> {
222        if min_chunk_bytes == MIN_CHUNK_BYTES {
223            return self.find_split_points(bytes, max_chunks);
224        }
225        default_find_split_points(self, bytes, max_chunks, min_chunk_bytes)
226    }
227}
228
229/// The engine's default chunk planning, shared by the `Splitter` methods.
230fn default_find_split_points<S: Splitter + ?Sized>(
231    splitter: &S,
232    bytes: &[u8],
233    max_chunks: usize,
234    min_chunk_bytes: usize,
235) -> Vec<usize> {
236    if max_chunks <= 1 || bytes.is_empty() {
237        return vec![0, bytes.len()];
238    }
239    let n = plan_chunk_count_with(
240        bytes.len(),
241        max_chunks,
242        SplitMode::Parallel,
243        min_chunk_bytes,
244    );
245    let skip = splitter.skip_regions();
246    // Planning must not start a global worker pool for bounded readers.
247    let mut points: Vec<usize> = (1..n)
248        .filter_map(|i| {
249            let approx = bytes.len() / n * i;
250            let pos = splitter.next_record_start(bytes, approx)?;
251            // Reject candidates inside skip regions.
252            if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
253                return None;
254            }
255            Some(pos)
256        })
257        .collect();
258
259    points.sort_unstable();
260    points.dedup();
261    // Drop candidates at 0: the prepend below already supplies the
262    // leading split point, and keeping them would duplicate it (this
263    // happens when bytes.len() < n, where bytes.len() / n == 0 puts
264    // every nominal offset at 0).
265    points.retain(|&p| p > 0);
266    points.insert(0, 0);
267    if *points.last().unwrap_or(&0) != bytes.len() {
268        points.push(bytes.len());
269    }
270    points
271}
272
273// ---------------------------------------------------------------------------
274// Declarative splitting helpers
275// ---------------------------------------------------------------------------
276
277/// Check whether the newline at `pos` is preceded by a continuation character.
278///
279/// Used by Splitters that need to skip continued newlines. For example, in
280/// properties files, `key = value \` continues on the next line: the `\n`
281/// after `\` is NOT a record boundary.
282///
283/// ```rust
284/// use rypipe_core::is_continued_newline;
285/// assert!(is_continued_newline(b"key=value\\\nnext\n", 10));
286/// ```
287#[inline]
288pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
289    if pos == 0 || bytes.get(pos) != Some(&b'\n') {
290        return false;
291    }
292    let mut run = 0;
293    let mut cursor = pos;
294    if bytes[cursor - 1] == b'\r' {
295        cursor -= 1;
296    }
297    while cursor > 0 && bytes[cursor - 1] == b'\\' {
298        run += 1;
299        cursor -= 1;
300    }
301    run % 2 == 1
302}
303
304/// Scan forward from `from` to find the next record boundary.
305///
306/// This is the declarative counterpart to manual byte scanning. It handles:
307/// - **Continuations**: skips `\n` preceded by `continuation` (e.g. `b'\\'`)
308/// - **Comments**: skips lines starting with any prefix in `comment_prefixes`
309/// - **Blank-line mode**: splits after an empty LF or CRLF line
310///
311/// Returns the byte position after the boundary newline, or `None` if no
312/// more boundaries exist.
313///
314/// # Example
315///
316/// ```rust
317/// use rypipe_core::find_next_record_boundary;
318/// let input = b"# comment\nfirst\nsecond\n";
319/// assert_eq!(find_next_record_boundary(input, 0, None, &[b"#"], false), Some(16));
320/// ```
321#[inline]
322pub fn find_next_record_boundary(
323    bytes: &[u8],
324    from: usize,
325    continuation: Option<u8>,
326    comment_prefixes: &[&[u8]],
327    blank_line_mode: bool,
328) -> Option<usize> {
329    if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
330        return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
331    }
332    let mut pos = from;
333    while pos < bytes.len() {
334        let rel = memchr::memchr(b'\n', &bytes[pos..])?;
335        let nl_pos = pos + rel;
336
337        // Skip continued newlines (preceded by continuation character)
338        let continued = |newline: usize| {
339            continuation.is_some_and(|character| {
340                let end = if newline > 0 && bytes[newline - 1] == b'\r' {
341                    newline - 1
342                } else {
343                    newline
344                };
345                if character == b'\\' {
346                    is_continued_newline(bytes, newline)
347                } else {
348                    end > 0 && bytes[end - 1] == character
349                }
350            })
351        };
352        if continued(nl_pos) {
353            pos = nl_pos + 1;
354            continue;
355        }
356
357        if !blank_line_mode && comment_prefixes.is_empty() {
358            return Some(nl_pos + 1);
359        }
360
361        if blank_line_mode {
362            let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
363            if end == 0 || bytes[end - 1] != b'\n' {
364                pos = nl_pos + 1;
365                continue;
366            }
367            if comment_prefixes.is_empty() && !continued(end - 1) {
368                return Some(nl_pos + 1);
369            }
370        }
371
372        // Compute line start for comment check
373        let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
374
375        // Skip comment lines
376        if !comment_prefixes.is_empty() {
377            let line_bytes = &bytes[line_start..nl_pos];
378            let is_comment = comment_prefixes
379                .iter()
380                .any(|prefix| line_bytes.starts_with(prefix));
381            if is_comment {
382                pos = nl_pos + 1;
383                continue;
384            }
385        }
386
387        // Blank-line mode: split on empty lines
388        if blank_line_mode {
389            let line = &bytes[line_start..nl_pos];
390            if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
391                let previous_start =
392                    memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
393                if !comment_prefixes
394                    .iter()
395                    .any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
396                {
397                    return Some(nl_pos + 1);
398                }
399            }
400            pos = nl_pos + 1;
401            continue;
402        }
403
404        // Line mode: every newline is a boundary
405        return Some(nl_pos + 1);
406    }
407    None
408}
409
410/// Format-specific parser: turns a byte chunk into field/value events sent to
411/// a `ColumnarSink`.
412pub trait RecordParser: Send + Sync {
413    /// Validate that the whole byte slice is well-formed for this format.
414    fn validate(&self, bytes: &[u8]) -> Result<()>;
415
416    /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
417    fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
418
419    /// Parse one chunk generically (monomorphized, inlinable).
420    ///
421    /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
422    /// have not yet migrated. Adapters that implement this directly get
423    /// devirtualized `sink` calls and cross-crate inlining.
424    #[inline]
425    fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
426    where
427        Self: Sized,
428    {
429        // Default impl delegates to the object-safe path. Adapters that override
430        // this method get devirtualized sink calls and cross-crate inlining.
431        self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
432    }
433}
434
435/// Sink for decoder events.  The decoder calls `begin_row` / `put_field` /
436/// `end_row` for each record.
437///
438/// Implementations must be `Send + Sync` when used with the parallel or
439/// parallel_streaming executors. `TableBuilder` satisfies this.
440pub trait ColumnarSink {
441    /// Begin a new row. Must be paired with exactly one `end_row` per row.
442    fn begin_row(&mut self);
443    fn put_field(&mut self, name: &str, value: Value<'_>);
444
445    /// Push a complete row.
446    ///
447    /// The default preserves event-oriented semantics. Sinks may override this
448    /// to optimize row handling while preserving filtering and duplicate-field
449    /// behavior.
450    #[inline]
451    fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
452        for (name, value) in fields {
453            self.put_field(name, value.clone());
454        }
455    }
456
457    /// Resolve a valid raw name without allocating a UTF-8 `String`.
458    #[inline]
459    fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
460        std::str::from_utf8(raw_name)
461            .ok()
462            .and_then(|name| self.resolve(name))
463    }
464
465    /// Push a value for a field whose name is still in its raw byte form.
466    #[inline]
467    fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
468        if let Ok(name) = std::str::from_utf8(raw_name) {
469            self.resolve_and_put(name, value);
470        }
471    }
472
473    /// End the current row. Must be paired with exactly one `begin_row` per row.
474    fn end_row(&mut self);
475
476    /// Return `false` to signal that the engine will drop this field.
477    #[inline]
478    fn wants(&self, _name: &str) -> bool {
479        true
480    }
481
482    /// Resolve a raw field name to its output column name, or `None` if dropped.
483    /// Default keeps the name as-is. Adapters that do expensive extraction can
484    /// call this once and then `put_field_resolved` to avoid a second
485    /// `resolve_field` inside `put_field`.
486    #[inline]
487    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
488        Some(name)
489    }
490
491    /// Push a field that has already been resolved via `resolve`.
492    /// Default delegates to `put_field` (which will resolve again). Overrides
493    /// should bypass the rename/drop lookup.
494    #[inline]
495    fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
496        self.put_field(resolved_name, value);
497    }
498
499    /// Push a value directly to a known slot index, bypassing all name
500    /// resolution, hash lookups, and attribute scanning.  Used by adapters
501    /// that have verified the field identity via `expect_slot` + memcmp.
502    ///
503    /// Default: discards the value. Engines MUST override this to push
504    /// directly to columns by slot index; the default silently drops data.
505    #[inline]
506    fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
507        let _ = value;
508    }
509
510    /// Push a known-`&str` field into a known slot, bypassing the `Value`
511    /// round trip. Default forwards to `put_field_at`; sinks that store strings
512    /// directly (e.g. `TableBuilder`) override it for the hot path.
513    #[inline]
514    fn put_str_at(&mut self, slot: u32, value: &str) {
515        self.put_field_at(slot, Value::Str(Cow::Borrowed(value)));
516    }
517
518    /// Resolve a field name and push it in one call, avoiding the double
519    /// `resolve_field` that `wants` + `put_field` would perform.
520    /// Default: resolve then put_field_resolved (with owned clone to satisfy
521    /// the borrow checker). Overrides should avoid the allocation.
522    #[inline]
523    fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
524        if let Some(resolved) = self.resolve(name) {
525            let owned = resolved.to_owned();
526            self.put_field_resolved(&owned, value);
527        }
528    }
529
530    /// Whether this sink needs decoded values from the parser.
531    ///
532    /// Return `false` for locate-only sinks that walk rows and resolve field
533    /// names but skip value extraction entirely.  The parser can use this to
534    /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
535    #[inline]
536    fn needs_value(&self) -> bool {
537        true
538    }
539
540    /// Whether this sink needs field-name resolution (`wants` + `resolve`).
541    ///
542    /// Return `false` for traversal-only sinks that walk rows and find field
543    /// extents but don't need to resolve names or check `wants`.  When false,
544    /// the parser skips `wants()` and `resolve()` calls entirely; it only
545    /// locates the byte extents of each field within a row.
546    ///
547    /// **Note:** this method is primarily for benchmarking/profiling harnesses.
548    /// A sink returning `false` may receive `put_field` calls with unresolved
549    /// names, or none at all, depending on the adapter.  Consider this
550    /// experimental until a non-benchmark consumer exists.
551    #[doc(hidden)]
552    #[inline]
553    fn needs_resolve(&self) -> bool {
554        true
555    }
556
557    /// Whether the current row has been rejected by the filter and the scanner
558    /// should skip the remainder of the row. Used for predicate-first
559    /// deferred materialization: once `Fail`, the scanner can byte-jump to
560    /// `</Details>` without further field scanning. Default false.
561    #[inline]
562    fn row_rejected(&self) -> bool {
563        false
564    }
565
566    /// Return true to abort parsing early. Parsers should check this between
567    /// records. Bounded single-pass sinks use it to stop promptly when the
568    /// consumer has gone away (e.g. a dropped streaming iterator), instead of
569    /// finishing the whole input. Default false (never stop early).
570    #[inline]
571    fn should_stop(&self) -> bool {
572        false
573    }
574
575    /// Whether every wanted column for the current row already has a value.
576    /// The adapter can byte-jump to the row close tag when this returns true,
577    /// skipping remaining fields.  Composes with `row_rejected` (a row can
578    /// be satisfied OR rejected, both short-circuit).
579    ///
580    /// Default: false (never short-circuit).
581    #[inline]
582    fn row_satisfied(&self) -> bool {
583        false
584    }
585
586    /// Bitmask of columns that the projection wants.  Bit `i` is set iff
587    /// column `i` is in the output schema.  The adapter can test membership
588    /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
589    /// field.  Returns empty mask (all zeros) when no projection is active.
590    #[inline]
591    fn wanted_mask(&self) -> u64 {
592        0
593    }
594
595    /// After the layout is learned, the expected (slot, raw name bytes) at
596    /// this ordinal position in the record.  The adapter can compare the
597    /// raw bytes in-place (one memcmp) instead of running the full
598    /// attribute scan → UTF-8 decode → hash → lookup path.
599    ///
600    /// Return `None` to fall back to the generic path.  Default: `None`.
601    #[inline]
602    fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
603        None
604    }
605
606    /// Adapter reports the slot it resolved for `ordinal` so the engine
607    /// can cache it for subsequent rows.  Called after the adapter resolves
608    /// a field via the generic path (wants + resolve + put_field_resolved).
609    #[inline]
610    fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
611
612    /// Adapter reports that the layout expected at `ordinal` did not match.
613    /// The engine should invalidate its cached expectation and fall back to
614    /// generic resolution for subsequent rows.
615    #[inline]
616    fn layout_broken(&mut self, _ordinal: u32) {}
617
618    /// Reset the ordinal counter for child-element tracking.  Called after
619    /// row-tag attributes are emitted so child ordinals start at 0.
620    #[inline]
621    fn reset_child_ordinal(&mut self) {}
622
623    /// Finalize the sink into an Arrow `RecordBatch`.
624    fn finish(&mut self) -> Result<RecordBatch>;
625}
626
627/// Convert split-point offsets returned by a `Splitter` into non-empty
628/// `Range<usize>` chunks.
629pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
630    if points.len() < 2 {
631        return std::iter::once(0..len).collect();
632    }
633    points
634        .windows(2)
635        .filter_map(|w| {
636            let (start, end) = (w[0], w[1]);
637            if start < end {
638                Some(start..end)
639            } else {
640                None
641            }
642        })
643        .collect()
644}
645
646#[cfg(test)]
647mod tests {
648    use super::*;
649
650    /// Records start at every multiple of 4, including 0.
651    struct FixedWidthSplitter;
652
653    impl Splitter for FixedWidthSplitter {
654        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
655            (from..bytes.len()).find(|&p| p % 4 == 0)
656        }
657
658        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
659            4
660        }
661    }
662
663    #[test]
664    fn find_split_points_no_duplicate_leading_zero() {
665        // Input shorter than the planned chunk count: bytes.len() / n == 0
666        // puts every nominal offset at 0, so without the guard the prepended
667        // 0 duplicates the candidate at 0.
668        let bytes = vec![0u8; 8];
669        let splitter = FixedWidthSplitter;
670        for max_chunks in [2, 8, 64, 1024] {
671            let points = splitter.find_split_points(&bytes, max_chunks);
672            assert_eq!(points.first(), Some(&0), "must start at 0");
673            assert_eq!(points.last(), Some(&bytes.len()), "must end at len");
674            assert!(
675                points.windows(2).all(|w| w[0] < w[1]),
676                "duplicate or unsorted points for max_chunks={max_chunks}: {points:?}"
677            );
678        }
679    }
680}
681
682#[cfg(test)]
683mod chunk_floor_tests {
684    use super::*;
685
686    /// A splitter that overrides the documented entry point with a sentinel.
687    struct FixedSplitter;
688    impl Splitter for FixedSplitter {
689        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
690            (from..bytes.len())
691                .find(|&p| bytes[p] == b'\n')
692                .map(|p| p + 1)
693        }
694        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
695            4
696        }
697        fn find_split_points(&self, _bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
698            vec![0, 7]
699        }
700    }
701
702    #[test]
703    fn adapter_override_is_honored_at_the_default_floor() {
704        // Engines call find_split_points_with; at the default floor it must
705        // defer to an adapter's find_split_points override.
706        assert_eq!(
707            FixedSplitter.find_split_points_with(b"ignored", 8, MIN_CHUNK_BYTES),
708            vec![0, 7]
709        );
710    }
711
712    #[test]
713    fn a_lower_floor_yields_more_chunks() {
714        let bytes = 8 << 20; // 8 MiB
715        let default = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, MIN_CHUNK_BYTES);
716        let small = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, 256 * 1024);
717        assert!(small > default, "default={default} small={small}");
718    }
719}