rypipe_core/decoder.rs
1use std::borrow::Cow;
2use std::ops::Range;
3
4use arrow::record_batch::RecordBatch;
5
6use crate::value::Value;
7use crate::Result;
8
9// ---------------------------------------------------------------------------
10// Chunk-size floor
11// ---------------------------------------------------------------------------
12
13/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
14/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
15/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
16pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
17
18/// Maximum number of split chunks. Above this, scheduling overhead dominates.
19pub const MAX_SPLIT_CHUNKS: usize = 1024;
20
21/// Mode for chunk planning: parallel and streaming have different optima.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum SplitMode {
24 /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
25 Parallel,
26 /// Streaming: peak at 2 MB chunks (measured with bounded memory).
27 Streaming,
28}
29
30/// Plan the number of chunks given input size, thread count, and mode.
31/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a floor of
32/// `min_chunk_bytes` (default [`MIN_CHUNK_BYTES`] via [`plan_chunk_count`]).
33pub fn plan_chunk_count_with(
34 bytes: usize,
35 threads: usize,
36 mode: SplitMode,
37 min_chunk_bytes: usize,
38) -> usize {
39 let by_size = bytes / min_chunk_bytes.max(1);
40 let cap = match mode {
41 SplitMode::Parallel => 16usize.saturating_mul(threads),
42 SplitMode::Streaming => 8usize.saturating_mul(threads),
43 };
44 by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
45}
46
47/// [`plan_chunk_count_with`] using the default [`MIN_CHUNK_BYTES`] floor.
48pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
49 plan_chunk_count_with(bytes, threads, mode, MIN_CHUNK_BYTES)
50}
51
52// ---------------------------------------------------------------------------
53// SkipRegionFinder: byte ranges where a candidate boundary is invalid
54// ---------------------------------------------------------------------------
55
56/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
57/// a candidate record boundary must be rejected. The engine calls this
58/// during split-point discovery to avoid splitting inside comments or strings.
59pub trait SkipRegionFinder: Send + Sync {
60 /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
61 fn openers(&self) -> &[&'static [u8]];
62
63 /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
64 fn closer_for(&self, opener: &[u8]) -> &'static [u8];
65
66 /// Maximum backward scan window in bytes. Default 64 KiB: sufficient
67 /// for most XML/CSV comments and JSON string literals.
68 fn window(&self) -> usize {
69 64 * 1024
70 }
71}
72
73/// Check whether byte position `at` falls inside a skip region by scanning
74/// backward up to `finder.window()` bytes, looking for an unclosed opener.
75///
76/// Returns `false` immediately when the file contains none of the openers;
77/// check that once per chunk, not per candidate.
78pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
79 if at > bytes.len() {
80 return false;
81 }
82 let openers = finder.openers();
83 if openers.is_empty() {
84 return false;
85 }
86 let scan_start = at.saturating_sub(finder.window());
87 let scan = &bytes[scan_start..at];
88 for opener in openers {
89 let closer = finder.closer_for(opener);
90 // Use memchr::memmem::find directly (no Finder::new preprocessing).
91 let mut search = 0;
92 while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
93 let open_pos = scan_start + search + rel;
94 // Check if there's a closer between the opener and `at`.
95 let closer_start = open_pos + opener.len();
96 if closer_start >= at {
97 // Opener extends past `at`; treat as unclosed (inside skip region).
98 return true;
99 }
100 let between = &bytes[closer_start..at];
101 if memchr::memmem::find(between, closer).is_none() {
102 // Unclosed opener; `at` is inside this skip region.
103 return true;
104 }
105 search += rel + 1;
106 }
107 }
108 false
109}
110
111// ---------------------------------------------------------------------------
112// Splitter trait with default
113// ---------------------------------------------------------------------------
114
115/// How records are separated in the input.
116#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
117pub enum RecordBoundary {
118 /// Split on every newline (default). Each `\n` is a record boundary.
119 #[default]
120 Line,
121 /// Split on blank lines. Records span multiple lines; a blank line
122 /// (`\n\n`) separates records.
123 BlankLine,
124}
125
126/// Format-specific splitter: decides where it is safe to divide an input byte
127/// stream into independent chunks.
128///
129/// The only required method is `next_record_start`; adapters provide the
130/// record boundary logic, and the engine handles chunk planning, skip-region
131/// rejection, and deduplication.
132///
133/// For multi-line formats, implement the declarative methods
134/// ([`record_boundary`](Self::record_boundary),
135/// [`continuation_char`](Self::continuation_char),
136/// [`comment_prefixes`](Self::comment_prefixes)) and call
137/// [`find_next_record_boundary`] from your `next_record_start`:
138///
139/// ```rust
140/// use rypipe_core::{find_next_record_boundary, RecordBoundary, Splitter};
141/// struct Lines;
142/// impl Splitter for Lines {
143/// fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
144/// find_next_record_boundary(bytes, from, self.continuation_char(), self.comment_prefixes(), self.record_boundary() == RecordBoundary::BlankLine)
145/// }
146/// fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize { sample.len().max(1) }
147/// }
148/// assert_eq!(Lines.next_record_start(b"one\ntwo\n", 0), Some(4));
149/// ```
150pub trait Splitter: Send + Sync {
151 /// Return the next record boundary at or after `from`. Must return a
152 /// position where a record starts, or `None` if no more records exist.
153 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
154
155 /// Estimate the average bytes per record from a sample of the input.
156 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
157
158 /// How records are separated. Default: [`RecordBoundary::Line`] (every
159 /// newline is a boundary). Override to [`RecordBoundary::BlankLine`] for
160 /// multi-line formats where blank lines separate records.
161 fn record_boundary(&self) -> RecordBoundary {
162 RecordBoundary::Line
163 }
164
165 /// Character that continues a record across lines (e.g. `b'\\'` for
166 /// properties files). When set, newlines preceded by this character are
167 /// NOT record boundaries. Default: `None` (no continuations).
168 fn continuation_char(&self) -> Option<u8> {
169 None
170 }
171
172 /// Line prefixes to skip (e.g. `&[b"#", b"!"]` for properties files).
173 /// Lines starting with any of these are ignored during splitting.
174 /// Default: empty (no comments).
175 fn comment_prefixes(&self) -> &[&[u8]] {
176 &[]
177 }
178
179 /// Optional: byte ranges where a candidate boundary must be rejected.
180 /// See `in_skip_region`.
181 fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
182 None
183 }
184
185 /// Find split points. **Do not override** without a measured reason.
186 ///
187 /// Returns a sorted list of byte offsets where it is safe to split the
188 /// input into independent chunks. The return value must satisfy:
189 /// - Sorted ascending, no duplicates.
190 /// - First element is `0`, last element is `bytes.len()`.
191 /// - All values are `<= bytes.len()`.
192 ///
193 /// Default implementation:
194 /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
195 /// 2. Scan each nominal offset with `next_record_start`.
196 /// 3. Reject candidates inside skip regions via `in_skip_region`.
197 /// 4. Dedup, sort, prepend 0.
198 /// 5. Apply chunk floor via `plan_chunk_count`.
199 ///
200 /// This is the default-floor entry point. Adapters that override it keep
201 /// working; engines call [`Splitter::find_split_points_with`] instead, and
202 /// that defers to this override at the default floor.
203 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
204 default_find_split_points(self, bytes, max_chunks, MIN_CHUNK_BYTES)
205 }
206
207 /// Find split points with an explicit chunk-size floor.
208 ///
209 /// The floor is opt-in: lowering it yields more, smaller chunks (better
210 /// load balancing on many or small cores, at more scheduling overhead);
211 /// raising it yields fewer, larger chunks. Default is [`MIN_CHUNK_BYTES`].
212 ///
213 /// At the default floor this calls [`Splitter::find_split_points`], so an
214 /// adapter's override is honored; a non-default floor uses the engine's
215 /// default planning (override `find_split_points_with` to customize it).
216 fn find_split_points_with(
217 &self,
218 bytes: &[u8],
219 max_chunks: usize,
220 min_chunk_bytes: usize,
221 ) -> Vec<usize> {
222 if min_chunk_bytes == MIN_CHUNK_BYTES {
223 return self.find_split_points(bytes, max_chunks);
224 }
225 default_find_split_points(self, bytes, max_chunks, min_chunk_bytes)
226 }
227}
228
229/// The engine's default chunk planning, shared by the `Splitter` methods.
230fn default_find_split_points<S: Splitter + ?Sized>(
231 splitter: &S,
232 bytes: &[u8],
233 max_chunks: usize,
234 min_chunk_bytes: usize,
235) -> Vec<usize> {
236 if max_chunks <= 1 || bytes.is_empty() {
237 return vec![0, bytes.len()];
238 }
239 let n = plan_chunk_count_with(
240 bytes.len(),
241 max_chunks,
242 SplitMode::Parallel,
243 min_chunk_bytes,
244 );
245 let skip = splitter.skip_regions();
246 // Planning must not start a global worker pool for bounded readers.
247 let mut points: Vec<usize> = (1..n)
248 .filter_map(|i| {
249 let approx = bytes.len() / n * i;
250 let pos = splitter.next_record_start(bytes, approx)?;
251 // Reject candidates inside skip regions.
252 if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
253 return None;
254 }
255 Some(pos)
256 })
257 .collect();
258
259 points.sort_unstable();
260 points.dedup();
261 // Drop candidates at 0: the prepend below already supplies the
262 // leading split point, and keeping them would duplicate it (this
263 // happens when bytes.len() < n, where bytes.len() / n == 0 puts
264 // every nominal offset at 0).
265 points.retain(|&p| p > 0);
266 points.insert(0, 0);
267 if *points.last().unwrap_or(&0) != bytes.len() {
268 points.push(bytes.len());
269 }
270 points
271}
272
273// ---------------------------------------------------------------------------
274// Declarative splitting helpers
275// ---------------------------------------------------------------------------
276
277/// Check whether the newline at `pos` is preceded by a continuation character.
278///
279/// Used by Splitters that need to skip continued newlines. For example, in
280/// properties files, `key = value \` continues on the next line: the `\n`
281/// after `\` is NOT a record boundary.
282///
283/// ```rust
284/// use rypipe_core::is_continued_newline;
285/// assert!(is_continued_newline(b"key=value\\\nnext\n", 10));
286/// ```
287#[inline]
288pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
289 if pos == 0 || bytes.get(pos) != Some(&b'\n') {
290 return false;
291 }
292 let mut run = 0;
293 let mut cursor = pos;
294 if bytes[cursor - 1] == b'\r' {
295 cursor -= 1;
296 }
297 while cursor > 0 && bytes[cursor - 1] == b'\\' {
298 run += 1;
299 cursor -= 1;
300 }
301 run % 2 == 1
302}
303
304/// Scan forward from `from` to find the next record boundary.
305///
306/// This is the declarative counterpart to manual byte scanning. It handles:
307/// - **Continuations**: skips `\n` preceded by `continuation` (e.g. `b'\\'`)
308/// - **Comments**: skips lines starting with any prefix in `comment_prefixes`
309/// - **Blank-line mode**: splits after an empty LF or CRLF line
310///
311/// Returns the byte position after the boundary newline, or `None` if no
312/// more boundaries exist.
313///
314/// # Example
315///
316/// ```rust
317/// use rypipe_core::find_next_record_boundary;
318/// let input = b"# comment\nfirst\nsecond\n";
319/// assert_eq!(find_next_record_boundary(input, 0, None, &[b"#"], false), Some(16));
320/// ```
321#[inline]
322pub fn find_next_record_boundary(
323 bytes: &[u8],
324 from: usize,
325 continuation: Option<u8>,
326 comment_prefixes: &[&[u8]],
327 blank_line_mode: bool,
328) -> Option<usize> {
329 if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
330 return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
331 }
332 let mut pos = from;
333 while pos < bytes.len() {
334 let rel = memchr::memchr(b'\n', &bytes[pos..])?;
335 let nl_pos = pos + rel;
336
337 // Skip continued newlines (preceded by continuation character)
338 let continued = |newline: usize| {
339 continuation.is_some_and(|character| {
340 let end = if newline > 0 && bytes[newline - 1] == b'\r' {
341 newline - 1
342 } else {
343 newline
344 };
345 if character == b'\\' {
346 is_continued_newline(bytes, newline)
347 } else {
348 end > 0 && bytes[end - 1] == character
349 }
350 })
351 };
352 if continued(nl_pos) {
353 pos = nl_pos + 1;
354 continue;
355 }
356
357 if !blank_line_mode && comment_prefixes.is_empty() {
358 return Some(nl_pos + 1);
359 }
360
361 if blank_line_mode {
362 let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
363 if end == 0 || bytes[end - 1] != b'\n' {
364 pos = nl_pos + 1;
365 continue;
366 }
367 if comment_prefixes.is_empty() && !continued(end - 1) {
368 return Some(nl_pos + 1);
369 }
370 }
371
372 // Compute line start for comment check
373 let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
374
375 // Skip comment lines
376 if !comment_prefixes.is_empty() {
377 let line_bytes = &bytes[line_start..nl_pos];
378 let is_comment = comment_prefixes
379 .iter()
380 .any(|prefix| line_bytes.starts_with(prefix));
381 if is_comment {
382 pos = nl_pos + 1;
383 continue;
384 }
385 }
386
387 // Blank-line mode: split on empty lines
388 if blank_line_mode {
389 let line = &bytes[line_start..nl_pos];
390 if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
391 let previous_start =
392 memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
393 if !comment_prefixes
394 .iter()
395 .any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
396 {
397 return Some(nl_pos + 1);
398 }
399 }
400 pos = nl_pos + 1;
401 continue;
402 }
403
404 // Line mode: every newline is a boundary
405 return Some(nl_pos + 1);
406 }
407 None
408}
409
410/// Format-specific parser: turns a byte chunk into field/value events sent to
411/// a `ColumnarSink`.
412pub trait RecordParser: Send + Sync {
413 /// Validate that the whole byte slice is well-formed for this format.
414 fn validate(&self, bytes: &[u8]) -> Result<()>;
415
416 /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
417 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
418
419 /// Parse one chunk generically (monomorphized, inlinable).
420 ///
421 /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
422 /// have not yet migrated. Adapters that implement this directly get
423 /// devirtualized `sink` calls and cross-crate inlining.
424 #[inline]
425 fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
426 where
427 Self: Sized,
428 {
429 // Default impl delegates to the object-safe path. Adapters that override
430 // this method get devirtualized sink calls and cross-crate inlining.
431 self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
432 }
433}
434
435/// Sink for decoder events. The decoder calls `begin_row` / `put_field` /
436/// `end_row` for each record.
437///
438/// Implementations must be `Send + Sync` when used with the parallel or
439/// parallel_streaming executors. `TableBuilder` satisfies this.
440pub trait ColumnarSink {
441 /// Begin a new row. Must be paired with exactly one `end_row` per row.
442 fn begin_row(&mut self);
443 fn put_field(&mut self, name: &str, value: Value<'_>);
444
445 /// Push a complete row.
446 ///
447 /// The default preserves event-oriented semantics. Sinks may override this
448 /// to optimize row handling while preserving filtering and duplicate-field
449 /// behavior.
450 #[inline]
451 fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
452 for (name, value) in fields {
453 self.put_field(name, value.clone());
454 }
455 }
456
457 /// Resolve a valid raw name without allocating a UTF-8 `String`.
458 #[inline]
459 fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
460 std::str::from_utf8(raw_name)
461 .ok()
462 .and_then(|name| self.resolve(name))
463 }
464
465 /// Push a value for a field whose name is still in its raw byte form.
466 #[inline]
467 fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
468 if let Ok(name) = std::str::from_utf8(raw_name) {
469 self.resolve_and_put(name, value);
470 }
471 }
472
473 /// End the current row. Must be paired with exactly one `begin_row` per row.
474 fn end_row(&mut self);
475
476 /// Return `false` to signal that the engine will drop this field.
477 #[inline]
478 fn wants(&self, _name: &str) -> bool {
479 true
480 }
481
482 /// Resolve a raw field name to its output column name, or `None` if dropped.
483 /// Default keeps the name as-is. Adapters that do expensive extraction can
484 /// call this once and then `put_field_resolved` to avoid a second
485 /// `resolve_field` inside `put_field`.
486 #[inline]
487 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
488 Some(name)
489 }
490
491 /// Push a field that has already been resolved via `resolve`.
492 /// Default delegates to `put_field` (which will resolve again). Overrides
493 /// should bypass the rename/drop lookup.
494 #[inline]
495 fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
496 self.put_field(resolved_name, value);
497 }
498
499 /// Push a value directly to a known slot index, bypassing all name
500 /// resolution, hash lookups, and attribute scanning. Used by adapters
501 /// that have verified the field identity via `expect_slot` + memcmp.
502 ///
503 /// Default: discards the value. Engines MUST override this to push
504 /// directly to columns by slot index; the default silently drops data.
505 #[inline]
506 fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
507 let _ = value;
508 }
509
510 /// Push a known-`&str` field into a known slot, bypassing the `Value`
511 /// round trip. Default forwards to `put_field_at`; sinks that store strings
512 /// directly (e.g. `TableBuilder`) override it for the hot path.
513 #[inline]
514 fn put_str_at(&mut self, slot: u32, value: &str) {
515 self.put_field_at(slot, Value::Str(Cow::Borrowed(value)));
516 }
517
518 /// Resolve a field name and push it in one call, avoiding the double
519 /// `resolve_field` that `wants` + `put_field` would perform.
520 /// Default: resolve then put_field_resolved (with owned clone to satisfy
521 /// the borrow checker). Overrides should avoid the allocation.
522 #[inline]
523 fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
524 if let Some(resolved) = self.resolve(name) {
525 let owned = resolved.to_owned();
526 self.put_field_resolved(&owned, value);
527 }
528 }
529
530 /// Whether this sink needs decoded values from the parser.
531 ///
532 /// Return `false` for locate-only sinks that walk rows and resolve field
533 /// names but skip value extraction entirely. The parser can use this to
534 /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
535 #[inline]
536 fn needs_value(&self) -> bool {
537 true
538 }
539
540 /// Whether this sink needs field-name resolution (`wants` + `resolve`).
541 ///
542 /// Return `false` for traversal-only sinks that walk rows and find field
543 /// extents but don't need to resolve names or check `wants`. When false,
544 /// the parser skips `wants()` and `resolve()` calls entirely; it only
545 /// locates the byte extents of each field within a row.
546 ///
547 /// **Note:** this method is primarily for benchmarking/profiling harnesses.
548 /// A sink returning `false` may receive `put_field` calls with unresolved
549 /// names, or none at all, depending on the adapter. Consider this
550 /// experimental until a non-benchmark consumer exists.
551 #[doc(hidden)]
552 #[inline]
553 fn needs_resolve(&self) -> bool {
554 true
555 }
556
557 /// Whether the current row has been rejected by the filter and the scanner
558 /// should skip the remainder of the row. Used for predicate-first
559 /// deferred materialization: once `Fail`, the scanner can byte-jump to
560 /// `</Details>` without further field scanning. Default false.
561 #[inline]
562 fn row_rejected(&self) -> bool {
563 false
564 }
565
566 /// Return true to abort parsing early. Parsers should check this between
567 /// records. Bounded single-pass sinks use it to stop promptly when the
568 /// consumer has gone away (e.g. a dropped streaming iterator), instead of
569 /// finishing the whole input. Default false (never stop early).
570 #[inline]
571 fn should_stop(&self) -> bool {
572 false
573 }
574
575 /// Whether every wanted column for the current row already has a value.
576 /// The adapter can byte-jump to the row close tag when this returns true,
577 /// skipping remaining fields. Composes with `row_rejected` (a row can
578 /// be satisfied OR rejected, both short-circuit).
579 ///
580 /// Default: false (never short-circuit).
581 #[inline]
582 fn row_satisfied(&self) -> bool {
583 false
584 }
585
586 /// Bitmask of columns that the projection wants. Bit `i` is set iff
587 /// column `i` is in the output schema. The adapter can test membership
588 /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
589 /// field. Returns empty mask (all zeros) when no projection is active.
590 #[inline]
591 fn wanted_mask(&self) -> u64 {
592 0
593 }
594
595 /// After the layout is learned, the expected (slot, raw name bytes) at
596 /// this ordinal position in the record. The adapter can compare the
597 /// raw bytes in-place (one memcmp) instead of running the full
598 /// attribute scan → UTF-8 decode → hash → lookup path.
599 ///
600 /// Return `None` to fall back to the generic path. Default: `None`.
601 #[inline]
602 fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
603 None
604 }
605
606 /// Adapter reports the slot it resolved for `ordinal` so the engine
607 /// can cache it for subsequent rows. Called after the adapter resolves
608 /// a field via the generic path (wants + resolve + put_field_resolved).
609 #[inline]
610 fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
611
612 /// Adapter reports that the layout expected at `ordinal` did not match.
613 /// The engine should invalidate its cached expectation and fall back to
614 /// generic resolution for subsequent rows.
615 #[inline]
616 fn layout_broken(&mut self, _ordinal: u32) {}
617
618 /// Reset the ordinal counter for child-element tracking. Called after
619 /// row-tag attributes are emitted so child ordinals start at 0.
620 #[inline]
621 fn reset_child_ordinal(&mut self) {}
622
623 /// Finalize the sink into an Arrow `RecordBatch`.
624 fn finish(&mut self) -> Result<RecordBatch>;
625}
626
627/// Convert split-point offsets returned by a `Splitter` into non-empty
628/// `Range<usize>` chunks.
629pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
630 if points.len() < 2 {
631 return std::iter::once(0..len).collect();
632 }
633 points
634 .windows(2)
635 .filter_map(|w| {
636 let (start, end) = (w[0], w[1]);
637 if start < end {
638 Some(start..end)
639 } else {
640 None
641 }
642 })
643 .collect()
644}
645
646#[cfg(test)]
647mod tests {
648 use super::*;
649
650 /// Records start at every multiple of 4, including 0.
651 struct FixedWidthSplitter;
652
653 impl Splitter for FixedWidthSplitter {
654 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
655 (from..bytes.len()).find(|&p| p % 4 == 0)
656 }
657
658 fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
659 4
660 }
661 }
662
663 #[test]
664 fn find_split_points_no_duplicate_leading_zero() {
665 // Input shorter than the planned chunk count: bytes.len() / n == 0
666 // puts every nominal offset at 0, so without the guard the prepended
667 // 0 duplicates the candidate at 0.
668 let bytes = vec![0u8; 8];
669 let splitter = FixedWidthSplitter;
670 for max_chunks in [2, 8, 64, 1024] {
671 let points = splitter.find_split_points(&bytes, max_chunks);
672 assert_eq!(points.first(), Some(&0), "must start at 0");
673 assert_eq!(points.last(), Some(&bytes.len()), "must end at len");
674 assert!(
675 points.windows(2).all(|w| w[0] < w[1]),
676 "duplicate or unsorted points for max_chunks={max_chunks}: {points:?}"
677 );
678 }
679 }
680}
681
682#[cfg(test)]
683mod chunk_floor_tests {
684 use super::*;
685
686 /// A splitter that overrides the documented entry point with a sentinel.
687 struct FixedSplitter;
688 impl Splitter for FixedSplitter {
689 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
690 (from..bytes.len())
691 .find(|&p| bytes[p] == b'\n')
692 .map(|p| p + 1)
693 }
694 fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
695 4
696 }
697 fn find_split_points(&self, _bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
698 vec![0, 7]
699 }
700 }
701
702 #[test]
703 fn adapter_override_is_honored_at_the_default_floor() {
704 // Engines call find_split_points_with; at the default floor it must
705 // defer to an adapter's find_split_points override.
706 assert_eq!(
707 FixedSplitter.find_split_points_with(b"ignored", 8, MIN_CHUNK_BYTES),
708 vec![0, 7]
709 );
710 }
711
712 #[test]
713 fn a_lower_floor_yields_more_chunks() {
714 let bytes = 8 << 20; // 8 MiB
715 let default = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, MIN_CHUNK_BYTES);
716 let small = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, 256 * 1024);
717 assert!(small > default, "default={default} small={small}");
718 }
719}