rypipe_core/decoder.rs
1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8// ---------------------------------------------------------------------------
9// Chunk-size floor
10// ---------------------------------------------------------------------------
11
12/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
13/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
14/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
15pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
16
17/// Maximum number of split chunks. Above this, scheduling overhead dominates.
18pub const MAX_SPLIT_CHUNKS: usize = 1024;
19
20/// Mode for chunk planning: parallel and streaming have different optima.
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub enum SplitMode {
23 /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
24 Parallel,
25 /// Streaming: peak at 2 MB chunks (measured with bounded memory).
26 Streaming,
27}
28
29/// Plan the number of chunks given input size, thread count, and mode.
30/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a floor of
31/// `min_chunk_bytes` (default [`MIN_CHUNK_BYTES`] via [`plan_chunk_count`]).
32pub fn plan_chunk_count_with(
33 bytes: usize,
34 threads: usize,
35 mode: SplitMode,
36 min_chunk_bytes: usize,
37) -> usize {
38 let by_size = bytes / min_chunk_bytes.max(1);
39 let cap = match mode {
40 SplitMode::Parallel => 16usize.saturating_mul(threads),
41 SplitMode::Streaming => 8usize.saturating_mul(threads),
42 };
43 by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
44}
45
46/// [`plan_chunk_count_with`] using the default [`MIN_CHUNK_BYTES`] floor.
47pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
48 plan_chunk_count_with(bytes, threads, mode, MIN_CHUNK_BYTES)
49}
50
51// ---------------------------------------------------------------------------
52// SkipRegionFinder: byte ranges where a candidate boundary is invalid
53// ---------------------------------------------------------------------------
54
55/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
56/// a candidate record boundary must be rejected. The engine calls this
57/// during split-point discovery to avoid splitting inside comments or strings.
58pub trait SkipRegionFinder: Send + Sync {
59 /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
60 fn openers(&self) -> &[&'static [u8]];
61
62 /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
63 fn closer_for(&self, opener: &[u8]) -> &'static [u8];
64
65 /// Maximum backward scan window in bytes. Default 64 KiB: sufficient
66 /// for most XML/CSV comments and JSON string literals.
67 fn window(&self) -> usize {
68 64 * 1024
69 }
70}
71
72/// Check whether byte position `at` falls inside a skip region by scanning
73/// backward up to `finder.window()` bytes, looking for an unclosed opener.
74///
75/// Returns `false` immediately when the file contains none of the openers;
76/// check that once per chunk, not per candidate.
77pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
78 if at > bytes.len() {
79 return false;
80 }
81 let openers = finder.openers();
82 if openers.is_empty() {
83 return false;
84 }
85 let scan_start = at.saturating_sub(finder.window());
86 let scan = &bytes[scan_start..at];
87 for opener in openers {
88 let closer = finder.closer_for(opener);
89 // Use memchr::memmem::find directly (no Finder::new preprocessing).
90 let mut search = 0;
91 while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
92 let open_pos = scan_start + search + rel;
93 // Check if there's a closer between the opener and `at`.
94 let closer_start = open_pos + opener.len();
95 if closer_start >= at {
96 // Opener extends past `at`; treat as unclosed (inside skip region).
97 return true;
98 }
99 let between = &bytes[closer_start..at];
100 if memchr::memmem::find(between, closer).is_none() {
101 // Unclosed opener; `at` is inside this skip region.
102 return true;
103 }
104 search += rel + 1;
105 }
106 }
107 false
108}
109
110// ---------------------------------------------------------------------------
111// Splitter trait with default
112// ---------------------------------------------------------------------------
113
114/// How records are separated in the input.
115#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
116pub enum RecordBoundary {
117 /// Split on every newline (default). Each `\n` is a record boundary.
118 #[default]
119 Line,
120 /// Split on blank lines. Records span multiple lines; a blank line
121 /// (`\n\n`) separates records.
122 BlankLine,
123}
124
125/// Format-specific splitter: decides where it is safe to divide an input byte
126/// stream into independent chunks.
127///
128/// The only required method is `next_record_start`; adapters provide the
129/// record boundary logic, and the engine handles chunk planning, skip-region
130/// rejection, and deduplication.
131///
132/// For multi-line formats, implement the declarative methods
133/// ([`record_boundary`](Self::record_boundary),
134/// [`continuation_char`](Self::continuation_char),
135/// [`comment_prefixes`](Self::comment_prefixes)) and call
136/// [`find_next_record_boundary`] from your `next_record_start`:
137///
138/// ```rust
139/// use rypipe_core::{find_next_record_boundary, RecordBoundary, Splitter};
140/// struct Lines;
141/// impl Splitter for Lines {
142/// fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
143/// find_next_record_boundary(bytes, from, self.continuation_char(), self.comment_prefixes(), self.record_boundary() == RecordBoundary::BlankLine)
144/// }
145/// fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize { sample.len().max(1) }
146/// }
147/// assert_eq!(Lines.next_record_start(b"one\ntwo\n", 0), Some(4));
148/// ```
149pub trait Splitter: Send + Sync {
150 /// Return the next record boundary at or after `from`. Must return a
151 /// position where a record starts, or `None` if no more records exist.
152 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
153
154 /// Estimate the average bytes per record from a sample of the input.
155 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
156
157 /// How records are separated. Default: [`RecordBoundary::Line`] (every
158 /// newline is a boundary). Override to [`RecordBoundary::BlankLine`] for
159 /// multi-line formats where blank lines separate records.
160 fn record_boundary(&self) -> RecordBoundary {
161 RecordBoundary::Line
162 }
163
164 /// Character that continues a record across lines (e.g. `b'\\'` for
165 /// properties files). When set, newlines preceded by this character are
166 /// NOT record boundaries. Default: `None` (no continuations).
167 fn continuation_char(&self) -> Option<u8> {
168 None
169 }
170
171 /// Line prefixes to skip (e.g. `&[b"#", b"!"]` for properties files).
172 /// Lines starting with any of these are ignored during splitting.
173 /// Default: empty (no comments).
174 fn comment_prefixes(&self) -> &[&[u8]] {
175 &[]
176 }
177
178 /// Optional: byte ranges where a candidate boundary must be rejected.
179 /// See `in_skip_region`.
180 fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
181 None
182 }
183
184 /// Find split points. **Do not override** without a measured reason.
185 ///
186 /// Returns a sorted list of byte offsets where it is safe to split the
187 /// input into independent chunks. The return value must satisfy:
188 /// - Sorted ascending, no duplicates.
189 /// - First element is `0`, last element is `bytes.len()`.
190 /// - All values are `<= bytes.len()`.
191 ///
192 /// Default implementation:
193 /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
194 /// 2. Scan each nominal offset with `next_record_start`.
195 /// 3. Reject candidates inside skip regions via `in_skip_region`.
196 /// 4. Dedup, sort, prepend 0.
197 /// 5. Apply chunk floor via `plan_chunk_count`.
198 ///
199 /// This is the default-floor entry point. Adapters that override it keep
200 /// working; engines call [`Splitter::find_split_points_with`] instead, and
201 /// that defers to this override at the default floor.
202 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
203 default_find_split_points(self, bytes, max_chunks, MIN_CHUNK_BYTES)
204 }
205
206 /// Find split points with an explicit chunk-size floor.
207 ///
208 /// The floor is opt-in: lowering it yields more, smaller chunks (better
209 /// load balancing on many or small cores, at more scheduling overhead);
210 /// raising it yields fewer, larger chunks. Default is [`MIN_CHUNK_BYTES`].
211 ///
212 /// At the default floor this calls [`Splitter::find_split_points`], so an
213 /// adapter's override is honored; a non-default floor uses the engine's
214 /// default planning (override `find_split_points_with` to customize it).
215 fn find_split_points_with(
216 &self,
217 bytes: &[u8],
218 max_chunks: usize,
219 min_chunk_bytes: usize,
220 ) -> Vec<usize> {
221 if min_chunk_bytes == MIN_CHUNK_BYTES {
222 return self.find_split_points(bytes, max_chunks);
223 }
224 default_find_split_points(self, bytes, max_chunks, min_chunk_bytes)
225 }
226}
227
228/// The engine's default chunk planning, shared by the `Splitter` methods.
229fn default_find_split_points<S: Splitter + ?Sized>(
230 splitter: &S,
231 bytes: &[u8],
232 max_chunks: usize,
233 min_chunk_bytes: usize,
234) -> Vec<usize> {
235 if max_chunks <= 1 || bytes.is_empty() {
236 return vec![0, bytes.len()];
237 }
238 let n = plan_chunk_count_with(
239 bytes.len(),
240 max_chunks,
241 SplitMode::Parallel,
242 min_chunk_bytes,
243 );
244 let skip = splitter.skip_regions();
245 // Planning must not start a global worker pool for bounded readers.
246 let mut points: Vec<usize> = (1..n)
247 .filter_map(|i| {
248 let approx = bytes.len() / n * i;
249 let pos = splitter.next_record_start(bytes, approx)?;
250 // Reject candidates inside skip regions.
251 if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
252 return None;
253 }
254 Some(pos)
255 })
256 .collect();
257
258 points.sort_unstable();
259 points.dedup();
260 // Drop candidates at 0: the prepend below already supplies the
261 // leading split point, and keeping them would duplicate it (this
262 // happens when bytes.len() < n, where bytes.len() / n == 0 puts
263 // every nominal offset at 0).
264 points.retain(|&p| p > 0);
265 points.insert(0, 0);
266 if *points.last().unwrap_or(&0) != bytes.len() {
267 points.push(bytes.len());
268 }
269 points
270}
271
272// ---------------------------------------------------------------------------
273// Declarative splitting helpers
274// ---------------------------------------------------------------------------
275
276/// Check whether the newline at `pos` is preceded by a continuation character.
277///
278/// Used by Splitters that need to skip continued newlines. For example, in
279/// properties files, `key = value \` continues on the next line: the `\n`
280/// after `\` is NOT a record boundary.
281///
282/// ```rust
283/// use rypipe_core::is_continued_newline;
284/// assert!(is_continued_newline(b"key=value\\\nnext\n", 10));
285/// ```
286#[inline]
287pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
288 if pos == 0 || bytes.get(pos) != Some(&b'\n') {
289 return false;
290 }
291 let mut run = 0;
292 let mut cursor = pos;
293 if bytes[cursor - 1] == b'\r' {
294 cursor -= 1;
295 }
296 while cursor > 0 && bytes[cursor - 1] == b'\\' {
297 run += 1;
298 cursor -= 1;
299 }
300 run % 2 == 1
301}
302
303/// Scan forward from `from` to find the next record boundary.
304///
305/// This is the declarative counterpart to manual byte scanning. It handles:
306/// - **Continuations**: skips `\n` preceded by `continuation` (e.g. `b'\\'`)
307/// - **Comments**: skips lines starting with any prefix in `comment_prefixes`
308/// - **Blank-line mode**: splits after an empty LF or CRLF line
309///
310/// Returns the byte position after the boundary newline, or `None` if no
311/// more boundaries exist.
312///
313/// # Example
314///
315/// ```rust
316/// use rypipe_core::find_next_record_boundary;
317/// let input = b"# comment\nfirst\nsecond\n";
318/// assert_eq!(find_next_record_boundary(input, 0, None, &[b"#"], false), Some(16));
319/// ```
320#[inline]
321pub fn find_next_record_boundary(
322 bytes: &[u8],
323 from: usize,
324 continuation: Option<u8>,
325 comment_prefixes: &[&[u8]],
326 blank_line_mode: bool,
327) -> Option<usize> {
328 if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
329 return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
330 }
331 let mut pos = from;
332 while pos < bytes.len() {
333 let rel = memchr::memchr(b'\n', &bytes[pos..])?;
334 let nl_pos = pos + rel;
335
336 // Skip continued newlines (preceded by continuation character)
337 let continued = |newline: usize| {
338 continuation.is_some_and(|character| {
339 let end = if newline > 0 && bytes[newline - 1] == b'\r' {
340 newline - 1
341 } else {
342 newline
343 };
344 if character == b'\\' {
345 is_continued_newline(bytes, newline)
346 } else {
347 end > 0 && bytes[end - 1] == character
348 }
349 })
350 };
351 if continued(nl_pos) {
352 pos = nl_pos + 1;
353 continue;
354 }
355
356 if !blank_line_mode && comment_prefixes.is_empty() {
357 return Some(nl_pos + 1);
358 }
359
360 if blank_line_mode {
361 let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
362 if end == 0 || bytes[end - 1] != b'\n' {
363 pos = nl_pos + 1;
364 continue;
365 }
366 if comment_prefixes.is_empty() && !continued(end - 1) {
367 return Some(nl_pos + 1);
368 }
369 }
370
371 // Compute line start for comment check
372 let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
373
374 // Skip comment lines
375 if !comment_prefixes.is_empty() {
376 let line_bytes = &bytes[line_start..nl_pos];
377 let is_comment = comment_prefixes
378 .iter()
379 .any(|prefix| line_bytes.starts_with(prefix));
380 if is_comment {
381 pos = nl_pos + 1;
382 continue;
383 }
384 }
385
386 // Blank-line mode: split on empty lines
387 if blank_line_mode {
388 let line = &bytes[line_start..nl_pos];
389 if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
390 let previous_start =
391 memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
392 if !comment_prefixes
393 .iter()
394 .any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
395 {
396 return Some(nl_pos + 1);
397 }
398 }
399 pos = nl_pos + 1;
400 continue;
401 }
402
403 // Line mode: every newline is a boundary
404 return Some(nl_pos + 1);
405 }
406 None
407}
408
409/// Format-specific parser: turns a byte chunk into field/value events sent to
410/// a `ColumnarSink`.
411pub trait RecordParser: Send + Sync {
412 /// Validate that the whole byte slice is well-formed for this format.
413 fn validate(&self, bytes: &[u8]) -> Result<()>;
414
415 /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
416 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
417
418 /// Parse one chunk generically (monomorphized, inlinable).
419 ///
420 /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
421 /// have not yet migrated. Adapters that implement this directly get
422 /// devirtualized `sink` calls and cross-crate inlining.
423 #[inline]
424 fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
425 where
426 Self: Sized,
427 {
428 // Default impl delegates to the object-safe path. Adapters that override
429 // this method get devirtualized sink calls and cross-crate inlining.
430 self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
431 }
432}
433
434/// Sink for decoder events. The decoder calls `begin_row` / `put_field` /
435/// `end_row` for each record.
436///
437/// Implementations must be `Send + Sync` when used with the parallel or
438/// parallel_streaming executors. `TableBuilder` satisfies this.
439pub trait ColumnarSink {
440 /// Begin a new row. Must be paired with exactly one `end_row` per row.
441 fn begin_row(&mut self);
442 fn put_field(&mut self, name: &str, value: Value<'_>);
443
444 /// Push a complete row.
445 ///
446 /// The default preserves event-oriented semantics. Sinks may override this
447 /// to optimize row handling while preserving filtering and duplicate-field
448 /// behavior.
449 #[inline]
450 fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
451 for (name, value) in fields {
452 self.put_field(name, value.clone());
453 }
454 }
455
456 /// Resolve a valid raw name without allocating a UTF-8 `String`.
457 #[inline]
458 fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
459 std::str::from_utf8(raw_name)
460 .ok()
461 .and_then(|name| self.resolve(name))
462 }
463
464 /// Push a value for a field whose name is still in its raw byte form.
465 #[inline]
466 fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
467 if let Ok(name) = std::str::from_utf8(raw_name) {
468 self.resolve_and_put(name, value);
469 }
470 }
471
472 /// End the current row. Must be paired with exactly one `begin_row` per row.
473 fn end_row(&mut self);
474
475 /// Return `false` to signal that the engine will drop this field.
476 #[inline]
477 fn wants(&self, _name: &str) -> bool {
478 true
479 }
480
481 /// Resolve a raw field name to its output column name, or `None` if dropped.
482 /// Default keeps the name as-is. Adapters that do expensive extraction can
483 /// call this once and then `put_field_resolved` to avoid a second
484 /// `resolve_field` inside `put_field`.
485 #[inline]
486 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
487 Some(name)
488 }
489
490 /// Push a field that has already been resolved via `resolve`.
491 /// Default delegates to `put_field` (which will resolve again). Overrides
492 /// should bypass the rename/drop lookup.
493 #[inline]
494 fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
495 self.put_field(resolved_name, value);
496 }
497
498 /// Push a value directly to a known slot index, bypassing all name
499 /// resolution, hash lookups, and attribute scanning. Used by adapters
500 /// that have verified the field identity via `expect_slot` + memcmp.
501 ///
502 /// Default: discards the value. Engines MUST override this to push
503 /// directly to columns by slot index; the default silently drops data.
504 #[inline]
505 fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
506 let _ = value;
507 }
508
509 /// Resolve a field name and push it in one call, avoiding the double
510 /// `resolve_field` that `wants` + `put_field` would perform.
511 /// Default: resolve then put_field_resolved (with owned clone to satisfy
512 /// the borrow checker). Overrides should avoid the allocation.
513 #[inline]
514 fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
515 if let Some(resolved) = self.resolve(name) {
516 let owned = resolved.to_owned();
517 self.put_field_resolved(&owned, value);
518 }
519 }
520
521 /// Whether this sink needs decoded values from the parser.
522 ///
523 /// Return `false` for locate-only sinks that walk rows and resolve field
524 /// names but skip value extraction entirely. The parser can use this to
525 /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
526 #[inline]
527 fn needs_value(&self) -> bool {
528 true
529 }
530
531 /// Whether this sink needs field-name resolution (`wants` + `resolve`).
532 ///
533 /// Return `false` for traversal-only sinks that walk rows and find field
534 /// extents but don't need to resolve names or check `wants`. When false,
535 /// the parser skips `wants()` and `resolve()` calls entirely; it only
536 /// locates the byte extents of each field within a row.
537 ///
538 /// **Note:** this method is primarily for benchmarking/profiling harnesses.
539 /// A sink returning `false` may receive `put_field` calls with unresolved
540 /// names, or none at all, depending on the adapter. Consider this
541 /// experimental until a non-benchmark consumer exists.
542 #[doc(hidden)]
543 #[inline]
544 fn needs_resolve(&self) -> bool {
545 true
546 }
547
548 /// Whether the current row has been rejected by the filter and the scanner
549 /// should skip the remainder of the row. Used for predicate-first
550 /// deferred materialization: once `Fail`, the scanner can byte-jump to
551 /// `</Details>` without further field scanning. Default false.
552 #[inline]
553 fn row_rejected(&self) -> bool {
554 false
555 }
556
557 /// Whether every wanted column for the current row already has a value.
558 /// The adapter can byte-jump to the row close tag when this returns true,
559 /// skipping remaining fields. Composes with `row_rejected` (a row can
560 /// be satisfied OR rejected, both short-circuit).
561 ///
562 /// Default: false (never short-circuit).
563 #[inline]
564 fn row_satisfied(&self) -> bool {
565 false
566 }
567
568 /// Bitmask of columns that the projection wants. Bit `i` is set iff
569 /// column `i` is in the output schema. The adapter can test membership
570 /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
571 /// field. Returns empty mask (all zeros) when no projection is active.
572 #[inline]
573 fn wanted_mask(&self) -> u64 {
574 0
575 }
576
577 /// After the layout is learned, the expected (slot, raw name bytes) at
578 /// this ordinal position in the record. The adapter can compare the
579 /// raw bytes in-place (one memcmp) instead of running the full
580 /// attribute scan → UTF-8 decode → hash → lookup path.
581 ///
582 /// Return `None` to fall back to the generic path. Default: `None`.
583 #[inline]
584 fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
585 None
586 }
587
588 /// Adapter reports the slot it resolved for `ordinal` so the engine
589 /// can cache it for subsequent rows. Called after the adapter resolves
590 /// a field via the generic path (wants + resolve + put_field_resolved).
591 #[inline]
592 fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
593
594 /// Adapter reports that the layout expected at `ordinal` did not match.
595 /// The engine should invalidate its cached expectation and fall back to
596 /// generic resolution for subsequent rows.
597 #[inline]
598 fn layout_broken(&mut self, _ordinal: u32) {}
599
600 /// Reset the ordinal counter for child-element tracking. Called after
601 /// row-tag attributes are emitted so child ordinals start at 0.
602 #[inline]
603 fn reset_child_ordinal(&mut self) {}
604
605 /// Finalize the sink into an Arrow `RecordBatch`.
606 fn finish(&mut self) -> Result<RecordBatch>;
607}
608
609/// Convert split-point offsets returned by a `Splitter` into non-empty
610/// `Range<usize>` chunks.
611pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
612 if points.len() < 2 {
613 return std::iter::once(0..len).collect();
614 }
615 points
616 .windows(2)
617 .filter_map(|w| {
618 let (start, end) = (w[0], w[1]);
619 if start < end {
620 Some(start..end)
621 } else {
622 None
623 }
624 })
625 .collect()
626}
627
628#[cfg(test)]
629mod tests {
630 use super::*;
631
632 /// Records start at every multiple of 4, including 0.
633 struct FixedWidthSplitter;
634
635 impl Splitter for FixedWidthSplitter {
636 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
637 (from..bytes.len()).find(|&p| p % 4 == 0)
638 }
639
640 fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
641 4
642 }
643 }
644
645 #[test]
646 fn find_split_points_no_duplicate_leading_zero() {
647 // Input shorter than the planned chunk count: bytes.len() / n == 0
648 // puts every nominal offset at 0, so without the guard the prepended
649 // 0 duplicates the candidate at 0.
650 let bytes = vec![0u8; 8];
651 let splitter = FixedWidthSplitter;
652 for max_chunks in [2, 8, 64, 1024] {
653 let points = splitter.find_split_points(&bytes, max_chunks);
654 assert_eq!(points.first(), Some(&0), "must start at 0");
655 assert_eq!(points.last(), Some(&bytes.len()), "must end at len");
656 assert!(
657 points.windows(2).all(|w| w[0] < w[1]),
658 "duplicate or unsorted points for max_chunks={max_chunks}: {points:?}"
659 );
660 }
661 }
662}
663
664#[cfg(test)]
665mod chunk_floor_tests {
666 use super::*;
667
668 /// A splitter that overrides the documented entry point with a sentinel.
669 struct FixedSplitter;
670 impl Splitter for FixedSplitter {
671 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
672 (from..bytes.len())
673 .find(|&p| bytes[p] == b'\n')
674 .map(|p| p + 1)
675 }
676 fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
677 4
678 }
679 fn find_split_points(&self, _bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
680 vec![0, 7]
681 }
682 }
683
684 #[test]
685 fn adapter_override_is_honored_at_the_default_floor() {
686 // Engines call find_split_points_with; at the default floor it must
687 // defer to an adapter's find_split_points override.
688 assert_eq!(
689 FixedSplitter.find_split_points_with(b"ignored", 8, MIN_CHUNK_BYTES),
690 vec![0, 7]
691 );
692 }
693
694 #[test]
695 fn a_lower_floor_yields_more_chunks() {
696 let bytes = 8 << 20; // 8 MiB
697 let default = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, MIN_CHUNK_BYTES);
698 let small = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, 256 * 1024);
699 assert!(small > default, "default={default} small={small}");
700 }
701}