rypipe_core/decoder.rs
1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8// ---------------------------------------------------------------------------
9// S4: Chunk-size floor
10// ---------------------------------------------------------------------------
11
12/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
13/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
14/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
15pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
16
17/// Maximum number of split chunks. Above this, scheduling overhead dominates.
18pub const MAX_SPLIT_CHUNKS: usize = 1024;
19
20/// Mode for chunk planning: parallel and streaming have different optima.
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub enum SplitMode {
23 /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
24 Parallel,
25 /// Streaming: peak at 2 MB chunks (measured with bounded memory).
26 Streaming,
27}
28
29/// Plan the number of chunks given input size, thread count, and mode.
30/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a 2 MiB floor.
31pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
32 let by_size = bytes / MIN_CHUNK_BYTES.max(1);
33 let cap = match mode {
34 SplitMode::Parallel => 16usize.saturating_mul(threads),
35 SplitMode::Streaming => 8usize.saturating_mul(threads),
36 };
37 by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
38}
39
40// ---------------------------------------------------------------------------
41// S3: SkipRegionFinder: byte ranges where a candidate boundary is invalid
42// ---------------------------------------------------------------------------
43
44/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
45/// a candidate record boundary must be rejected. The engine calls this
46/// during split-point discovery to avoid splitting inside comments or strings.
47pub trait SkipRegionFinder: Send + Sync {
48 /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
49 fn openers(&self) -> &[&'static [u8]];
50
51 /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
52 fn closer_for(&self, opener: &[u8]) -> &'static [u8];
53
54 /// Maximum backward scan window in bytes. Default 64 KiB: sufficient
55 /// for most XML/CSV comments and JSON string literals.
56 fn window(&self) -> usize {
57 64 * 1024
58 }
59}
60
61/// Check whether byte position `at` falls inside a skip region by scanning
62/// backward up to `finder.window()` bytes, looking for an unclosed opener.
63///
64/// Returns `false` immediately when the file contains none of the openers;
65/// check that once per chunk, not per candidate.
66pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
67 if at > bytes.len() {
68 return false;
69 }
70 let openers = finder.openers();
71 if openers.is_empty() {
72 return false;
73 }
74 let scan_start = at.saturating_sub(finder.window());
75 let scan = &bytes[scan_start..at];
76 for opener in openers {
77 let closer = finder.closer_for(opener);
78 // Use memchr::memmem::find directly (no Finder::new preprocessing).
79 let mut search = 0;
80 while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
81 let open_pos = scan_start + search + rel;
82 // Check if there's a closer between the opener and `at`.
83 let closer_start = open_pos + opener.len();
84 if closer_start >= at {
85 // Opener extends past `at`; treat as unclosed (inside skip region).
86 return true;
87 }
88 let between = &bytes[closer_start..at];
89 if memchr::memmem::find(between, closer).is_none() {
90 // Unclosed opener; `at` is inside this skip region.
91 return true;
92 }
93 search += rel + 1;
94 }
95 }
96 false
97}
98
99// ---------------------------------------------------------------------------
100// S1: Splitter trait with default
101// ---------------------------------------------------------------------------
102
103/// How records are separated in the input.
104#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
105pub enum RecordBoundary {
106 /// Split on every newline (default). Each `\n` is a record boundary.
107 #[default]
108 Line,
109 /// Split on blank lines. Records span multiple lines; a blank line
110 /// (`\n\n`) separates records.
111 BlankLine,
112}
113
114/// Format-specific splitter: decides where it is safe to divide an input byte
115/// stream into independent chunks.
116///
117/// The only required method is `next_record_start`; adapters provide the
118/// record boundary logic, and the engine handles chunk planning, skip-region
119/// rejection, and deduplication.
120///
121/// For multi-line formats, implement the declarative methods
122/// ([`record_boundary`](Self::record_boundary),
123/// [`continuation_char`](Self::continuation_char),
124/// [`comment_prefixes`](Self::comment_prefixes)) and call
125/// [`find_next_record_boundary`] from your `next_record_start`:
126///
127/// ```rust
128/// use rypipe_core::{find_next_record_boundary, RecordBoundary, Splitter};
129/// struct Lines;
130/// impl Splitter for Lines {
131/// fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
132/// find_next_record_boundary(bytes, from, self.continuation_char(), self.comment_prefixes(), self.record_boundary() == RecordBoundary::BlankLine)
133/// }
134/// fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize { sample.len().max(1) }
135/// }
136/// assert_eq!(Lines.next_record_start(b"one\ntwo\n", 0), Some(4));
137/// ```
138pub trait Splitter: Send + Sync {
139 /// Return the next record boundary at or after `from`. Must return a
140 /// position where a record starts, or `None` if no more records exist.
141 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
142
143 /// Estimate the average bytes per record from a sample of the input.
144 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
145
146 /// How records are separated. Default: [`RecordBoundary::Line`] (every
147 /// newline is a boundary). Override to [`RecordBoundary::BlankLine`] for
148 /// multi-line formats where blank lines separate records.
149 fn record_boundary(&self) -> RecordBoundary {
150 RecordBoundary::Line
151 }
152
153 /// Character that continues a record across lines (e.g. `b'\\'` for
154 /// properties files). When set, newlines preceded by this character are
155 /// NOT record boundaries. Default: `None` (no continuations).
156 fn continuation_char(&self) -> Option<u8> {
157 None
158 }
159
160 /// Line prefixes to skip (e.g. `&[b"#", b"!"]` for properties files).
161 /// Lines starting with any of these are ignored during splitting.
162 /// Default: empty (no comments).
163 fn comment_prefixes(&self) -> &[&[u8]] {
164 &[]
165 }
166
167 /// Optional: byte ranges where a candidate boundary must be rejected.
168 /// See `in_skip_region`.
169 fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
170 None
171 }
172
173 /// Find split points. **Do not override** without a measured reason.
174 ///
175 /// Returns a sorted list of byte offsets where it is safe to split the
176 /// input into independent chunks. The return value must satisfy:
177 /// - Sorted ascending, no duplicates.
178 /// - First element is `0`, last element is `bytes.len()`.
179 /// - All values are `<= bytes.len()`.
180 ///
181 /// Default implementation:
182 /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
183 /// 2. Scan each nominal offset with `next_record_start`.
184 /// 3. Reject candidates inside skip regions via `in_skip_region`.
185 /// 4. Dedup, sort, prepend 0.
186 /// 5. Apply chunk floor via `plan_chunk_count`.
187 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
188 if max_chunks <= 1 || bytes.is_empty() {
189 return vec![0, bytes.len()];
190 }
191 let n = plan_chunk_count(bytes.len(), max_chunks, SplitMode::Parallel);
192 let skip = self.skip_regions();
193 // Planning must not start a global worker pool for bounded readers.
194 let mut points: Vec<usize> = (1..n)
195 .filter_map(|i| {
196 let approx = bytes.len() / n * i;
197 let pos = self.next_record_start(bytes, approx)?;
198 // Reject candidates inside skip regions.
199 if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
200 return None;
201 }
202 Some(pos)
203 })
204 .collect();
205
206 points.sort_unstable();
207 points.dedup();
208 points.insert(0, 0);
209 if *points.last().unwrap_or(&0) != bytes.len() {
210 points.push(bytes.len());
211 }
212 points
213 }
214}
215
216// ---------------------------------------------------------------------------
217// Declarative splitting helpers
218// ---------------------------------------------------------------------------
219
220/// Check whether the newline at `pos` is preceded by a continuation character.
221///
222/// Used by Splitters that need to skip continued newlines. For example, in
223/// properties files, `key = value \` continues on the next line: the `\n`
224/// after `\` is NOT a record boundary.
225///
226/// ```rust
227/// use rypipe_core::is_continued_newline;
228/// assert!(is_continued_newline(b"key=value\\\nnext\n", 10));
229/// ```
230#[inline]
231pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
232 if pos == 0 || bytes.get(pos) != Some(&b'\n') {
233 return false;
234 }
235 let mut run = 0;
236 let mut cursor = pos;
237 if bytes[cursor - 1] == b'\r' {
238 cursor -= 1;
239 }
240 while cursor > 0 && bytes[cursor - 1] == b'\\' {
241 run += 1;
242 cursor -= 1;
243 }
244 run % 2 == 1
245}
246
247/// Scan forward from `from` to find the next record boundary.
248///
249/// This is the declarative counterpart to manual byte scanning. It handles:
250/// - **Continuations**: skips `\n` preceded by `continuation` (e.g. `b'\\'`)
251/// - **Comments**: skips lines starting with any prefix in `comment_prefixes`
252/// - **Blank-line mode**: splits after an empty LF or CRLF line
253///
254/// Returns the byte position after the boundary newline, or `None` if no
255/// more boundaries exist.
256///
257/// # Example
258///
259/// ```rust
260/// use rypipe_core::find_next_record_boundary;
261/// let input = b"# comment\nfirst\nsecond\n";
262/// assert_eq!(find_next_record_boundary(input, 0, None, &[b"#"], false), Some(16));
263/// ```
264#[inline]
265pub fn find_next_record_boundary(
266 bytes: &[u8],
267 from: usize,
268 continuation: Option<u8>,
269 comment_prefixes: &[&[u8]],
270 blank_line_mode: bool,
271) -> Option<usize> {
272 if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
273 return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
274 }
275 let mut pos = from;
276 while pos < bytes.len() {
277 let rel = memchr::memchr(b'\n', &bytes[pos..])?;
278 let nl_pos = pos + rel;
279
280 // Skip continued newlines (preceded by continuation character)
281 let continued = |newline: usize| {
282 continuation.is_some_and(|character| {
283 let end = if newline > 0 && bytes[newline - 1] == b'\r' {
284 newline - 1
285 } else {
286 newline
287 };
288 if character == b'\\' {
289 is_continued_newline(bytes, newline)
290 } else {
291 end > 0 && bytes[end - 1] == character
292 }
293 })
294 };
295 if continued(nl_pos) {
296 pos = nl_pos + 1;
297 continue;
298 }
299
300 if !blank_line_mode && comment_prefixes.is_empty() {
301 return Some(nl_pos + 1);
302 }
303
304 if blank_line_mode {
305 let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
306 if end == 0 || bytes[end - 1] != b'\n' {
307 pos = nl_pos + 1;
308 continue;
309 }
310 if comment_prefixes.is_empty() && !continued(end - 1) {
311 return Some(nl_pos + 1);
312 }
313 }
314
315 // Compute line start for comment check
316 let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
317
318 // Skip comment lines
319 if !comment_prefixes.is_empty() {
320 let line_bytes = &bytes[line_start..nl_pos];
321 let is_comment = comment_prefixes
322 .iter()
323 .any(|prefix| line_bytes.starts_with(prefix));
324 if is_comment {
325 pos = nl_pos + 1;
326 continue;
327 }
328 }
329
330 // Blank-line mode: split on empty lines
331 if blank_line_mode {
332 let line = &bytes[line_start..nl_pos];
333 if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
334 let previous_start =
335 memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
336 if !comment_prefixes
337 .iter()
338 .any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
339 {
340 return Some(nl_pos + 1);
341 }
342 }
343 pos = nl_pos + 1;
344 continue;
345 }
346
347 // Line mode: every newline is a boundary
348 return Some(nl_pos + 1);
349 }
350 None
351}
352
353/// Format-specific parser: turns a byte chunk into field/value events sent to
354/// a `ColumnarSink`.
355pub trait RecordParser: Send + Sync {
356 /// Validate that the whole byte slice is well-formed for this format.
357 fn validate(&self, bytes: &[u8]) -> Result<()>;
358
359 /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
360 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
361
362 /// Parse one chunk generically (monomorphized, inlinable).
363 ///
364 /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
365 /// have not yet migrated. Adapters that implement this directly get
366 /// devirtualized `sink` calls and cross-crate inlining.
367 #[inline]
368 fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
369 where
370 Self: Sized,
371 {
372 // Default impl delegates to the object-safe path. Adapters that override
373 // this method get devirtualized sink calls and cross-crate inlining.
374 self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
375 }
376}
377
378/// Sink for decoder events. The decoder calls `begin_row` / `put_field` /
379/// `end_row` for each record.
380///
381/// Implementations must be `Send + Sync` when used with the parallel or
382/// parallel_streaming executors. `TableBuilder` satisfies this.
383pub trait ColumnarSink {
384 /// Begin a new row. Must be paired with exactly one `end_row` per row.
385 fn begin_row(&mut self);
386 fn put_field(&mut self, name: &str, value: Value<'_>);
387
388 /// Push a complete row.
389 ///
390 /// The default preserves event-oriented semantics. Sinks may override this
391 /// to optimize row handling while preserving filtering and duplicate-field
392 /// behavior.
393 #[inline]
394 fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
395 for (name, value) in fields {
396 self.put_field(name, value.clone());
397 }
398 }
399
400 /// Resolve a valid raw name without allocating a UTF-8 `String`.
401 #[inline]
402 fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
403 std::str::from_utf8(raw_name)
404 .ok()
405 .and_then(|name| self.resolve(name))
406 }
407
408 /// Push a value for a field whose name is still in its raw byte form.
409 #[inline]
410 fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
411 if let Ok(name) = std::str::from_utf8(raw_name) {
412 self.resolve_and_put(name, value);
413 }
414 }
415
416 /// End the current row. Must be paired with exactly one `begin_row` per row.
417 fn end_row(&mut self);
418
419 /// Return `false` to signal that the engine will drop this field.
420 #[inline]
421 fn wants(&self, _name: &str) -> bool {
422 true
423 }
424
425 /// Resolve a raw field name to its output column name, or `None` if dropped.
426 /// Default keeps the name as-is. Adapters that do expensive extraction can
427 /// call this once and then `put_field_resolved` to avoid a second
428 /// `resolve_field` inside `put_field`.
429 #[inline]
430 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
431 Some(name)
432 }
433
434 /// Push a field that has already been resolved via `resolve`.
435 /// Default delegates to `put_field` (which will resolve again). Overrides
436 /// should bypass the rename/drop lookup.
437 #[inline]
438 fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
439 self.put_field(resolved_name, value);
440 }
441
442 /// Push a value directly to a known slot index, bypassing all name
443 /// resolution, hash lookups, and attribute scanning. Used by adapters
444 /// that have verified the field identity via `expect_slot` + memcmp.
445 ///
446 /// Default: discards the value. Engines MUST override this to push
447 /// directly to columns by slot index; the default silently drops data.
448 #[inline]
449 fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
450 let _ = value;
451 }
452
453 /// Resolve a field name and push it in one call, avoiding the double
454 /// `resolve_field` that `wants` + `put_field` would perform.
455 /// Default: resolve then put_field_resolved (with owned clone to satisfy
456 /// the borrow checker). Overrides should avoid the allocation.
457 #[inline]
458 fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
459 if let Some(resolved) = self.resolve(name) {
460 let owned = resolved.to_owned();
461 self.put_field_resolved(&owned, value);
462 }
463 }
464
465 /// Whether this sink needs decoded values from the parser.
466 ///
467 /// Return `false` for locate-only sinks that walk rows and resolve field
468 /// names but skip value extraction entirely. The parser can use this to
469 /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
470 #[inline]
471 fn needs_value(&self) -> bool {
472 true
473 }
474
475 /// Whether this sink needs field-name resolution (`wants` + `resolve`).
476 ///
477 /// Return `false` for traversal-only sinks that walk rows and find field
478 /// extents but don't need to resolve names or check `wants`. When false,
479 /// the parser skips `wants()` and `resolve()` calls entirely; it only
480 /// locates the byte extents of each field within a row.
481 ///
482 /// **Note:** this method is primarily for benchmarking/profiling harnesses.
483 /// A sink returning `false` may receive `put_field` calls with unresolved
484 /// names, or none at all, depending on the adapter. Consider this
485 /// experimental until a non-benchmark consumer exists.
486 #[doc(hidden)]
487 #[inline]
488 fn needs_resolve(&self) -> bool {
489 true
490 }
491
492 /// Whether the current row has been rejected by the filter and the scanner
493 /// should skip the remainder of the row. Used for predicate-first
494 /// deferred materialization: once `Fail`, the scanner can byte-jump to
495 /// `</Details>` without further field scanning. Default false.
496 #[inline]
497 fn row_rejected(&self) -> bool {
498 false
499 }
500
501 /// Whether every wanted column for the current row already has a value.
502 /// The adapter can byte-jump to the row close tag when this returns true,
503 /// skipping remaining fields. Composes with `row_rejected` (a row can
504 /// be satisfied OR rejected, both short-circuit).
505 ///
506 /// Default: false (never short-circuit).
507 #[inline]
508 fn row_satisfied(&self) -> bool {
509 false
510 }
511
512 /// Bitmask of columns that the projection wants. Bit `i` is set iff
513 /// column `i` is in the output schema. The adapter can test membership
514 /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
515 /// field. Returns empty mask (all zeros) when no projection is active.
516 #[inline]
517 fn wanted_mask(&self) -> u64 {
518 0
519 }
520
521 /// After the layout is learned, the expected (slot, raw name bytes) at
522 /// this ordinal position in the record. The adapter can compare the
523 /// raw bytes in-place (one memcmp) instead of running the full
524 /// attribute scan → UTF-8 decode → hash → lookup path.
525 ///
526 /// Return `None` to fall back to the generic path. Default: `None`.
527 #[inline]
528 fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
529 None
530 }
531
532 /// Adapter reports the slot it resolved for `ordinal` so the engine
533 /// can cache it for subsequent rows. Called after the adapter resolves
534 /// a field via the generic path (wants + resolve + put_field_resolved).
535 #[inline]
536 fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
537
538 /// Adapter reports that the layout expected at `ordinal` did not match.
539 /// The engine should invalidate its cached expectation and fall back to
540 /// generic resolution for subsequent rows.
541 #[inline]
542 fn layout_broken(&mut self, _ordinal: u32) {}
543
544 /// Reset the ordinal counter for child-element tracking. Called after
545 /// row-tag attributes are emitted so child ordinals start at 0.
546 #[inline]
547 fn reset_child_ordinal(&mut self) {}
548
549 /// Finalize the sink into an Arrow `RecordBatch`.
550 fn finish(&mut self) -> Result<RecordBatch>;
551}
552
553/// Convert split-point offsets returned by a `Splitter` into non-empty
554/// `Range<usize>` chunks.
555pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
556 if points.len() < 2 {
557 return std::iter::once(0..len).collect();
558 }
559 points
560 .windows(2)
561 .filter_map(|w| {
562 let (start, end) = (w[0], w[1]);
563 if start < end {
564 Some(start..end)
565 } else {
566 None
567 }
568 })
569 .collect()
570}