Skip to main content

rypipe_core/
decoder.rs

1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8// ---------------------------------------------------------------------------
9// S4: Chunk-size floor
10// ---------------------------------------------------------------------------
11
12/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
13/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
14/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
15pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
16
17/// Maximum number of split chunks. Above this, scheduling overhead dominates.
18pub const MAX_SPLIT_CHUNKS: usize = 1024;
19
20/// Mode for chunk planning: parallel and streaming have different optima.
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub enum SplitMode {
23    /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
24    Parallel,
25    /// Streaming: peak at 2 MB chunks (measured with bounded memory).
26    Streaming,
27}
28
29/// Plan the number of chunks given input size, thread count, and mode.
30/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a 2 MiB floor.
31pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
32    let by_size = bytes / MIN_CHUNK_BYTES.max(1);
33    let cap = match mode {
34        SplitMode::Parallel => 16 * threads,
35        SplitMode::Streaming => 8 * threads,
36    };
37    by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
38}
39
40// ---------------------------------------------------------------------------
41// S3: SkipRegionFinder: byte ranges where a candidate boundary is invalid
42// ---------------------------------------------------------------------------
43
44/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
45/// a candidate record boundary must be rejected.  The engine calls this
46/// during split-point discovery to avoid splitting inside comments or strings.
47pub trait SkipRegionFinder: Send + Sync {
48    /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
49    fn openers(&self) -> &[&'static [u8]];
50
51    /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
52    fn closer_for(&self, opener: &[u8]) -> &'static [u8];
53
54    /// Maximum backward scan window in bytes.  Default 64 KiB: sufficient
55    /// for most XML/CSV comments and JSON string literals.
56    fn window(&self) -> usize {
57        64 * 1024
58    }
59}
60
61/// Check whether byte position `at` falls inside a skip region by scanning
62/// backward up to `finder.window()` bytes, looking for an unclosed opener.
63///
64/// Returns `false` immediately when the file contains none of the openers;
65/// check that once per chunk, not per candidate.
66pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
67    let openers = finder.openers();
68    if openers.is_empty() {
69        return false;
70    }
71    let scan_start = at.saturating_sub(finder.window());
72    let scan = &bytes[scan_start..at];
73    for opener in openers {
74        let closer = finder.closer_for(opener);
75        // Use memchr::memmem::find directly (no Finder::new preprocessing).
76        let mut search = 0;
77        while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
78            let open_pos = scan_start + search + rel;
79            // Check if there's a closer between the opener and `at`.
80            let between = &bytes[open_pos + opener.len()..at];
81            if memchr::memmem::find(between, closer).is_none() {
82                // Unclosed opener; `at` is inside this skip region.
83                return true;
84            }
85            search += rel + 1;
86        }
87    }
88    false
89}
90
91// ---------------------------------------------------------------------------
92// S1: Splitter trait with default
93// ---------------------------------------------------------------------------
94
95/// Format-specific splitter: decides where it is safe to divide an input byte
96/// stream into independent chunks.
97///
98/// The only required method is `next_record_start`; adapters provide the
99/// record boundary logic, and the engine handles chunk planning, skip-region
100/// rejection, and deduplication.
101pub trait Splitter: Send + Sync {
102    /// Return the next record boundary at or after `from`.  Must return a
103    /// position where a record starts, or `None` if no more records exist.
104    fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
105
106    /// Estimate the average bytes per record from a sample of the input.
107    fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
108
109    /// Optional: byte ranges where a candidate boundary must be rejected.
110    /// See `in_skip_region`.
111    fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
112        None
113    }
114
115    /// Find split points.  **Do not override** without a measured reason.
116    ///
117    /// Default implementation:
118    /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
119    /// 2. `par_iter` over nominals (rayon), each calling `next_record_start`.
120    /// 3. Reject candidates inside skip regions via `in_skip_region`.
121    /// 4. Dedup, sort, prepend 0.
122    /// 5. Apply chunk floor via `plan_chunk_count`.
123    fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
124        if max_chunks <= 1 || bytes.is_empty() {
125            return vec![0, bytes.len()];
126        }
127        let n = plan_chunk_count(bytes.len(), max_chunks, SplitMode::Parallel);
128        let nominals: Vec<usize> = (1..n).map(|i| bytes.len() * i / n).collect();
129
130        use rayon::prelude::*;
131
132        let skip = self.skip_regions();
133        let has_skip = skip.is_some();
134
135        let mut points: Vec<usize> = nominals
136            .par_iter()
137            .filter_map(|&approx| {
138                let pos = self.next_record_start(bytes, approx)?;
139                // Reject candidates inside skip regions.
140                if has_skip && in_skip_region(bytes, pos, skip.unwrap()) {
141                    return None;
142                }
143                Some(pos)
144            })
145            .collect();
146
147        points.sort_unstable();
148        points.dedup();
149        points.insert(0, 0);
150        if *points.last().unwrap_or(&0) != bytes.len() {
151            points.push(bytes.len());
152        }
153        points
154    }
155}
156
157/// Format-specific parser: turns a byte chunk into field/value events sent to
158/// a `ColumnarSink`.
159pub trait RecordParser: Send + Sync {
160    /// Validate that the whole byte slice is well-formed for this format.
161    fn validate(&self, bytes: &[u8]) -> Result<()>;
162
163    /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
164    fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
165
166    /// Parse one chunk generically (monomorphized, inlinable).
167    ///
168    /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
169    /// have not yet migrated. Adapters that implement this directly get
170    /// devirtualized `sink` calls and cross-crate inlining.
171    #[inline]
172    fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
173    where
174        Self: Sized,
175    {
176        // Default impl delegates to the object-safe path. Adapters that override
177        // this method get devirtualized sink calls and cross-crate inlining.
178        self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
179    }
180}
181
182/// Sink for decoder events.  The decoder calls `begin_row` / `put_field` /
183/// `end_row` for each record.
184pub trait ColumnarSink {
185    fn begin_row(&mut self);
186    fn put_field(&mut self, name: &str, value: Value<'_>);
187
188    /// Push a complete row.
189    ///
190    /// The default preserves event-oriented semantics. Sinks may override this
191    /// to optimize row handling while preserving filtering and duplicate-field
192    /// behavior.
193    #[inline]
194    fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
195        for (name, value) in fields {
196            self.put_field(name, value.clone());
197        }
198    }
199
200    /// Resolve a valid raw name without allocating a UTF-8 `String`.
201    #[inline]
202    fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
203        std::str::from_utf8(raw_name)
204            .ok()
205            .and_then(|name| self.resolve(name))
206    }
207
208    /// Push a value for a field whose name is still in its raw byte form.
209    #[inline]
210    fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
211        if let Ok(name) = std::str::from_utf8(raw_name) {
212            self.resolve_and_put(name, value);
213        }
214    }
215
216    fn end_row(&mut self);
217
218    /// Return `false` to signal that the engine will drop this field.
219    #[inline]
220    fn wants(&self, _name: &str) -> bool {
221        true
222    }
223
224    /// Resolve a raw field name to its output column name, or `None` if dropped.
225    /// Default keeps the name as-is. Adapters that do expensive extraction can
226    /// call this once and then `put_field_resolved` to avoid a second
227    /// `resolve_field` inside `put_field`.
228    #[inline]
229    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
230        Some(name)
231    }
232
233    /// Push a field that has already been resolved via `resolve`.
234    /// Default delegates to `put_field` (which will resolve again). Overrides
235    /// should bypass the rename/drop lookup.
236    #[inline]
237    fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
238        self.put_field(resolved_name, value);
239    }
240
241    /// Push a value directly to a known slot index, bypassing all name
242    /// resolution, hash lookups, and attribute scanning.  Used by adapters
243    /// that have verified the field identity via `expect_slot` + memcmp.
244    ///
245    /// Default: delegates to `put_field` with a synthetic name (slow path).
246    /// Engines that track slot indices should override this.
247    #[inline]
248    fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
249        // Fallback: can't push without a name; discard.
250        // Engines override this to push directly to columns[slot].
251        let _ = value;
252    }
253
254    /// Resolve a field name and push it in one call, avoiding the double
255    /// `resolve_field` that `wants` + `put_field` would perform.
256    /// Default: resolve then put_field_resolved (with owned clone to satisfy
257    /// the borrow checker). Overrides should avoid the allocation.
258    #[inline]
259    fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
260        if let Some(resolved) = self.resolve(name) {
261            let owned = resolved.to_owned();
262            self.put_field_resolved(&owned, value);
263        }
264    }
265
266    /// Whether this sink needs decoded values from the parser.
267    ///
268    /// Return `false` for locate-only sinks that walk rows and resolve field
269    /// names but skip value extraction entirely.  The parser can use this to
270    /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
271    #[inline]
272    fn needs_value(&self) -> bool {
273        true
274    }
275
276    /// Whether this sink needs field-name resolution (`wants` + `resolve`).
277    ///
278    /// Return `false` for traversal-only sinks that walk rows and find field
279    /// extents but don't need to resolve names or check `wants`.  When false,
280    /// the parser skips `wants()` and `resolve()` calls entirely; it only
281    /// locates the byte extents of each field within a row.
282    ///
283    /// **Note:** this method is primarily for benchmarking/profiling harnesses.
284    /// A sink returning `false` may receive `put_field` calls with unresolved
285    /// names, or none at all, depending on the adapter.  Consider this
286    /// experimental until a non-benchmark consumer exists.
287    #[doc(hidden)]
288    #[inline]
289    fn needs_resolve(&self) -> bool {
290        true
291    }
292
293    /// Whether the current row has been rejected by the filter and the scanner
294    /// should skip the remainder of the row. Used for predicate-first
295    /// deferred materialization: once `Fail`, the scanner can byte-jump to
296    /// `</Details>` without further field scanning. Default false.
297    #[inline]
298    fn row_rejected(&self) -> bool {
299        false
300    }
301
302    /// Whether every wanted column for the current row already has a value.
303    /// The adapter can byte-jump to the row close tag when this returns true,
304    /// skipping remaining fields.  Composes with `row_rejected` (a row can
305    /// be satisfied OR rejected, both short-circuit).
306    ///
307    /// Default: false (never short-circuit).
308    #[inline]
309    fn row_satisfied(&self) -> bool {
310        false
311    }
312
313    /// Bitmask of columns that the projection wants.  Bit `i` is set iff
314    /// column `i` is in the output schema.  The adapter can test membership
315    /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
316    /// field.  Returns empty mask (all zeros) when no projection is active.
317    #[inline]
318    fn wanted_mask(&self) -> u64 {
319        0
320    }
321
322    /// After the layout is learned, the expected (slot, raw name bytes) at
323    /// this ordinal position in the record.  The adapter can compare the
324    /// raw bytes in-place (one memcmp) instead of running the full
325    /// attribute scan → UTF-8 decode → hash → lookup path.
326    ///
327    /// Return `None` to fall back to the generic path.  Default: `None`.
328    #[inline]
329    fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
330        None
331    }
332
333    /// Adapter reports the slot it resolved for `ordinal` so the engine
334    /// can cache it for subsequent rows.  Called after the adapter resolves
335    /// a field via the generic path (wants + resolve + put_field_resolved).
336    #[inline]
337    fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
338
339    /// Adapter reports that the layout expected at `ordinal` did not match.
340    /// The engine should invalidate its cached expectation and fall back to
341    /// generic resolution for subsequent rows.
342    #[inline]
343    fn layout_broken(&mut self, _ordinal: u32) {}
344
345    /// Reset the ordinal counter for child-element tracking.  Called after
346    /// row-tag attributes are emitted so child ordinals start at 0.
347    #[inline]
348    fn reset_child_ordinal(&mut self) {}
349
350    /// Finalize the sink into an Arrow `RecordBatch`.
351    fn finish(&mut self) -> Result<RecordBatch>;
352}
353
354/// Convert split-point offsets returned by a `Splitter` into non-empty
355/// `Range<usize>` chunks.
356pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
357    if points.len() < 2 {
358        return std::iter::once(0..len).collect();
359    }
360    points
361        .windows(2)
362        .filter_map(|w| {
363            let (start, end) = (w[0], w[1]);
364            if start < end {
365                Some(start..end)
366            } else {
367                None
368            }
369        })
370        .collect()
371}