rypipe_core/decoder.rs
1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8// ---------------------------------------------------------------------------
9// S4: Chunk-size floor
10// ---------------------------------------------------------------------------
11
12/// Minimum chunk size in bytes. Sub-MB chunks collapse throughput due to
13/// per-chunk fixed cost (thread dispatch, cache cold start). Measured:
14/// 100 MB at par128 (0.78 MB chunks) = 2,265 MB/s vs par16 (6.25 MB) = 3,735.
15pub const MIN_CHUNK_BYTES: usize = 2 << 20; // 2 MiB
16
17/// Maximum number of split chunks. Above this, scheduling overhead dominates.
18pub const MAX_SPLIT_CHUNKS: usize = 1024;
19
20/// Mode for chunk planning: parallel and streaming have different optima.
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
22pub enum SplitMode {
23 /// Parallel: peak at 4 MB chunks (measured on 533 MB, 16 cores).
24 Parallel,
25 /// Streaming: peak at 2 MB chunks (measured with bounded memory).
26 Streaming,
27}
28
29/// Plan the number of chunks given input size, thread count, and mode.
30/// Returns a value in `[threads, MAX_SPLIT_CHUNKS]` with a 2 MiB floor.
31pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
32 let by_size = bytes / MIN_CHUNK_BYTES.max(1);
33 let cap = match mode {
34 SplitMode::Parallel => 16 * threads,
35 SplitMode::Streaming => 8 * threads,
36 };
37 by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
38}
39
40// ---------------------------------------------------------------------------
41// S3: SkipRegionFinder: byte ranges where a candidate boundary is invalid
42// ---------------------------------------------------------------------------
43
44/// Finds byte ranges (comments, CDATA, quoted fields, string literals) where
45/// a candidate record boundary must be rejected. The engine calls this
46/// during split-point discovery to avoid splitting inside comments or strings.
47pub trait SkipRegionFinder: Send + Sync {
48 /// Openers that start a skip region (e.g. `b"<!--"`, `b"<![CDATA["`).
49 fn openers(&self) -> &[&'static [u8]];
50
51 /// The closer for a given opener (e.g. `"-->"` for `"<!--"`).
52 fn closer_for(&self, opener: &[u8]) -> &'static [u8];
53
54 /// Maximum backward scan window in bytes. Default 64 KiB: sufficient
55 /// for most XML/CSV comments and JSON string literals.
56 fn window(&self) -> usize {
57 64 * 1024
58 }
59}
60
61/// Check whether byte position `at` falls inside a skip region by scanning
62/// backward up to `finder.window()` bytes, looking for an unclosed opener.
63///
64/// Returns `false` immediately when the file contains none of the openers;
65/// check that once per chunk, not per candidate.
66pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
67 let openers = finder.openers();
68 if openers.is_empty() {
69 return false;
70 }
71 let scan_start = at.saturating_sub(finder.window());
72 let scan = &bytes[scan_start..at];
73 for opener in openers {
74 let closer = finder.closer_for(opener);
75 // Use memchr::memmem::find directly (no Finder::new preprocessing).
76 let mut search = 0;
77 while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
78 let open_pos = scan_start + search + rel;
79 // Check if there's a closer between the opener and `at`.
80 let between = &bytes[open_pos + opener.len()..at];
81 if memchr::memmem::find(between, closer).is_none() {
82 // Unclosed opener; `at` is inside this skip region.
83 return true;
84 }
85 search += rel + 1;
86 }
87 }
88 false
89}
90
91// ---------------------------------------------------------------------------
92// S1: Splitter trait with default
93// ---------------------------------------------------------------------------
94
95/// Format-specific splitter: decides where it is safe to divide an input byte
96/// stream into independent chunks.
97///
98/// The only required method is `next_record_start`; adapters provide the
99/// record boundary logic, and the engine handles chunk planning, skip-region
100/// rejection, and deduplication.
101pub trait Splitter: Send + Sync {
102 /// Return the next record boundary at or after `from`. Must return a
103 /// position where a record starts, or `None` if no more records exist.
104 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
105
106 /// Estimate the average bytes per record from a sample of the input.
107 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
108
109 /// Optional: byte ranges where a candidate boundary must be rejected.
110 /// See `in_skip_region`.
111 fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
112 None
113 }
114
115 /// Find split points. **Do not override** without a measured reason.
116 ///
117 /// Default implementation:
118 /// 1. Nominal offsets at `bytes.len() * i / n` for `i in 1..n`.
119 /// 2. `par_iter` over nominals (rayon), each calling `next_record_start`.
120 /// 3. Reject candidates inside skip regions via `in_skip_region`.
121 /// 4. Dedup, sort, prepend 0.
122 /// 5. Apply chunk floor via `plan_chunk_count`.
123 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
124 if max_chunks <= 1 || bytes.is_empty() {
125 return vec![0, bytes.len()];
126 }
127 let n = plan_chunk_count(bytes.len(), max_chunks, SplitMode::Parallel);
128 let nominals: Vec<usize> = (1..n).map(|i| bytes.len() * i / n).collect();
129
130 use rayon::prelude::*;
131
132 let skip = self.skip_regions();
133 let has_skip = skip.is_some();
134
135 let mut points: Vec<usize> = nominals
136 .par_iter()
137 .filter_map(|&approx| {
138 let pos = self.next_record_start(bytes, approx)?;
139 // Reject candidates inside skip regions.
140 if has_skip && in_skip_region(bytes, pos, skip.unwrap()) {
141 return None;
142 }
143 Some(pos)
144 })
145 .collect();
146
147 points.sort_unstable();
148 points.dedup();
149 points.insert(0, 0);
150 if *points.last().unwrap_or(&0) != bytes.len() {
151 points.push(bytes.len());
152 }
153 points
154 }
155}
156
157/// Format-specific parser: turns a byte chunk into field/value events sent to
158/// a `ColumnarSink`.
159pub trait RecordParser: Send + Sync {
160 /// Validate that the whole byte slice is well-formed for this format.
161 fn validate(&self, bytes: &[u8]) -> Result<()>;
162
163 /// Parse one chunk and feed all row events into `sink` (object-safe, virtual).
164 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
165
166 /// Parse one chunk generically (monomorphized, inlinable).
167 ///
168 /// Default impl delegates to `parse_chunk` (object-safe) for adapters that
169 /// have not yet migrated. Adapters that implement this directly get
170 /// devirtualized `sink` calls and cross-crate inlining.
171 #[inline]
172 fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
173 where
174 Self: Sized,
175 {
176 // Default impl delegates to the object-safe path. Adapters that override
177 // this method get devirtualized sink calls and cross-crate inlining.
178 self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
179 }
180}
181
182/// Sink for decoder events. The decoder calls `begin_row` / `put_field` /
183/// `end_row` for each record.
184pub trait ColumnarSink {
185 fn begin_row(&mut self);
186 fn put_field(&mut self, name: &str, value: Value<'_>);
187
188 /// Push a complete row.
189 ///
190 /// The default preserves event-oriented semantics. Sinks may override this
191 /// to optimize row handling while preserving filtering and duplicate-field
192 /// behavior.
193 #[inline]
194 fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
195 for (name, value) in fields {
196 self.put_field(name, value.clone());
197 }
198 }
199
200 /// Resolve a valid raw name without allocating a UTF-8 `String`.
201 #[inline]
202 fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
203 std::str::from_utf8(raw_name)
204 .ok()
205 .and_then(|name| self.resolve(name))
206 }
207
208 /// Push a value for a field whose name is still in its raw byte form.
209 #[inline]
210 fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
211 if let Ok(name) = std::str::from_utf8(raw_name) {
212 self.resolve_and_put(name, value);
213 }
214 }
215
216 fn end_row(&mut self);
217
218 /// Return `false` to signal that the engine will drop this field.
219 #[inline]
220 fn wants(&self, _name: &str) -> bool {
221 true
222 }
223
224 /// Resolve a raw field name to its output column name, or `None` if dropped.
225 /// Default keeps the name as-is. Adapters that do expensive extraction can
226 /// call this once and then `put_field_resolved` to avoid a second
227 /// `resolve_field` inside `put_field`.
228 #[inline]
229 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
230 Some(name)
231 }
232
233 /// Push a field that has already been resolved via `resolve`.
234 /// Default delegates to `put_field` (which will resolve again). Overrides
235 /// should bypass the rename/drop lookup.
236 #[inline]
237 fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
238 self.put_field(resolved_name, value);
239 }
240
241 /// Push a value directly to a known slot index, bypassing all name
242 /// resolution, hash lookups, and attribute scanning. Used by adapters
243 /// that have verified the field identity via `expect_slot` + memcmp.
244 ///
245 /// Default: delegates to `put_field` with a synthetic name (slow path).
246 /// Engines that track slot indices should override this.
247 #[inline]
248 fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
249 // Fallback: can't push without a name; discard.
250 // Engines override this to push directly to columns[slot].
251 let _ = value;
252 }
253
254 /// Resolve a field name and push it in one call, avoiding the double
255 /// `resolve_field` that `wants` + `put_field` would perform.
256 /// Default: resolve then put_field_resolved (with owned clone to satisfy
257 /// the borrow checker). Overrides should avoid the allocation.
258 #[inline]
259 fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
260 if let Some(resolved) = self.resolve(name) {
261 let owned = resolved.to_owned();
262 self.put_field_resolved(&owned, value);
263 }
264 }
265
266 /// Whether this sink needs decoded values from the parser.
267 ///
268 /// Return `false` for locate-only sinks that walk rows and resolve field
269 /// names but skip value extraction entirely. The parser can use this to
270 /// bypass expensive text extraction (e.g. `raw_text_until` in XML scanners).
271 #[inline]
272 fn needs_value(&self) -> bool {
273 true
274 }
275
276 /// Whether this sink needs field-name resolution (`wants` + `resolve`).
277 ///
278 /// Return `false` for traversal-only sinks that walk rows and find field
279 /// extents but don't need to resolve names or check `wants`. When false,
280 /// the parser skips `wants()` and `resolve()` calls entirely; it only
281 /// locates the byte extents of each field within a row.
282 ///
283 /// **Note:** this method is primarily for benchmarking/profiling harnesses.
284 /// A sink returning `false` may receive `put_field` calls with unresolved
285 /// names, or none at all, depending on the adapter. Consider this
286 /// experimental until a non-benchmark consumer exists.
287 #[doc(hidden)]
288 #[inline]
289 fn needs_resolve(&self) -> bool {
290 true
291 }
292
293 /// Whether the current row has been rejected by the filter and the scanner
294 /// should skip the remainder of the row. Used for predicate-first
295 /// deferred materialization: once `Fail`, the scanner can byte-jump to
296 /// `</Details>` without further field scanning. Default false.
297 #[inline]
298 fn row_rejected(&self) -> bool {
299 false
300 }
301
302 /// Whether every wanted column for the current row already has a value.
303 /// The adapter can byte-jump to the row close tag when this returns true,
304 /// skipping remaining fields. Composes with `row_rejected` (a row can
305 /// be satisfied OR rejected, both short-circuit).
306 ///
307 /// Default: false (never short-circuit).
308 #[inline]
309 fn row_satisfied(&self) -> bool {
310 false
311 }
312
313 /// Bitmask of columns that the projection wants. Bit `i` is set iff
314 /// column `i` is in the output schema. The adapter can test membership
315 /// with `(wanted_mask >> slot) & 1 == 1` instead of a virtual call per
316 /// field. Returns empty mask (all zeros) when no projection is active.
317 #[inline]
318 fn wanted_mask(&self) -> u64 {
319 0
320 }
321
322 /// After the layout is learned, the expected (slot, raw name bytes) at
323 /// this ordinal position in the record. The adapter can compare the
324 /// raw bytes in-place (one memcmp) instead of running the full
325 /// attribute scan → UTF-8 decode → hash → lookup path.
326 ///
327 /// Return `None` to fall back to the generic path. Default: `None`.
328 #[inline]
329 fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
330 None
331 }
332
333 /// Adapter reports the slot it resolved for `ordinal` so the engine
334 /// can cache it for subsequent rows. Called after the adapter resolves
335 /// a field via the generic path (wants + resolve + put_field_resolved).
336 #[inline]
337 fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
338
339 /// Adapter reports that the layout expected at `ordinal` did not match.
340 /// The engine should invalidate its cached expectation and fall back to
341 /// generic resolution for subsequent rows.
342 #[inline]
343 fn layout_broken(&mut self, _ordinal: u32) {}
344
345 /// Reset the ordinal counter for child-element tracking. Called after
346 /// row-tag attributes are emitted so child ordinals start at 0.
347 #[inline]
348 fn reset_child_ordinal(&mut self) {}
349
350 /// Finalize the sink into an Arrow `RecordBatch`.
351 fn finish(&mut self) -> Result<RecordBatch>;
352}
353
354/// Convert split-point offsets returned by a `Splitter` into non-empty
355/// `Range<usize>` chunks.
356pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
357 if points.len() < 2 {
358 return std::iter::once(0..len).collect();
359 }
360 points
361 .windows(2)
362 .filter_map(|w| {
363 let (start, end) = (w[0], w[1]);
364 if start < end {
365 Some(start..end)
366 } else {
367 None
368 }
369 })
370 .collect()
371}