rypipe_core/decoder.rs
1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8/// Format-specific splitter: decides where it is safe to divide an input byte
9/// stream into independent chunks.
10pub trait Splitter: Send + Sync {
11 /// Return sorted byte offsets where the input may be split. The first
12 /// offset should normally be `0` and the last should be `bytes.len()`.
13 /// Adjacent offsets produce one chunk range.
14 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize>;
15
16 /// Estimate the average bytes per record from a sample of the input.
17 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
18}
19
20/// Format-specific parser: turns a byte chunk into field/value events sent to
21/// a `ColumnarSink`.
22pub trait RecordParser: Send + Sync {
23 /// Validate that the whole byte slice is well-formed for this format.
24 fn validate(&self, bytes: &[u8]) -> Result<()>;
25
26 /// Parse one chunk and feed all row events into `sink`.
27 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
28}
29
30/// Sink for decoder events. The decoder calls `begin_row` / `put_field` /
31/// `end_row` for each record.
32pub trait ColumnarSink {
33 fn begin_row(&mut self);
34 fn put_field(&mut self, name: &str, value: Value<'_>);
35 fn end_row(&mut self);
36
37 /// Return `false` to signal that the engine will drop this field.
38 fn wants(&self, _name: &str) -> bool {
39 true
40 }
41
42 /// Finalize the sink into an Arrow `RecordBatch`.
43 fn finish(&mut self) -> Result<RecordBatch>;
44}
45
46/// Convert split-point offsets returned by a `Splitter` into non-empty
47/// `Range<usize>` chunks.
48pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
49 if points.len() < 2 {
50 return std::iter::once(0..len).collect();
51 }
52 points
53 .windows(2)
54 .filter_map(|w| {
55 let (start, end) = (w[0], w[1]);
56 if start < end {
57 Some(start..end)
58 } else {
59 None
60 }
61 })
62 .collect()
63}