Skip to main content

rypipe_core/
decoder.rs

1use std::ops::Range;
2
3use arrow::record_batch::RecordBatch;
4
5use crate::value::Value;
6use crate::Result;
7
8/// Format-specific splitter: decides where it is safe to divide an input byte
9/// stream into independent chunks.
10pub trait Splitter: Send + Sync {
11    /// Return sorted byte offsets where the input may be split.  The first
12    /// offset should normally be `0` and the last should be `bytes.len()`.
13    /// Adjacent offsets produce one chunk range.
14    fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize>;
15
16    /// Estimate the average bytes per record from a sample of the input.
17    fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
18}
19
20/// Format-specific parser: turns a byte chunk into field/value events sent to
21/// a `ColumnarSink`.
22pub trait RecordParser: Send + Sync {
23    /// Validate that the whole byte slice is well-formed for this format.
24    fn validate(&self, bytes: &[u8]) -> Result<()>;
25
26    /// Parse one chunk and feed all row events into `sink`.
27    fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
28}
29
30/// Sink for decoder events.  The decoder calls `begin_row` / `put_field` /
31/// `end_row` for each record.
32pub trait ColumnarSink {
33    fn begin_row(&mut self);
34    fn put_field(&mut self, name: &str, value: Value<'_>);
35    fn end_row(&mut self);
36
37    /// Return `false` to signal that the engine will drop this field.
38    fn wants(&self, _name: &str) -> bool {
39        true
40    }
41
42    /// Finalize the sink into an Arrow `RecordBatch`.
43    fn finish(&mut self) -> Result<RecordBatch>;
44}
45
46/// Convert split-point offsets returned by a `Splitter` into non-empty
47/// `Range<usize>` chunks.
48pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
49    if points.len() < 2 {
50        return std::iter::once(0..len).collect();
51    }
52    points
53        .windows(2)
54        .filter_map(|w| {
55            let (start, end) = (w[0], w[1]);
56            if start < end {
57                Some(start..end)
58            } else {
59                None
60            }
61        })
62        .collect()
63}