use std::ops::Range;
use arrow::record_batch::RecordBatch;
use crate::value::Value;
use crate::Result;
pub const MIN_CHUNK_BYTES: usize = 2 << 20;
pub const MAX_SPLIT_CHUNKS: usize = 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SplitMode {
Parallel,
Streaming,
}
pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
let by_size = bytes / MIN_CHUNK_BYTES.max(1);
let cap = match mode {
SplitMode::Parallel => 16 * threads,
SplitMode::Streaming => 8 * threads,
};
by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
}
pub trait SkipRegionFinder: Send + Sync {
fn openers(&self) -> &[&'static [u8]];
fn closer_for(&self, opener: &[u8]) -> &'static [u8];
fn window(&self) -> usize {
64 * 1024
}
}
pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
let openers = finder.openers();
if openers.is_empty() {
return false;
}
let scan_start = at.saturating_sub(finder.window());
let scan = &bytes[scan_start..at];
for opener in openers {
let closer = finder.closer_for(opener);
let mut search = 0;
while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
let open_pos = scan_start + search + rel;
let between = &bytes[open_pos + opener.len()..at];
if memchr::memmem::find(between, closer).is_none() {
return true;
}
search += rel + 1;
}
}
false
}
pub trait Splitter: Send + Sync {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
None
}
fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
if max_chunks <= 1 || bytes.is_empty() {
return vec![0, bytes.len()];
}
let n = plan_chunk_count(bytes.len(), max_chunks, SplitMode::Parallel);
let nominals: Vec<usize> = (1..n).map(|i| bytes.len() * i / n).collect();
use rayon::prelude::*;
let skip = self.skip_regions();
let has_skip = skip.is_some();
let mut points: Vec<usize> = nominals
.par_iter()
.filter_map(|&approx| {
let pos = self.next_record_start(bytes, approx)?;
if has_skip && in_skip_region(bytes, pos, skip.unwrap()) {
return None;
}
Some(pos)
})
.collect();
points.sort_unstable();
points.dedup();
points.insert(0, 0);
if *points.last().unwrap_or(&0) != bytes.len() {
points.push(bytes.len());
}
points
}
}
pub trait RecordParser: Send + Sync {
fn validate(&self, bytes: &[u8]) -> Result<()>;
fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
#[inline]
fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
where
Self: Sized,
{
self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
}
}
pub trait ColumnarSink {
fn begin_row(&mut self);
fn put_field(&mut self, name: &str, value: Value<'_>);
#[inline]
fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
for (name, value) in fields {
self.put_field(name, value.clone());
}
}
#[inline]
fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
std::str::from_utf8(raw_name)
.ok()
.and_then(|name| self.resolve(name))
}
#[inline]
fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
if let Ok(name) = std::str::from_utf8(raw_name) {
self.resolve_and_put(name, value);
}
}
fn end_row(&mut self);
#[inline]
fn wants(&self, _name: &str) -> bool {
true
}
#[inline]
fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
Some(name)
}
#[inline]
fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
self.put_field(resolved_name, value);
}
#[inline]
fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
let _ = value;
}
#[inline]
fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
if let Some(resolved) = self.resolve(name) {
let owned = resolved.to_owned();
self.put_field_resolved(&owned, value);
}
}
#[inline]
fn needs_value(&self) -> bool {
true
}
#[doc(hidden)]
#[inline]
fn needs_resolve(&self) -> bool {
true
}
#[inline]
fn row_rejected(&self) -> bool {
false
}
#[inline]
fn row_satisfied(&self) -> bool {
false
}
#[inline]
fn wanted_mask(&self) -> u64 {
0
}
#[inline]
fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
None
}
#[inline]
fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
#[inline]
fn layout_broken(&mut self, _ordinal: u32) {}
#[inline]
fn reset_child_ordinal(&mut self) {}
fn finish(&mut self) -> Result<RecordBatch>;
}
pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
if points.len() < 2 {
return std::iter::once(0..len).collect();
}
points
.windows(2)
.filter_map(|w| {
let (start, end) = (w[0], w[1]);
if start < end {
Some(start..end)
} else {
None
}
})
.collect()
}