use std::ops::Range;
use arrow::record_batch::RecordBatch;
use crate::value::Value;
use crate::Result;
pub const MIN_CHUNK_BYTES: usize = 2 << 20;
pub const MAX_SPLIT_CHUNKS: usize = 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SplitMode {
Parallel,
Streaming,
}
pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
let by_size = bytes / MIN_CHUNK_BYTES.max(1);
let cap = match mode {
SplitMode::Parallel => 16usize.saturating_mul(threads),
SplitMode::Streaming => 8usize.saturating_mul(threads),
};
by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
}
pub trait SkipRegionFinder: Send + Sync {
fn openers(&self) -> &[&'static [u8]];
fn closer_for(&self, opener: &[u8]) -> &'static [u8];
fn window(&self) -> usize {
64 * 1024
}
}
pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
if at > bytes.len() {
return false;
}
let openers = finder.openers();
if openers.is_empty() {
return false;
}
let scan_start = at.saturating_sub(finder.window());
let scan = &bytes[scan_start..at];
for opener in openers {
let closer = finder.closer_for(opener);
let mut search = 0;
while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
let open_pos = scan_start + search + rel;
let closer_start = open_pos + opener.len();
if closer_start >= at {
return true;
}
let between = &bytes[closer_start..at];
if memchr::memmem::find(between, closer).is_none() {
return true;
}
search += rel + 1;
}
}
false
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum RecordBoundary {
#[default]
Line,
BlankLine,
}
pub trait Splitter: Send + Sync {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
fn record_boundary(&self) -> RecordBoundary {
RecordBoundary::Line
}
fn continuation_char(&self) -> Option<u8> {
None
}
fn comment_prefixes(&self) -> &[&[u8]] {
&[]
}
fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
None
}
fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
if max_chunks <= 1 || bytes.is_empty() {
return vec![0, bytes.len()];
}
let n = plan_chunk_count(bytes.len(), max_chunks, SplitMode::Parallel);
let skip = self.skip_regions();
let mut points: Vec<usize> = (1..n)
.filter_map(|i| {
let approx = bytes.len() / n * i;
let pos = self.next_record_start(bytes, approx)?;
if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
return None;
}
Some(pos)
})
.collect();
points.sort_unstable();
points.dedup();
points.insert(0, 0);
if *points.last().unwrap_or(&0) != bytes.len() {
points.push(bytes.len());
}
points
}
}
#[inline]
pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
if pos == 0 || bytes.get(pos) != Some(&b'\n') {
return false;
}
let mut run = 0;
let mut cursor = pos;
if bytes[cursor - 1] == b'\r' {
cursor -= 1;
}
while cursor > 0 && bytes[cursor - 1] == b'\\' {
run += 1;
cursor -= 1;
}
run % 2 == 1
}
#[inline]
pub fn find_next_record_boundary(
bytes: &[u8],
from: usize,
continuation: Option<u8>,
comment_prefixes: &[&[u8]],
blank_line_mode: bool,
) -> Option<usize> {
if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
}
let mut pos = from;
while pos < bytes.len() {
let rel = memchr::memchr(b'\n', &bytes[pos..])?;
let nl_pos = pos + rel;
let continued = |newline: usize| {
continuation.is_some_and(|character| {
let end = if newline > 0 && bytes[newline - 1] == b'\r' {
newline - 1
} else {
newline
};
if character == b'\\' {
is_continued_newline(bytes, newline)
} else {
end > 0 && bytes[end - 1] == character
}
})
};
if continued(nl_pos) {
pos = nl_pos + 1;
continue;
}
if !blank_line_mode && comment_prefixes.is_empty() {
return Some(nl_pos + 1);
}
if blank_line_mode {
let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
if end == 0 || bytes[end - 1] != b'\n' {
pos = nl_pos + 1;
continue;
}
if comment_prefixes.is_empty() && !continued(end - 1) {
return Some(nl_pos + 1);
}
}
let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
if !comment_prefixes.is_empty() {
let line_bytes = &bytes[line_start..nl_pos];
let is_comment = comment_prefixes
.iter()
.any(|prefix| line_bytes.starts_with(prefix));
if is_comment {
pos = nl_pos + 1;
continue;
}
}
if blank_line_mode {
let line = &bytes[line_start..nl_pos];
if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
let previous_start =
memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
if !comment_prefixes
.iter()
.any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
{
return Some(nl_pos + 1);
}
}
pos = nl_pos + 1;
continue;
}
return Some(nl_pos + 1);
}
None
}
pub trait RecordParser: Send + Sync {
fn validate(&self, bytes: &[u8]) -> Result<()>;
fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
#[inline]
fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
where
Self: Sized,
{
self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
}
}
pub trait ColumnarSink {
fn begin_row(&mut self);
fn put_field(&mut self, name: &str, value: Value<'_>);
#[inline]
fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
for (name, value) in fields {
self.put_field(name, value.clone());
}
}
#[inline]
fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
std::str::from_utf8(raw_name)
.ok()
.and_then(|name| self.resolve(name))
}
#[inline]
fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
if let Ok(name) = std::str::from_utf8(raw_name) {
self.resolve_and_put(name, value);
}
}
fn end_row(&mut self);
#[inline]
fn wants(&self, _name: &str) -> bool {
true
}
#[inline]
fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
Some(name)
}
#[inline]
fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
self.put_field(resolved_name, value);
}
#[inline]
fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
let _ = value;
}
#[inline]
fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
if let Some(resolved) = self.resolve(name) {
let owned = resolved.to_owned();
self.put_field_resolved(&owned, value);
}
}
#[inline]
fn needs_value(&self) -> bool {
true
}
#[doc(hidden)]
#[inline]
fn needs_resolve(&self) -> bool {
true
}
#[inline]
fn row_rejected(&self) -> bool {
false
}
#[inline]
fn row_satisfied(&self) -> bool {
false
}
#[inline]
fn wanted_mask(&self) -> u64 {
0
}
#[inline]
fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
None
}
#[inline]
fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
#[inline]
fn layout_broken(&mut self, _ordinal: u32) {}
#[inline]
fn reset_child_ordinal(&mut self) {}
fn finish(&mut self) -> Result<RecordBatch>;
}
pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
if points.len() < 2 {
return std::iter::once(0..len).collect();
}
points
.windows(2)
.filter_map(|w| {
let (start, end) = (w[0], w[1]);
if start < end {
Some(start..end)
} else {
None
}
})
.collect()
}