use std::borrow::Cow;
use std::ops::Range;
use arrow::record_batch::RecordBatch;
use crate::value::Value;
use crate::Result;
pub const MIN_CHUNK_BYTES: usize = 2 << 20;
pub const MAX_SPLIT_CHUNKS: usize = 1024;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SplitMode {
Parallel,
Streaming,
}
pub fn plan_chunk_count_with(
bytes: usize,
threads: usize,
mode: SplitMode,
min_chunk_bytes: usize,
) -> usize {
let by_size = bytes / min_chunk_bytes.max(1);
let cap = match mode {
SplitMode::Parallel => 16usize.saturating_mul(threads),
SplitMode::Streaming => 8usize.saturating_mul(threads),
};
by_size.min(cap).max(threads).min(MAX_SPLIT_CHUNKS)
}
pub fn plan_chunk_count(bytes: usize, threads: usize, mode: SplitMode) -> usize {
plan_chunk_count_with(bytes, threads, mode, MIN_CHUNK_BYTES)
}
pub trait SkipRegionFinder: Send + Sync {
fn openers(&self) -> &[&'static [u8]];
fn closer_for(&self, opener: &[u8]) -> &'static [u8];
fn window(&self) -> usize {
64 * 1024
}
}
pub fn in_skip_region(bytes: &[u8], at: usize, finder: &dyn SkipRegionFinder) -> bool {
if at > bytes.len() {
return false;
}
let openers = finder.openers();
if openers.is_empty() {
return false;
}
let scan_start = at.saturating_sub(finder.window());
let scan = &bytes[scan_start..at];
for opener in openers {
let closer = finder.closer_for(opener);
let mut search = 0;
while let Some(rel) = memchr::memmem::find(&scan[search..], opener) {
let open_pos = scan_start + search + rel;
let closer_start = open_pos + opener.len();
if closer_start >= at {
return true;
}
let between = &bytes[closer_start..at];
if memchr::memmem::find(between, closer).is_none() {
return true;
}
search += rel + 1;
}
}
false
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum RecordBoundary {
#[default]
Line,
BlankLine,
}
pub trait Splitter: Send + Sync {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize>;
fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize;
fn record_boundary(&self) -> RecordBoundary {
RecordBoundary::Line
}
fn continuation_char(&self) -> Option<u8> {
None
}
fn comment_prefixes(&self) -> &[&[u8]] {
&[]
}
fn skip_regions(&self) -> Option<&dyn SkipRegionFinder> {
None
}
fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
default_find_split_points(self, bytes, max_chunks, MIN_CHUNK_BYTES)
}
fn find_split_points_with(
&self,
bytes: &[u8],
max_chunks: usize,
min_chunk_bytes: usize,
) -> Vec<usize> {
if min_chunk_bytes == MIN_CHUNK_BYTES {
return self.find_split_points(bytes, max_chunks);
}
default_find_split_points(self, bytes, max_chunks, min_chunk_bytes)
}
}
fn default_find_split_points<S: Splitter + ?Sized>(
splitter: &S,
bytes: &[u8],
max_chunks: usize,
min_chunk_bytes: usize,
) -> Vec<usize> {
if max_chunks <= 1 || bytes.is_empty() {
return vec![0, bytes.len()];
}
let n = plan_chunk_count_with(
bytes.len(),
max_chunks,
SplitMode::Parallel,
min_chunk_bytes,
);
let skip = splitter.skip_regions();
let mut points: Vec<usize> = (1..n)
.filter_map(|i| {
let approx = bytes.len() / n * i;
let pos = splitter.next_record_start(bytes, approx)?;
if skip.is_some_and(|finder| in_skip_region(bytes, pos, finder)) {
return None;
}
Some(pos)
})
.collect();
points.sort_unstable();
points.dedup();
points.retain(|&p| p > 0);
points.insert(0, 0);
if *points.last().unwrap_or(&0) != bytes.len() {
points.push(bytes.len());
}
points
}
#[inline]
pub fn is_continued_newline(bytes: &[u8], pos: usize) -> bool {
if pos == 0 || bytes.get(pos) != Some(&b'\n') {
return false;
}
let mut run = 0;
let mut cursor = pos;
if bytes[cursor - 1] == b'\r' {
cursor -= 1;
}
while cursor > 0 && bytes[cursor - 1] == b'\\' {
run += 1;
cursor -= 1;
}
run % 2 == 1
}
#[inline]
pub fn find_next_record_boundary(
bytes: &[u8],
from: usize,
continuation: Option<u8>,
comment_prefixes: &[&[u8]],
blank_line_mode: bool,
) -> Option<usize> {
if continuation.is_none() && comment_prefixes.is_empty() && !blank_line_mode {
return memchr::memchr(b'\n', bytes.get(from..)?).map(|offset| from + offset + 1);
}
let mut pos = from;
while pos < bytes.len() {
let rel = memchr::memchr(b'\n', &bytes[pos..])?;
let nl_pos = pos + rel;
let continued = |newline: usize| {
continuation.is_some_and(|character| {
let end = if newline > 0 && bytes[newline - 1] == b'\r' {
newline - 1
} else {
newline
};
if character == b'\\' {
is_continued_newline(bytes, newline)
} else {
end > 0 && bytes[end - 1] == character
}
})
};
if continued(nl_pos) {
pos = nl_pos + 1;
continue;
}
if !blank_line_mode && comment_prefixes.is_empty() {
return Some(nl_pos + 1);
}
if blank_line_mode {
let end = nl_pos - usize::from(nl_pos > 0 && bytes[nl_pos - 1] == b'\r');
if end == 0 || bytes[end - 1] != b'\n' {
pos = nl_pos + 1;
continue;
}
if comment_prefixes.is_empty() && !continued(end - 1) {
return Some(nl_pos + 1);
}
}
let line_start = memchr::memrchr(b'\n', &bytes[..nl_pos]).map_or(0, |p| p + 1);
if !comment_prefixes.is_empty() {
let line_bytes = &bytes[line_start..nl_pos];
let is_comment = comment_prefixes
.iter()
.any(|prefix| line_bytes.starts_with(prefix));
if is_comment {
pos = nl_pos + 1;
continue;
}
}
if blank_line_mode {
let line = &bytes[line_start..nl_pos];
if line_start > 0 && (line.is_empty() || line == b"\r") && !continued(line_start - 1) {
let previous_start =
memchr::memrchr(b'\n', &bytes[..line_start - 1]).map_or(0, |p| p + 1);
if !comment_prefixes
.iter()
.any(|prefix| bytes[previous_start..line_start - 1].starts_with(prefix))
{
return Some(nl_pos + 1);
}
}
pos = nl_pos + 1;
continue;
}
return Some(nl_pos + 1);
}
None
}
pub trait RecordParser: Send + Sync {
fn validate(&self, bytes: &[u8]) -> Result<()>;
fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()>;
#[inline]
fn parse_chunk_generic<S: ColumnarSink>(&self, bytes: &[u8], sink: &mut S) -> Result<()>
where
Self: Sized,
{
self.parse_chunk(bytes, sink as &mut dyn ColumnarSink)
}
}
pub trait ColumnarSink {
fn begin_row(&mut self);
fn put_field(&mut self, name: &str, value: Value<'_>);
#[inline]
fn put_row(&mut self, fields: &[(&str, Value<'_>)]) {
for (name, value) in fields {
self.put_field(name, value.clone());
}
}
#[inline]
fn resolve_raw<'a>(&'a self, raw_name: &'a [u8]) -> Option<&'a str> {
std::str::from_utf8(raw_name)
.ok()
.and_then(|name| self.resolve(name))
}
#[inline]
fn resolve_and_put_raw(&mut self, raw_name: &[u8], value: Value<'_>) {
if let Ok(name) = std::str::from_utf8(raw_name) {
self.resolve_and_put(name, value);
}
}
fn end_row(&mut self);
#[inline]
fn wants(&self, _name: &str) -> bool {
true
}
#[inline]
fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
Some(name)
}
#[inline]
fn put_field_resolved(&mut self, resolved_name: &str, value: Value<'_>) {
self.put_field(resolved_name, value);
}
#[inline]
fn put_field_at(&mut self, _slot: u32, value: Value<'_>) {
let _ = value;
}
#[inline]
fn put_str_at(&mut self, slot: u32, value: &str) {
self.put_field_at(slot, Value::Str(Cow::Borrowed(value)));
}
#[inline]
fn resolve_and_put(&mut self, name: &str, value: Value<'_>) {
if let Some(resolved) = self.resolve(name) {
let owned = resolved.to_owned();
self.put_field_resolved(&owned, value);
}
}
#[inline]
fn needs_value(&self) -> bool {
true
}
#[doc(hidden)]
#[inline]
fn needs_resolve(&self) -> bool {
true
}
#[inline]
fn row_rejected(&self) -> bool {
false
}
#[inline]
fn should_stop(&self) -> bool {
false
}
#[inline]
fn row_satisfied(&self) -> bool {
false
}
#[inline]
fn wanted_mask(&self) -> u64 {
0
}
#[inline]
fn expect_slot(&self, _ordinal: u32) -> Option<(u32, &[u8])> {
None
}
#[inline]
fn record_slot(&mut self, _ordinal: u32, _slot: u32, _raw_name: &[u8]) {}
#[inline]
fn layout_broken(&mut self, _ordinal: u32) {}
#[inline]
fn reset_child_ordinal(&mut self) {}
fn finish(&mut self) -> Result<RecordBatch>;
}
pub fn split_points_to_ranges(points: &[usize], len: usize) -> Vec<Range<usize>> {
if points.len() < 2 {
return std::iter::once(0..len).collect();
}
points
.windows(2)
.filter_map(|w| {
let (start, end) = (w[0], w[1]);
if start < end {
Some(start..end)
} else {
None
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
struct FixedWidthSplitter;
impl Splitter for FixedWidthSplitter {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
(from..bytes.len()).find(|&p| p % 4 == 0)
}
fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
4
}
}
#[test]
fn find_split_points_no_duplicate_leading_zero() {
let bytes = vec![0u8; 8];
let splitter = FixedWidthSplitter;
for max_chunks in [2, 8, 64, 1024] {
let points = splitter.find_split_points(&bytes, max_chunks);
assert_eq!(points.first(), Some(&0), "must start at 0");
assert_eq!(points.last(), Some(&bytes.len()), "must end at len");
assert!(
points.windows(2).all(|w| w[0] < w[1]),
"duplicate or unsorted points for max_chunks={max_chunks}: {points:?}"
);
}
}
}
#[cfg(test)]
mod chunk_floor_tests {
use super::*;
struct FixedSplitter;
impl Splitter for FixedSplitter {
fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
(from..bytes.len())
.find(|&p| bytes[p] == b'\n')
.map(|p| p + 1)
}
fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
4
}
fn find_split_points(&self, _bytes: &[u8], _max_chunks: usize) -> Vec<usize> {
vec![0, 7]
}
}
#[test]
fn adapter_override_is_honored_at_the_default_floor() {
assert_eq!(
FixedSplitter.find_split_points_with(b"ignored", 8, MIN_CHUNK_BYTES),
vec![0, 7]
);
}
#[test]
fn a_lower_floor_yields_more_chunks() {
let bytes = 8 << 20; let default = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, MIN_CHUNK_BYTES);
let small = plan_chunk_count_with(bytes, 4, SplitMode::Parallel, 256 * 1024);
assert!(small > default, "default={default} small={small}");
}
}