use std::borrow::Cow;
use std::collections::VecDeque;
use std::fs::File;
use std::io::{self, BufRead, BufReader};
use std::sync::Arc;
use super::{
backend::Backend,
chunk_types::{Chunk, ChunkConfig},
chunker::smart_chunk_mapped_profile,
chunker::SmartChunker,
normalization::{NormBase, NormMap, Normalization, OrigSpan},
split_engine::char_len,
ChipError,
};
const BOM: char = '\u{feff}';
pub const FLAT_FILE_MAX_BYTES: u64 = 32 * 1024 * 1024;
pub(crate) const MAX_INPUT_LINE_BYTES: usize = 1024 * 1024;
fn read_line_bounded<R: BufRead>(
reader: &mut R,
buf: &mut Vec<u8>,
max: usize,
) -> io::Result<usize> {
loop {
let (taken, newline, eof) = {
let available = reader.fill_buf()?;
if available.is_empty() {
(0, false, true)
} else {
let through_newline = available
.iter()
.position(|byte| *byte == b'\n')
.map_or(available.len(), |index| index + 1);
let remaining = max.saturating_add(1).saturating_sub(buf.len());
let taken = through_newline.min(remaining);
buf.extend_from_slice(&available[..taken]);
(taken, buf.last() == Some(&b'\n'), false)
}
};
reader.consume(taken);
if buf.len() > max {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("physical input line exceeds the {max}-byte streaming limit"),
));
}
if eof || newline || taken == 0 {
return Ok(buf.len());
}
}
}
fn invalid_utf8(path: &str, e: impl std::fmt::Display) -> ChipError {
ChipError::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!("invalid UTF-8 in '{path}': {e} (use errors=\"replace\" to substitute)"),
))
}
pub fn read_text_file(path: &str, lossy: bool) -> Result<String, ChipError> {
let bytes = std::fs::read(path)?;
let mut text = if lossy {
String::from_utf8_lossy(&bytes).into_owned()
} else {
String::from_utf8(bytes).map_err(|e| invalid_utf8(path, e))?
};
if text.starts_with(BOM) {
text.replace_range(..BOM.len_utf8(), "");
}
Ok(text)
}
pub struct FileChunker {
path: String,
reader: BufReader<File>,
chunker: SmartChunker,
norm: Normalization,
lossy: bool,
queue: VecDeque<Chunk>,
fast_queue: Option<VecDeque<(Chunk, OrigSpan)>>,
scratch: Vec<Chunk>,
buf: Vec<u8>,
first_line: bool,
done: bool,
map: NormMap,
base: NormBase,
peak_norm_map_segments: usize,
peak_parser_open_bytes: usize,
peak_pending_bytes: usize,
peak_parser_gap_bytes: usize,
peak_input_line_bytes: usize,
}
impl FileChunker {
pub fn open(
path: &str,
cfg: &ChunkConfig,
backend: &Arc<Backend>,
lossy: bool,
) -> Result<Self, ChipError> {
cfg.validate()?;
let file = File::open(path)?;
let file_len = file.metadata()?.len();
let cfg = cfg.clone();
let norm = cfg.normalization;
let (fast_queue, fast_peak_norm_map_segments) = if !cfg.include_context {
if file_len > FLAT_FILE_MAX_BYTES {
return Err(ChipError::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"'{path}' is {file_len} bytes; flat mode (include_context=false) reads the \
whole file and accepts at most {FLAT_FILE_MAX_BYTES} bytes. Use \
include_context=true to stream it, or split the file."
),
)));
}
let text = read_text_file(path, lossy)?;
let (chunks, spans, peak_segments) = smart_chunk_mapped_profile(&text, &cfg, backend)?;
(Some(chunks.into_iter().zip(spans).collect()), peak_segments)
} else {
(None, 0)
};
let streaming = fast_queue.is_none();
let reader_capacity = if streaming { 1 << 20 } else { 8 * 1024 };
Ok(FileChunker {
path: path.to_string(),
reader: BufReader::with_capacity(reader_capacity, file),
chunker: SmartChunker::new(cfg, Arc::clone(backend)),
norm,
lossy,
queue: VecDeque::new(),
fast_queue,
scratch: Vec::new(),
buf: if streaming {
Vec::with_capacity(MAX_INPUT_LINE_BYTES + 1)
} else {
Vec::new()
},
first_line: true,
done: false,
map: NormMap::default(),
base: NormBase::default(),
peak_norm_map_segments: fast_peak_norm_map_segments,
peak_parser_open_bytes: 0,
peak_pending_bytes: 0,
peak_parser_gap_bytes: 0,
peak_input_line_bytes: 0,
})
}
#[cfg_attr(not(feature = "python"), allow(dead_code))]
pub(crate) fn peak_norm_map_segments(&self) -> usize {
self.peak_norm_map_segments
}
#[cfg_attr(not(feature = "python"), allow(dead_code))]
pub(crate) fn current_norm_map_segments(&self) -> usize {
self.map.segs.len()
}
#[cfg_attr(not(feature = "python"), allow(dead_code))]
pub(crate) fn peak_parser_open_bytes(&self) -> usize {
self.peak_parser_open_bytes
}
#[cfg_attr(not(feature = "python"), allow(dead_code))]
pub(crate) fn peak_pending_bytes(&self) -> usize {
self.peak_pending_bytes
}
#[cfg_attr(not(feature = "python"), allow(dead_code))]
pub(crate) fn peak_parser_gap_bytes(&self) -> usize {
self.peak_parser_gap_bytes
}
#[cfg_attr(not(feature = "python"), allow(dead_code))]
pub(crate) fn peak_input_line_bytes(&self) -> usize {
self.peak_input_line_bytes
}
pub fn next_chunk(&mut self) -> Result<Option<(Chunk, OrigSpan)>, ChipError> {
if let Some(fast) = self.fast_queue.as_mut() {
return Ok(fast.pop_front());
}
loop {
if let Some(c) = self.queue.pop_front() {
let span = self.map.span(&c);
if self.queue.is_empty() {
let (safe_byte, safe_char) = if self.done {
(self.base.nb, self.base.nc)
} else {
self.chunker.earliest_future_start()
};
self.map.prune(safe_byte, safe_char);
}
return Ok(Some((c, span)));
}
if self.done {
return Ok(None);
}
self.buf.clear();
let n = match read_line_bounded(&mut self.reader, &mut self.buf, MAX_INPUT_LINE_BYTES) {
Ok(n) => n,
Err(error) => {
self.peak_input_line_bytes = self.peak_input_line_bytes.max(self.buf.len());
self.done = true;
return Err(error.into());
}
};
self.peak_input_line_bytes = self.peak_input_line_bytes.max(n);
if n == 0 {
self.done = true;
if let Err(error) = self.chunker.finish(&mut self.scratch) {
self.done = true;
return Err(error);
}
self.peak_parser_open_bytes = self
.peak_parser_open_bytes
.max(self.chunker.parser.open_bytes());
self.peak_pending_bytes = self
.peak_pending_bytes
.max(self.chunker.peak_pending_bytes());
self.peak_parser_gap_bytes = self
.peak_parser_gap_bytes
.max(self.chunker.parser.gap_bytes());
} else {
let decoded: Cow<'_, str> = if self.lossy {
Cow::Owned(String::from_utf8_lossy(&self.buf).into_owned())
} else {
match std::str::from_utf8(&self.buf) {
Ok(text) => Cow::Borrowed(text),
Err(error) => {
self.done = true;
return Err(invalid_utf8(&self.path, error));
}
}
};
let line = if self.first_line {
self.first_line = false;
match decoded {
Cow::Borrowed(s) => Cow::Borrowed(s.strip_prefix(BOM).unwrap_or(s)),
Cow::Owned(mut s) => {
if s.starts_with(BOM) {
s.drain(..BOM.len_utf8());
}
Cow::Owned(s)
}
}
} else {
decoded
};
let owned = match self
.norm
.apply_mapped(line.as_ref(), self.base, &mut self.map)
{
Cow::Owned(s) => Some(s),
Cow::Borrowed(_) => None,
};
self.peak_norm_map_segments = self.peak_norm_map_segments.max(self.map.segs.len());
let normalized: &str = owned.as_deref().unwrap_or(line.as_ref());
self.base.ob += line.len();
self.base.oc += char_len(line.as_ref());
self.base.nb += normalized.len();
self.base.nc += char_len(normalized);
if let Err(error) = self.chunker.push_line(normalized, &mut self.scratch) {
self.done = true;
return Err(error);
}
self.peak_parser_open_bytes = self
.peak_parser_open_bytes
.max(self.chunker.parser.open_bytes());
self.peak_pending_bytes = self
.peak_pending_bytes
.max(self.chunker.peak_pending_bytes());
self.peak_parser_gap_bytes = self
.peak_parser_gap_bytes
.max(self.chunker.parser.gap_bytes());
}
self.queue.extend(self.scratch.drain(..));
}
}
}