use crate::diagnostics::ParseDiagnostics;
use crate::Result;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamState {
Rows,
NeedMore,
Eof,
}
pub trait StreamingRecordParser: Send {
type Record;
fn validate(&self, bytes: &[u8]) -> Result<()>;
fn parse_available(
&mut self,
bytes: &[u8],
eof: bool,
emit: &mut dyn FnMut(Self::Record),
diag: &dyn ParseDiagnostics,
) -> Result<usize>;
}
pub struct RecordStream<P: StreamingRecordParser> {
parser: P,
buffer: Vec<u8>,
start: usize,
eof: bool,
}
const COMPACT_AFTER: usize = 64 * 1024;
impl<P: StreamingRecordParser> RecordStream<P> {
pub fn new(parser: P) -> Self {
Self {
parser,
buffer: Vec::new(),
start: 0,
eof: false,
}
}
pub fn parser(&self) -> &P {
&self.parser
}
pub fn feed(&mut self, chunk: &[u8]) -> Result<()> {
self.parser.validate(chunk)?;
self.buffer.extend_from_slice(chunk);
Ok(())
}
pub fn finish(&mut self) {
self.eof = true;
}
pub fn is_eof(&self) -> bool {
self.eof
}
pub fn pending(&self) -> usize {
self.buffer.len() - self.start
}
pub fn parse(
&mut self,
emit: &mut dyn FnMut(P::Record),
diag: &dyn ParseDiagnostics,
) -> Result<StreamState> {
let RecordStream {
parser,
buffer,
start,
eof,
} = self;
let consumed = parser.parse_available(&buffer[*start..], *eof, emit, diag)?;
*start += consumed;
if *start >= buffer.len() || *start >= COMPACT_AFTER {
buffer.drain(..*start);
*start = 0;
}
if *eof {
return Ok(StreamState::Eof);
}
if consumed == 0 {
Ok(StreamState::NeedMore)
} else {
Ok(StreamState::Rows)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::diagnostics::NoopDiagnostics;
struct LineParser;
impl StreamingRecordParser for LineParser {
type Record = String;
fn validate(&self, bytes: &[u8]) -> Result<()> {
simdutf8::basic::from_utf8(bytes).map_err(crate::Error::Utf8)?;
Ok(())
}
fn parse_available(
&mut self,
bytes: &[u8],
_eof: bool,
emit: &mut dyn FnMut(String),
_diag: &dyn ParseDiagnostics,
) -> Result<usize> {
let mut consumed = 0usize;
for (idx, &b) in bytes.iter().enumerate() {
if b == b'\n' {
let line = std::str::from_utf8(&bytes[consumed..idx]).unwrap_or("");
if !line.is_empty() {
emit(line.to_string());
}
consumed = idx + 1;
}
}
Ok(consumed)
}
}
#[test]
fn emits_records_across_chunks_and_drops_the_trailing_partial() {
let mut stream = RecordStream::new(LineParser);
let diag = NoopDiagnostics;
let mut rows: Vec<String> = Vec::new();
let data = b"A=1 B=2\nA=3 B=4\nA=5";
stream.feed(&data[..10]).unwrap();
assert_eq!(
stream.parse(&mut |r| rows.push(r), &diag).unwrap(),
StreamState::Rows
);
assert_eq!(stream.pending(), 2);
stream.feed(&data[10..]).unwrap();
assert_eq!(
stream.parse(&mut |r| rows.push(r), &diag).unwrap(),
StreamState::Rows
);
assert_eq!(stream.pending(), 3);
stream.finish();
assert_eq!(
stream.parse(&mut |r| rows.push(r), &diag).unwrap(),
StreamState::Eof
);
assert_eq!(rows, vec!["A=1 B=2".to_string(), "A=3 B=4".to_string()]);
}
#[test]
fn empty_feed_is_eof() {
let mut stream = RecordStream::new(LineParser);
stream.finish();
let mut n = 0;
assert_eq!(
stream.parse(&mut |_| n += 1, &NoopDiagnostics).unwrap(),
StreamState::Eof
);
assert_eq!(n, 0);
}
#[test]
fn invalid_utf8_is_rejected_on_feed() {
let mut stream = RecordStream::new(LineParser);
assert!(stream.feed(b"\xff\n").is_err());
}
}