use super::{Failure, annotated, failure, options::Options, os_failure, records, standard_io};
use std::io::{self, BufRead, Write};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum Filter {
None,
Comments,
Vnlog,
}
impl Filter {
pub(super) fn of(options: &Options) -> Self {
if options.vnlog {
Self::Vnlog
} else if options.skip_comments {
Self::Comments
} else {
Self::None
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum Header {
None,
First,
}
impl Header {
pub(super) fn of(options: &Options) -> Self {
if options.header_in {
Self::First
} else {
Self::None
}
}
}
pub(super) struct Record<'b> {
bytes: &'b [u8],
data: usize,
}
impl<'b> Record<'b> {
pub(super) fn raw(&self) -> &'b [u8] {
self.bytes
}
pub(super) fn data(&self) -> &'b [u8] {
&self.bytes[..self.data]
}
}
pub(super) enum Accepted {
Skipped,
Header,
Data(usize),
}
pub(super) struct Intake<'p> {
terminator: u8,
filter: Filter,
header_pending: bool,
lines: u64,
ended: bool,
error: Option<io::Error>,
warning: Option<&'p [u8]>,
}
impl<'p> Intake<'p> {
pub(super) fn new(options: &Options, header: Header) -> Self {
Self {
terminator: options.record_end,
filter: Filter::of(options),
header_pending: header == Header::First,
lines: 0,
ended: false,
error: None,
warning: None,
}
}
pub(super) fn with_filter(mut self, filter: Filter) -> Self {
self.filter = filter;
self
}
pub(super) fn after(mut self, lines: u64) -> Self {
self.lines = lines;
self
}
pub(super) fn warn_full(mut self, options: &Options, program: &'p [u8]) -> Self {
if options.warns_full() {
self.warning = Some(program);
if !self.header_pending {
self.warn();
}
}
self
}
pub(super) fn line(&self) -> u64 {
self.lines
}
pub(super) fn read_error(&self) -> Option<&io::Error> {
self.error.as_ref()
}
pub(super) fn header<R: BufRead + ?Sized>(
&mut self,
reader: &mut R,
buffer: &mut Vec<u8>,
resolve: impl FnOnce(&[u8]) -> Result<(), Failure>,
) -> Result<bool, Failure> {
self.header_captured(reader, buffer, |_| (), |bytes, ()| resolve(bytes))
}
pub(super) fn header_captured<R: BufRead + ?Sized, C>(
&mut self,
reader: &mut R,
buffer: &mut Vec<u8>,
mut capture: impl FnMut(&[u8]) -> C,
resolve: impl FnOnce(&[u8], C) -> Result<(), Failure>,
) -> Result<bool, Failure> {
while self.header_pending {
if !self.read(reader, buffer)? {
return Ok(false);
}
let captured = capture(buffer);
if let Accepted::Header = self.accept(buffer)? {
resolve(buffer, captured)?;
self.warn();
return Ok(true);
}
}
Ok(false)
}
#[inline]
pub(super) fn next<'b, R: BufRead + ?Sized>(
&mut self,
reader: &mut R,
buffer: &'b mut Vec<u8>,
) -> Result<Option<Record<'b>>, Failure> {
let data = loop {
if !self.read(reader, buffer)? {
return Ok(None);
}
match self.accept(buffer)? {
Accepted::Data(data) => break data,
Accepted::Skipped => {}
Accepted::Header => unreachable!("the Input header is read by `header`"),
}
};
Ok(Some(Record {
bytes: buffer,
data,
}))
}
#[inline]
pub(super) fn accept(&mut self, buffer: &mut Vec<u8>) -> Result<Accepted, Failure> {
if self.header_pending {
return self.accept_header(buffer);
}
self.accept_data(buffer)
}
fn accept_data(&mut self, buffer: &[u8]) -> Result<Accepted, Failure> {
let data = match self.filter {
Filter::None => buffer.len(),
Filter::Comments if records::is_comment(buffer) => return Ok(Accepted::Skipped),
Filter::Comments => buffer.len(),
Filter::Vnlog if annotated::skip_data(buffer) => return Ok(Accepted::Skipped),
Filter::Vnlog => annotated::data(buffer).len(),
};
self.count()?;
Ok(Accepted::Data(data))
}
pub(super) fn for_each<R: BufRead + ?Sized>(
&mut self,
reader: &mut R,
buffer: &mut Vec<u8>,
mut visit: impl FnMut(Record<'_>, u64) -> Result<(), Failure>,
) -> Result<(), Failure> {
loop {
if self.ended {
self.warn();
return Ok(());
}
let available = match reader.fill_buf() {
Ok(bytes) => bytes,
Err(error) => {
self.ended = true;
self.error = Some(error);
return Ok(());
}
};
if available.is_empty() {
self.ended = true;
return Ok(());
}
let mut consumed = 0;
while let Some(end) = memchr::memchr(self.terminator, &available[consumed..]) {
let bytes = &available[consumed..consumed + end];
if let Accepted::Data(data) = self.accept_data(bytes)? {
visit(Record { bytes, data }, self.lines)?;
}
consumed += end + 1;
}
if consumed != 0 {
reader.consume(consumed);
} else if let Some(record) = self.next(reader, buffer)? {
visit(record, self.lines)?;
} else {
return Ok(());
}
}
}
#[cold]
fn accept_header(&mut self, buffer: &mut Vec<u8>) -> Result<Accepted, Failure> {
let accepted = match self.filter {
Filter::None => true,
Filter::Comments => !records::is_comment(buffer),
Filter::Vnlog => annotated::header(buffer)?,
};
if !accepted {
return Ok(Accepted::Skipped);
}
self.count()?;
self.header_pending = false;
Ok(Accepted::Header)
}
#[inline]
fn count(&mut self) -> Result<(), Failure> {
self.lines = self
.lines
.checked_add(1)
.ok_or_else(|| super::unsupported("record count exceeds u64 limit"))?;
Ok(())
}
#[inline]
fn read<R: BufRead + ?Sized>(
&mut self,
reader: &mut R,
buffer: &mut Vec<u8>,
) -> Result<bool, Failure> {
if self.ended {
self.warn();
return Ok(false);
}
match records::read_record_terminated(reader, buffer, self.terminator) {
Ok(true) => Ok(true),
Ok(false) => {
self.ended = true;
Ok(false)
}
Err(records::ReadError::Io(error)) => {
self.ended = true;
self.error = Some(error);
Ok(false)
}
Err(records::ReadError::Allocation) => {
Err(super::unsupported("record memory allocation failed"))
}
}
}
#[cold]
fn warn(&mut self) {
if let Some(program) = self.warning.take() {
write_full_warning(program);
}
}
pub(super) fn finish(self) -> Result<(), Failure> {
self.error.map_or(Ok(()), |error| Err(read_failure(&error)))
}
}
#[cold]
pub(super) fn write_full_warning(program: &[u8]) {
let mut stderr = standard_io::Stderr;
let _ = stderr.write_all(program).and_then(|()| {
stderr.write_all(b": Using -f/--full with non-linewise operations is deprecated and will be disabled in a future release.\n")
});
}
pub(super) fn read_failure(error: &io::Error) -> Failure {
if error.raw_os_error() == Some(5) {
failure(b"read error: Input/output error\n".to_vec())
} else {
os_failure(error, true)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::options;
fn options(args: &[&str]) -> Options {
let args: Vec<std::ffi::OsString> = args.iter().map(Into::into).collect();
match options::parse(&args, b"fastmash", false) {
Ok(options::Action::Calculate(options)) => *options,
_ => panic!("calculation options"),
}
}
type Row = (Vec<u8>, Vec<u8>, u64);
fn take(options: &Options, header: Header, input: &[u8]) -> (Option<Vec<u8>>, Vec<Row>) {
let mut reader = input;
let mut intake = Intake::new(options, header);
let mut buffer = Vec::new();
let found = intake
.header(&mut reader, &mut buffer, |_| Ok(()))
.ok()
.unwrap()
.then(|| buffer.clone());
let mut rows = Vec::new();
while let Some(record) = intake.next(&mut reader, &mut buffer).ok().unwrap() {
rows.push((record.raw().to_vec(), record.data().to_vec(), intake.line()));
}
(found, rows)
}
#[test]
fn filters_skip_records_and_lines_count_only_accepted_ones() {
let input = b"h\n#c\n ;d\n1 # x\n\n";
let (header, rows) = take(&options(&["-C", "count", "1"]), Header::First, input);
assert_eq!(header.as_deref(), Some(b"h".as_slice()));
assert_eq!(
rows,
[
(b"1 # x".to_vec(), b"1 # x".to_vec(), 2),
(b"".to_vec(), b"".to_vec(), 3)
]
);
let (header, rows) = take(&options(&["count", "1"]), Header::None, b"#c\n;d\n");
assert_eq!(header, None);
assert_eq!(rows.len(), 2);
assert_eq!(rows[1].2, 2);
}
#[test]
fn vnlog_header_is_stripped_and_data_views_are_raw_prefixes() {
let input = b"## meta\n\n#!x\n# a b \n1 2 # note\n#c\n;3 4\n";
let (header, rows) = take(&options(&["--vnlog", "count", "a"]), Header::First, input);
assert_eq!(header.as_deref(), Some(b"a b".as_slice()));
assert_eq!(
rows,
[
(b"1 2 # note".to_vec(), b"1 2".to_vec(), 2),
(b";3 4".to_vec(), b";3 4".to_vec(), 3)
]
);
let (_, rows) = take(&options(&["--vnlog", "count", "a"]), Header::None, input);
assert_eq!(rows.len(), 2);
let mut intake = Intake::new(&options(&["--vnlog", "count", "a"]), Header::First);
let error = intake
.header(&mut b"1 2\n".as_slice(), &mut Vec::new(), |_| Ok(()))
.err()
.unwrap();
assert_eq!(
error.message,
b"invalid vnlog data: received record before header: '1 2'\n"
);
}
#[test]
fn pushed_records_follow_the_same_policy() {
let options = options(&["--vnlog", "count", "a"]);
let mut intake = Intake::new(&options, Header::First).after(1);
let mut kinds = Vec::new();
for record in [b"#late".as_slice(), b"# p q", b"#x", b"a 1 #n"] {
let mut buffer = record.to_vec();
kinds.push(match intake.accept(&mut buffer).ok().unwrap() {
Accepted::Skipped => "skipped".to_owned(),
Accepted::Header => format!("header {}", String::from_utf8_lossy(&buffer)),
Accepted::Data(data) => {
format!("data {}", String::from_utf8_lossy(&buffer[..data]))
}
});
}
assert_eq!(kinds, ["header late", "skipped", "skipped", "data a 1"]);
assert_eq!(intake.line(), 3);
}
#[test]
fn visiting_records_preserves_headers_views_and_buffer_boundaries() {
for capacity in [1, 4, 128] {
let options = options(&["--vnlog", "count", "h"]);
let mut intake = Intake::new(&options, Header::First);
let mut reader =
io::BufReader::with_capacity(capacity, b"# h\n1#x\n#c\n123456789#n\n4".as_slice());
let mut buffer = Vec::new();
assert!(
intake
.header(&mut reader, &mut buffer, |header| {
assert_eq!(header, b"h");
Ok(())
})
.ok()
.unwrap()
);
let mut rows = Vec::new();
intake
.for_each(&mut reader, &mut buffer, |record, line| {
rows.push((record.raw().to_vec(), record.data().to_vec(), line));
Ok(())
})
.ok()
.unwrap();
assert_eq!(
rows,
[
(b"1#x".to_vec(), b"1".to_vec(), 2),
(b"123456789#n".to_vec(), b"123456789".to_vec(), 3),
(b"4".to_vec(), b"4".to_vec(), 4),
]
);
assert!(intake.finish().is_ok());
}
}
#[test]
fn visiting_nul_records_preserves_empty_and_unterminated_records() {
for capacity in [1, 4, 128] {
let options = options(&["-z", "count", "1"]);
let mut intake = Intake::new(&options, Header::None);
let mut reader = io::BufReader::with_capacity(capacity, b"1\nx\0\0last".as_slice());
let mut rows = Vec::new();
intake
.for_each(&mut reader, &mut Vec::new(), |record, line| {
rows.push((record.raw().to_vec(), line));
Ok(())
})
.ok()
.unwrap();
assert_eq!(
rows,
[
(b"1\nx".to_vec(), 1),
(Vec::new(), 2),
(b"last".to_vec(), 3)
]
);
assert!(intake.finish().is_ok());
}
}
#[test]
fn visiting_records_stops_at_the_first_eof() {
struct OneEof(bool);
impl io::Read for OneEof {
fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {
assert!(!self.0, "read after EOF");
self.0 = true;
Ok(0)
}
}
let options = options(&["count", "1"]);
let mut intake = Intake::new(&options, Header::None);
let mut reader = io::BufReader::new(OneEof(false));
let mut buffer = Vec::new();
for _ in 0..2 {
intake
.for_each(&mut reader, &mut buffer, |_, _| panic!("no data"))
.ok()
.unwrap();
}
assert!(intake.finish().is_ok());
}
struct Failing(u8);
impl io::Read for Failing {
fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {
unreachable!()
}
}
impl BufRead for Failing {
fn fill_buf(&mut self) -> io::Result<&[u8]> {
self.0 += 1;
match self.0 {
1 => Ok(b"a
partial"),
2 => Err(io::Error::from_raw_os_error(5)),
_ => panic!("read after the input ended"),
}
}
fn consume(&mut self, amount: usize) {
assert_eq!(amount, 2);
}
}
#[test]
fn visiting_records_defers_read_errors_and_does_not_read_after_failure() {
let options = options(&["count", "1"]);
let mut intake = Intake::new(&options, Header::None);
let mut reader = Failing(0);
let mut buffer = Vec::new();
let mut rows = Vec::new();
intake
.for_each(&mut reader, &mut buffer, |record, line| {
rows.push((record.raw().to_vec(), line));
Ok(())
})
.ok()
.unwrap();
assert_eq!(rows, [(b"a".to_vec(), 1)]);
intake
.for_each(&mut reader, &mut buffer, |_, _| panic!("input ended"))
.ok()
.unwrap();
let error = intake.finish().err().unwrap();
assert_eq!(error.status, 1);
assert_eq!(error.message, b"read error: Input/output error\n");
}
#[test]
fn a_read_error_ends_the_input_once_and_is_reported_last() {
let options = options(&["count", "1"]);
let mut intake = Intake::new(&options, Header::None);
let mut reader = Failing(0);
let mut buffer = Vec::new();
assert_eq!(
intake
.next(&mut reader, &mut buffer)
.ok()
.unwrap()
.map(|r| r.raw()),
Some(b"a".as_slice())
);
assert!(
intake
.next(&mut reader, &mut buffer)
.ok()
.unwrap()
.is_none()
);
assert!(
intake
.next(&mut reader, &mut buffer)
.ok()
.unwrap()
.is_none()
);
assert_eq!(
intake.read_error().and_then(io::Error::raw_os_error),
Some(5)
);
let error = intake.finish().err().unwrap();
assert_eq!(
(error.status, error.message),
(1, b"read error: Input/output error\n".to_vec())
);
}
}