use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use crate::error::{Error, Result};
use crate::samples::Samples;
use crate::wfdisc::{CSS_LINE_LEN, NNSA_LINE_LEN, Wfdisc};
use crate::DEFAULT_MAX_SAMPLE_BYTES;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Schema {
Css30,
NnsaKbCore,
Loose,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct Positions {
pub(crate) line_len: usize,
pub(crate) sta: (usize, usize),
pub(crate) chan: (usize, usize),
pub(crate) time: (usize, usize),
pub(crate) wfid: (usize, usize),
pub(crate) chanid: (usize, usize),
pub(crate) jdate: (usize, usize),
pub(crate) endtime: (usize, usize),
pub(crate) nsamp: (usize, usize),
pub(crate) samprate: (usize, usize),
pub(crate) calib: (usize, usize),
pub(crate) calper: (usize, usize),
pub(crate) instype: (usize, usize),
pub(crate) segtype: (usize, usize),
pub(crate) datatype: (usize, usize),
pub(crate) clip: (usize, usize),
pub(crate) dir: (usize, usize),
pub(crate) dfile: (usize, usize),
pub(crate) foff: (usize, usize),
pub(crate) commid: (usize, usize),
pub(crate) lddate: (usize, usize),
}
const CSS_POSITIONS: Positions = Positions {
line_len: CSS_LINE_LEN,
sta: (0, 6),
chan: (7, 15),
time: (16, 33),
wfid: (34, 42),
chanid: (43, 51),
jdate: (52, 60),
endtime: (61, 78),
nsamp: (79, 87),
samprate: (88, 99),
calib: (100, 116),
calper: (117, 133),
instype: (134, 140),
segtype: (141, 142),
datatype: (143, 145),
clip: (146, 147),
dir: (148, 212),
dfile: (213, 245),
foff: (246, 256),
commid: (257, 265),
lddate: (266, 283),
};
const NNSA_POSITIONS: Positions = Positions {
line_len: NNSA_LINE_LEN,
sta: (0, 6),
chan: (7, 15),
time: (16, 33),
wfid: (34, 43),
chanid: (44, 53),
jdate: (54, 62),
endtime: (62, 79),
nsamp: (80, 88),
samprate: (89, 100),
calib: (101, 117),
calper: (118, 134),
instype: (135, 141),
segtype: (142, 143),
datatype: (144, 146),
clip: (147, 148),
dir: (149, 213),
dfile: (214, 246),
foff: (247, 257),
commid: (258, 267),
lddate: (268, 287),
};
impl Schema {
pub fn detect(line_len: usize) -> Option<Self> {
match line_len {
CSS_LINE_LEN => Some(Schema::Css30),
NNSA_LINE_LEN => Some(Schema::NnsaKbCore),
_ => None,
}
}
pub const fn line_len(self) -> Option<usize> {
match self {
Schema::Css30 => Some(CSS_LINE_LEN),
Schema::NnsaKbCore => Some(NNSA_LINE_LEN),
Schema::Loose => None,
}
}
pub(crate) fn positions(self) -> Option<Positions> {
match self {
Schema::Css30 => Some(CSS_POSITIONS),
Schema::NnsaKbCore => Some(NNSA_POSITIONS),
Schema::Loose => None,
}
}
}
#[derive(Debug)]
pub struct WfdiscReader<R: BufRead> {
inner: R,
base_dir: Option<PathBuf>,
locked_schema: Option<Schema>,
line_buf: String,
eof: bool,
lenient: bool,
max_sample_bytes: usize,
}
impl WfdiscReader<BufReader<File>> {
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref();
let file = File::open(path)?;
let base_dir = path.parent().map(|p| p.to_path_buf());
Ok(WfdiscReader {
inner: BufReader::new(file),
base_dir,
locked_schema: None,
line_buf: String::with_capacity(NNSA_LINE_LEN + 2),
eof: false,
lenient: true,
max_sample_bytes: DEFAULT_MAX_SAMPLE_BYTES,
})
}
}
impl<R: BufRead> WfdiscReader<R> {
pub fn new(reader: R) -> Self {
WfdiscReader {
inner: reader,
base_dir: None,
locked_schema: None,
line_buf: String::new(),
eof: false,
lenient: true,
max_sample_bytes: DEFAULT_MAX_SAMPLE_BYTES,
}
}
pub fn new_with_base(reader: R, base_dir: impl AsRef<Path>) -> Self {
WfdiscReader {
inner: reader,
base_dir: Some(base_dir.as_ref().to_path_buf()),
locked_schema: None,
line_buf: String::new(),
eof: false,
lenient: true,
max_sample_bytes: DEFAULT_MAX_SAMPLE_BYTES,
}
}
#[must_use]
pub fn max_sample_bytes(mut self, max: usize) -> Self {
self.max_sample_bytes = max;
self
}
pub fn read_samples(&self, row: &Wfdisc) -> Result<Samples> {
let base = self
.base_dir
.as_deref()
.unwrap_or_else(|| Path::new("."));
row.read_samples_with_limit(base, self.max_sample_bytes)
}
pub fn with_schema(mut self, schema: Schema) -> Self {
self.locked_schema = Some(schema);
self
}
pub fn strict(mut self) -> Self {
self.lenient = false;
self
}
pub fn is_lenient(&self) -> bool {
self.lenient
}
pub fn base_dir(&self) -> Option<&Path> {
self.base_dir.as_deref()
}
pub fn schema(&self) -> Option<Schema> {
self.locked_schema
}
fn read_line(&mut self) -> Result<Option<&str>> {
loop {
self.line_buf.clear();
let n = self.inner.read_line(&mut self.line_buf)?;
if n == 0 {
self.eof = true;
return Ok(None);
}
let trimmed = self
.line_buf
.trim_end_matches(['\r', '\n'])
.trim_end_matches(' ')
.len();
if trimmed == 0 {
continue;
}
return Ok(Some(self.line_buf.trim_end_matches(['\r', '\n'])));
}
}
}
impl WfdiscReader<BufReader<File>> {
pub fn read_all(self) -> Result<Vec<Wfdisc>> {
self.collect()
}
}
impl<R: BufRead> Iterator for WfdiscReader<R> {
type Item = Result<Wfdisc>;
fn next(&mut self) -> Option<Self::Item> {
loop {
if self.eof {
return None;
}
let line = match self.read_line() {
Ok(Some(l)) => l.to_string(),
Ok(None) => return None,
Err(e) => return Some(Err(e)),
};
if let Some(schema) = self.locked_schema {
return Some(match schema {
Schema::Loose => {
let token_count = line.split_whitespace().count();
if token_count < 20 {
if self.lenient {
continue;
}
return Some(Err(Error::InvalidLineLength { got: line.len() }));
}
Wfdisc::parse_loose(&line)
}
Schema::Css30 | Schema::NnsaKbCore => {
let positions = schema.positions().expect("fixed-width schema");
if line.len() == positions.line_len {
Wfdisc::parse_with(&line, schema)
} else {
Err(Error::MixedSchemaInLockedMode {
got: line.len(),
schema,
expected: positions.line_len,
})
}
}
});
}
if let Some(schema) = Schema::detect(line.len()) {
self.locked_schema = Some(schema);
return Some(Wfdisc::parse_with(&line, schema));
}
if self.lenient {
let token_count = line.split_whitespace().count();
if token_count < 20 {
continue;
}
if self.locked_schema.is_none() {
self.locked_schema = Some(Schema::Loose);
}
return Some(Wfdisc::parse_loose(&line));
}
return Some(Err(Error::InvalidLineLength { got: line.len() }));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::DataType;
use std::io::Cursor;
const CSS_BYTES: &[u8] = include_bytes!("../tests/data/test_css.wfdisc");
const NNSA_BYTES: &[u8] = include_bytes!("../tests/data/test_nnsa.wfdisc");
#[test]
fn iter_yields_six_rows_for_test_css() {
let r = WfdiscReader::new(BufReader::new(Cursor::new(CSS_BYTES)));
let rows: Vec<_> = r.collect::<Result<_>>().unwrap();
assert_eq!(rows.len(), 6);
assert_eq!(rows[0].schema, Schema::Css30);
}
#[test]
fn iter_yields_six_rows_for_test_nnsa() {
let r = WfdiscReader::new(BufReader::new(Cursor::new(NNSA_BYTES)));
let rows: Vec<_> = r.collect::<Result<_>>().unwrap();
assert_eq!(rows.len(), 6);
assert_eq!(rows[0].schema, Schema::NnsaKbCore);
assert_eq!(rows[5].chan, "HHN");
assert_eq!(rows[5].datatype, DataType::I4);
}
#[test]
fn schema_detect_recognises_known_lengths() {
assert_eq!(Schema::detect(283), Some(Schema::Css30));
assert_eq!(Schema::detect(287), Some(Schema::NnsaKbCore));
assert_eq!(Schema::detect(100), None);
}
#[test]
fn schema_line_len() {
assert_eq!(Schema::Css30.line_len(), Some(283));
assert_eq!(Schema::NnsaKbCore.line_len(), Some(287));
assert_eq!(Schema::Loose.line_len(), None);
}
#[test]
fn open_records_base_dir() {
let path = Path::new("tests/data/test_css.wfdisc");
let r = WfdiscReader::open(path).unwrap();
assert_eq!(r.base_dir(), Some(Path::new("tests/data")));
}
#[test]
fn skips_blank_lines() {
let mut bytes: Vec<u8> = Vec::new();
bytes.extend_from_slice(CSS_BYTES);
bytes.extend_from_slice(b"\n\n \n");
let r = WfdiscReader::new(BufReader::new(Cursor::new(bytes)));
let rows: Vec<_> = r.collect::<Result<_>>().unwrap();
assert_eq!(rows.len(), 6);
}
#[test]
fn malformed_line_surfaces_invalid_length_in_strict_mode() {
let bytes = b"too short\n";
let r = WfdiscReader::new(BufReader::new(Cursor::new(&bytes[..]))).strict();
let err = r.collect::<Result<Vec<_>>>().unwrap_err();
assert!(matches!(err, Error::InvalidLineLength { .. }));
}
#[test]
fn lenient_default_skips_garbage_lines() {
let mut bytes: Vec<u8> = Vec::new();
bytes.extend_from_slice(b"USER@db > spool off;\n");
bytes.extend_from_slice(CSS_BYTES);
let r = WfdiscReader::new(BufReader::new(Cursor::new(bytes)));
let rows: Vec<_> = r.collect::<std::result::Result<Vec<_>, _>>().unwrap();
assert_eq!(rows.len(), 6);
}
const COMPACT_LINE: &str = "TEST01 BHZ 1000000000.00000 -1 -1 2001252 1000000009.99000 1000 100.0 1.0 1.0 - - s4 - . synthetic.w 0 -1 1000000000.00000\n";
#[test]
fn lenient_recovers_non_canonical_compact_line() {
let r = WfdiscReader::new(BufReader::new(Cursor::new(COMPACT_LINE.as_bytes())));
let rows: Vec<_> = r.collect::<std::result::Result<Vec<_>, _>>().unwrap();
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].schema, Schema::Loose);
assert_eq!(rows[0].sta, "TEST01");
assert_eq!(rows[0].chan, "BHZ");
assert_eq!(rows[0].nsamp, 1000);
assert_eq!(rows[0].datatype, crate::DataType::S4);
assert_eq!(rows[0].dfile, "synthetic.w");
assert_eq!(rows[0].foff, 0);
}
#[test]
fn strict_mode_rejects_non_canonical_line() {
let line = "TEST01 BHZ 1000000000.0 -1 -1 2001252 1000000009.99 1000 100.0 1.0 1.0 - - s4 - . synthetic.w 0 -1 1000000000.0\n";
let r = WfdiscReader::new(BufReader::new(Cursor::new(line.as_bytes()))).strict();
let err = r.collect::<Result<Vec<_>>>().unwrap_err();
assert!(matches!(err, Error::InvalidLineLength { .. }));
}
#[test]
fn locked_schema_overrides_autodetect() {
let r = WfdiscReader::new(BufReader::new(Cursor::new(CSS_BYTES)))
.with_schema(Schema::Css30);
let rows: Vec<_> = r.collect::<Result<_>>().unwrap();
assert_eq!(rows.len(), 6);
}
#[test]
fn locked_loose_schema_parses_fixed_width_lines_as_loose() {
let r = WfdiscReader::new(BufReader::new(Cursor::new(CSS_BYTES)))
.with_schema(Schema::Loose);
let rows: Vec<_> = r.collect::<Result<_>>().unwrap();
assert_eq!(rows.len(), 6);
assert!(rows.iter().all(|r| r.schema == Schema::Loose));
}
#[test]
fn locked_css_schema_rejects_mismatched_length_in_lenient_mode() {
let r = WfdiscReader::new(BufReader::new(Cursor::new(COMPACT_LINE.as_bytes())))
.with_schema(Schema::Css30);
let err = r.collect::<Result<Vec<_>>>().unwrap_err();
assert!(matches!(
err,
Error::MixedSchemaInLockedMode {
schema: Schema::Css30,
expected: 283,
..
}
));
}
}