Struct cdx::prelude::TextFileMode

pub struct TextFileMode {
    pub cdx: bool,
    pub head_mode: HeadMode,
    pub col_mode: ColumnMode,
    pub delim: u8,
    pub line_break: u8,
}

how to pase a text file into lines and columns

Fields

cdx: bool

does it have a cdx header

head_mode: HeadMode

yes, no, maybe

col_mode: ColumnMode

plain, quote, backslash

delim: u8

column delimiter

line_break: u8

line delimiter

Implementations

impl TextFileMode

pub fn new(spec: &str) -> Result<Self>

new from spec

Examples found in repository

src/bin/cdx/globals.rs (line 75)

≺ ≻↕

    pub fn consume(&mut self, args: &[ArgValue]) -> Result<()> {
        for x in args {
            if x.name == "header" {
                self.checker.mode = HeaderMode::from_str(&x.value)?;
            } else if x.name == "text-in" {
                self.text_in = TextFileMode::new(&x.value)?;
            } else if x.name == "text-out" {
                self.text_out = Some(TextFileMode::new(&x.value)?);
            } else if x.name == "std-agg" {
                AggMaker::help();
                return cdx_err(CdxError::NoError);
            } else if x.name == "std-comp" {
                CompMaker::help();
                return cdx_err(CdxError::NoError);
            } else if x.name == "std-const" {
                expr::show_const();
                return cdx_err(CdxError::NoError);
            } else if x.name == "std-func" {
                expr::show_func();
                return cdx_err(CdxError::NoError);
            } else if x.name == "std-gen" {
                GenMaker::help();
                return cdx_err(CdxError::NoError);
            } else if x.name == "std-match" {
                MatchMaker::help();
                return cdx_err(CdxError::NoError);
            } else if x.name == "std-trans" {
                TransMaker::help();
                return cdx_err(CdxError::NoError);
            } else if x.name == "foo" {
                eprintln!("Something with side effect");
            } else {
                unreachable!();
            }
        }
        Ok(())
    }

pub fn new_with(spec: &str, dflt: &Self) -> Result<Self>

new from spec, with defaults

Examples found in repository

src/util.rs (line 209)

    pub fn new(spec: &str) -> Result<Self> {
        Self::new_with(spec, &Self::default())
    }

pub fn split(&self, line: &mut TextLine)

split data line into columns

Examples found in repository

src/util.rs (line 1354)

↕

    pub fn getline(&mut self) -> Result<bool> {
        self.loc.bytes += self.curr().line.len();
        self.incr();
        if self
            .cont
            .text
            .read_line(&mut *self.file, &mut self.lines[self.curr].line)?
        {
            self.cont.is_done = true;
        } else if self.do_split {
            self.cont.text.split(&mut self.lines[self.curr]);
        }
        Ok(self.cont.is_done)
    }

pub fn split_head(&self, line: &mut StringLine)

split data line into columns

Examples found in repository

src/util.rs (line 1049)

↕

    fn read_header(&mut self, file: &mut impl BufRead, line: &mut TextLine) -> Result<()> {
        self.is_empty = self.text.read_header(file, &mut self.header.line)?;
        if self.is_empty {
            return Ok(());
        }
        self.has_header = !self.header.line.is_empty();
        if self.has_header {
            self.text.split_head(&mut self.header);
        }
        self.is_done = self.text.read_line(file, &mut line.line)?;
        if self.is_done {
            return Ok(());
        }
        line.split(self.text.delim);

        if !self.has_header {
            let mut head_str = String::new();
            for i in 1..=line.len() {
                head_str += "c";
                head_str += &i.to_string();
                head_str += "\t";
            }
            head_str.pop();
            head_str += "\n";
            let mut fake_head = head_str.as_bytes();
            self.header.read(&mut fake_head)?;
            self.header.split(self.text.delim);
        }
        Ok(())
    }

pub fn ensure_eof(&self, line: &mut Vec<u8>) -> Result<bool>

append a newline, if line does not already end with one. Return false.

Examples found in repository

src/util.rs (line 348)

≺ ≻↕

    pub fn read_line<T: BufRead>(&self, f: &mut T, line: &mut Vec<u8>) -> Result<bool> {
        line.clear();
        let sz = f.read_until(self.line_break, line)?;
        if sz == 0 {
            return Ok(true);
        }
        if self.col_mode != ColumnMode::Quote {
            return self.ensure_eof(line);
        }
        let mut within = false;
        let mut skip = 0;
        loop {
            for ch in line.iter().skip(skip) {
                if *ch == b'"' {
                    within = !within;
                }
            }
            skip = line.len();
            if !within {
                return self.ensure_eof(line);
            }
            let sz = f.read_until(self.line_break, line)?;
            if sz == 0 {
                return self.ensure_eof(line);
            }
        }
    }

pub fn read_string<T: BufRead>(
    &self,
    f: &mut T,
    line: &mut String
) -> Result<bool>

read a line, dealing with quotes if necessary, return as String

Examples found in repository

src/util.rs (line 320)

≺ ≻↕

    pub fn read_header<T: BufRead>(&self, f: &mut T, line: &mut String) -> Result<bool> {
        line.clear();
        let start = f.fill_buf()?;
        if start.is_empty() {
            return Ok(true);
        }
        match self.head_mode {
            HeadMode::Yes => self.read_string(f, line),
            HeadMode::No => Ok(false),
            HeadMode::Maybe => {
                if start.starts_with(b" CDX") {
                    self.read_string(f, line)
                } else {
                    Ok(false)
                }
            }
            HeadMode::Skip => {
                let start = f.fill_buf()?;
                if start.starts_with(b" CDX") {
                    self.read_string(f, line)?;
                    line.clear();
                }
                let start = f.fill_buf()?;
                Ok(start.is_empty())
            }
        }
    }

pub fn read_header<T: BufRead>(
    &self,
    f: &mut T,
    line: &mut String
) -> Result<bool>

read a line, dealing with quotes if necessary. Return true if file empty

Examples found in repository

src/util.rs (line 1043)

↕

    fn read_header(&mut self, file: &mut impl BufRead, line: &mut TextLine) -> Result<()> {
        self.is_empty = self.text.read_header(file, &mut self.header.line)?;
        if self.is_empty {
            return Ok(());
        }
        self.has_header = !self.header.line.is_empty();
        if self.has_header {
            self.text.split_head(&mut self.header);
        }
        self.is_done = self.text.read_line(file, &mut line.line)?;
        if self.is_done {
            return Ok(());
        }
        line.split(self.text.delim);

        if !self.has_header {
            let mut head_str = String::new();
            for i in 1..=line.len() {
                head_str += "c";
                head_str += &i.to_string();
                head_str += "\t";
            }
            head_str.pop();
            head_str += "\n";
            let mut fake_head = head_str.as_bytes();
            self.header.read(&mut fake_head)?;
            self.header.split(self.text.delim);
        }
        Ok(())
    }

pub fn read_line<T: BufRead>(
    &self,
    f: &mut T,
    line: &mut Vec<u8>
) -> Result<bool>

read a line, dealing with quotes if necessary. return true if no bytes read.

Examples found in repository

src/util.rs (line 306)

≺ ≻↕

    pub fn read_string<T: BufRead>(&self, f: &mut T, line: &mut String) -> Result<bool> {
        let mut x = Vec::new();
        if self.read_line(f, &mut x)? {
            return Ok(true);
        }
        *line = String::from_utf8(x)?;
        Ok(false)
    }
    /// read a line, dealing with quotes if necessary. Return true if file empty
    pub fn read_header<T: BufRead>(&self, f: &mut T, line: &mut String) -> Result<bool> {
        line.clear();
        let start = f.fill_buf()?;
        if start.is_empty() {
            return Ok(true);
        }
        match self.head_mode {
            HeadMode::Yes => self.read_string(f, line),
            HeadMode::No => Ok(false),
            HeadMode::Maybe => {
                if start.starts_with(b" CDX") {
                    self.read_string(f, line)
                } else {
                    Ok(false)
                }
            }
            HeadMode::Skip => {
                let start = f.fill_buf()?;
                if start.starts_with(b" CDX") {
                    self.read_string(f, line)?;
                    line.clear();
                }
                let start = f.fill_buf()?;
                Ok(start.is_empty())
            }
        }
    }
    /// read a line, dealing with quotes if necessary. return true if no bytes read.
    pub fn read_line<T: BufRead>(&self, f: &mut T, line: &mut Vec<u8>) -> Result<bool> {
        line.clear();
        let sz = f.read_until(self.line_break, line)?;
        if sz == 0 {
            return Ok(true);
        }
        if self.col_mode != ColumnMode::Quote {
            return self.ensure_eof(line);
        }
        let mut within = false;
        let mut skip = 0;
        loop {
            for ch in line.iter().skip(skip) {
                if *ch == b'"' {
                    within = !within;
                }
            }
            skip = line.len();
            if !within {
                return self.ensure_eof(line);
            }
            let sz = f.read_until(self.line_break, line)?;
            if sz == 0 {
                return self.ensure_eof(line);
            }
        }
    }
}

/*
CDX mode yes/no (cx)
if cdx mode, yes,no,maybe,skip (ynms)
else yes or no (yn)
delim : auto escape stnl
quoting : omit, quote, backslash (oqb)
line delimiter auto escape stnr, C for crlf

default is cmton
regular csv is xy,qC
regular tsv is xytbn
plain tsv xy
cdx.maybe.Ct.literal.Ln
--text-in
 */

/// pointers into a vector, simulating a slice without the ownership issues
#[derive(Debug, Clone, Copy, Default)]
pub struct FakeSlice {
    begin: u32,
    end: u32,
}
impl FakeSlice {
    /// new from text, like '7' or '3-6'
    pub fn new(spec: &str) -> Result<Self> {
        if let Some((a, b)) = spec.split_once('-') {
            let begin = a.to_usize_whole(spec.as_bytes(), "range")? as u32;
            let end = b.to_usize_whole(spec.as_bytes(), "range")? as u32;
            if begin == 0 || end == 0 {
                err!("Invalid range, both number must be greater than zero")
            } else if begin > end {
                err!("Invalid range, begin is greater than end")
            } else {
                Ok(Self {
                    begin: begin - 1,
                    end,
                })
            }
        } else {
            let num = spec.to_usize_whole(spec.as_bytes(), "position")? as u32;
            if num == 0 {
                err!("Invalid offset, must be greater than zero")
            } else {
                Ok(Self {
                    begin: num - 1,
                    end: num,
                })
            }
        }
    }
    /// get slice from FakeSlice
    pub fn get<'a>(&self, data: &'a [u8]) -> &'a [u8] {
        &data[self.begin as usize..self.end as usize]
    }
    /// get slice from FakeSlice, but don't fall off the end.
    pub fn get_safe<'a>(&self, data: &'a [u8]) -> &'a [u8] {
        &data[cmp::min(self.begin as usize, data.len())..cmp::min(self.end as usize, data.len())]
    }
    /// len
    pub const fn len(&self) -> usize {
        (self.end - self.begin) as usize
    }
    /// is empty?
    pub const fn is_empty(&self) -> bool {
        self.begin == self.end
    }
}

/// split a line by a delimiter. Remove trailing newline, if any.
pub fn split_plain(parts: &mut Vec<FakeSlice>, line: &[u8], delim: u8) {
    parts.clear();
    let mut begin: u32 = 0;
    let mut end: u32 = 0;
    #[allow(clippy::explicit_counter_loop)] // I need the counter to be u32
    for ch in line.iter() {
        if *ch == delim {
            parts.push(FakeSlice { begin, end });
            begin = end + 1;
        }
        end += 1;
    }
    if begin != end {
        let mut f = FakeSlice { begin, end };
        if line[(end - 1) as usize] == b'\n' {
            f.end -= 1;
        }
        parts.push(f);
    }
}

/// A line of a text file, broken into columns.
/// A line ends with a newline character, but column values do not.
/// An empty line contains one empty column
///```
/// use std::io::BufRead;
/// let mut data = b"one\ttwo\tthree\n";
/// let mut dp = &data[..];
/// let mut line = cdx::util::TextLine::new();
/// let eof = line.read(&mut dp).unwrap();
/// assert_eq!(eof, false);
/// assert_eq!(line.strlen(), 14);
/// line.split(b'\t');
/// assert_eq!(line.len(), 3);
/// assert_eq!(line.get(1), b"two");
///```
#[derive(Debug, Clone, Default)]
// pub(crate) because the borrow checker can be a nuisance.
pub struct TextLine {
    // The whole input line, with newline, decoded as necessary
    pub(crate) line: Vec<u8>,
    // the individual columns, without newline
    pub(crate) parts: Vec<FakeSlice>,
    // the original input line, if any decoding was necessary
    pub(crate) orig: Vec<u8>,
}

/// as TextLine, but [String] rather than `Vec<u8>`
#[derive(Debug, Clone, Default)]
pub struct StringLine {
    /// The whole input line, with newline
    pub line: String,
    /// the individual columns
    pub parts: Vec<FakeSlice>,
    // the original input line, if any decoding was necessary
    pub(crate) orig: String,
}

impl fmt::Display for TextLine {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        for i in self {
            write!(f, "{} ", str::from_utf8(i).unwrap())?;
        }
        Ok(())
    }
}

impl fmt::Display for StringLine {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        for i in self {
            write!(f, "{} ", i)?;
        }
        Ok(())
    }
}

impl std::ops::Index<usize> for TextLine {
    type Output = [u8];
    fn index(&self, pos: usize) -> &Self::Output {
        self.get(pos)
    }
}

impl std::ops::Index<usize> for StringLine {
    type Output = str;
    fn index(&self, pos: usize) -> &Self::Output {
        self.get(pos)
    }
}

/// write a buffer. If non-empty, ensure trailing newline
pub fn write_all_nl(w: &mut impl Write, buf: &[u8]) -> Result<()> {
    if !buf.is_empty() {
        w.write_all(buf)?;
        if buf[buf.len() - 1] != b'\n' {
            w.write_all(&[b'\n'])?;
        }
    }
    Ok(())
}

impl TextLine {
    /// whole line, with newline
    pub fn parts(&mut self) -> &mut Vec<FakeSlice> {
        &mut self.parts
    }
    /// whole line, with newline
    pub fn line(&self) -> &[u8] {
        &self.line
    }
    /// whole line, without newline
    pub fn line_nl(&self) -> &[u8] {
        &self.line[..self.line.len() - 1]
    }
    /// whole line, with newline, as Vec
    pub fn raw(&mut self) -> &mut Vec<u8> {
        &mut self.line
    }
    /// assign TextLine into existing TextLine, avoiding allocation if possible
    pub fn assign(&mut self, x: &Self) {
        self.line.clear();
        self.line.extend_from_slice(&x.line[..]);
        self.parts.clear();
        self.parts.extend_from_slice(&x.parts[..]);
    }
    /// make a new TextLine
    pub const fn new() -> Self {
        Self {
            line: Vec::new(),
            parts: Vec::new(),
            orig: Vec::new(),
        }
    }
    /// Iterator over columns in the line
    pub const fn iter(&self) -> TextLineIter<'_> {
        TextLineIter {
            line: self,
            index: 0,
        }
    }
    /// empty the line
    pub fn clear(&mut self) {
        self.parts.clear();
        self.line.clear();
    }
    /// How many column in the line
    pub fn len(&self) -> usize {
        self.parts.len()
    }
    /// How many bytes in the line
    pub fn strlen(&self) -> usize {
        self.line.len()
    }
    /// should always be false, but required by clippy
    pub fn is_empty(&self) -> bool {
        self.parts.is_empty()
    }
    /// Get one column. Return an empty column if index is too big.
    pub fn get(&self, index: usize) -> &[u8] {
        if index >= self.parts.len() {
            &self.line[0..0]
        } else {
            &self.line[self.parts[index].begin as usize..self.parts[index].end as usize]
        }
    }
    /// Read a new line from a file, should generally be followed by `split`
    pub fn read<T: BufRead>(&mut self, f: &mut T) -> Result<bool> {
        self.clear();
        let sz = f.read_until(b'\n', &mut self.line)?;
        if sz == 0 {
            Ok(true)
        } else {
            if self.line.last() != Some(&b'\n') {
                self.line.push(b'\n');
            }
            Ok(false)
        }
    }
    /// split the line into columns
    /// hypothetically you could split on one delimiter, do some work, then split on a different delimiter.
    pub fn split(&mut self, delim: u8) {
        split_plain(&mut self.parts, &self.line, delim);
    }
    /// return all parts as a vector
    pub fn vec(&self) -> Vec<&[u8]> {
        self.iter().collect()
    }
}

impl StringLine {
    /// make a new StringLine
    pub const fn new() -> Self {
        Self {
            line: String::new(),
            parts: Vec::new(),
            orig: String::new(),
        }
    }
    /// Iterator over columns in the line
    pub const fn iter(&self) -> StringLineIter<'_> {
        StringLineIter {
            line: self,
            index: 0,
        }
    }
    /// create a fake CDX header with columns c1,c2...
    pub fn fake(&mut self, num_cols: usize, delim: u8) {
        self.line = " CDX".to_string();
        for i in 1..=num_cols {
            self.line += std::str::from_utf8(&[delim]).unwrap();
            self.line += "c";
            self.line += &i.to_string();
        }
        self.line += "\n";
    }
    fn clear(&mut self) {
        self.parts.clear();
        self.line.clear();
    }
    /// How many column in the line
    pub fn len(&self) -> usize {
        self.parts.len()
    }
    /// How many bytes in the line
    pub fn strlen(&self) -> usize {
        self.line.len()
    }
    /// should always be false, but required by clippy
    pub fn is_empty(&self) -> bool {
        self.parts.is_empty()
    }
    /// Get one column. Return an empty column if index is too big.
    pub fn get(&self, index: usize) -> &str {
        if index >= self.parts.len() {
            &self.line[0..0]
        } else {
            &self.line[self.parts[index].begin as usize..self.parts[index].end as usize]
        }
    }
    /// Read a new line from a file, should generally be followed by `split`
    pub fn read<T: BufRead>(&mut self, f: &mut T) -> Result<bool> {
        self.clear();
        let sz = f.read_line(&mut self.line)?;
        if sz == 0 {
            Ok(true)
        } else {
            if self.line.as_bytes().last() != Some(&b'\n') {
                self.line.push('\n');
            }
            Ok(false)
        }
    }
    /// split the line into columns
    /// hypothetically you could split on one delimiter, do some work, then split on a different delimiter.
    pub fn split(&mut self, delim: u8) {
        split_plain(&mut self.parts, self.line.as_bytes(), delim);
    }
    /// return all parts as a vector
    pub fn vec(&self) -> Vec<&str> {
        self.iter().collect()
    }
}

impl<'a> IntoIterator for &'a TextLine {
    type Item = &'a [u8];
    type IntoIter = TextLineIter<'a>;

    fn into_iter(self) -> Self::IntoIter {
        TextLineIter {
            line: self,
            index: 0,
        }
    }
}

impl<'a> IntoIterator for &'a StringLine {
    type Item = &'a str;
    type IntoIter = StringLineIter<'a>;

    fn into_iter(self) -> Self::IntoIter {
        StringLineIter {
            line: self,
            index: 0,
        }
    }
}

/// Iterator over the columns in a TextLine
#[derive(Debug, Clone)]
pub struct TextLineIter<'a> {
    line: &'a TextLine,
    index: usize,
}

/// Iterator over the columns in a StringLine
#[derive(Debug, Clone)]
pub struct StringLineIter<'a> {
    line: &'a StringLine,
    index: usize,
}

impl<'a> Iterator for TextLineIter<'a> {
    // we will be counting with usize
    type Item = &'a [u8];

    fn next(&mut self) -> Option<Self::Item> {
        if self.index >= self.line.len() {
            None
        } else {
            self.index += 1;
            Some(&self.line[self.index - 1])
        }
    }
}

impl<'a> Iterator for StringLineIter<'a> {
    // we will be counting with usize
    type Item = &'a str;

    fn next(&mut self) -> Option<Self::Item> {
        if self.index >= self.line.parts.len() {
            None
        } else {
            self.index += 1;
            Some(&self.line[self.index - 1])
        }
    }
}

struct S3Reader {
    //    name : String,
    rt: tokio::runtime::Runtime,
    //    client : aws_sdk_s3::Client,
    f: aws_sdk_s3::output::GetObjectOutput,
    left: Option<bytes::Bytes>,
}
impl S3Reader {
    fn new(bucket: &str, key: &str) -> Result<Self> {
        let rt = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()?;
        let shared_config = rt.block_on(aws_config::load_from_env());
        let client = aws_sdk_s3::Client::new(&shared_config);
        let obj = rt.block_on(client.get_object().bucket(bucket).key(key).send())?;
        Ok(Self {
            //	    name : key.to_string(),
            rt,
            //	    client,
            f: obj,
            left: None,
        })
    }
    fn new_path(spec: &str) -> Result<Self> {
        if let Some(name) = spec.strip_prefix("s3://") {
            if let Some((a, b)) = name.split_once('/') {
                Self::new(a, b)
            } else {
                err!("Not an S3 file spec '{}'", spec)
            }
        } else {
            err!("Not an S3 file '{}'", spec)
        }
    }
}
impl Read for S3Reader {
    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
        if let Some(bytes) = &self.left {
            if bytes.len() > buf.len() {
                buf.clone_from_slice(&bytes[..buf.len()]);
                self.left = Some(bytes.slice(buf.len()..));
                return Ok(buf.len());
            } else {
                let len = bytes.len();
                buf[0..len].clone_from_slice(bytes);
                self.left = None;
                return Ok(len);
            }
        }
        let bytes_res = self.rt.block_on(self.f.body.try_next());
        if bytes_res.is_err() {
            return Err(std::io::Error::new(std::io::ErrorKind::Other, "oh no"));
        }
        self.left = bytes_res.unwrap();
        if let Some(bytes) = &self.left {
            if bytes.len() > buf.len() {
                buf.clone_from_slice(&bytes[..buf.len()]);
                self.left = Some(bytes.slice(buf.len()..));
                Ok(buf.len())
            } else {
                let len = bytes.len();
                buf[0..bytes.len()].clone_from_slice(bytes);
                self.left = None;
                Ok(len)
            }
        } else {
            Ok(0)
        }
    }
}

/// Input file. Wrapped in a type so I can 'impl Debug'
pub struct Infile(
    /// The file being read
    pub io::BufReader<Box<dyn Read>>,
    pub String,
);

impl Infile {
    /// create a new input file
    pub fn new(f: io::BufReader<Box<dyn Read>>, n: &str) -> Self {
        Self(f, n.to_string())
    }
}

impl Default for Infile {
    fn default() -> Self {
        Self::new(io::BufReader::new(Box::new(io::empty())), "")
    }
}

impl fmt::Debug for Infile {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "Infile : {}", self.1)
    }
}

impl Deref for Infile {
    type Target = io::BufReader<Box<dyn Read>>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl DerefMut for Infile {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.0
    }
}

impl AsRef<io::BufReader<Box<dyn Read>>> for Infile {
    fn as_ref(&self) -> &io::BufReader<Box<dyn Read>> {
        &self.0
    }
}

impl AsMut<io::BufReader<Box<dyn Read>>> for Infile {
    fn as_mut(&mut self) -> &mut io::BufReader<Box<dyn Read>> {
        &mut self.0
    }
}

/// output file type
pub struct Outfile(pub io::BufWriter<Box<dyn Write>>, pub String);

impl Outfile {
    /// create a new input file
    pub fn new(f: io::BufWriter<Box<dyn Write>>, n: &str) -> Self {
        Self(f, n.to_string())
    }
}

impl Default for Outfile {
    fn default() -> Self {
        Self::new(io::BufWriter::new(Box::new(io::sink())), "")
    }
}

impl fmt::Debug for Outfile {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "Outfile : {}", self.1)
    }
}

impl Deref for Outfile {
    type Target = io::BufWriter<Box<dyn Write>>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl DerefMut for Outfile {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.0
    }
}

impl AsRef<io::BufWriter<Box<dyn Write>>> for Outfile {
    fn as_ref(&self) -> &io::BufWriter<Box<dyn Write>> {
        &self.0
    }
}

impl AsMut<io::BufWriter<Box<dyn Write>>> for Outfile {
    fn as_mut(&mut self) -> &mut io::BufWriter<Box<dyn Write>> {
        &mut self.0
    }
}

/// Make an Outfile from a file name
pub fn get_writer(name: &str) -> Result<Outfile> {
    let inner: Box<dyn Write> = {
        if name == "-" {
            Box::new(io::stdout())
        } else if name == "--" {
            Box::new(io::stderr())
        } else {
            Box::new(fs::OpenOptions::new().write(true).create(true).open(name)?)
        }
    };
    Ok(Outfile::new(io::BufWriter::new(inner), name))
}

// should return Cow<>
fn unescape_vec(data: &[u8]) -> Vec<u8> {
    let mut ret: Vec<u8> = Vec::with_capacity(data.len());
    let mut last_was_slash = false;
    for x in data {
        if last_was_slash {
            ret.push(match x {
                b'n' => b'\n',
                b't' => b'\t',
                b's' => b' ',
                ch => *ch,
            });
            last_was_slash = false;
        } else if x == &b'\\' {
            last_was_slash = true;
        } else {
            ret.push(*x);
        }
    }
    if last_was_slash {
        ret.push(b'\\');
    }
    ret
}

/// Make an Infile from a file name
pub fn get_reader(name: &str) -> Result<Infile> {
    let inner: Box<dyn Read> = {
        if name == "-" {
            //	    unsafe { Box::new(std::fs::File::from_raw_fd(1)) }
            Box::new(io::stdin())
        } else if name.starts_with("s3://") {
            Box::new(S3Reader::new_path(name)?)
        } else if let Some(stripped) = name.strip_prefix("<<") {
            Box::new(std::io::Cursor::new(unescape_vec(stripped.as_bytes())))
        } else {
            Box::new(fs::File::open(name)?)
        }
    };
    let mut outer = io::BufReader::new(inner);
    let start = outer.fill_buf()?;
    if start.starts_with(&[0x1fu8, 0x8bu8, 0x08u8]) {
        outer = io::BufReader::new(Box::new(MultiGzDecoder::new(outer)));
    }
    Ok(Infile::new(outer, name))
}

#[derive(Debug, Default)]
/// shared context for any input file type
pub struct InfileContext {
    // CDX header, contructed if necessary
    header: StringLine,
    // have we read all the btes of the file
    is_done: bool,
    // is the file length zero
    is_empty: bool,
    // was there a CDX header?
    has_header: bool,
    text: TextFileMode,
}

/// create appropriate header from first line of file
pub fn make_header(line: &[u8]) -> StringLine {
    let mut s = StringLine::new();
    if is_cdx(line) {
        s.line = String::from_utf8_lossy(&line[5..]).to_string();
    } else {
        s.line = String::new();
        for x in 1..=line.split(|ch| *ch == b'\t').count() {
            s.line.push_str(&format!("c{}\t", x));
        }
        s.line.pop();
    }
    s.split(b'\t');
    s
}

// if CDX and specified and different, then strip header
/// Reader header line, if any, and first line of text
impl InfileContext {
    const fn new(text_in: &TextFileMode) -> Self {
        Self {
            header: StringLine::new(),
            is_done: true,
            is_empty: true,
            has_header: false,
            text: *text_in,
        }
    }
    fn read_header(&mut self, file: &mut impl BufRead, line: &mut TextLine) -> Result<()> {
        self.is_empty = self.text.read_header(file, &mut self.header.line)?;
        if self.is_empty {
            return Ok(());
        }
        self.has_header = !self.header.line.is_empty();
        if self.has_header {
            self.text.split_head(&mut self.header);
        }
        self.is_done = self.text.read_line(file, &mut line.line)?;
        if self.is_done {
            return Ok(());
        }
        line.split(self.text.delim);

        if !self.has_header {
            let mut head_str = String::new();
            for i in 1..=line.len() {
                head_str += "c";
                head_str += &i.to_string();
                head_str += "\t";
            }
            head_str.pop();
            head_str += "\n";
            let mut fake_head = head_str.as_bytes();
            self.header.read(&mut fake_head)?;
            self.header.split(self.text.delim);
        }
        Ok(())
    }
}

/// where are we, in which file?
#[derive(Debug, Default, Clone)]
pub struct FileLocData {
    /// full file name
    name: String,
    /// byte offset, uncompressed
    bytes: usize,
    /// line number
    line: usize,
}

/// types of file location data
#[derive(Debug, Copy, Clone)]
enum FileLocItem {
    /// byte offset of start of line
    Bytes,
    /// 1-based line number
    Line,
    /// file name, with given number of parts
    Name(usize),
}
impl FileLocItem {
    fn new(spec: &str) -> Result<Self> {
        if spec.eq_ignore_ascii_case("bytes") {
            Ok(Self::Bytes)
        } else if spec.eq_ignore_ascii_case("line") {
            Ok(Self::Line)
        } else if spec.eq_ignore_ascii_case("name") {
            Ok(Self::Name(0))
        } else if let Some((a, b)) = spec.split_once('.') {
            if a.eq_ignore_ascii_case("name") {
                Ok(Self::Name(
                    b.to_usize_whole(spec.as_bytes(), "File location")?,
                ))
            } else {
                err!("File Loc must be once of Bytes, Line, Name : '{}'", spec)
            }
        } else {
            err!("File Loc must be once of Bytes, Line, Name : '{}'", spec)
        }
    }
    const fn dflt_name(&self) -> &'static str {
        match self {
            Self::Bytes => "bytes",
            Self::Line => "line",
            Self::Name(_) => "filename",
        }
    }
    fn write_data(&mut self, data: &mut impl Write, loc: &FileLocData) -> Result<()> {
        match self {
            Self::Bytes => write!(data, "{}", loc.bytes).unwrap(),
            Self::Line => write!(data, "{}", loc.line).unwrap(),
            Self::Name(n) => {
                if *n == 0 {
                    data.write_all(loc.name.as_bytes())?;
                } else {
                    // FIXME - this should really be cached somehow
                    data.write_all(loc.name.tail_path_u8(*n, b'/').as_bytes())?;
                }
            }
        }
        Ok(())
    }
}

/// FileLocItem with column name
#[derive(Debug, Clone)]
struct FileLoc {
    col_name: String,
    item: FileLocItem,
}
impl FileLoc {
    fn new(spec: &str) -> Result<Self> {
        if let Some((a, b)) = spec.split_once(':') {
            Ok(Self {
                col_name: a.to_string(),
                item: FileLocItem::new(b)?,
            })
        } else {
            let item = FileLocItem::new(spec)?;
            Ok(Self {
                col_name: item.dflt_name().to_string(),
                item,
            })
        }
    }
    fn write_data(&mut self, data: &mut impl Write, loc: &FileLocData) -> Result<()> {
        self.item.write_data(data, loc)
    }
}

/// List of FileLoc
#[derive(Default, Debug, Clone)]
pub struct FileLocList {
    v: Vec<FileLoc>,
}
impl FileLocList {
    /// new
    pub fn new() -> Self {
        Self::default()
    }
    /// new
    pub fn is_empty(&self) -> bool {
        self.v.is_empty()
    }
    /// add Name:Spec
    pub fn push(&mut self, spec: &str) -> Result<()> {
        for x in spec.split(',') {
            self.v.push(FileLoc::new(x)?);
        }
        Ok(())
    }
    /// fill data with file loc data
    pub fn write_data(
        &mut self,
        data: &mut impl Write,
        delim: u8,
        loc: &FileLocData,
    ) -> Result<()> {
        for x in &mut self.v {
            x.write_data(data, loc)?;
            data.write_all(&[delim])?;
        }
        Ok(())
    }
    /// fill data with column names
    pub fn write_names(&mut self, data: &mut String, delim: u8) {
        for x in &mut self.v {
            data.push_str(&x.col_name);
            data.push(delim as char);
        }
    }
    /// add new columns to header
    pub fn add(&self, header: &mut ColumnHeader) -> Result<()> {
        for x in &self.v {
            header.push(&x.col_name)?;
        }
        Ok(())
    }
}

/// Text file reader. Lines broken into columns, with lookback
pub struct Reader {
    file: Infile,
    lines: Vec<TextLine>,
    cont: InfileContext,
    do_split: bool,
    curr: usize,
    loc: FileLocData,
}
impl fmt::Debug for Reader {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "Reader")
    }
}

impl Default for Reader {
    fn default() -> Self {
        Self::new(&TextFileMode::default())
    }
}

impl Reader {
    /// loc
    pub const fn loc(&self) -> &FileLocData {
        &self.loc
    }
    /// make a new Reader
    pub fn new(text_in: &TextFileMode) -> Self {
        Self::new_with(1, text_in)
    }
    /// set to false to skip breaking into columns
    pub fn do_split(&mut self, val: bool) {
        self.do_split = val;
    }
    /// make a new Reader, with explicit lookback
    pub fn new_with(lookback: usize, text_in: &TextFileMode) -> Self {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        Self {
            file: Infile::default(),
            lines,
            cont: InfileContext::new(text_in),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        }
    }
    /// make a new Reader, with deault text, FIXME - delete this
    pub fn new_open2(name: &str) -> Result<Self> {
        Self::new_open_with(name, 1, &TextFileMode::default())
    }
    /// make a new Reader
    pub fn new_open(name: &str, text_in: &TextFileMode) -> Result<Self> {
        Self::new_open_with(name, 1, text_in)
    }
    /// make a new Reader
    pub fn new_open_with(name: &str, lookback: usize, text_in: &TextFileMode) -> Result<Self> {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        let mut tmp = Self {
            file: get_reader(name)?,
            lines,
            cont: InfileContext::new(text_in),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        };
        tmp.cont.read_header(&mut *tmp.file, &mut tmp.lines[0])?;
        tmp.loc.name = name.to_string();
        tmp.loc.line = 1;
        tmp.loc.bytes = if tmp.has_header() {
            tmp.header().line.len()
        } else {
            0
        };
        Ok(tmp)
    }
    /// get current line contents, without the trailing newline
    pub fn curr_nl(&self) -> &[u8] {
        let line = self.curr_line();
        &line.line[0..line.line.len() - 1]
    }
    /// get previous line contents, without the trailing newline
    pub fn prev_nl(&self, n: usize) -> &[u8] {
        let line = self.prev_line(n);
        &line.line[0..line.line.len() - 1]
    }
    /// get delimiter
    pub const fn delim(&self) -> u8 {
        self.cont.text.delim
    }
    /// get column names
    pub fn names(&self) -> Vec<&str> {
        self.cont.header.vec()
    }
    /// write the current text line with newline
    pub fn write(&self, w: &mut impl Write) -> Result<()> {
        w.write_all(&self.curr_line().line)?;
        Ok(())
    }
    /// open file for reading
    pub fn open(&mut self, name: &str) -> Result<()> {
        self.file = get_reader(name)?;
        self.cont.read_header(&mut *self.file, &mut self.lines[0])
    }
    /// The full text of the header, without the trailing newline
    pub const fn header_line(&self) -> &String {
        &self.cont.header.line
    }
    /// was file zero bytes?
    pub const fn is_empty(&self) -> bool {
        self.cont.is_empty
    }
    /// have we hit EOF?
    pub const fn is_done(&self) -> bool {
        self.cont.is_done
    }
    /// line number of curr_line
    pub const fn line_number(&self) -> usize {
        self.loc.line
    }
    fn incr(&mut self) {
        self.loc.line += 1;
        self.curr += 1;
        if self.curr >= self.lines.len() {
            self.curr = 0;
        }
    }
    /// get next line of text
    pub fn getline(&mut self) -> Result<bool> {
        self.loc.bytes += self.curr().line.len();
        self.incr();
        if self
            .cont
            .text
            .read_line(&mut *self.file, &mut self.lines[self.curr].line)?
        {
            self.cont.is_done = true;
        } else if self.do_split {
            self.cont.text.split(&mut self.lines[self.curr]);
        }
        Ok(self.cont.is_done)
    }

Trait Implementations

impl Clone for TextFileMode

fn clone(&self) -> TextFileMode

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TextFileMode

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TextFileMode

fn default() -> Self

Returns the “default value” for a type.

impl Copy for TextFileMode