Struct cdx::util::TextLine

pub struct TextLine { /* private fields */ }

A line of a text file, broken into columns. A line ends with a newline character, but column values do not. An empty line contains one empty column

 use std::io::BufRead;
 let mut data = b"one\ttwo\tthree\n";
 let mut dp = &data[..];
 let mut line = cdx::util::TextLine::new();
 let eof = line.read(&mut dp).unwrap();
 assert_eq!(eof, false);
 assert_eq!(line.strlen(), 14);
 line.split(b'\t');
 assert_eq!(line.len(), 3);
 assert_eq!(line.get(1), b"two");

Implementations

impl TextLine

pub fn parts(&mut self) -> &mut Vec<FakeSlice>

whole line, with newline

Examples found in repository

src/agg.rs (line 813)

↕

    fn result(&mut self, w: &mut dyn Write, fmt: NumFormat) -> Result<()> {
        self.data.split(self.delim);
        if self.do_sort {
            self.data.parts.sort_by(|a, b| {
                self.comp
                    .comp(a.get(&self.data.line), b.get(&self.data.line))
            });
        }
        if self.do_uniq {
            self.data.parts.dedup_by(|a, b| {
                self.comp
                    .equal(a.get(&self.data.line), b.get(&self.data.line))
            });
        }
        if self.do_count {
            fmt.print(self.data.parts().len() as f64, w)?;
        } else {
            let mut num_written = 0;
            for x in &self.data.parts {
                if x.len() >= self.min_len && x.len() <= self.max_len {
                    if num_written > 0 {
                        w.write_all(&[self.out_delim])?;
                    }
                    w.write_all(x.get(self.data.line()))?;
                    num_written += 1;
                    if num_written >= self.max_parts {
                        break;
                    }
                }
            }
        }
        Ok(())
    }

pub fn line(&self) -> &[u8]

whole line, with newline

Examples found in repository

src/column.rs (line 623)

≺ ≻

    fn write(&mut self, w: &mut dyn Write, line: &TextLine, _delim: u8) -> Result<()> {
        w.write_all(&line.line()[0..line.line().len() - 1])?;
        Ok(())
    }

src/comp.rs (line 391)

≺ ≻↕

    fn comp_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        _left_file: usize,
        _right_file: usize,
    ) -> Ordering {
        self.comp.comp(left.line(), right.line())
    }
    /// compare lines
    fn equal_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        _left_file: usize,
        _right_file: usize,
    ) -> bool {
        self.comp.equal(left.line(), right.line())
    }
    /// compare lines
    fn comp_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        _delim: u8,
        _left_file: usize,
        _right_file: usize,
    ) -> Ordering {
        self.comp.comp(left, right)
    }
    /// compare lines
    fn equal_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        _delim: u8,
        _left_file: usize,
        _right_file: usize,
    ) -> bool {
        self.comp.equal(left, right)
    }
    /// resolve named columns; illegal to call any of the others with a file that has not been looked up
    fn lookup(&mut self, _fieldnames: &[&str], _file_num: usize) -> Result<()> {
        Ok(())
    }
    fn need_split(&self) -> bool {
        false
    }

    fn fill_cache_cols(&mut self, item: &mut Item, value: &TextLine) {
        self.comp.fill_cache(item, value.line())
    }

    fn fill_cache_line(&mut self, item: &mut Item, value: &[u8], _delim: u8) {
        self.comp.fill_cache(item, value)
    }

    fn set(&mut self, value: &[u8]) {
        self.comp.set(value)
    }

    fn comp_self_cols(&mut self, right: &TextLine) -> Ordering {
        self.comp.comp_self(right.line())
    }

    fn equal_self_cols(&mut self, right: &TextLine) -> bool {
        self.comp.equal_self(right.line())
    }

    fn comp_self_line(&mut self, right: &[u8], _delim: u8) -> Ordering {
        self.comp.comp_self(right)
    }

    fn equal_self_line(&mut self, right: &[u8], _delim: u8) -> bool {
        self.comp.equal_self(right)
    }
}

/// compare the whole line with the [Compare]
struct LineCompCol {
    cols: Vec<NamedCol>,
    comp: Comp,
}

impl LineCompCol {
    /// cols must not be empty, use LineCompWhole for that
    fn new(comp: Comp, spec: &str) -> Result<Self> {
        let mut cols = Vec::new();
        for x in spec.split('.') {
            cols.push(NamedCol::new_from(x)?);
        }
        Ok(Self { cols, comp })
    }
}

impl LineCompare for LineCompCol {
    fn used_cols(&self, v: &mut Vec<usize>, file_num: usize) {
        v.push(self.cols[file_num].num);
    }
    /// compare lines
    fn comp_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        self.comp.comp(
            left.get(self.cols[left_file].num),
            right.get(self.cols[right_file].num),
        )
    }
    /// compare lines
    fn equal_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        self.comp.equal(
            left.get(self.cols[left_file].num),
            right.get(self.cols[right_file].num),
        )
    }
    /// compare lines
    fn comp_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        delim: u8,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        self.comp.comp(
            get_col(left, self.cols[left_file].num, delim),
            get_col(right, self.cols[right_file].num, delim),
        )
    }
    /// compare lines
    fn equal_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        delim: u8,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        self.comp.equal(
            get_col(left, self.cols[left_file].num, delim),
            get_col(right, self.cols[right_file].num, delim),
        )
    }
    /// resolve named columns; illegal to call any of the others with a file that has not been looked up
    fn lookup(&mut self, fieldnames: &[&str], file_num: usize) -> Result<()> {
        while self.cols.len() < (file_num + 1) {
            self.cols.push(self.cols[self.cols.len() - 1].clone());
        }
        self.cols[file_num].lookup(fieldnames)
    }
    fn fill_cache_cols(&mut self, item: &mut Item, value: &TextLine) {
        self.comp.fill_cache(item, value.get(self.cols[0].num))
    }

    fn fill_cache_line(&mut self, item: &mut Item, value: &[u8], delim: u8) {
        self.comp
            .fill_cache(item, get_col(item.get(value), self.cols[0].num, delim))
    }

    fn set(&mut self, value: &[u8]) {
        self.comp.set(value)
    }

    fn comp_self_cols(&mut self, right: &TextLine) -> Ordering {
        self.comp.comp_self(right.get(self.cols[0].num))
    }

    fn equal_self_cols(&mut self, right: &TextLine) -> bool {
        self.comp.equal_self(right.get(self.cols[0].num))
    }

    fn comp_self_line(&mut self, right: &[u8], delim: u8) -> Ordering {
        self.comp.comp_self(get_col(right, self.cols[0].num, delim))
    }

    fn equal_self_line(&mut self, right: &[u8], delim: u8) -> bool {
        self.comp
            .equal_self(get_col(right, self.cols[0].num, delim))
    }
}

/*
    /// Compare lines
    fn comp_items(&self, base: &[u8], left: &Item, right: &Item) -> Ordering {
        if left.cache < right.cache {
            Ordering::Less
        } else if left.cache > right.cache {
            Ordering::Greater
        } else if left.complete() && right.complete() {
            Ordering::Equal
        } else {
            self.comp_lines(left.get(base), right.get(base))
        }
    }
    /// Compare lines
    fn equal_items(&self, base: &[u8], left: &Item, right: &Item) -> bool {
        if left.cache != right.cache {
            return false;
        }
        if left.complete() && right.complete() {
            return true;
        }
        self.equal_lines(left.get(base), right.get(base))
    }
}
*/

/// make fixed length array from slice
fn make_array(val: &[u8]) -> [u8; 8] {
    let mut res = [0; 8];
    match val.len() {
        0 => {}
        1 => res[..1].copy_from_slice(val),
        2 => res[..2].copy_from_slice(val),
        3 => res[..3].copy_from_slice(val),
        4 => res[..4].copy_from_slice(val),
        5 => res[..5].copy_from_slice(val),
        6 => res[..6].copy_from_slice(val),
        7 => res[..7].copy_from_slice(val),
        _ => res[..8].copy_from_slice(&val[..8]),
        //	_ => val[..8].try_into().unwrap(),
    }
    res
}
/*
fn make_array(val: &[u8]) -> [u8; 8] {
    let mut result = [0; 8];
    let len = std::cmp::min(val.len(), 8);
    result[..len].copy_from_slice(&val[..len]);
    result
}
*/

fn skip_comma_zero(mut a: &[u8]) -> &[u8] {
    while !a.is_empty() && (a[0] == b'0' || a[0] == b',') {
        a = &a[1..];
    }
    a
}

fn skip_comma(mut a: &[u8]) -> &[u8] {
    while !a.is_empty() && a[0] == b',' {
        a = &a[1..];
    }
    a
}

fn frac_cmp(mut a: &[u8], mut b: &[u8]) -> Ordering {
    debug_assert!(!a.is_empty());
    debug_assert!(!b.is_empty());
    debug_assert!(a[0] == b'.');
    debug_assert!(b[0] == b'.');
    a = &a[1..];
    b = &b[1..];
    while !a.is_empty() && !b.is_empty() && (a[0] == b[0]) && a[0].is_ascii_digit() {
        a = &a[1..];
        b = &b[1..];
    }
    if a.is_empty() {
        if b.is_empty() || !b[0].is_ascii_digit() {
            Ordering::Equal
        } else {
            Ordering::Less
        }
    } else if b.is_empty() {
        if a[0].is_ascii_digit() {
            Ordering::Greater
        } else {
            Ordering::Equal
        }
    } else if a[0].is_ascii_digit() {
        if b[0].is_ascii_digit() {
            a[0].cmp(&b[0])
        } else {
            Ordering::Greater
        }
    } else if b[0].is_ascii_digit() {
        Ordering::Less
    } else {
        Ordering::Equal
    }
}

fn num_cmp_unsigned(mut a: &[u8], mut b: &[u8]) -> Ordering {
    a = skip_comma_zero(a);
    b = skip_comma_zero(b);
    while !a.is_empty() && !b.is_empty() && (a[0] == b[0]) && a[0].is_ascii_digit() {
        a = &a[1..];
        b = &b[1..];
        a = skip_comma(a);
        b = skip_comma(b);
    }
    if a.is_empty() && b.is_empty() {
        return Ordering::Equal;
    }
    if a.is_empty() {
        if b[0].is_ascii_digit() {
            return Ordering::Less;
        }
        return Ordering::Equal;
    }
    if b.is_empty() {
        if a[0].is_ascii_digit() {
            return Ordering::Greater;
        }
        return Ordering::Equal;
    }
    if a[0] == b'.' && b[0] == b'.' {
        return frac_cmp(a, b);
    }
    let tmp = a[0].cmp(&b[0]);

    let mut log_a = 0;
    while !a.is_empty() && a[0].is_ascii_digit() {
        log_a += 1;
        a = &a[1..];
        a = skip_comma(a);
    }

    let mut log_b = 0;
    while !b.is_empty() && b[0].is_ascii_digit() {
        log_b += 1;
        b = &b[1..];
        b = skip_comma(b);
    }

    if log_a != log_b {
        return log_a.cmp(&log_b);
    }
    if log_a == 0 {
        return Ordering::Equal;
    }
    tmp
}

fn num_cmp_signed(mut a: &[u8], mut b: &[u8]) -> Ordering {
    a = a.trimw_start();
    b = b.trimw_start();

    let left_minus = if !a.is_empty() && a[0] == b'-' {
        a = &a[1..];
        true
    } else {
        false
    };

    let right_minus = if !b.is_empty() && b[0] == b'-' {
        b = &b[1..];
        true
    } else {
        false
    };

    if left_minus {
        if right_minus {
            return num_cmp_unsigned(b, a);
        } else {
            return Ordering::Less;
        }
    }
    if right_minus {
        return Ordering::Greater;
    }
    num_cmp_unsigned(a, b)
}

fn ip_cmp(mut a: &[u8], mut b: &[u8]) -> Ordering {
    a = a.trimw_start();
    b = b.trimw_start();

    loop {
        let mut a2: usize = 0;
        while a.len() > a2 && a[a2].is_ascii_digit() {
            a2 += 1;
        }
        let mut b2: usize = 0;
        while b.len() > b2 && b[b2].is_ascii_digit() {
            b2 += 1;
        }
        if a2 != b2 {
            return a2.cmp(&b2);
        }
        for x in 0..a2 {
            if a[x] != b[x] {
                return a[x].cmp(&b[x]);
            }
        }
        let a_dot = a.len() > a2 && a[a2] == b'.';
        let b_dot = b.len() > b2 && b[b2] == b'.';
        if !a_dot && !b_dot {
            return Ordering::Equal;
        }
        if !a_dot {
            return Ordering::Less;
        }
        if !b_dot {
            return Ordering::Greater;
        }
        a = &a[a2 + 1..];
        b = &b[b2 + 1..];
    }
}

/// One line in a buffer of text, suitable for sorting
#[derive(Debug, Clone, Copy)]
pub struct Item {
    /// offset from beginning of buffer
    pub offset: u32,
    /// length of line, including newline, with high bit reserved
    pub size_plus: u32,
    /// Compare caches, and then if necessary compare actual data
    pub cache: u64,
}
const _: () = assert!(std::mem::size_of::<Item>() == 16);

/*
    Reverse, // compare right to left
    LowerUtf, // unicode lowercase
    NormUtf, // unicode normalized
    LowerNormUtf, // unicode lowercase normalized
    Human,  // 2.3K or 4.5G
    Fuzzy(u32 n), // integer compare, but equal if within n
    Date(String fmt), // formatted date compare
    Url, //. Compare backwards from first slash, then forwards from first slash
    Prefix, // columns are equal if either is a prefix of the other
    Enum (JAN,FEB,MAR,...)
*/

type MakerBox = Box<dyn Fn(&Comp) -> Result<Box<dyn Compare>> + Send>;
/// A named constructor for a [Compare], used by [CompMaker]
struct CompMakerItem {
    /// matched against Comparator::ctype
    tag: &'static str,
    /// what this matcher does
    help: &'static str,
    /// Create a dyn Match from a pattern
    maker: MakerBox,
}

type LineMakerBox = Box<dyn Fn(&LineComp) -> Result<Box<dyn LineCompare>> + Send>;
/// A named constructor for a [LineCompare], used by [CompMaker]
struct LineCompMakerItem {
    /// matched against Comparator::ctype
    tag: &'static str,
    /// what this matcher does
    help: &'static str,
    /// Create a dyn Match from a pattern
    maker: LineMakerBox,
}

struct CompMakerAlias {
    old_name: &'static str,
    new_name: &'static str,
}

lazy_static! {
    static ref COMP_MAKER: Mutex<Vec<CompMakerItem>> = Mutex::new(Vec::new());
    static ref LINE_MAKER: Mutex<Vec<LineCompMakerItem>> = Mutex::new(Vec::new());
    static ref COMP_ALIAS: Mutex<Vec<CompMakerAlias>> = Mutex::new(Vec::new());
    static ref MODIFIERS: Vec<&'static str> = vec!["rev", "strict", "trail", "low"];
}

/// Makes a [Compare or LineComp]
#[derive(Debug, Clone, Default)]
pub struct CompMaker {}

impl CompMaker {
    /// add standard match makers
    fn init() -> Result<()> {
        if !COMP_MAKER.lock().unwrap().is_empty() {
            return Ok(());
        }
        Self::do_add_alias("numeric", "num")?;
        Self::do_add_alias("length", "len")?;
        Self::do_add_alias("plain", "")?;
        Self::do_push_line("expr", "Sort by value of expr", |p| {
            Ok(Box::new(LineCompExpr::new(&p.pattern)?))
        })?;
        Self::do_push("length", "Sort by length of string", |_p| {
            Ok(Box::new(CompareLen::new()))
        })?;
        Self::do_push("random", "Sort randomly", |_p| {
            Ok(Box::new(CompareRandom::new()))
        })?;
        Self::do_push("ip", "Sort as IP address or 1.2.3 section numbers", |_p| {
            Ok(Box::new(CompareIP::new()))
        })?;
        Self::do_push("plain", "Sort the plain bytes", |_p| {
            Ok(Box::new(ComparePlain::new()))
        })?;
        Self::do_push("lower", "Sort as the ascii lowercase of the string", |_p| {
            Ok(Box::new(CompareLower::new()))
        })?;
        Self::do_push(
            "float",
            "Convert to floating point, and sort the result.",
            |_p| Ok(Box::new(Comparef64::new())),
        )?;
        Self::do_push("numeric", "Convert NNN.nnn of arbitrary length.", |_p| {
            Ok(Box::new(CompareNumeric::new()))
        })?;
        Self::do_push("equal", "Everything always compares equal.", |_p| {
            Ok(Box::new(CompareEqual {}))
        })?;
        Ok(())
    }
    /// Add a new Compare. If a Compare already exists by that name, replace it.
    pub fn push<F: 'static>(tag: &'static str, help: &'static str, maker: F) -> Result<()>
    where
        F: Fn(&Comp) -> Result<Box<dyn Compare>> + Send,
    {
        Self::init()?;
        Self::do_push(tag, help, maker)
    }
    /// Add a new LineCompare. If a Compare already exists by that name, replace it.
    pub fn push_line<F: 'static>(tag: &'static str, help: &'static str, maker: F) -> Result<()>
    where
        F: Fn(&LineComp) -> Result<Box<dyn LineCompare>> + Send,
    {
        Self::init()?;
        Self::do_push_line(tag, help, maker)
    }
    /// Add a new alias. If an alias already exists by that name, replace it.
    pub fn add_alias(old_name: &'static str, new_name: &'static str) -> Result<()> {
        Self::init()?;
        Self::do_add_alias(old_name, new_name)
    }
    /// Return name, replaced by its alias, if any.
    fn resolve_alias(name: &str) -> &str {
        let mut mm = COMP_ALIAS.lock().unwrap();
        for x in mm.iter_mut() {
            if x.new_name == name {
                return x.old_name;
            }
        }
        name
    }
    fn do_add_alias(old_name: &'static str, new_name: &'static str) -> Result<()> {
        if MODIFIERS.contains(&new_name) {
            return err!("You can't add an alias named {} because that is reserved for a modifier");
        }
        let m = CompMakerAlias { old_name, new_name };
        let mut mm = COMP_ALIAS.lock().unwrap();
        for x in mm.iter_mut() {
            if x.new_name == m.new_name {
                *x = m;
                return Ok(());
            }
        }
        mm.push(m);
        Ok(())
    }
    fn do_push<F: 'static>(tag: &'static str, help: &'static str, maker: F) -> Result<()>
    where
        F: Fn(&Comp) -> Result<Box<dyn Compare>> + Send,
    {
        if MODIFIERS.contains(&tag) {
            return err!(
                "You can't add a matcher named {} because that is reserved for a modifier"
            );
        }
        let m = CompMakerItem {
            tag,
            help,
            maker: Box::new(maker),
        };
        let mut mm = COMP_MAKER.lock().unwrap();
        for x in mm.iter_mut() {
            if x.tag == m.tag {
                *x = m;
                return Ok(());
            }
        }
        mm.push(m);
        Ok(())
    }
    fn do_push_line<F: 'static>(tag: &'static str, help: &'static str, maker: F) -> Result<()>
    where
        F: Fn(&LineComp) -> Result<Box<dyn LineCompare>> + Send,
    {
        if MODIFIERS.contains(&tag) {
            return err!(
                "You can't add a matcher named {} because that is reserved for a modifier"
            );
        }
        let m = LineCompMakerItem {
            tag,
            help,
            maker: Box::new(maker),
        };
        let mut mm = LINE_MAKER.lock().unwrap();
        for x in mm.iter_mut() {
            if x.tag == m.tag {
                *x = m;
                return Ok(());
            }
        }
        mm.push(m);
        Ok(())
    }
    /// Print all available Matchers to stdout.
    pub fn help() {
        println!("Modifers :");
        println!("rev     reverse to ordering");
        println!("strict  compare as junk if not exactly right");
        println!("trail   compare as junk if no leading goodness");
        println!("low     junk should compare low, rather than high");
        println!("Methods :");
        Self::init().unwrap();
        let mm = COMP_MAKER.lock().unwrap();
        for x in &*mm {
            println!("{:12}{}", x.tag, x.help);
        }
        let mm = LINE_MAKER.lock().unwrap();
        for x in &*mm {
            println!("{:12}{}", x.tag, x.help);
        }
        println!("See also https://avjewe.github.io/cdxdoc/Comparator.html.");
    }
    /// create Box<dyn Compare> from spec
    pub fn make_comp_box(spec: &str) -> Result<Box<dyn Compare>> {
        Ok(Self::make_comp(spec)?.comp)
    }
    /// create Box<dyn LineCompare> from spec
    pub fn make_line_comp_box(spec: &str) -> Result<Box<dyn LineCompare>> {
        Ok(Self::make_line_comp(spec)?.comp)
    }
    /// create Comp from spec
    pub fn make_comp(spec: &str) -> Result<Comp> {
        if let Some((a, b)) = spec.split_once(',') {
            Self::make_comp_parts(a, b)
        } else {
            Self::make_comp_parts(spec, "")
        }
    }
    /// create Comp from method and pattern
    pub fn make_comp_parts(method: &str, pattern: &str) -> Result<Comp> {
        let mut comp = Comp::new();
        comp.pattern = pattern.to_string();
        if !method.is_empty() {
            for x in method.split('.') {
                if x.eq_ignore_ascii_case("rev") {
                    comp.reverse = true;
                } else if x.eq_ignore_ascii_case("strict") {
                    comp.junk.junk_type = JunkType::None;
                } else if x.eq_ignore_ascii_case("trail") {
                    comp.junk.junk_type = JunkType::Trailing;
                } else if x.eq_ignore_ascii_case("low") {
                    comp.junk.junk_val = JunkVal::Min;
                } else {
                    comp.ctype = x.to_string();
                }
            }
        }
        Self::remake_comp(&mut comp)?;
        Ok(comp)
    }
    /// create LineComp from spec
    pub fn make_line_comp(spec: &str) -> Result<LineComp> {
        if let Some((a, b)) = spec.split_once(',') {
            if let Some((c, d)) = b.split_once(',') {
                Self::make_line_comp_parts(a, c, d)
            } else {
                Self::make_line_comp_parts(a, b, "")
            }
        } else {
            Self::make_line_comp_parts(spec, "", "")
        }
    }
    /// create LineComp from columns, method and pattern
    pub fn make_line_comp_parts(cols: &str, method: &str, pattern: &str) -> Result<LineComp> {
        let mut comp = LineComp::new();
        comp.pattern = pattern.to_string();
        comp.cols = cols.to_string();
        if !method.is_empty() {
            for x in method.split('.') {
                if x.eq_ignore_ascii_case("rev") {
                    comp.reverse = true;
                } else if x.eq_ignore_ascii_case("strict") {
                    comp.junk.junk_type = JunkType::None;
                } else if x.eq_ignore_ascii_case("trail") {
                    comp.junk.junk_type = JunkType::Trailing;
                } else if x.eq_ignore_ascii_case("low") {
                    comp.junk.junk_val = JunkVal::Min;
                } else {
                    comp.ctype = x.to_string();
                }
            }
        }
        Self::remake_line_comp(&mut comp)?;
        Ok(comp)
    }
    /// reset the Compare inside the Comp
    pub fn remake_comp(comp: &mut Comp) -> Result<()> {
        Self::init()?;
        let ctype = Self::resolve_alias(&comp.ctype);
        let mm = COMP_MAKER.lock().unwrap();
        for x in &*mm {
            if ctype.eq_ignore_ascii_case(x.tag) {
                comp.comp = (x.maker)(comp)?;
                return Ok(());
            }
        }
        err!("No such compare type : '{}'", comp.ctype)
    }
    /// reset the LineCompare inside the LineComp
    pub fn remake_line_comp(comp: &mut LineComp) -> Result<()> {
        Self::init()?;
        let ctype = Self::resolve_alias(&comp.ctype);
        let mm = LINE_MAKER.lock().unwrap();
        for x in &*mm {
            if ctype.eq_ignore_ascii_case(x.tag) {
                comp.comp = (x.maker)(comp)?;
                return Ok(());
            }
        }
        let mm = COMP_MAKER.lock().unwrap();
        let mut new_comp = Comp::with_line_comp(comp);
        for x in &*mm {
            if ctype.eq_ignore_ascii_case(x.tag) {
                new_comp.comp = (x.maker)(&new_comp)?;
                if comp.cols.is_empty() {
                    comp.comp = Box::new(LineCompWhole::new(new_comp));
                } else {
                    comp.comp = Box::new(LineCompCol::new(new_comp, &comp.cols)?);
                }
                return Ok(());
            }
        }
        err!("No such compare type : '{}'", comp.ctype)
    }
}

#[derive(Default, Debug)]
/// Ordered list of [Comp]
pub struct CompList {
    c: Vec<Comp>,
}
#[derive(Default, Debug)]
/// Ordered list of [LineComp]
pub struct LineCompList {
    c: Vec<LineComp>,
    value: Vec<u8>,
}

impl CompList {
    /// new
    pub fn new() -> Self {
        Self::default()
    }
    /// any [Comp]s in the list?
    pub fn is_empty(&self) -> bool {
        self.c.is_empty()
    }
    /// add
    pub fn push(&mut self, x: Comp) {
        self.c.push(x);
    }
    /// add
    pub fn add(&mut self, x: &str) -> Result<()> {
        self.c.push(CompMaker::make_comp(x)?);
        Ok(())
    }
    /// Compare two slices, usually column values
    pub fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        for x in &self.c {
            let ret = x.comp(left, right);
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// Compare two slices for equality
    pub fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        for x in &self.c {
            if !x.comp.equal(left, right) {
                return false;
            }
        }
        true
    }
    /// set cache for this value
    pub fn fill_cache(&self, item: &mut Item, value: &[u8]) {
        if !self.c.is_empty() {
            self.c[0].fill_cache(item, value);
        }
    }
    /// set my value
    pub fn set(&mut self, value: &[u8], delim: u8) -> Result<()> {
        if self.c.len() == 1 {
            self.c[0].set(value);
        } else {
            let values: Vec<&[u8]> = value.split(|ch| *ch == delim).collect();
            if values.len() != self.c.len() {
                return err!(
                    "Tried to use a {} part value for a {} part Comparison",
                    values.len(),
                    self.c.len()
                );
            }
            for (n, x) in self.c.iter_mut().enumerate() {
                x.set(values[n]);
            }
        }
        Ok(())
    }
    /// Compare self to slice
    pub fn comp_self(&self, right: &[u8]) -> Ordering {
        for x in &self.c {
            let ret = x.comp_self(right);
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// Compare self to slice for equality
    pub fn equal_self(&self, right: &[u8]) -> bool {
        for x in &self.c {
            if !x.equal_self(right) {
                return false;
            }
        }
        true
    }
}

impl LineCompList {
    /// new
    pub fn new() -> Self {
        Self::default()
    }
    /// any [LineComp]s in the list?
    pub fn is_empty(&self) -> bool {
        self.c.is_empty()
    }
    /// which columns used as part of key?
    pub fn used_cols(&self, file_num: usize) -> Vec<usize> {
        let mut v = Vec::new();
        for x in &self.c {
            x.used_cols(&mut v, file_num);
        }
        v
    }
    /// add
    pub fn add(&mut self, x: &str) -> Result<()> {
        self.c.push(CompMaker::make_line_comp(x)?);
        Ok(())
    }
    /// add
    pub fn push(&mut self, x: LineComp) {
        self.c.push(x);
    }
    /// compare [Item]s
    pub fn comp_items(&mut self, base: &[u8], left: &Item, right: &Item) -> Ordering {
        if self.c.is_empty() {
            return Ordering::Equal;
        }
        let ret = self.c[0].comp_items(base, left, right);
        if ret != Ordering::Equal {
            return ret;
        }
        for x in self.c.iter_mut().skip(1) {
            let ret = x.comp_lines(left.get(base), right.get(base));
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// compare [Item]s
    pub fn equal_items(&mut self, base: &[u8], left: &Item, right: &Item) -> bool {
        if self.c.is_empty() {
            return true;
        }
        if !self.c[0].equal_items(base, left, right) {
            return false;
        }
        for x in self.c.iter_mut().skip(1) {
            if !x.equal_lines(left.get(base), right.get(base)) {
                return false;
            }
        }
        true
    }
    /// compare [TextLine]s in the same file
    pub fn comp_cols(&mut self, left: &TextLine, right: &TextLine) -> Ordering {
        self.comp_cols_n(left, right, 0, 0)
    }
    /// compare [TextLine]s in different files
    pub fn comp_cols_n(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        for x in &mut self.c {
            let ret = x.comp_cols_n(left, right, left_file, right_file);
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// compare [TextLine]s in the same file
    pub fn equal_cols(&mut self, left: &TextLine, right: &TextLine) -> bool {
        self.equal_cols_n(left, right, 0, 0)
    }
    /// compare [TextLine]s in different files
    pub fn equal_cols_n(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        for x in &mut self.c {
            if !x.equal_cols_n(left, right, left_file, right_file) {
                return false;
            }
        }
        true
    }
    /// compare liness from the same
    pub fn comp_lines(&mut self, left: &[u8], right: &[u8]) -> Ordering {
        self.comp_lines_n(left, right, 0, 0)
    }
    /// compare lines from different files
    pub fn comp_lines_n(
        &mut self,
        left: &[u8],
        right: &[u8],
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        for x in &mut self.c {
            let ret = x.comp_lines_n(left, right, left_file, right_file);
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// compare lines from the same file
    pub fn equal_lines(&mut self, left: &[u8], right: &[u8]) -> bool {
        self.equal_lines_n(left, right, 0, 0)
    }
    /// compare lines from different files
    pub fn equal_lines_n(
        &mut self,
        left: &[u8],
        right: &[u8],
        left_file: usize,
        right_file: usize,
    ) -> bool {
        for x in &mut self.c {
            if !x.equal_lines_n(left, right, left_file, right_file) {
                return false;
            }
        }
        true
    }
    /// resolve named columns
    pub fn lookup(&mut self, fieldnames: &[&str]) -> Result<()> {
        self.lookup_n(fieldnames, 0)
    }
    /// resolve named columns in the given file
    pub fn lookup_n(&mut self, fieldnames: &[&str], file_num: usize) -> Result<()> {
        for x in &mut self.c {
            x.lookup_n(fieldnames, file_num)?
        }
        Ok(())
    }
    /// do [TextLine]s need their columns initialized
    pub fn need_split(&self) -> bool {
        for x in &self.c {
            if x.need_split() {
                return true;
            }
        }
        false
    }
    /// fill [Item]'s cache
    pub fn fill_cache_cols(&mut self, item: &mut Item, value: &TextLine) {
        if !self.c.is_empty() {
            self.c[0].fill_cache_cols(item, value);
        }
    }
    /// fill [Item]'s cache
    pub fn fill_cache_line(&mut self, item: &mut Item, value: &[u8]) {
        if !self.c.is_empty() {
            self.c[0].fill_cache_line(item, value);
        }
    }
    /// get the value previously set
    pub fn get_value(&self) -> &[u8] {
        &self.value
    }
    /// set value fo later comparison
    pub fn set(&mut self, value: &[u8], delim: u8) -> Result<()> {
        self.value = value.to_owned();
        if self.c.len() == 1 {
            self.c[0].set(value);
        } else {
            let values: Vec<&[u8]> = value.split(|ch| *ch == delim).collect();
            if values.len() != self.c.len() {
                return err!(
                    "Tried to use a {} part value for a {} part Comparison",
                    values.len(),
                    self.c.len()
                );
            }
            for (n, x) in self.c.iter_mut().enumerate() {
                x.set(values[n]);
            }
        }
        Ok(())
    }
    /// compare my value to this line
    pub fn comp_self_cols(&mut self, right: &TextLine) -> Ordering {
        for x in &mut self.c {
            let ret = x.comp_self_cols(right);
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// compare my value to this line
    pub fn equal_self_cols(&mut self, right: &TextLine) -> bool {
        for x in &mut self.c {
            if !x.equal_self_cols(right) {
                return false;
            }
        }
        true
    }
    /// compare my value to this line
    pub fn comp_self_line(&mut self, right: &[u8]) -> Ordering {
        for x in &mut self.c {
            let ret = x.comp_self_line(right);
            if ret != Ordering::Equal {
                return ret;
            }
        }
        Ordering::Equal
    }
    /// compare my value to this line
    pub fn equal_self_line(&mut self, right: &[u8]) -> bool {
        for x in &mut self.c {
            if !x.equal_self_line(right) {
                return false;
            }
        }
        true
    }
}

#[derive(Default, Debug)]
struct CompareRandom {
    rng: fastrand::Rng,
}
impl CompareRandom {
    fn new() -> Self {
        Self::default()
    }
    fn ord(&self) -> Ordering {
        if self.rng.bool() {
            Ordering::Less
        } else {
            Ordering::Greater
        }
    }
}
impl Compare for CompareRandom {
    fn comp(&self, _left: &[u8], _right: &[u8]) -> Ordering {
        self.ord()
    }
    fn equal(&self, _left: &[u8], _right: &[u8]) -> bool {
        self.rng.bool()
    }
    fn fill_cache(&self, _item: &mut Item, _value: &[u8]) {}
    fn set(&mut self, _value: &[u8]) {}
    fn comp_self(&self, _right: &[u8]) -> Ordering {
        self.ord()
    }
    fn equal_self(&self, _right: &[u8]) -> bool {
        self.rng.bool()
    }
}

/// IP Address Comparison
#[derive(Default, Debug)]
struct CompareIP {
    value: Vec<u8>,
}

/// Default comparison
#[derive(Default, Debug)]
struct ComparePlain {
    value: Vec<u8>,
}

/// Default comparison
#[derive(Default, Debug)]
struct CompareLower {
    value: Vec<u8>,
}

/// Compare by length of string
#[derive(Default, Debug)]
struct CompareLen {
    value: u32,
}

/// always equal comparison
#[derive(Default, Debug)]
struct CompareEqual {}

/// f64 comparison
#[derive(Default, Debug)]
struct Comparef64 {
    value: f64,
}

impl Comparef64 {
    const fn new() -> Self {
        Self { value: 0.0 }
    }
}

/// nnn.nnn comparison
#[derive(Default, Debug)]
struct CompareNumeric {
    value: Vec<u8>,
}

impl CompareNumeric {
    const fn new() -> Self {
        Self { value: Vec::new() }
    }
}

impl ComparePlain {
    const fn new() -> Self {
        Self { value: Vec::new() }
    }
}

impl CompareLower {
    const fn new() -> Self {
        Self { value: Vec::new() }
    }
}

impl CompareIP {
    const fn new() -> Self {
        Self { value: Vec::new() }
    }
}

impl CompareLen {
    const fn new() -> Self {
        Self { value: 0 }
    }
}

impl Compare for ComparePlain {
    fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        left.cmp(right)
    }
    fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        left == right
    }
    fn fill_cache(&self, item: &mut Item, value: &[u8]) {
        item.cache = u64::from_be_bytes(make_array(value));
        item.assign_complete(value.len() <= 8);
    }
    fn set(&mut self, value: &[u8]) {
        self.value = value.to_vec();
    }
    fn comp_self(&self, right: &[u8]) -> Ordering {
        self.value[..].cmp(right)
    }
    fn equal_self(&self, right: &[u8]) -> bool {
        self.value == right
    }
}

impl Compare for CompareLower {
    fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        left.cmp_insens(right)
    }
    fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        left.equal_insens(right)
    }
    fn fill_cache(&self, item: &mut Item, value: &[u8]) {
        let mut aa = make_array(value);
        for x in &mut aa {
            x.make_ascii_lowercase()
        }
        item.cache = u64::from_be_bytes(aa);
        item.assign_complete(value.len() <= 8);
    }
    fn set(&mut self, value: &[u8]) {
        value.assign_lower(&mut self.value);
    }
    fn comp_self(&self, right: &[u8]) -> Ordering {
        self.value.cmp_insens_quick(right)
    }
    fn equal_self(&self, right: &[u8]) -> bool {
        self.value.equal_insens_quick(right)
    }
}

impl Compare for CompareIP {
    fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        ip_cmp(left, right)
    }
    fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        ip_cmp(left, right) == Ordering::Equal
    }
    fn fill_cache(&self, item: &mut Item, _value: &[u8]) {
        item.cache = 0; // FIXME
        item.clear_complete();
    }
    fn set(&mut self, value: &[u8]) {
        self.value = value.to_vec();
    }
    fn comp_self(&self, right: &[u8]) -> Ordering {
        ip_cmp(&self.value, right)
    }
    fn equal_self(&self, right: &[u8]) -> bool {
        ip_cmp(&self.value, right) == Ordering::Equal
    }
}

impl Compare for CompareNumeric {
    fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        num_cmp_signed(left, right)
    }
    fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        num_cmp_signed(left, right) == Ordering::Equal
    }
    fn fill_cache(&self, item: &mut Item, value: &[u8]) {
        item.cache = ulp_to_ulong(value.to_f64_lossy());
    }
    fn set(&mut self, value: &[u8]) {
        self.value = value.to_vec();
    }
    fn comp_self(&self, right: &[u8]) -> Ordering {
        num_cmp_signed(&self.value, right)
    }
    fn equal_self(&self, right: &[u8]) -> bool {
        num_cmp_signed(&self.value, right) == Ordering::Equal
    }
}

impl Compare for CompareEqual {
    fn comp(&self, _left: &[u8], _right: &[u8]) -> Ordering {
        Ordering::Equal
    }
    fn equal(&self, _left: &[u8], _right: &[u8]) -> bool {
        true
    }
    fn fill_cache(&self, item: &mut Item, _value: &[u8]) {
        item.cache = 0;
        item.set_complete();
    }
    fn set(&mut self, _value: &[u8]) {}
    fn comp_self(&self, _right: &[u8]) -> Ordering {
        Ordering::Equal
    }
    fn equal_self(&self, _right: &[u8]) -> bool {
        true
    }
}

impl Compare for Comparef64 {
    fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        fcmp(left.to_f64_lossy(), right.to_f64_lossy())
    }
    fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        left.to_f64_lossy() == right.to_f64_lossy()
    }
    fn fill_cache(&self, item: &mut Item, value: &[u8]) {
        item.cache = ulp_to_ulong(value.to_f64_lossy());
        item.set_complete();
    }
    fn set(&mut self, value: &[u8]) {
        self.value = value.to_f64_lossy();
    }
    fn comp_self(&self, right: &[u8]) -> Ordering {
        fcmp(self.value, right.to_f64_lossy())
    }
    fn equal_self(&self, right: &[u8]) -> bool {
        self.value == right.to_f64_lossy()
    }
}

impl Compare for CompareLen {
    fn comp(&self, left: &[u8], right: &[u8]) -> Ordering {
        left.len().cmp(&right.len())
    }
    fn equal(&self, left: &[u8], right: &[u8]) -> bool {
        left.len() == right.len()
    }
    fn fill_cache(&self, item: &mut Item, value: &[u8]) {
        item.cache = value.len() as u64;
        item.set_complete();
    }
    fn set(&mut self, value: &[u8]) {
        self.value = value.len() as u32;
    }
    fn comp_self(&self, right: &[u8]) -> Ordering {
        self.value.cmp(&(right.len() as u32))
    }
    fn equal_self(&self, right: &[u8]) -> bool {
        self.value == right.len() as u32
    }
}

impl Item {
    /// new item
    pub const fn new() -> Self {
        Self {
            offset: 0,
            size_plus: 0,
            cache: 0,
        }
    }
    /// return line as slice
    pub fn get<'a>(&self, base: &'a [u8]) -> &'a [u8] {
        &base[self.begin()..self.end()]
    }
    /// size in bytes of line
    pub const fn size(&self) -> u32 {
        self.size_plus & 0x7fffffff
    }
    /// test complete bit
    pub const fn complete(&self) -> bool {
        (self.size_plus & 0x80000000) != 0
    }
    /// set complete bit
    pub fn set_complete(&mut self) {
        self.size_plus |= 0x80000000;
    }
    /// clear complete bit
    pub fn clear_complete(&mut self) {
        self.size_plus &= 0x7fffffff;
    }
    /// conditionally set complete bit
    pub fn assign_complete(&mut self, tag: bool) {
        if tag {
            self.set_complete();
        } else {
            self.clear_complete();
        }
    }
    /// offset of begin
    pub const fn begin(&self) -> usize {
        self.offset as usize
    }
    /// offset of end
    pub const fn end(&self) -> usize {
        self.offset as usize + self.size() as usize
    }
    /// Test an Item for equality to myself
    pub fn equal(&self, other: &Self, base: &[u8]) -> bool {
        if self.cache != other.cache {
            return false;
        }
        if self.complete() && other.complete() {
            return true;
        }
        self.get(base) == other.get(base)
    }
    /// Compare Item to myself with standard ordering
    pub fn compare(&self, other: &Self, base: &[u8]) -> Ordering {
        if self.cache < other.cache {
            return Ordering::Less;
        }
        if self.cache > other.cache {
            return Ordering::Greater;
        }
        if (self.cache == other.cache) && self.complete() && other.complete() {
            return Ordering::Equal;
        }
        self.get(base).cmp(other.get(base))
    }
    /// Compare an Item to myself, with custom ordering
    pub fn compare2(&self, other: &Self, base: &[u8], cmp: &dyn Compare) -> Ordering {
        if self.cache < other.cache {
            return Ordering::Less;
        }
        if self.cache > other.cache {
            return Ordering::Greater;
        }
        if (self.cache == other.cache) && self.complete() && other.complete() {
            return Ordering::Equal;
        }
        cmp.comp(self.get(base), other.get(base))
    }
}

impl Default for Item {
    fn default() -> Self {
        Self::new()
    }
}

#[derive(Debug, Default)]
struct LineCompExpr {
    exprs: Vec<Expr>,
    value: f64,
}

impl LineCompExpr {
    fn new(expr: &str) -> Result<Self> {
        Ok(Self {
            exprs: vec![Expr::new(expr)?],
            value: 0.0,
        })
    }
}

impl LineCompare for LineCompExpr {
    fn comp_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        let left_val = self.exprs[left_file].eval(left);
        let right_val = self.exprs[right_file].eval(right);
        fcmp(left_val, right_val)
    }
    fn used_cols(&self, v: &mut Vec<usize>, file_num: usize) {
        self.exprs[file_num].used_cols(v);
    }
    fn equal_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        let left_val = self.exprs[left_file].eval(left);
        let right_val = self.exprs[right_file].eval(right);
        left_val == right_val
    }
    fn comp_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        delim: u8,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        let left_val = self.exprs[left_file].eval_line(left, delim);
        let right_val = self.exprs[right_file].eval_line(right, delim);
        fcmp(left_val, right_val)
    }
    fn equal_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        delim: u8,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        let left_val = self.exprs[left_file].eval_line(left, delim);
        let right_val = self.exprs[right_file].eval_line(right, delim);
        left_val == right_val
    }
    fn lookup(&mut self, fieldnames: &[&str], file_num: usize) -> Result<()> {
        while self.exprs.len() < (file_num + 1) {
            self.exprs.push(Expr::new(self.exprs[0].expr())?);
        }
        self.exprs[file_num].lookup(fieldnames)
    }

    fn fill_cache_cols(&mut self, item: &mut Item, value: &TextLine) {
        item.cache = ulp_to_ulong(self.exprs[0].eval(value));
        item.set_complete();
    }
    fn fill_cache_line(&mut self, item: &mut Item, value: &[u8], delim: u8) {
        let val = self.exprs[0].eval_line(item.get(value), delim);
        item.cache = ulp_to_ulong(val);
        item.set_complete();
    }
    fn set(&mut self, value: &[u8]) {
        self.value = value.to_f64_lossy()
    }
    fn comp_self_cols(&mut self, right: &TextLine) -> Ordering {
        let value = self.exprs[0].eval(right);
        fcmp(self.value, value)
    }
    fn equal_self_cols(&mut self, right: &TextLine) -> bool {
        let value = self.exprs[0].eval(right);
        self.value == value
    }
    fn comp_self_line(&mut self, right: &[u8], delim: u8) -> Ordering {
        let value = self.exprs[0].eval_line(right, delim);
        fcmp(self.value, value)
    }
    fn equal_self_line(&mut self, right: &[u8], delim: u8) -> bool {
        let value = self.exprs[0].eval_line(right, delim);
        self.value == value
    }
}

/// return true if ordering is bad.
/// print appropriate message to stderr
pub fn comp_check(f: &Reader, cmp: &mut LineCompList, unique: bool) -> bool {
    let c = cmp.comp_cols(f.prev_line(1), f.curr_line());
    let bad = match c {
        Ordering::Less => false,
        Ordering::Equal => unique,
        Ordering::Greater => true,
    };
    if c == Ordering::Equal && unique {
        eprintln!("Lines are equal when they should be unique.");
    } else if bad {
        eprintln!("Lines are out of order");
    }
    if bad {
        eprint!("{} : ", f.line_number() - 1);
        prerr_n(&[f.prev_line(1).line()]);
        eprint!("{} : ", f.line_number());
        prerr_n(&[f.curr_line().line()]);
    }
    bad
}

src/agg.rs (line 792)

≺ ≻↕

    fn add(&mut self, data: &[u8]) {
        if !data.is_empty() {
            if !self.data.line().is_empty() {
                self.data.raw().push(self.delim);
            }
            self.data.raw().extend_from_slice(data);
        }
    }
    fn result(&mut self, w: &mut dyn Write, fmt: NumFormat) -> Result<()> {
        self.data.split(self.delim);
        if self.do_sort {
            self.data.parts.sort_by(|a, b| {
                self.comp
                    .comp(a.get(&self.data.line), b.get(&self.data.line))
            });
        }
        if self.do_uniq {
            self.data.parts.dedup_by(|a, b| {
                self.comp
                    .equal(a.get(&self.data.line), b.get(&self.data.line))
            });
        }
        if self.do_count {
            fmt.print(self.data.parts().len() as f64, w)?;
        } else {
            let mut num_written = 0;
            for x in &self.data.parts {
                if x.len() >= self.min_len && x.len() <= self.max_len {
                    if num_written > 0 {
                        w.write_all(&[self.out_delim])?;
                    }
                    w.write_all(x.get(self.data.line()))?;
                    num_written += 1;
                    if num_written >= self.max_parts {
                        break;
                    }
                }
            }
        }
        Ok(())
    }

src/matcher.rs (line 602)

≺ ≻↕

fn load_hashset(data: &mut HashSet<Vec<u8>>, fname: &str) -> Result<()> {
    let mut f = Reader::new();
    f.do_split(false);
    f.open(fname)?;
    if f.is_done() {
        return Ok(());
    }
    loop {
        let line = &f.curr().line();
        if line.len() > 1 {
            data.insert(line[0..line.len() - 1].to_vec());
        }
        if f.getline()? {
            break;
        }
    }
    Ok(())
}

#[derive(Debug, Clone)]
/// pattern is file name. String exactly matches one line of file.
struct FileExactMatch {
    data: HashSet<Vec<u8>>,
    file_name: String,
}
impl FileExactMatch {
    fn new(file_name: &str) -> Result<Self> {
        let mut d = HashSet::new();
        load_hashset(&mut d, file_name)?;
        Ok(Self {
            data: d,
            file_name: file_name.to_string(),
        })
    }
}
impl Match for FileExactMatch {
    fn smatch(&self, buff: &str) -> bool {
        self.data.contains(buff.as_bytes())
    }
    fn umatch(&self, buff: &[u8]) -> bool {
        self.data.contains(buff)
    }
    fn show(&self) -> String {
        format!("Exact Match of one line in file {}", self.file_name)
    }
}

fn load_hashset_c(data: &mut HashSet<Vec<u8>>, fname: &str, unicode: bool) -> Result<()> {
    let mut f = Reader::new();
    f.do_split(false);
    f.open(fname)?;
    if f.is_done() {
        return Ok(());
    }
    loop {
        let mut line: &[u8] = f.curr().line();
        if line.len() > 1 {
            if line.last().unwrap() == &b'\n' {
                line = &line[..line.len() - 1];
            }
            if unicode {
                data.insert(String::from_utf8(line.to_vec())?.new_lower().into_bytes());
            // PERF - 2 allocations
            } else {
                data.insert(line.new_lower());
            }
        }
        if f.getline()? {
            break;
        }
    }
    Ok(())
}

src/tabs.rs (line 76)

≺ ≻↕

pub fn show(file: &str, screen: &Rect) -> Result<()> {
    let mut f = Reader::new();
    f.open(file)?;
    if f.is_empty() {
        return Ok(());
    }
    let mut lines: Vec<StringLine> = Vec::new();
    let mut sizes: Vec<usize> = Vec::new();
    sizes.resize(f.header().len(), 0);
    if f.has_header() {
        lines.push(f.header().clone());
    }
    if !f.is_done() {
        while lines.len() < screen.height {
            let mut s = StringLine::new();
            s.line = String::from_utf8_lossy(f.curr().line()).to_string();
            s.split(f.delim());
            lines.push(s);
            if f.getline()? {
                break;
            }
        }
    }
    for x in &lines {
        for (i, c) in x.iter().enumerate() {
            let width = UnicodeWidthStr::width(c);
            if sizes[i] < width {
                sizes[i] = width;
            }
        }
    }
    let mut total: usize = sizes.iter().sum();
    let target = screen.width - sizes.len();
    while total > target {
        dec_max(&mut sizes);
        total = sizes.iter().sum();
    }
    let mut w = get_writer("-")?;
    let mut do_center = f.has_header();
    for x in &lines {
        let mut need_space = false;
        for (c, y) in x.iter().enumerate() {
            let (nstr, width) = y.unicode_truncate(sizes[c]);
            if need_space {
                w.write_all(b" ")?;
            }
            let num = (sizes[c] - width) / 2;
            if do_center {
                for _ in 0..num {
                    w.write_all(b" ")?;
                }
            }
            w.write_all(nstr.as_bytes())?;
            if do_center {
                let num2 = (sizes[c] - width) - num;
                for _ in 0..num2 {
                    w.write_all(b" ")?;
                }
            } else {
                for _ in width..sizes[c] {
                    w.write_all(b" ")?;
                }
            }
            need_space = true;
        }
        do_center = false;
        w.write_all(b"\n")?;
    }
    Ok(())
}

/// show the file in a specific rectangle
pub fn show2(file: &str, screen: &Rect, w: &mut Vec<String>) -> Result<usize> {
    let mut f = Reader::new();
    f.open(file)?;
    if f.is_empty() {
        return Ok(0);
    }
    let mut lines: Vec<StringLine> = Vec::new();
    let mut sizes: Vec<usize> = Vec::new();
    sizes.resize(f.header().len(), 0);
    if f.has_header() {
        lines.push(f.header().clone());
    }
    if !f.is_done() {
        while lines.len() < screen.height {
            let mut s = StringLine::new();
            s.line = String::from_utf8_lossy(f.curr().line()).to_string();
            s.split(f.delim());
            lines.push(s);
            if f.getline()? {
                break;
            }
        }
    }
    for x in &lines {
        for (i, c) in x.iter().enumerate() {
            let width = UnicodeWidthStr::width(c);
            if sizes[i] < width {
                sizes[i] = width;
            }
        }
    }
    let mut total: usize = sizes.iter().sum();
    let target = screen.width - sizes.len();
    while total > target {
        dec_max(&mut sizes);
        total = sizes.iter().sum();
    }

    let mut do_center = f.has_header();
    w.clear();
    for x in &lines {
        let mut s = String::new();
        let mut need_space = false;
        for (c, y) in x.iter().enumerate() {
            let (nstr, width) = y.unicode_truncate(sizes[c]);
            if need_space {
                s.push(' ');
            }
            let num = (sizes[c] - width) / 2;
            if do_center {
                for _ in 0..num {
                    s.push(' ');
                }
            }
            s.push_str(nstr);
            if do_center {
                let num2 = (sizes[c] - width) - num;
                for _ in 0..num2 {
                    s.push(' ');
                }
            } else {
                for _ in width..sizes[c] {
                    s.push(' ');
                }
            }
            need_space = true;
        }
        do_center = false;
        w.push(s);
    }
    Ok(sizes.iter().sum::<usize>() + sizes.len())
}

src/sort.rs (line 53)

≺ ≻↕

pub fn merge_t(
    in_files: &[String],
    cmp: &mut LineCompList,
    mut w: impl Write,
    unique: bool,
    _tmp: &TempDir,
) -> Result<()> {
    if in_files.is_empty() {
        return Ok(());
    }
    if in_files.len() == 1 && !unique {
        let r = get_reader(&in_files[0])?;
        return copy(r.0, w);
    }
    let mut open_files: Vec<Reader> = Vec::with_capacity(in_files.len());
    for x in in_files {
        open_files.push(Reader::new_open(x)?);
    }
    if !cmp.need_split() {
        for x in &mut open_files {
            x.do_split(false);
        }
    }
    // FIXME -- Check Header
    if open_files[0].has_header() {
        w.write_all(open_files[0].header().line.as_bytes())?;
    }

    let nums: Vec<usize> = (0..open_files.len()).collect();
    let mut mm = MergeTreeItem::new_tree(&open_files, &nums);
    if unique {
        let x = mm.next(cmp, &mut open_files)?;
        if x.is_none() {
            return Ok(());
        }
        let x = x.unwrap();
        w.write_all(open_files[x].curr_line().line())?;
        let mut prev = open_files[x].curr_line().clone();
        loop {
            let x = mm.next(cmp, &mut open_files)?;
            if x.is_none() {
                break;
            }
            let x = x.unwrap();
            if !cmp.equal_cols(&prev, open_files[x].curr_line()) {
                w.write_all(open_files[x].curr_line().line())?;
            }
            prev.assign(open_files[x].curr_line());
        }
    } else {
        loop {
            let x = mm.next(cmp, &mut open_files)?;
            if x.is_none() {
                break;
            }
            let x = x.unwrap();
            w.write_all(open_files[x].curr_line().line())?;
        }
    }
    Ok(())
    /*
        let mut files = in_files.to_owned();
        let mut n = 0;
        loop {
            if files.len() == 2 {
                return merge_2(&files[0], &files[1], cmp, w, unique);
            }
            let mut tmp_file = tmp.path().to_owned();
            tmp_file.push(format!("merge_{}.txt", n));
            n += 1;
            let tmp_name = tmp_file.to_str().unwrap();
            let new_w = get_writer(tmp_name)?;
            merge_2(&files[0], &files[1], cmp, new_w, unique)?;
            files.remove(0);
            files.remove(0);
            files.push(tmp_name.to_string());
        }

    */
}

/// merge all the files into w
pub fn merge(files: &[String], cmp: &mut LineCompList, w: impl Write, unique: bool) -> Result<()> {
    let tmp = TempDir::new("merge")?;
    merge_t(files, cmp, w, unique, &tmp)
}

/// given two file names, merge them into output
pub fn merge_2(
    left: &str,
    right: &str,
    cmp: &mut LineCompList,
    mut w: impl Write,
    unique: bool,
) -> Result<()> {
    let mut left_file = Reader::new();
    let mut right_file = Reader::new();
    left_file.open(left)?;
    right_file.open(right)?;
    left_file.do_split(false);
    right_file.do_split(false);
    cmp.lookup(&left_file.names())?;

    // FIXME -- Check Header
    if left_file.has_header() {
        w.write_all(left_file.header().line.as_bytes())?;
    }

    if unique {
        let mut prev: Vec<u8> = Vec::new();
        while !left_file.is_done() && !right_file.is_done() {
            let ord = cmp.comp_lines(left_file.curr().line(), right_file.curr().line());
            if ord == Ordering::Less {
                left_file.write(&mut w)?;
                mem::swap(&mut prev, left_file.curr_mut().raw());
                left_file.getline()?;
            } else if ord == Ordering::Greater {
                right_file.write(&mut w)?;
                mem::swap(&mut prev, left_file.curr_mut().raw());
                right_file.getline()?;
            } else {
                left_file.write(&mut w)?;
                mem::swap(&mut prev, left_file.curr_mut().raw());
                left_file.getline()?;
                right_file.getline()?;
            }
            while !left_file.is_done() && cmp.equal_lines(left_file.curr().line(), &prev) {
                left_file.getline()?;
            }
            while !right_file.is_done() && cmp.equal_lines(right_file.curr().line(), &prev) {
                right_file.getline()?;
            }
        }
    } else {
        while !left_file.is_done() && !right_file.is_done() {
            let ord = cmp.comp_lines(left_file.curr().line(), right_file.curr().line());
            // if Equal, write both lines
            if ord != Ordering::Less {
                right_file.write(&mut w)?;
                right_file.getline()?;
            }
            if ord != Ordering::Greater {
                left_file.write(&mut w)?;
                left_file.getline()?;
            }
        }
    }
    while !left_file.is_done() {
        left_file.write(&mut w)?;
        left_file.getline()?;
    }
    while !right_file.is_done() {
        right_file.write(&mut w)?;
        right_file.getline()?;
    }
    Ok(())
}

Additional examples can be found in:

• src/bin/cdx/sample_main.rs
• src/bin/cdx/uniq_main.rs
• src/bin/cdx/wc_main.rs

pub fn line_nl(&self) -> &[u8]

whole line, without newline

Examples found in repository

src/matcher.rs (line 1021)

≺ ≻

    fn ok(&mut self, line: &TextLine) -> bool {
        self.matcher.negate
            ^ if self.matcher.string {
                self.matcher
                    .smatch(&String::from_utf8_lossy(line.line_nl()))
            } else {
                self.matcher.umatch(line.line_nl())
            }
    }

pub fn raw(&mut self) -> &mut Vec<u8>

whole line, with newline, as Vec

Examples found in repository

src/agg.rs (line 793)

≺ ≻

    fn add(&mut self, data: &[u8]) {
        if !data.is_empty() {
            if !self.data.line().is_empty() {
                self.data.raw().push(self.delim);
            }
            self.data.raw().extend_from_slice(data);
        }
    }

src/sort.rs (line 131)

≺ ≻↕

pub fn merge_2(
    left: &str,
    right: &str,
    cmp: &mut LineCompList,
    mut w: impl Write,
    unique: bool,
) -> Result<()> {
    let mut left_file = Reader::new();
    let mut right_file = Reader::new();
    left_file.open(left)?;
    right_file.open(right)?;
    left_file.do_split(false);
    right_file.do_split(false);
    cmp.lookup(&left_file.names())?;

    // FIXME -- Check Header
    if left_file.has_header() {
        w.write_all(left_file.header().line.as_bytes())?;
    }

    if unique {
        let mut prev: Vec<u8> = Vec::new();
        while !left_file.is_done() && !right_file.is_done() {
            let ord = cmp.comp_lines(left_file.curr().line(), right_file.curr().line());
            if ord == Ordering::Less {
                left_file.write(&mut w)?;
                mem::swap(&mut prev, left_file.curr_mut().raw());
                left_file.getline()?;
            } else if ord == Ordering::Greater {
                right_file.write(&mut w)?;
                mem::swap(&mut prev, left_file.curr_mut().raw());
                right_file.getline()?;
            } else {
                left_file.write(&mut w)?;
                mem::swap(&mut prev, left_file.curr_mut().raw());
                left_file.getline()?;
                right_file.getline()?;
            }
            while !left_file.is_done() && cmp.equal_lines(left_file.curr().line(), &prev) {
                left_file.getline()?;
            }
            while !right_file.is_done() && cmp.equal_lines(right_file.curr().line(), &prev) {
                right_file.getline()?;
            }
        }
    } else {
        while !left_file.is_done() && !right_file.is_done() {
            let ord = cmp.comp_lines(left_file.curr().line(), right_file.curr().line());
            // if Equal, write both lines
            if ord != Ordering::Less {
                right_file.write(&mut w)?;
                right_file.getline()?;
            }
            if ord != Ordering::Greater {
                left_file.write(&mut w)?;
                left_file.getline()?;
            }
        }
    }
    while !left_file.is_done() {
        left_file.write(&mut w)?;
        left_file.getline()?;
    }
    while !right_file.is_done() {
        right_file.write(&mut w)?;
        right_file.getline()?;
    }
    Ok(())
}

pub fn assign(&mut self, x: &Self)

assign TextLine into existing TextLine, avoiding allocation if possible

Examples found in repository

src/bin/cdx/uniq_main.rs (line 137)

≺ ≻↕

    fn assign(&mut self, tmp: &mut TextLine, new: &TextLine) {
        match self.which {
            Which::First => {}
            Which::Last => tmp.assign(new),
            Which::Min => {
                if self.comp.comp_cols(tmp, new) == Ordering::Greater {
                    tmp.assign(new);
                }
            }
            Which::Max => {
                if self.comp.comp_cols(tmp, new) == Ordering::Less {
                    tmp.assign(new);
                }
            }
        }
    }
    fn lookup(&mut self, fieldnames: &[&str]) -> Result<()> {
        self.comp.lookup(fieldnames)
    }
}

pub fn main(argv: &[String]) -> Result<()> {
    let prog = args::ProgSpec::new("Select uniq lines.", args::FileCount::One);
    const A: [ArgSpec; 7] = [
        arg! {"agg", "a", "Col,Spec", "Merge value from this column, in place."},
        arg! {"agg-pre", "", "NewCol,SrcCol,Spec", "Merge value from SrcCol into new column, before other columns."},
        arg! {"agg-post", "", "NewCol,SrcCol,Spec", "Merge value from SrcCol into new column, after other columns."},
        arg! {"key", "k", "Spec", "How to compare adjacent lines"},
        arg! {"count", "c", "ColName,Position", "Write the count of matching line."},
        arg! {"which", "w", "(First,Last,Min,Max)[,LineCompare]", "Which of the matching lines should be printed."},
        arg! {"agg-help", "", "", "Print help for aggregators"},
    ];
    let (args, files) = args::parse(&prog, &A, argv)?;

    let mut agg = LineAggList::new();
    let mut comp = LineCompList::new();
    let mut count = Count::default();

    for x in args {
        if x.name == "key" {
            comp.add(&x.value)?;
        } else if x.name == "count" {
            count.get_count(&x.value)?;
        } else if x.name == "which" {
            count.get_which(&x.value)?;
        } else if x.name == "agg" {
            agg.push_replace(&x.value)?;
        } else if x.name == "agg-post" {
            agg.push_append(&x.value)?;
        } else if x.name == "agg-pre" {
            agg.push_prefix(&x.value)?;
        } else {
            unreachable!();
        }
    }

    assert_eq!(files.len(), 1);

    let mut f = Reader::new();
    f.open(&files[0])?;
    if f.is_empty() {
        return Ok(());
    }
    comp.lookup(&f.names())?;
    count.lookup(&f.names())?;
    let mut c_write = Writer::new(f.delim());
    if !agg.is_empty() {
        if count.pos == CountPos::Begin {
            agg.push_first_prefix(&format!("{},1,count", count.name))?;
        }
        if count.pos == CountPos::End {
            agg.push_append(&format!("{},1,count", count.name))?;
        }
        agg.lookup(&f.names())?;
        agg.fill(&mut c_write, f.header());
        c_write.lookup(&f.names())?;
    }

    let mut w = get_writer("-")?;
    if f.has_header() {
        let mut ch = ColumnHeader::new();
        if agg.is_empty() {
            if count.pos == CountPos::Begin {
                ch.push(&count.name)?;
            }
            ch.push_all(f.header())?;
            if count.pos == CountPos::End {
                ch.push(&count.name)?;
            }
        } else {
            c_write.add_names(&mut ch, f.header())?;
        }
        w.write_all(ch.get_head(f.delim()).as_bytes())?;
    }
    if f.is_done() {
        return Ok(());
    }

    f.do_split(comp.need_split());
    let mut matches = 1;
    if !agg.is_empty() {
        agg.add(f.curr_line());
        let mut tmp = f.curr_line().clone();
        loop {
            if f.getline()? {
                c_write.write(&mut w, &tmp)?;
                break;
            }
            if comp.equal_cols(f.prev_line(1), f.curr_line()) {
                count.assign(&mut tmp, f.curr_line());
                agg.add(f.curr_line());
            } else {
                c_write.write(&mut w, &tmp)?;
                tmp.assign(f.curr_line());
                agg.reset();
                agg.add(f.curr_line());
            }
        }
    } else if count.which == Which::Last {
        loop {
            if f.getline()? {
                count.write(&mut w, matches, f.prev_line(1).line(), f.delim())?;
                break;
            }
            if comp.equal_cols(f.prev_line(1), f.curr_line()) {
                matches += 1;
            } else {
                count.write(&mut w, matches, f.prev_line(1).line(), f.delim())?;
                matches = 1;
            }
        }
    } else if count.which == Which::First && count.is_plain() {
        f.write_curr(&mut w)?;
        loop {
            if f.getline()? {
                break;
            }
            if !comp.equal_cols(f.prev_line(1), f.curr_line()) {
                f.write_curr(&mut w)?;
            }
        }
    } else {
        let mut tmp = f.curr_line().clone();
        loop {
            if f.getline()? {
                count.write(&mut w, matches, tmp.line(), f.delim())?;
                break;
            }
            if comp.equal_cols(f.prev_line(1), f.curr_line()) {
                count.assign(&mut tmp, f.curr_line());
                matches += 1;
            } else {
                count.write(&mut w, matches, tmp.line(), f.delim())?;
                tmp.assign(f.curr_line());
                matches = 1;
            }
        }
    }
    Ok(())
}

src/sort.rs (line 64)

↕

pub fn merge_t(
    in_files: &[String],
    cmp: &mut LineCompList,
    mut w: impl Write,
    unique: bool,
    _tmp: &TempDir,
) -> Result<()> {
    if in_files.is_empty() {
        return Ok(());
    }
    if in_files.len() == 1 && !unique {
        let r = get_reader(&in_files[0])?;
        return copy(r.0, w);
    }
    let mut open_files: Vec<Reader> = Vec::with_capacity(in_files.len());
    for x in in_files {
        open_files.push(Reader::new_open(x)?);
    }
    if !cmp.need_split() {
        for x in &mut open_files {
            x.do_split(false);
        }
    }
    // FIXME -- Check Header
    if open_files[0].has_header() {
        w.write_all(open_files[0].header().line.as_bytes())?;
    }

    let nums: Vec<usize> = (0..open_files.len()).collect();
    let mut mm = MergeTreeItem::new_tree(&open_files, &nums);
    if unique {
        let x = mm.next(cmp, &mut open_files)?;
        if x.is_none() {
            return Ok(());
        }
        let x = x.unwrap();
        w.write_all(open_files[x].curr_line().line())?;
        let mut prev = open_files[x].curr_line().clone();
        loop {
            let x = mm.next(cmp, &mut open_files)?;
            if x.is_none() {
                break;
            }
            let x = x.unwrap();
            if !cmp.equal_cols(&prev, open_files[x].curr_line()) {
                w.write_all(open_files[x].curr_line().line())?;
            }
            prev.assign(open_files[x].curr_line());
        }
    } else {
        loop {
            let x = mm.next(cmp, &mut open_files)?;
            if x.is_none() {
                break;
            }
            let x = x.unwrap();
            w.write_all(open_files[x].curr_line().line())?;
        }
    }
    Ok(())
    /*
        let mut files = in_files.to_owned();
        let mut n = 0;
        loop {
            if files.len() == 2 {
                return merge_2(&files[0], &files[1], cmp, w, unique);
            }
            let mut tmp_file = tmp.path().to_owned();
            tmp_file.push(format!("merge_{}.txt", n));
            n += 1;
            let tmp_name = tmp_file.to_str().unwrap();
            let new_w = get_writer(tmp_name)?;
            merge_2(&files[0], &files[1], cmp, new_w, unique)?;
            files.remove(0);
            files.remove(0);
            files.push(tmp_name.to_string());
        }

    */
}

pub const fn new() -> Self

make a new TextLine

Examples found in repository

src/util.rs (line 1014)

≺ ≻↕

    pub fn new_with(lookback: usize) -> Self {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        Self {
            file: Infile::default(),
            lines,
            cont: InfileContext::new(),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        }
    }
    /// make a new Reader
    pub fn new_open(name: &str) -> Result<Self> {
        Self::new_open_with(name, 1)
    }
    /// make a new Reader
    pub fn new_open_with(name: &str, lookback: usize) -> Result<Self> {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        let mut tmp = Self {
            file: get_reader(name)?,
            lines,
            cont: InfileContext::new(),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        };
        tmp.cont.read_header(&mut *tmp.file, &mut tmp.lines[0])?;
        tmp.loc.name = name.to_string();
        tmp.loc.line = 1;
        tmp.loc.bytes = if tmp.has_header() {
            tmp.header().line.len()
        } else {
            0
        };
        Ok(tmp)
    }

src/bin/cdx/wc_main.rs (line 80)

↕

pub fn main(argv: &[String]) -> Result<()> {
    let prog = args::ProgSpec::new("Aggregate info on whole lines.", args::FileCount::Many);
    const A: [ArgSpec; 10] = [
        arg! {"agg", "a", "NewCol,Spec", "Merge values into new column."},
        arg! {"lines", "l", "", "Shortcut for '--agg lines,count'"},
        arg! {"bytes", "b", "", "Shortcut for '--agg bytes,asum,chars'"},
        arg! {"chars", "c", "", "Shortcut for '--agg chars,asum,utf8.chars'"},
        arg! {"words", "w", "", "Shortcut for '--agg words,asum,swords'"},
        arg! {"file", "f", "Tri,ColName", "Should we add the filename as the first column?"},
        arg! {"header", "h", "Tri", "Should we write a cdx header?"},
        arg! {"total", "t", "yes,no,maybe,only", "Should we write the totals line?"},
        arg! {"format", "F", "plain,float,power2,power10", "Format for output numbers."},
        arg! {"columns", "C", "", "Count each column separately."},
    ];
    let (args, files) = args::parse(&prog, &A, argv)?;

    let mut agg = AggList::new();
    let mut file_name_col = "file".to_string();
    let mut show_file_name = Tri::Maybe;
    let mut show_header = Tri::Maybe;
    let mut show_totals = Tri::Maybe;
    let mut total_only = false;
    let mut do_columns = false;
    let mut fmt = NumFormat::default();
    for x in args {
        if x.name == "agg" {
            agg.push(&x.value)?;
        } else if x.name == "lines" {
            agg.push("lines,count")?;
        } else if x.name == "bytes" {
            agg.push("bytes,asum,chars")?;
        } else if x.name == "chars" {
            agg.push("chars,asum,utf8.chars")?;
        } else if x.name == "words" {
            agg.push("words,asum,swords")?;
        } else if x.name == "format" {
            fmt = NumFormat::new(&x.value)?;
        } else if x.name == "columns" {
            do_columns = true;
        } else if x.name == "total" {
            if x.value.eq_ignore_ascii_case("only") {
                total_only = true;
            } else {
                show_totals = Tri::new(&x.value)?;
            }
        } else if x.name == "header" {
            show_header = Tri::new(&x.value)?;
        } else if x.name == "file" {
            if let Some((a, b)) = x.value.split_once(',') {
                show_file_name = Tri::new(a)?;
                file_name_col = b.to_string();
            } else {
                show_file_name = Tri::new(&x.value)?;
            }
        } else {
            unreachable!();
        }
    }
    let nada = TextLine::new();
    let mut w = get_writer("-")?;
    let mut first_file = true;
    let mut totals = Vec::new();
    let do_totals;
    let show_file;
    agg.fmt(fmt);
    if do_columns {
        if agg.is_empty() {
            agg.push("bytes,asum,chars")?;
        }
        totals.resize(agg.len(), 0.0);
        let mut aggs: Vec<NamedAgg> = Vec::new();
        let mut colmap: Vec<usize> = Vec::new();
        for x in &files {
            let mut f = Reader::new();
            f.open(x)?;
            if f.is_empty() {
                continue;
            }
            colmap.clear();
            for x in &f.names() {
                if let Some(pos) = aggs.iter().position(|agg| agg.name == *x) {
                    colmap.push(pos);
                } else {
                    colmap.push(aggs.len());
                    aggs.push(NamedAgg::new(x, agg.deep_clone()));
                }
            }
            if f.is_done() {
                break;
            }
            loop {
                for (i, x) in f.curr_line().iter().enumerate() {
                    aggs[colmap[i]].agg.add(x);
                }
                if f.getline()? {
                    break;
                }
            }
        }
        show_file = match show_file_name {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => aggs.len() > 1 && !total_only,
        };
        do_totals = match show_totals {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => aggs.len() > 1,
        } || total_only;
        let do_header = match show_header {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => agg.len() > 1 || show_file,
        };
        if do_header {
            let mut ch = ColumnHeader::new();
            if show_file_name != Tri::No {
                ch.push("column")?;
            }
            let mut c_write = Writer::new(b'\t');
            agg.fill(&mut c_write);
            c_write.add_names(&mut ch, &StringLine::new())?;
            w.write_all(ch.get_head(b'\t').as_bytes())?;
        }
        if !total_only {
            for x in &aggs {
                if show_file_name != Tri::No {
                    w.write_all(x.name.as_bytes())?;
                    w.write_all(b"\t")?;
                }
                for i in 0..x.agg.len() {
                    if i != 0 {
                        w.write_all(b"\t")?;
                    }
                    x.agg.get(i).agg.borrow_mut().result(&mut w, fmt)?;
                }
                w.write_all(b"\n")?;
            }
        }
        if do_totals {
            for x in &aggs {
                #[allow(clippy::needless_range_loop)]
                for i in 0..agg.len() {
                    totals[i] += x.agg.get(i).agg.borrow().value();
                }
            }
        }
    } else {
        if agg.is_empty() {
            agg.push("lines,count")?;
        }
        totals.resize(agg.len(), 0.0);
        show_file = match show_file_name {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => files.len() > 1 && !total_only,
        };
        do_totals = match show_totals {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => files.len() > 1,
        } || total_only;
        let do_header = match show_header {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => agg.len() > 1 || show_file,
        };
        for x in &files {
            let mut f = Reader::new();
            f.open(x)?;
            if f.is_empty() {
                continue;
            }
            let mut c_write = Writer::new(f.delim());
            agg.fill(&mut c_write);
            c_write.lookup(&f.names())?;

            if do_header && first_file {
                first_file = false;
                let mut ch = ColumnHeader::new();
                if show_file {
                    ch.push(&file_name_col)?;
                }
                c_write.add_names(&mut ch, f.header())?;
                w.write_all(ch.get_head(f.delim()).as_bytes())?;
            }

            if f.is_done() {
                continue;
            }

            f.do_split(false);
            loop {
                agg.add(f.curr_line().line());
                if f.getline()? {
                    break;
                }
            }
            if !total_only {
                if show_file {
                    w.write_all(x.as_bytes())?;
                    w.write_all(b"\t")?;
                }
                c_write.write(&mut w, &nada)?;
            }
            #[allow(clippy::needless_range_loop)]
            for i in 0..agg.len() {
                totals[i] += agg.get(i).agg.borrow().value();
            }
            agg.reset();
        }
    }
    if do_totals {
        if show_file {
            w.write_all(b"totals\t")?;
        }
        for (i, t) in totals.iter().enumerate() {
            if i != 0 {
                w.write_all(b"\t")?;
            }
            fmt.print(*t, &mut w)?;
        }
        w.write_all(b"\n")?;
    }
    Ok(())
}

pub const fn iter(&self) -> TextLineIter<'_>

Iterator over columns in the line

Examples found in repository

src/util.rs (line 404)

    pub fn vec(&self) -> Vec<&[u8]> {
        self.iter().collect()
    }

src/bin/cdx/wc_main.rs (line 113)

↕

pub fn main(argv: &[String]) -> Result<()> {
    let prog = args::ProgSpec::new("Aggregate info on whole lines.", args::FileCount::Many);
    const A: [ArgSpec; 10] = [
        arg! {"agg", "a", "NewCol,Spec", "Merge values into new column."},
        arg! {"lines", "l", "", "Shortcut for '--agg lines,count'"},
        arg! {"bytes", "b", "", "Shortcut for '--agg bytes,asum,chars'"},
        arg! {"chars", "c", "", "Shortcut for '--agg chars,asum,utf8.chars'"},
        arg! {"words", "w", "", "Shortcut for '--agg words,asum,swords'"},
        arg! {"file", "f", "Tri,ColName", "Should we add the filename as the first column?"},
        arg! {"header", "h", "Tri", "Should we write a cdx header?"},
        arg! {"total", "t", "yes,no,maybe,only", "Should we write the totals line?"},
        arg! {"format", "F", "plain,float,power2,power10", "Format for output numbers."},
        arg! {"columns", "C", "", "Count each column separately."},
    ];
    let (args, files) = args::parse(&prog, &A, argv)?;

    let mut agg = AggList::new();
    let mut file_name_col = "file".to_string();
    let mut show_file_name = Tri::Maybe;
    let mut show_header = Tri::Maybe;
    let mut show_totals = Tri::Maybe;
    let mut total_only = false;
    let mut do_columns = false;
    let mut fmt = NumFormat::default();
    for x in args {
        if x.name == "agg" {
            agg.push(&x.value)?;
        } else if x.name == "lines" {
            agg.push("lines,count")?;
        } else if x.name == "bytes" {
            agg.push("bytes,asum,chars")?;
        } else if x.name == "chars" {
            agg.push("chars,asum,utf8.chars")?;
        } else if x.name == "words" {
            agg.push("words,asum,swords")?;
        } else if x.name == "format" {
            fmt = NumFormat::new(&x.value)?;
        } else if x.name == "columns" {
            do_columns = true;
        } else if x.name == "total" {
            if x.value.eq_ignore_ascii_case("only") {
                total_only = true;
            } else {
                show_totals = Tri::new(&x.value)?;
            }
        } else if x.name == "header" {
            show_header = Tri::new(&x.value)?;
        } else if x.name == "file" {
            if let Some((a, b)) = x.value.split_once(',') {
                show_file_name = Tri::new(a)?;
                file_name_col = b.to_string();
            } else {
                show_file_name = Tri::new(&x.value)?;
            }
        } else {
            unreachable!();
        }
    }
    let nada = TextLine::new();
    let mut w = get_writer("-")?;
    let mut first_file = true;
    let mut totals = Vec::new();
    let do_totals;
    let show_file;
    agg.fmt(fmt);
    if do_columns {
        if agg.is_empty() {
            agg.push("bytes,asum,chars")?;
        }
        totals.resize(agg.len(), 0.0);
        let mut aggs: Vec<NamedAgg> = Vec::new();
        let mut colmap: Vec<usize> = Vec::new();
        for x in &files {
            let mut f = Reader::new();
            f.open(x)?;
            if f.is_empty() {
                continue;
            }
            colmap.clear();
            for x in &f.names() {
                if let Some(pos) = aggs.iter().position(|agg| agg.name == *x) {
                    colmap.push(pos);
                } else {
                    colmap.push(aggs.len());
                    aggs.push(NamedAgg::new(x, agg.deep_clone()));
                }
            }
            if f.is_done() {
                break;
            }
            loop {
                for (i, x) in f.curr_line().iter().enumerate() {
                    aggs[colmap[i]].agg.add(x);
                }
                if f.getline()? {
                    break;
                }
            }
        }
        show_file = match show_file_name {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => aggs.len() > 1 && !total_only,
        };
        do_totals = match show_totals {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => aggs.len() > 1,
        } || total_only;
        let do_header = match show_header {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => agg.len() > 1 || show_file,
        };
        if do_header {
            let mut ch = ColumnHeader::new();
            if show_file_name != Tri::No {
                ch.push("column")?;
            }
            let mut c_write = Writer::new(b'\t');
            agg.fill(&mut c_write);
            c_write.add_names(&mut ch, &StringLine::new())?;
            w.write_all(ch.get_head(b'\t').as_bytes())?;
        }
        if !total_only {
            for x in &aggs {
                if show_file_name != Tri::No {
                    w.write_all(x.name.as_bytes())?;
                    w.write_all(b"\t")?;
                }
                for i in 0..x.agg.len() {
                    if i != 0 {
                        w.write_all(b"\t")?;
                    }
                    x.agg.get(i).agg.borrow_mut().result(&mut w, fmt)?;
                }
                w.write_all(b"\n")?;
            }
        }
        if do_totals {
            for x in &aggs {
                #[allow(clippy::needless_range_loop)]
                for i in 0..agg.len() {
                    totals[i] += x.agg.get(i).agg.borrow().value();
                }
            }
        }
    } else {
        if agg.is_empty() {
            agg.push("lines,count")?;
        }
        totals.resize(agg.len(), 0.0);
        show_file = match show_file_name {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => files.len() > 1 && !total_only,
        };
        do_totals = match show_totals {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => files.len() > 1,
        } || total_only;
        let do_header = match show_header {
            Tri::Yes => true,
            Tri::No => false,
            Tri::Maybe => agg.len() > 1 || show_file,
        };
        for x in &files {
            let mut f = Reader::new();
            f.open(x)?;
            if f.is_empty() {
                continue;
            }
            let mut c_write = Writer::new(f.delim());
            agg.fill(&mut c_write);
            c_write.lookup(&f.names())?;

            if do_header && first_file {
                first_file = false;
                let mut ch = ColumnHeader::new();
                if show_file {
                    ch.push(&file_name_col)?;
                }
                c_write.add_names(&mut ch, f.header())?;
                w.write_all(ch.get_head(f.delim()).as_bytes())?;
            }

            if f.is_done() {
                continue;
            }

            f.do_split(false);
            loop {
                agg.add(f.curr_line().line());
                if f.getline()? {
                    break;
                }
            }
            if !total_only {
                if show_file {
                    w.write_all(x.as_bytes())?;
                    w.write_all(b"\t")?;
                }
                c_write.write(&mut w, &nada)?;
            }
            #[allow(clippy::needless_range_loop)]
            for i in 0..agg.len() {
                totals[i] += agg.get(i).agg.borrow().value();
            }
            agg.reset();
        }
    }
    if do_totals {
        if show_file {
            w.write_all(b"totals\t")?;
        }
        for (i, t) in totals.iter().enumerate() {
            if i != 0 {
                w.write_all(b"\t")?;
            }
            fmt.print(*t, &mut w)?;
        }
        w.write_all(b"\n")?;
    }
    Ok(())
}

pub fn clear(&mut self)

empty the line

Examples found in repository

src/agg.rs (line 832)

    fn reset(&mut self) {
        self.data.clear();
    }

src/util.rs (line 369)

↕

    pub fn read<T: BufRead>(&mut self, f: &mut T) -> Result<bool> {
        self.clear();
        let sz = f.read_until(b'\n', &mut self.line)?;
        if sz == 0 {
            Ok(true)
        } else {
            if self.line.last() != Some(&b'\n') {
                self.line.push(b'\n');
            }
            Ok(false)
        }
    }

pub fn len(&self) -> usize

How many column in the line

Examples found in repository

src/matcher.rs (line 1052)

≺ ≻↕

    fn ok(&mut self, line: &TextLine) -> bool {
        line.len() == self.count
    }

    fn lookup(&mut self, _fieldnames: &[&str]) -> Result<()> {
        Ok(())
    }
    fn show(&self) -> String {
        format!("Does line have {} columns", self.count)
    }
    fn ok_verbose(&mut self, line: &TextLine, line_num: usize, fname: &str) -> bool {
        let ret = self.ok(line);
        if !ret {
            eprintln!(
                "Line {} of {} had {} columns where {} were expected.",
                line_num,
                fname,
                line.len(),
                self.count
            );
        }
        ret
    }

src/util.rs (line 540)

≺ ≻↕

    fn next(&mut self) -> Option<Self::Item> {
        if self.index >= self.line.len() {
            None
        } else {
            self.index += 1;
            Some(&self.line[self.index - 1])
        }
    }
}

impl<'a> Iterator for StringLineIter<'a> {
    // we will be counting with usize
    type Item = &'a str;

    fn next(&mut self) -> Option<Self::Item> {
        if self.index >= self.line.parts.len() {
            None
        } else {
            self.index += 1;
            Some(&self.line[self.index - 1])
        }
    }
}

struct S3Reader {
    //    name : String,
    rt: tokio::runtime::Runtime,
    //    client : aws_sdk_s3::Client,
    f: aws_sdk_s3::output::GetObjectOutput,
    left: Option<bytes::Bytes>,
}
impl S3Reader {
    fn new(bucket: &str, key: &str) -> Result<Self> {
        let rt = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()?;
        let shared_config = rt.block_on(aws_config::load_from_env());
        let client = aws_sdk_s3::Client::new(&shared_config);
        let obj = rt.block_on(client.get_object().bucket(bucket).key(key).send())?;
        Ok(Self {
            //	    name : key.to_string(),
            rt,
            //	    client,
            f: obj,
            left: None,
        })
    }
    fn new_path(spec: &str) -> Result<Self> {
        if let Some(name) = spec.strip_prefix("s3://") {
            if let Some((a, b)) = name.split_once('/') {
                Self::new(a, b)
            } else {
                err!("Not an S3 file spec '{}'", spec)
            }
        } else {
            err!("Not an S3 file '{}'", spec)
        }
    }
}
impl Read for S3Reader {
    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
        if let Some(bytes) = &self.left {
            if bytes.len() > buf.len() {
                buf.clone_from_slice(&bytes[..buf.len()]);
                self.left = Some(bytes.slice(buf.len()..));
                return Ok(buf.len());
            } else {
                let len = bytes.len();
                buf[0..len].clone_from_slice(bytes);
                self.left = None;
                return Ok(len);
            }
        }
        let bytes_res = self.rt.block_on(self.f.body.try_next());
        if bytes_res.is_err() {
            return Err(std::io::Error::new(std::io::ErrorKind::Other, "oh no"));
        }
        self.left = bytes_res.unwrap();
        if let Some(bytes) = &self.left {
            if bytes.len() > buf.len() {
                buf.clone_from_slice(&bytes[..buf.len()]);
                self.left = Some(bytes.slice(buf.len()..));
                Ok(buf.len())
            } else {
                let len = bytes.len();
                buf[0..bytes.len()].clone_from_slice(bytes);
                self.left = None;
                Ok(len)
            }
        } else {
            Ok(0)
        }
    }
}

/// Input file. Wrapped in a type so I can 'impl Debug'
pub struct Infile(
    /// The file being read
    pub io::BufReader<Box<dyn Read>>,
);

impl Infile {
    /// create a new input file
    pub fn new(f: io::BufReader<Box<dyn Read>>) -> Self {
        Self(f)
    }
}

impl Default for Infile {
    fn default() -> Self {
        Self::new(io::BufReader::new(Box::new(io::empty())))
    }
}

impl fmt::Debug for Infile {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "Infile")
    }
}

impl Deref for Infile {
    type Target = io::BufReader<Box<dyn Read>>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl DerefMut for Infile {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.0
    }
}

impl AsRef<io::BufReader<Box<dyn Read>>> for Infile {
    fn as_ref(&self) -> &io::BufReader<Box<dyn Read>> {
        &self.0
    }
}

impl AsMut<io::BufReader<Box<dyn Read>>> for Infile {
    fn as_mut(&mut self) -> &mut io::BufReader<Box<dyn Read>> {
        &mut self.0
    }
}

/// output file type
pub type Outfile = io::BufWriter<Box<dyn Write>>;

/// Make an Outfile from a file name
pub fn get_writer2<P: AsRef<Path>>(name: P) -> Result<Outfile> {
    let name = name.as_ref().as_os_str();
    let inner: Box<dyn Write> = {
        if name == OsStr::new("-") {
            Box::new(io::stdout())
        } else if name == OsStr::new("--") {
            Box::new(io::stderr())
        } else {
            Box::new(fs::OpenOptions::new().write(true).create(true).open(name)?)
        }
    };
    Ok(io::BufWriter::new(inner))
}

/// Make an Outfile from a file name
pub fn get_writer(name: &str) -> Result<Outfile> {
    let inner: Box<dyn Write> = {
        if name == "-" {
            Box::new(io::stdout())
        } else if name == "--" {
            Box::new(io::stderr())
        } else {
            Box::new(fs::OpenOptions::new().write(true).create(true).open(name)?)
        }
    };
    Ok(io::BufWriter::new(inner))
}

// should return Cow<>
fn unescape_vec(data: &[u8]) -> Vec<u8> {
    let mut ret: Vec<u8> = Vec::with_capacity(data.len());
    let mut last_was_slash = false;
    for x in data {
        if last_was_slash {
            ret.push(match x {
                b'n' => b'\n',
                b't' => b'\t',
                b's' => b' ',
                ch => *ch,
            });
            last_was_slash = false;
        } else if x == &b'\\' {
            last_was_slash = true;
        } else {
            ret.push(*x);
        }
    }
    if last_was_slash {
        ret.push(b'\\');
    }
    ret
}

/// Make an Infile from a file name
pub fn get_reader(name: &str) -> Result<Infile> {
    let inner: Box<dyn Read> = {
        if name == "-" {
            //	    unsafe { Box::new(std::fs::File::from_raw_fd(1)) }
            Box::new(io::stdin())
        } else if name.starts_with("s3://") {
            Box::new(S3Reader::new_path(name)?)
        } else if let Some(stripped) = name.strip_prefix("<<") {
            Box::new(std::io::Cursor::new(unescape_vec(stripped.as_bytes())))
        } else {
            Box::new(fs::File::open(name)?)
        }
    };
    let mut outer = io::BufReader::new(inner);
    let start = outer.fill_buf()?;
    if start.starts_with(&[0x1fu8, 0x8bu8, 0x08u8]) {
        outer = io::BufReader::new(Box::new(MultiGzDecoder::new(outer)));
    }
    Ok(Infile::new(outer))
}

#[derive(Debug, Default)]
/// shared context for any input file type
struct InfileContext {
    // CDX header, contructed if necessary
    header: StringLine,
    // delimter
    delim: u8,
    // have we read all the btes of the file
    is_done: bool,
    // is the file length zero
    is_empty: bool,
    // was there a CDX header?
    has_header: bool,
}

/// create appropriate header from first line of file
pub fn make_header(line: &[u8]) -> StringLine {
    let mut s = StringLine::new();
    if is_cdx(line) {
        s.line = String::from_utf8_lossy(&line[5..]).to_string();
    } else {
        s.line = String::new();
        for x in 1..=line.split(|ch| *ch == b'\t').count() {
            s.line.push_str(&format!("c{}\t", x));
        }
        s.line.pop();
    }
    s.split(b'\t');
    s
}

// FIXME -- specify delimiter
// if CDX and specified and different, then strip header
/// Reader header line, if any, and first line of text
impl InfileContext {
    const fn new() -> Self {
        Self {
            header: StringLine::new(),
            delim: b'\t',
            is_done: true,
            is_empty: true,
            has_header: false,
        }
    }
    fn read_header(&mut self, file: &mut impl BufRead, line: &mut TextLine) -> Result<()> {
        if self.header.read(file)? {
            self.is_done = true;
            self.is_empty = true;
            return Ok(());
        }
        self.is_done = false;
        self.is_empty = false;
        if self.header.line.starts_with(" CDX") {
            self.has_header = true;
            self.delim = self.header.line.as_bytes()[4];
            self.header.split(self.delim);
            self.header.parts.remove(0);
            if line.read(file)? {
                self.is_done = true;
                return Ok(());
            }
            line.split(self.delim);
        } else {
            self.delim = b'\t';
            line.line = self.header.line.as_bytes().to_vec();
            line.split(self.delim);
            let mut head_str = String::new();
            for i in 1..=line.len() {
                head_str += "c";
                head_str += &i.to_string();
                head_str += "\t";
            }
            head_str.pop();
            head_str += "\n";
            let mut fake_head = head_str.as_bytes();
            self.header.read(&mut fake_head)?;
            self.header.split(self.delim);
        }
        Ok(())
    }

src/bin/cdx/verify_main.rs (line 100)

≺ ≻↕

pub fn main(argv: &[String]) -> Result<()> {
    let prog = args::ProgSpec::new("Verify file contents.", args::FileCount::Many);
    const A: [ArgSpec; 10] = [
        arg! {"report", "r", "Number", "How many failures to report before exit."},
        arg! {"first", "f", "Op,Value", "'FirstLine Op Value' must be true. E.g LT,a for first line is less than 'a'."},
        arg! {"last", "l", "Op,Value", "'LastLine Op Value' must be true."},
        arg! {"key", "k", "Spec", "How to compare adjacent lines"},
        arg! {"sort", "s", "", "Check that the file is sorted."},
        arg! {"unique", "u", "", "Check that the file is sorted, with unique lines."},
        arg! {"pattern", "p", "Col,Spec,Pattern", "Select line where this col matches this pattern."},
        arg! {"show-matchers", "", "", "Print available matchers"},
        arg! {"show-const", "", "", "Print available constants"},
        arg! {"show-func", "", "", "Print available functions"},
    ];
    let (args, files) = args::parse(&prog, &A, argv)?;

    let mut list = LineMatcherList::new_with(Combiner::And);
    let mut comp = LineCompList::new();
    let mut do_sort = false;
    let mut do_unique = false;
    let mut max_fails = 5;
    let mut first: Option<CheckLine> = None;
    let mut last: Option<CheckLine> = None;

    for x in args {
        if x.name == "pattern" {
            list.push(&x.value)?;
        } else if x.name == "key" {
            comp.add(&x.value)?;
        } else if x.name == "or" {
            list.multi = Combiner::Or;
        } else if x.name == "fail" {
            max_fails = x.value.to_usize_whole(x.value.as_bytes(), "max fails")?;
        } else if x.name == "sort" {
            do_sort = true;
        } else if x.name == "first" {
            first = Some(CheckLine::new(&x.value)?);
        } else if x.name == "last" {
            last = Some(CheckLine::new(&x.value)?);
        } else if x.name == "unique" {
            do_sort = true;
            do_unique = true;
        } else if x.name == "show-const" {
            expr::show_const();
            return Ok(());
        } else if x.name == "show-func" {
            expr::show_func();
            return Ok(());
        } else {
            unreachable!();
        }
    }
    if comp.is_empty() {
        comp.add("")?;
    }

    let mut fails = 0;
    for x in &files {
        let mut f = Reader::new();
        f.open(x)?;
        if f.is_empty() {
            continue;
        }
        list.lookup(&f.names())?;
        comp.lookup(&f.names())?;
        if f.is_done() {
            continue;
        }
        if first.is_some()
            && !first.as_ref().unwrap().line_ok_verbose(
                f.curr_line(),
                &mut comp,
                f.line_number(),
            )?
        {
            fails += 1;
        }
        let num_cols = f.names().len();
        loop {
            let mut did_fail = false;
            if f.curr().len() != num_cols {
                eprintln!(
                    "Expected {num_cols} columns, but line {} of {} had {}",
                    f.line_number() + 1,
                    x,
                    f.curr().len()
                );
                did_fail = true;
            }
            if !list.ok_verbose(f.curr_line(), f.line_number(), x) {
                did_fail = true;
            }
            if f.getline()? {
                if last.is_some()
                    && !last.as_ref().unwrap().line_ok_verbose(
                        f.prev_line(1),
                        &mut comp,
                        f.line_number() - 1,
                    )?
                {
                    fails += 1;
                }
                break;
            }
            if do_sort {
                did_fail = did_fail || comp_check(&f, &mut comp, do_unique);
            }
            if did_fail {
                fails += 1;
                if fails >= max_fails {
                    break;
                }
            }
        }
        if fails > 0 {
            return Err(Error::Silent);
        }
    }
    Ok(())
}

pub fn strlen(&self) -> usize

How many bytes in the line

pub fn is_empty(&self) -> bool

should always be false, but required by clippy

pub fn get(&self, index: usize) -> &[u8]

Get one column. Return an empty column if index is too big.

Examples found in repository

src/util.rs (line 282)

    fn index(&self, pos: usize) -> &Self::Output {
        self.get(pos)
    }

src/agg.rs (line 378)

    fn add(&mut self, data: &TextLine) {
        self.agg.add(data.get(self.src.num));
    }

src/join.rs (line 120)

    fn write(&self, mut w: impl Write, f: &[Reader]) -> Result<()> {
        w.write_all(f[self.file].curr().get(self.col.num))?;
        Ok(())
    }

src/expr.rs (line 492)

    fn prepare(&mut self, t: &TextLine) {
        for x in &mut self.vars {
            if let Some(c) = x.col {
                x.val = t.get(c).to_f64_lossy();
            }
        }
    }

src/comp.rs (line 492)

≺ ≻↕

    fn comp_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        self.comp.comp(
            left.get(self.cols[left_file].num),
            right.get(self.cols[right_file].num),
        )
    }
    /// compare lines
    fn equal_cols(
        &mut self,
        left: &TextLine,
        right: &TextLine,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        self.comp.equal(
            left.get(self.cols[left_file].num),
            right.get(self.cols[right_file].num),
        )
    }
    /// compare lines
    fn comp_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        delim: u8,
        left_file: usize,
        right_file: usize,
    ) -> Ordering {
        self.comp.comp(
            get_col(left, self.cols[left_file].num, delim),
            get_col(right, self.cols[right_file].num, delim),
        )
    }
    /// compare lines
    fn equal_lines(
        &mut self,
        left: &[u8],
        right: &[u8],
        delim: u8,
        left_file: usize,
        right_file: usize,
    ) -> bool {
        self.comp.equal(
            get_col(left, self.cols[left_file].num, delim),
            get_col(right, self.cols[right_file].num, delim),
        )
    }
    /// resolve named columns; illegal to call any of the others with a file that has not been looked up
    fn lookup(&mut self, fieldnames: &[&str], file_num: usize) -> Result<()> {
        while self.cols.len() < (file_num + 1) {
            self.cols.push(self.cols[self.cols.len() - 1].clone());
        }
        self.cols[file_num].lookup(fieldnames)
    }
    fn fill_cache_cols(&mut self, item: &mut Item, value: &TextLine) {
        self.comp.fill_cache(item, value.get(self.cols[0].num))
    }

    fn fill_cache_line(&mut self, item: &mut Item, value: &[u8], delim: u8) {
        self.comp
            .fill_cache(item, get_col(item.get(value), self.cols[0].num, delim))
    }

    fn set(&mut self, value: &[u8]) {
        self.comp.set(value)
    }

    fn comp_self_cols(&mut self, right: &TextLine) -> Ordering {
        self.comp.comp_self(right.get(self.cols[0].num))
    }

    fn equal_self_cols(&mut self, right: &TextLine) -> bool {
        self.comp.equal_self(right.get(self.cols[0].num))
    }

src/matcher.rs (line 975)

≺ ≻↕

    fn ok(&mut self, line: &TextLine) -> bool {
        let mut yes: usize = 0;
        let mut no: usize = 0;
        for x in self.col.col.get_cols() {
            let o = self.comp.comp(line.get(self.target.num), line.get(x.num));
            if self.op.ord_ok(o) {
                yes += 1;
            } else {
                no += 1;
            }
            let t = self.col.det.match_mid(yes, no);
            if t != Tri::Maybe {
                return t == Tri::Yes;
            }
        }
        self.col.det.match_final(yes, no)
    }
    fn lookup(&mut self, fieldnames: &[&str]) -> Result<()> {
        self.target.lookup(fieldnames)?;
        self.col.lookup(fieldnames)
    }

    fn show(&self) -> String {
        format!(
            "CompMatcher {:?} {:?} {:?} {:?}",
            self.target, self.op, self.col, self.comp
        )
    }
}

/// Does the whole line match a pattern. Implements LineMatch
#[derive(Debug)]
struct WholeMatcher {
    matcher: Matcher,
}

impl WholeMatcher {
    fn new(method: &str, pattern: &str) -> Result<Self> {
        Ok(Self {
            matcher: MatchMaker::make2(method, pattern)?,
        })
    }
}

// LineMatch::matcher could return Result<bool> and then we can put strict back
impl LineMatch for WholeMatcher {
    fn ok(&mut self, line: &TextLine) -> bool {
        self.matcher.negate
            ^ if self.matcher.string {
                self.matcher
                    .smatch(&String::from_utf8_lossy(line.line_nl()))
            } else {
                self.matcher.umatch(line.line_nl())
            }
    }

    fn lookup(&mut self, _fieldnames: &[&str]) -> Result<()> {
        Ok(())
    }
    fn show(&self) -> String {
        format!("match whole line against {}", self.matcher)
    }
}

/// Does the line have a certain number of columns
#[derive(Debug)]
struct CountMatcher {
    count: usize,
}

impl CountMatcher {
    fn new(pattern: &str) -> Result<Self> {
        Ok(Self {
            count: pattern.to_usize_whole(pattern.as_bytes(), "count matcher")?,
        })
    }
}

// LineMatch::matcher could return Result<bool> and then we can put strict back
impl LineMatch for CountMatcher {
    fn ok(&mut self, line: &TextLine) -> bool {
        line.len() == self.count
    }

    fn lookup(&mut self, _fieldnames: &[&str]) -> Result<()> {
        Ok(())
    }
    fn show(&self) -> String {
        format!("Does line have {} columns", self.count)
    }
    fn ok_verbose(&mut self, line: &TextLine, line_num: usize, fname: &str) -> bool {
        let ret = self.ok(line);
        if !ret {
            eprintln!(
                "Line {} of {} had {} columns where {} were expected.",
                line_num,
                fname,
                line.len(),
                self.count
            );
        }
        ret
    }
}

/// Does a particular column of a line match a pattern. Implements LineMatch
#[derive(Debug)]
struct ColSetMatcher {
    matcher: Matcher,
    col: ColumnSet,
    det: Determiner,
}

impl ColSetMatcher {
    /// new from parts
    fn new(cols: &str, method: &str, pattern: &str) -> Result<Self> {
        if let Some((a, b)) = cols.split_once(',') {
            Ok(Self {
                matcher: MatchMaker::make2(method, pattern)?,
                col: ColumnSet::from_spec(b)?,
                det: Determiner::new(a)?,
            })
        } else {
            err!("ColumnGroup format is Determiner,ColumnSet")
        }
    }
}

impl LineMatch for ColSetMatcher {
    fn ok(&mut self, line: &TextLine) -> bool {
        let mut yes = 0;
        let mut no = 0;
        for x in self.col.get_cols() {
            let did_match = if self.matcher.string {
                self.matcher
                    .smatch(&String::from_utf8_lossy(line.get(x.num)))
            } else {
                self.matcher.umatch(line.get(x.num))
            };
            if did_match {
                yes += 1;
            } else {
                no += 1;
            }
            let res = self.det.match_mid(yes, no);
            if res != Tri::Maybe {
                return self.matcher.negate ^ (res == Tri::Yes);
            }
        }
        self.matcher.negate ^ self.det.match_final(yes, no)
    }

    fn lookup(&mut self, fieldnames: &[&str]) -> Result<()> {
        self.col.lookup(fieldnames)
    }
    fn show(&self) -> String {
        format!("match {:?} against {}", self.col, self.matcher)
    }
}

/// Does a particular column of a line match a pattern. Implements LineMatch
#[derive(Debug)]
struct ColMatcher {
    matcher: Matcher,
    col: NamedCol,
}

impl ColMatcher {
    /// Column,Spec,Pattern
    /// Pattern may have additional commas
    fn new(cols: &str, method: &str, pattern: &str) -> Result<Self> {
        let mut nc = NamedCol::new();
        nc.parse(cols)?;
        Ok(Self {
            matcher: MatchMaker::make2(method, pattern)?,
            col: nc,
        })
    }
}

// LineMatch::matcher could return Result<bool> and then we can put strict back
impl LineMatch for ColMatcher {
    fn ok(&mut self, line: &TextLine) -> bool {
        self.matcher.negate
            ^ if self.matcher.string {
                self.matcher
                    .smatch(&String::from_utf8_lossy(line.get(self.col.num)))
            } else {
                self.matcher.umatch(line.get(self.col.num))
            }
    }

Additional examples can be found in:

• src/column.rs

pub fn read<T: BufRead>(&mut self, f: &mut T) -> Result<bool>

Read a new line from a file, should generally be followed by split

Examples found in repository

src/util.rs (line 821)

≺ ≻↕

    fn read_header(&mut self, file: &mut impl BufRead, line: &mut TextLine) -> Result<()> {
        if self.header.read(file)? {
            self.is_done = true;
            self.is_empty = true;
            return Ok(());
        }
        self.is_done = false;
        self.is_empty = false;
        if self.header.line.starts_with(" CDX") {
            self.has_header = true;
            self.delim = self.header.line.as_bytes()[4];
            self.header.split(self.delim);
            self.header.parts.remove(0);
            if line.read(file)? {
                self.is_done = true;
                return Ok(());
            }
            line.split(self.delim);
        } else {
            self.delim = b'\t';
            line.line = self.header.line.as_bytes().to_vec();
            line.split(self.delim);
            let mut head_str = String::new();
            for i in 1..=line.len() {
                head_str += "c";
                head_str += &i.to_string();
                head_str += "\t";
            }
            head_str.pop();
            head_str += "\n";
            let mut fake_head = head_str.as_bytes();
            self.header.read(&mut fake_head)?;
            self.header.split(self.delim);
        }
        Ok(())
    }
}

/// where are we, in which file?
#[derive(Debug, Default, Clone)]
pub struct FileLocData {
    /// full file name
    name: String,
    /// byte offset, uncompressed
    bytes: usize,
    /// line number
    line: usize,
}

/// types of file location data
#[derive(Debug, Copy, Clone)]
enum FileLocItem {
    /// byte offset of start of line
    Bytes,
    /// 1-based line number
    Line,
    /// file name, with given number of parts
    Name(usize),
}
impl FileLocItem {
    fn new(spec: &str) -> Result<Self> {
        if spec.eq_ignore_ascii_case("bytes") {
            Ok(Self::Bytes)
        } else if spec.eq_ignore_ascii_case("line") {
            Ok(Self::Line)
        } else if spec.eq_ignore_ascii_case("name") {
            Ok(Self::Name(0))
        } else if let Some((a, b)) = spec.split_once('.') {
            if a.eq_ignore_ascii_case("name") {
                Ok(Self::Name(
                    b.to_usize_whole(spec.as_bytes(), "File location")?,
                ))
            } else {
                err!("File Loc must be once of Bytes, Line, Name : '{}'", spec)
            }
        } else {
            err!("File Loc must be once of Bytes, Line, Name : '{}'", spec)
        }
    }
    const fn dflt_name(&self) -> &'static str {
        match self {
            Self::Bytes => "bytes",
            Self::Line => "line",
            Self::Name(_) => "filename",
        }
    }
    fn write_data(&mut self, data: &mut impl Write, loc: &FileLocData) -> Result<()> {
        match self {
            Self::Bytes => write!(data, "{}", loc.bytes).unwrap(),
            Self::Line => write!(data, "{}", loc.line).unwrap(),
            Self::Name(n) => {
                if *n == 0 {
                    data.write_all(loc.name.as_bytes())?;
                } else {
                    // FIXME - this should really be cached somehow
                    data.write_all(loc.name.tail_path_u8(*n, b'/').as_bytes())?;
                }
            }
        }
        Ok(())
    }
}

/// FileLocItem with column name
#[derive(Debug, Clone)]
struct FileLoc {
    col_name: String,
    item: FileLocItem,
}
impl FileLoc {
    fn new(spec: &str) -> Result<Self> {
        if let Some((a, b)) = spec.split_once(':') {
            Ok(Self {
                col_name: a.to_string(),
                item: FileLocItem::new(b)?,
            })
        } else {
            let item = FileLocItem::new(spec)?;
            Ok(Self {
                col_name: item.dflt_name().to_string(),
                item,
            })
        }
    }
    fn write_data(&mut self, data: &mut impl Write, loc: &FileLocData) -> Result<()> {
        self.item.write_data(data, loc)
    }
}

/// List of FileLoc
#[derive(Default, Debug, Clone)]
pub struct FileLocList {
    v: Vec<FileLoc>,
}
impl FileLocList {
    /// new
    pub fn new() -> Self {
        Self::default()
    }
    /// new
    pub fn is_empty(&self) -> bool {
        self.v.is_empty()
    }
    /// add Name:Spec
    pub fn push(&mut self, spec: &str) -> Result<()> {
        for x in spec.split(',') {
            self.v.push(FileLoc::new(x)?);
        }
        Ok(())
    }
    /// fill data with file loc data
    pub fn write_data(
        &mut self,
        data: &mut impl Write,
        delim: u8,
        loc: &FileLocData,
    ) -> Result<()> {
        for x in &mut self.v {
            x.write_data(data, loc)?;
            data.write_all(&[delim])?;
        }
        Ok(())
    }
    /// fill data with column names
    pub fn write_names(&mut self, data: &mut String, delim: u8) {
        for x in &mut self.v {
            data.push_str(&x.col_name);
            data.push(delim as char);
        }
    }
    /// add new columns to header
    pub fn add(&self, header: &mut ColumnHeader) -> Result<()> {
        for x in &self.v {
            header.push(&x.col_name)?;
        }
        Ok(())
    }
}

#[derive(Debug, Default)]
/// Text file reader. Lines broken into columns, with lookback
pub struct Reader {
    file: Infile,
    lines: Vec<TextLine>,
    cont: InfileContext,
    do_split: bool,
    curr: usize,
    loc: FileLocData,
}

impl Reader {
    /// loc
    pub const fn loc(&self) -> &FileLocData {
        &self.loc
    }
    /// make a new Reader
    pub fn new() -> Self {
        Self::new_with(1)
    }
    /// set to false to skip breaking into columns
    pub fn do_split(&mut self, val: bool) {
        self.do_split = val;
    }
    /// make a new Reader, with explicit lookback
    pub fn new_with(lookback: usize) -> Self {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        Self {
            file: Infile::default(),
            lines,
            cont: InfileContext::new(),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        }
    }
    /// make a new Reader
    pub fn new_open(name: &str) -> Result<Self> {
        Self::new_open_with(name, 1)
    }
    /// make a new Reader
    pub fn new_open_with(name: &str, lookback: usize) -> Result<Self> {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        let mut tmp = Self {
            file: get_reader(name)?,
            lines,
            cont: InfileContext::new(),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        };
        tmp.cont.read_header(&mut *tmp.file, &mut tmp.lines[0])?;
        tmp.loc.name = name.to_string();
        tmp.loc.line = 1;
        tmp.loc.bytes = if tmp.has_header() {
            tmp.header().line.len()
        } else {
            0
        };
        Ok(tmp)
    }
    /// get current line contents, without the trailing newline
    pub fn curr_nl(&self) -> &[u8] {
        let line = self.curr_line();
        &line.line[0..line.line.len() - 1]
    }
    /// get previous line contents, without the trailing newline
    pub fn prev_nl(&self, n: usize) -> &[u8] {
        let line = self.prev_line(n);
        &line.line[0..line.line.len() - 1]
    }
    /// get delimiter
    pub const fn delim(&self) -> u8 {
        self.cont.delim
    }
    /// get column names
    pub fn names(&self) -> Vec<&str> {
        self.cont.header.vec()
    }
    /// write the current text line with newline
    pub fn write(&self, w: &mut impl Write) -> Result<()> {
        w.write_all(&self.curr_line().line)?;
        Ok(())
    }
    /// open file for reading
    pub fn open(&mut self, name: &str) -> Result<()> {
        self.file = get_reader(name)?;
        self.cont.read_header(&mut *self.file, &mut self.lines[0])
    }
    /// The full text of the header, without the trailing newline
    pub const fn header_line(&self) -> &String {
        &self.cont.header.line
    }
    /// was file zero bytes?
    pub const fn is_empty(&self) -> bool {
        self.cont.is_empty
    }
    /// have we hit EOF?
    pub const fn is_done(&self) -> bool {
        self.cont.is_done
    }
    /// line number of curr_line
    pub const fn line_number(&self) -> usize {
        self.loc.line
    }
    fn incr(&mut self) {
        self.loc.line += 1;
        self.curr += 1;
        if self.curr >= self.lines.len() {
            self.curr = 0;
        }
    }
    /// get next line of text
    pub fn getline(&mut self) -> Result<bool> {
        self.loc.bytes += self.curr().line.len();
        self.incr();
        if self.lines[self.curr].read(&mut *self.file)? {
            self.cont.is_done = true;
        } else if self.do_split {
            self.lines[self.curr].split(self.cont.delim);
        }
        Ok(self.cont.is_done)
    }

pub fn split(&mut self, delim: u8)

split the line into columns hypothetically you could split on one delimiter, do some work, then split on a different delimiter.

Examples found in repository

src/agg.rs (line 799)

↕

    fn result(&mut self, w: &mut dyn Write, fmt: NumFormat) -> Result<()> {
        self.data.split(self.delim);
        if self.do_sort {
            self.data.parts.sort_by(|a, b| {
                self.comp
                    .comp(a.get(&self.data.line), b.get(&self.data.line))
            });
        }
        if self.do_uniq {
            self.data.parts.dedup_by(|a, b| {
                self.comp
                    .equal(a.get(&self.data.line), b.get(&self.data.line))
            });
        }
        if self.do_count {
            fmt.print(self.data.parts().len() as f64, w)?;
        } else {
            let mut num_written = 0;
            for x in &self.data.parts {
                if x.len() >= self.min_len && x.len() <= self.max_len {
                    if num_written > 0 {
                        w.write_all(&[self.out_delim])?;
                    }
                    w.write_all(x.get(self.data.line()))?;
                    num_written += 1;
                    if num_written >= self.max_parts {
                        break;
                    }
                }
            }
        }
        Ok(())
    }

src/util.rs (line 825)

≺ ≻↕

    fn read_header(&mut self, file: &mut impl BufRead, line: &mut TextLine) -> Result<()> {
        if self.header.read(file)? {
            self.is_done = true;
            self.is_empty = true;
            return Ok(());
        }
        self.is_done = false;
        self.is_empty = false;
        if self.header.line.starts_with(" CDX") {
            self.has_header = true;
            self.delim = self.header.line.as_bytes()[4];
            self.header.split(self.delim);
            self.header.parts.remove(0);
            if line.read(file)? {
                self.is_done = true;
                return Ok(());
            }
            line.split(self.delim);
        } else {
            self.delim = b'\t';
            line.line = self.header.line.as_bytes().to_vec();
            line.split(self.delim);
            let mut head_str = String::new();
            for i in 1..=line.len() {
                head_str += "c";
                head_str += &i.to_string();
                head_str += "\t";
            }
            head_str.pop();
            head_str += "\n";
            let mut fake_head = head_str.as_bytes();
            self.header.read(&mut fake_head)?;
            self.header.split(self.delim);
        }
        Ok(())
    }
}

/// where are we, in which file?
#[derive(Debug, Default, Clone)]
pub struct FileLocData {
    /// full file name
    name: String,
    /// byte offset, uncompressed
    bytes: usize,
    /// line number
    line: usize,
}

/// types of file location data
#[derive(Debug, Copy, Clone)]
enum FileLocItem {
    /// byte offset of start of line
    Bytes,
    /// 1-based line number
    Line,
    /// file name, with given number of parts
    Name(usize),
}
impl FileLocItem {
    fn new(spec: &str) -> Result<Self> {
        if spec.eq_ignore_ascii_case("bytes") {
            Ok(Self::Bytes)
        } else if spec.eq_ignore_ascii_case("line") {
            Ok(Self::Line)
        } else if spec.eq_ignore_ascii_case("name") {
            Ok(Self::Name(0))
        } else if let Some((a, b)) = spec.split_once('.') {
            if a.eq_ignore_ascii_case("name") {
                Ok(Self::Name(
                    b.to_usize_whole(spec.as_bytes(), "File location")?,
                ))
            } else {
                err!("File Loc must be once of Bytes, Line, Name : '{}'", spec)
            }
        } else {
            err!("File Loc must be once of Bytes, Line, Name : '{}'", spec)
        }
    }
    const fn dflt_name(&self) -> &'static str {
        match self {
            Self::Bytes => "bytes",
            Self::Line => "line",
            Self::Name(_) => "filename",
        }
    }
    fn write_data(&mut self, data: &mut impl Write, loc: &FileLocData) -> Result<()> {
        match self {
            Self::Bytes => write!(data, "{}", loc.bytes).unwrap(),
            Self::Line => write!(data, "{}", loc.line).unwrap(),
            Self::Name(n) => {
                if *n == 0 {
                    data.write_all(loc.name.as_bytes())?;
                } else {
                    // FIXME - this should really be cached somehow
                    data.write_all(loc.name.tail_path_u8(*n, b'/').as_bytes())?;
                }
            }
        }
        Ok(())
    }
}

/// FileLocItem with column name
#[derive(Debug, Clone)]
struct FileLoc {
    col_name: String,
    item: FileLocItem,
}
impl FileLoc {
    fn new(spec: &str) -> Result<Self> {
        if let Some((a, b)) = spec.split_once(':') {
            Ok(Self {
                col_name: a.to_string(),
                item: FileLocItem::new(b)?,
            })
        } else {
            let item = FileLocItem::new(spec)?;
            Ok(Self {
                col_name: item.dflt_name().to_string(),
                item,
            })
        }
    }
    fn write_data(&mut self, data: &mut impl Write, loc: &FileLocData) -> Result<()> {
        self.item.write_data(data, loc)
    }
}

/// List of FileLoc
#[derive(Default, Debug, Clone)]
pub struct FileLocList {
    v: Vec<FileLoc>,
}
impl FileLocList {
    /// new
    pub fn new() -> Self {
        Self::default()
    }
    /// new
    pub fn is_empty(&self) -> bool {
        self.v.is_empty()
    }
    /// add Name:Spec
    pub fn push(&mut self, spec: &str) -> Result<()> {
        for x in spec.split(',') {
            self.v.push(FileLoc::new(x)?);
        }
        Ok(())
    }
    /// fill data with file loc data
    pub fn write_data(
        &mut self,
        data: &mut impl Write,
        delim: u8,
        loc: &FileLocData,
    ) -> Result<()> {
        for x in &mut self.v {
            x.write_data(data, loc)?;
            data.write_all(&[delim])?;
        }
        Ok(())
    }
    /// fill data with column names
    pub fn write_names(&mut self, data: &mut String, delim: u8) {
        for x in &mut self.v {
            data.push_str(&x.col_name);
            data.push(delim as char);
        }
    }
    /// add new columns to header
    pub fn add(&self, header: &mut ColumnHeader) -> Result<()> {
        for x in &self.v {
            header.push(&x.col_name)?;
        }
        Ok(())
    }
}

#[derive(Debug, Default)]
/// Text file reader. Lines broken into columns, with lookback
pub struct Reader {
    file: Infile,
    lines: Vec<TextLine>,
    cont: InfileContext,
    do_split: bool,
    curr: usize,
    loc: FileLocData,
}

impl Reader {
    /// loc
    pub const fn loc(&self) -> &FileLocData {
        &self.loc
    }
    /// make a new Reader
    pub fn new() -> Self {
        Self::new_with(1)
    }
    /// set to false to skip breaking into columns
    pub fn do_split(&mut self, val: bool) {
        self.do_split = val;
    }
    /// make a new Reader, with explicit lookback
    pub fn new_with(lookback: usize) -> Self {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        Self {
            file: Infile::default(),
            lines,
            cont: InfileContext::new(),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        }
    }
    /// make a new Reader
    pub fn new_open(name: &str) -> Result<Self> {
        Self::new_open_with(name, 1)
    }
    /// make a new Reader
    pub fn new_open_with(name: &str, lookback: usize) -> Result<Self> {
        let mut lines: Vec<TextLine> = Vec::new();
        lines.resize(lookback + 1, TextLine::new());
        let mut tmp = Self {
            file: get_reader(name)?,
            lines,
            cont: InfileContext::new(),
            do_split: true,
            curr: 0,
            loc: FileLocData::default(),
        };
        tmp.cont.read_header(&mut *tmp.file, &mut tmp.lines[0])?;
        tmp.loc.name = name.to_string();
        tmp.loc.line = 1;
        tmp.loc.bytes = if tmp.has_header() {
            tmp.header().line.len()
        } else {
            0
        };
        Ok(tmp)
    }
    /// get current line contents, without the trailing newline
    pub fn curr_nl(&self) -> &[u8] {
        let line = self.curr_line();
        &line.line[0..line.line.len() - 1]
    }
    /// get previous line contents, without the trailing newline
    pub fn prev_nl(&self, n: usize) -> &[u8] {
        let line = self.prev_line(n);
        &line.line[0..line.line.len() - 1]
    }
    /// get delimiter
    pub const fn delim(&self) -> u8 {
        self.cont.delim
    }
    /// get column names
    pub fn names(&self) -> Vec<&str> {
        self.cont.header.vec()
    }
    /// write the current text line with newline
    pub fn write(&self, w: &mut impl Write) -> Result<()> {
        w.write_all(&self.curr_line().line)?;
        Ok(())
    }
    /// open file for reading
    pub fn open(&mut self, name: &str) -> Result<()> {
        self.file = get_reader(name)?;
        self.cont.read_header(&mut *self.file, &mut self.lines[0])
    }
    /// The full text of the header, without the trailing newline
    pub const fn header_line(&self) -> &String {
        &self.cont.header.line
    }
    /// was file zero bytes?
    pub const fn is_empty(&self) -> bool {
        self.cont.is_empty
    }
    /// have we hit EOF?
    pub const fn is_done(&self) -> bool {
        self.cont.is_done
    }
    /// line number of curr_line
    pub const fn line_number(&self) -> usize {
        self.loc.line
    }
    fn incr(&mut self) {
        self.loc.line += 1;
        self.curr += 1;
        if self.curr >= self.lines.len() {
            self.curr = 0;
        }
    }
    /// get next line of text
    pub fn getline(&mut self) -> Result<bool> {
        self.loc.bytes += self.curr().line.len();
        self.incr();
        if self.lines[self.curr].read(&mut *self.file)? {
            self.cont.is_done = true;
        } else if self.do_split {
            self.lines[self.curr].split(self.cont.delim);
        }
        Ok(self.cont.is_done)
    }

pub fn vec(&self) -> Vec<&[u8]>

return all parts as a vector

Trait Implementations

impl Clone for TextLine

fn clone(&self) -> TextLine

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TextLine

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for TextLine

fn default() -> TextLine

Returns the “default value” for a type.

impl Display for TextLine

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Index<usize> for TextLine

type Output = [u8]

The returned type after indexing.

fn index(&self, pos: usize) -> &Self::Output

Performs the indexing (container[index]) operation.

impl<'a> IntoIterator for &'a TextLine

type Item = &'a [u8]

The type of the elements being iterated over.

type IntoIter = TextLineIter<'a>

Which kind of iterator are we turning this into?

fn into_iter(self) -> Self::IntoIter

Creates an iterator from a value.