kobold_csv/

dialect.rs

//! The delimited-file DIALECT layer -- what makes kobold-csv CSV-aware.
//!
//! A [`Dialect`] fixes the four parameters that decide how a flat record of string fields becomes one line
//! of delimited text and back: the field `delimiter` (`,` `|` `\t` ...), the `quote` character, whether to
//! quote `quote_all` fields unconditionally, and the [`LineTerminator`] used between rows.
//!
//! ## Quoting (RFC-4180 style)
//!
//! A field is QUOTED when it contains the delimiter, the quote character, a carriage return, or a line feed
//! (or always, when `quote_all`). Inside a quoted field an embedded quote is DOUBLED (`"` -> `""`). This is
//! the de-facto CSV escaping convention (RFC 4180, and what spreadsheets/`COPY ... CSV` agree on). Doing it
//! by hand -- rather than depending on a CSV crate -- keeps kobold-csv std-only and lets the escaping itself
//! be evidence (`KOBOLD.CSV.ESCAPE`): the writer and the fail-closed reader are exact inverses.
//!
//! ## Fail-closed reader
//!
//! [`parse_row`] is deliberately strict: a quote that opens a field but never closes, or stray text after a
//! closing quote, is a [`Finding`], NEVER a best-effort guess. A reconciliation tool must be able to trust
//! that a row either parsed exactly or was rejected with a reason -- silent recovery would corrupt custody.
//!
//! This module is independent of GnuCOBOL/libcob.

use crate::model::Finding;

// Character literals are spelled as hex byte constants where a bare literal would be visually ambiguous
// (a comma, a tab, a double quote). This keeps the source unambiguous and ASCII-only.
const COMMA: u8 = 0x2c; // ','
const PIPE: u8 = 0x7c; // '|'
const TAB: u8 = 0x09; // '\t'
const DQUOTE: u8 = 0x22; // '"'
const CR: u8 = 0x0d; // '\r'
const LF: u8 = 0x0a; // '\n'

/// The line terminator a dialect writes between rows (and tolerates when reading).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LineTerminator {
    /// A single line feed (`\n`) -- the Unix convention.
    Lf,
    /// A carriage return + line feed (`\r\n`) -- the RFC-4180 / DOS convention.
    CrLf,
}

impl LineTerminator {
    /// The bytes this terminator emits.
    pub fn bytes(self) -> &'static [u8] {
        match self {
            LineTerminator::Lf => b"\n",
            LineTerminator::CrLf => b"\r\n",
        }
    }
}

/// A delimited-file dialect: delimiter, quote char, quote-all flag, and line terminator.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Dialect {
    /// The field delimiter byte (e.g. `,` `|` tab).
    pub delimiter: u8,
    /// The quote byte (conventionally `"`).
    pub quote: u8,
    /// When true, every field is quoted regardless of content (some downstream loaders require it).
    pub quote_all: bool,
    /// The terminator written between rows.
    pub line_terminator: LineTerminator,
}

impl Dialect {
    /// Comma-separated, double-quoted, LF terminated -- the default CSV dialect.
    pub fn csv() -> Self {
        Dialect { delimiter: COMMA, quote: DQUOTE, quote_all: false, line_terminator: LineTerminator::Lf }
    }

    /// Pipe-separated, double-quoted, LF terminated -- common in mainframe extract files where commas occur
    /// inside data.
    pub fn pipe() -> Self {
        Dialect { delimiter: PIPE, quote: DQUOTE, quote_all: false, line_terminator: LineTerminator::Lf }
    }

    /// Tab-separated, double-quoted, LF terminated (TSV).
    pub fn tab() -> Self {
        Dialect { delimiter: TAB, quote: DQUOTE, quote_all: false, line_terminator: LineTerminator::Lf }
    }
}

/// Decide whether `field` must be quoted under dialect `d`: it must when `quote_all` is set, or when the
/// text contains the delimiter, the quote char, CR, or LF.
fn needs_quoting(field: &str, d: &Dialect) -> bool {
    if d.quote_all {
        return true;
    }
    field.bytes().any(|b| b == d.delimiter || b == d.quote || b == CR || b == LF)
}

/// Write one `field` to `out` under dialect `d`, quoting and doubling embedded quotes as required. The
/// inverse of [`parse_row`]'s field handling: `write_field`/`parse_row` are an exact round-trip pair.
pub fn write_field(field: &str, d: &Dialect, out: &mut String) {
    if !needs_quoting(field, d) {
        out.push_str(field);
        return;
    }
    out.push(d.quote as char);
    for b in field.bytes() {
        if b == d.quote {
            // Double the embedded quote.
            out.push(d.quote as char);
            out.push(d.quote as char);
        } else {
            out.push(b as char);
        }
    }
    out.push(d.quote as char);
}

/// Write a full `row` of fields to `out` under dialect `d`, joining with the delimiter and appending the
/// line terminator. Does NOT add a trailing terminator beyond this row's own.
pub fn write_row(row: &[String], d: &Dialect, out: &mut String) {
    for (i, field) in row.iter().enumerate() {
        if i > 0 {
            out.push(d.delimiter as char);
        }
        write_field(field, d, out);
    }
    for &b in d.line_terminator.bytes() {
        out.push(b as char);
    }
}

/// `KOBOLD.CSV.ESCAPE` / parse evidence: FAIL-CLOSED parse of one `line` of delimited bytes into its fields
/// under dialect `d`.
///
/// `line` is one logical record's bytes WITHOUT its line terminator (a trailing CR is tolerated and stripped
/// to support CrLf splitting). Handles: unquoted fields, quoted fields, doubled quotes inside quoted fields,
/// and embedded delimiters/CR/LF inside quoted fields.
///
/// Fails closed (returns a [`Finding`], never a best-effort row) on a quote that opens a field but never
/// closes, or any non-delimiter text immediately after a closing quote.
pub fn parse_row(line: &[u8], d: &Dialect) -> Result<Vec<String>, Finding> {
    // Tolerate a single trailing CR (the case where a CrLf file was split on LF only).
    let line = if line.last() == Some(&CR) { &line[..line.len() - 1] } else { line };

    let mut fields: Vec<String> = Vec::new();
    let mut i = 0usize;
    let n = line.len();

    loop {
        // Parse one field starting at i.
        let mut field = String::new();
        if i < n && line[i] == d.quote {
            // Quoted field.
            i += 1; // consume opening quote
            loop {
                if i >= n {
                    return Err(Finding::new(
                        "CSV_UNTERMINATED_QUOTE",
                        "quoted field opened but never closed before end of line".to_string(),
                    ));
                }
                let b = line[i];
                if b == d.quote {
                    // Either a doubled quote (escaped) or the closing quote.
                    if i + 1 < n && line[i + 1] == d.quote {
                        field.push(d.quote as char);
                        i += 2;
                        continue;
                    }
                    // Closing quote.
                    i += 1;
                    // Must be at end-of-line or a delimiter now; anything else is malformed.
                    if i < n && line[i] != d.delimiter {
                        return Err(Finding::new(
                            "CSV_TEXT_AFTER_QUOTE",
                            format!(
                                "unexpected byte 0x{:02x} after closing quote (field {})",
                                line[i],
                                fields.len()
                            ),
                        ));
                    }
                    break;
                }
                field.push(b as char);
                i += 1;
            }
        } else {
            // Unquoted field: read up to the next delimiter or end of line. A quote inside an unquoted
            // field is taken literally (it is not a CSV special there).
            while i < n && line[i] != d.delimiter {
                field.push(line[i] as char);
                i += 1;
            }
        }

        fields.push(field);

        if i >= n {
            break;
        }
        // line[i] is the delimiter.
        i += 1;
        // A trailing delimiter at end-of-line means a final empty field.
        if i >= n {
            fields.push(String::new());
            break;
        }
    }

    Ok(fields)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn plain_fields_unquoted() {
        let d = Dialect::csv();
        let mut s = String::new();
        write_row(&["A".into(), "B".into(), "C".into()], &d, &mut s);
        assert_eq!(s, "A,B,C\n");
        assert_eq!(parse_row(b"A,B,C", &d).unwrap(), vec!["A", "B", "C"]);
    }

    #[test]
    fn field_with_delimiter_is_quoted_and_roundtrips() {
        let d = Dialect::csv();
        let mut s = String::new();
        write_field("a,b", &d, &mut s);
        assert_eq!(s, "\"a,b\"");
        assert_eq!(parse_row(s.as_bytes(), &d).unwrap(), vec!["a,b"]);
    }

    #[test]
    fn embedded_quote_is_doubled_and_roundtrips() {
        // KOBOLD.CSV.ESCAPE: a field containing both the delimiter and an embedded quote.
        let d = Dialect::csv();
        let field = "say \"hi\", now"; // contains a comma and double quotes
        let mut s = String::new();
        write_field(field, &d, &mut s);
        // -> "say ""hi"", now"
        assert_eq!(s, "\"say \"\"hi\"\", now\"");
        assert_eq!(parse_row(s.as_bytes(), &d).unwrap(), vec![field]);
    }

    #[test]
    fn quote_all_quotes_plain_fields() {
        let mut d = Dialect::csv();
        d.quote_all = true;
        let mut s = String::new();
        write_field("AB", &d, &mut s);
        assert_eq!(s, "\"AB\"");
    }

    #[test]
    fn trailing_empty_field() {
        let d = Dialect::csv();
        assert_eq!(parse_row(b"A,", &d).unwrap(), vec!["A", ""]);
        assert_eq!(parse_row(b",", &d).unwrap(), vec!["", ""]);
    }

    #[test]
    fn crlf_trailing_cr_tolerated() {
        let d = Dialect::csv();
        assert_eq!(parse_row(b"A,B\r", &d).unwrap(), vec!["A", "B"]);
    }

    #[test]
    fn fail_closed_unterminated_quote() {
        let d = Dialect::csv();
        let f = parse_row(b"\"abc", &d).expect_err("must fail closed");
        assert_eq!(f.code, "CSV_UNTERMINATED_QUOTE");
    }

    #[test]
    fn fail_closed_text_after_quote() {
        let d = Dialect::csv();
        let f = parse_row(b"\"ab\"c", &d).expect_err("must fail closed");
        assert_eq!(f.code, "CSV_TEXT_AFTER_QUOTE");
    }

    #[test]
    fn pipe_dialect() {
        let d = Dialect::pipe();
        let mut s = String::new();
        write_row(&["X".into(), "a|b".into()], &d, &mut s);
        assert_eq!(s, "X|\"a|b\"\n");
        assert_eq!(parse_row(b"X|\"a|b\"", &d).unwrap(), vec!["X", "a|b"]);
    }

    #[test]
    fn tab_dialect() {
        let d = Dialect::tab();
        assert_eq!(d.delimiter, 0x09);
        let mut s = String::new();
        write_row(&["A".into(), "B".into()], &d, &mut s);
        assert_eq!(s, "A\tB\n");
    }
}