csv-spectrum 0.1.1

CSV parser conformance test corpus
Documentation
//! Loader-contract tests.
//!
//! These prove the public surface: 12 samples, ascending order by name, raw
//! byte slices that match the files on disk exactly, with line endings and
//! special bytes intact.

use csv_spectrum::{spectrum, Sample, SAMPLES};

/// The 12 base names in the order `spectrum` must return them.
const EXPECTED_NAMES: [&str; 12] = [
    "comma_in_quotes",
    "empty",
    "empty_crlf",
    "escaped_quotes",
    "json",
    "location_coordinates",
    "newlines",
    "newlines_crlf",
    "quotes_and_newlines",
    "simple",
    "simple_crlf",
    "utf8",
];

#[test]
fn returns_twelve_samples() {
    assert_eq!(spectrum().len(), 12);
    assert_eq!(SAMPLES.len(), 12);
}

#[test]
fn names_match_expected_set_and_order() {
    let names: Vec<&str> = spectrum().iter().map(|s| s.name).collect();
    assert_eq!(names, EXPECTED_NAMES);
}

#[test]
fn names_are_ascending() {
    let names: Vec<&str> = spectrum().iter().map(|s| s.name).collect();
    let mut sorted = names.clone();
    sorted.sort_unstable();
    assert_eq!(names, sorted, "samples must be in ascending name order");
}

#[test]
fn spectrum_matches_const_table() {
    assert_eq!(spectrum(), SAMPLES.as_slice());
}

/// Each sample's bytes must equal the matching file, byte for byte.
#[test]
fn bytes_match_files_on_disk() {
    macro_rules! check {
        ($name:literal) => {{
            let s = find($name);
            assert_eq!(
                s.csv,
                include_bytes!(concat!("../fixtures/csvs/", $name, ".csv")),
                "csv bytes differ for {}",
                $name
            );
            assert_eq!(
                s.json,
                include_bytes!(concat!("../fixtures/json/", $name, ".json")),
                "json bytes differ for {}",
                $name
            );
        }};
    }
    check!("comma_in_quotes");
    check!("empty");
    check!("empty_crlf");
    check!("escaped_quotes");
    check!("json");
    check!("location_coordinates");
    check!("newlines");
    check!("newlines_crlf");
    check!("quotes_and_newlines");
    check!("simple");
    check!("simple_crlf");
    check!("utf8");
}

/// Exact byte sizes. A mismatch means an editor or git rewrote a fixture.
#[test]
fn byte_sizes_are_exact() {
    let expected: &[(&str, usize, usize)] = &[
        ("comma_in_quotes", 68, 127),
        ("empty", 19, 72),
        ("empty_crlf", 21, 72),
        ("escaped_quotes", 25, 85),
        ("json", 65, 93),
        ("location_coordinates", 117, 155),
        ("newlines", 42, 170),
        ("newlines_crlf", 47, 172),
        ("quotes_and_newlines", 27, 89),
        ("simple", 12, 53),
        ("simple_crlf", 14, 52),
        ("utf8", 18, 103),
    ];
    for &(name, csv_len, json_len) in expected {
        let s = find(name);
        assert_eq!(s.csv.len(), csv_len, "csv length for {name}");
        assert_eq!(s.json.len(), json_len, "json length for {name}");
    }
}

/// The CRLF files carry `\r\n`. Their LF siblings carry no `\r` at all.
#[test]
fn crlf_files_keep_carriage_returns() {
    for name in ["empty_crlf", "simple_crlf", "newlines_crlf"] {
        let s = find(name);
        assert!(
            s.csv.windows(2).any(|w| w == b"\r\n"),
            "{name}.csv should contain CRLF"
        );
    }
    for name in ["empty", "simple", "newlines"] {
        let s = find(name);
        assert!(
            !s.csv.contains(&b'\r'),
            "{name}.csv should contain no carriage returns"
        );
    }
}

/// A CRLF file is longer than its LF sibling by exactly one byte per line break.
#[test]
fn crlf_adds_one_byte_per_line_break() {
    // simple has two line breaks, so simple_crlf is two bytes longer.
    assert_eq!(find("simple_crlf").csv.len(), find("simple").csv.len() + 2);
    // empty has two line breaks too.
    assert_eq!(find("empty_crlf").csv.len(), find("empty").csv.len() + 2);
}

/// Trailing newline state is per file and significant. Spot check both ways.
#[test]
fn trailing_bytes_are_preserved() {
    // No trailing newline.
    assert_eq!(*find("comma_in_quotes").csv.last().unwrap(), b'3');
    assert_eq!(*find("utf8").csv.last().unwrap(), 0xA4);
    assert_eq!(*find("comma_in_quotes").json.last().unwrap(), b']');
    assert_eq!(*find("simple_crlf").json.last().unwrap(), b']');
    // Trailing newline.
    assert_eq!(*find("simple").csv.last().unwrap(), b'\n');
    assert_eq!(*find("simple").json.last().unwrap(), b'\n');
    assert_eq!(*find("location_coordinates").json.last().unwrap(), b'\n');
    assert_eq!(*find("newlines").json.last().unwrap(), b'\n');
}

/// The JSON bytes are the on-disk file, not a re-serialized structure. The
/// `empty` rows are compact one-liners, so a re-serializer would have changed
/// the whitespace.
#[test]
fn json_is_raw_not_reserialized() {
    let s = find("empty");
    let json = std::str::from_utf8(s.json).unwrap();
    assert!(json.contains(r#"{ "a": "1", "b": "", "c": "" }"#));
    assert!(json.starts_with("[\n"));
}

/// `utf8` ends with the two-byte encoding of U+02A4 and decodes to a string
/// ending in that character.
#[test]
fn utf8_sample_passes_through_multibyte() {
    let s = find("utf8");
    assert_eq!(&s.csv[s.csv.len() - 2..], &[0xCA, 0xA4]);
    let text = std::str::from_utf8(s.csv).unwrap();
    assert!(text.ends_with('\u{02A4}'));
}

/// `location_coordinates` carries the U+FFFD replacement bytes twice in both
/// the CSV and the JSON.
#[test]
fn location_coordinates_keeps_replacement_bytes() {
    let s = find("location_coordinates");
    assert_eq!(count_subslice(s.csv, &[0xEF, 0xBF, 0xBD]), 2);
    assert_eq!(count_subslice(s.json, &[0xEF, 0xBF, 0xBD]), 2);
}

/// The phone value in the expected JSON is anonymized and differs from the CSV
/// cell. This is a corpus fact, not a bug to fix.
#[test]
fn location_coordinates_phone_is_anonymized() {
    let s = find("location_coordinates");
    let csv = std::str::from_utf8(s.csv).unwrap();
    let json = std::str::from_utf8(s.json).unwrap();
    assert!(csv.contains("2095257564"));
    assert!(json.contains("1234567890"));
    assert!(!json.contains("2095257564"));
}

/// `empty.json` and `empty_crlf.json` are byte identical. The CRLF distinction
/// lives in the CSV, not the parsed result.
#[test]
fn empty_and_empty_crlf_json_are_identical() {
    assert_eq!(find("empty").json, find("empty_crlf").json);
}

/// `simple.json` and `simple_crlf.json` differ only by the trailing newline.
#[test]
fn simple_json_differs_from_crlf_only_by_trailing_newline() {
    let simple = find("simple").json;
    let crlf = find("simple_crlf").json;
    assert_eq!(simple.len(), crlf.len() + 1);
    assert_eq!(&simple[..crlf.len()], crlf);
    assert_eq!(*simple.last().unwrap(), b'\n');
}

fn find(name: &str) -> Sample {
    *SAMPLES.iter().find(|s| s.name == name).unwrap()
}

fn count_subslice(haystack: &[u8], needle: &[u8]) -> usize {
    haystack
        .windows(needle.len())
        .filter(|w| *w == needle)
        .count()
}