rudb_csv/dialect.rs
1//! What a CSV file's punctuation is, and working it out from the bytes.
2//!
3//! A CSV file does not say how it is written. The delimiter, the quote, the escape and whether the
4//! first line is a header are all conventions, and a reader that demands to be told them is a
5//! reader every loader script has to be rewritten for. DuckDB sniffs, so this sniffs, and every
6//! rule here was read off duckdb v1.4.1's `sniff_csv` rather than reasoned about.
7//!
8//! The candidates are DuckDB's: comma, pipe, semicolon and tab for the delimiter, and the double
9//! quote for the quote and the escape. A file with no quote character in it is reported as having
10//! no quote at all rather than as having the default one, which is visible in `sniff_csv` and is
11//! reproduced because the same field is printed back in an error message.
12
13use rudb_common::{Error, Result};
14
15/// The delimiters tried, in the order they are tried.
16///
17/// Order settles a tie and ties happen: a one column file of `a;b` has neither a comma nor a tab in
18/// it and is one column under both. Comma first is DuckDB's order and is the one that matters,
19/// since the file that arrives with no clue in it is a comma separated one often enough.
20pub const DELIMITERS: [u8; 4] = *b",|;\t";
21
22/// How a CSV file is punctuated.
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub struct Dialect {
25 /// The byte between two fields.
26 pub delimiter: u8,
27 /// The byte that opens and closes a field that may hold a delimiter or a newline, if the file
28 /// has one.
29 pub quote: Option<u8>,
30 /// The byte that makes the next quote a literal one, if the file has one. When it is the quote
31 /// itself, which is what RFC 4180 says and what every writer does, a doubled quote is one
32 /// quote.
33 pub escape: Option<u8>,
34 /// Whether the first line names the columns rather than being one of them.
35 pub header: bool,
36}
37
38impl Dialect {
39 /// The dialect a file with nothing unusual in it has.
40 #[must_use]
41 pub const fn comma_separated() -> Self {
42 Self { delimiter: b',', quote: None, escape: None, header: true }
43 }
44
45 /// The quote byte, or the double quote when the file has none.
46 ///
47 /// A file with no quote in it still needs a byte to compare against while splitting, and the
48 /// one that cannot appear is the one that never appeared. Splitting on the default is what makes
49 /// a quote that turns up after the sample still read as a quote.
50 #[must_use]
51 pub const fn quote_byte(self) -> u8 {
52 match self.quote {
53 Some(quote) => quote,
54 None => b'"',
55 }
56 }
57
58 /// The escape byte, or the quote byte when the file has none.
59 #[must_use]
60 pub const fn escape_byte(self) -> u8 {
61 match self.escape {
62 Some(escape) => escape,
63 None => self.quote_byte(),
64 }
65 }
66
67 /// How a byte is written in the block DuckDB prints under a CSV error.
68 ///
69 /// A tab is `\t` there and a byte that is nothing is `(empty)`, which is why this takes the
70 /// option rather than the byte.
71 #[must_use]
72 pub fn shown(byte: Option<u8>) -> String {
73 match byte {
74 None => "(empty)".to_string(),
75 Some(b'\t') => "\\t".to_string(),
76 Some(byte) => (byte as char).to_string(),
77 }
78 }
79}
80
81/// Which delimiter splits `sample` into the most columns, consistently.
82///
83/// Consistently is the whole test. Every candidate splits every line into some number of fields,
84/// and the one to take is the candidate under which all the lines agree, because a delimiter that
85/// is really just a character inside the data will land in some lines and not others. Among the
86/// candidates that agree, the one that found the most columns wins, since a file is more likely to
87/// have three columns separated by something than one column containing it.
88///
89/// # Errors
90///
91/// When the sample has no complete line in it, which is a file of one unterminated line and is the
92/// one case where there is nothing to count.
93pub fn delimiter(sample: &[u8], quote: Option<u8>) -> Result<u8> {
94 let mut best = (1usize, DELIMITERS[0]);
95 for candidate in DELIMITERS {
96 let dialect = Dialect { delimiter: candidate, quote, escape: quote, header: false };
97 let Some(width) = consistent_width(sample, dialect) else { continue };
98 if width > best.0 {
99 best = (width, candidate);
100 }
101 }
102 if sample.iter().all(|&byte| byte != b'\n' && byte != b'\r') && sample.is_empty() {
103 return Err(Error::io("the file is empty"));
104 }
105 Ok(best.1)
106}
107
108/// The number of fields every line has under `dialect`, when they all have the same number.
109fn consistent_width(sample: &[u8], dialect: Dialect) -> Option<usize> {
110 let mut at = 0;
111 let mut fields = Vec::new();
112 let mut width = None;
113 let mut lines = 0;
114 while at < sample.len() {
115 let next = crate::scan::record(sample, at, dialect, true, &mut fields).ok()??;
116 at = next;
117 lines += 1;
118 match width {
119 None => width = Some(fields.len()),
120 Some(held) if held == fields.len() => {}
121 Some(_) => return None,
122 }
123 }
124 if lines == 0 { None } else { width }
125}
126
127/// Whether the file uses a quote character at all, and which one.
128///
129/// Only the double quote is looked for, which is what `sniff_csv` reports and what every writer
130/// emits. A field is quoted when the first byte after a delimiter or a line start is the quote, so
131/// a double quote sitting in the middle of a field does not make the file a quoted one.
132#[must_use]
133pub fn quote(sample: &[u8]) -> Option<u8> {
134 let mut at_field_start = true;
135 for &byte in sample {
136 if at_field_start && byte == b'"' {
137 return Some(b'"');
138 }
139 at_field_start = byte == b'\n' || byte == b'\r' || DELIMITERS.contains(&byte);
140 }
141 None
142}
143
144#[cfg(test)]
145mod tests {
146 use super::*;
147
148 #[test]
149 fn the_delimiter_is_the_one_every_line_agrees_on() {
150 assert_eq!(delimiter(b"a,b,c\n1,2,3\n", None).unwrap(), b',');
151 assert_eq!(delimiter(b"a|b\n1|x\n", None).unwrap(), b'|');
152 assert_eq!(delimiter(b"a;b\n1;x\n", None).unwrap(), b';');
153 assert_eq!(delimiter(b"a\tb\n1\tx\n", None).unwrap(), b'\t');
154 }
155
156 #[test]
157 fn a_character_that_lands_in_some_lines_and_not_others_is_not_the_delimiter() {
158 // The semicolon splits the first line into two and the second into one, so it is a
159 // character in the data. The comma splits both into two and is the answer.
160 let sample = b"a,b;c\n1,2\n";
161 assert_eq!(delimiter(sample, None).unwrap(), b',');
162 }
163
164 #[test]
165 fn the_delimiter_that_finds_more_columns_wins_among_the_ones_that_agree() {
166 // Every line is one field under a tab and three under a comma, and both are consistent.
167 assert_eq!(delimiter(b"a,b,c\nx,y,z\n", None).unwrap(), b',');
168 }
169
170 #[test]
171 fn a_file_with_nothing_to_split_on_is_comma_separated_and_one_column() {
172 assert_eq!(delimiter(b"a\nb\n", None).unwrap(), b',');
173 }
174
175 #[test]
176 fn a_delimiter_inside_a_quoted_field_does_not_count() {
177 let sample = b"a,b\n1,\"x,y\"\n";
178 assert_eq!(delimiter(sample, Some(b'"')).unwrap(), b',');
179 }
180
181 #[test]
182 fn a_quote_is_found_where_a_field_starts_and_not_in_the_middle_of_one() {
183 assert_eq!(quote(b"a,b\n1,\"x\"\n"), Some(b'"'));
184 assert_eq!(quote(b"a,b\n1,x\n"), None);
185 assert_eq!(quote(b"a,b\n1,he said \"hi\"\n"), None);
186 }
187
188 #[test]
189 fn the_block_duckdb_prints_writes_a_tab_as_two_characters() {
190 assert_eq!(Dialect::shown(None), "(empty)");
191 assert_eq!(Dialect::shown(Some(b'\t')), "\\t");
192 assert_eq!(Dialect::shown(Some(b',')), ",");
193 }
194}