rudb_csv/dialect.rs
1//! What a CSV file's punctuation is, and working it out from the bytes.
2//!
3//! A CSV file does not say how it is written. The delimiter, the quote, the escape and whether the
4//! first line is a header are all conventions, and a reader that demands to be told them is a
5//! reader every loader script has to be rewritten for. DuckDB sniffs, so this sniffs, and every
6//! rule here was read off duckdb v1.4.1's `sniff_csv` rather than reasoned about.
7//!
8//! The candidates are DuckDB's: comma, pipe, semicolon and tab for the delimiter, and the double
9//! quote for the quote and the escape. A file with no quote character in it is reported as having
10//! no quote at all rather than as having the default one, which is visible in `sniff_csv` and is
11//! reproduced because the same field is printed back in an error message.
12
13use rudb_common::{Error, Result};
14
15/// The delimiters tried, in the order they are tried.
16///
17/// Order settles a tie and ties happen: a one column file of `a;b` has neither a comma nor a tab in
18/// it and is one column under both. Comma first is DuckDB's order and is the one that matters,
19/// since the file that arrives with no clue in it is a comma separated one often enough.
20pub const DELIMITERS: [u8; 4] = *b",|;\t";
21
22/// How a CSV file is punctuated.
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub struct Dialect {
25 /// The byte between two fields.
26 pub delimiter: u8,
27 /// The byte that opens and closes a field that may hold a delimiter or a newline, if the file
28 /// has one.
29 pub quote: Option<u8>,
30 /// The byte that makes the next quote a literal one, if the file has one. When it is the quote
31 /// itself, which is what RFC 4180 says and what every writer does, a doubled quote is one
32 /// quote.
33 pub escape: Option<u8>,
34 /// Whether the first line names the columns rather than being one of them.
35 pub header: bool,
36}
37
38/// What the caller said about how the file is written, where the sniffer would otherwise decide.
39///
40/// Every field is optional and a `None` means nothing was said, which is the common case and is the
41/// one the sniffer is for. What is given is not sniffed: `read_csv('f.csv', delim=';')` does not try
42/// the four candidates and pick one, it uses the semicolon, and a file that is really comma
43/// separated then comes back as one column. That is DuckDB's behaviour and it is the useful one,
44/// since somebody who wrote the delimiter down knows something the first megabyte of the file does
45/// not say.
46///
47/// A given value also changes the block DuckDB prints under a conversion error, where a line reads
48/// `(Set By User)` rather than `(Auto-Detected)`, which is why this is carried into the reader
49/// rather than folded into a [`Dialect`] and forgotten.
50#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
51pub struct Given {
52 /// The byte between two fields.
53 pub delimiter: Option<u8>,
54 /// The byte that opens and closes a field.
55 pub quote: Option<u8>,
56 /// The byte that makes the next quote a literal one.
57 pub escape: Option<u8>,
58 /// Whether the first line names the columns.
59 pub header: Option<bool>,
60}
61
62impl Given {
63 /// How a byte is written in the block under a conversion error, and where it came from.
64 #[must_use]
65 pub fn shown(given: Option<u8>, sniffed: Option<u8>) -> String {
66 format!("{} {}", Dialect::shown(sniffed), Self::source(given.is_some()))
67 }
68
69 /// What the block calls a value the caller gave and one it worked out.
70 #[must_use]
71 pub const fn source(given: bool) -> &'static str {
72 if given { "(Set By User)" } else { "(Auto-Detected)" }
73 }
74}
75
76impl Dialect {
77 /// The dialect a file with nothing unusual in it has.
78 #[must_use]
79 pub const fn comma_separated() -> Self {
80 Self { delimiter: b',', quote: None, escape: None, header: true }
81 }
82
83 /// The quote byte, or the double quote when the file has none.
84 ///
85 /// A file with no quote in it still needs a byte to compare against while splitting, and the
86 /// one that cannot appear is the one that never appeared. Splitting on the default is what makes
87 /// a quote that turns up after the sample still read as a quote.
88 #[must_use]
89 pub const fn quote_byte(self) -> u8 {
90 match self.quote {
91 Some(quote) => quote,
92 None => b'"',
93 }
94 }
95
96 /// The escape byte, or the quote byte when the file has none.
97 #[must_use]
98 pub const fn escape_byte(self) -> u8 {
99 match self.escape {
100 Some(escape) => escape,
101 None => self.quote_byte(),
102 }
103 }
104
105 /// How a byte is written in the block DuckDB prints under a CSV error.
106 ///
107 /// A tab is `\t` there and a byte that is nothing is `(empty)`, which is why this takes the
108 /// option rather than the byte.
109 #[must_use]
110 pub fn shown(byte: Option<u8>) -> String {
111 match byte {
112 None => "(empty)".to_string(),
113 Some(b'\t') => "\\t".to_string(),
114 Some(byte) => (byte as char).to_string(),
115 }
116 }
117}
118
119/// Which delimiter splits `sample` into the most columns, consistently.
120///
121/// Consistently is the whole test. Every candidate splits every line into some number of fields,
122/// and the one to take is the candidate under which all the lines agree, because a delimiter that
123/// is really just a character inside the data will land in some lines and not others. Among the
124/// candidates that agree, the one that found the most columns wins, since a file is more likely to
125/// have three columns separated by something than one column containing it.
126///
127/// # Errors
128///
129/// When the sample has no complete line in it, which is a file of one unterminated line and is the
130/// one case where there is nothing to count.
131pub fn delimiter(sample: &[u8], quote: Option<u8>) -> Result<u8> {
132 let mut best = (1usize, DELIMITERS[0]);
133 for candidate in DELIMITERS {
134 let dialect = Dialect { delimiter: candidate, quote, escape: quote, header: false };
135 let Some(width) = consistent_width(sample, dialect) else { continue };
136 if width > best.0 {
137 best = (width, candidate);
138 }
139 }
140 if sample.iter().all(|&byte| byte != b'\n' && byte != b'\r') && sample.is_empty() {
141 return Err(Error::io("the file is empty"));
142 }
143 Ok(best.1)
144}
145
146/// The number of fields every line has under `dialect`, when they all have the same number.
147fn consistent_width(sample: &[u8], dialect: Dialect) -> Option<usize> {
148 let mut at = 0;
149 let mut fields = Vec::new();
150 let mut width = None;
151 let mut lines = 0;
152 while at < sample.len() {
153 let next = crate::scan::record(sample, at, dialect, true, &mut fields).ok()??;
154 at = next;
155 lines += 1;
156 match width {
157 None => width = Some(fields.len()),
158 Some(held) if held == fields.len() => {}
159 Some(_) => return None,
160 }
161 }
162 if lines == 0 { None } else { width }
163}
164
165/// Whether the file uses a quote character at all, and which one.
166///
167/// Only the double quote is looked for, which is what `sniff_csv` reports and what every writer
168/// emits. A field is quoted when the first byte after a delimiter or a line start is the quote, so
169/// a double quote sitting in the middle of a field does not make the file a quoted one.
170#[must_use]
171pub fn quote(sample: &[u8]) -> Option<u8> {
172 let mut at_field_start = true;
173 for &byte in sample {
174 if at_field_start && byte == b'"' {
175 return Some(b'"');
176 }
177 at_field_start = byte == b'\n' || byte == b'\r' || DELIMITERS.contains(&byte);
178 }
179 None
180}
181
182#[cfg(test)]
183mod tests {
184 use super::*;
185
186 #[test]
187 fn the_delimiter_is_the_one_every_line_agrees_on() {
188 assert_eq!(delimiter(b"a,b,c\n1,2,3\n", None).unwrap(), b',');
189 assert_eq!(delimiter(b"a|b\n1|x\n", None).unwrap(), b'|');
190 assert_eq!(delimiter(b"a;b\n1;x\n", None).unwrap(), b';');
191 assert_eq!(delimiter(b"a\tb\n1\tx\n", None).unwrap(), b'\t');
192 }
193
194 #[test]
195 fn a_character_that_lands_in_some_lines_and_not_others_is_not_the_delimiter() {
196 // The semicolon splits the first line into two and the second into one, so it is a
197 // character in the data. The comma splits both into two and is the answer.
198 let sample = b"a,b;c\n1,2\n";
199 assert_eq!(delimiter(sample, None).unwrap(), b',');
200 }
201
202 #[test]
203 fn the_delimiter_that_finds_more_columns_wins_among_the_ones_that_agree() {
204 // Every line is one field under a tab and three under a comma, and both are consistent.
205 assert_eq!(delimiter(b"a,b,c\nx,y,z\n", None).unwrap(), b',');
206 }
207
208 #[test]
209 fn a_file_with_nothing_to_split_on_is_comma_separated_and_one_column() {
210 assert_eq!(delimiter(b"a\nb\n", None).unwrap(), b',');
211 }
212
213 #[test]
214 fn a_delimiter_inside_a_quoted_field_does_not_count() {
215 let sample = b"a,b\n1,\"x,y\"\n";
216 assert_eq!(delimiter(sample, Some(b'"')).unwrap(), b',');
217 }
218
219 #[test]
220 fn a_quote_is_found_where_a_field_starts_and_not_in_the_middle_of_one() {
221 assert_eq!(quote(b"a,b\n1,\"x\"\n"), Some(b'"'));
222 assert_eq!(quote(b"a,b\n1,x\n"), None);
223 assert_eq!(quote(b"a,b\n1,he said \"hi\"\n"), None);
224 }
225
226 #[test]
227 fn the_block_duckdb_prints_writes_a_tab_as_two_characters() {
228 assert_eq!(Dialect::shown(None), "(empty)");
229 assert_eq!(Dialect::shown(Some(b'\t')), "\\t");
230 assert_eq!(Dialect::shown(Some(b',')), ",");
231 }
232}