Skip to main content

dataprof_csv/
quote_state.rs

1//! Whether a CSV source ended inside a quoted field.
2//!
3//! The `csv` crate, and `arrow-csv` which shares its `csv-core` tokenizer,
4//! treat end of input inside a quoted field as the end of that field. A quote
5//! that is never closed therefore swallows every following row into the last
6//! field of one record, and nothing the parser returns says so (#782).
7//!
8//! [`QuoteTrackingReader`] sits between the source and the parser and follows
9//! the same quote rules as `csv-core` with the options dataprof uses: a quote
10//! opens a field only at the start of a field, a doubled quote inside a quoted
11//! field is a literal quote, and CR, LF and CRLF all end a record. Only a
12//! quote changes that state, so it jumps from quote to quote with `memchr`
13//! and costs little on data that is mostly unquoted.
14
15use std::io::{self, Read};
16use std::sync::Arc;
17use std::sync::atomic::{AtomicU8, Ordering};
18
19use dataprof_core::DataProfilerError;
20
21const UTF8_BOM: &[u8; 3] = b"\xef\xbb\xbf";
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24enum State {
25    /// Outside any quoted field, where a quote opens one only at a field start.
26    Outside,
27    /// Inside a quoted field.
28    InQuoted,
29    /// Inside a quoted field, having just read a quote at the end of a read:
30    /// the next byte decides whether it closed the field or was the first half
31    /// of an escaped quote.
32    QuoteInQuoted,
33}
34
35const NOT_AT_END: u8 = 0;
36const ENDED_OUTSIDE_QUOTES: u8 = 1;
37const ENDED_INSIDE_QUOTES: u8 = 2;
38
39/// Shared answer from a [`QuoteTrackingReader`], readable after the reader has
40/// been moved into a parser.
41#[derive(Debug, Clone, Default)]
42pub struct QuoteOutcome(Arc<AtomicU8>);
43
44impl QuoteOutcome {
45    /// Whether the source ended inside a quoted field, or `None` while it has
46    /// not been read to its end.
47    ///
48    /// A parser reads ahead of the records it returns, so the end can be
49    /// reached by a scan that stopped at a row cap. Callers take the answer
50    /// only for a scan that consumed its whole source.
51    pub fn at_end(&self) -> Option<bool> {
52        match self.0.load(Ordering::Relaxed) {
53            NOT_AT_END => None,
54            state => Some(state == ENDED_INSIDE_QUOTES),
55        }
56    }
57}
58
59/// Pass-through reader that records whether its source ended inside a quoted
60/// field. See the module docs.
61#[derive(Debug)]
62pub struct QuoteTrackingReader<R> {
63    inner: R,
64    delimiter: u8,
65    quote: u8,
66    state: State,
67    /// The last byte observed, `None` at the start of the source, which is a
68    /// field start. A quote opens a field only when the byte before it ends
69    /// one, so this carries that answer across reads.
70    previous: Option<u8>,
71    /// Bytes of a leading UTF-8 BOM matched so far, `None` once past it.
72    /// `csv-core` strips the BOM, so the byte after it starts a field.
73    bom_matched: Option<usize>,
74    outcome: QuoteOutcome,
75}
76
77impl<R: Read> QuoteTrackingReader<R> {
78    /// Track a source from its first byte, where a UTF-8 BOM is skipped.
79    pub fn new(inner: R, delimiter: u8, quote: u8) -> Self {
80        Self {
81            inner,
82            delimiter,
83            quote,
84            state: State::Outside,
85            previous: None,
86            bom_matched: Some(0),
87            outcome: QuoteOutcome::default(),
88        }
89    }
90
91    /// Handle to the answer, set when the source reaches its end.
92    pub fn outcome(&self) -> QuoteOutcome {
93        self.outcome.clone()
94    }
95}
96
97impl<R> QuoteTrackingReader<R> {
98    fn ends_a_field(&self, byte: u8) -> bool {
99        byte == self.delimiter || byte == b'\r' || byte == b'\n'
100    }
101
102    /// Follow the quote state over the next bytes of the source, visiting only
103    /// the quotes: nothing else changes it.
104    fn observe(&mut self, mut bytes: &[u8]) {
105        while let Some(matched) = self.bom_matched {
106            let Some((&byte, rest)) = bytes.split_first() else {
107                return;
108            };
109            if byte != UTF8_BOM[matched] {
110                // A partial BOM is data, so the byte after it is mid-field.
111                self.bom_matched = None;
112                self.previous = matched.checked_sub(1).map(|last| UTF8_BOM[last]);
113                break;
114            }
115            bytes = rest;
116            self.bom_matched = (matched + 1 < UTF8_BOM.len()).then_some(matched + 1);
117        }
118        let Some(&last) = bytes.last() else {
119            return;
120        };
121
122        let mut index = 0;
123        if self.state == State::QuoteInQuoted {
124            if bytes[0] == self.quote {
125                index = 1;
126                self.state = State::InQuoted;
127            } else {
128                self.state = State::Outside;
129            }
130        }
131        while let Some(offset) = memchr::memchr(self.quote, &bytes[index..]) {
132            let at = index + offset;
133            index = at + 1;
134            if self.state == State::InQuoted {
135                match bytes.get(index) {
136                    // A doubled quote is a literal one; the field goes on.
137                    Some(&next) if next == self.quote => index += 1,
138                    Some(_) => self.state = State::Outside,
139                    None => self.state = State::QuoteInQuoted,
140                }
141            } else {
142                let before = at.checked_sub(1).map_or(self.previous, |i| Some(bytes[i]));
143                if before.is_none_or(|byte| self.ends_a_field(byte)) {
144                    self.state = State::InQuoted;
145                }
146            }
147        }
148        self.previous = Some(last);
149    }
150}
151
152impl<R: Read> Read for QuoteTrackingReader<R> {
153    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
154        let read = self.inner.read(buf)?;
155        if read == 0 {
156            if !buf.is_empty() {
157                let state = if self.state == State::InQuoted {
158                    ENDED_INSIDE_QUOTES
159                } else {
160                    ENDED_OUTSIDE_QUOTES
161                };
162                self.outcome.0.store(state, Ordering::Relaxed);
163            }
164        } else {
165            self.observe(&buf[..read]);
166        }
167        Ok(read)
168    }
169}
170
171/// Whether `bytes`, which start at a record boundary and run to the end of the
172/// source, end inside a quoted field. `at_source_start` says whether they begin
173/// at the source's first byte, where a UTF-8 BOM is skipped.
174pub(crate) fn ends_inside_quotes(
175    bytes: &[u8],
176    delimiter: u8,
177    quote: u8,
178    at_source_start: bool,
179) -> bool {
180    let mut tracker = QuoteTrackingReader::new(io::empty(), delimiter, quote);
181    if !at_source_start {
182        tracker.bom_matched = None;
183    }
184    tracker.observe(bytes);
185    tracker.state == State::InQuoted
186}
187
188/// The error strict mode (`flexible = false`) raises for a source that ended
189/// inside a quoted field.
190pub fn unterminated_quote_error() -> DataProfilerError {
191    DataProfilerError::csv_structure(
192        "The CSV source ends inside a quoted field: a quote was opened and never closed, \
193         so every row after it was read into that one field.",
194        "Close the quote in the source. Set csv_flexible=true to profile it anyway; the \
195         report then sets execution.unterminated_quote.",
196    )
197}
198
199#[cfg(test)]
200mod tests {
201    use super::*;
202
203    fn tracked(input: &[u8], read_size: usize) -> bool {
204        struct Trickle<'a>(&'a [u8], usize);
205        impl Read for Trickle<'_> {
206            fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
207                let n = self.0.len().min(self.1).min(buf.len());
208                buf[..n].copy_from_slice(&self.0[..n]);
209                self.0 = &self.0[n..];
210                Ok(n)
211            }
212        }
213        let mut reader = QuoteTrackingReader::new(Trickle(input, read_size), b',', b'"');
214        let outcome = reader.outcome();
215        assert_eq!(outcome.at_end(), None);
216        io::copy(&mut reader, &mut io::sink()).unwrap();
217        let answer = outcome.at_end().expect("read to the end");
218        assert_eq!(answer, ends_inside_quotes(input, b',', b'"', true));
219        answer
220    }
221
222    fn records(input: &[u8]) -> usize {
223        let mut reader = csv::ReaderBuilder::new()
224            .has_headers(false)
225            .flexible(true)
226            .from_reader(input);
227        let mut count = 0;
228        for record in reader.byte_records() {
229            record.unwrap();
230            count += 1;
231        }
232        count
233    }
234
235    /// The `csv` crate's own answer. Text appended after a record terminator
236    /// starts a new record, unless the input ended inside a quoted field, where
237    /// it lands in that field instead.
238    fn csv_ended_inside_quotes(input: &[u8]) -> bool {
239        let extended = [input, b"\nZ"].concat();
240        records(&extended) == records(input)
241    }
242
243    #[test]
244    fn agrees_with_the_csv_crate_on_every_short_input() {
245        const ALPHABET: &[u8] = b"a,\"\r\n";
246        let mut checked = 0;
247        let mut inside = 0;
248        for len in 0..=7u32 {
249            for code in 0..ALPHABET.len().pow(len) {
250                let mut rest = code;
251                let input: Vec<u8> = (0..len)
252                    .map(|_| {
253                        let byte = ALPHABET[rest % ALPHABET.len()];
254                        rest /= ALPHABET.len();
255                        byte
256                    })
257                    .collect();
258                let expected = csv_ended_inside_quotes(&input);
259                for read_size in [1, 2, 64] {
260                    assert_eq!(
261                        tracked(&input, read_size),
262                        expected,
263                        "{:?} read {read_size} bytes at a time",
264                        String::from_utf8_lossy(&input)
265                    );
266                }
267                checked += 1;
268                inside += usize::from(expected);
269            }
270        }
271        assert!(checked > 90_000 && inside > 10_000, "{checked} / {inside}");
272    }
273
274    #[test]
275    fn a_leading_bom_is_skipped_as_csv_core_skips_it() {
276        // Past the BOM the quote opens a field, so the comma inside it is data.
277        // A partial BOM is data, so the quote after it is mid-field and literal.
278        for input in [
279            &b"\xef\xbb\xbf\"x,\"\n"[..],
280            b"\xef\xbb\xbf\"open\n1,2\n",
281            b"\xef\xbb\"x,\"\n",
282            b"\xef\"open\n1,2\n",
283        ] {
284            let expected = csv_ended_inside_quotes(input);
285            for read_size in [1, 2, 64] {
286                assert_eq!(tracked(input, read_size), expected);
287            }
288        }
289        assert!(!tracked(b"\xef\xbb\xbf\"x,\"\n", 1));
290        assert!(tracked(b"\xef\xbb\xbf\"open\n1,2\n", 1));
291    }
292
293    #[test]
294    fn the_issue_example_ends_inside_quotes() {
295        let input = b"id,text\n1,\"never closed\n2,x\n3,y\n";
296        assert!(csv_ended_inside_quotes(input));
297        assert!(tracked(input, 4096));
298        assert!(!tracked(b"id,text\n1,\"line one\nline two\"\n2,x\n", 4096));
299    }
300}