1use std::io::{self, Read};
16use std::sync::Arc;
17use std::sync::atomic::{AtomicU8, Ordering};
18
19use dataprof_core::DataProfilerError;
20
21const UTF8_BOM: &[u8; 3] = b"\xef\xbb\xbf";
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24enum State {
25 Outside,
27 InQuoted,
29 QuoteInQuoted,
33}
34
35const NOT_AT_END: u8 = 0;
36const ENDED_OUTSIDE_QUOTES: u8 = 1;
37const ENDED_INSIDE_QUOTES: u8 = 2;
38
39#[derive(Debug, Clone, Default)]
42pub struct QuoteOutcome(Arc<AtomicU8>);
43
44impl QuoteOutcome {
45 pub fn at_end(&self) -> Option<bool> {
52 match self.0.load(Ordering::Relaxed) {
53 NOT_AT_END => None,
54 state => Some(state == ENDED_INSIDE_QUOTES),
55 }
56 }
57}
58
59#[derive(Debug)]
62pub struct QuoteTrackingReader<R> {
63 inner: R,
64 delimiter: u8,
65 quote: u8,
66 state: State,
67 previous: Option<u8>,
71 bom_matched: Option<usize>,
74 outcome: QuoteOutcome,
75}
76
77impl<R: Read> QuoteTrackingReader<R> {
78 pub fn new(inner: R, delimiter: u8, quote: u8) -> Self {
80 Self {
81 inner,
82 delimiter,
83 quote,
84 state: State::Outside,
85 previous: None,
86 bom_matched: Some(0),
87 outcome: QuoteOutcome::default(),
88 }
89 }
90
91 pub fn outcome(&self) -> QuoteOutcome {
93 self.outcome.clone()
94 }
95}
96
97impl<R> QuoteTrackingReader<R> {
98 fn ends_a_field(&self, byte: u8) -> bool {
99 byte == self.delimiter || byte == b'\r' || byte == b'\n'
100 }
101
102 fn observe(&mut self, mut bytes: &[u8]) {
105 while let Some(matched) = self.bom_matched {
106 let Some((&byte, rest)) = bytes.split_first() else {
107 return;
108 };
109 if byte != UTF8_BOM[matched] {
110 self.bom_matched = None;
112 self.previous = matched.checked_sub(1).map(|last| UTF8_BOM[last]);
113 break;
114 }
115 bytes = rest;
116 self.bom_matched = (matched + 1 < UTF8_BOM.len()).then_some(matched + 1);
117 }
118 let Some(&last) = bytes.last() else {
119 return;
120 };
121
122 let mut index = 0;
123 if self.state == State::QuoteInQuoted {
124 if bytes[0] == self.quote {
125 index = 1;
126 self.state = State::InQuoted;
127 } else {
128 self.state = State::Outside;
129 }
130 }
131 while let Some(offset) = memchr::memchr(self.quote, &bytes[index..]) {
132 let at = index + offset;
133 index = at + 1;
134 if self.state == State::InQuoted {
135 match bytes.get(index) {
136 Some(&next) if next == self.quote => index += 1,
138 Some(_) => self.state = State::Outside,
139 None => self.state = State::QuoteInQuoted,
140 }
141 } else {
142 let before = at.checked_sub(1).map_or(self.previous, |i| Some(bytes[i]));
143 if before.is_none_or(|byte| self.ends_a_field(byte)) {
144 self.state = State::InQuoted;
145 }
146 }
147 }
148 self.previous = Some(last);
149 }
150}
151
152impl<R: Read> Read for QuoteTrackingReader<R> {
153 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
154 let read = self.inner.read(buf)?;
155 if read == 0 {
156 if !buf.is_empty() {
157 let state = if self.state == State::InQuoted {
158 ENDED_INSIDE_QUOTES
159 } else {
160 ENDED_OUTSIDE_QUOTES
161 };
162 self.outcome.0.store(state, Ordering::Relaxed);
163 }
164 } else {
165 self.observe(&buf[..read]);
166 }
167 Ok(read)
168 }
169}
170
171pub(crate) fn ends_inside_quotes(
175 bytes: &[u8],
176 delimiter: u8,
177 quote: u8,
178 at_source_start: bool,
179) -> bool {
180 let mut tracker = QuoteTrackingReader::new(io::empty(), delimiter, quote);
181 if !at_source_start {
182 tracker.bom_matched = None;
183 }
184 tracker.observe(bytes);
185 tracker.state == State::InQuoted
186}
187
188pub fn unterminated_quote_error() -> DataProfilerError {
191 DataProfilerError::csv_structure(
192 "The CSV source ends inside a quoted field: a quote was opened and never closed, \
193 so every row after it was read into that one field.",
194 "Close the quote in the source. Set csv_flexible=true to profile it anyway; the \
195 report then sets execution.unterminated_quote.",
196 )
197}
198
199#[cfg(test)]
200mod tests {
201 use super::*;
202
203 fn tracked(input: &[u8], read_size: usize) -> bool {
204 struct Trickle<'a>(&'a [u8], usize);
205 impl Read for Trickle<'_> {
206 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
207 let n = self.0.len().min(self.1).min(buf.len());
208 buf[..n].copy_from_slice(&self.0[..n]);
209 self.0 = &self.0[n..];
210 Ok(n)
211 }
212 }
213 let mut reader = QuoteTrackingReader::new(Trickle(input, read_size), b',', b'"');
214 let outcome = reader.outcome();
215 assert_eq!(outcome.at_end(), None);
216 io::copy(&mut reader, &mut io::sink()).unwrap();
217 let answer = outcome.at_end().expect("read to the end");
218 assert_eq!(answer, ends_inside_quotes(input, b',', b'"', true));
219 answer
220 }
221
222 fn records(input: &[u8]) -> usize {
223 let mut reader = csv::ReaderBuilder::new()
224 .has_headers(false)
225 .flexible(true)
226 .from_reader(input);
227 let mut count = 0;
228 for record in reader.byte_records() {
229 record.unwrap();
230 count += 1;
231 }
232 count
233 }
234
235 fn csv_ended_inside_quotes(input: &[u8]) -> bool {
239 let extended = [input, b"\nZ"].concat();
240 records(&extended) == records(input)
241 }
242
243 #[test]
244 fn agrees_with_the_csv_crate_on_every_short_input() {
245 const ALPHABET: &[u8] = b"a,\"\r\n";
246 let mut checked = 0;
247 let mut inside = 0;
248 for len in 0..=7u32 {
249 for code in 0..ALPHABET.len().pow(len) {
250 let mut rest = code;
251 let input: Vec<u8> = (0..len)
252 .map(|_| {
253 let byte = ALPHABET[rest % ALPHABET.len()];
254 rest /= ALPHABET.len();
255 byte
256 })
257 .collect();
258 let expected = csv_ended_inside_quotes(&input);
259 for read_size in [1, 2, 64] {
260 assert_eq!(
261 tracked(&input, read_size),
262 expected,
263 "{:?} read {read_size} bytes at a time",
264 String::from_utf8_lossy(&input)
265 );
266 }
267 checked += 1;
268 inside += usize::from(expected);
269 }
270 }
271 assert!(checked > 90_000 && inside > 10_000, "{checked} / {inside}");
272 }
273
274 #[test]
275 fn a_leading_bom_is_skipped_as_csv_core_skips_it() {
276 for input in [
279 &b"\xef\xbb\xbf\"x,\"\n"[..],
280 b"\xef\xbb\xbf\"open\n1,2\n",
281 b"\xef\xbb\"x,\"\n",
282 b"\xef\"open\n1,2\n",
283 ] {
284 let expected = csv_ended_inside_quotes(input);
285 for read_size in [1, 2, 64] {
286 assert_eq!(tracked(input, read_size), expected);
287 }
288 }
289 assert!(!tracked(b"\xef\xbb\xbf\"x,\"\n", 1));
290 assert!(tracked(b"\xef\xbb\xbf\"open\n1,2\n", 1));
291 }
292
293 #[test]
294 fn the_issue_example_ends_inside_quotes() {
295 let input = b"id,text\n1,\"never closed\n2,x\n3,y\n";
296 assert!(csv_ended_inside_quotes(input));
297 assert!(tracked(input, 4096));
298 assert!(!tracked(b"id,text\n1,\"line one\nline two\"\n2,x\n", 4096));
299 }
300}