1use bytes::Bytes;
16use datafusion::object_store::{Error, Result};
17use futures::{Stream, StreamExt, stream::BoxStream};
18use transform_stream::AsyncTryStream;
19
20use crate::SelectError;
21
22pub fn csv_input_requires_normalization(quote: Option<&str>, escape: Option<&str>) -> bool {
24 quote.is_some_and(|quote| quote.len() > 1) || escape.is_some_and(|escape| escape.len() > 1)
25}
26
27#[derive(Default)]
29pub(crate) struct CsvSyntax<'a> {
30 pub quote: Option<&'a str>,
31 pub escape: Option<&'a str>,
32 pub field: Option<&'a str>,
33 pub record: Option<&'a str>,
34 pub comment: Option<u8>,
35}
36
37#[derive(Clone, Copy, PartialEq, Eq)]
38enum State {
39 FieldStart,
40 Unquoted,
41 Quoted,
42 AfterQuote,
43 Escaped,
44 Comment,
45}
46
47struct CsvInputNormalizer {
51 quote: Vec<u8>,
52 escape: Vec<u8>,
53 field: Vec<u8>,
54 record: Vec<u8>,
55 comment: Option<u8>,
56 default_records: bool,
57 state: State,
58 record_start: bool,
59 carry: Vec<u8>,
60 token_size: usize,
61}
62
63impl CsvInputNormalizer {
64 fn new(csv: &CsvSyntax<'_>) -> Self {
65 let quote = csv
66 .quote
67 .filter(|value| !value.is_empty())
68 .unwrap_or("\"")
69 .as_bytes()
70 .to_vec();
71 let escape = csv
72 .escape
73 .filter(|value| !value.is_empty())
74 .unwrap_or("\"")
75 .as_bytes()
76 .to_vec();
77 let field = csv.field.filter(|value| !value.is_empty()).unwrap_or(",").as_bytes().to_vec();
78 let record = csv
79 .record
80 .filter(|value| !value.is_empty())
81 .unwrap_or("\n")
82 .as_bytes()
83 .to_vec();
84 let token_size = quote.len().max(escape.len()).max(field.len()).max(record.len()).max(2);
85 Self {
86 quote,
87 escape,
88 field,
89 record,
90 comment: csv.comment,
91 default_records: csv.record.is_none(),
92 state: State::FieldStart,
93 record_start: true,
94 carry: Vec::new(),
95 token_size,
96 }
97 }
98
99 fn record_len(&self, bytes: &[u8]) -> usize {
100 if self.default_records && bytes.starts_with(b"\r\n") {
101 2
102 } else if self.default_records && bytes.starts_with(b"\r") {
103 1
104 } else if bytes.starts_with(&self.record) {
105 self.record.len()
106 } else {
107 0
108 }
109 }
110
111 fn push_value(output: &mut Vec<u8>, bytes: &[u8]) {
112 for byte in bytes {
113 if *byte == b'"' {
114 output.push(b'"');
115 }
116 output.push(*byte);
117 }
118 }
119
120 fn convert(&mut self, chunk: &[u8], last: bool) -> std::result::Result<Vec<u8>, SelectError> {
121 let mut bytes = std::mem::take(&mut self.carry);
122 bytes.extend_from_slice(chunk);
123 let end = if last {
124 bytes.len()
125 } else {
126 bytes.len().saturating_sub(self.token_size - 1)
127 };
128 let mut output = Vec::with_capacity(bytes.len());
129 let mut pos = 0;
130 while pos < end {
131 let rest = &bytes[pos..];
132 let record_len = self.record_len(rest);
133 let field = rest.starts_with(&self.field) && self.field.len() > record_len;
134 match self.state {
135 State::Comment => {
136 if record_len > 0 {
137 self.state = State::FieldStart;
138 pos += record_len;
139 } else {
140 pos += 1;
141 }
142 }
143 State::Escaped => {
144 if record_len > 0 {
145 return Err(SelectError::CsvParsingError);
146 }
147 Self::push_value(&mut output, &rest[..1]);
148 self.state = State::Quoted;
149 pos += 1;
150 }
151 State::Quoted if rest.starts_with(&self.quote) => {
152 self.state = State::AfterQuote;
153 pos += self.quote.len();
154 }
155 State::Quoted if rest.starts_with(&self.escape) => {
156 self.state = State::Escaped;
157 pos += self.escape.len();
158 }
159 State::Quoted => {
160 if record_len > 0 {
161 return Err(SelectError::CsvParsingError);
162 }
163 Self::push_value(&mut output, &rest[..1]);
164 pos += 1;
165 }
166 State::AfterQuote if rest.starts_with(&self.quote) => {
167 Self::push_value(&mut output, &self.quote);
168 self.state = State::Quoted;
169 pos += self.quote.len();
170 }
171 State::FieldStart if self.record_start && self.comment == Some(rest[0]) => {
172 self.state = State::Comment;
173 pos += 1;
174 }
175 State::FieldStart if rest.starts_with(&self.quote) => {
176 output.push(b'"');
177 self.state = State::Quoted;
178 self.record_start = false;
179 pos += self.quote.len();
180 }
181 _ if field || record_len > 0 => {
182 if self.state == State::FieldStart {
183 if field || !self.record_start {
184 output.extend_from_slice(b"\"\"");
185 }
186 } else {
187 output.push(b'"');
188 }
189 output.push(if field { b',' } else { b'\n' });
190 self.state = State::FieldStart;
191 self.record_start = !field;
192 pos += if field { self.field.len() } else { record_len };
193 }
194 _ => {
195 if self.state == State::FieldStart {
196 output.push(b'"');
197 }
198 self.state = State::Unquoted;
199 self.record_start = false;
200 Self::push_value(&mut output, &rest[..1]);
201 pos += 1;
202 }
203 }
204 }
205 self.carry.extend_from_slice(&bytes[pos..]);
206 if last {
207 match self.state {
208 State::Quoted | State::Escaped => return Err(SelectError::CsvParsingError),
209 State::Unquoted | State::AfterQuote => output.push(b'"'),
210 State::FieldStart if !self.record_start => output.extend_from_slice(b"\"\""),
211 State::FieldStart | State::Comment => {}
212 }
213 }
214 Ok(output)
215 }
216}
217
218#[cfg(test)]
219mod tests {
220 use super::*;
221
222 fn normalize_chunks(csv: &CsvSyntax<'_>, input: &[u8], chunk_size: usize) -> Vec<u8> {
223 let mut normalizer = CsvInputNormalizer::new(csv);
224 let mut output = Vec::new();
225 for chunk in input.chunks(chunk_size) {
226 output.extend(normalizer.convert(chunk, false).expect("normalize complete CSV input"));
227 assert!(normalizer.carry.len() < normalizer.token_size, "only a partial token may be retained");
228 }
229 output.extend(normalizer.convert(&[], true).expect("finish complete CSV input"));
230 output
231 }
232
233 #[test]
234 fn unicode_csv_quotes_preserve_values_at_every_chunk_boundary() {
235 let cases = [
236 ("ع", "\"", "عcol1ع,عcol2ع,عcol3ع\n", "\"col1\",\"col2\",\"col3\"\n"),
237 ("ع", "\"", "\"left\",tail\n", "\"\"\"left\"\"\",\"tail\"\n"),
238 ("ع", "\"", "عA,Bع,plain\n", "\"A,B\",\"plain\"\n"),
239 ("ع", "\"", "عAععBع,tail\n", "\"AعB\",\"tail\"\n"),
240 ("ع", "\"", "عA\"عBع,tail\n", "\"AعB\",\"tail\"\n"),
241 ("ع", "\\", "عA\\\"Bع,tail\n", "\"A\"\"B\",\"tail\"\n"),
242 ("\"", "界", "\"A界\"B\",\"C\"\n", "\"A\"\"B\",\"C\"\n"),
243 ("🦀", "🦀", "🦀A🦀🦀B🦀,C\n", "\"A🦀B\",\"C\"\n"),
244 ("ع", "\"", "a\0b,عc\0dع\n", "\"a\0b\",\"c\0d\"\n"),
245 ("ع", "\"", "AعB,tail\n", "\"AعB\",\"tail\"\n"),
246 ("ع", "\"", "عaعsuffix,tail\n", "\"asuffix\",\"tail\"\n"),
247 ("ع", "\"", ",\n", "\"\",\"\"\n"),
248 ("ع", "\"", "a,", "\"a\",\"\""),
249 ("ع", "\"", "عع", "\"\""),
250 ("ع", "\"", "\n", "\n"),
251 ("ع", "\"", "", ""),
252 ];
253 for (quote, escape, input, expected) in cases {
254 let csv = CsvSyntax {
255 quote: Some(quote),
256 escape: Some(escape),
257 record: Some("\n"),
258 ..Default::default()
259 };
260 for chunk_size in 1..=input.len().max(1) {
261 assert_eq!(
262 normalize_chunks(&csv, input.as_bytes(), chunk_size),
263 expected.as_bytes(),
264 "input={input:?}, chunk_size={chunk_size}"
265 );
266 }
267 }
268 }
269
270 #[test]
271 fn unicode_csv_quotes_keep_custom_delimiters_and_comments_out_of_values() {
272 let csv = CsvSyntax {
273 quote: Some("ع"),
274 escape: Some("\\"),
275 field: Some("界"),
276 record: Some("^Y"),
277 comment: Some(b'#'),
278 };
279 let input = "#skipع界^Yعa界bع界\"literal\"^Yعline\nbreakع界end^Y";
280 let expected = "\"a界b\",\"\"\"literal\"\"\"\n\"line\nbreak\",\"end\"\n";
281 for chunk_size in 1..=input.len() {
282 assert_eq!(normalize_chunks(&csv, input.as_bytes(), chunk_size), expected.as_bytes());
283 }
284 }
285
286 #[test]
287 fn unicode_csv_quotes_reject_unterminated_fields_and_quoted_record_delimiters() {
288 for input in ["عunfinished", "عescape\\", "عline\nbreakع\n", "عline\\\nbreakع\n"] {
289 let csv = CsvSyntax {
290 quote: Some("ع"),
291 escape: Some("\\"),
292 ..Default::default()
293 };
294 let mut normalizer = CsvInputNormalizer::new(&csv);
295 assert_eq!(normalizer.convert(input.as_bytes(), true), Err(SelectError::CsvParsingError));
296 }
297 }
298
299 #[test]
300 fn unicode_csv_quotes_preserve_omitted_syntax_defaults() {
301 assert!(!csv_input_requires_normalization(None, None));
302 assert!(!csv_input_requires_normalization(Some("\""), Some("\\")));
303 assert!(csv_input_requires_normalization(Some("ع"), None));
304 assert!(csv_input_requires_normalization(None, Some("界")));
305
306 let quote_only = CsvSyntax {
307 quote: Some("ع"),
308 ..Default::default()
309 };
310 assert_eq!(
311 normalize_chunks("e_only, "عA\"عBع,tail\r\n".as_bytes(), 1),
312 "\"AعB\",\"tail\"\n".as_bytes()
313 );
314 let escape_only = CsvSyntax {
315 escape: Some("界"),
316 ..Default::default()
317 };
318 assert_eq!(
319 normalize_chunks(&escape_only, "\"A界\"B\",tail\r\n".as_bytes(), 1),
320 b"\"A\"\"B\",\"tail\"\n"
321 );
322 }
323
324 #[test]
325 fn unicode_csv_quotes_stream_large_fields_without_retaining_records() {
326 let csv = CsvSyntax {
327 quote: Some("ع"),
328 ..Default::default()
329 };
330 let mut normalizer = CsvInputNormalizer::new(&csv);
331 let chunk = vec![b'x'; 64 * 1024];
332 let mut output_len = normalizer.convert("ع".as_bytes(), false).expect("opening quote").len();
333 for _ in 0..64 {
334 let output = normalizer.convert(&chunk, false).expect("stream field chunk");
335 assert!(output.len() >= chunk.len() - 3, "field data must be emitted before its closing quote");
336 assert!(normalizer.carry.len() < 4);
337 output_len += output.len();
338 }
339 output_len += normalizer.convert("ع\n".as_bytes(), true).expect("close field").len();
340 assert_eq!(output_len, chunk.len() * 64 + 3);
341 }
342}
343
344pub(crate) fn normalize_csv_stream<S>(stream: S, csv: &CsvSyntax<'_>) -> BoxStream<'static, Result<Bytes>>
345where
346 S: Stream<Item = Result<Bytes>> + Send + 'static,
347{
348 let mut normalizer = CsvInputNormalizer::new(csv);
349 AsyncTryStream::<Bytes, Error, _>::new(|mut y| async move {
350 futures::pin_mut!(stream);
351 while let Some(chunk) = stream.next().await {
352 let converted = normalizer.convert(&chunk?, false).map_err(|source| Error::Generic {
353 store: "EcObjectStore",
354 source: Box::new(source),
355 })?;
356 if !converted.is_empty() {
357 y.yield_ok(Bytes::from(converted)).await;
358 }
359 }
360 let converted = normalizer.convert(&[], true).map_err(|source| Error::Generic {
361 store: "EcObjectStore",
362 source: Box::new(source),
363 })?;
364 if !converted.is_empty() {
365 y.yield_ok(Bytes::from(converted)).await;
366 }
367 Ok(())
368 })
369 .boxed()
370}
371
372#[cfg(test)]
373mod stream_tests {
374 use super::*;
375 use std::sync::{
376 Arc,
377 atomic::{AtomicBool, AtomicUsize, Ordering},
378 };
379
380 struct DropProbe(Arc<AtomicBool>);
381
382 impl Drop for DropProbe {
383 fn drop(&mut self) {
384 self.0.store(true, Ordering::SeqCst);
385 }
386 }
387
388 #[tokio::test]
389 async fn unicode_csv_quotes_drop_the_source_without_reading_ahead() {
390 let polls = Arc::new(AtomicUsize::new(0));
391 let dropped = Arc::new(AtomicBool::new(false));
392 let source =
393 futures::stream::unfold((DropProbe(Arc::clone(&dropped)), Arc::clone(&polls)), |(guard, polls)| async move {
394 polls.fetch_add(1, Ordering::SeqCst);
395 Some((Ok(Bytes::from_static("عvalueع\n".as_bytes())), (guard, polls)))
396 });
397 let csv = CsvSyntax {
398 quote: Some("ع"),
399 ..Default::default()
400 };
401 let mut stream = normalize_csv_stream(source, &csv);
402 assert!(!stream.next().await.expect("first output").expect("valid CSV").is_empty());
403 assert_eq!(polls.load(Ordering::SeqCst), 1);
404 drop(stream);
405 assert!(dropped.load(Ordering::SeqCst), "cancellation must release the source reader");
406 assert_eq!(polls.load(Ordering::SeqCst), 1);
407 }
408
409 #[tokio::test]
410 async fn unicode_csv_quotes_preserve_source_errors_after_partial_output() {
411 let source = futures::stream::iter([
412 Ok(Bytes::from_static("عvalueع\n".as_bytes())),
413 Err(Error::Generic {
414 store: "fixture",
415 source: std::io::Error::other("source read failed").into(),
416 }),
417 ]);
418 let csv = CsvSyntax {
419 quote: Some("ع"),
420 ..Default::default()
421 };
422 let mut stream = normalize_csv_stream(source, &csv);
423 assert!(!stream.next().await.expect("partial output").expect("valid prefix").is_empty());
424 let error = stream
425 .next()
426 .await
427 .expect("source failure must remain visible")
428 .expect_err("must not return a successful tail");
429 assert!(error.to_string().contains("source read failed"));
430 assert!(stream.next().await.is_none());
431 }
432}