Skip to main content

rustfs_s3select_api/
csv_input.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use bytes::Bytes;
16use datafusion::object_store::{Error, Result};
17use futures::{Stream, StreamExt, stream::BoxStream};
18use transform_stream::AsyncTryStream;
19
20use crate::SelectError;
21
22/// Arrow accepts byte-sized CSV controls. Unicode quotes need streaming normalization.
23pub fn csv_input_requires_normalization(quote: Option<&str>, escape: Option<&str>) -> bool {
24    quote.is_some_and(|quote| quote.len() > 1) || escape.is_some_and(|escape| escape.len() > 1)
25}
26
27/// CSV syntax independent of request headers, serialization formats, or S3 DTOs.
28#[derive(Default)]
29pub(crate) struct CsvSyntax<'a> {
30    pub quote: Option<&'a str>,
31    pub escape: Option<&'a str>,
32    pub field: Option<&'a str>,
33    pub record: Option<&'a str>,
34    pub comment: Option<u8>,
35}
36
37#[derive(Clone, Copy, PartialEq, Eq)]
38enum State {
39    FieldStart,
40    Unquoted,
41    Quoted,
42    AfterQuote,
43    Escaped,
44    Comment,
45}
46
47/// Emits ordinary CSV with every field quoted. This avoids reserving a sentinel
48/// byte that might also appear in a UTF-8 field. Only a partial control token is
49/// retained between chunks; neither records nor objects are buffered.
50struct CsvInputNormalizer {
51    quote: Vec<u8>,
52    escape: Vec<u8>,
53    field: Vec<u8>,
54    record: Vec<u8>,
55    comment: Option<u8>,
56    default_records: bool,
57    state: State,
58    record_start: bool,
59    carry: Vec<u8>,
60    token_size: usize,
61}
62
63impl CsvInputNormalizer {
64    fn new(csv: &CsvSyntax<'_>) -> Self {
65        let quote = csv
66            .quote
67            .filter(|value| !value.is_empty())
68            .unwrap_or("\"")
69            .as_bytes()
70            .to_vec();
71        let escape = csv
72            .escape
73            .filter(|value| !value.is_empty())
74            .unwrap_or("\"")
75            .as_bytes()
76            .to_vec();
77        let field = csv.field.filter(|value| !value.is_empty()).unwrap_or(",").as_bytes().to_vec();
78        let record = csv
79            .record
80            .filter(|value| !value.is_empty())
81            .unwrap_or("\n")
82            .as_bytes()
83            .to_vec();
84        let token_size = quote.len().max(escape.len()).max(field.len()).max(record.len()).max(2);
85        Self {
86            quote,
87            escape,
88            field,
89            record,
90            comment: csv.comment,
91            default_records: csv.record.is_none(),
92            state: State::FieldStart,
93            record_start: true,
94            carry: Vec::new(),
95            token_size,
96        }
97    }
98
99    fn record_len(&self, bytes: &[u8]) -> usize {
100        if self.default_records && bytes.starts_with(b"\r\n") {
101            2
102        } else if self.default_records && bytes.starts_with(b"\r") {
103            1
104        } else if bytes.starts_with(&self.record) {
105            self.record.len()
106        } else {
107            0
108        }
109    }
110
111    fn push_value(output: &mut Vec<u8>, bytes: &[u8]) {
112        for byte in bytes {
113            if *byte == b'"' {
114                output.push(b'"');
115            }
116            output.push(*byte);
117        }
118    }
119
120    fn convert(&mut self, chunk: &[u8], last: bool) -> std::result::Result<Vec<u8>, SelectError> {
121        let mut bytes = std::mem::take(&mut self.carry);
122        bytes.extend_from_slice(chunk);
123        let end = if last {
124            bytes.len()
125        } else {
126            bytes.len().saturating_sub(self.token_size - 1)
127        };
128        let mut output = Vec::with_capacity(bytes.len());
129        let mut pos = 0;
130        while pos < end {
131            let rest = &bytes[pos..];
132            let record_len = self.record_len(rest);
133            let field = rest.starts_with(&self.field) && self.field.len() > record_len;
134            match self.state {
135                State::Comment => {
136                    if record_len > 0 {
137                        self.state = State::FieldStart;
138                        pos += record_len;
139                    } else {
140                        pos += 1;
141                    }
142                }
143                State::Escaped => {
144                    if record_len > 0 {
145                        return Err(SelectError::CsvParsingError);
146                    }
147                    Self::push_value(&mut output, &rest[..1]);
148                    self.state = State::Quoted;
149                    pos += 1;
150                }
151                State::Quoted if rest.starts_with(&self.quote) => {
152                    self.state = State::AfterQuote;
153                    pos += self.quote.len();
154                }
155                State::Quoted if rest.starts_with(&self.escape) => {
156                    self.state = State::Escaped;
157                    pos += self.escape.len();
158                }
159                State::Quoted => {
160                    if record_len > 0 {
161                        return Err(SelectError::CsvParsingError);
162                    }
163                    Self::push_value(&mut output, &rest[..1]);
164                    pos += 1;
165                }
166                State::AfterQuote if rest.starts_with(&self.quote) => {
167                    Self::push_value(&mut output, &self.quote);
168                    self.state = State::Quoted;
169                    pos += self.quote.len();
170                }
171                State::FieldStart if self.record_start && self.comment == Some(rest[0]) => {
172                    self.state = State::Comment;
173                    pos += 1;
174                }
175                State::FieldStart if rest.starts_with(&self.quote) => {
176                    output.push(b'"');
177                    self.state = State::Quoted;
178                    self.record_start = false;
179                    pos += self.quote.len();
180                }
181                _ if field || record_len > 0 => {
182                    if self.state == State::FieldStart {
183                        if field || !self.record_start {
184                            output.extend_from_slice(b"\"\"");
185                        }
186                    } else {
187                        output.push(b'"');
188                    }
189                    output.push(if field { b',' } else { b'\n' });
190                    self.state = State::FieldStart;
191                    self.record_start = !field;
192                    pos += if field { self.field.len() } else { record_len };
193                }
194                _ => {
195                    if self.state == State::FieldStart {
196                        output.push(b'"');
197                    }
198                    self.state = State::Unquoted;
199                    self.record_start = false;
200                    Self::push_value(&mut output, &rest[..1]);
201                    pos += 1;
202                }
203            }
204        }
205        self.carry.extend_from_slice(&bytes[pos..]);
206        if last {
207            match self.state {
208                State::Quoted | State::Escaped => return Err(SelectError::CsvParsingError),
209                State::Unquoted | State::AfterQuote => output.push(b'"'),
210                State::FieldStart if !self.record_start => output.extend_from_slice(b"\"\""),
211                State::FieldStart | State::Comment => {}
212            }
213        }
214        Ok(output)
215    }
216}
217
218#[cfg(test)]
219mod tests {
220    use super::*;
221
222    fn normalize_chunks(csv: &CsvSyntax<'_>, input: &[u8], chunk_size: usize) -> Vec<u8> {
223        let mut normalizer = CsvInputNormalizer::new(csv);
224        let mut output = Vec::new();
225        for chunk in input.chunks(chunk_size) {
226            output.extend(normalizer.convert(chunk, false).expect("normalize complete CSV input"));
227            assert!(normalizer.carry.len() < normalizer.token_size, "only a partial token may be retained");
228        }
229        output.extend(normalizer.convert(&[], true).expect("finish complete CSV input"));
230        output
231    }
232
233    #[test]
234    fn unicode_csv_quotes_preserve_values_at_every_chunk_boundary() {
235        let cases = [
236            ("ع", "\"", "عcol1ع,عcol2ع,عcol3ع\n", "\"col1\",\"col2\",\"col3\"\n"),
237            ("ع", "\"", "\"left\",tail\n", "\"\"\"left\"\"\",\"tail\"\n"),
238            ("ع", "\"", "عA,Bع,plain\n", "\"A,B\",\"plain\"\n"),
239            ("ع", "\"", "عAععBع,tail\n", "\"AعB\",\"tail\"\n"),
240            ("ع", "\"", "عA\"عBع,tail\n", "\"AعB\",\"tail\"\n"),
241            ("ع", "\\", "عA\\\"Bع,tail\n", "\"A\"\"B\",\"tail\"\n"),
242            ("\"", "界", "\"A界\"B\",\"C\"\n", "\"A\"\"B\",\"C\"\n"),
243            ("🦀", "🦀", "🦀A🦀🦀B🦀,C\n", "\"A🦀B\",\"C\"\n"),
244            ("ع", "\"", "a\0b,عc\0dع\n", "\"a\0b\",\"c\0d\"\n"),
245            ("ع", "\"", "AعB,tail\n", "\"AعB\",\"tail\"\n"),
246            ("ع", "\"", "عaعsuffix,tail\n", "\"asuffix\",\"tail\"\n"),
247            ("ع", "\"", ",\n", "\"\",\"\"\n"),
248            ("ع", "\"", "a,", "\"a\",\"\""),
249            ("ع", "\"", "عع", "\"\""),
250            ("ع", "\"", "\n", "\n"),
251            ("ع", "\"", "", ""),
252        ];
253        for (quote, escape, input, expected) in cases {
254            let csv = CsvSyntax {
255                quote: Some(quote),
256                escape: Some(escape),
257                record: Some("\n"),
258                ..Default::default()
259            };
260            for chunk_size in 1..=input.len().max(1) {
261                assert_eq!(
262                    normalize_chunks(&csv, input.as_bytes(), chunk_size),
263                    expected.as_bytes(),
264                    "input={input:?}, chunk_size={chunk_size}"
265                );
266            }
267        }
268    }
269
270    #[test]
271    fn unicode_csv_quotes_keep_custom_delimiters_and_comments_out_of_values() {
272        let csv = CsvSyntax {
273            quote: Some("ع"),
274            escape: Some("\\"),
275            field: Some("界"),
276            record: Some("^Y"),
277            comment: Some(b'#'),
278        };
279        let input = "#skipع界^Yعa界bع界\"literal\"^Yعline\nbreakع界end^Y";
280        let expected = "\"a界b\",\"\"\"literal\"\"\"\n\"line\nbreak\",\"end\"\n";
281        for chunk_size in 1..=input.len() {
282            assert_eq!(normalize_chunks(&csv, input.as_bytes(), chunk_size), expected.as_bytes());
283        }
284    }
285
286    #[test]
287    fn unicode_csv_quotes_reject_unterminated_fields_and_quoted_record_delimiters() {
288        for input in ["عunfinished", "عescape\\", "عline\nbreakع\n", "عline\\\nbreakع\n"] {
289            let csv = CsvSyntax {
290                quote: Some("ع"),
291                escape: Some("\\"),
292                ..Default::default()
293            };
294            let mut normalizer = CsvInputNormalizer::new(&csv);
295            assert_eq!(normalizer.convert(input.as_bytes(), true), Err(SelectError::CsvParsingError));
296        }
297    }
298
299    #[test]
300    fn unicode_csv_quotes_preserve_omitted_syntax_defaults() {
301        assert!(!csv_input_requires_normalization(None, None));
302        assert!(!csv_input_requires_normalization(Some("\""), Some("\\")));
303        assert!(csv_input_requires_normalization(Some("ع"), None));
304        assert!(csv_input_requires_normalization(None, Some("界")));
305
306        let quote_only = CsvSyntax {
307            quote: Some("ع"),
308            ..Default::default()
309        };
310        assert_eq!(
311            normalize_chunks(&quote_only, "عA\"عBع,tail\r\n".as_bytes(), 1),
312            "\"AعB\",\"tail\"\n".as_bytes()
313        );
314        let escape_only = CsvSyntax {
315            escape: Some("界"),
316            ..Default::default()
317        };
318        assert_eq!(
319            normalize_chunks(&escape_only, "\"A界\"B\",tail\r\n".as_bytes(), 1),
320            b"\"A\"\"B\",\"tail\"\n"
321        );
322    }
323
324    #[test]
325    fn unicode_csv_quotes_stream_large_fields_without_retaining_records() {
326        let csv = CsvSyntax {
327            quote: Some("ع"),
328            ..Default::default()
329        };
330        let mut normalizer = CsvInputNormalizer::new(&csv);
331        let chunk = vec![b'x'; 64 * 1024];
332        let mut output_len = normalizer.convert("ع".as_bytes(), false).expect("opening quote").len();
333        for _ in 0..64 {
334            let output = normalizer.convert(&chunk, false).expect("stream field chunk");
335            assert!(output.len() >= chunk.len() - 3, "field data must be emitted before its closing quote");
336            assert!(normalizer.carry.len() < 4);
337            output_len += output.len();
338        }
339        output_len += normalizer.convert("ع\n".as_bytes(), true).expect("close field").len();
340        assert_eq!(output_len, chunk.len() * 64 + 3);
341    }
342}
343
344pub(crate) fn normalize_csv_stream<S>(stream: S, csv: &CsvSyntax<'_>) -> BoxStream<'static, Result<Bytes>>
345where
346    S: Stream<Item = Result<Bytes>> + Send + 'static,
347{
348    let mut normalizer = CsvInputNormalizer::new(csv);
349    AsyncTryStream::<Bytes, Error, _>::new(|mut y| async move {
350        futures::pin_mut!(stream);
351        while let Some(chunk) = stream.next().await {
352            let converted = normalizer.convert(&chunk?, false).map_err(|source| Error::Generic {
353                store: "EcObjectStore",
354                source: Box::new(source),
355            })?;
356            if !converted.is_empty() {
357                y.yield_ok(Bytes::from(converted)).await;
358            }
359        }
360        let converted = normalizer.convert(&[], true).map_err(|source| Error::Generic {
361            store: "EcObjectStore",
362            source: Box::new(source),
363        })?;
364        if !converted.is_empty() {
365            y.yield_ok(Bytes::from(converted)).await;
366        }
367        Ok(())
368    })
369    .boxed()
370}
371
372#[cfg(test)]
373mod stream_tests {
374    use super::*;
375    use std::sync::{
376        Arc,
377        atomic::{AtomicBool, AtomicUsize, Ordering},
378    };
379
380    struct DropProbe(Arc<AtomicBool>);
381
382    impl Drop for DropProbe {
383        fn drop(&mut self) {
384            self.0.store(true, Ordering::SeqCst);
385        }
386    }
387
388    #[tokio::test]
389    async fn unicode_csv_quotes_drop_the_source_without_reading_ahead() {
390        let polls = Arc::new(AtomicUsize::new(0));
391        let dropped = Arc::new(AtomicBool::new(false));
392        let source =
393            futures::stream::unfold((DropProbe(Arc::clone(&dropped)), Arc::clone(&polls)), |(guard, polls)| async move {
394                polls.fetch_add(1, Ordering::SeqCst);
395                Some((Ok(Bytes::from_static("عvalueع\n".as_bytes())), (guard, polls)))
396            });
397        let csv = CsvSyntax {
398            quote: Some("ع"),
399            ..Default::default()
400        };
401        let mut stream = normalize_csv_stream(source, &csv);
402        assert!(!stream.next().await.expect("first output").expect("valid CSV").is_empty());
403        assert_eq!(polls.load(Ordering::SeqCst), 1);
404        drop(stream);
405        assert!(dropped.load(Ordering::SeqCst), "cancellation must release the source reader");
406        assert_eq!(polls.load(Ordering::SeqCst), 1);
407    }
408
409    #[tokio::test]
410    async fn unicode_csv_quotes_preserve_source_errors_after_partial_output() {
411        let source = futures::stream::iter([
412            Ok(Bytes::from_static("عvalueع\n".as_bytes())),
413            Err(Error::Generic {
414                store: "fixture",
415                source: std::io::Error::other("source read failed").into(),
416            }),
417        ]);
418        let csv = CsvSyntax {
419            quote: Some("ع"),
420            ..Default::default()
421        };
422        let mut stream = normalize_csv_stream(source, &csv);
423        assert!(!stream.next().await.expect("partial output").expect("valid prefix").is_empty());
424        let error = stream
425            .next()
426            .await
427            .expect("source failure must remain visible")
428            .expect_err("must not return a successful tail");
429        assert!(error.to_string().contains("source read failed"));
430        assert!(stream.next().await.is_none());
431    }
432}