simd_csv/
utils.rs

1use std::borrow::Cow;
2use std::io::{self, Read, Seek, SeekFrom};
3
4use memchr::memchr;
5
6#[inline]
7pub fn trim_trailing_crlf(slice: &[u8]) -> &[u8] {
8    let mut len = slice.len();
9
10    let has_lf = len >= 1 && slice[len - 1] == b'\n';
11    let has_crlf = has_lf && len >= 2 && slice[len - 2] == b'\r';
12
13    len -= (has_lf as usize) + (has_crlf as usize);
14
15    &slice[..len]
16}
17
18#[inline(always)]
19pub fn trim_bom(slice: &[u8]) -> usize {
20    if slice.len() >= 3 && &slice[..3] == b"\xef\xbb\xbf" {
21        3
22    } else {
23        0
24    }
25}
26
27#[inline]
28pub fn unquoted(cell: &[u8], quote: u8) -> Option<&[u8]> {
29    let len = cell.len();
30
31    if len >= 2 && cell[0] == quote && cell[len - 1] == quote {
32        Some(&cell[1..len - 1])
33    } else {
34        None
35    }
36}
37
38/// Unescape a potentially escaped but unquoted (no leading/trailing quotes) CSV
39/// cell.
40///
41/// Returns a [`Cow::Borrowed`] if nothing needed unescaping, and a
42/// [`Cow::Owned`] if something was actually unescaped.
43///
44/// This function will therefore not allocate if this is not actually required.
45pub fn unescape(cell: &[u8], quote: u8) -> Cow<[u8]> {
46    let len = cell.len();
47    let mut output = Vec::new();
48
49    let mut pos: usize = 0;
50
51    while pos < len {
52        if let Some(offset) = memchr(quote, &cell[pos..]) {
53            if output.is_empty() {
54                output.reserve_exact(len);
55            }
56
57            let limit = pos + offset + 1;
58
59            output.extend_from_slice(&cell[pos..limit]);
60
61            if limit < len && cell[limit] == quote {
62                pos = limit + 1;
63            } else {
64                pos = limit;
65                break;
66            }
67        } else {
68            break;
69        }
70    }
71
72    if output.is_empty() {
73        Cow::Borrowed(cell)
74    } else {
75        output.extend_from_slice(&cell[pos..]);
76        Cow::Owned(output)
77    }
78}
79
80pub fn unescape_to(cell: &[u8], quote: u8, out: &mut Vec<u8>) {
81    let len = cell.len();
82    let mut pos: usize = 0;
83
84    while pos < len {
85        if let Some(offset) = memchr(quote, &cell[pos..]) {
86            let limit = pos + offset + 1;
87
88            out.extend_from_slice(&cell[pos..limit]);
89
90            if limit < len && cell[limit] == quote {
91                pos = limit + 1;
92            } else {
93                pos = limit;
94                break;
95            }
96        } else {
97            break;
98        }
99    }
100
101    out.extend_from_slice(&cell[pos..]);
102}
103
104pub struct ReverseReader<R> {
105    input: R,
106    offset: u64,
107    ptr: u64,
108}
109
110impl<R: Seek + Read> ReverseReader<R> {
111    pub fn new(input: R, filesize: u64, offset: u64) -> Self {
112        Self {
113            input,
114            offset,
115            ptr: filesize,
116        }
117    }
118}
119
120impl<R: Seek + Read> Read for ReverseReader<R> {
121    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
122        let buff_size = buf.len() as u64;
123
124        if self.ptr == self.offset {
125            return Ok(0);
126        }
127
128        if self.offset + buff_size > self.ptr {
129            let e = (self.ptr - self.offset) as usize;
130
131            self.input.seek(SeekFrom::Start(self.offset))?;
132            self.input.read_exact(&mut buf[0..e])?;
133
134            buf[0..e].reverse();
135
136            self.ptr = self.offset;
137
138            Ok(e)
139        } else {
140            let new_position = self.ptr - buff_size;
141
142            self.input.seek(SeekFrom::Start(new_position))?;
143            self.input.read_exact(buf)?;
144            buf.reverse();
145
146            self.ptr -= buff_size;
147
148            Ok(buff_size as usize)
149        }
150    }
151}
152
153#[cfg(test)]
154mod tests {
155    use super::*;
156
157    #[test]
158    fn test_unescape() {
159        assert_eq!(unescape(b"test", b'"'), Cow::Borrowed(b"test"));
160        assert_eq!(
161            unescape(b"\"\"hello\"\"", b'"'),
162            Cow::<[u8]>::Owned(b"\"hello\"".to_vec())
163        );
164        assert_eq!(
165            unescape(b"this is \"\"hello\"\" then?", b'"'),
166            Cow::<[u8]>::Owned(b"this is \"hello\" then?".to_vec())
167        );
168
169        // It should remain safe with incomplete/invalid data
170        assert_eq!(
171            unescape(b"goettigen\"\"", b'"'),
172            Cow::<[u8]>::Owned(b"goettigen\"".to_vec())
173        );
174        assert_eq!(
175            unescape(b"goettigen\"", b'"'),
176            Cow::<[u8]>::Owned(b"goettigen\"".to_vec())
177        );
178        assert_eq!(
179            unescape(b"goettigen\"whatever", b'"'),
180            Cow::<[u8]>::Owned(b"goettigen\"whatever".to_vec())
181        );
182    }
183
184    #[test]
185    fn test_unescape_to() {
186        let mut scratch = Vec::new();
187
188        unescape_to(b"test", b'"', &mut scratch);
189        assert_eq!(scratch, b"test");
190
191        scratch.clear();
192        unescape_to(b"\"\"hello\"\"", b'"', &mut scratch);
193        assert_eq!(scratch, b"\"hello\"");
194
195        scratch.clear();
196        unescape_to(b"this is \"\"hello\"\" then?", b'"', &mut scratch);
197        assert_eq!(scratch, b"this is \"hello\" then?");
198
199        // It should remain safe with incomplete/invalid data
200        scratch.clear();
201        unescape_to(b"goettigen\"\"", b'"', &mut scratch);
202        assert_eq!(scratch, b"goettigen\"");
203
204        scratch.clear();
205        unescape_to(b"goettigen\"", b'"', &mut scratch);
206        assert_eq!(scratch, b"goettigen\"");
207
208        scratch.clear();
209        unescape_to(b"goettigen\"whatever", b'"', &mut scratch);
210        assert_eq!(scratch, b"goettigen\"whatever");
211    }
212}
simd_csv/utils.rs

simd_csv/
utils.rs
