simd_csv/
utils.rs

1use std::borrow::Cow;
2use std::io::{self, Read, Seek, SeekFrom};
3
4use memchr::memchr;
5
6#[inline]
7pub fn trim_trailing_crlf(slice: &[u8]) -> &[u8] {
8    let mut len = slice.len();
9
10    let has_lf = len >= 1 && slice[len - 1] == b'\n';
11    let has_crlf = has_lf && len >= 2 && slice[len - 2] == b'\r';
12
13    len -= (has_lf as usize) + (has_crlf as usize);
14
15    &slice[..len]
16}
17
18#[inline(always)]
19pub fn trim_bom(slice: &[u8]) -> usize {
20    if slice.len() >= 3 && &slice[..3] == b"\xef\xbb\xbf" {
21        3
22    } else {
23        0
24    }
25}
26
27#[inline]
28pub fn unquoted(cell: &[u8], quote: u8) -> Option<&[u8]> {
29    let len = cell.len();
30
31    if len >= 2 && cell[0] == quote && cell[len - 1] == quote {
32        Some(&cell[1..len - 1])
33    } else {
34        None
35    }
36}
37
38/// Unescape a potentially escaped but unquoted (no leading/trailing quotes) CSV
39/// cell.
40///
41/// Returns a [`Cow::Borrowed`] if nothing needed unescaping, and a
42/// [`Cow::Owned`] if something was actually unescaped.
43///
44/// This function will therefore not allocate if this is not actually required.
45pub fn unescape(cell: &[u8], quote: u8) -> Cow<[u8]> {
46    let len = cell.len();
47    let mut output = Vec::new();
48
49    let mut pos: usize = 0;
50
51    while pos < len {
52        if let Some(offset) = memchr(quote, &cell[pos..]) {
53            if output.is_empty() {
54                output.reserve_exact(len);
55            }
56
57            output.extend_from_slice(&cell[pos..pos + offset + 1]);
58
59            // NOTE: we assume, next character MUST be a quote
60            pos += offset + 2;
61        } else {
62            break;
63        }
64    }
65
66    if output.is_empty() {
67        Cow::Borrowed(cell)
68    } else {
69        output.extend_from_slice(&cell[pos..]);
70        Cow::Owned(output)
71    }
72}
73
74pub fn unescape_to(cell: &[u8], quote: u8, out: &mut Vec<u8>) {
75    let len = cell.len();
76    let mut pos: usize = 0;
77
78    while pos < len {
79        if let Some(offset) = memchr(quote, &cell[pos..]) {
80            out.extend_from_slice(&cell[pos..pos + offset + 1]);
81
82            // NOTE: we assume, next character MUST be a quote
83            pos += offset + 2;
84        } else {
85            break;
86        }
87    }
88
89    out.extend_from_slice(&cell[pos..]);
90}
91
92pub struct ReverseReader<R> {
93    input: R,
94    offset: u64,
95    ptr: u64,
96}
97
98impl<R: Seek + Read> ReverseReader<R> {
99    pub fn new(input: R, filesize: u64, offset: u64) -> Self {
100        Self {
101            input,
102            offset,
103            ptr: filesize,
104        }
105    }
106}
107
108impl<R: Seek + Read> Read for ReverseReader<R> {
109    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
110        let buff_size = buf.len() as u64;
111
112        if self.ptr == self.offset {
113            return Ok(0);
114        }
115
116        if self.offset + buff_size > self.ptr {
117            let e = (self.ptr - self.offset) as usize;
118
119            self.input.seek(SeekFrom::Start(self.offset))?;
120            self.input.read_exact(&mut buf[0..e])?;
121
122            buf[0..e].reverse();
123
124            self.ptr = self.offset;
125
126            Ok(e)
127        } else {
128            let new_position = self.ptr - buff_size;
129
130            self.input.seek(SeekFrom::Start(new_position))?;
131            self.input.read_exact(buf)?;
132            buf.reverse();
133
134            self.ptr -= buff_size;
135
136            Ok(buff_size as usize)
137        }
138    }
139}
140
141#[cfg(test)]
142mod tests {
143    use super::*;
144
145    #[test]
146    fn test_unescape() {
147        assert_eq!(unescape(b"test", b'"'), Cow::Borrowed(b"test"));
148        assert_eq!(
149            unescape(b"\"\"hello\"\"", b'"'),
150            Cow::<[u8]>::Owned(b"\"hello\"".to_vec())
151        );
152        assert_eq!(
153            unescape(b"this is \"\"hello\"\" then?", b'"'),
154            Cow::<[u8]>::Owned(b"this is \"hello\" then?".to_vec())
155        );
156    }
157
158    #[test]
159    fn test_unescape_to() {
160        let mut scratch = Vec::new();
161
162        unescape_to(b"test", b'"', &mut scratch);
163        assert_eq!(scratch, b"test");
164
165        scratch.clear();
166        unescape_to(b"\"\"hello\"\"", b'"', &mut scratch);
167        assert_eq!(scratch, b"\"hello\"");
168
169        scratch.clear();
170        unescape_to(b"this is \"\"hello\"\" then?", b'"', &mut scratch);
171        assert_eq!(scratch, b"this is \"hello\" then?");
172    }
173}
simd_csv/utils.rs

simd_csv/
utils.rs
