Skip to main content

dataprof_csv/
memmap_reader.rs

1use anyhow::Result;
2use memmap2::Mmap;
3use std::fs::File;
4use std::path::Path;
5
6use crate::CsvParserConfig;
7
8/// Memory-mapped CSV reader for efficient processing of large files.
9pub struct MemoryMappedCsvReader {
10    mmap: Mmap,
11    file_size: u64,
12}
13
14impl MemoryMappedCsvReader {
15    pub fn new(path: &Path) -> Result<Self> {
16        let file = File::open(path)?;
17        let file_size = file.metadata()?.len();
18
19        // Safety: The file is opened read-only and we hold the File handle for the
20        // lifetime of the Mmap. The file must not be concurrently modified.
21        #[allow(unsafe_code)]
22        let mmap = unsafe { Mmap::map(&file)? };
23
24        Ok(Self { mmap, file_size })
25    }
26
27    /// Get file size in bytes.
28    pub fn file_size(&self) -> u64 {
29        self.file_size
30    }
31
32    /// Parse CSV records from memory-mapped data in chunks.
33    ///
34    /// Returns `(headers, records, actual_bytes_consumed)`.
35    pub fn read_csv_chunk(
36        &self,
37        offset: u64,
38        chunk_size: usize,
39        has_headers: bool,
40        csv_config: Option<&CsvParserConfig>,
41    ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
42        let start = offset as usize;
43        if start >= self.mmap.len() {
44            return Ok((None, Vec::new(), 0));
45        }
46
47        let quote = csv_config.map_or(b'"', |config| config.quote_char);
48        let delimiter = csv_config
49            .and_then(|config| config.delimiter)
50            .unwrap_or(b',');
51        let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
52
53        // A chunk containing only leading record terminators makes the csv
54        // crate synthesize an empty header at EOF. The following chunk would
55        // then treat the real header as data. When the first target lands in a
56        // run of blank lines, include the first non-blank logical record too so
57        // header discovery has the same view as a full-file reader.
58        if has_headers && offset == 0 {
59            // The CSV decoder strips a leading UTF-8 BOM, so the blank lines
60            // to step over start after it. Counting from byte 0 stopped at the
61            // BOM, and a chunk of BOM and line end became an empty header with
62            // the real one read as data (#1101).
63            let bom = self.leading_bom_len();
64            let header_start = bom
65                + self.mmap[bom..]
66                    .iter()
67                    .take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
68                    .count();
69            if end <= header_start && header_start < self.mmap.len() {
70                end = self.csv_record_boundary(header_start, 1, quote, delimiter);
71            }
72        }
73        let chunk_data = &self.mmap[start..end];
74        let actual_bytes = chunk_data.len();
75
76        let mut builder = csv::ReaderBuilder::new();
77        builder.has_headers(has_headers && offset == 0);
78        if let Some(config) = csv_config {
79            if let Some(delim) = config.delimiter {
80                builder.delimiter(delim);
81            }
82            builder.flexible(config.flexible);
83            builder.quote(config.quote_char);
84            if config.trim_whitespace {
85                builder.trim(csv::Trim::All);
86            }
87        }
88        let mut reader = builder.from_reader(chunk_data);
89
90        let headers = if has_headers && offset == 0 {
91            Some(reader.headers()?.clone())
92        } else {
93            None
94        };
95
96        let mut records = Vec::new();
97        for result in reader.records() {
98            records.push(result?);
99        }
100
101        Ok((headers, records, actual_bytes))
102    }
103
104    /// Whether the file ends inside a quoted field, scanning from `offset` on.
105    ///
106    /// `offset` must be a record boundary, such as the offset of a chunk
107    /// [`Self::read_csv_chunk`] returned. Passing the last chunk's offset keeps
108    /// the scan to that chunk. The CSV parser accepts a quote that is never
109    /// closed as a field running to the end of the file, so this is the only
110    /// sign of it.
111    pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
112        let start = usize::try_from(offset)
113            .unwrap_or(usize::MAX)
114            .min(self.mmap.len());
115        let quote = csv_config.map_or(b'"', |config| config.quote_char);
116        let delimiter = csv_config
117            .and_then(|config| config.delimiter)
118            .unwrap_or(b',');
119        crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
120    }
121
122    /// Return a byte boundary that never splits an RFC 4180 record.
123    ///
124    /// `chunk_size` is a working-set target, not permission to hand a partial
125    /// header or record to the CSV decoder. Prefer the last complete record at
126    /// or before the target. If the first record itself is larger, extend just
127    /// far enough to include it. The latter is unavoidable for any parser that
128    /// returns a complete cell, and prevents a long header/record from being
129    /// silently discarded.
130    fn csv_record_boundary(
131        &self,
132        start: usize,
133        chunk_size: usize,
134        quote: u8,
135        delimiter: u8,
136    ) -> usize {
137        let target = start.saturating_add(chunk_size).min(self.mmap.len());
138        // A leading BOM is no field content: the byte after it starts the
139        // first field, so a quote there opens a quoted header (#1101).
140        let mut index = if start == 0 {
141            self.leading_bom_len()
142        } else {
143            start
144        };
145        let mut in_quotes = false;
146        let mut at_field_start = true;
147        let mut last_boundary = None;
148
149        while index < target {
150            let byte = self.mmap[index];
151            if in_quotes && byte == quote {
152                // Two quotes inside a quoted field encode one literal quote;
153                // neither ends the field.
154                if self.mmap.get(index + 1) == Some(&quote) {
155                    index += 2;
156                    continue;
157                }
158                in_quotes = false;
159            } else if !in_quotes && at_field_start && byte == quote {
160                in_quotes = true;
161                at_field_start = false;
162            } else if !in_quotes && byte == delimiter {
163                at_field_start = true;
164            } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
165                // A CRLF whose CR is the target's last byte crosses the target
166                // by one. Prefer an earlier complete record when one exists;
167                // otherwise include the LF so the pair is never split.
168                if boundary > target && last_boundary.is_some() {
169                    break;
170                }
171                last_boundary = Some(boundary);
172                at_field_start = true;
173                index = boundary;
174                continue;
175            } else if !in_quotes {
176                at_field_start = false;
177            }
178            index += 1;
179        }
180
181        // A target at EOF owns the final unterminated record as well as every
182        // earlier terminated one. Returning the last separator here would
183        // unnecessarily defer that final record to a second chunk and would
184        // make a whole-file logical-record sample incomplete.
185        if target == self.mmap.len() {
186            return target;
187        }
188
189        if let Some(boundary) = last_boundary {
190            return boundary;
191        }
192
193        // No complete record fit. Continue from the quote state at `target`
194        // until the current logical record ends, or include the final
195        // unterminated record so the CSV crate can validate it honestly.
196        while index < self.mmap.len() {
197            let byte = self.mmap[index];
198            if in_quotes && byte == quote {
199                if self.mmap.get(index + 1) == Some(&quote) {
200                    index += 2;
201                    continue;
202                }
203                in_quotes = false;
204            } else if !in_quotes && at_field_start && byte == quote {
205                in_quotes = true;
206                at_field_start = false;
207            } else if !in_quotes && byte == delimiter {
208                at_field_start = true;
209            } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
210                return boundary;
211            } else if !in_quotes {
212                at_field_start = false;
213            }
214            index += 1;
215        }
216
217        self.mmap.len()
218    }
219
220    /// Length of a UTF-8 BOM at the start of the file, which the CSV decoder
221    /// strips; 0 when there is none.
222    fn leading_bom_len(&self) -> usize {
223        if self.mmap.starts_with(b"\xEF\xBB\xBF") {
224            3
225        } else {
226            0
227        }
228    }
229
230    /// End offset for the CSV crate's default CR, LF, or CRLF terminator.
231    fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
232        match self.mmap[index] {
233            b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
234            b'\r' | b'\n' => Some(index + 1),
235            _ => None,
236        }
237    }
238
239    /// Count logical CSV records, including the header, with a full scan.
240    /// This shares the record parser with partial counting and avoids prefix bias.
241    pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
242        let delimiter = csv_config
243            .and_then(|config| config.delimiter)
244            .unwrap_or(b',');
245        let quote = csv_config.map_or(b'"', |config| config.quote_char);
246        let count = crate::row_count::csv_record_count(&self.mmap[..], delimiter, quote)?;
247        Ok(usize::try_from(count).unwrap_or(usize::MAX))
248    }
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254    use std::io::Write;
255    use tempfile::NamedTempFile;
256
257    #[test]
258    fn test_memory_mapped_reader() -> Result<()> {
259        let mut temp_file = NamedTempFile::new()?;
260        writeln!(temp_file, "name,age,city")?;
261        writeln!(temp_file, "Alice,25,New York")?;
262        writeln!(temp_file, "Bob,30,London")?;
263        writeln!(temp_file, "Charlie,35,Tokyo")?;
264        temp_file.flush()?;
265
266        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
267
268        assert!(reader.file_size() > 0);
269
270        let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
271
272        assert!(headers.is_some());
273        assert_eq!(records.len(), 3);
274
275        let header_record = headers.expect("Headers should be present in test data");
276        assert_eq!(header_record.get(0), Some("name"));
277        assert_eq!(header_record.get(1), Some("age"));
278        assert_eq!(header_record.get(2), Some("city"));
279
280        assert_eq!(records[0].get(0), Some("Alice"));
281        assert_eq!(records[0].get(1), Some("25"));
282
283        Ok(())
284    }
285
286    #[test]
287    fn test_row_estimation() -> Result<()> {
288        let mut temp_file = NamedTempFile::new()?;
289        writeln!(temp_file, "a,b,c")?;
290        for i in 0..100 {
291            writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
292        }
293        temp_file.flush()?;
294
295        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
296        let estimated = reader.estimate_csv_record_count(None)?;
297
298        assert!(estimated > 90 && estimated < 120);
299
300        Ok(())
301    }
302
303    #[test]
304    fn row_estimation_counts_logical_csv_records() -> Result<()> {
305        let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
306            (
307                "quoted multiline",
308                b"id,bio\n1,\"hello\nworld\"\n2,plain",
309                None,
310            ),
311            ("lone CR", b"id,value\r1,x\r2,y", None),
312            (
313                "custom delimiter",
314                b"id;bio\r1;\"hello\nworld\"\r2;plain",
315                Some(CsvParserConfig::default().with_delimiter(b';')),
316            ),
317        ];
318
319        for (case, payload, config) in cases {
320            let mut temp_file = NamedTempFile::new()?;
321            temp_file.write_all(payload)?;
322            temp_file.flush()?;
323
324            let reader = MemoryMappedCsvReader::new(temp_file.path())?;
325            assert_eq!(
326                reader.estimate_csv_record_count(config.as_ref())?,
327                3,
328                "case={case}"
329            );
330        }
331        Ok(())
332    }
333
334    #[test]
335    fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
336        let expected_rows = 1000;
337        let mut temp_file = NamedTempFile::new()?;
338        writeln!(temp_file, "id,name,value")?;
339        for i in 0..expected_rows {
340            let padding = "x".repeat(i % 50);
341            writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
342        }
343        temp_file.flush()?;
344
345        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
346
347        let chunk_size = 512;
348        let mut offset = 0u64;
349        let mut total_records = 0;
350        let mut first = true;
351
352        loop {
353            let (headers, records, bytes) =
354                reader.read_csv_chunk(offset, chunk_size, first, None)?;
355            if records.is_empty() && bytes == 0 {
356                break;
357            }
358            if first && headers.is_some() {
359                first = false;
360            }
361            total_records += records.len();
362            offset += bytes as u64;
363        }
364
365        assert_eq!(
366            total_records, expected_rows,
367            "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
368        );
369
370        Ok(())
371    }
372
373    #[test]
374    fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
375        let mut temp_file = NamedTempFile::new()?;
376        write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
377        temp_file.flush()?;
378
379        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
380        let mut offset = 0u64;
381        let mut headers = None;
382        let mut records = Vec::new();
383
384        loop {
385            let (chunk_headers, chunk_records, bytes) =
386                reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
387            if bytes == 0 {
388                break;
389            }
390            if headers.is_none() {
391                headers = chunk_headers;
392            }
393            records.extend(chunk_records);
394            offset += bytes as u64;
395        }
396
397        assert_eq!(
398            headers.expect("header").iter().collect::<Vec<_>>(),
399            ["alpha", "beta"]
400        );
401        assert_eq!(records.len(), 2);
402        assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
403        assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
404        Ok(())
405    }
406
407    #[test]
408    fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
409        let payload = b"\n\r\n\ralpha,beta\n1,2";
410        let mut temp_file = NamedTempFile::new()?;
411        temp_file.write_all(payload)?;
412        temp_file.flush()?;
413
414        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
415        let mut offset = 0u64;
416        let mut headers = None;
417        let mut records = Vec::new();
418
419        loop {
420            let (chunk_headers, chunk_records, bytes) =
421                reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
422            if bytes == 0 {
423                break;
424            }
425            if headers.is_none() {
426                headers = chunk_headers;
427            }
428            records.extend(chunk_records);
429            offset += bytes as u64;
430        }
431
432        assert_eq!(
433            headers.expect("header").iter().collect::<Vec<_>>(),
434            ["alpha", "beta"]
435        );
436        assert_eq!(records.len(), 1);
437        assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
438        Ok(())
439    }
440
441    #[test]
442    fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
443        let cases: [(&str, &[u8], &str); 2] = [
444            (
445                "crlf",
446                b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
447                "hello \"quoted\"\nworld",
448            ),
449            (
450                "lone-cr",
451                b"id,bio\r1,\"hello\nworld\"\r2,plain",
452                "hello\nworld",
453            ),
454        ];
455
456        for (terminator, payload, expected_bio) in cases {
457            let mut temp_file = NamedTempFile::new()?;
458            temp_file.write_all(payload)?;
459            temp_file.flush()?;
460
461            let reader = MemoryMappedCsvReader::new(temp_file.path())?;
462            for chunk_size in 1..=payload.len() + 1 {
463                let mut offset = 0u64;
464                let mut headers = None;
465                let mut records = Vec::new();
466
467                loop {
468                    let (chunk_headers, chunk_records, bytes) =
469                        reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
470                    if bytes == 0 {
471                        break;
472                    }
473                    if headers.is_none() {
474                        headers = chunk_headers;
475                    }
476                    records.extend(chunk_records);
477                    offset += bytes as u64;
478                }
479
480                assert_eq!(
481                    headers.expect("header").iter().collect::<Vec<_>>(),
482                    ["id", "bio"],
483                    "terminator={terminator}, chunk_size={chunk_size}"
484                );
485                assert_eq!(
486                    records.len(),
487                    2,
488                    "terminator={terminator}, chunk_size={chunk_size}"
489                );
490                assert_eq!(
491                    records[0].get(0),
492                    Some("1"),
493                    "terminator={terminator}, chunk_size={chunk_size}"
494                );
495                assert_eq!(
496                    records[0].get(1),
497                    Some(expected_bio),
498                    "terminator={terminator}, chunk_size={chunk_size}"
499                );
500                assert_eq!(
501                    records[1].iter().collect::<Vec<_>>(),
502                    ["2", "plain"],
503                    "terminator={terminator}, chunk_size={chunk_size}"
504                );
505            }
506        }
507        Ok(())
508    }
509}