Skip to main content

dataprof_csv/
memmap_reader.rs

1use anyhow::Result;
2use memmap2::Mmap;
3use std::fs::File;
4use std::path::Path;
5
6use crate::CsvParserConfig;
7
8/// Memory-mapped CSV reader for efficient processing of large files.
9pub struct MemoryMappedCsvReader {
10    mmap: Mmap,
11    file_size: u64,
12}
13
14impl MemoryMappedCsvReader {
15    pub fn new(path: &Path) -> Result<Self> {
16        let file = File::open(path)?;
17        let file_size = file.metadata()?.len();
18
19        // Safety: The file is opened read-only and we hold the File handle for the
20        // lifetime of the Mmap. The file must not be concurrently modified.
21        #[allow(unsafe_code)]
22        let mmap = unsafe { Mmap::map(&file)? };
23
24        Ok(Self { mmap, file_size })
25    }
26
27    /// Get file size in bytes.
28    pub fn file_size(&self) -> u64 {
29        self.file_size
30    }
31
32    /// Parse CSV records from memory-mapped data in chunks.
33    ///
34    /// Returns `(headers, records, actual_bytes_consumed)`.
35    pub fn read_csv_chunk(
36        &self,
37        offset: u64,
38        chunk_size: usize,
39        has_headers: bool,
40        csv_config: Option<&CsvParserConfig>,
41    ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
42        let (headers, records, actual_bytes, ragged) =
43            self.read_csv_chunk_until_ragged(offset, chunk_size, has_headers, csv_config)?;
44        match ragged {
45            Some(error) => Err(error),
46            None => Ok((headers, records, actual_bytes)),
47        }
48    }
49
50    /// [`Self::read_csv_chunk`], except that a record strict parsing refuses
51    /// for its field count ends the chunk instead of failing it: the records
52    /// before it are returned with the refusal. A caller under a row cap
53    /// raises it only if the cap is not reached first, so a record the
54    /// profile never reaches is never judged (#757).
55    #[allow(clippy::type_complexity)]
56    pub fn read_csv_chunk_until_ragged(
57        &self,
58        offset: u64,
59        chunk_size: usize,
60        has_headers: bool,
61        csv_config: Option<&CsvParserConfig>,
62    ) -> Result<(
63        Option<csv::StringRecord>,
64        Vec<csv::StringRecord>,
65        usize,
66        Option<anyhow::Error>,
67    )> {
68        let start = offset as usize;
69        if start >= self.mmap.len() {
70            return Ok((None, Vec::new(), 0, None));
71        }
72
73        let quote = csv_config.map_or(b'"', |config| config.quote_char);
74        let delimiter = csv_config
75            .and_then(|config| config.delimiter)
76            .unwrap_or(b',');
77        let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
78
79        // A chunk containing only leading record terminators makes the csv
80        // crate synthesize an empty header at EOF. The following chunk would
81        // then treat the real header as data. When the first target lands in a
82        // run of blank lines, include the first non-blank logical record too so
83        // header discovery has the same view as a full-file reader.
84        if has_headers && offset == 0 {
85            // The CSV decoder strips a leading UTF-8 BOM, so the blank lines
86            // to step over start after it. Counting from byte 0 stopped at the
87            // BOM, and a chunk of BOM and line end became an empty header with
88            // the real one read as data (#1101).
89            let bom = self.leading_bom_len();
90            let header_start = bom
91                + self.mmap[bom..]
92                    .iter()
93                    .take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
94                    .count();
95            if end <= header_start && header_start < self.mmap.len() {
96                end = self.csv_record_boundary(header_start, 1, quote, delimiter);
97            }
98        }
99        let chunk_data = &self.mmap[start..end];
100        let actual_bytes = chunk_data.len();
101
102        let mut builder = csv::ReaderBuilder::new();
103        builder.has_headers(has_headers && offset == 0);
104        if let Some(config) = csv_config {
105            if let Some(delim) = config.delimiter {
106                builder.delimiter(delim);
107            }
108            builder.flexible(config.flexible);
109            builder.quote(config.quote_char);
110            if config.trim_whitespace {
111                builder.trim(csv::Trim::All);
112            }
113        }
114        let mut reader = builder.from_reader(chunk_data);
115
116        let headers = if has_headers && offset == 0 {
117            Some(reader.headers()?.clone())
118        } else {
119            None
120        };
121
122        let mut records = Vec::new();
123        for result in reader.records() {
124            match result {
125                Ok(record) => records.push(record),
126                Err(error) if matches!(error.kind(), csv::ErrorKind::UnequalLengths { .. }) => {
127                    return Ok((headers, records, actual_bytes, Some(error.into())));
128                }
129                Err(error) => return Err(error.into()),
130            }
131        }
132
133        Ok((headers, records, actual_bytes, None))
134    }
135
136    /// Whether the file ends inside a quoted field, scanning from `offset` on.
137    ///
138    /// `offset` must be a record boundary, such as the offset of a chunk
139    /// [`Self::read_csv_chunk`] returned. Passing the last chunk's offset keeps
140    /// the scan to that chunk. The CSV parser accepts a quote that is never
141    /// closed as a field running to the end of the file, so this is the only
142    /// sign of it.
143    pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
144        let start = usize::try_from(offset)
145            .unwrap_or(usize::MAX)
146            .min(self.mmap.len());
147        let quote = csv_config.map_or(b'"', |config| config.quote_char);
148        let delimiter = csv_config
149            .and_then(|config| config.delimiter)
150            .unwrap_or(b',');
151        crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
152    }
153
154    /// Return a byte boundary that never splits an RFC 4180 record.
155    ///
156    /// `chunk_size` is a working-set target, not permission to hand a partial
157    /// header or record to the CSV decoder. Prefer the last complete record at
158    /// or before the target. If the first record itself is larger, extend just
159    /// far enough to include it. The latter is unavoidable for any parser that
160    /// returns a complete cell, and prevents a long header/record from being
161    /// silently discarded.
162    fn csv_record_boundary(
163        &self,
164        start: usize,
165        chunk_size: usize,
166        quote: u8,
167        delimiter: u8,
168    ) -> usize {
169        let target = start.saturating_add(chunk_size).min(self.mmap.len());
170        // A leading BOM is no field content: the byte after it starts the
171        // first field, so a quote there opens a quoted header (#1101).
172        let mut index = if start == 0 {
173            self.leading_bom_len()
174        } else {
175            start
176        };
177        let mut in_quotes = false;
178        let mut at_field_start = true;
179        let mut last_boundary = None;
180
181        while index < target {
182            let byte = self.mmap[index];
183            if in_quotes && byte == quote {
184                // Two quotes inside a quoted field encode one literal quote;
185                // neither ends the field.
186                if self.mmap.get(index + 1) == Some(&quote) {
187                    index += 2;
188                    continue;
189                }
190                in_quotes = false;
191            } else if !in_quotes && at_field_start && byte == quote {
192                in_quotes = true;
193                at_field_start = false;
194            } else if !in_quotes && byte == delimiter {
195                at_field_start = true;
196            } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
197                // A CRLF whose CR is the target's last byte crosses the target
198                // by one. Prefer an earlier complete record when one exists;
199                // otherwise include the LF so the pair is never split.
200                if boundary > target && last_boundary.is_some() {
201                    break;
202                }
203                last_boundary = Some(boundary);
204                at_field_start = true;
205                index = boundary;
206                continue;
207            } else if !in_quotes {
208                at_field_start = false;
209            }
210            index += 1;
211        }
212
213        // A target at EOF owns the final unterminated record as well as every
214        // earlier terminated one. Returning the last separator here would
215        // unnecessarily defer that final record to a second chunk and would
216        // make a whole-file logical-record sample incomplete.
217        if target == self.mmap.len() {
218            return target;
219        }
220
221        if let Some(boundary) = last_boundary {
222            return boundary;
223        }
224
225        // No complete record fit. Continue from the quote state at `target`
226        // until the current logical record ends, or include the final
227        // unterminated record so the CSV crate can validate it honestly.
228        while index < self.mmap.len() {
229            let byte = self.mmap[index];
230            if in_quotes && byte == quote {
231                if self.mmap.get(index + 1) == Some(&quote) {
232                    index += 2;
233                    continue;
234                }
235                in_quotes = false;
236            } else if !in_quotes && at_field_start && byte == quote {
237                in_quotes = true;
238                at_field_start = false;
239            } else if !in_quotes && byte == delimiter {
240                at_field_start = true;
241            } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
242                return boundary;
243            } else if !in_quotes {
244                at_field_start = false;
245            }
246            index += 1;
247        }
248
249        self.mmap.len()
250    }
251
252    /// Length of a UTF-8 BOM at the start of the file, which the CSV decoder
253    /// strips; 0 when there is none.
254    fn leading_bom_len(&self) -> usize {
255        if self.mmap.starts_with(b"\xEF\xBB\xBF") {
256            3
257        } else {
258            0
259        }
260    }
261
262    /// End offset for the CSV crate's default CR, LF, or CRLF terminator.
263    fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
264        match self.mmap[index] {
265            b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
266            b'\r' | b'\n' => Some(index + 1),
267            _ => None,
268        }
269    }
270
271    /// Count logical CSV records, including the header, with a full scan.
272    /// This shares the record parser with partial counting and avoids prefix bias.
273    pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
274        let delimiter = csv_config
275            .and_then(|config| config.delimiter)
276            .unwrap_or(b',');
277        let quote = csv_config.map_or(b'"', |config| config.quote_char);
278        let count = crate::row_count::csv_record_count(&self.mmap[..], delimiter, quote)?;
279        Ok(usize::try_from(count).unwrap_or(usize::MAX))
280    }
281}
282
283#[cfg(test)]
284mod tests {
285    use super::*;
286    use std::io::Write;
287    use tempfile::NamedTempFile;
288
289    #[test]
290    fn test_memory_mapped_reader() -> Result<()> {
291        let mut temp_file = NamedTempFile::new()?;
292        writeln!(temp_file, "name,age,city")?;
293        writeln!(temp_file, "Alice,25,New York")?;
294        writeln!(temp_file, "Bob,30,London")?;
295        writeln!(temp_file, "Charlie,35,Tokyo")?;
296        temp_file.flush()?;
297
298        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
299
300        assert!(reader.file_size() > 0);
301
302        let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
303
304        assert!(headers.is_some());
305        assert_eq!(records.len(), 3);
306
307        let header_record = headers.expect("Headers should be present in test data");
308        assert_eq!(header_record.get(0), Some("name"));
309        assert_eq!(header_record.get(1), Some("age"));
310        assert_eq!(header_record.get(2), Some("city"));
311
312        assert_eq!(records[0].get(0), Some("Alice"));
313        assert_eq!(records[0].get(1), Some("25"));
314
315        Ok(())
316    }
317
318    #[test]
319    fn test_row_estimation() -> Result<()> {
320        let mut temp_file = NamedTempFile::new()?;
321        writeln!(temp_file, "a,b,c")?;
322        for i in 0..100 {
323            writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
324        }
325        temp_file.flush()?;
326
327        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
328        let estimated = reader.estimate_csv_record_count(None)?;
329
330        assert!(estimated > 90 && estimated < 120);
331
332        Ok(())
333    }
334
335    #[test]
336    fn row_estimation_counts_logical_csv_records() -> Result<()> {
337        let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
338            (
339                "quoted multiline",
340                b"id,bio\n1,\"hello\nworld\"\n2,plain",
341                None,
342            ),
343            ("lone CR", b"id,value\r1,x\r2,y", None),
344            (
345                "custom delimiter",
346                b"id;bio\r1;\"hello\nworld\"\r2;plain",
347                Some(CsvParserConfig::default().with_delimiter(b';')),
348            ),
349        ];
350
351        for (case, payload, config) in cases {
352            let mut temp_file = NamedTempFile::new()?;
353            temp_file.write_all(payload)?;
354            temp_file.flush()?;
355
356            let reader = MemoryMappedCsvReader::new(temp_file.path())?;
357            assert_eq!(
358                reader.estimate_csv_record_count(config.as_ref())?,
359                3,
360                "case={case}"
361            );
362        }
363        Ok(())
364    }
365
366    #[test]
367    fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
368        let expected_rows = 1000;
369        let mut temp_file = NamedTempFile::new()?;
370        writeln!(temp_file, "id,name,value")?;
371        for i in 0..expected_rows {
372            let padding = "x".repeat(i % 50);
373            writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
374        }
375        temp_file.flush()?;
376
377        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
378
379        let chunk_size = 512;
380        let mut offset = 0u64;
381        let mut total_records = 0;
382        let mut first = true;
383
384        loop {
385            let (headers, records, bytes) =
386                reader.read_csv_chunk(offset, chunk_size, first, None)?;
387            if records.is_empty() && bytes == 0 {
388                break;
389            }
390            if first && headers.is_some() {
391                first = false;
392            }
393            total_records += records.len();
394            offset += bytes as u64;
395        }
396
397        assert_eq!(
398            total_records, expected_rows,
399            "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
400        );
401
402        Ok(())
403    }
404
405    #[test]
406    fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
407        let mut temp_file = NamedTempFile::new()?;
408        write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
409        temp_file.flush()?;
410
411        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
412        let mut offset = 0u64;
413        let mut headers = None;
414        let mut records = Vec::new();
415
416        loop {
417            let (chunk_headers, chunk_records, bytes) =
418                reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
419            if bytes == 0 {
420                break;
421            }
422            if headers.is_none() {
423                headers = chunk_headers;
424            }
425            records.extend(chunk_records);
426            offset += bytes as u64;
427        }
428
429        assert_eq!(
430            headers.expect("header").iter().collect::<Vec<_>>(),
431            ["alpha", "beta"]
432        );
433        assert_eq!(records.len(), 2);
434        assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
435        assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
436        Ok(())
437    }
438
439    #[test]
440    fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
441        let payload = b"\n\r\n\ralpha,beta\n1,2";
442        let mut temp_file = NamedTempFile::new()?;
443        temp_file.write_all(payload)?;
444        temp_file.flush()?;
445
446        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
447        let mut offset = 0u64;
448        let mut headers = None;
449        let mut records = Vec::new();
450
451        loop {
452            let (chunk_headers, chunk_records, bytes) =
453                reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
454            if bytes == 0 {
455                break;
456            }
457            if headers.is_none() {
458                headers = chunk_headers;
459            }
460            records.extend(chunk_records);
461            offset += bytes as u64;
462        }
463
464        assert_eq!(
465            headers.expect("header").iter().collect::<Vec<_>>(),
466            ["alpha", "beta"]
467        );
468        assert_eq!(records.len(), 1);
469        assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
470        Ok(())
471    }
472
473    #[test]
474    fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
475        let cases: [(&str, &[u8], &str); 2] = [
476            (
477                "crlf",
478                b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
479                "hello \"quoted\"\nworld",
480            ),
481            (
482                "lone-cr",
483                b"id,bio\r1,\"hello\nworld\"\r2,plain",
484                "hello\nworld",
485            ),
486        ];
487
488        for (terminator, payload, expected_bio) in cases {
489            let mut temp_file = NamedTempFile::new()?;
490            temp_file.write_all(payload)?;
491            temp_file.flush()?;
492
493            let reader = MemoryMappedCsvReader::new(temp_file.path())?;
494            for chunk_size in 1..=payload.len() + 1 {
495                let mut offset = 0u64;
496                let mut headers = None;
497                let mut records = Vec::new();
498
499                loop {
500                    let (chunk_headers, chunk_records, bytes) =
501                        reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
502                    if bytes == 0 {
503                        break;
504                    }
505                    if headers.is_none() {
506                        headers = chunk_headers;
507                    }
508                    records.extend(chunk_records);
509                    offset += bytes as u64;
510                }
511
512                assert_eq!(
513                    headers.expect("header").iter().collect::<Vec<_>>(),
514                    ["id", "bio"],
515                    "terminator={terminator}, chunk_size={chunk_size}"
516                );
517                assert_eq!(
518                    records.len(),
519                    2,
520                    "terminator={terminator}, chunk_size={chunk_size}"
521                );
522                assert_eq!(
523                    records[0].get(0),
524                    Some("1"),
525                    "terminator={terminator}, chunk_size={chunk_size}"
526                );
527                assert_eq!(
528                    records[0].get(1),
529                    Some(expected_bio),
530                    "terminator={terminator}, chunk_size={chunk_size}"
531                );
532                assert_eq!(
533                    records[1].iter().collect::<Vec<_>>(),
534                    ["2", "plain"],
535                    "terminator={terminator}, chunk_size={chunk_size}"
536                );
537            }
538        }
539        Ok(())
540    }
541}