Skip to main content

dataprof_csv/
memmap_reader.rs

1use anyhow::Result;
2use dataprof_core::MemoryTracker;
3use memmap2::Mmap;
4use std::fs::File;
5use std::io::{BufRead, BufReader, Cursor};
6use std::path::Path;
7
8use crate::CsvParserConfig;
9
10/// Memory-mapped CSV reader for efficient processing of large files.
11pub struct MemoryMappedCsvReader {
12    mmap: Mmap,
13    file_size: u64,
14    memory_tracker: MemoryTracker,
15    resource_id: String,
16}
17
18impl MemoryMappedCsvReader {
19    pub fn new(path: &Path) -> Result<Self> {
20        Self::new_with_tracker(path, MemoryTracker::default())
21    }
22
23    pub fn new_with_tracker(path: &Path, memory_tracker: MemoryTracker) -> Result<Self> {
24        let file = File::open(path)?;
25        let file_size = file.metadata()?.len();
26
27        // Safety: The file is opened read-only and we hold the File handle for the
28        // lifetime of the Mmap. The file must not be concurrently modified.
29        #[allow(unsafe_code)]
30        let mmap = unsafe { Mmap::map(&file)? };
31
32        let resource_id = format!("mmap_{}", path.display());
33
34        memory_tracker.track_allocation(resource_id.clone(), file_size as usize, "memory_map");
35
36        Ok(Self {
37            mmap,
38            file_size,
39            memory_tracker,
40            resource_id,
41        })
42    }
43
44    /// Get file size in bytes.
45    pub fn file_size(&self) -> u64 {
46        self.file_size
47    }
48
49    /// Read a chunk of the file starting at the given byte offset.
50    ///
51    /// Returns `(lines, actual_bytes_consumed)` where `actual_bytes_consumed`
52    /// accounts for line-boundary trimming and may be less than `chunk_size`.
53    pub fn read_chunk(&self, offset: u64, chunk_size: usize) -> Result<(Vec<String>, usize)> {
54        let start = offset as usize;
55        let end = std::cmp::min(start + chunk_size, self.mmap.len());
56
57        if start >= self.mmap.len() {
58            return Ok((Vec::new(), 0));
59        }
60
61        let chunk_data = &self.mmap[start..end];
62
63        let at_eof = end == self.mmap.len();
64        let starts_mid_line = start > 0 && self.mmap[start - 1] != b'\n';
65        let (adjusted_chunk, bytes_consumed) =
66            self.find_line_boundary(chunk_data, starts_mid_line, at_eof);
67
68        let cursor = Cursor::new(adjusted_chunk);
69        let reader = BufReader::new(cursor);
70
71        let mut lines = Vec::new();
72        for line in reader.lines() {
73            lines.push(line?);
74        }
75
76        Ok((lines, bytes_consumed))
77    }
78
79    /// Parse CSV records from memory-mapped data in chunks.
80    ///
81    /// Returns `(headers, records, actual_bytes_consumed)`.
82    pub fn read_csv_chunk(
83        &self,
84        offset: u64,
85        chunk_size: usize,
86        has_headers: bool,
87        csv_config: Option<&CsvParserConfig>,
88    ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
89        let start = offset as usize;
90        if start >= self.mmap.len() {
91            return Ok((None, Vec::new(), 0));
92        }
93
94        let quote = csv_config.map_or(b'"', |config| config.quote_char);
95        let delimiter = csv_config
96            .and_then(|config| config.delimiter)
97            .unwrap_or(b',');
98        let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
99
100        // A chunk containing only leading record terminators makes the csv
101        // crate synthesize an empty header at EOF. The following chunk would
102        // then treat the real header as data. When the first target lands in a
103        // run of blank lines, include the first non-blank logical record too so
104        // header discovery has the same view as a full-file reader.
105        if has_headers && offset == 0 {
106            let header_start = self
107                .mmap
108                .iter()
109                .take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
110                .count();
111            if end <= header_start && header_start < self.mmap.len() {
112                end = self.csv_record_boundary(header_start, 1, quote, delimiter);
113            }
114        }
115        let chunk_data = &self.mmap[start..end];
116        let actual_bytes = chunk_data.len();
117
118        let mut builder = csv::ReaderBuilder::new();
119        builder.has_headers(has_headers && offset == 0);
120        if let Some(config) = csv_config {
121            if let Some(delim) = config.delimiter {
122                builder.delimiter(delim);
123            }
124            builder.flexible(config.flexible);
125            builder.quote(config.quote_char);
126            if config.trim_whitespace {
127                builder.trim(csv::Trim::All);
128            }
129        }
130        let mut reader = builder.from_reader(chunk_data);
131
132        let headers = if has_headers && offset == 0 {
133            Some(reader.headers()?.clone())
134        } else {
135            None
136        };
137
138        let mut records = Vec::new();
139        for result in reader.records() {
140            records.push(result?);
141        }
142
143        Ok((headers, records, actual_bytes))
144    }
145
146    /// Whether the file ends inside a quoted field, scanning from `offset` on.
147    ///
148    /// `offset` must be a record boundary, such as the offset of a chunk
149    /// [`Self::read_csv_chunk`] returned. Passing the last chunk's offset keeps
150    /// the scan to that chunk. The CSV parser accepts a quote that is never
151    /// closed as a field running to the end of the file, so this is the only
152    /// sign of it.
153    pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
154        let start = usize::try_from(offset)
155            .unwrap_or(usize::MAX)
156            .min(self.mmap.len());
157        let quote = csv_config.map_or(b'"', |config| config.quote_char);
158        let delimiter = csv_config
159            .and_then(|config| config.delimiter)
160            .unwrap_or(b',');
161        crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
162    }
163
164    /// Return a byte boundary that never splits an RFC 4180 record.
165    ///
166    /// `chunk_size` is a working-set target, not permission to hand a partial
167    /// header or record to the CSV decoder. Prefer the last complete record at
168    /// or before the target. If the first record itself is larger, extend just
169    /// far enough to include it. The latter is unavoidable for any parser that
170    /// returns a complete cell, and prevents a long header/record from being
171    /// silently discarded.
172    fn csv_record_boundary(
173        &self,
174        start: usize,
175        chunk_size: usize,
176        quote: u8,
177        delimiter: u8,
178    ) -> usize {
179        let target = start.saturating_add(chunk_size).min(self.mmap.len());
180        let mut index = start;
181        let mut in_quotes = false;
182        let mut at_field_start = true;
183        let mut last_boundary = None;
184
185        while index < target {
186            let byte = self.mmap[index];
187            if in_quotes && byte == quote {
188                // Two quotes inside a quoted field encode one literal quote;
189                // neither ends the field.
190                if self.mmap.get(index + 1) == Some(&quote) {
191                    index += 2;
192                    continue;
193                }
194                in_quotes = false;
195            } else if !in_quotes && at_field_start && byte == quote {
196                in_quotes = true;
197                at_field_start = false;
198            } else if !in_quotes && byte == delimiter {
199                at_field_start = true;
200            } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
201                // A CRLF whose CR is the target's last byte crosses the target
202                // by one. Prefer an earlier complete record when one exists;
203                // otherwise include the LF so the pair is never split.
204                if boundary > target && last_boundary.is_some() {
205                    break;
206                }
207                last_boundary = Some(boundary);
208                at_field_start = true;
209                index = boundary;
210                continue;
211            } else if !in_quotes {
212                at_field_start = false;
213            }
214            index += 1;
215        }
216
217        // A target at EOF owns the final unterminated record as well as every
218        // earlier terminated one. Returning the last separator here would
219        // unnecessarily defer that final record to a second chunk and would
220        // make a whole-file logical-record sample incomplete.
221        if target == self.mmap.len() {
222            return target;
223        }
224
225        if let Some(boundary) = last_boundary {
226            return boundary;
227        }
228
229        // No complete record fit. Continue from the quote state at `target`
230        // until the current logical record ends, or include the final
231        // unterminated record so the CSV crate can validate it honestly.
232        while index < self.mmap.len() {
233            let byte = self.mmap[index];
234            if in_quotes && byte == quote {
235                if self.mmap.get(index + 1) == Some(&quote) {
236                    index += 2;
237                    continue;
238                }
239                in_quotes = false;
240            } else if !in_quotes && at_field_start && byte == quote {
241                in_quotes = true;
242                at_field_start = false;
243            } else if !in_quotes && byte == delimiter {
244                at_field_start = true;
245            } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
246                return boundary;
247            } else if !in_quotes {
248                at_field_start = false;
249            }
250            index += 1;
251        }
252
253        self.mmap.len()
254    }
255
256    /// End offset for the CSV crate's default CR, LF, or CRLF terminator.
257    fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
258        match self.mmap[index] {
259            b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
260            b'\r' | b'\n' => Some(index + 1),
261            _ => None,
262        }
263    }
264
265    /// Find the next line boundary to avoid cutting CSV records in half.
266    ///
267    /// When `at_eof` is true, any trailing data after the last newline is
268    /// included because there is no subsequent chunk that will pick it up.
269    ///
270    /// Returns `(data_slice, bytes_consumed)` where `bytes_consumed` is the
271    /// number of bytes from the original chunk that were consumed (including
272    /// any skipped partial-line prefix), so the caller can correctly advance
273    /// the file offset.
274    fn find_line_boundary<'a>(
275        &self,
276        chunk: &'a [u8],
277        skip_first_partial: bool,
278        at_eof: bool,
279    ) -> (&'a [u8], usize) {
280        if chunk.is_empty() {
281            return (chunk, 0);
282        }
283
284        let mut start_pos = 0;
285
286        if skip_first_partial {
287            if let Some(first_newline) = chunk.iter().position(|&b| b == b'\n') {
288                start_pos = first_newline + 1;
289            } else {
290                return (&chunk[chunk.len()..], chunk.len());
291            }
292        }
293
294        let mut end_pos = chunk.len();
295
296        if !at_eof {
297            if let Some(last_newline) = chunk[start_pos..].iter().rposition(|&b| b == b'\n') {
298                end_pos = start_pos + last_newline + 1;
299            } else if start_pos > 0 {
300                return (&chunk[chunk.len()..], start_pos);
301            }
302        }
303
304        (&chunk[start_pos..end_pos], end_pos)
305    }
306
307    /// Estimate the number of logical CSV records using the default parser
308    /// configuration.
309    ///
310    /// Retained as the backwards-compatible form of
311    /// [`Self::estimate_csv_record_count`].
312    pub fn estimate_row_count(&self) -> Result<usize> {
313        self.estimate_csv_record_count(None)
314    }
315
316    /// Estimate the number of logical CSV records in the file by sampling.
317    ///
318    /// The sample uses [`Self::read_csv_chunk`], so quoted multiline fields,
319    /// CR/LF/CRLF terminators, delimiters, and blank-record handling match the
320    /// records that incremental profiling will actually consume.
321    pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
322        const SAMPLE_SIZE: usize = 64 * 1024;
323
324        if self.mmap.is_empty() {
325            return Ok(0);
326        }
327
328        let (_, records, sampled_bytes) = self.read_csv_chunk(0, SAMPLE_SIZE, false, csv_config)?;
329        if sampled_bytes == 0 || records.is_empty() {
330            return Ok(0);
331        }
332        if sampled_bytes == self.mmap.len() {
333            return Ok(records.len());
334        }
335
336        let estimated_records =
337            (self.file_size as u128 * records.len() as u128) / sampled_bytes as u128;
338        Ok(estimated_records.min(usize::MAX as u128) as usize)
339    }
340
341    /// Check for memory leaks in the memory tracker.
342    pub fn check_memory_leaks(&self) -> String {
343        self.memory_tracker.report_leaks()
344    }
345
346    /// Get memory usage statistics.
347    pub fn get_memory_stats(&self) -> (usize, usize, usize) {
348        self.memory_tracker.get_memory_stats()
349    }
350}
351
352impl Drop for MemoryMappedCsvReader {
353    fn drop(&mut self) {
354        self.memory_tracker.track_deallocation(&self.resource_id);
355    }
356}
357
358#[cfg(test)]
359mod tests {
360    use super::*;
361    use std::io::Write;
362    use tempfile::NamedTempFile;
363
364    #[test]
365    fn test_memory_mapped_reader() -> Result<()> {
366        let mut temp_file = NamedTempFile::new()?;
367        writeln!(temp_file, "name,age,city")?;
368        writeln!(temp_file, "Alice,25,New York")?;
369        writeln!(temp_file, "Bob,30,London")?;
370        writeln!(temp_file, "Charlie,35,Tokyo")?;
371        temp_file.flush()?;
372
373        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
374
375        assert!(reader.file_size() > 0);
376
377        let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
378
379        assert!(headers.is_some());
380        assert_eq!(records.len(), 3);
381
382        let header_record = headers.expect("Headers should be present in test data");
383        assert_eq!(header_record.get(0), Some("name"));
384        assert_eq!(header_record.get(1), Some("age"));
385        assert_eq!(header_record.get(2), Some("city"));
386
387        assert_eq!(records[0].get(0), Some("Alice"));
388        assert_eq!(records[0].get(1), Some("25"));
389
390        Ok(())
391    }
392
393    #[test]
394    fn test_row_estimation() -> Result<()> {
395        let mut temp_file = NamedTempFile::new()?;
396        writeln!(temp_file, "a,b,c")?;
397        for i in 0..100 {
398            writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
399        }
400        temp_file.flush()?;
401
402        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
403        let estimated = reader.estimate_row_count()?;
404
405        assert!(estimated > 90 && estimated < 120);
406
407        Ok(())
408    }
409
410    #[test]
411    fn row_estimation_counts_logical_csv_records() -> Result<()> {
412        let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
413            (
414                "quoted multiline",
415                b"id,bio\n1,\"hello\nworld\"\n2,plain",
416                None,
417            ),
418            ("lone CR", b"id,value\r1,x\r2,y", None),
419            (
420                "custom delimiter",
421                b"id;bio\r1;\"hello\nworld\"\r2;plain",
422                Some(CsvParserConfig::default().with_delimiter(b';')),
423            ),
424        ];
425
426        for (case, payload, config) in cases {
427            let mut temp_file = NamedTempFile::new()?;
428            temp_file.write_all(payload)?;
429            temp_file.flush()?;
430
431            let reader = MemoryMappedCsvReader::new(temp_file.path())?;
432            assert_eq!(
433                reader.estimate_csv_record_count(config.as_ref())?,
434                3,
435                "case={case}"
436            );
437        }
438        Ok(())
439    }
440
441    #[test]
442    fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
443        let expected_rows = 1000;
444        let mut temp_file = NamedTempFile::new()?;
445        writeln!(temp_file, "id,name,value")?;
446        for i in 0..expected_rows {
447            let padding = "x".repeat(i % 50);
448            writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
449        }
450        temp_file.flush()?;
451
452        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
453
454        let chunk_size = 512;
455        let mut offset = 0u64;
456        let mut total_records = 0;
457        let mut first = true;
458
459        loop {
460            let (headers, records, bytes) =
461                reader.read_csv_chunk(offset, chunk_size, first, None)?;
462            if records.is_empty() && bytes == 0 {
463                break;
464            }
465            if first && headers.is_some() {
466                first = false;
467            }
468            total_records += records.len();
469            offset += bytes as u64;
470        }
471
472        assert_eq!(
473            total_records, expected_rows,
474            "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
475        );
476
477        Ok(())
478    }
479
480    #[test]
481    fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
482        let mut temp_file = NamedTempFile::new()?;
483        write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
484        temp_file.flush()?;
485
486        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
487        let mut offset = 0u64;
488        let mut headers = None;
489        let mut records = Vec::new();
490
491        loop {
492            let (chunk_headers, chunk_records, bytes) =
493                reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
494            if bytes == 0 {
495                break;
496            }
497            if headers.is_none() {
498                headers = chunk_headers;
499            }
500            records.extend(chunk_records);
501            offset += bytes as u64;
502        }
503
504        assert_eq!(
505            headers.expect("header").iter().collect::<Vec<_>>(),
506            ["alpha", "beta"]
507        );
508        assert_eq!(records.len(), 2);
509        assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
510        assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
511        Ok(())
512    }
513
514    #[test]
515    fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
516        let payload = b"\n\r\n\ralpha,beta\n1,2";
517        let mut temp_file = NamedTempFile::new()?;
518        temp_file.write_all(payload)?;
519        temp_file.flush()?;
520
521        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
522        let mut offset = 0u64;
523        let mut headers = None;
524        let mut records = Vec::new();
525
526        loop {
527            let (chunk_headers, chunk_records, bytes) =
528                reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
529            if bytes == 0 {
530                break;
531            }
532            if headers.is_none() {
533                headers = chunk_headers;
534            }
535            records.extend(chunk_records);
536            offset += bytes as u64;
537        }
538
539        assert_eq!(
540            headers.expect("header").iter().collect::<Vec<_>>(),
541            ["alpha", "beta"]
542        );
543        assert_eq!(records.len(), 1);
544        assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
545        Ok(())
546    }
547
548    #[test]
549    fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
550        let cases: [(&str, &[u8], &str); 2] = [
551            (
552                "crlf",
553                b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
554                "hello \"quoted\"\nworld",
555            ),
556            (
557                "lone-cr",
558                b"id,bio\r1,\"hello\nworld\"\r2,plain",
559                "hello\nworld",
560            ),
561        ];
562
563        for (terminator, payload, expected_bio) in cases {
564            let mut temp_file = NamedTempFile::new()?;
565            temp_file.write_all(payload)?;
566            temp_file.flush()?;
567
568            let reader = MemoryMappedCsvReader::new(temp_file.path())?;
569            for chunk_size in 1..=payload.len() + 1 {
570                let mut offset = 0u64;
571                let mut headers = None;
572                let mut records = Vec::new();
573
574                loop {
575                    let (chunk_headers, chunk_records, bytes) =
576                        reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
577                    if bytes == 0 {
578                        break;
579                    }
580                    if headers.is_none() {
581                        headers = chunk_headers;
582                    }
583                    records.extend(chunk_records);
584                    offset += bytes as u64;
585                }
586
587                assert_eq!(
588                    headers.expect("header").iter().collect::<Vec<_>>(),
589                    ["id", "bio"],
590                    "terminator={terminator}, chunk_size={chunk_size}"
591                );
592                assert_eq!(
593                    records.len(),
594                    2,
595                    "terminator={terminator}, chunk_size={chunk_size}"
596                );
597                assert_eq!(
598                    records[0].get(0),
599                    Some("1"),
600                    "terminator={terminator}, chunk_size={chunk_size}"
601                );
602                assert_eq!(
603                    records[0].get(1),
604                    Some(expected_bio),
605                    "terminator={terminator}, chunk_size={chunk_size}"
606                );
607                assert_eq!(
608                    records[1].iter().collect::<Vec<_>>(),
609                    ["2", "plain"],
610                    "terminator={terminator}, chunk_size={chunk_size}"
611                );
612            }
613        }
614        Ok(())
615    }
616}