Skip to main content

dataprof_csv/
memmap_reader.rs

1use anyhow::Result;
2use dataprof_core::MemoryTracker;
3use memmap2::Mmap;
4use std::fs::File;
5use std::io::{BufRead, BufReader, Cursor};
6use std::path::Path;
7
8use crate::CsvParserConfig;
9
10/// Memory-mapped CSV reader for efficient processing of large files.
11pub struct MemoryMappedCsvReader {
12    mmap: Mmap,
13    file_size: u64,
14    memory_tracker: MemoryTracker,
15    resource_id: String,
16}
17
18impl MemoryMappedCsvReader {
19    pub fn new(path: &Path) -> Result<Self> {
20        Self::new_with_tracker(path, MemoryTracker::default())
21    }
22
23    pub fn new_with_tracker(path: &Path, memory_tracker: MemoryTracker) -> Result<Self> {
24        let file = File::open(path)?;
25        let file_size = file.metadata()?.len();
26
27        // Safety: The file is opened read-only and we hold the File handle for the
28        // lifetime of the Mmap. The file must not be concurrently modified.
29        #[allow(unsafe_code)]
30        let mmap = unsafe { Mmap::map(&file)? };
31
32        let resource_id = format!("mmap_{}", path.display());
33
34        memory_tracker.track_allocation(resource_id.clone(), file_size as usize, "memory_map");
35
36        Ok(Self {
37            mmap,
38            file_size,
39            memory_tracker,
40            resource_id,
41        })
42    }
43
44    /// Get file size in bytes.
45    pub fn file_size(&self) -> u64 {
46        self.file_size
47    }
48
49    /// Read a chunk of the file starting at the given byte offset.
50    ///
51    /// Returns `(lines, actual_bytes_consumed)` where `actual_bytes_consumed`
52    /// accounts for line-boundary trimming and may be less than `chunk_size`.
53    pub fn read_chunk(&self, offset: u64, chunk_size: usize) -> Result<(Vec<String>, usize)> {
54        let start = offset as usize;
55        let end = std::cmp::min(start + chunk_size, self.mmap.len());
56
57        if start >= self.mmap.len() {
58            return Ok((Vec::new(), 0));
59        }
60
61        let chunk_data = &self.mmap[start..end];
62
63        let at_eof = end == self.mmap.len();
64        let starts_mid_line = start > 0 && self.mmap[start - 1] != b'\n';
65        let (adjusted_chunk, bytes_consumed) =
66            self.find_line_boundary(chunk_data, starts_mid_line, at_eof);
67
68        let cursor = Cursor::new(adjusted_chunk);
69        let reader = BufReader::new(cursor);
70
71        let mut lines = Vec::new();
72        for line in reader.lines() {
73            lines.push(line?);
74        }
75
76        Ok((lines, bytes_consumed))
77    }
78
79    /// Parse CSV records from memory-mapped data in chunks.
80    ///
81    /// Returns `(headers, records, actual_bytes_consumed)`.
82    pub fn read_csv_chunk(
83        &self,
84        offset: u64,
85        chunk_size: usize,
86        has_headers: bool,
87        csv_config: Option<&CsvParserConfig>,
88    ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
89        let (lines, actual_bytes) = self.read_chunk(offset, chunk_size)?;
90
91        if lines.is_empty() {
92            return Ok((None, Vec::new(), 0));
93        }
94
95        let chunk_data = lines.join("\n");
96        let mut builder = csv::ReaderBuilder::new();
97        builder.has_headers(has_headers && offset == 0);
98        if let Some(config) = csv_config {
99            if let Some(delim) = config.delimiter {
100                builder.delimiter(delim);
101            }
102            builder.flexible(config.flexible);
103            builder.quote(config.quote_char);
104            if config.trim_whitespace {
105                builder.trim(csv::Trim::All);
106            }
107        }
108        let mut reader = builder.from_reader(Cursor::new(chunk_data.as_bytes()));
109
110        let headers = if has_headers && offset == 0 {
111            Some(reader.headers()?.clone())
112        } else {
113            None
114        };
115
116        let mut records = Vec::new();
117        for result in reader.records() {
118            records.push(result?);
119        }
120
121        Ok((headers, records, actual_bytes))
122    }
123
124    /// Find the next line boundary to avoid cutting CSV records in half.
125    ///
126    /// When `at_eof` is true, any trailing data after the last newline is
127    /// included because there is no subsequent chunk that will pick it up.
128    ///
129    /// Returns `(data_slice, bytes_consumed)` where `bytes_consumed` is the
130    /// number of bytes from the original chunk that were consumed (including
131    /// any skipped partial-line prefix), so the caller can correctly advance
132    /// the file offset.
133    fn find_line_boundary<'a>(
134        &self,
135        chunk: &'a [u8],
136        skip_first_partial: bool,
137        at_eof: bool,
138    ) -> (&'a [u8], usize) {
139        if chunk.is_empty() {
140            return (chunk, 0);
141        }
142
143        let mut start_pos = 0;
144
145        if skip_first_partial {
146            if let Some(first_newline) = chunk.iter().position(|&b| b == b'\n') {
147                start_pos = first_newline + 1;
148            } else {
149                return (&chunk[chunk.len()..], chunk.len());
150            }
151        }
152
153        let mut end_pos = chunk.len();
154
155        if !at_eof {
156            if let Some(last_newline) = chunk[start_pos..].iter().rposition(|&b| b == b'\n') {
157                end_pos = start_pos + last_newline + 1;
158            } else if start_pos > 0 {
159                return (&chunk[chunk.len()..], start_pos);
160            }
161        }
162
163        (&chunk[start_pos..end_pos], end_pos)
164    }
165
166    /// Estimate the number of rows in the file by sampling.
167    pub fn estimate_row_count(&self) -> Result<usize> {
168        const SAMPLE_SIZE: usize = 64 * 1024;
169
170        if self.file_size < SAMPLE_SIZE as u64 {
171            let cursor = Cursor::new(&*self.mmap);
172            let reader = BufReader::new(cursor);
173            return Ok(reader.lines().count());
174        }
175
176        let sample_data = &self.mmap[0..SAMPLE_SIZE];
177        let cursor = Cursor::new(sample_data);
178        let reader = BufReader::new(cursor);
179
180        let sample_lines = reader.lines().count();
181        if sample_lines == 0 {
182            return Ok(0);
183        }
184
185        let estimated_rows = (self.file_size * sample_lines as u64) / SAMPLE_SIZE as u64;
186        Ok(estimated_rows as usize)
187    }
188
189    /// Check for memory leaks in the memory tracker.
190    pub fn check_memory_leaks(&self) -> String {
191        self.memory_tracker.report_leaks()
192    }
193
194    /// Get memory usage statistics.
195    pub fn get_memory_stats(&self) -> (usize, usize, usize) {
196        self.memory_tracker.get_memory_stats()
197    }
198}
199
200impl Drop for MemoryMappedCsvReader {
201    fn drop(&mut self) {
202        self.memory_tracker.track_deallocation(&self.resource_id);
203    }
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209    use std::io::Write;
210    use tempfile::NamedTempFile;
211
212    #[test]
213    fn test_memory_mapped_reader() -> Result<()> {
214        let mut temp_file = NamedTempFile::new()?;
215        writeln!(temp_file, "name,age,city")?;
216        writeln!(temp_file, "Alice,25,New York")?;
217        writeln!(temp_file, "Bob,30,London")?;
218        writeln!(temp_file, "Charlie,35,Tokyo")?;
219        temp_file.flush()?;
220
221        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
222
223        assert!(reader.file_size() > 0);
224
225        let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
226
227        assert!(headers.is_some());
228        assert_eq!(records.len(), 3);
229
230        let header_record = headers.expect("Headers should be present in test data");
231        assert_eq!(header_record.get(0), Some("name"));
232        assert_eq!(header_record.get(1), Some("age"));
233        assert_eq!(header_record.get(2), Some("city"));
234
235        assert_eq!(records[0].get(0), Some("Alice"));
236        assert_eq!(records[0].get(1), Some("25"));
237
238        Ok(())
239    }
240
241    #[test]
242    fn test_row_estimation() -> Result<()> {
243        let mut temp_file = NamedTempFile::new()?;
244        writeln!(temp_file, "a,b,c")?;
245        for i in 0..100 {
246            writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
247        }
248        temp_file.flush()?;
249
250        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
251        let estimated = reader.estimate_row_count()?;
252
253        assert!(estimated > 90 && estimated < 120);
254
255        Ok(())
256    }
257
258    #[test]
259    fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
260        let expected_rows = 1000;
261        let mut temp_file = NamedTempFile::new()?;
262        writeln!(temp_file, "id,name,value")?;
263        for i in 0..expected_rows {
264            let padding = "x".repeat(i % 50);
265            writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
266        }
267        temp_file.flush()?;
268
269        let reader = MemoryMappedCsvReader::new(temp_file.path())?;
270
271        let chunk_size = 512;
272        let mut offset = 0u64;
273        let mut total_records = 0;
274        let mut first = true;
275
276        loop {
277            let (headers, records, bytes) =
278                reader.read_csv_chunk(offset, chunk_size, first, None)?;
279            if records.is_empty() && bytes == 0 {
280                break;
281            }
282            if first && headers.is_some() {
283                first = false;
284            }
285            total_records += records.len();
286            offset += bytes as u64;
287        }
288
289        assert_eq!(
290            total_records, expected_rows,
291            "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
292        );
293
294        Ok(())
295    }
296}