1use anyhow::Result;
2use dataprof_core::MemoryTracker;
3use memmap2::Mmap;
4use std::fs::File;
5use std::io::{BufRead, BufReader, Cursor};
6use std::path::Path;
7
8use crate::CsvParserConfig;
9
10pub struct MemoryMappedCsvReader {
12 mmap: Mmap,
13 file_size: u64,
14 memory_tracker: MemoryTracker,
15 resource_id: String,
16}
17
18impl MemoryMappedCsvReader {
19 pub fn new(path: &Path) -> Result<Self> {
20 Self::new_with_tracker(path, MemoryTracker::default())
21 }
22
23 pub fn new_with_tracker(path: &Path, memory_tracker: MemoryTracker) -> Result<Self> {
24 let file = File::open(path)?;
25 let file_size = file.metadata()?.len();
26
27 #[allow(unsafe_code)]
30 let mmap = unsafe { Mmap::map(&file)? };
31
32 let resource_id = format!("mmap_{}", path.display());
33
34 memory_tracker.track_allocation(resource_id.clone(), file_size as usize, "memory_map");
35
36 Ok(Self {
37 mmap,
38 file_size,
39 memory_tracker,
40 resource_id,
41 })
42 }
43
44 pub fn file_size(&self) -> u64 {
46 self.file_size
47 }
48
49 pub fn read_chunk(&self, offset: u64, chunk_size: usize) -> Result<(Vec<String>, usize)> {
54 let start = offset as usize;
55 let end = std::cmp::min(start + chunk_size, self.mmap.len());
56
57 if start >= self.mmap.len() {
58 return Ok((Vec::new(), 0));
59 }
60
61 let chunk_data = &self.mmap[start..end];
62
63 let at_eof = end == self.mmap.len();
64 let starts_mid_line = start > 0 && self.mmap[start - 1] != b'\n';
65 let (adjusted_chunk, bytes_consumed) =
66 self.find_line_boundary(chunk_data, starts_mid_line, at_eof);
67
68 let cursor = Cursor::new(adjusted_chunk);
69 let reader = BufReader::new(cursor);
70
71 let mut lines = Vec::new();
72 for line in reader.lines() {
73 lines.push(line?);
74 }
75
76 Ok((lines, bytes_consumed))
77 }
78
79 pub fn read_csv_chunk(
83 &self,
84 offset: u64,
85 chunk_size: usize,
86 has_headers: bool,
87 csv_config: Option<&CsvParserConfig>,
88 ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
89 let (lines, actual_bytes) = self.read_chunk(offset, chunk_size)?;
90
91 if lines.is_empty() {
92 return Ok((None, Vec::new(), 0));
93 }
94
95 let chunk_data = lines.join("\n");
96 let mut builder = csv::ReaderBuilder::new();
97 builder.has_headers(has_headers && offset == 0);
98 if let Some(config) = csv_config {
99 if let Some(delim) = config.delimiter {
100 builder.delimiter(delim);
101 }
102 builder.flexible(config.flexible);
103 builder.quote(config.quote_char);
104 if config.trim_whitespace {
105 builder.trim(csv::Trim::All);
106 }
107 }
108 let mut reader = builder.from_reader(Cursor::new(chunk_data.as_bytes()));
109
110 let headers = if has_headers && offset == 0 {
111 Some(reader.headers()?.clone())
112 } else {
113 None
114 };
115
116 let mut records = Vec::new();
117 for result in reader.records() {
118 records.push(result?);
119 }
120
121 Ok((headers, records, actual_bytes))
122 }
123
124 fn find_line_boundary<'a>(
134 &self,
135 chunk: &'a [u8],
136 skip_first_partial: bool,
137 at_eof: bool,
138 ) -> (&'a [u8], usize) {
139 if chunk.is_empty() {
140 return (chunk, 0);
141 }
142
143 let mut start_pos = 0;
144
145 if skip_first_partial {
146 if let Some(first_newline) = chunk.iter().position(|&b| b == b'\n') {
147 start_pos = first_newline + 1;
148 } else {
149 return (&chunk[chunk.len()..], chunk.len());
150 }
151 }
152
153 let mut end_pos = chunk.len();
154
155 if !at_eof {
156 if let Some(last_newline) = chunk[start_pos..].iter().rposition(|&b| b == b'\n') {
157 end_pos = start_pos + last_newline + 1;
158 } else if start_pos > 0 {
159 return (&chunk[chunk.len()..], start_pos);
160 }
161 }
162
163 (&chunk[start_pos..end_pos], end_pos)
164 }
165
166 pub fn estimate_row_count(&self) -> Result<usize> {
168 const SAMPLE_SIZE: usize = 64 * 1024;
169
170 if self.file_size < SAMPLE_SIZE as u64 {
171 let cursor = Cursor::new(&*self.mmap);
172 let reader = BufReader::new(cursor);
173 return Ok(reader.lines().count());
174 }
175
176 let sample_data = &self.mmap[0..SAMPLE_SIZE];
177 let cursor = Cursor::new(sample_data);
178 let reader = BufReader::new(cursor);
179
180 let sample_lines = reader.lines().count();
181 if sample_lines == 0 {
182 return Ok(0);
183 }
184
185 let estimated_rows = (self.file_size * sample_lines as u64) / SAMPLE_SIZE as u64;
186 Ok(estimated_rows as usize)
187 }
188
189 pub fn check_memory_leaks(&self) -> String {
191 self.memory_tracker.report_leaks()
192 }
193
194 pub fn get_memory_stats(&self) -> (usize, usize, usize) {
196 self.memory_tracker.get_memory_stats()
197 }
198}
199
200impl Drop for MemoryMappedCsvReader {
201 fn drop(&mut self) {
202 self.memory_tracker.track_deallocation(&self.resource_id);
203 }
204}
205
206#[cfg(test)]
207mod tests {
208 use super::*;
209 use std::io::Write;
210 use tempfile::NamedTempFile;
211
212 #[test]
213 fn test_memory_mapped_reader() -> Result<()> {
214 let mut temp_file = NamedTempFile::new()?;
215 writeln!(temp_file, "name,age,city")?;
216 writeln!(temp_file, "Alice,25,New York")?;
217 writeln!(temp_file, "Bob,30,London")?;
218 writeln!(temp_file, "Charlie,35,Tokyo")?;
219 temp_file.flush()?;
220
221 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
222
223 assert!(reader.file_size() > 0);
224
225 let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
226
227 assert!(headers.is_some());
228 assert_eq!(records.len(), 3);
229
230 let header_record = headers.expect("Headers should be present in test data");
231 assert_eq!(header_record.get(0), Some("name"));
232 assert_eq!(header_record.get(1), Some("age"));
233 assert_eq!(header_record.get(2), Some("city"));
234
235 assert_eq!(records[0].get(0), Some("Alice"));
236 assert_eq!(records[0].get(1), Some("25"));
237
238 Ok(())
239 }
240
241 #[test]
242 fn test_row_estimation() -> Result<()> {
243 let mut temp_file = NamedTempFile::new()?;
244 writeln!(temp_file, "a,b,c")?;
245 for i in 0..100 {
246 writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
247 }
248 temp_file.flush()?;
249
250 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
251 let estimated = reader.estimate_row_count()?;
252
253 assert!(estimated > 90 && estimated < 120);
254
255 Ok(())
256 }
257
258 #[test]
259 fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
260 let expected_rows = 1000;
261 let mut temp_file = NamedTempFile::new()?;
262 writeln!(temp_file, "id,name,value")?;
263 for i in 0..expected_rows {
264 let padding = "x".repeat(i % 50);
265 writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
266 }
267 temp_file.flush()?;
268
269 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
270
271 let chunk_size = 512;
272 let mut offset = 0u64;
273 let mut total_records = 0;
274 let mut first = true;
275
276 loop {
277 let (headers, records, bytes) =
278 reader.read_csv_chunk(offset, chunk_size, first, None)?;
279 if records.is_empty() && bytes == 0 {
280 break;
281 }
282 if first && headers.is_some() {
283 first = false;
284 }
285 total_records += records.len();
286 offset += bytes as u64;
287 }
288
289 assert_eq!(
290 total_records, expected_rows,
291 "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
292 );
293
294 Ok(())
295 }
296}