1use anyhow::Result;
2use dataprof_core::MemoryTracker;
3use memmap2::Mmap;
4use std::fs::File;
5use std::io::{BufRead, BufReader, Cursor};
6use std::path::Path;
7
8use crate::CsvParserConfig;
9
10pub struct MemoryMappedCsvReader {
12 mmap: Mmap,
13 file_size: u64,
14 memory_tracker: MemoryTracker,
15 resource_id: String,
16}
17
18impl MemoryMappedCsvReader {
19 pub fn new(path: &Path) -> Result<Self> {
20 Self::new_with_tracker(path, MemoryTracker::default())
21 }
22
23 pub fn new_with_tracker(path: &Path, memory_tracker: MemoryTracker) -> Result<Self> {
24 let file = File::open(path)?;
25 let file_size = file.metadata()?.len();
26
27 #[allow(unsafe_code)]
30 let mmap = unsafe { Mmap::map(&file)? };
31
32 let resource_id = format!("mmap_{}", path.display());
33
34 memory_tracker.track_allocation(resource_id.clone(), file_size as usize, "memory_map");
35
36 Ok(Self {
37 mmap,
38 file_size,
39 memory_tracker,
40 resource_id,
41 })
42 }
43
44 pub fn file_size(&self) -> u64 {
46 self.file_size
47 }
48
49 pub fn read_chunk(&self, offset: u64, chunk_size: usize) -> Result<(Vec<String>, usize)> {
54 let start = offset as usize;
55 let end = std::cmp::min(start + chunk_size, self.mmap.len());
56
57 if start >= self.mmap.len() {
58 return Ok((Vec::new(), 0));
59 }
60
61 let chunk_data = &self.mmap[start..end];
62
63 let at_eof = end == self.mmap.len();
64 let starts_mid_line = start > 0 && self.mmap[start - 1] != b'\n';
65 let (adjusted_chunk, bytes_consumed) =
66 self.find_line_boundary(chunk_data, starts_mid_line, at_eof);
67
68 let cursor = Cursor::new(adjusted_chunk);
69 let reader = BufReader::new(cursor);
70
71 let mut lines = Vec::new();
72 for line in reader.lines() {
73 lines.push(line?);
74 }
75
76 Ok((lines, bytes_consumed))
77 }
78
79 pub fn read_csv_chunk(
83 &self,
84 offset: u64,
85 chunk_size: usize,
86 has_headers: bool,
87 csv_config: Option<&CsvParserConfig>,
88 ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
89 let start = offset as usize;
90 if start >= self.mmap.len() {
91 return Ok((None, Vec::new(), 0));
92 }
93
94 let quote = csv_config.map_or(b'"', |config| config.quote_char);
95 let delimiter = csv_config
96 .and_then(|config| config.delimiter)
97 .unwrap_or(b',');
98 let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
99
100 if has_headers && offset == 0 {
106 let header_start = self
107 .mmap
108 .iter()
109 .take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
110 .count();
111 if end <= header_start && header_start < self.mmap.len() {
112 end = self.csv_record_boundary(header_start, 1, quote, delimiter);
113 }
114 }
115 let chunk_data = &self.mmap[start..end];
116 let actual_bytes = chunk_data.len();
117
118 let mut builder = csv::ReaderBuilder::new();
119 builder.has_headers(has_headers && offset == 0);
120 if let Some(config) = csv_config {
121 if let Some(delim) = config.delimiter {
122 builder.delimiter(delim);
123 }
124 builder.flexible(config.flexible);
125 builder.quote(config.quote_char);
126 if config.trim_whitespace {
127 builder.trim(csv::Trim::All);
128 }
129 }
130 let mut reader = builder.from_reader(chunk_data);
131
132 let headers = if has_headers && offset == 0 {
133 Some(reader.headers()?.clone())
134 } else {
135 None
136 };
137
138 let mut records = Vec::new();
139 for result in reader.records() {
140 records.push(result?);
141 }
142
143 Ok((headers, records, actual_bytes))
144 }
145
146 pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
154 let start = usize::try_from(offset)
155 .unwrap_or(usize::MAX)
156 .min(self.mmap.len());
157 let quote = csv_config.map_or(b'"', |config| config.quote_char);
158 let delimiter = csv_config
159 .and_then(|config| config.delimiter)
160 .unwrap_or(b',');
161 crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
162 }
163
164 fn csv_record_boundary(
173 &self,
174 start: usize,
175 chunk_size: usize,
176 quote: u8,
177 delimiter: u8,
178 ) -> usize {
179 let target = start.saturating_add(chunk_size).min(self.mmap.len());
180 let mut index = start;
181 let mut in_quotes = false;
182 let mut at_field_start = true;
183 let mut last_boundary = None;
184
185 while index < target {
186 let byte = self.mmap[index];
187 if in_quotes && byte == quote {
188 if self.mmap.get(index + 1) == Some("e) {
191 index += 2;
192 continue;
193 }
194 in_quotes = false;
195 } else if !in_quotes && at_field_start && byte == quote {
196 in_quotes = true;
197 at_field_start = false;
198 } else if !in_quotes && byte == delimiter {
199 at_field_start = true;
200 } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
201 if boundary > target && last_boundary.is_some() {
205 break;
206 }
207 last_boundary = Some(boundary);
208 at_field_start = true;
209 index = boundary;
210 continue;
211 } else if !in_quotes {
212 at_field_start = false;
213 }
214 index += 1;
215 }
216
217 if target == self.mmap.len() {
222 return target;
223 }
224
225 if let Some(boundary) = last_boundary {
226 return boundary;
227 }
228
229 while index < self.mmap.len() {
233 let byte = self.mmap[index];
234 if in_quotes && byte == quote {
235 if self.mmap.get(index + 1) == Some("e) {
236 index += 2;
237 continue;
238 }
239 in_quotes = false;
240 } else if !in_quotes && at_field_start && byte == quote {
241 in_quotes = true;
242 at_field_start = false;
243 } else if !in_quotes && byte == delimiter {
244 at_field_start = true;
245 } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
246 return boundary;
247 } else if !in_quotes {
248 at_field_start = false;
249 }
250 index += 1;
251 }
252
253 self.mmap.len()
254 }
255
256 fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
258 match self.mmap[index] {
259 b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
260 b'\r' | b'\n' => Some(index + 1),
261 _ => None,
262 }
263 }
264
265 fn find_line_boundary<'a>(
275 &self,
276 chunk: &'a [u8],
277 skip_first_partial: bool,
278 at_eof: bool,
279 ) -> (&'a [u8], usize) {
280 if chunk.is_empty() {
281 return (chunk, 0);
282 }
283
284 let mut start_pos = 0;
285
286 if skip_first_partial {
287 if let Some(first_newline) = chunk.iter().position(|&b| b == b'\n') {
288 start_pos = first_newline + 1;
289 } else {
290 return (&chunk[chunk.len()..], chunk.len());
291 }
292 }
293
294 let mut end_pos = chunk.len();
295
296 if !at_eof {
297 if let Some(last_newline) = chunk[start_pos..].iter().rposition(|&b| b == b'\n') {
298 end_pos = start_pos + last_newline + 1;
299 } else if start_pos > 0 {
300 return (&chunk[chunk.len()..], start_pos);
301 }
302 }
303
304 (&chunk[start_pos..end_pos], end_pos)
305 }
306
307 pub fn estimate_row_count(&self) -> Result<usize> {
313 self.estimate_csv_record_count(None)
314 }
315
316 pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
322 const SAMPLE_SIZE: usize = 64 * 1024;
323
324 if self.mmap.is_empty() {
325 return Ok(0);
326 }
327
328 let (_, records, sampled_bytes) = self.read_csv_chunk(0, SAMPLE_SIZE, false, csv_config)?;
329 if sampled_bytes == 0 || records.is_empty() {
330 return Ok(0);
331 }
332 if sampled_bytes == self.mmap.len() {
333 return Ok(records.len());
334 }
335
336 let estimated_records =
337 (self.file_size as u128 * records.len() as u128) / sampled_bytes as u128;
338 Ok(estimated_records.min(usize::MAX as u128) as usize)
339 }
340
341 pub fn check_memory_leaks(&self) -> String {
343 self.memory_tracker.report_leaks()
344 }
345
346 pub fn get_memory_stats(&self) -> (usize, usize, usize) {
348 self.memory_tracker.get_memory_stats()
349 }
350}
351
352impl Drop for MemoryMappedCsvReader {
353 fn drop(&mut self) {
354 self.memory_tracker.track_deallocation(&self.resource_id);
355 }
356}
357
358#[cfg(test)]
359mod tests {
360 use super::*;
361 use std::io::Write;
362 use tempfile::NamedTempFile;
363
364 #[test]
365 fn test_memory_mapped_reader() -> Result<()> {
366 let mut temp_file = NamedTempFile::new()?;
367 writeln!(temp_file, "name,age,city")?;
368 writeln!(temp_file, "Alice,25,New York")?;
369 writeln!(temp_file, "Bob,30,London")?;
370 writeln!(temp_file, "Charlie,35,Tokyo")?;
371 temp_file.flush()?;
372
373 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
374
375 assert!(reader.file_size() > 0);
376
377 let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
378
379 assert!(headers.is_some());
380 assert_eq!(records.len(), 3);
381
382 let header_record = headers.expect("Headers should be present in test data");
383 assert_eq!(header_record.get(0), Some("name"));
384 assert_eq!(header_record.get(1), Some("age"));
385 assert_eq!(header_record.get(2), Some("city"));
386
387 assert_eq!(records[0].get(0), Some("Alice"));
388 assert_eq!(records[0].get(1), Some("25"));
389
390 Ok(())
391 }
392
393 #[test]
394 fn test_row_estimation() -> Result<()> {
395 let mut temp_file = NamedTempFile::new()?;
396 writeln!(temp_file, "a,b,c")?;
397 for i in 0..100 {
398 writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
399 }
400 temp_file.flush()?;
401
402 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
403 let estimated = reader.estimate_row_count()?;
404
405 assert!(estimated > 90 && estimated < 120);
406
407 Ok(())
408 }
409
410 #[test]
411 fn row_estimation_counts_logical_csv_records() -> Result<()> {
412 let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
413 (
414 "quoted multiline",
415 b"id,bio\n1,\"hello\nworld\"\n2,plain",
416 None,
417 ),
418 ("lone CR", b"id,value\r1,x\r2,y", None),
419 (
420 "custom delimiter",
421 b"id;bio\r1;\"hello\nworld\"\r2;plain",
422 Some(CsvParserConfig::default().with_delimiter(b';')),
423 ),
424 ];
425
426 for (case, payload, config) in cases {
427 let mut temp_file = NamedTempFile::new()?;
428 temp_file.write_all(payload)?;
429 temp_file.flush()?;
430
431 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
432 assert_eq!(
433 reader.estimate_csv_record_count(config.as_ref())?,
434 3,
435 "case={case}"
436 );
437 }
438 Ok(())
439 }
440
441 #[test]
442 fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
443 let expected_rows = 1000;
444 let mut temp_file = NamedTempFile::new()?;
445 writeln!(temp_file, "id,name,value")?;
446 for i in 0..expected_rows {
447 let padding = "x".repeat(i % 50);
448 writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
449 }
450 temp_file.flush()?;
451
452 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
453
454 let chunk_size = 512;
455 let mut offset = 0u64;
456 let mut total_records = 0;
457 let mut first = true;
458
459 loop {
460 let (headers, records, bytes) =
461 reader.read_csv_chunk(offset, chunk_size, first, None)?;
462 if records.is_empty() && bytes == 0 {
463 break;
464 }
465 if first && headers.is_some() {
466 first = false;
467 }
468 total_records += records.len();
469 offset += bytes as u64;
470 }
471
472 assert_eq!(
473 total_records, expected_rows,
474 "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
475 );
476
477 Ok(())
478 }
479
480 #[test]
481 fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
482 let mut temp_file = NamedTempFile::new()?;
483 write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
484 temp_file.flush()?;
485
486 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
487 let mut offset = 0u64;
488 let mut headers = None;
489 let mut records = Vec::new();
490
491 loop {
492 let (chunk_headers, chunk_records, bytes) =
493 reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
494 if bytes == 0 {
495 break;
496 }
497 if headers.is_none() {
498 headers = chunk_headers;
499 }
500 records.extend(chunk_records);
501 offset += bytes as u64;
502 }
503
504 assert_eq!(
505 headers.expect("header").iter().collect::<Vec<_>>(),
506 ["alpha", "beta"]
507 );
508 assert_eq!(records.len(), 2);
509 assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
510 assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
511 Ok(())
512 }
513
514 #[test]
515 fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
516 let payload = b"\n\r\n\ralpha,beta\n1,2";
517 let mut temp_file = NamedTempFile::new()?;
518 temp_file.write_all(payload)?;
519 temp_file.flush()?;
520
521 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
522 let mut offset = 0u64;
523 let mut headers = None;
524 let mut records = Vec::new();
525
526 loop {
527 let (chunk_headers, chunk_records, bytes) =
528 reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
529 if bytes == 0 {
530 break;
531 }
532 if headers.is_none() {
533 headers = chunk_headers;
534 }
535 records.extend(chunk_records);
536 offset += bytes as u64;
537 }
538
539 assert_eq!(
540 headers.expect("header").iter().collect::<Vec<_>>(),
541 ["alpha", "beta"]
542 );
543 assert_eq!(records.len(), 1);
544 assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
545 Ok(())
546 }
547
548 #[test]
549 fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
550 let cases: [(&str, &[u8], &str); 2] = [
551 (
552 "crlf",
553 b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
554 "hello \"quoted\"\nworld",
555 ),
556 (
557 "lone-cr",
558 b"id,bio\r1,\"hello\nworld\"\r2,plain",
559 "hello\nworld",
560 ),
561 ];
562
563 for (terminator, payload, expected_bio) in cases {
564 let mut temp_file = NamedTempFile::new()?;
565 temp_file.write_all(payload)?;
566 temp_file.flush()?;
567
568 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
569 for chunk_size in 1..=payload.len() + 1 {
570 let mut offset = 0u64;
571 let mut headers = None;
572 let mut records = Vec::new();
573
574 loop {
575 let (chunk_headers, chunk_records, bytes) =
576 reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
577 if bytes == 0 {
578 break;
579 }
580 if headers.is_none() {
581 headers = chunk_headers;
582 }
583 records.extend(chunk_records);
584 offset += bytes as u64;
585 }
586
587 assert_eq!(
588 headers.expect("header").iter().collect::<Vec<_>>(),
589 ["id", "bio"],
590 "terminator={terminator}, chunk_size={chunk_size}"
591 );
592 assert_eq!(
593 records.len(),
594 2,
595 "terminator={terminator}, chunk_size={chunk_size}"
596 );
597 assert_eq!(
598 records[0].get(0),
599 Some("1"),
600 "terminator={terminator}, chunk_size={chunk_size}"
601 );
602 assert_eq!(
603 records[0].get(1),
604 Some(expected_bio),
605 "terminator={terminator}, chunk_size={chunk_size}"
606 );
607 assert_eq!(
608 records[1].iter().collect::<Vec<_>>(),
609 ["2", "plain"],
610 "terminator={terminator}, chunk_size={chunk_size}"
611 );
612 }
613 }
614 Ok(())
615 }
616}