1use anyhow::Result;
2use memmap2::Mmap;
3use std::fs::File;
4use std::path::Path;
5
6use crate::CsvParserConfig;
7
8pub struct MemoryMappedCsvReader {
10 mmap: Mmap,
11 file_size: u64,
12}
13
14impl MemoryMappedCsvReader {
15 pub fn new(path: &Path) -> Result<Self> {
16 let file = File::open(path)?;
17 let file_size = file.metadata()?.len();
18
19 #[allow(unsafe_code)]
22 let mmap = unsafe { Mmap::map(&file)? };
23
24 Ok(Self { mmap, file_size })
25 }
26
27 pub fn file_size(&self) -> u64 {
29 self.file_size
30 }
31
32 pub fn read_csv_chunk(
36 &self,
37 offset: u64,
38 chunk_size: usize,
39 has_headers: bool,
40 csv_config: Option<&CsvParserConfig>,
41 ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
42 let (headers, records, actual_bytes, ragged) =
43 self.read_csv_chunk_until_ragged(offset, chunk_size, has_headers, csv_config)?;
44 match ragged {
45 Some(error) => Err(error),
46 None => Ok((headers, records, actual_bytes)),
47 }
48 }
49
50 #[allow(clippy::type_complexity)]
56 pub fn read_csv_chunk_until_ragged(
57 &self,
58 offset: u64,
59 chunk_size: usize,
60 has_headers: bool,
61 csv_config: Option<&CsvParserConfig>,
62 ) -> Result<(
63 Option<csv::StringRecord>,
64 Vec<csv::StringRecord>,
65 usize,
66 Option<anyhow::Error>,
67 )> {
68 let start = offset as usize;
69 if start >= self.mmap.len() {
70 return Ok((None, Vec::new(), 0, None));
71 }
72
73 let quote = csv_config.map_or(b'"', |config| config.quote_char);
74 let delimiter = csv_config
75 .and_then(|config| config.delimiter)
76 .unwrap_or(b',');
77 let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
78
79 if has_headers && offset == 0 {
85 let bom = self.leading_bom_len();
90 let header_start = bom
91 + self.mmap[bom..]
92 .iter()
93 .take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
94 .count();
95 if end <= header_start && header_start < self.mmap.len() {
96 end = self.csv_record_boundary(header_start, 1, quote, delimiter);
97 }
98 }
99 let chunk_data = &self.mmap[start..end];
100 let actual_bytes = chunk_data.len();
101
102 let mut builder = csv::ReaderBuilder::new();
103 builder.has_headers(has_headers && offset == 0);
104 if let Some(config) = csv_config {
105 if let Some(delim) = config.delimiter {
106 builder.delimiter(delim);
107 }
108 builder.flexible(config.flexible);
109 builder.quote(config.quote_char);
110 if config.trim_whitespace {
111 builder.trim(csv::Trim::All);
112 }
113 }
114 let mut reader = builder.from_reader(chunk_data);
115
116 let headers = if has_headers && offset == 0 {
117 Some(reader.headers()?.clone())
118 } else {
119 None
120 };
121
122 let mut records = Vec::new();
123 for result in reader.records() {
124 match result {
125 Ok(record) => records.push(record),
126 Err(error) if matches!(error.kind(), csv::ErrorKind::UnequalLengths { .. }) => {
127 return Ok((headers, records, actual_bytes, Some(error.into())));
128 }
129 Err(error) => return Err(error.into()),
130 }
131 }
132
133 Ok((headers, records, actual_bytes, None))
134 }
135
136 pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
144 let start = usize::try_from(offset)
145 .unwrap_or(usize::MAX)
146 .min(self.mmap.len());
147 let quote = csv_config.map_or(b'"', |config| config.quote_char);
148 let delimiter = csv_config
149 .and_then(|config| config.delimiter)
150 .unwrap_or(b',');
151 crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
152 }
153
154 fn csv_record_boundary(
163 &self,
164 start: usize,
165 chunk_size: usize,
166 quote: u8,
167 delimiter: u8,
168 ) -> usize {
169 let target = start.saturating_add(chunk_size).min(self.mmap.len());
170 let mut index = if start == 0 {
173 self.leading_bom_len()
174 } else {
175 start
176 };
177 let mut in_quotes = false;
178 let mut at_field_start = true;
179 let mut last_boundary = None;
180
181 while index < target {
182 let byte = self.mmap[index];
183 if in_quotes && byte == quote {
184 if self.mmap.get(index + 1) == Some("e) {
187 index += 2;
188 continue;
189 }
190 in_quotes = false;
191 } else if !in_quotes && at_field_start && byte == quote {
192 in_quotes = true;
193 at_field_start = false;
194 } else if !in_quotes && byte == delimiter {
195 at_field_start = true;
196 } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
197 if boundary > target && last_boundary.is_some() {
201 break;
202 }
203 last_boundary = Some(boundary);
204 at_field_start = true;
205 index = boundary;
206 continue;
207 } else if !in_quotes {
208 at_field_start = false;
209 }
210 index += 1;
211 }
212
213 if target == self.mmap.len() {
218 return target;
219 }
220
221 if let Some(boundary) = last_boundary {
222 return boundary;
223 }
224
225 while index < self.mmap.len() {
229 let byte = self.mmap[index];
230 if in_quotes && byte == quote {
231 if self.mmap.get(index + 1) == Some("e) {
232 index += 2;
233 continue;
234 }
235 in_quotes = false;
236 } else if !in_quotes && at_field_start && byte == quote {
237 in_quotes = true;
238 at_field_start = false;
239 } else if !in_quotes && byte == delimiter {
240 at_field_start = true;
241 } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
242 return boundary;
243 } else if !in_quotes {
244 at_field_start = false;
245 }
246 index += 1;
247 }
248
249 self.mmap.len()
250 }
251
252 fn leading_bom_len(&self) -> usize {
255 if self.mmap.starts_with(b"\xEF\xBB\xBF") {
256 3
257 } else {
258 0
259 }
260 }
261
262 fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
264 match self.mmap[index] {
265 b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
266 b'\r' | b'\n' => Some(index + 1),
267 _ => None,
268 }
269 }
270
271 pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
274 let delimiter = csv_config
275 .and_then(|config| config.delimiter)
276 .unwrap_or(b',');
277 let quote = csv_config.map_or(b'"', |config| config.quote_char);
278 let count = crate::row_count::csv_record_count(&self.mmap[..], delimiter, quote)?;
279 Ok(usize::try_from(count).unwrap_or(usize::MAX))
280 }
281}
282
283#[cfg(test)]
284mod tests {
285 use super::*;
286 use std::io::Write;
287 use tempfile::NamedTempFile;
288
289 #[test]
290 fn test_memory_mapped_reader() -> Result<()> {
291 let mut temp_file = NamedTempFile::new()?;
292 writeln!(temp_file, "name,age,city")?;
293 writeln!(temp_file, "Alice,25,New York")?;
294 writeln!(temp_file, "Bob,30,London")?;
295 writeln!(temp_file, "Charlie,35,Tokyo")?;
296 temp_file.flush()?;
297
298 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
299
300 assert!(reader.file_size() > 0);
301
302 let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
303
304 assert!(headers.is_some());
305 assert_eq!(records.len(), 3);
306
307 let header_record = headers.expect("Headers should be present in test data");
308 assert_eq!(header_record.get(0), Some("name"));
309 assert_eq!(header_record.get(1), Some("age"));
310 assert_eq!(header_record.get(2), Some("city"));
311
312 assert_eq!(records[0].get(0), Some("Alice"));
313 assert_eq!(records[0].get(1), Some("25"));
314
315 Ok(())
316 }
317
318 #[test]
319 fn test_row_estimation() -> Result<()> {
320 let mut temp_file = NamedTempFile::new()?;
321 writeln!(temp_file, "a,b,c")?;
322 for i in 0..100 {
323 writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
324 }
325 temp_file.flush()?;
326
327 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
328 let estimated = reader.estimate_csv_record_count(None)?;
329
330 assert!(estimated > 90 && estimated < 120);
331
332 Ok(())
333 }
334
335 #[test]
336 fn row_estimation_counts_logical_csv_records() -> Result<()> {
337 let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
338 (
339 "quoted multiline",
340 b"id,bio\n1,\"hello\nworld\"\n2,plain",
341 None,
342 ),
343 ("lone CR", b"id,value\r1,x\r2,y", None),
344 (
345 "custom delimiter",
346 b"id;bio\r1;\"hello\nworld\"\r2;plain",
347 Some(CsvParserConfig::default().with_delimiter(b';')),
348 ),
349 ];
350
351 for (case, payload, config) in cases {
352 let mut temp_file = NamedTempFile::new()?;
353 temp_file.write_all(payload)?;
354 temp_file.flush()?;
355
356 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
357 assert_eq!(
358 reader.estimate_csv_record_count(config.as_ref())?,
359 3,
360 "case={case}"
361 );
362 }
363 Ok(())
364 }
365
366 #[test]
367 fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
368 let expected_rows = 1000;
369 let mut temp_file = NamedTempFile::new()?;
370 writeln!(temp_file, "id,name,value")?;
371 for i in 0..expected_rows {
372 let padding = "x".repeat(i % 50);
373 writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
374 }
375 temp_file.flush()?;
376
377 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
378
379 let chunk_size = 512;
380 let mut offset = 0u64;
381 let mut total_records = 0;
382 let mut first = true;
383
384 loop {
385 let (headers, records, bytes) =
386 reader.read_csv_chunk(offset, chunk_size, first, None)?;
387 if records.is_empty() && bytes == 0 {
388 break;
389 }
390 if first && headers.is_some() {
391 first = false;
392 }
393 total_records += records.len();
394 offset += bytes as u64;
395 }
396
397 assert_eq!(
398 total_records, expected_rows,
399 "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
400 );
401
402 Ok(())
403 }
404
405 #[test]
406 fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
407 let mut temp_file = NamedTempFile::new()?;
408 write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
409 temp_file.flush()?;
410
411 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
412 let mut offset = 0u64;
413 let mut headers = None;
414 let mut records = Vec::new();
415
416 loop {
417 let (chunk_headers, chunk_records, bytes) =
418 reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
419 if bytes == 0 {
420 break;
421 }
422 if headers.is_none() {
423 headers = chunk_headers;
424 }
425 records.extend(chunk_records);
426 offset += bytes as u64;
427 }
428
429 assert_eq!(
430 headers.expect("header").iter().collect::<Vec<_>>(),
431 ["alpha", "beta"]
432 );
433 assert_eq!(records.len(), 2);
434 assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
435 assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
436 Ok(())
437 }
438
439 #[test]
440 fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
441 let payload = b"\n\r\n\ralpha,beta\n1,2";
442 let mut temp_file = NamedTempFile::new()?;
443 temp_file.write_all(payload)?;
444 temp_file.flush()?;
445
446 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
447 let mut offset = 0u64;
448 let mut headers = None;
449 let mut records = Vec::new();
450
451 loop {
452 let (chunk_headers, chunk_records, bytes) =
453 reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
454 if bytes == 0 {
455 break;
456 }
457 if headers.is_none() {
458 headers = chunk_headers;
459 }
460 records.extend(chunk_records);
461 offset += bytes as u64;
462 }
463
464 assert_eq!(
465 headers.expect("header").iter().collect::<Vec<_>>(),
466 ["alpha", "beta"]
467 );
468 assert_eq!(records.len(), 1);
469 assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
470 Ok(())
471 }
472
473 #[test]
474 fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
475 let cases: [(&str, &[u8], &str); 2] = [
476 (
477 "crlf",
478 b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
479 "hello \"quoted\"\nworld",
480 ),
481 (
482 "lone-cr",
483 b"id,bio\r1,\"hello\nworld\"\r2,plain",
484 "hello\nworld",
485 ),
486 ];
487
488 for (terminator, payload, expected_bio) in cases {
489 let mut temp_file = NamedTempFile::new()?;
490 temp_file.write_all(payload)?;
491 temp_file.flush()?;
492
493 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
494 for chunk_size in 1..=payload.len() + 1 {
495 let mut offset = 0u64;
496 let mut headers = None;
497 let mut records = Vec::new();
498
499 loop {
500 let (chunk_headers, chunk_records, bytes) =
501 reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
502 if bytes == 0 {
503 break;
504 }
505 if headers.is_none() {
506 headers = chunk_headers;
507 }
508 records.extend(chunk_records);
509 offset += bytes as u64;
510 }
511
512 assert_eq!(
513 headers.expect("header").iter().collect::<Vec<_>>(),
514 ["id", "bio"],
515 "terminator={terminator}, chunk_size={chunk_size}"
516 );
517 assert_eq!(
518 records.len(),
519 2,
520 "terminator={terminator}, chunk_size={chunk_size}"
521 );
522 assert_eq!(
523 records[0].get(0),
524 Some("1"),
525 "terminator={terminator}, chunk_size={chunk_size}"
526 );
527 assert_eq!(
528 records[0].get(1),
529 Some(expected_bio),
530 "terminator={terminator}, chunk_size={chunk_size}"
531 );
532 assert_eq!(
533 records[1].iter().collect::<Vec<_>>(),
534 ["2", "plain"],
535 "terminator={terminator}, chunk_size={chunk_size}"
536 );
537 }
538 }
539 Ok(())
540 }
541}