1use anyhow::Result;
2use memmap2::Mmap;
3use std::fs::File;
4use std::path::Path;
5
6use crate::CsvParserConfig;
7
8pub struct MemoryMappedCsvReader {
10 mmap: Mmap,
11 file_size: u64,
12}
13
14impl MemoryMappedCsvReader {
15 pub fn new(path: &Path) -> Result<Self> {
16 let file = File::open(path)?;
17 let file_size = file.metadata()?.len();
18
19 #[allow(unsafe_code)]
22 let mmap = unsafe { Mmap::map(&file)? };
23
24 Ok(Self { mmap, file_size })
25 }
26
27 pub fn file_size(&self) -> u64 {
29 self.file_size
30 }
31
32 pub fn read_csv_chunk(
36 &self,
37 offset: u64,
38 chunk_size: usize,
39 has_headers: bool,
40 csv_config: Option<&CsvParserConfig>,
41 ) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
42 let start = offset as usize;
43 if start >= self.mmap.len() {
44 return Ok((None, Vec::new(), 0));
45 }
46
47 let quote = csv_config.map_or(b'"', |config| config.quote_char);
48 let delimiter = csv_config
49 .and_then(|config| config.delimiter)
50 .unwrap_or(b',');
51 let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
52
53 if has_headers && offset == 0 {
59 let bom = self.leading_bom_len();
64 let header_start = bom
65 + self.mmap[bom..]
66 .iter()
67 .take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
68 .count();
69 if end <= header_start && header_start < self.mmap.len() {
70 end = self.csv_record_boundary(header_start, 1, quote, delimiter);
71 }
72 }
73 let chunk_data = &self.mmap[start..end];
74 let actual_bytes = chunk_data.len();
75
76 let mut builder = csv::ReaderBuilder::new();
77 builder.has_headers(has_headers && offset == 0);
78 if let Some(config) = csv_config {
79 if let Some(delim) = config.delimiter {
80 builder.delimiter(delim);
81 }
82 builder.flexible(config.flexible);
83 builder.quote(config.quote_char);
84 if config.trim_whitespace {
85 builder.trim(csv::Trim::All);
86 }
87 }
88 let mut reader = builder.from_reader(chunk_data);
89
90 let headers = if has_headers && offset == 0 {
91 Some(reader.headers()?.clone())
92 } else {
93 None
94 };
95
96 let mut records = Vec::new();
97 for result in reader.records() {
98 records.push(result?);
99 }
100
101 Ok((headers, records, actual_bytes))
102 }
103
104 pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
112 let start = usize::try_from(offset)
113 .unwrap_or(usize::MAX)
114 .min(self.mmap.len());
115 let quote = csv_config.map_or(b'"', |config| config.quote_char);
116 let delimiter = csv_config
117 .and_then(|config| config.delimiter)
118 .unwrap_or(b',');
119 crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
120 }
121
122 fn csv_record_boundary(
131 &self,
132 start: usize,
133 chunk_size: usize,
134 quote: u8,
135 delimiter: u8,
136 ) -> usize {
137 let target = start.saturating_add(chunk_size).min(self.mmap.len());
138 let mut index = if start == 0 {
141 self.leading_bom_len()
142 } else {
143 start
144 };
145 let mut in_quotes = false;
146 let mut at_field_start = true;
147 let mut last_boundary = None;
148
149 while index < target {
150 let byte = self.mmap[index];
151 if in_quotes && byte == quote {
152 if self.mmap.get(index + 1) == Some("e) {
155 index += 2;
156 continue;
157 }
158 in_quotes = false;
159 } else if !in_quotes && at_field_start && byte == quote {
160 in_quotes = true;
161 at_field_start = false;
162 } else if !in_quotes && byte == delimiter {
163 at_field_start = true;
164 } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
165 if boundary > target && last_boundary.is_some() {
169 break;
170 }
171 last_boundary = Some(boundary);
172 at_field_start = true;
173 index = boundary;
174 continue;
175 } else if !in_quotes {
176 at_field_start = false;
177 }
178 index += 1;
179 }
180
181 if target == self.mmap.len() {
186 return target;
187 }
188
189 if let Some(boundary) = last_boundary {
190 return boundary;
191 }
192
193 while index < self.mmap.len() {
197 let byte = self.mmap[index];
198 if in_quotes && byte == quote {
199 if self.mmap.get(index + 1) == Some("e) {
200 index += 2;
201 continue;
202 }
203 in_quotes = false;
204 } else if !in_quotes && at_field_start && byte == quote {
205 in_quotes = true;
206 at_field_start = false;
207 } else if !in_quotes && byte == delimiter {
208 at_field_start = true;
209 } else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
210 return boundary;
211 } else if !in_quotes {
212 at_field_start = false;
213 }
214 index += 1;
215 }
216
217 self.mmap.len()
218 }
219
220 fn leading_bom_len(&self) -> usize {
223 if self.mmap.starts_with(b"\xEF\xBB\xBF") {
224 3
225 } else {
226 0
227 }
228 }
229
230 fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
232 match self.mmap[index] {
233 b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
234 b'\r' | b'\n' => Some(index + 1),
235 _ => None,
236 }
237 }
238
239 pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
242 let delimiter = csv_config
243 .and_then(|config| config.delimiter)
244 .unwrap_or(b',');
245 let quote = csv_config.map_or(b'"', |config| config.quote_char);
246 let count = crate::row_count::csv_record_count(&self.mmap[..], delimiter, quote)?;
247 Ok(usize::try_from(count).unwrap_or(usize::MAX))
248 }
249}
250
251#[cfg(test)]
252mod tests {
253 use super::*;
254 use std::io::Write;
255 use tempfile::NamedTempFile;
256
257 #[test]
258 fn test_memory_mapped_reader() -> Result<()> {
259 let mut temp_file = NamedTempFile::new()?;
260 writeln!(temp_file, "name,age,city")?;
261 writeln!(temp_file, "Alice,25,New York")?;
262 writeln!(temp_file, "Bob,30,London")?;
263 writeln!(temp_file, "Charlie,35,Tokyo")?;
264 temp_file.flush()?;
265
266 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
267
268 assert!(reader.file_size() > 0);
269
270 let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
271
272 assert!(headers.is_some());
273 assert_eq!(records.len(), 3);
274
275 let header_record = headers.expect("Headers should be present in test data");
276 assert_eq!(header_record.get(0), Some("name"));
277 assert_eq!(header_record.get(1), Some("age"));
278 assert_eq!(header_record.get(2), Some("city"));
279
280 assert_eq!(records[0].get(0), Some("Alice"));
281 assert_eq!(records[0].get(1), Some("25"));
282
283 Ok(())
284 }
285
286 #[test]
287 fn test_row_estimation() -> Result<()> {
288 let mut temp_file = NamedTempFile::new()?;
289 writeln!(temp_file, "a,b,c")?;
290 for i in 0..100 {
291 writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
292 }
293 temp_file.flush()?;
294
295 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
296 let estimated = reader.estimate_csv_record_count(None)?;
297
298 assert!(estimated > 90 && estimated < 120);
299
300 Ok(())
301 }
302
303 #[test]
304 fn row_estimation_counts_logical_csv_records() -> Result<()> {
305 let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
306 (
307 "quoted multiline",
308 b"id,bio\n1,\"hello\nworld\"\n2,plain",
309 None,
310 ),
311 ("lone CR", b"id,value\r1,x\r2,y", None),
312 (
313 "custom delimiter",
314 b"id;bio\r1;\"hello\nworld\"\r2;plain",
315 Some(CsvParserConfig::default().with_delimiter(b';')),
316 ),
317 ];
318
319 for (case, payload, config) in cases {
320 let mut temp_file = NamedTempFile::new()?;
321 temp_file.write_all(payload)?;
322 temp_file.flush()?;
323
324 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
325 assert_eq!(
326 reader.estimate_csv_record_count(config.as_ref())?,
327 3,
328 "case={case}"
329 );
330 }
331 Ok(())
332 }
333
334 #[test]
335 fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
336 let expected_rows = 1000;
337 let mut temp_file = NamedTempFile::new()?;
338 writeln!(temp_file, "id,name,value")?;
339 for i in 0..expected_rows {
340 let padding = "x".repeat(i % 50);
341 writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
342 }
343 temp_file.flush()?;
344
345 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
346
347 let chunk_size = 512;
348 let mut offset = 0u64;
349 let mut total_records = 0;
350 let mut first = true;
351
352 loop {
353 let (headers, records, bytes) =
354 reader.read_csv_chunk(offset, chunk_size, first, None)?;
355 if records.is_empty() && bytes == 0 {
356 break;
357 }
358 if first && headers.is_some() {
359 first = false;
360 }
361 total_records += records.len();
362 offset += bytes as u64;
363 }
364
365 assert_eq!(
366 total_records, expected_rows,
367 "Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
368 );
369
370 Ok(())
371 }
372
373 #[test]
374 fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
375 let mut temp_file = NamedTempFile::new()?;
376 write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
377 temp_file.flush()?;
378
379 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
380 let mut offset = 0u64;
381 let mut headers = None;
382 let mut records = Vec::new();
383
384 loop {
385 let (chunk_headers, chunk_records, bytes) =
386 reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
387 if bytes == 0 {
388 break;
389 }
390 if headers.is_none() {
391 headers = chunk_headers;
392 }
393 records.extend(chunk_records);
394 offset += bytes as u64;
395 }
396
397 assert_eq!(
398 headers.expect("header").iter().collect::<Vec<_>>(),
399 ["alpha", "beta"]
400 );
401 assert_eq!(records.len(), 2);
402 assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
403 assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
404 Ok(())
405 }
406
407 #[test]
408 fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
409 let payload = b"\n\r\n\ralpha,beta\n1,2";
410 let mut temp_file = NamedTempFile::new()?;
411 temp_file.write_all(payload)?;
412 temp_file.flush()?;
413
414 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
415 let mut offset = 0u64;
416 let mut headers = None;
417 let mut records = Vec::new();
418
419 loop {
420 let (chunk_headers, chunk_records, bytes) =
421 reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
422 if bytes == 0 {
423 break;
424 }
425 if headers.is_none() {
426 headers = chunk_headers;
427 }
428 records.extend(chunk_records);
429 offset += bytes as u64;
430 }
431
432 assert_eq!(
433 headers.expect("header").iter().collect::<Vec<_>>(),
434 ["alpha", "beta"]
435 );
436 assert_eq!(records.len(), 1);
437 assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
438 Ok(())
439 }
440
441 #[test]
442 fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
443 let cases: [(&str, &[u8], &str); 2] = [
444 (
445 "crlf",
446 b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
447 "hello \"quoted\"\nworld",
448 ),
449 (
450 "lone-cr",
451 b"id,bio\r1,\"hello\nworld\"\r2,plain",
452 "hello\nworld",
453 ),
454 ];
455
456 for (terminator, payload, expected_bio) in cases {
457 let mut temp_file = NamedTempFile::new()?;
458 temp_file.write_all(payload)?;
459 temp_file.flush()?;
460
461 let reader = MemoryMappedCsvReader::new(temp_file.path())?;
462 for chunk_size in 1..=payload.len() + 1 {
463 let mut offset = 0u64;
464 let mut headers = None;
465 let mut records = Vec::new();
466
467 loop {
468 let (chunk_headers, chunk_records, bytes) =
469 reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
470 if bytes == 0 {
471 break;
472 }
473 if headers.is_none() {
474 headers = chunk_headers;
475 }
476 records.extend(chunk_records);
477 offset += bytes as u64;
478 }
479
480 assert_eq!(
481 headers.expect("header").iter().collect::<Vec<_>>(),
482 ["id", "bio"],
483 "terminator={terminator}, chunk_size={chunk_size}"
484 );
485 assert_eq!(
486 records.len(),
487 2,
488 "terminator={terminator}, chunk_size={chunk_size}"
489 );
490 assert_eq!(
491 records[0].get(0),
492 Some("1"),
493 "terminator={terminator}, chunk_size={chunk_size}"
494 );
495 assert_eq!(
496 records[0].get(1),
497 Some(expected_bio),
498 "terminator={terminator}, chunk_size={chunk_size}"
499 );
500 assert_eq!(
501 records[1].iter().collect::<Vec<_>>(),
502 ["2", "plain"],
503 "terminator={terminator}, chunk_size={chunk_size}"
504 );
505 }
506 }
507 Ok(())
508 }
509}