use anyhow::Result;
use memmap2::Mmap;
use std::fs::File;
use std::path::Path;
use crate::CsvParserConfig;
pub struct MemoryMappedCsvReader {
mmap: Mmap,
file_size: u64,
}
impl MemoryMappedCsvReader {
pub fn new(path: &Path) -> Result<Self> {
let file = File::open(path)?;
let file_size = file.metadata()?.len();
#[allow(unsafe_code)]
let mmap = unsafe { Mmap::map(&file)? };
Ok(Self { mmap, file_size })
}
pub fn file_size(&self) -> u64 {
self.file_size
}
pub fn read_csv_chunk(
&self,
offset: u64,
chunk_size: usize,
has_headers: bool,
csv_config: Option<&CsvParserConfig>,
) -> Result<(Option<csv::StringRecord>, Vec<csv::StringRecord>, usize)> {
let (headers, records, actual_bytes, ragged) =
self.read_csv_chunk_until_ragged(offset, chunk_size, has_headers, csv_config)?;
match ragged {
Some(error) => Err(error),
None => Ok((headers, records, actual_bytes)),
}
}
#[allow(clippy::type_complexity)]
pub fn read_csv_chunk_until_ragged(
&self,
offset: u64,
chunk_size: usize,
has_headers: bool,
csv_config: Option<&CsvParserConfig>,
) -> Result<(
Option<csv::StringRecord>,
Vec<csv::StringRecord>,
usize,
Option<anyhow::Error>,
)> {
let start = offset as usize;
if start >= self.mmap.len() {
return Ok((None, Vec::new(), 0, None));
}
let quote = csv_config.map_or(b'"', |config| config.quote_char);
let delimiter = csv_config
.and_then(|config| config.delimiter)
.unwrap_or(b',');
let mut end = self.csv_record_boundary(start, chunk_size.max(1), quote, delimiter);
if has_headers && offset == 0 {
let bom = self.leading_bom_len();
let header_start = bom
+ self.mmap[bom..]
.iter()
.take_while(|&&byte| matches!(byte, b'\r' | b'\n'))
.count();
if end <= header_start && header_start < self.mmap.len() {
end = self.csv_record_boundary(header_start, 1, quote, delimiter);
}
}
let chunk_data = &self.mmap[start..end];
let actual_bytes = chunk_data.len();
let mut builder = csv::ReaderBuilder::new();
builder.has_headers(has_headers && offset == 0);
if let Some(config) = csv_config {
if let Some(delim) = config.delimiter {
builder.delimiter(delim);
}
builder.flexible(config.flexible);
builder.quote(config.quote_char);
if config.trim_whitespace {
builder.trim(csv::Trim::All);
}
}
let mut reader = builder.from_reader(chunk_data);
let headers = if has_headers && offset == 0 {
Some(reader.headers()?.clone())
} else {
None
};
let mut records = Vec::new();
for result in reader.records() {
match result {
Ok(record) => records.push(record),
Err(error) if matches!(error.kind(), csv::ErrorKind::UnequalLengths { .. }) => {
return Ok((headers, records, actual_bytes, Some(error.into())));
}
Err(error) => return Err(error.into()),
}
}
Ok((headers, records, actual_bytes, None))
}
pub fn ends_inside_quotes(&self, offset: u64, csv_config: Option<&CsvParserConfig>) -> bool {
let start = usize::try_from(offset)
.unwrap_or(usize::MAX)
.min(self.mmap.len());
let quote = csv_config.map_or(b'"', |config| config.quote_char);
let delimiter = csv_config
.and_then(|config| config.delimiter)
.unwrap_or(b',');
crate::quote_state::ends_inside_quotes(&self.mmap[start..], delimiter, quote, start == 0)
}
fn csv_record_boundary(
&self,
start: usize,
chunk_size: usize,
quote: u8,
delimiter: u8,
) -> usize {
let target = start.saturating_add(chunk_size).min(self.mmap.len());
let mut index = if start == 0 {
self.leading_bom_len()
} else {
start
};
let mut in_quotes = false;
let mut at_field_start = true;
let mut last_boundary = None;
while index < target {
let byte = self.mmap[index];
if in_quotes && byte == quote {
if self.mmap.get(index + 1) == Some("e) {
index += 2;
continue;
}
in_quotes = false;
} else if !in_quotes && at_field_start && byte == quote {
in_quotes = true;
at_field_start = false;
} else if !in_quotes && byte == delimiter {
at_field_start = true;
} else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
if boundary > target && last_boundary.is_some() {
break;
}
last_boundary = Some(boundary);
at_field_start = true;
index = boundary;
continue;
} else if !in_quotes {
at_field_start = false;
}
index += 1;
}
if target == self.mmap.len() {
return target;
}
if let Some(boundary) = last_boundary {
return boundary;
}
while index < self.mmap.len() {
let byte = self.mmap[index];
if in_quotes && byte == quote {
if self.mmap.get(index + 1) == Some("e) {
index += 2;
continue;
}
in_quotes = false;
} else if !in_quotes && at_field_start && byte == quote {
in_quotes = true;
at_field_start = false;
} else if !in_quotes && byte == delimiter {
at_field_start = true;
} else if !in_quotes && let Some(boundary) = self.csv_record_terminator_end(index) {
return boundary;
} else if !in_quotes {
at_field_start = false;
}
index += 1;
}
self.mmap.len()
}
fn leading_bom_len(&self) -> usize {
if self.mmap.starts_with(b"\xEF\xBB\xBF") {
3
} else {
0
}
}
fn csv_record_terminator_end(&self, index: usize) -> Option<usize> {
match self.mmap[index] {
b'\r' if self.mmap.get(index + 1) == Some(&b'\n') => Some(index + 2),
b'\r' | b'\n' => Some(index + 1),
_ => None,
}
}
pub fn estimate_csv_record_count(&self, csv_config: Option<&CsvParserConfig>) -> Result<usize> {
let delimiter = csv_config
.and_then(|config| config.delimiter)
.unwrap_or(b',');
let quote = csv_config.map_or(b'"', |config| config.quote_char);
let count = crate::row_count::csv_record_count(&self.mmap[..], delimiter, quote)?;
Ok(usize::try_from(count).unwrap_or(usize::MAX))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
use tempfile::NamedTempFile;
#[test]
fn test_memory_mapped_reader() -> Result<()> {
let mut temp_file = NamedTempFile::new()?;
writeln!(temp_file, "name,age,city")?;
writeln!(temp_file, "Alice,25,New York")?;
writeln!(temp_file, "Bob,30,London")?;
writeln!(temp_file, "Charlie,35,Tokyo")?;
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
assert!(reader.file_size() > 0);
let (headers, records, _bytes) = reader.read_csv_chunk(0, 1024, true, None)?;
assert!(headers.is_some());
assert_eq!(records.len(), 3);
let header_record = headers.expect("Headers should be present in test data");
assert_eq!(header_record.get(0), Some("name"));
assert_eq!(header_record.get(1), Some("age"));
assert_eq!(header_record.get(2), Some("city"));
assert_eq!(records[0].get(0), Some("Alice"));
assert_eq!(records[0].get(1), Some("25"));
Ok(())
}
#[test]
fn test_row_estimation() -> Result<()> {
let mut temp_file = NamedTempFile::new()?;
writeln!(temp_file, "a,b,c")?;
for i in 0..100 {
writeln!(temp_file, "{},{},{}", i, i * 2, i * 3)?;
}
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
let estimated = reader.estimate_csv_record_count(None)?;
assert!(estimated > 90 && estimated < 120);
Ok(())
}
#[test]
fn row_estimation_counts_logical_csv_records() -> Result<()> {
let cases: [(&str, &[u8], Option<CsvParserConfig>); 3] = [
(
"quoted multiline",
b"id,bio\n1,\"hello\nworld\"\n2,plain",
None,
),
("lone CR", b"id,value\r1,x\r2,y", None),
(
"custom delimiter",
b"id;bio\r1;\"hello\nworld\"\r2;plain",
Some(CsvParserConfig::default().with_delimiter(b';')),
),
];
for (case, payload, config) in cases {
let mut temp_file = NamedTempFile::new()?;
temp_file.write_all(payload)?;
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
assert_eq!(
reader.estimate_csv_record_count(config.as_ref())?,
3,
"case={case}"
);
}
Ok(())
}
#[test]
fn test_no_row_loss_at_chunk_boundaries() -> Result<()> {
let expected_rows = 1000;
let mut temp_file = NamedTempFile::new()?;
writeln!(temp_file, "id,name,value")?;
for i in 0..expected_rows {
let padding = "x".repeat(i % 50);
writeln!(temp_file, "{},name_{}{},{}", i, i, padding, i * 10)?;
}
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
let chunk_size = 512;
let mut offset = 0u64;
let mut total_records = 0;
let mut first = true;
loop {
let (headers, records, bytes) =
reader.read_csv_chunk(offset, chunk_size, first, None)?;
if records.is_empty() && bytes == 0 {
break;
}
if first && headers.is_some() {
first = false;
}
total_records += records.len();
offset += bytes as u64;
}
assert_eq!(
total_records, expected_rows,
"Expected {expected_rows} rows but got {total_records} — rows lost at chunk boundaries"
);
Ok(())
}
#[test]
fn csv_chunk_smaller_than_header_preserves_schema_and_rows() -> Result<()> {
let mut temp_file = NamedTempFile::new()?;
write!(temp_file, "alpha,beta\n1,2\n3,4\n")?;
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
let mut offset = 0u64;
let mut headers = None;
let mut records = Vec::new();
loop {
let (chunk_headers, chunk_records, bytes) =
reader.read_csv_chunk(offset, 5, headers.is_none(), None)?;
if bytes == 0 {
break;
}
if headers.is_none() {
headers = chunk_headers;
}
records.extend(chunk_records);
offset += bytes as u64;
}
assert_eq!(
headers.expect("header").iter().collect::<Vec<_>>(),
["alpha", "beta"]
);
assert_eq!(records.len(), 2);
assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
assert_eq!(records[1].iter().collect::<Vec<_>>(), ["3", "4"]);
Ok(())
}
#[test]
fn csv_chunk_skips_leading_blank_lines_before_header() -> Result<()> {
let payload = b"\n\r\n\ralpha,beta\n1,2";
let mut temp_file = NamedTempFile::new()?;
temp_file.write_all(payload)?;
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
let mut offset = 0u64;
let mut headers = None;
let mut records = Vec::new();
loop {
let (chunk_headers, chunk_records, bytes) =
reader.read_csv_chunk(offset, 1, headers.is_none(), None)?;
if bytes == 0 {
break;
}
if headers.is_none() {
headers = chunk_headers;
}
records.extend(chunk_records);
offset += bytes as u64;
}
assert_eq!(
headers.expect("header").iter().collect::<Vec<_>>(),
["alpha", "beta"]
);
assert_eq!(records.len(), 1);
assert_eq!(records[0].iter().collect::<Vec<_>>(), ["1", "2"]);
Ok(())
}
#[test]
fn csv_chunk_preserves_multiline_records_for_every_terminator() -> Result<()> {
let cases: [(&str, &[u8], &str); 2] = [
(
"crlf",
b"id,bio\r\n1,\"hello \"\"quoted\"\"\nworld\"\r\n2,plain",
"hello \"quoted\"\nworld",
),
(
"lone-cr",
b"id,bio\r1,\"hello\nworld\"\r2,plain",
"hello\nworld",
),
];
for (terminator, payload, expected_bio) in cases {
let mut temp_file = NamedTempFile::new()?;
temp_file.write_all(payload)?;
temp_file.flush()?;
let reader = MemoryMappedCsvReader::new(temp_file.path())?;
for chunk_size in 1..=payload.len() + 1 {
let mut offset = 0u64;
let mut headers = None;
let mut records = Vec::new();
loop {
let (chunk_headers, chunk_records, bytes) =
reader.read_csv_chunk(offset, chunk_size, headers.is_none(), None)?;
if bytes == 0 {
break;
}
if headers.is_none() {
headers = chunk_headers;
}
records.extend(chunk_records);
offset += bytes as u64;
}
assert_eq!(
headers.expect("header").iter().collect::<Vec<_>>(),
["id", "bio"],
"terminator={terminator}, chunk_size={chunk_size}"
);
assert_eq!(
records.len(),
2,
"terminator={terminator}, chunk_size={chunk_size}"
);
assert_eq!(
records[0].get(0),
Some("1"),
"terminator={terminator}, chunk_size={chunk_size}"
);
assert_eq!(
records[0].get(1),
Some(expected_bio),
"terminator={terminator}, chunk_size={chunk_size}"
);
assert_eq!(
records[1].iter().collect::<Vec<_>>(),
["2", "plain"],
"terminator={terminator}, chunk_size={chunk_size}"
);
}
}
Ok(())
}
}