use std::fs::File;
use std::io::{BufRead, BufReader, Cursor};
use std::path::Path;
use crate::error::{ExcelError, Result};
use crate::header::{HeaderMap, resolve_headers};
use crate::import::report::{ImportReport, RowError, SheetHeader};
use crate::model::{ExcelRow, RowData};
use crate::read::ReadOptions;
use crate::read::styles::Styles;
use crate::value::{Cell, CellValue};
pub struct CsvReader<R: BufRead> {
reader: R,
line_no: u32,
eof: bool,
first: bool,
}
impl CsvReader<BufReader<File>> {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let file = File::open(path)?;
Ok(Self::new(BufReader::with_capacity(128 * 1024, file)))
}
}
impl CsvReader<Cursor<Vec<u8>>> {
pub fn from_bytes(bytes: Vec<u8>) -> Self {
Self::new(Cursor::new(bytes))
}
}
impl<R: BufRead> CsvReader<R> {
pub fn new(reader: R) -> Self {
Self {
reader,
line_no: 0,
eof: false,
first: true,
}
}
pub fn next_record(&mut self) -> Result<Option<(Vec<String>, u32)>> {
if self.eof {
return Ok(None);
}
let start_line = self.line_no + 1;
let mut raw: Vec<u8> = Vec::new();
loop {
let mut line: Vec<u8> = Vec::new();
let n = self.reader.read_until(b'\n', &mut line)?;
if n == 0 {
self.eof = true;
if raw.is_empty() {
return Ok(None);
}
break;
}
self.line_no += 1;
if self.first {
self.first = false;
if line.starts_with(&[0xEF, 0xBB, 0xBF]) {
line.drain(0..3);
}
}
raw.extend_from_slice(&line);
if !has_unclosed_quote(&raw) {
break;
}
}
Ok(Some((split_fields(&raw), start_line)))
}
pub fn read_into<T, F>(&mut self, options: &ReadOptions, mut on_row: F) -> Result<ImportReport>
where
T: ExcelRow,
F: FnMut(&RowData<'_>, T) -> std::result::Result<(), Vec<String>>,
{
let started = std::time::Instant::now();
let styles = Styles::empty();
let columns = T::columns();
let span = options.header.row_span.max(1) as usize;
let header_start = options.header.row_index as usize;
let mut idx = 0usize;
let mut header_rows: Vec<Vec<String>> = Vec::new();
while idx < header_start + span {
match self.next_record()? {
Some((rec, _)) => {
if idx >= header_start {
header_rows.push(rec);
}
idx += 1;
}
None => break,
}
}
let mut report = ImportReport {
sheets: vec!["CSV".to_string()],
..Default::default()
};
if header_rows.is_empty() {
report.headers.push(SheetHeader {
sheet: "CSV".to_string(),
analysis: HeaderMap::dynamic(Vec::new()).analysis(),
});
return Ok(report);
}
let ncols = header_rows.iter().map(|r| r.len()).max().unwrap_or(0);
let mut file_headers: Vec<String> = Vec::with_capacity(ncols);
for c in 0..ncols {
let parts: Vec<String> = header_rows
.iter()
.filter_map(|r| r.get(c))
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect();
file_headers.push(parts.join(" "));
}
for cc in &options.computed {
if !file_headers.iter().any(|h| h.as_str() == cc.header()) {
file_headers.push(cc.header().to_string());
}
}
let map = resolve_headers(&file_headers, &columns, &options.header);
let missing = map.missing_required();
if !missing.is_empty() {
return Err(ExcelError::MissingColumns(missing));
}
report.headers.push(SheetHeader {
sheet: "CSV".to_string(),
analysis: map.analysis(),
});
let defaults: Vec<(u16, String)> = columns
.iter()
.filter(|d| d.is_importable())
.filter_map(|d| d.default.as_ref().map(|v| (d, v)))
.filter_map(|(d, v)| map.index_of(&d.header).map(|col| (col, v.clone())))
.collect();
while let Some((rec, line)) = self.next_record()? {
report.total_rows += 1;
let mut cells: Vec<Cell> = Vec::new();
for (c, text) in rec.iter().enumerate() {
if text.is_empty() {
continue;
}
let value = if options.trim_text {
CellValue::Text(text.trim().to_string())
} else {
CellValue::Text(text.clone())
};
cells.push(Cell {
row: line,
col: c as u16,
value,
style: 0,
});
}
for (col, text) in &defaults {
let empty = cells
.iter()
.find(|c| c.col == *col)
.map(|c| c.value.is_empty())
.unwrap_or(true);
if empty {
cells.push(Cell {
row: line,
col: *col,
value: CellValue::Text(text.clone()),
style: 0,
});
}
}
if !options.computed.is_empty() {
let mut values: Vec<(u16, CellValue)> = Vec::new();
{
let row = RowData {
row_index: line,
sheet: "CSV",
header: &map,
cells: &cells,
images: &[],
merges: &[],
styles: &styles,
hyperlinks: &[],
comments: &[],
validations: &[],
conditional_formats: &[],
};
for cc in &options.computed {
let Some(col) =
row.header.file_headers.iter().position(|h| h.as_str() == cc.header())
else {
continue;
};
values.push((col as u16, cc.eval(&row)));
}
}
for (col, value) in values {
match cells.iter_mut().find(|c| c.col == col) {
Some(c) => c.value = value,
None => cells.push(Cell {
row: line,
col,
value,
style: 0,
}),
}
}
}
cells.sort_by_key(|c| c.col);
let row = RowData {
row_index: line,
sheet: "CSV",
header: &map,
cells: &cells,
images: &[],
merges: &[],
styles: &styles,
hyperlinks: &[],
comments: &[],
validations: &[],
conditional_formats: &[],
};
if let Some(filter) = &options.row_filter {
if !filter.keep(&row) {
report.skipped_rows += 1;
continue;
}
}
if cells.iter().all(|c| c.value.is_empty()) && options.skip_empty_rows {
report.skipped_rows += 1;
continue;
}
match T::from_row(&row) {
Ok(data) => {
let mut messages = data.validate().err().unwrap_or_default();
if messages.is_empty() {
if let Err(errs) = on_row(&row, data) {
messages = errs;
}
}
if messages.is_empty() {
report.success_rows += 1;
} else {
push_error(&mut report, line, messages);
}
}
Err(column_errors) => {
let messages: Vec<String> = column_errors
.iter()
.map(|e| {
if e.file_header.is_empty() {
format!("{}:{}", e.column, e.message)
} else {
format!("{}({}):{}", e.column, e.file_header, e.message)
}
})
.collect();
push_error(&mut report, line, messages);
}
}
}
report.elapsed_ms = started.elapsed().as_millis() as u64;
Ok(report)
}
pub fn parse_all<T: ExcelRow>(&mut self, options: &ReadOptions) -> Result<Vec<T>> {
let mut out = Vec::new();
let mut first_error: Option<ExcelError> = None;
let report = self.read_into::<T, _>(options, |_row, data| {
out.push(data);
Ok(())
})?;
if !report.errors.is_empty() {
let e = &report.errors[0];
first_error = Some(ExcelError::InvalidRow {
row: e.row,
messages: e.messages.clone(),
});
}
if let Some(e) = first_error {
return Err(e);
}
Ok(out)
}
}
fn push_error(report: &mut ImportReport, line: u32, messages: Vec<String>) {
report.error_rows += 1;
if report.errors.len() < 1000 {
report.errors.push(RowError {
row: line,
sheet: "CSV".to_string(),
messages,
});
} else {
report.errors_truncated = true;
}
}
fn has_unclosed_quote(data: &[u8]) -> bool {
let mut in_quotes = false;
let mut field_started = false;
let mut i = 0;
while i < data.len() {
let b = data[i];
if in_quotes {
if b == b'"' {
if i + 1 < data.len() && data[i + 1] == b'"' {
i += 2;
continue;
}
in_quotes = false;
}
i += 1;
} else {
match b {
b'"' if !field_started => {
in_quotes = true;
field_started = true;
}
b',' => field_started = false,
_ => field_started = true,
}
i += 1;
}
}
in_quotes
}
fn split_fields(raw: &[u8]) -> Vec<String> {
let mut end = raw.len();
if end > 0 && raw[end - 1] == b'\n' {
end -= 1;
}
if end > 0 && raw[end - 1] == b'\r' {
end -= 1;
}
let data = &raw[..end];
let mut fields: Vec<String> = Vec::new();
let mut cur: Vec<u8> = Vec::new();
let mut in_quotes = false;
let mut field_started = false;
let mut i = 0;
while i < data.len() {
let b = data[i];
if in_quotes {
if b == b'"' {
if i + 1 < data.len() && data[i + 1] == b'"' {
cur.push(b'"');
i += 2;
continue;
}
in_quotes = false;
i += 1;
continue;
}
cur.push(b);
i += 1;
continue;
}
match b {
b'"' if !field_started => {
in_quotes = true;
field_started = true;
i += 1;
}
b',' => {
fields.push(String::from_utf8_lossy(&cur).into_owned());
cur.clear();
field_started = false;
i += 1;
}
_ => {
cur.push(b);
field_started = true;
i += 1;
}
}
}
fields.push(String::from_utf8_lossy(&cur).into_owned());
fields
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_split_fields() {
assert_eq!(split_fields(b"a,b,c\n"), vec!["a", "b", "c"]);
assert_eq!(split_fields(b"a,\"b,c\",d"), vec!["a", "b,c", "d"]);
assert_eq!(split_fields(b"\"he said \"\"hi\"\"\",x"), vec!["he said \"hi\"", "x"]);
assert_eq!(split_fields(b"a,b\r\n"), vec!["a", "b"]);
assert_eq!(split_fields(b"only"), vec!["only"]);
}
#[test]
fn test_unclosed_quote() {
assert!(!has_unclosed_quote(b"a,\"b\",c"));
assert!(has_unclosed_quote(b"a,\"b"));
assert!(!has_unclosed_quote(b"a,\"b\"\"c\""));
}
}