use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_csv::reader::{Decoder, Format, ReaderBuilder};
use arrow_schema::{DataType, Field, Schema};
use regex::Regex;
use crate::files_source::_helpers::text::bytes_reader::BytesReader;
use crate::files_source::_helpers::text::csv_records::{CsvQuoting, FirstRecord, QuoteTracker};
use crate::files_source::constants::{ENCODING_HINT, MAX_RECORD_BYTES, QUOTE_OFF_BYTE};
use crate::files_source::main::text_charset::text_charset;
use crate::files_source::models::CsvOptions;
use crate::load_naming::main::numbered_header::numbered_header;
pub(crate) struct CsvReader {
bytes: BytesReader,
decoder: Option<Decoder>,
tracker: QuoteTracker,
pending: Vec<u8>,
offset: usize,
at_end: bool,
end_sent: bool,
names: Vec<String>,
repeats: Vec<Option<String>>,
next_line: u64,
skipped: u64,
}
impl CsvReader {
pub(crate) async fn open(
bytes: BytesReader,
options: &CsvOptions,
batch_rows: usize,
) -> Result<Self, String> {
Self::open_as(bytes, (options, batch_rows), true).await
}
pub(crate) async fn open_lines(
bytes: BytesReader,
options: &CsvOptions,
batch_rows: usize,
) -> Result<Self, String> {
Self::open_as(bytes, (options, batch_rows), false).await
}
pub(crate) async fn open_as(
bytes: BytesReader,
(options, batch_rows): (&CsvOptions, usize),
named: bool,
) -> Result<Self, String> {
let quoting = quoting(options)?;
let skipped = options.skip_rows;
let mut bytes = bytes.decoding(
text_charset(&options.encoding)?,
(skipped, options.encoding_check),
);
let mut first_record = FirstRecord::new(quoting, MAX_RECORD_BYTES);
let mut pending: Vec<u8> = Vec::new();
let (first, at_end) = loop {
let chunk = bytes.next_chunk().await?;
pending.extend_from_slice(&chunk);
let fed = first_record
.feed(&chunk)
.map_err(|error| shifted(&error, skipped))?;
if let Some(first) = fed {
break (first, chunk.is_empty());
}
};
let given = options.column_names.as_deref().filter(|_| named);
let (names, repeats): (Vec<String>, Vec<Option<String>>) =
column_names(first, options.header && named, given)
.map_err(|error| shifted(&error, skipped))?
.into_iter()
.unzip();
let decoder = match names.is_empty() {
true => None,
false => Some(decoder(&names, (options, quoting), batch_rows)?),
};
Ok(Self {
bytes,
decoder,
tracker: QuoteTracker::new(quoting, MAX_RECORD_BYTES),
pending,
offset: 0,
at_end,
end_sent: false,
names,
repeats,
next_line: 1 + skipped + u64::from(options.header),
skipped,
})
}
pub(crate) fn names(&self) -> &[String] {
&self.names
}
pub(crate) fn repeats(&self) -> &[Option<String>] {
&self.repeats
}
pub(crate) async fn next_batch(&mut self) -> Result<Option<(RecordBatch, u64)>, String> {
let skipped = self.skipped;
let parsed = |error: String| shifted(&error, skipped);
let Some(decoder) = self.decoder.as_mut() else {
return Ok(None);
};
loop {
if self.offset == self.pending.len() && !self.at_end {
self.pending = self.bytes.next_chunk().await?;
self.offset = 0;
self.at_end = self.pending.is_empty();
}
let input = &self.pending[self.offset..];
if !input.is_empty() {
let decoded = decoder.decode(input).map_err(csv_error).map_err(parsed)?;
self.tracker.push(&input[..decoded]).map_err(parsed)?;
self.offset += decoded;
} else if self.at_end && !self.end_sent && decoder.capacity() > 0 {
decoder.decode(&[]).map_err(csv_error).map_err(parsed)?;
self.end_sent = true;
}
let done = self.end_sent && self.offset == self.pending.len();
if done {
self.tracker.finish().map_err(parsed)?;
}
if decoder.capacity() == 0 || done {
match decoder.flush().map_err(csv_error).map_err(parsed)? {
Some(batch) => {
let line = self.next_line;
self.next_line += batch.num_rows() as u64;
return Ok(Some((batch, line)));
}
None if done => return Ok(None),
None => {}
}
}
}
}
}
fn decoder(
names: &[String],
(options, quoting): (&CsvOptions, CsvQuoting),
batch_rows: usize,
) -> Result<Decoder, String> {
let fields: Vec<Field> = names
.iter()
.map(|name| Field::new(name, DataType::Utf8, true))
.collect();
let format = Format::default()
.with_header(options.header)
.with_delimiter(quoting.delimiter)
.with_quote(quoting.quote.unwrap_or(QUOTE_OFF_BYTE))
.with_null_regex(null_regex(&options.null_values)?);
let format = match quoting.escape {
Some(escape) => format.with_escape(escape),
None => format,
};
Ok(ReaderBuilder::new(Arc::new(Schema::new(fields)))
.with_format(format)
.with_batch_size(batch_rows)
.build_decoder())
}
fn null_regex(null_values: &[String]) -> Result<Regex, String> {
let pattern = match null_values.is_empty() {
true => r"[^\s\S]".to_owned(),
false => {
let alternatives: Vec<String> = null_values
.iter()
.map(|value| regex::escape(value))
.collect();
format!(r"\A(?:{})\z", alternatives.join("|"))
}
};
Regex::new(&pattern).map_err(|error| format!("csv null_values: {error}"))
}
fn quoting(options: &CsvOptions) -> Result<CsvQuoting, String> {
Ok(CsvQuoting {
delimiter: ascii(options.delimiter, "delimiter")?,
quote: options
.quote
.character()
.map(|quote| ascii(quote, "quote"))
.transpose()?,
escape: options
.escape
.map(|escape| ascii(escape, "escape"))
.transpose()?,
})
}
fn ascii(character: char, setting: &str) -> Result<u8, String> {
match u8::try_from(character) {
Ok(byte) if byte.is_ascii() => Ok(byte),
_ => Err(format!(
"csv {setting} must be one ASCII character, not {character:?}"
)),
}
}
fn column_names(
first: Vec<String>,
header: bool,
given: Option<&[String]>,
) -> Result<Vec<(String, Option<String>)>, String> {
let plain = |names: Vec<String>| names.into_iter().map(|name| (name, None)).collect();
match (header, given) {
(false, Some(given)) if first.len() == given.len() => return Ok(plain(given.to_vec())),
(false, Some(given)) if !first.is_empty() => {
return Err(format!(
"line 1: the file has {} columns, but column_names names {}; name each column",
first.len(),
given.len()
));
}
(false, _) => {
return Ok(plain(
(1..=first.len())
.map(|position| format!("column_{position}"))
.collect(),
));
}
(true, _) => {}
}
for (position, name) in first.iter().enumerate() {
if name.is_empty() {
return Err(format!("line 1: header field {} is empty", position + 1));
}
}
Ok(numbered_header(&first))
}
fn csv_error(error: arrow_schema::ArrowError) -> String {
let text = error.to_string();
let text = text.strip_prefix("Csv error: ").unwrap_or(&text);
let invalid = text
.strip_prefix("Encountered invalid UTF-8 data for line ")
.and_then(|rest| rest.split_once(" and field "));
match invalid {
Some((line, field)) => {
format!("line {line}, field {field}: the bytes are not valid UTF-8; {ENCODING_HINT}")
}
None => text.to_owned(),
}
}
fn shifted(error: &str, skipped: u64) -> String {
if skipped == 0 {
return error.to_owned();
}
let Ok(lines) = Regex::new(r"\bline (\d+)") else {
return error.to_owned();
};
lines
.replace_all(error, |found: ®ex::Captures<'_>| {
let line = found[1].parse::<u64>().map_or(0, |line| line + skipped);
format!("line {line}")
})
.into_owned()
}