use arrow_array::{Array, RecordBatch, StringArray};
use crate::column_types::models::ColumnType;
use crate::file_store::models::{OpenFolder, StoredFile};
use crate::files_source::_helpers::cells::cell_reader::{CellReader, cells_batch};
use crate::files_source::_helpers::text::bytes_reader::{BytesReader, open_bytes};
use crate::files_source::_helpers::text::checks::{ColumnChecks, misfit, value_fits};
use crate::files_source::_helpers::text::csv_reader::CsvReader;
use crate::files_source::_helpers::text::inference::csv_value_type;
use crate::files_source::_helpers::text::json_reader::{JsonReader, records_batch};
use crate::files_source::_helpers::text::whole_records::{csv_lines_batch, json_records_batch};
use crate::files_source::models::{Codec, FilesRead, SourceFormat};
enum Records {
Json {
reader: JsonReader,
columns: Vec<(String, ColumnType)>,
batch_rows: usize,
},
Csv(Box<CsvReader>),
JsonWhole {
reader: JsonReader,
column: String,
batch_rows: usize,
},
CsvWhole {
reader: Box<CsvReader>,
column: String,
},
Cells {
reader: CellReader,
columns: Vec<(String, ColumnType)>,
batch_rows: usize,
},
}
pub(crate) struct TextReader {
records: Records,
checks: ColumnChecks,
}
impl TextReader {
pub(crate) async fn open(
folder: &OpenFolder,
file: &StoredFile,
codec: Codec,
request: &FilesRead,
) -> Result<Self, String> {
let checks = ColumnChecks::new(request)?;
let bytes = open_bytes(folder, file, codec).await?;
let records = match (request.encoding.format, &request.encoding.record) {
(SourceFormat::Xlsx | SourceFormat::Xml, _) => Records::Cells {
reader: CellReader::open(bytes, &request.encoding).await?,
columns: request
.columns
.iter()
.map(|column| (column.name.clone(), column.column_type.clone()))
.collect(),
batch_rows: request.batch_rows,
},
(SourceFormat::Csv, Some(column)) => Records::CsvWhole {
reader: Box::new(
CsvReader::open_lines(bytes, &request.encoding.csv, request.batch_rows).await?,
),
column: column.clone(),
},
(format, Some(column)) => Records::JsonWhole {
reader: JsonReader::new(bytes, format == SourceFormat::Ndjson),
column: column.clone(),
batch_rows: request.batch_rows,
},
(format, None) => typed_records(bytes, format, (request, &checks)).await?,
};
Ok(Self { records, checks })
}
pub(crate) async fn next_batch(&mut self) -> Result<Option<(RecordBatch, Vec<u64>)>, String> {
match &mut self.records {
Records::Json {
reader,
columns,
batch_rows,
} => {
let records = reader.next_records(*batch_rows).await?;
match records.is_empty() {
true => Ok(None),
false => records_batch(&records, columns, (&self.checks, reader.flattens()))
.map(Some),
}
}
Records::Cells {
reader,
columns,
batch_rows,
} => {
let records = reader.next_records(*batch_rows);
match records.is_empty() {
true => Ok(None),
false => cells_batch(&records, columns, &self.checks).map(Some),
}
}
Records::JsonWhole {
reader,
column,
batch_rows,
} => {
let records = reader.next_records(*batch_rows).await?;
match records.is_empty() {
true => Ok(None),
false => json_records_batch(&records, column).map(Some),
}
}
Records::CsvWhole { reader, column } => {
let Some((batch, first)) = reader.next_batch().await? else {
return Ok(None);
};
let lines: Vec<u64> = (first..first + batch.num_rows() as u64).collect();
Ok(Some((csv_lines_batch(&batch, column)?, lines)))
}
Records::Csv(reader) => {
let Some((batch, first)) = reader.next_batch().await? else {
return Ok(None);
};
let lines: Vec<u64> = (first..first + batch.num_rows() as u64).collect();
check_csv_values(&batch, &lines, &self.checks)?;
Ok(Some((batch, lines)))
}
}
}
}
async fn typed_records(
bytes: BytesReader,
format: SourceFormat,
(request, checks): (&FilesRead, &ColumnChecks),
) -> Result<Records, String> {
Ok(match format {
SourceFormat::Csv => {
let reader = CsvReader::open(bytes, &request.encoding.csv, request.batch_rows).await?;
for name in reader.names() {
checks.check_name("header", name, 1 + request.encoding.csv.skip_rows)?;
}
Records::Csv(Box::new(reader))
}
format => Records::Json {
reader: JsonReader::new(bytes, format == SourceFormat::Ndjson)
.flattening(request.encoding.flatten_objects),
columns: request
.columns
.iter()
.map(|column| (column.name.clone(), column.column_type.clone()))
.collect(),
batch_rows: request.batch_rows,
},
})
}
fn check_csv_values(
batch: &RecordBatch,
lines: &[u64],
checks: &ColumnChecks,
) -> Result<(), String> {
for (field, column) in batch.schema().fields().iter().zip(batch.columns()) {
let Some(column_type) = checks.inferred_type(field.name()) else {
continue;
};
let Some(values) = column.as_any().downcast_ref::<StringArray>() else {
continue;
};
let misfit_row = values
.iter()
.position(|value| !value_fits(column_type, value.map(csv_value_type)));
if let Some(row) = misfit_row {
return Err(misfit(
lines[row],
field.name(),
&format!("{:?}", values.value(row)),
column_type,
));
}
}
Ok(())
}