use std::collections::VecDeque;
use std::collections::{BTreeSet, HashMap};
use std::sync::Arc;
use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StringArray};
use arrow_schema::{DataType, Field, Schema};
use serde_json::Value;
use crate::column_types::models::ColumnType;
use crate::files_source::_helpers::text::bytes_reader::BytesReader;
use crate::files_source::_helpers::text::checks::{ColumnChecks, misfit, value_fits};
use crate::files_source::_helpers::text::inference::json_value_type;
use crate::files_source::_helpers::text::json_records::{
JsonRecord, JsonSplitter, flattened_record,
};
use crate::files_source::constants::MAX_RECORD_BYTES;
use crate::row_batches::models::Flatten;
pub(crate) struct JsonReader {
bytes: BytesReader,
splitter: JsonSplitter,
ready: VecDeque<JsonRecord>,
finished: bool,
flatten: Flatten,
flattened: BTreeSet<String>,
objects: BTreeSet<String>,
}
impl JsonReader {
pub(crate) fn new(bytes: BytesReader, by_line: bool) -> Self {
Self {
bytes,
splitter: JsonSplitter::new(by_line, MAX_RECORD_BYTES),
ready: VecDeque::new(),
finished: false,
flatten: Flatten::default(),
flattened: BTreeSet::new(),
objects: BTreeSet::new(),
}
}
pub(crate) fn flattening(mut self, flatten: &Flatten) -> Self {
self.flatten = flatten.clone();
self
}
pub(crate) fn flattens(&self) -> bool {
!self.flatten.is_none()
}
pub(crate) fn flattened(&self) -> &BTreeSet<String> {
&self.flattened
}
pub(crate) fn objects(&self) -> &BTreeSet<String> {
&self.objects
}
pub(crate) async fn next_records(&mut self, limit: usize) -> Result<Vec<JsonRecord>, String> {
while self.ready.len() < limit && !self.finished {
let chunk = self.bytes.next_chunk().await?;
self.finished = chunk.is_empty();
let records = match self.finished {
true => self.splitter.finish()?,
false => self.splitter.push(&chunk)?,
};
self.ready.extend(records);
}
let take = limit.min(self.ready.len());
let records: Vec<JsonRecord> = self.ready.drain(..take).collect();
if self.flatten.is_none() {
return Ok(records);
}
let mut flat: Vec<JsonRecord> = Vec::with_capacity(records.len());
for record in records {
let (record, made, objects) = flattened_record(record, &self.flatten)?;
self.flattened.extend(made);
self.objects.extend(objects);
flat.push(record);
}
Ok(flat)
}
}
pub(crate) fn records_batch(
records: &[JsonRecord],
columns: &[(String, ColumnType)],
(checks, flattened): (&ColumnChecks, bool),
) -> Result<(RecordBatch, Vec<u64>), String> {
let index: HashMap<&str, usize> = columns
.iter()
.enumerate()
.map(|(position, (name, _))| (name.as_str(), position))
.collect();
let mut values: Vec<Vec<Option<String>>> = vec![vec![None; records.len()]; columns.len()];
for (row, record) in records.iter().enumerate() {
for field in &record.fields {
let (key, value, nested) = field;
let Some(position) = index.get(key.as_str()) else {
if !(flattened && value.is_null() && checks.is_flattened_object(key)) {
checks.check_name("key", key, record.line)?;
}
continue;
};
check_value(checks, field, record.line)?;
values[*position][row] = match nested {
Some(text) => Some(text.clone()),
None => value_text(value, &columns[*position].1),
};
}
}
let fields: Vec<Field> = columns
.iter()
.map(|(name, _)| Field::new(name, DataType::Utf8, true))
.collect();
let arrays: Vec<ArrayRef> = values
.into_iter()
.map(|column| Arc::new(StringArray::from(column)) as ArrayRef)
.collect();
let options = RecordBatchOptions::new().with_row_count(Some(records.len()));
let batch = RecordBatch::try_new_with_options(Arc::new(Schema::new(fields)), arrays, &options)
.map_err(|error| error.to_string())?;
Ok((batch, records.iter().map(|record| record.line).collect()))
}
fn check_value(
checks: &ColumnChecks,
(key, value, nested): &(String, Value, Option<String>),
line: u64,
) -> Result<(), String> {
match checks.inferred_type(key) {
Some(column_type) if !value_fits(column_type, json_value_type(value)) => {
let shown = nested.clone().unwrap_or_else(|| value.to_string());
Err(misfit(line, key, &shown, column_type))
}
_ => Ok(()),
}
}
fn value_text(value: &Value, column_type: &ColumnType) -> Option<String> {
match (value, column_type) {
(Value::Null, _) => None,
(Value::String(text), column_type) if *column_type != ColumnType::Json => {
Some(text.clone())
}
(value, _) => Some(value.to_string()),
}
}