use std::collections::HashMap;
use std::collections::VecDeque;
use std::sync::Arc;
use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StringArray};
use arrow_schema::{DataType, Field, Schema};
use serde_json::Value;
use crate::column_types::models::ColumnType;
use crate::files_source::_helpers::text::bytes_reader::BytesReader;
use crate::files_source::_helpers::text::checks::{ColumnChecks, misfit, value_fits};
use crate::files_source::_helpers::text::inference::json_value_type;
use crate::files_source::_helpers::text::json_records::{JsonRecord, JsonSplitter};
use crate::files_source::constants::MAX_RECORD_BYTES;
pub(crate) struct JsonReader {
bytes: BytesReader,
splitter: JsonSplitter,
ready: VecDeque<JsonRecord>,
finished: bool,
}
impl JsonReader {
pub(crate) fn new(bytes: BytesReader, by_line: bool) -> Self {
Self {
bytes,
splitter: JsonSplitter::new(by_line, MAX_RECORD_BYTES),
ready: VecDeque::new(),
finished: false,
}
}
pub(crate) async fn next_records(&mut self, limit: usize) -> Result<Vec<JsonRecord>, String> {
while self.ready.len() < limit && !self.finished {
let chunk = self.bytes.next_chunk().await?;
self.finished = chunk.is_empty();
let records = match self.finished {
true => self.splitter.finish()?,
false => self.splitter.push(&chunk)?,
};
self.ready.extend(records);
}
let take = limit.min(self.ready.len());
Ok(self.ready.drain(..take).collect())
}
}
pub(crate) fn records_batch(
records: &[JsonRecord],
columns: &[(String, ColumnType)],
checks: &ColumnChecks,
) -> Result<(RecordBatch, Vec<u64>), String> {
let index: HashMap<&str, usize> = columns
.iter()
.enumerate()
.map(|(position, (name, _))| (name.as_str(), position))
.collect();
let mut values: Vec<Vec<Option<String>>> = vec![vec![None; records.len()]; columns.len()];
for (row, record) in records.iter().enumerate() {
for (key, value) in &record.fields {
let Some(position) = index.get(key.as_str()) else {
checks.check_name("key", key, record.line)?;
continue;
};
check_value(checks, key, value, record.line)?;
values[*position][row] = value_text(value, &columns[*position].1);
}
}
let fields: Vec<Field> = columns
.iter()
.map(|(name, _)| Field::new(name, DataType::Utf8, true))
.collect();
let arrays: Vec<ArrayRef> = values
.into_iter()
.map(|column| Arc::new(StringArray::from(column)) as ArrayRef)
.collect();
let options = RecordBatchOptions::new().with_row_count(Some(records.len()));
let batch = RecordBatch::try_new_with_options(Arc::new(Schema::new(fields)), arrays, &options)
.map_err(|error| error.to_string())?;
Ok((batch, records.iter().map(|record| record.line).collect()))
}
fn check_value(checks: &ColumnChecks, key: &str, value: &Value, line: u64) -> Result<(), String> {
match checks.inferred_type(key) {
Some(column_type) if !value_fits(column_type, json_value_type(value)) => {
Err(misfit(line, key, &value.to_string(), column_type))
}
_ => Ok(()),
}
}
fn value_text(value: &Value, column_type: &ColumnType) -> Option<String> {
match (value, column_type) {
(Value::Null, _) => None,
(Value::String(text), column_type) if *column_type != ColumnType::Json => {
Some(text.clone())
}
(value, _) => Some(value.to_string()),
}
}