use std::collections::HashMap;
use std::sync::Arc;
use arrow_array::cast::AsArray;
use arrow_array::{Array, ArrayRef, BooleanArray, RecordBatch, RecordBatchOptions};
use arrow_schema::{DataType, Field, Schema};
use crate::column_types::constants::ARROW_TYPE_METADATA_KEY;
use crate::column_types::models::ColumnType;
use crate::row_batches::_helpers::arrow_json::normalise_column;
use crate::row_batches::constants::{
FLATTEN_SEPARATOR, FLATTENED_METADATA_KEY, FLATTENED_SEPARATOR,
};
use crate::row_batches::main::columns_batch::columns_batch;
use crate::row_batches::main::json_rows::json_rows;
use crate::row_batches::models::{JsonLayout, JsonRead};
use crate::row_batches::types::NamedColumn;
pub(crate) fn rebuilt(
columns: Vec<(Arc<Field>, ArrayRef)>,
rows: usize,
flattened: &[String],
) -> Result<RecordBatch, String> {
let (fields, arrays): (Vec<Arc<Field>>, Vec<ArrayRef>) = columns.into_iter().unzip();
let options = RecordBatchOptions::new().with_row_count(Some(rows));
let schema = marked(Schema::new(fields), flattened);
RecordBatch::try_new_with_options(Arc::new(schema), arrays, &options)
.map_err(|error| error.to_string())
}
pub(crate) fn marked(schema: Schema, flattened: &[String]) -> Schema {
match flattened.is_empty() {
true => schema,
false => {
let mut metadata = schema.metadata().clone();
metadata.insert(
FLATTENED_METADATA_KEY.to_owned(),
flattened.join(FLATTENED_SEPARATOR),
);
schema.with_metadata(metadata)
}
}
}
pub(crate) fn flattened_names(schema: &Schema) -> Vec<String> {
schema
.metadata()
.get(FLATTENED_METADATA_KEY)
.map(|names| {
names
.split(FLATTENED_SEPARATOR)
.map(str::to_owned)
.collect()
})
.unwrap_or_default()
}
pub(crate) struct Names<'a> {
pub(crate) plain: &'a Schema,
pub(crate) made: HashMap<String, String>,
}
impl Names<'_> {
pub(crate) fn claim(&mut self, object: &str, key: &str) -> Result<String, String> {
let name = format!("{object}{FLATTEN_SEPARATOR}{key}");
let field = self
.plain
.fields()
.iter()
.any(|field| *field.name() == name);
let other = self.made.get(&name).filter(|from| from.as_str() != object);
let clash = match (field, other) {
(true, _) => Some("which the data already has".to_owned()),
(false, Some(from)) => Some(format!("which flattening {from} also gives")),
(false, None) => None,
};
match clash {
Some(clash) => Err(format!(
"flatten {object}: the key {key} gives column {name}, {clash}; rename one of them or leave {object} unflattened"
)),
None => {
self.made.insert(name.clone(), object.to_owned());
Ok(name)
}
}
}
}
pub(crate) fn expanded_columns(
batch: &RecordBatch,
flatten: &[String],
) -> Result<(Vec<NamedColumn>, Vec<String>), String> {
let schema = batch.schema();
let mut names = Names {
plain: &schema,
made: HashMap::new(),
};
let mut columns: Vec<(Arc<Field>, ArrayRef)> = Vec::with_capacity(schema.fields().len());
for (field, array) in schema.fields().iter().zip(batch.columns()) {
match flatten.contains(field.name()) {
false => columns.push((field.clone(), array.clone())),
true => columns.extend(key_columns(field, array, &mut names)?),
}
}
let mut flattened = flattened_names(&schema);
flattened.extend(names.made.into_keys());
flattened.sort();
Ok((columns, flattened))
}
pub(crate) fn key_columns(
field: &Field,
array: &ArrayRef,
names: &mut Names<'_>,
) -> Result<Vec<(Arc<Field>, ArrayRef)>, String> {
let is_json = field
.metadata()
.get(ARROW_TYPE_METADATA_KEY)
.is_some_and(|kind| *kind == ColumnType::Json.to_string());
let keys = match (array.data_type(), is_json) {
_ if array.logical_null_count() == array.len() => Vec::new(),
(DataType::Struct(_), _) => struct_children(array)?,
(DataType::Utf8, true) => json_keys(field, array)?,
(DataType::Map(..), _) => {
let (json_field, json_array) = normalise_column(field.name(), array)?;
json_keys(&json_field, &json_array)?
}
(DataType::Utf8, false) => {
let (row, shape) = text_shape(array);
return Err(shape_error(field.name(), row, shape));
}
(data_type, _) => {
return Err(shape_error(
field.name(),
first_value(array),
&shape(data_type),
));
}
};
keys.into_iter()
.map(|(key, values)| {
let name = names.claim(field.name(), key.name())?;
Ok((Arc::new(key.as_ref().clone().with_name(name)), values))
})
.collect()
}
fn struct_children(array: &ArrayRef) -> Result<Vec<(Arc<Field>, ArrayRef)>, String> {
let structs = array.as_struct();
let missing: BooleanArray = (0..structs.len())
.map(|row| Some(structs.is_null(row)))
.collect();
let fields = structs.fields().iter().cloned();
fields
.zip(structs.columns())
.map(|(child_field, child)| {
let child =
arrow_select::nullif::nullif(child, &missing).map_err(|error| error.to_string())?;
Ok((
Arc::new(child_field.as_ref().clone().with_nullable(true)),
child,
))
})
.collect()
}
fn json_keys(field: &Field, array: &ArrayRef) -> Result<Vec<(Arc<Field>, ArrayRef)>, String> {
let document = object_lines(field, array)?;
let read = JsonRead {
layout: JsonLayout::Document,
path: None,
batch_rows: array.len().max(1),
projection: None,
};
let mut keys: Vec<(Arc<Field>, ArrayRef)> = Vec::new();
json_rows(document.as_bytes(), &read, |columns| {
let batch = columns_batch(columns)?;
keys.extend(
batch
.schema()
.fields()
.iter()
.cloned()
.zip(batch.columns().iter().cloned()),
);
Ok(())
})
.map_err(|error| format!("flatten {}: {error}", field.name()))?;
Ok(keys)
}
fn object_lines(field: &Field, array: &ArrayRef) -> Result<String, String> {
let texts = array.as_string::<i32>();
let mut document = String::new();
for row in 0..array.len() {
let text = match texts.is_valid(row) {
true => texts.value(row).trim(),
false => "null",
};
let line = match (text, text.starts_with('{'), text.starts_with('[')) {
("null", _, _) => "{}",
(object, true, _) => object,
(_, false, true) => return Err(shape_error(field.name(), row, "a list")),
(_, false, false) => return Err(shape_error(field.name(), row, "a scalar")),
};
document.push_str(line);
document.push('\n');
}
Ok(document)
}
fn text_shape(array: &ArrayRef) -> (usize, &'static str) {
let texts = array.as_string::<i32>();
let object = |row: usize| texts.value(row).trim_start().starts_with('{');
let valid = (0..array.len()).filter(|row| texts.is_valid(*row));
let mixed = valid.clone().any(object);
let row = valid
.clone()
.find(|row| !object(*row))
.unwrap_or_else(|| first_value(array));
match mixed {
true => (row, "a scalar (other rows hold objects)"),
false => (row, "a text value"),
}
}
fn first_value(array: &ArrayRef) -> usize {
(0..array.len())
.find(|row| array.is_valid(*row))
.unwrap_or_default()
}
pub(crate) fn shape(data_type: &DataType) -> String {
match data_type {
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => "a text value".to_owned(),
DataType::Boolean => "a boolean".to_owned(),
DataType::Date32 | DataType::Date64 => "a date".to_owned(),
DataType::Timestamp(..) => "a timestamp".to_owned(),
DataType::Decimal128(..) | DataType::Decimal256(..) => "a decimal".to_owned(),
DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(..) => {
"a list".to_owned()
}
DataType::Map(..) => "a map".to_owned(),
numeric if numeric.is_numeric() => "a number".to_owned(),
other => format!("a {other} value"),
}
}
pub(crate) fn shape_error(object: &str, row: usize, shape: &str) -> String {
format!(
"flatten {object}: row {} of a batch holds {shape}, not an object; the shape changed, so leave {object} unflattened or fix the source",
row + 1
)
}