use std::collections::HashMap;
use std::sync::Arc;
use arrow_array::cast::AsArray;
use arrow_array::{Array, ArrayRef, BooleanArray, RecordBatch, RecordBatchOptions};
use arrow_schema::{DataType, Field, Schema};
use crate::column_types::constants::ARROW_TYPE_METADATA_KEY;
use crate::column_types::models::ColumnType;
use crate::load_naming::main::loaded_name::loaded_name;
use crate::load_naming::models::Naming;
use crate::row_batches::_helpers::arrow_json::normalise_column;
use crate::row_batches::constants::{
DATA_KEYS_SHARE, FLATTEN_SEPARATOR, FLATTENED_METADATA_KEY, FLATTENED_SEPARATOR, MANY_KEYS,
};
use crate::row_batches::main::columns_batch::columns_batch;
use crate::row_batches::main::json_rows::json_rows;
use crate::row_batches::models::{Flatten, FlattenObjects, JsonLayout, JsonRead};
use crate::row_batches::types::FlattenedColumns;
pub(crate) fn rebuilt(
columns: Vec<(Arc<Field>, ArrayRef)>,
rows: usize,
flattened: &[String],
) -> Result<RecordBatch, String> {
let (fields, arrays): (Vec<Arc<Field>>, Vec<ArrayRef>) = columns.into_iter().unzip();
let options = RecordBatchOptions::new().with_row_count(Some(rows));
let schema = marked(Schema::new(fields), flattened);
RecordBatch::try_new_with_options(Arc::new(schema), arrays, &options)
.map_err(|error| error.to_string())
}
pub(crate) fn marked(schema: Schema, flattened: &[String]) -> Schema {
match flattened.is_empty() {
true => schema,
false => {
let mut metadata = schema.metadata().clone();
metadata.insert(
FLATTENED_METADATA_KEY.to_owned(),
flattened.join(FLATTENED_SEPARATOR),
);
schema.with_metadata(metadata)
}
}
}
pub(crate) fn flattened_names(schema: &Schema) -> Vec<String> {
schema
.metadata()
.get(FLATTENED_METADATA_KEY)
.map(|names| {
names
.split(FLATTENED_SEPARATOR)
.map(str::to_owned)
.collect()
})
.unwrap_or_default()
}
pub(crate) struct Names<'a> {
pub(crate) plain: &'a Schema,
pub(crate) made: HashMap<String, String>,
}
impl Names<'_> {
pub(crate) fn claim(&mut self, object: &str, key: &str) -> Result<String, String> {
let name = format!("{object}{FLATTEN_SEPARATOR}{key}");
let field = self
.plain
.fields()
.iter()
.any(|field| *field.name() == name);
let other = self.made.get(&name).filter(|from| from.as_str() != object);
let clash = match (field, other) {
(true, _) => Some("which the data already has".to_owned()),
(false, Some(from)) => Some(format!("which flattening {from} also gives")),
(false, None) => None,
};
match clash {
Some(clash) => Err(format!(
"flatten {object}: the key {key} gives column {name}, {clash}; rename one of them or leave {object} unflattened"
)),
None => {
self.made.insert(name.clone(), object.to_owned());
Ok(name)
}
}
}
}
pub(crate) fn expanded_columns(
batch: &RecordBatch,
flatten: &Flatten,
) -> Result<Option<FlattenedColumns>, String> {
let schema = batch.schema();
let earlier = flattened_names(&schema);
let mut names = Names {
plain: &schema,
made: HashMap::new(),
};
let mut columns: Vec<(Arc<Field>, ArrayRef)> = Vec::with_capacity(schema.fields().len());
let mut expanded = false;
for (field, array) in schema.fields().iter().zip(batch.columns()) {
let path: Vec<&str> = match earlier.contains(field.name()) {
true => field.name().split(FLATTEN_SEPARATOR).collect(),
false => vec![field.name().as_str()],
};
expanded |= expand((field, array), &path, flatten, (&mut names, &mut columns))?;
}
if !expanded {
return Ok(None);
}
let made: Vec<&str> = columns
.iter()
.map(|(field, _)| field.name().as_str())
.filter(|name| names.made.contains_key(*name))
.collect();
let mut flattened: Vec<String> = earlier;
flattened.extend(made.into_iter().map(str::to_owned));
flattened.sort();
flattened.dedup();
Ok(Some((columns, flattened)))
}
fn expand(
(field, array): (&Arc<Field>, &ArrayRef),
path: &[&str],
flatten: &Flatten,
(names, columns): (&mut Names<'_>, &mut Vec<(Arc<Field>, ArrayRef)>),
) -> Result<bool, String> {
let object = flatten.lists(path)
|| (flatten.objects == FlattenObjects::All && holds_objects(field, array));
if !(object && flatten.flattens(path)?) {
columns.push((field.clone(), array.clone()));
return Ok(false);
}
let parent = path.join(FLATTEN_SEPARATOR);
for (key, values) in key_columns(field, array, flatten.as_text)? {
let name = names.claim(&parent, key.name())?;
let below: Vec<&str> = path.iter().copied().chain([key.name().as_str()]).collect();
let named = Arc::new(key.as_ref().clone().with_name(name));
expand((&named, &values), &below, flatten, (names, columns))?;
}
Ok(true)
}
fn holds_objects(field: &Field, array: &ArrayRef) -> bool {
if array.logical_null_count() == array.len() {
return false;
}
match array.data_type() {
DataType::Struct(_) | DataType::Map(..) => true,
DataType::Utf8 if is_json(field) => {
let texts = array.as_string::<i32>();
texts
.iter()
.flatten()
.all(|text| text.trim_start().starts_with('{'))
}
_ => false,
}
}
fn is_json(field: &Field) -> bool {
field
.metadata()
.get(ARROW_TYPE_METADATA_KEY)
.is_some_and(|kind| *kind == ColumnType::Json.to_string())
}
pub(crate) fn key_columns(
field: &Field,
array: &ArrayRef,
as_text: bool,
) -> Result<Vec<(Arc<Field>, ArrayRef)>, String> {
match (array.data_type(), is_json(field)) {
_ if array.logical_null_count() == array.len() => Ok(Vec::new()),
(DataType::Struct(_), _) => struct_children(array),
(DataType::Utf8, true) => json_keys(field, array, as_text),
(DataType::Map(..), _) => {
let (json_field, json_array) = normalise_column(field.name(), array)?;
json_keys(&json_field, &json_array, as_text)
}
(DataType::Utf8, false) => {
let (row, shape) = text_shape(array);
Err(shape_error(field.name(), row, shape))
}
(data_type, _) => Err(shape_error(
field.name(),
first_value(array),
&shape(data_type),
)),
}
}
fn struct_children(array: &ArrayRef) -> Result<Vec<(Arc<Field>, ArrayRef)>, String> {
let structs = array.as_struct();
let missing: BooleanArray = (0..structs.len())
.map(|row| Some(structs.is_null(row)))
.collect();
let fields = structs.fields().iter().cloned();
fields
.zip(structs.columns())
.map(|(child_field, child)| {
let child =
arrow_select::nullif::nullif(child, &missing).map_err(|error| error.to_string())?;
Ok((
Arc::new(child_field.as_ref().clone().with_nullable(true)),
child,
))
})
.collect()
}
fn json_keys(
field: &Field,
array: &ArrayRef,
as_text: bool,
) -> Result<Vec<(Arc<Field>, ArrayRef)>, String> {
let document = object_lines(field, array)?;
let read = JsonRead {
layout: JsonLayout::Document,
path: None,
batch_rows: array.len().max(1),
projection: None,
};
let mut keys: Vec<(Arc<Field>, ArrayRef)> = Vec::new();
json_rows(document.as_bytes(), &read, |columns| {
let batch = columns_batch(columns, as_text)?;
keys.extend(
batch
.schema()
.fields()
.iter()
.cloned()
.zip(batch.columns().iter().cloned()),
);
Ok(())
})
.map_err(|error| format!("flatten {}: {error}", field.name()))?;
Ok(keys)
}
fn object_lines(field: &Field, array: &ArrayRef) -> Result<String, String> {
let texts = array.as_string::<i32>();
let mut document = String::new();
for row in 0..array.len() {
let text = match texts.is_valid(row) {
true => texts.value(row).trim(),
false => "null",
};
let line = match (text, text.starts_with('{'), text.starts_with('[')) {
("null", _, _) => "{}",
(object, true, _) => object,
(_, false, true) => return Err(shape_error(field.name(), row, "a list")),
(_, false, false) => return Err(shape_error(field.name(), row, "a scalar")),
};
document.push_str(line);
document.push('\n');
}
Ok(document)
}
fn text_shape(array: &ArrayRef) -> (usize, &'static str) {
let texts = array.as_string::<i32>();
let object = |row: usize| texts.value(row).trim_start().starts_with('{');
let valid = (0..array.len()).filter(|row| texts.is_valid(*row));
let mixed = valid.clone().any(object);
let row = valid
.clone()
.find(|row| !object(*row))
.unwrap_or_else(|| first_value(array));
match mixed {
true => (row, "a scalar (other rows hold objects)"),
false => (row, "a text value"),
}
}
fn first_value(array: &ArrayRef) -> usize {
(0..array.len())
.find(|row| array.is_valid(*row))
.unwrap_or_default()
}
pub(crate) fn shape(data_type: &DataType) -> String {
match data_type {
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => "a text value".to_owned(),
DataType::Boolean => "a boolean".to_owned(),
DataType::Date32 | DataType::Date64 => "a date".to_owned(),
DataType::Timestamp(..) => "a timestamp".to_owned(),
DataType::Decimal128(..) | DataType::Decimal256(..) => "a decimal".to_owned(),
DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(..) => {
"a list".to_owned()
}
DataType::Map(..) => "a map".to_owned(),
numeric if numeric.is_numeric() => "a number".to_owned(),
other => format!("a {other} value"),
}
}
pub(crate) fn shape_error(object: &str, row: usize, shape: &str) -> String {
format!(
"flatten {object}: row {} of a batch holds {shape}, not an object; the shape changed, so leave {object} unflattened or fix the source",
row + 1
)
}
pub(crate) fn leaf_keys(object: &str, (objects, columns): (&[String], &[String])) -> Vec<String> {
let prefix = format!("{object}{FLATTEN_SEPARATOR}");
columns
.iter()
.filter_map(|column| column.strip_prefix(&prefix))
.filter(|key| !objects.contains(&format!("{prefix}{key}")))
.map(str::to_owned)
.collect()
}
pub(crate) fn keys_are_data(keys: &[String]) -> bool {
let data = keys.iter().filter(|key| is_data_key(key)).count();
keys.len() > MANY_KEYS || (!keys.is_empty() && data * 100 > keys.len() * DATA_KEYS_SHARE)
}
pub(crate) fn is_data_key(key: &str) -> bool {
key.bytes().any(|byte| byte.is_ascii_digit())
|| loaded_name(Naming::SnakeCase, key, None) != key.to_lowercase()
}