use arrow_schema::{DataType, Field, Schema};
use crate::column_types::main::arrow_field::arrow_field;
use crate::column_types::models::ColumnType;
pub fn merge_schema(seen: &Schema, batch: &Schema) -> Schema {
let mut fields: Vec<Field> = seen
.fields()
.iter()
.map(|field| field.as_ref().clone())
.collect();
for field in batch.fields() {
match fields.iter().position(|seen| seen.name() == field.name()) {
Some(position) => fields[position] = merge_field(&fields[position], field),
None => fields.push(field.as_ref().clone()),
}
}
Schema::new(fields)
}
fn merge_field(seen: &Field, field: &Field) -> Field {
match (seen.data_type(), field.data_type()) {
(left, right) if left == right && seen.metadata() == field.metadata() => seen.clone(),
(DataType::Null, _) => field.clone(),
(_, DataType::Null) => seen.clone(),
(DataType::Int64, DataType::Float64) | (DataType::Float64, DataType::Int64) => {
Field::new(seen.name(), DataType::Float64, true)
}
(DataType::List(left), DataType::List(right)) if list_elements_merge(left, right) => {
let element = merge_field(left, right);
Field::new(seen.name(), DataType::List(element.into()), true)
}
(DataType::List(_), _) | (_, DataType::List(_)) => {
arrow_field(seen.name(), &ColumnType::Json)
.unwrap_or_else(|| Field::new(seen.name(), DataType::Utf8, true))
}
_ => Field::new(seen.name(), DataType::Utf8, true),
}
}
fn list_elements_merge(left: &Field, right: &Field) -> bool {
*merge_field(left, right).data_type() != DataType::Utf8
|| (*left.data_type() == DataType::Utf8 && *right.data_type() == DataType::Utf8)
}