use std::sync::Arc;
use arrow_array::RecordBatch;
use arrow_schema::{FieldRef, Schema};
use crate::load_naming::_helpers::excluded::unexcluded;
use crate::load_naming::_helpers::numbering::clashes;
use crate::load_naming::main::loaded_columns::loaded_columns;
use crate::load_naming::models::{ColumnName, NameMemo, NameRules, NumberedName};
use crate::load_naming::types::RenameFunction;
pub fn named_batch(
batch: RecordBatch,
memo: NameMemo,
rules: &NameRules,
rename: RenameFunction<'_>,
) -> (Result<RecordBatch, String>, NameMemo) {
let batch = match unexcluded(batch, rules) {
Ok(batch) => batch,
Err(problem) => return (Err(problem), memo),
};
let new: Vec<String> = batch
.schema()
.fields()
.iter()
.map(|field| field.name().clone())
.filter(|name| !memo.loaded.contains_key(name))
.collect();
if new.is_empty() {
return (renamed_batch(&batch, &memo), memo);
}
match remembered(&memo, new, rules, rename) {
Ok(memo) => (renamed_batch(&batch, &memo), memo),
Err(problem) => (Err(problem), memo),
}
}
fn remembered(
memo: &NameMemo,
new: Vec<String>,
rules: &NameRules,
rename: RenameFunction<'_>,
) -> Result<NameMemo, String> {
let mut memo = memo.clone();
memo.seen.extend(new);
let columns: Vec<ColumnName> = memo
.seen
.iter()
.enumerate()
.map(|(index, name)| ColumnName {
position: index + 1,
source: name.clone(),
read_as: name.clone(),
})
.collect();
let mut earlier = rules.kept.clone();
earlier.extend(memo.numbered.iter().cloned());
let given = memo.loaded.iter().map(|(source, loaded)| NumberedName {
column: loaded.clone(),
source: source.clone(),
repeat: 0,
});
let rules = NameRules {
kept: earlier.iter().cloned().chain(given).collect(),
..rules.clone()
};
let names = loaded_columns(&columns, &rules, rename).map_err(|problems| problems.join("; "))?;
for column in &names.columns {
if let Some(earlier) = memo.loaded.get(&column.source)
&& *earlier != column.loaded
{
return Err(format!(
"field {:?} loaded as {earlier} earlier in this load, and the rename now gives it {}; a rename function must give a field the same name every time",
column.source, column.loaded
));
}
memo.loaded
.insert(column.source.clone(), column.loaded.clone());
}
memo.numbered = names
.numbered
.into_iter()
.filter(|name| earlier.contains(name) || clashes(name, &names.columns))
.collect();
memo.columns = names.columns;
Ok(memo)
}
fn renamed_batch(batch: &RecordBatch, memo: &NameMemo) -> Result<RecordBatch, String> {
let schema = batch.schema();
let fields: Vec<FieldRef> = schema
.fields()
.iter()
.map(|field| match memo.loaded.get(field.name()) {
Some(loaded) if loaded != field.name() => {
Arc::new(field.as_ref().clone().with_name(loaded))
}
_ => field.clone(),
})
.collect();
RecordBatch::try_new(
Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())),
batch.columns().to_vec(),
)
.map_err(|error| error.to_string())
}