use crate::files_source::_helpers::plan::columns::{
ColumnMerge, SampledColumn, drop_loader_columns, file_columns, merge_columns, parquet_columns,
};
use crate::files_source::models::{BatchFooters, DataFile, FileColumn};
pub fn batch_columns(
footers: &BatchFooters,
files: &[DataFile],
known: &[FileColumn],
replaced: &[&str],
) -> (Vec<FileColumn>, Vec<String>) {
let mut columns: Vec<SampledColumn> = known
.iter()
.map(|column| SampledColumn {
name: column.name.clone(),
column_type: Some(column.column_type.clone()),
direct_refusal: column.direct_refusal.clone(),
stored_read: column.stored_read,
lists: column.lists,
})
.collect();
let mut problems: Vec<String> = Vec::new();
for data in files {
let Some(footer) = footers.parquet.get(&data.file.relative_key) else {
continue;
};
let (mut found, _) = drop_loader_columns(parquet_columns(footer));
for column in &mut found {
column.direct_refusal = None;
if let Some(seen) = known.iter().find(|seen| seen.name == column.name) {
column.stored_read = seen.stored_read;
}
}
let merge = ColumnMerge {
key: &data.file.relative_key,
replaced,
widen_numbers: false,
};
let (merged, conflicts) = merge_columns(columns, found, merge);
columns = merged;
problems.extend(conflicts);
}
(file_columns(columns), problems)
}