use std::sync::Arc;
use arrow_array::{ArrayRef, RecordBatch, StringArray, new_null_array};
use arrow_cast::{CastOptions, cast_with_options};
use arrow_schema::{Field, Schema, SchemaRef};
use crate::column_types::main::arrow_field::arrow_field;
use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN};
use crate::files_source::models::DataFile;
use crate::schema_plan::models::DesiredColumn;
pub(crate) fn output_schema(columns: &[DesiredColumn]) -> Result<SchemaRef, String> {
let fields: Vec<Field> = columns
.iter()
.map(|column| {
arrow_field(&column.name, &column.column_type)
.ok_or_else(|| format!("{} has no Arrow representation", column.name))
})
.collect::<Result<_, String>>()?;
Ok(Arc::new(Schema::new(fields)))
}
pub(crate) fn shape_batch(
batch: &RecordBatch,
file: &DataFile,
file_name: &str,
schema: &SchemaRef,
) -> Result<RecordBatch, String> {
let mut constants: Vec<(&str, &str)> = file
.partitions
.iter()
.map(|(key, value)| (key.as_str(), value.as_str()))
.collect();
let version = file.file.version();
constants.push((SOURCE_FILE_COLUMN, file_name));
constants.push((SOURCE_VERSION_COLUMN, &version));
let columns: Vec<ArrayRef> = schema
.fields()
.iter()
.map(|field| shape_column(batch, &constants, field))
.collect::<Result<_, String>>()?;
RecordBatch::try_new(schema.clone(), columns).map_err(|error| error.to_string())
}
fn shape_column(
batch: &RecordBatch,
constants: &[(&str, &str)],
field: &Field,
) -> Result<ArrayRef, String> {
let rows = batch.num_rows();
let constant = constants.iter().find(|(key, _)| key == field.name());
let source: ArrayRef = match (constant, batch.column_by_name(field.name())) {
(Some((_, value)), _) => Arc::new(StringArray::from(vec![*value; rows])),
(None, Some(column)) => column.clone(),
(None, None) => return Ok(new_null_array(field.data_type(), rows)),
};
let strict = CastOptions {
safe: false,
..CastOptions::default()
};
cast_with_options(&source, field.data_type(), &strict).map_err(|error| {
format!(
"{}: cannot convert to {}: {error}",
field.name(),
field.data_type()
)
})
}