use std::sync::Arc;
use arrow_array::{ArrayRef, RecordBatch};
use arrow_cast::{CastOptions, cast_with_options};
use arrow_schema::{DataType, Field, Schema, TimeUnit};
use crate::column_types::models::ColumnType;
use crate::schema_plan::models::DesiredColumn;
fn clickhouse_type(column_type: &ColumnType, arrived: &DataType) -> Result<DataType, String> {
if arrived.is_nested() && !matches!(column_type, ColumnType::Native(_)) {
return Err(format!(
"a nested value loads only into a native(...) column, not {column_type}"
));
}
Ok(match column_type {
ColumnType::Bool => DataType::Boolean,
ColumnType::SmallInt => DataType::Int16,
ColumnType::Int => DataType::Int64,
ColumnType::Float => DataType::Float64,
ColumnType::Decimal { precision, scale } => {
let scale = i8::try_from(*scale)
.map_err(|_| format!("a decimal scale of {scale} has no Arrow type"))?;
DataType::Decimal128(*precision, scale)
}
ColumnType::Time
| ColumnType::Text
| ColumnType::Varchar(_)
| ColumnType::Uuid
| ColumnType::Json => DataType::Utf8,
ColumnType::Binary => DataType::Binary,
ColumnType::TimestampTz => DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
ColumnType::TimestampNtz => DataType::Timestamp(TimeUnit::Microsecond, None),
ColumnType::Date => DataType::Date32,
ColumnType::Native(_) => plain_type(arrived),
})
}
fn plain_type(arrived: &DataType) -> DataType {
match arrived {
DataType::LargeUtf8 | DataType::Utf8View => DataType::Utf8,
DataType::LargeBinary | DataType::BinaryView => DataType::Binary,
DataType::Dictionary(_, value) => plain_type(value),
other => other.clone(),
}
}
pub(crate) fn clickhouse_batch(
batch: &RecordBatch,
(columns, builds): (&[DesiredColumn], bool),
) -> Result<RecordBatch, String> {
let options = CastOptions {
safe: false,
..CastOptions::default()
};
let schema = batch.schema();
let mut fields: Vec<Field> = Vec::with_capacity(batch.num_columns());
let mut arrays: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());
for (field, array) in schema.fields().iter().zip(batch.columns()) {
let name = field.name();
let column = match (columns.iter().find(|column| &column.name == name), builds) {
(Some(column), _) => column,
(None, true) => continue,
(None, false) => {
return Err(format!(
"column {name} is not among the table's planned columns"
));
}
};
let target = clickhouse_type(&column.column_type, field.data_type()).map_err(|reason| {
format!(
"column {name} ({}) cannot load into ClickHouse: {reason}",
field.data_type()
)
})?;
let cast = match field.data_type() == &target {
true => Arc::clone(array),
false => cast_with_options(array, &target, &options).map_err(|error| {
format!("column {name} cannot be sent to ClickHouse as {target}: {error}")
})?,
};
fields.push(Field::new(name, target, true));
arrays.push(cast);
}
RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).map_err(|error| error.to_string())
}