use std::sync::Arc;
use arrow_array::cast::AsArray;
use arrow_array::{Array, ArrayRef, ListArray, RecordBatch};
use arrow_cast::{CastOptions, cast_with_options};
use arrow_schema::{DataType, Field, Schema, TimeUnit};
use prost::Message;
use crate::adapters::bigquery::_helpers::dialect::type_names::decimal_digits;
use crate::bigquery_api::main::arrow_request::arrow_request;
use crate::bigquery_api::models::AppendRowsRequest;
use crate::column_types::constants::ARRAY_ITEM;
use crate::column_types::models::ColumnType;
use crate::schema_plan::models::DesiredColumn;
fn write_api_type(column_type: &ColumnType, arrived: &DataType) -> Result<DataType, String> {
if let (ColumnType::Array(element), DataType::List(item)) = (column_type, arrived) {
let element = match write_api_type(element, item.data_type())? {
DataType::Decimal128(_, _) => DataType::Decimal128(38, 9),
DataType::Decimal256(_, _) => DataType::Decimal256(76, 38),
other => other,
};
return Ok(DataType::List(Arc::new(Field::new(
ARRAY_ITEM, element, true,
))));
}
if arrived.is_nested() || matches!(column_type, ColumnType::Array(_)) {
return Err("nested values are not streamed".to_owned());
}
Ok(match column_type {
ColumnType::Bool => DataType::Boolean,
ColumnType::SmallInt | ColumnType::Int => DataType::Int64,
ColumnType::Decimal { precision, scale } => {
let (precision, scale, big) = decimal_digits(*precision, *scale);
let scale = i8::try_from(scale)
.map_err(|_| format!("a decimal scale of {scale} has no Arrow type"))?;
match big {
true => DataType::Decimal256(precision, scale),
false => DataType::Decimal128(precision, scale),
}
}
ColumnType::Float => DataType::Float64,
ColumnType::Text | ColumnType::Varchar(_) | ColumnType::Uuid | ColumnType::Json => {
DataType::Utf8
}
ColumnType::Date => DataType::Date32,
ColumnType::Time => DataType::Time64(TimeUnit::Microsecond),
ColumnType::TimestampTz => DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
ColumnType::TimestampNtz => DataType::Timestamp(TimeUnit::Microsecond, None),
ColumnType::Binary => DataType::Binary,
ColumnType::Native(_) => match arrived {
DataType::LargeUtf8 | DataType::Utf8View | DataType::Dictionary(_, _) => DataType::Utf8,
DataType::LargeBinary | DataType::BinaryView => DataType::Binary,
other => other.clone(),
},
ColumnType::Array(_) => return Err("an array arrives as a list".to_owned()),
})
}
pub(crate) fn write_api_batch(
batch: &RecordBatch,
(columns, builds): (&[DesiredColumn], bool),
) -> Result<RecordBatch, String> {
let options = CastOptions {
safe: false,
..CastOptions::default()
};
let schema = batch.schema();
let mut fields: Vec<Field> = Vec::with_capacity(batch.num_columns());
let mut arrays: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());
for (field, array) in schema.fields().iter().zip(batch.columns()) {
let name = field.name();
let column = match (columns.iter().find(|column| &column.name == name), builds) {
(Some(column), _) => column,
(None, true) => continue,
(None, false) => {
return Err(format!(
"column {name} is not among the table's planned columns"
));
}
};
let target = write_api_type(&column.column_type, field.data_type()).map_err(|reason| {
format!(
"column {name} ({}) cannot load into BigQuery yet: {reason}",
field.data_type()
)
})?;
let cast = cast_with_options(array, &target, &options).map_err(|error| {
format!("column {name} cannot be sent to BigQuery as {target}: {error}")
})?;
let cast = repeated(cast).map_err(|reason| format!("column {name}: {reason}"))?;
fields.push(Field::new(name, cast.data_type().clone(), true));
arrays.push(cast);
}
RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).map_err(|error| error.to_string())
}
fn repeated(array: ArrayRef) -> Result<ArrayRef, String> {
let DataType::List(item) = array.data_type() else {
return Ok(array);
};
let lists = array.as_list::<i32>();
for row in 0..lists.len() {
if lists.is_valid(row) && lists.value(row).null_count() > 0 {
return Err(format!(
"row {}: an array holds a NULL element, which BigQuery arrays cannot hold",
row + 1
));
}
}
let lists = ListArray::try_new(
Arc::new(item.as_ref().clone().with_nullable(false)),
lists.offsets().clone(),
lists.values().clone(),
None,
)
.map_err(|error| error.to_string())?;
Ok(Arc::new(lists))
}
pub(crate) fn append_requests(
batch: &RecordBatch,
max_bytes: usize,
) -> Result<Vec<(AppendRowsRequest, usize)>, String> {
let rows = batch.num_rows();
let memory = batch
.columns()
.iter()
.map(|column| column.to_data().get_slice_memory_size())
.sum::<Result<usize, _>>()
.map_err(|error| error.to_string())?
.max(1);
let mut step = (rows.saturating_mul(max_bytes) / memory).clamp(1, rows.max(1));
let mut requests: Vec<(AppendRowsRequest, usize)> = Vec::new();
let mut start = 0;
while start < rows {
let length = step.min(rows - start);
let request = arrow_request("", 0, &batch.slice(start, length), true)
.map_err(|error| error.to_string())?;
if request.encoded_len() > max_bytes && length > 1 {
step = length / 2;
continue;
}
requests.push((request, length));
start += length;
}
Ok(requests)
}