sqlload 0.33.4

SQL data loading tooling.
//! Rows as the Storage Write API reads them: Arrow types it accepts, cut into requests it takes.

use std::sync::Arc;

use arrow_array::cast::AsArray;
use arrow_array::{Array, ArrayRef, ListArray, RecordBatch};
use arrow_cast::{CastOptions, cast_with_options};
use arrow_schema::{DataType, Field, Schema, TimeUnit};
use prost::Message;

use crate::adapters::bigquery::_helpers::dialect::type_names::decimal_digits;
use crate::bigquery_api::main::arrow_request::arrow_request;
use crate::bigquery_api::models::AppendRowsRequest;
use crate::column_types::constants::ARRAY_ITEM;
use crate::column_types::models::ColumnType;
use crate::schema_plan::models::DesiredColumn;

/// The Arrow type the Write API reads into a column of `column_type`, given the type the rows arrived as, or why it cannot load it yet. A decimal inside an array goes at its column's own scale (9 for NUMERIC, 38 for BIGNUMERIC), as the Write API reads an element at that scale whatever scale the Arrow type gives.
fn write_api_type(column_type: &ColumnType, arrived: &DataType) -> Result<DataType, String> {
    if let (ColumnType::Array(element), DataType::List(item)) = (column_type, arrived) {
        let element = match write_api_type(element, item.data_type())? {
            DataType::Decimal128(_, _) => DataType::Decimal128(38, 9),
            DataType::Decimal256(_, _) => DataType::Decimal256(76, 38),
            other => other,
        };
        return Ok(DataType::List(Arc::new(Field::new(
            ARRAY_ITEM, element, true,
        ))));
    }
    if arrived.is_nested() || matches!(column_type, ColumnType::Array(_)) {
        return Err("nested values are not streamed".to_owned());
    }
    Ok(match column_type {
        ColumnType::Bool => DataType::Boolean,
        ColumnType::SmallInt | ColumnType::Int => DataType::Int64,
        ColumnType::Decimal { precision, scale } => {
            let (precision, scale, big) = decimal_digits(*precision, *scale);
            let scale = i8::try_from(scale)
                .map_err(|_| format!("a decimal scale of {scale} has no Arrow type"))?;
            match big {
                true => DataType::Decimal256(precision, scale),
                false => DataType::Decimal128(precision, scale),
            }
        }
        ColumnType::Float => DataType::Float64,
        ColumnType::Text | ColumnType::Varchar(_) | ColumnType::Uuid | ColumnType::Json => {
            DataType::Utf8
        }
        ColumnType::Date => DataType::Date32,
        ColumnType::Time => DataType::Time64(TimeUnit::Microsecond),
        ColumnType::TimestampTz => DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
        ColumnType::TimestampNtz => DataType::Timestamp(TimeUnit::Microsecond, None),
        ColumnType::Binary => DataType::Binary,
        ColumnType::Native(_) => match arrived {
            DataType::LargeUtf8 | DataType::Utf8View | DataType::Dictionary(_, _) => DataType::Utf8,
            DataType::LargeBinary | DataType::BinaryView => DataType::Binary,
            other => other.clone(),
        },
        ColumnType::Array(_) => return Err("an array arrives as a list".to_owned()),
    })
}

/// `batch` in the Arrow types the Write API reads for `columns`, matched by name, every field nullable and without metadata. A field not among `columns` is left out for a build table (which holds only the table's own columns) and an error otherwise; a value that does not fit its column is an error, never a silent NULL.
pub(crate) fn write_api_batch(
    batch: &RecordBatch,
    (columns, builds): (&[DesiredColumn], bool),
) -> Result<RecordBatch, String> {
    let options = CastOptions {
        safe: false,
        ..CastOptions::default()
    };
    let schema = batch.schema();
    let mut fields: Vec<Field> = Vec::with_capacity(batch.num_columns());
    let mut arrays: Vec<ArrayRef> = Vec::with_capacity(batch.num_columns());
    for (field, array) in schema.fields().iter().zip(batch.columns()) {
        let name = field.name();
        let column = match (columns.iter().find(|column| &column.name == name), builds) {
            (Some(column), _) => column,
            (None, true) => continue,
            (None, false) => {
                return Err(format!(
                    "column {name} is not among the table's planned columns"
                ));
            }
        };
        let target = write_api_type(&column.column_type, field.data_type()).map_err(|reason| {
            format!(
                "column {name} ({}) cannot load into BigQuery yet: {reason}",
                field.data_type()
            )
        })?;
        let cast = cast_with_options(array, &target, &options).map_err(|error| {
            format!("column {name} cannot be sent to BigQuery as {target}: {error}")
        })?;
        let cast = repeated(cast).map_err(|reason| format!("column {name}: {reason}"))?;
        fields.push(Field::new(name, cast.data_type().clone(), true));
        arrays.push(cast);
    }
    RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).map_err(|error| error.to_string())
}

/// A list column as a repeated field takes it: elements that cannot be NULL, a NULL list as an empty one, and a NULL element an error, as BigQuery's arrays hold no NULLs; any other column unchanged.
fn repeated(array: ArrayRef) -> Result<ArrayRef, String> {
    let DataType::List(item) = array.data_type() else {
        return Ok(array);
    };
    let lists = array.as_list::<i32>();
    for row in 0..lists.len() {
        if lists.is_valid(row) && lists.value(row).null_count() > 0 {
            return Err(format!(
                "row {}: an array holds a NULL element, which BigQuery arrays cannot hold",
                row + 1
            ));
        }
    }
    let lists = ListArray::try_new(
        Arc::new(item.as_ref().clone().with_nullable(false)),
        lists.offsets().clone(),
        lists.values().clone(),
        None,
    )
    .map_err(|error| error.to_string())?;
    Ok(Arc::new(lists))
}

/// `batch` cut into append requests of at most `max_bytes` each (sized by the rows it holds, not the buffers a slice shares), every one carrying the writer schema and no stream yet, with its rows; a single row larger than `max_bytes` goes alone, for BigQuery to refuse by name.
pub(crate) fn append_requests(
    batch: &RecordBatch,
    max_bytes: usize,
) -> Result<Vec<(AppendRowsRequest, usize)>, String> {
    let rows = batch.num_rows();
    let memory = batch
        .columns()
        .iter()
        .map(|column| column.to_data().get_slice_memory_size())
        .sum::<Result<usize, _>>()
        .map_err(|error| error.to_string())?
        .max(1);
    let mut step = (rows.saturating_mul(max_bytes) / memory).clamp(1, rows.max(1));
    let mut requests: Vec<(AppendRowsRequest, usize)> = Vec::new();
    let mut start = 0;
    while start < rows {
        let length = step.min(rows - start);
        let request = arrow_request("", 0, &batch.slice(start, length), true)
            .map_err(|error| error.to_string())?;
        if request.encoded_len() > max_bytes && length > 1 {
            step = length / 2;
            continue;
        }
        requests.push((request, length));
        start += length;
    }
    Ok(requests)
}