sqlload 0.60.0

SQL data loading tooling.
//! The Arrow reader over an ODBC cursor, and what its failures say.

use std::sync::Arc;

use arrow_odbc::odbc_api::Cursor;
use arrow_odbc::{ColumnFailure, OdbcReaderBuilder};
use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};

use crate::odbc_source::constants::TOO_LARGE_VALUE;
use crate::odbc_source::errors::OdbcSourceError;
use crate::odbc_source::models::OdbcReader;

/// The reader of `cursor`'s rows as `schema` (from `fetch_schema`) in batches of `batch_rows`, each buffer capped by the reader's limits; a column the driver gives no length for is refused, naming the setting that bounds it. The rows per batch are chosen by the caller from its memory budget, so arrow-odbc's own byte limit is lifted.
pub(crate) fn batch_reader<C: Cursor>(
    reader: &OdbcReader,
    cursor: C,
    (dbms, schema, batch_rows): (String, SchemaRef, usize),
) -> Result<arrow_odbc::OdbcReader<C>, OdbcSourceError> {
    let mut builder = OdbcReaderBuilder::new();
    builder
        .with_schema(schema)
        .with_dbms_name(dbms)
        .with_max_num_rows_per_batch(batch_rows.max(1))
        .with_max_bytes_per_batch(usize::MAX)
        .with_fallibale_allocations(true);
    if let Some(size) = reader.limits.max_text_size {
        builder.with_max_text_size(size);
    }
    if let Some(size) = reader.limits.max_binary_size {
        builder.with_max_binary_size(size);
    }
    builder.build(cursor).map_err(|error| match &error {
        arrow_odbc::Error::ColumnFailure {
            name,
            source: ColumnFailure::ZeroSizedColumn { .. } | ColumnFailure::UnknownStringLength { .. },
            ..
        } => OdbcSourceError::Invalid(format!(
            "the driver gives no length for column {name}; set max_text_size (or max_binary_size, for binary columns) on the source"
        )),
        _ => OdbcSourceError::Arrow(error),
    })
}

/// A failed fetch, with the setting to raise when a value outgrew its buffer.
pub(crate) fn fetch_error(text: &str) -> OdbcSourceError {
    match text.contains(TOO_LARGE_VALUE) {
        true => OdbcSourceError::Invalid(format!(
            "{text}; a value longer than its buffer fails the read rather than being cut, so raise max_text_size or max_binary_size on the source"
        )),
        false => OdbcSourceError::Invalid(text.to_owned()),
    }
}

/// `schema` with every decimal field fetched as text, and every timestamp coarser than microseconds fetched in microseconds: a driver may describe fewer fractional digits than its values hold (MySQL Connector/ODBC describes every `DATETIME(n)` with none), and the reader would drop the rest.
pub(crate) fn fetch_schema(schema: &Schema) -> SchemaRef {
    let fields: Vec<Field> = schema
        .fields()
        .iter()
        .map(|field| match field.data_type() {
            DataType::Decimal128(..) => field.as_ref().clone().with_data_type(DataType::Utf8),
            DataType::Timestamp(TimeUnit::Second | TimeUnit::Millisecond, zone) => field
                .as_ref()
                .clone()
                .with_data_type(DataType::Timestamp(TimeUnit::Microsecond, zone.clone())),
            _ => field.as_ref().clone(),
        })
        .collect();
    Arc::new(Schema::new(fields))
}