use std::sync::Arc;
use arrow_array::builder::{
BinaryBuilder, BooleanBuilder, Date32Builder, FixedSizeBinaryBuilder, Float32Builder,
Float64Builder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, StringBuilder,
Time32MillisecondBuilder, Time32SecondBuilder, Time64MicrosecondBuilder,
Time64NanosecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder, UInt8Builder,
};
use arrow_array::{ArrayRef, RecordBatch};
use arrow_odbc::odbc_api::buffers::Indicator;
use arrow_odbc::odbc_api::parameter::{Binary, Text, VarCell, VarKind, WideText};
use arrow_odbc::odbc_api::sys::{Date, Time, Timestamp};
use arrow_odbc::odbc_api::{Bit, Cursor, CursorRow, Nullable, Pod};
use arrow_schema::{DataType, Field, SchemaRef, TimeUnit};
use chrono::{NaiveDate, NaiveTime, Timelike};
use crate::odbc_source::constants::{KEPT_VALUE_BUFFER, VALUE_OVERHEAD};
use crate::odbc_source::errors::OdbcSourceError;
use crate::odbc_source::models::ValueLimits;
struct ValueColumn {
name: String,
values: Values,
bytes: Vec<u8>,
wide: Vec<u16>,
}
enum Values {
Boolean(BooleanBuilder),
Int8(Int8Builder),
Int16(Int16Builder),
Int32(Int32Builder),
Int64(Int64Builder),
UInt8(UInt8Builder),
Float32(Float32Builder),
Float64(Float64Builder),
Date(Date32Builder),
TimeSeconds(Time32SecondBuilder),
TimeMilliseconds(Time32MillisecondBuilder),
TimeMicroseconds(Time64MicrosecondBuilder),
TimeNanoseconds(Time64NanosecondBuilder),
TimestampMicroseconds(TimestampMicrosecondBuilder, Option<Arc<str>>),
TimestampNanoseconds(TimestampNanosecondBuilder, Option<Arc<str>>),
Text(StringBuilder),
Binary(BinaryBuilder),
FixedBinary(FixedSizeBinaryBuilder, i32),
}
pub(crate) fn read_values<C: Cursor>(
(mut cursor, schema): (C, &SchemaRef),
(budget, max_rows): (usize, usize),
(limits, table): (&ValueLimits, &str),
mut send: impl FnMut(RecordBatch) -> bool,
) -> Result<(), OdbcSourceError> {
let mut columns: Vec<ValueColumn> = schema
.fields()
.iter()
.map(|field| ValueColumn::new(field))
.collect::<Result<_, _>>()?;
loop {
let (mut rows, mut bytes, mut ended) = (0usize, 0usize, false);
while rows < max_rows.max(1) && bytes < budget {
let Some(mut row) = cursor.next_row()? else {
ended = true;
break;
};
for (column, number) in columns.iter_mut().zip(1u16..) {
let (next, size) = column.read(row, number, limits)?;
row = next;
if size > budget {
tracing::warn!(
table,
column = %column.name,
bytes = size,
budget,
"a value is larger than the table's fetch buffer (max_batch_memory); it loads, and its fetch ends after its row"
);
}
bytes = bytes.saturating_add(size + VALUE_OVERHEAD);
}
rows += 1;
}
if rows > 0 {
tracing::debug!(table, rows, bytes, budget, "fetched rows value by value");
let arrays: Vec<ArrayRef> = columns.iter_mut().map(ValueColumn::finish).collect();
let batch = RecordBatch::try_new(schema.clone(), arrays)
.map_err(|error| OdbcSourceError::Invalid(error.to_string()))?;
if !send(batch) {
return Ok(());
}
}
if ended {
return Ok(());
}
}
}
impl ValueColumn {
fn new(field: &Field) -> Result<Self, OdbcSourceError> {
let values = match field.data_type() {
DataType::Boolean => Values::Boolean(BooleanBuilder::new()),
DataType::Int8 => Values::Int8(Int8Builder::new()),
DataType::Int16 => Values::Int16(Int16Builder::new()),
DataType::Int32 => Values::Int32(Int32Builder::new()),
DataType::Int64 => Values::Int64(Int64Builder::new()),
DataType::UInt8 => Values::UInt8(UInt8Builder::new()),
DataType::Float32 => Values::Float32(Float32Builder::new()),
DataType::Float64 => Values::Float64(Float64Builder::new()),
DataType::Date32 => Values::Date(Date32Builder::new()),
DataType::Time32(TimeUnit::Second) => Values::TimeSeconds(Time32SecondBuilder::new()),
DataType::Time32(TimeUnit::Millisecond) => {
Values::TimeMilliseconds(Time32MillisecondBuilder::new())
}
DataType::Time64(TimeUnit::Microsecond) => {
Values::TimeMicroseconds(Time64MicrosecondBuilder::new())
}
DataType::Time64(TimeUnit::Nanosecond) => {
Values::TimeNanoseconds(Time64NanosecondBuilder::new())
}
DataType::Timestamp(TimeUnit::Microsecond, zone) => {
Values::TimestampMicroseconds(TimestampMicrosecondBuilder::new(), zone.clone())
}
DataType::Timestamp(TimeUnit::Nanosecond, zone) => {
Values::TimestampNanoseconds(TimestampNanosecondBuilder::new(), zone.clone())
}
DataType::Utf8 => Values::Text(StringBuilder::new()),
DataType::Binary => Values::Binary(BinaryBuilder::new()),
DataType::FixedSizeBinary(length) => {
Values::FixedBinary(FixedSizeBinaryBuilder::new(*length), *length)
}
other => {
return Err(OdbcSourceError::Invalid(format!(
"column {} is fetched as {other}, which SQLLoad doesn't read value by value",
field.name()
)));
}
};
Ok(Self {
name: field.name().clone(),
values,
bytes: Vec::new(),
wide: Vec::new(),
})
}
fn read<'r>(
&mut self,
mut row: CursorRow<'r>,
number: u16,
limits: &ValueLimits,
) -> Result<(CursorRow<'r>, usize), OdbcSourceError> {
let row_ref = &mut row;
let name = self.name.as_str();
let failed = |error: arrow_odbc::odbc_api::Error| {
OdbcSourceError::Invalid(format!("column {name}: ODBC: {error}"))
};
let size = match &mut self.values {
Values::Boolean(builder) => {
builder.append_option(
fixed::<Bit>(row_ref, number)
.map_err(failed)?
.map(Bit::as_bool),
);
size_of::<bool>()
}
Values::Int8(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<i8>()
}
Values::Int16(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<i16>()
}
Values::Int32(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<i32>()
}
Values::Int64(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<i64>()
}
Values::UInt8(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<u8>()
}
Values::Float32(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<f32>()
}
Values::Float64(builder) => {
builder.append_option(fixed(row_ref, number).map_err(failed)?);
size_of::<f64>()
}
Values::Date(builder) => {
let days = match fixed::<Date>(row_ref, number).map_err(failed)? {
Some(date) => Some(epoch_days(&date).ok_or_else(|| invalid(name, "date"))?),
None => None,
};
builder.append_option(days);
size_of::<i32>()
}
Values::TimeSeconds(builder) => {
let time = fixed::<Time>(row_ref, number).map_err(failed)?;
builder.append_option(time.map(|time| {
(i32::from(time.hour) * 60 + i32::from(time.minute)) * 60
+ i32::from(time.second)
}));
size_of::<i32>()
}
Values::TimeMilliseconds(builder) => {
let time = time_text((&mut self.bytes, &self.name), row_ref, number)?;
builder.append_option(time.map(|(seconds, nanoseconds)| {
i32::try_from(seconds * 1_000 + nanoseconds / 1_000_000).unwrap_or(i32::MAX)
}));
size_of::<i32>()
}
Values::TimeMicroseconds(builder) => {
let time = time_text((&mut self.bytes, &self.name), row_ref, number)?;
builder.append_option(
time.map(|(seconds, nanoseconds)| seconds * 1_000_000 + nanoseconds / 1_000),
);
size_of::<i64>()
}
Values::TimeNanoseconds(builder) => {
let time = time_text((&mut self.bytes, &self.name), row_ref, number)?;
builder.append_option(
time.map(|(seconds, nanoseconds)| seconds * 1_000_000_000 + nanoseconds),
);
size_of::<i64>()
}
Values::TimestampMicroseconds(builder, _) => {
let micros = match fixed::<Timestamp>(row_ref, number).map_err(failed)? {
Some(time) => {
Some(epoch_micros(&time).ok_or_else(|| invalid(name, "timestamp"))?)
}
None => None,
};
builder.append_option(micros);
size_of::<i64>()
}
Values::TimestampNanoseconds(builder, _) => {
let nanos = match fixed::<Timestamp>(row_ref, number).map_err(failed)? {
Some(time) => Some(
epoch_nanos(&time)
.ok_or_else(|| invalid(name, "timestamp in nanoseconds"))?,
),
None => None,
};
builder.append_option(nanos);
size_of::<i64>()
}
Values::Text(_) => self.read_text(row_ref, number, limits.max_text_size)?,
Values::Binary(_) | Values::FixedBinary(..) => {
self.read_binary(row_ref, number, limits.max_binary_size)?
}
};
Ok((row, size))
}
fn read_text(
&mut self,
row: &mut CursorRow<'_>,
number: u16,
cap: Option<usize>,
) -> Result<usize, OdbcSourceError> {
let Values::Text(builder) = &mut self.values else {
return Ok(0);
};
let name = self.name.as_str();
let size = match cfg!(target_os = "windows") {
true => {
let capped = (cap, "max_text_size");
let wide = std::mem::take(&mut self.wide);
let present;
(self.wide, present) = row.read_capped::<WideText>(number, wide, (name, capped))?;
if !present {
builder.append_null();
return Ok(0);
}
let text = String::from_utf16(&self.wide).map_err(|_| invalid(name, "text"))?;
over_cap(name, (text.len(), true), cap, "max_text_size")?;
builder.append_value(&text);
self.wide = kept(std::mem::take(&mut self.wide));
text.len()
}
false => {
let capped = (cap, "max_text_size");
let bytes = std::mem::take(&mut self.bytes);
let present;
(self.bytes, present) = row.read_capped::<Text>(number, bytes, (name, capped))?;
if !present {
builder.append_null();
return Ok(0);
}
over_cap(name, (self.bytes.len(), true), cap, "max_text_size")?;
let text = std::str::from_utf8(&self.bytes).map_err(|_| {
OdbcSourceError::Invalid(format!("column {name} holds text that is not UTF-8"))
})?;
builder.append_value(text);
let size = self.bytes.len();
self.bytes = kept(std::mem::take(&mut self.bytes));
size
}
};
Ok(size)
}
fn read_binary(
&mut self,
row: &mut CursorRow<'_>,
number: u16,
cap: Option<usize>,
) -> Result<usize, OdbcSourceError> {
let name = self.name.as_str();
let capped = (cap, "max_binary_size");
let bytes = std::mem::take(&mut self.bytes);
let present;
(self.bytes, present) = row.read_capped::<Binary>(number, bytes, (name, capped))?;
over_cap(name, (self.bytes.len(), true), cap, "max_binary_size")?;
let size = self.bytes.len();
match (&mut self.values, present) {
(Values::Binary(builder), true) => builder.append_value(&self.bytes),
(Values::Binary(builder), false) => builder.append_null(),
(Values::FixedBinary(builder, width), true) => {
builder.append_value(&self.bytes).map_err(|_| {
OdbcSourceError::Invalid(format!(
"column {name} holds a value of {size} bytes, not its {width}"
))
})?;
}
(Values::FixedBinary(builder, _), false) => builder.append_null(),
_ => {}
}
self.bytes = kept(std::mem::take(&mut self.bytes));
Ok(size)
}
fn finish(&mut self) -> ArrayRef {
match &mut self.values {
Values::Boolean(builder) => Arc::new(builder.finish()),
Values::Int8(builder) => Arc::new(builder.finish()),
Values::Int16(builder) => Arc::new(builder.finish()),
Values::Int32(builder) => Arc::new(builder.finish()),
Values::Int64(builder) => Arc::new(builder.finish()),
Values::UInt8(builder) => Arc::new(builder.finish()),
Values::Float32(builder) => Arc::new(builder.finish()),
Values::Float64(builder) => Arc::new(builder.finish()),
Values::Date(builder) => Arc::new(builder.finish()),
Values::TimeSeconds(builder) => Arc::new(builder.finish()),
Values::TimeMilliseconds(builder) => Arc::new(builder.finish()),
Values::TimeMicroseconds(builder) => Arc::new(builder.finish()),
Values::TimeNanoseconds(builder) => Arc::new(builder.finish()),
Values::TimestampMicroseconds(builder, zone) => {
Arc::new(builder.finish().with_timezone_opt(zone.clone()))
}
Values::TimestampNanoseconds(builder, zone) => {
Arc::new(builder.finish().with_timezone_opt(zone.clone()))
}
Values::Text(builder) => Arc::new(builder.finish()),
Values::Binary(builder) => Arc::new(builder.finish()),
Values::FixedBinary(builder, _) => Arc::new(builder.finish()),
}
}
}
fn time_text(
(bytes, name): (&mut Vec<u8>, &str),
row: &mut CursorRow<'_>,
number: u16,
) -> Result<Option<(i64, i64)>, OdbcSourceError> {
let present = row
.get_text(number, bytes)
.map_err(|error| OdbcSourceError::Invalid(format!("column {name}: ODBC: {error}")))?;
if !present {
return Ok(None);
}
let text = String::from_utf8_lossy(bytes);
let time =
NaiveTime::parse_from_str(text.trim(), "%H:%M:%S%.f").map_err(|_| invalid(name, "time"))?;
Ok(Some((
i64::from(time.num_seconds_from_midnight()),
i64::from(time.nanosecond()),
)))
}
fn fixed<T: Pod>(
row: &mut CursorRow<'_>,
number: u16,
) -> Result<Option<T>, arrow_odbc::odbc_api::Error> {
let mut value = Nullable::<T>::null();
row.get_data(number, &mut value)?;
Ok(value.into_opt())
}
fn day(year: i16, month: u16, day: u16) -> Option<NaiveDate> {
NaiveDate::from_ymd_opt(i32::from(year), u32::from(month), u32::from(day))
}
fn moment(time: &Timestamp) -> Option<chrono::NaiveDateTime> {
day(time.year, time.month, time.day)?.and_hms_nano_opt(
u32::from(time.hour),
u32::from(time.minute),
u32::from(time.second),
time.fraction,
)
}
fn epoch_days(date: &Date) -> Option<i32> {
Some(day(date.year, date.month, date.day)?.to_epoch_days())
}
fn epoch_micros(time: &Timestamp) -> Option<i64> {
Some(moment(time)?.and_utc().timestamp_micros())
}
fn epoch_nanos(time: &Timestamp) -> Option<i64> {
moment(time)?.and_utc().timestamp_nanos_opt()
}
fn invalid(name: &str, what: &str) -> OdbcSourceError {
OdbcSourceError::Invalid(format!(
"column {name} holds a value that is not a valid {what}"
))
}
fn over_cap(
name: &str,
(length, exact): (usize, bool),
cap: Option<usize>,
setting: &str,
) -> Result<(), OdbcSourceError> {
let at_least = match exact {
true => "",
false => "at least ",
};
match cap {
Some(cap) if length > cap => Err(OdbcSourceError::Invalid(format!(
"column {name} holds a value of {at_least}{length} bytes, longer than {setting} = {cap}; a value longer than its cap fails the read rather than being cut, so raise {setting} on the source, or remove it"
))),
_ => Ok(()),
}
}
trait CappedValue {
fn read_capped<K: VarKind>(
&mut self,
number: u16,
buffer: Vec<K::Element>,
capped: (&str, (Option<usize>, &str)),
) -> Result<(Vec<K::Element>, bool), OdbcSourceError>;
}
impl CappedValue for CursorRow<'_> {
fn read_capped<K: VarKind>(
&mut self,
number: u16,
mut buffer: Vec<K::Element>,
(name, (cap, setting)): (&str, (Option<usize>, &str)),
) -> Result<(Vec<K::Element>, bool), OdbcSourceError> {
let failed = |error: arrow_odbc::odbc_api::Error| {
OdbcSourceError::Invalid(format!("column {name}: ODBC: {error}"))
};
let element = size_of::<K::Element>();
if buffer.capacity() == 0 {
buffer.reserve(256);
}
buffer.resize(buffer.capacity(), K::ZERO);
let mut start = 0;
loop {
let mut piece = VarCell::<&mut [K::Element], K>::from_buffer(
&mut buffer[start..],
Indicator::NoTotal,
);
self.get_data(number, &mut piece).map_err(failed)?;
let (complete, indicator) = (piece.is_complete(), piece.indicator());
let Some(fetched) = piece.len_in_bytes() else {
buffer.clear();
return Ok((buffer, false));
};
if complete {
buffer.truncate(start + fetched / element);
return Ok((buffer, true));
}
let known = match indicator {
Indicator::Length(length) => (start + length / element, true),
_ => (start + fetched / element, false),
};
over_cap(name, known, cap, setting)?;
let old = buffer.len();
let grown = match indicator {
Indicator::Length(length) => old + (length - fetched) / element,
_ => old * 2,
};
buffer.resize(grown, K::ZERO);
start = old - K::TERMINATING_ZEROES;
}
}
}
fn kept<T>(buffer: Vec<T>) -> Vec<T> {
match buffer.capacity() > KEPT_VALUE_BUFFER {
true => Vec::new(),
false => buffer,
}
}