use std::sync::Arc;
use arrow_array::builder::{
BinaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, Float64Builder, Int64Builder,
StringBuilder, TimestampMicrosecondBuilder,
};
use arrow_array::cast::AsArray;
use arrow_array::types::Float64Type;
use arrow_array::{Array, ArrayRef, NullArray, TimestampMicrosecondArray};
use arrow_schema::{DataType, Field};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use chrono::DateTime;
use crate::column_types::constants::MAX_DECIMAL_PRECISION;
use crate::column_types::main::combined_type::combined_type;
use crate::column_types::main::portable_type::portable_type;
use crate::column_types::models::ColumnType;
use crate::row_batches::constants::{INT_DIGITS, MIN_TIMESTAMP_LENGTH, SECONDS_PER_DAY, UTC_ZONE};
use crate::row_batches::models::Cell;
pub(crate) fn column_type(cells: &[Cell]) -> (Option<ColumnType>, Vec<Option<i64>>) {
let mut seen: Option<ColumnType> = None;
let mut micros: Vec<Option<i64>> = Vec::new();
for (row, cell) in cells.iter().enumerate() {
let parsed = match cell {
Cell::Text(text) => timestamp_micros(text),
_ => None,
};
let Some(value_type) = cell_type(cell, parsed.is_some()) else {
continue;
};
let merged = match seen {
Some(seen) => merged_type(seen, value_type),
None => value_type,
};
match merged {
ColumnType::Text => return (Some(ColumnType::Text), Vec::new()),
ColumnType::TimestampTz => {
micros.resize(row, None);
micros.push(parsed);
}
_ => {}
}
seen = Some(merged);
}
micros.resize(cells.len(), None);
let widest = seen.map(|column_type| match column_type {
ColumnType::Decimal { precision, scale } => decimal(
precision
.saturating_sub(scale)
.max(INT_DIGITS.min(MAX_DECIMAL_PRECISION.saturating_sub(scale))),
scale,
),
other => other,
});
(widest, micros)
}
fn merged_type(seen: ColumnType, value: ColumnType) -> ColumnType {
match (seen, value) {
(
ColumnType::Decimal {
precision: left,
scale: left_scale,
},
ColumnType::Decimal {
precision: right,
scale: right_scale,
},
) => decimal(
left.saturating_sub(left_scale)
.max(right.saturating_sub(right_scale)),
left_scale.max(right_scale),
),
(ColumnType::Decimal { precision, scale }, ColumnType::Int)
| (ColumnType::Int, ColumnType::Decimal { precision, scale }) => {
decimal(precision.saturating_sub(scale).max(INT_DIGITS), scale)
}
(seen, value) => combined_type(seen, value),
}
}
fn decimal(integer: u8, scale: u8) -> ColumnType {
match u16::from(integer) + u16::from(scale) <= u16::from(MAX_DECIMAL_PRECISION) {
true => ColumnType::Decimal {
precision: integer + scale,
scale,
},
false => ColumnType::Text,
}
}
fn decimal_type(text: &str) -> ColumnType {
let digits = text.trim_start_matches(['-', '+']);
let (integer, fraction) = digits.split_once('.').unwrap_or((digits, ""));
let integer = integer.trim_start_matches('0').len();
match (u8::try_from(integer), u8::try_from(fraction.len())) {
(Ok(integer), Ok(scale)) => decimal(integer, scale),
_ => ColumnType::Text,
}
}
fn scaled(text: &str, scale: u8) -> Option<i128> {
let (negative, digits) = match text.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, text.trim_start_matches('+')),
};
let (integer, fraction) = digits.split_once('.').unwrap_or((digits, ""));
let padding = usize::from(scale).checked_sub(fraction.len())?;
match format!("{integer}{fraction}{}", "0".repeat(padding)).parse::<i128>() {
Ok(value) if negative => Some(-value),
Ok(value) => Some(value),
Err(_) => None,
}
}
pub(crate) fn number_cell(text: &str) -> Cell {
match text.bytes().any(|byte| matches!(byte, b'.' | b'e' | b'E')) {
false => text
.parse::<i64>()
.map_or_else(|_| Cell::Number(text.to_owned()), Cell::Int),
true => match text.parse::<f64>() {
Ok(value) if value.is_finite() => Cell::Float(value),
Ok(_) | Err(_) => Cell::Number(text.to_owned()),
},
}
}
pub(crate) fn column_array(
cells: Vec<Cell>,
column_type: Option<&ColumnType>,
micros: Vec<Option<i64>>,
) -> ArrayRef {
match column_type {
None => Arc::new(NullArray::new(cells.len())),
Some(ColumnType::Bool) => {
let mut builder = BooleanBuilder::with_capacity(cells.len());
cells.iter().for_each(|cell| match cell {
Cell::Bool(value) => builder.append_value(*value),
_ => builder.append_null(),
});
Arc::new(builder.finish())
}
Some(ColumnType::Int) => {
let mut builder = Int64Builder::with_capacity(cells.len());
cells.iter().for_each(|cell| match cell {
Cell::Int(value) => builder.append_value(*value),
_ => builder.append_null(),
});
Arc::new(builder.finish())
}
Some(ColumnType::Float) => {
let mut builder = Float64Builder::with_capacity(cells.len());
cells.iter().for_each(|cell| match cell {
Cell::Int(value) => builder.append_value(*value as f64),
Cell::Float(value) => builder.append_value(*value),
_ => builder.append_null(),
});
Arc::new(builder.finish())
}
Some(ColumnType::TimestampTz) => {
Arc::new(TimestampMicrosecondArray::from(micros).with_timezone(UTC_ZONE))
}
Some(ColumnType::TimestampNtz) => {
let mut builder = TimestampMicrosecondBuilder::with_capacity(cells.len());
cells.iter().for_each(|cell| match cell {
Cell::TimestampNtz(value) => builder.append_value(*value),
_ => builder.append_null(),
});
Arc::new(builder.finish())
}
Some(ColumnType::Date) => {
let mut builder = Date32Builder::with_capacity(cells.len());
cells.iter().for_each(|cell| match cell {
Cell::Date(value) => builder.append_value(*value),
_ => builder.append_null(),
});
Arc::new(builder.finish())
}
Some(ColumnType::Decimal { precision, scale }) => {
let mut builder = Decimal128Builder::with_capacity(cells.len());
cells.iter().for_each(|cell| {
let value = match cell {
Cell::Decimal(text) => scaled(text, *scale),
Cell::Int(value) => scaled(&value.to_string(), *scale),
_ => None,
};
builder.append_option(value);
});
let arrow_scale = i8::try_from(*scale).unwrap_or(i8::MAX);
match builder
.finish()
.with_precision_and_scale(*precision, arrow_scale)
{
Ok(array) => Arc::new(array),
Err(_) => text_array(cells),
}
}
Some(ColumnType::Binary) => {
let mut builder = BinaryBuilder::with_capacity(cells.len(), cells.len() * 16);
cells.iter().for_each(|cell| match cell {
Cell::Binary(bytes) => builder.append_value(bytes),
_ => builder.append_null(),
});
Arc::new(builder.finish())
}
Some(_) => text_array(cells),
}
}
fn text_array(cells: Vec<Cell>) -> ArrayRef {
let mut builder = StringBuilder::with_capacity(cells.len(), cells.len() * 16);
for cell in cells {
match cell {
Cell::Null => builder.append_null(),
Cell::Bool(value) => builder.append_value(if value { "true" } else { "false" }),
Cell::Int(value) => builder.append_value(value.to_string()),
Cell::Float(value) => {
builder.append_value(float_text(value));
}
Cell::Number(text) | Cell::Text(text) | Cell::Json(text) | Cell::Decimal(text) => {
builder.append_value(text);
}
Cell::Date(days) => builder.append_value(
chrono::DateTime::from_timestamp(i64::from(days) * SECONDS_PER_DAY, 0)
.map(|time| time.date_naive().to_string())
.unwrap_or_default(),
),
Cell::TimestampNtz(value) => builder.append_value(
chrono::DateTime::from_timestamp_micros(value)
.map(|time| time.naive_utc().format("%Y-%m-%dT%H:%M:%S%.f").to_string())
.unwrap_or_default(),
),
Cell::Binary(bytes) => builder.append_value(STANDARD.encode(bytes)),
}
}
Arc::new(builder.finish())
}
fn cell_type(cell: &Cell, timestamp: bool) -> Option<ColumnType> {
match cell {
Cell::Null => None,
Cell::Bool(_) => Some(ColumnType::Bool),
Cell::Int(_) => Some(ColumnType::Int),
Cell::Float(_) => Some(ColumnType::Float),
Cell::Number(_) => Some(ColumnType::Text),
Cell::Text(_) if timestamp => Some(ColumnType::TimestampTz),
Cell::Text(_) => Some(ColumnType::Text),
Cell::Json(_) => Some(ColumnType::Json),
Cell::Date(_) => Some(ColumnType::Date),
Cell::TimestampNtz(_) => Some(ColumnType::TimestampNtz),
Cell::Decimal(text) => Some(decimal_type(text)),
Cell::Binary(_) => Some(ColumnType::Binary),
}
}
fn timestamp_micros(text: &str) -> Option<i64> {
let bytes = text.as_bytes();
let shaped = bytes.len() >= MIN_TIMESTAMP_LENGTH
&& bytes[4] == b'-'
&& bytes[7] == b'-'
&& matches!(bytes[10], b'T' | b't' | b' ');
match shaped {
true => match DateTime::parse_from_rfc3339(text) {
Ok(time) => Some(time.timestamp_micros()),
Err(_) => None,
},
false => None,
}
}
fn float_text(value: f64) -> String {
match serde_json::Number::from_f64(value) {
Some(number) => number.to_string(),
None => value.to_string(),
}
}
pub(crate) fn nonfinite_problem(
field: &Field,
column: &ArrayRef,
first_row: u64,
) -> Option<String> {
let exact = field.data_type().is_integer()
|| matches!(
field.data_type(),
DataType::Decimal128(..) | DataType::Decimal256(..)
);
if !exact {
return None;
}
let floats = column.as_primitive_opt::<Float64Type>()?;
let row =
(0..floats.len()).find(|row| floats.is_valid(*row) && !floats.value(*row).is_finite())?;
let value = floats.value(row);
let type_text = match portable_type(field) {
Ok(column_type) => column_type.to_string(),
Err(_) => field.data_type().to_string(),
};
Some(format!(
"column {}: row {} holds {value}, which {type_text} can't hold; pin {} as float, or set on_nonfinite = \"null\" to load NaN and infinities as NULL",
field.name(),
first_row + row as u64 + 1,
field.name()
))
}