use arrow_array::builder::StringBuilder;
use arrow_array::cast::AsArray;
use arrow_array::types::{
Date32Type, Decimal128Type, Float64Type, Int16Type, Int64Type, Time64MicrosecondType,
TimestampMicrosecondType,
};
use arrow_array::{Array, ArrayRef, RecordBatch};
use arrow_cast::display::{ArrayFormatter, FormatOptions};
use arrow_cast::{CastOptions, cast_with_options};
use arrow_schema::{DataType, TimeUnit};
use bytes::{BufMut, Bytes, BytesMut};
use crate::column_types::main::arrow_field::arrow_field;
use crate::column_types::models::ColumnType;
use crate::postgres_source::constants::{
COPY_SIGNATURE, COPY_TRAILER, NULL_FIELD_LENGTH, NUMERIC_DIGIT_WIDTH, NUMERIC_NEGATIVE,
NUMERIC_POSITIVE, POSTGRES_EPOCH_DAYS, POSTGRES_EPOCH_MICROS,
};
use crate::schema_plan::models::DesiredColumn;
#[derive(Debug, Clone, Copy)]
enum Wire {
Bool,
Int2,
Int8,
Numeric { scale: u8 },
Float8,
Text,
Date,
Time,
Timestamp,
Bytea,
}
fn wire(column_type: &ColumnType) -> (Wire, DataType) {
match column_type {
ColumnType::Bool => (Wire::Bool, DataType::Boolean),
ColumnType::SmallInt => (Wire::Int2, DataType::Int16),
ColumnType::Int => (Wire::Int8, DataType::Int64),
ColumnType::Decimal { precision, scale } => (
Wire::Numeric { scale: *scale },
DataType::Decimal128(*precision, i8::try_from(*scale).unwrap_or(i8::MAX)),
),
ColumnType::Float => (Wire::Float8, DataType::Float64),
ColumnType::Text
| ColumnType::Varchar(_)
| ColumnType::Json
| ColumnType::Uuid
| ColumnType::Native(_) => (Wire::Text, DataType::Utf8),
ColumnType::Date => (Wire::Date, DataType::Date32),
ColumnType::Time => (Wire::Time, DataType::Time64(TimeUnit::Microsecond)),
ColumnType::TimestampTz => (
Wire::Timestamp,
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
),
ColumnType::TimestampNtz => (
Wire::Timestamp,
DataType::Timestamp(TimeUnit::Microsecond, None),
),
ColumnType::Binary => (Wire::Bytea, DataType::Binary),
ColumnType::Array(_) => (Wire::Text, DataType::Utf8),
}
}
fn array_literals(array: &ArrayRef, column_type: &ColumnType) -> Result<ArrayRef, String> {
let strict = CastOptions {
safe: false,
..CastOptions::default()
};
let target = arrow_field("", column_type)
.ok_or("an array of this type has no Arrow form")?
.data_type()
.clone();
let lists = cast_with_options(array, &target, &strict).map_err(|error| error.to_string())?;
let lists = lists.as_list::<i32>();
let options = FormatOptions::default();
let mut texts = StringBuilder::with_capacity(lists.len(), lists.values().len() * 8);
for row in 0..lists.len() {
if lists.is_null(row) {
texts.append_null();
continue;
}
let values = lists.value(row);
let formatter = ArrayFormatter::try_new(values.as_ref(), &options)
.map_err(|error| error.to_string())?;
let elements: Vec<String> = (0..values.len())
.map(|position| match values.is_null(position) {
true => "NULL".to_owned(),
false => format!(
"\"{}\"",
formatter
.value(position)
.to_string()
.replace('\\', "\\\\")
.replace('"', "\\\"")
),
})
.collect();
texts.append_value(format!("{{{}}}", elements.join(",")));
}
Ok(std::sync::Arc::new(texts.finish()))
}
pub(crate) fn numeric_bytes(value: i128, scale: u8) -> Vec<u8> {
let width = NUMERIC_DIGIT_WIDTH as usize;
let scale = usize::from(scale);
let mut digits = value.unsigned_abs().to_string();
if digits.len() <= scale {
digits = format!("{}{digits}", "0".repeat(scale + 1 - digits.len()));
}
let (whole, fraction) = digits.split_at(digits.len() - scale);
let whole = format!(
"{}{whole}",
"0".repeat((width - whole.len() % width) % width)
);
let fraction = format!(
"{fraction}{}",
"0".repeat((width - fraction.len() % width) % width)
);
let group = |text: &str| -> Vec<i16> {
let mut groups: Vec<i16> = Vec::with_capacity(text.len() / width);
for chunk in text.as_bytes().chunks(width) {
let mut sum = 0i16;
for digit in chunk {
sum = sum * 10 + i16::from(digit - b'0');
}
groups.push(sum);
}
groups
};
let mut groups: Vec<i16> = group(&whole);
let mut weight = i16::try_from(groups.len()).unwrap_or(i16::MAX) - 1;
groups.extend(group(&fraction));
let leading = groups.iter().take_while(|group| **group == 0).count();
groups.drain(..leading);
weight -= i16::try_from(leading).unwrap_or(i16::MAX);
while groups.last() == Some(&0) {
groups.pop();
}
if groups.is_empty() {
weight = 0;
}
let sign = match value < 0 && !groups.is_empty() {
true => NUMERIC_NEGATIVE,
false => NUMERIC_POSITIVE,
};
let mut bytes: Vec<u8> = Vec::with_capacity(8 + groups.len() * 2);
bytes.extend_from_slice(
&i16::try_from(groups.len())
.unwrap_or(i16::MAX)
.to_be_bytes(),
);
bytes.extend_from_slice(&weight.to_be_bytes());
bytes.extend_from_slice(&sign.to_be_bytes());
bytes.extend_from_slice(&u16::try_from(scale).unwrap_or(u16::MAX).to_be_bytes());
for group in groups {
bytes.extend_from_slice(&group.to_be_bytes());
}
bytes
}
fn put_field(
out: &mut BytesMut,
(array, wire): (&ArrayRef, Wire),
row: usize,
) -> Result<(), String> {
if array.is_null(row) {
out.put_i32(NULL_FIELD_LENGTH);
return Ok(());
}
let out_of_range = || format!("row {row}: the value is out of Postgres's range");
let mut fixed = |bytes: &[u8]| {
out.put_i32(i32::try_from(bytes.len()).unwrap_or(i32::MAX));
out.put_slice(bytes);
};
match wire {
Wire::Bool => fixed(&[u8::from(array.as_boolean().value(row))]),
Wire::Int2 => fixed(&array.as_primitive::<Int16Type>().value(row).to_be_bytes()),
Wire::Int8 => fixed(&array.as_primitive::<Int64Type>().value(row).to_be_bytes()),
Wire::Float8 => fixed(&array.as_primitive::<Float64Type>().value(row).to_be_bytes()),
Wire::Numeric { scale } => fixed(&numeric_bytes(
array.as_primitive::<Decimal128Type>().value(row),
scale,
)),
Wire::Text => fixed(array.as_string::<i32>().value(row).as_bytes()),
Wire::Bytea => fixed(array.as_binary::<i32>().value(row)),
Wire::Date => fixed(
&array
.as_primitive::<Date32Type>()
.value(row)
.checked_sub(POSTGRES_EPOCH_DAYS)
.ok_or_else(out_of_range)?
.to_be_bytes(),
),
Wire::Time => fixed(
&array
.as_primitive::<Time64MicrosecondType>()
.value(row)
.to_be_bytes(),
),
Wire::Timestamp => fixed(
&array
.as_primitive::<TimestampMicrosecondType>()
.value(row)
.checked_sub(POSTGRES_EPOCH_MICROS)
.ok_or_else(out_of_range)?
.to_be_bytes(),
),
}
Ok(())
}
fn wire_columns(
batch: &RecordBatch,
columns: &[DesiredColumn],
) -> Result<Vec<(ArrayRef, Wire)>, String> {
let options = CastOptions {
safe: false,
..CastOptions::default()
};
let schema = batch.schema();
columns
.iter()
.map(|column| {
let index = schema
.index_of(&column.name)
.map_err(|_| format!("column {} is not in the batch", column.name))?;
let (wire, data_type) = wire(&column.column_type);
let arrived = batch.column(index);
let cast = match &column.column_type {
ColumnType::Array(_) => array_literals(arrived, &column.column_type),
_ => cast_with_options(arrived, &data_type, &options)
.map_err(|error| error.to_string()),
}
.map_err(|error| {
format!(
"column {} ({}) cannot be sent to Postgres as {}: {error}",
column.name,
arrived.data_type(),
column.column_type
)
})?;
Ok((cast, wire))
})
.collect()
}
pub(crate) fn copy_chunks(
batch: &RecordBatch,
columns: &[DesiredColumn],
chunk_bytes: usize,
) -> Result<Vec<Bytes>, String> {
let fields = wire_columns(batch, columns)?;
let count = i16::try_from(fields.len())
.map_err(|_| format!("{} columns are more than COPY takes", fields.len()))?;
let mut chunks: Vec<Bytes> = Vec::new();
let mut out = BytesMut::with_capacity(chunk_bytes.min(batch.get_array_memory_size() + 64));
out.put_slice(COPY_SIGNATURE);
out.put_i32(0);
out.put_i32(0);
for row in 0..batch.num_rows() {
out.put_i16(count);
for ((array, wire), column) in fields.iter().zip(columns) {
put_field(&mut out, (array, *wire), row)
.map_err(|reason| format!("column {}: {reason}", column.name))?;
}
if out.len() >= chunk_bytes {
chunks.push(out.split().freeze());
}
}
out.put_i16(COPY_TRAILER);
chunks.push(out.freeze());
Ok(chunks)
}