use std::sync::Arc;
use arrow_array::builder::{
Date32Builder, Int32Builder, Int64Builder, StringBuilder, TimestampMillisecondBuilder,
};
use arrow_array::{ArrayRef, RecordBatch};
use arrow_schema::{DataType, Field, Schema, TimeUnit};
use parquet::arrow::ArrowWriter;
use parquet::basic::{Compression, ZstdLevel};
use parquet::file::properties::WriterProperties;
use serde_json::{Map, Value};
use crate::config::models::ColumnConfig;
use crate::file_format::_helpers::columns::{is_semi_structured, parquet_kind};
use crate::file_format::models::ParquetKind;
const UNIX_EPOCH_DAYS_FROM_CE: i32 = 719_163;
enum ColumnBuilder {
Int32(Int32Builder),
Int64(Int64Builder),
Timestamp(TimestampMillisecondBuilder),
Date(Date32Builder),
Text { builder: StringBuilder, json: bool },
}
impl ColumnBuilder {
fn new(column: &ColumnConfig) -> Self {
match parquet_kind(column) {
ParquetKind::Int32 => Self::Int32(Int32Builder::new()),
ParquetKind::Int64 => Self::Int64(Int64Builder::new()),
ParquetKind::Timestamp => {
Self::Timestamp(TimestampMillisecondBuilder::new().with_timezone("UTC"))
}
ParquetKind::Date => Self::Date(Date32Builder::new()),
ParquetKind::Text => Self::Text {
builder: StringBuilder::new(),
json: is_semi_structured(&column.sql_type),
},
}
}
fn data_type(&self) -> DataType {
match self {
Self::Int32(_) => DataType::Int32,
Self::Int64(_) => DataType::Int64,
Self::Timestamp(_) => DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())),
Self::Date(_) => DataType::Date32,
Self::Text { .. } => DataType::Utf8,
}
}
fn push(&mut self, value: Option<&Value>) {
let value = value.filter(|value| !value.is_null());
match self {
Self::Int32(builder) => {
builder.append_option(integer(value).and_then(narrow));
}
Self::Int64(builder) => builder.append_option(integer(value)),
Self::Timestamp(builder) => builder.append_option(value.and_then(instant_ms)),
Self::Date(builder) => builder.append_option(value.and_then(epoch_days)),
Self::Text { builder, json } => {
builder.append_option(value.map(|value| text(value, *json)))
}
}
}
fn finish(&mut self) -> ArrayRef {
match self {
Self::Int32(builder) => Arc::new(builder.finish()),
Self::Int64(builder) => Arc::new(builder.finish()),
Self::Timestamp(builder) => Arc::new(builder.finish().with_timezone("UTC")),
Self::Date(builder) => Arc::new(builder.finish()),
Self::Text { builder, .. } => Arc::new(builder.finish()),
}
}
}
pub(crate) fn parquet_file(columns: &[ColumnConfig], lines: &[&str]) -> Result<Vec<u8>, String> {
let mut builders: Vec<ColumnBuilder> = columns.iter().map(ColumnBuilder::new).collect();
for line in lines {
let row: Map<String, Value> = serde_json::from_str(line)
.map_err(|error| format!("row is not a JSON object: {error}"))?;
for (column, builder) in columns.iter().zip(builders.iter_mut()) {
builder.push(row.get(&column.name));
}
}
let fields: Vec<Field> = columns
.iter()
.zip(&builders)
.map(|(column, builder)| Field::new(&column.name, builder.data_type(), true))
.collect();
let schema = Arc::new(Schema::new(fields));
let arrays: Vec<ArrayRef> = builders.iter_mut().map(ColumnBuilder::finish).collect();
let batch = RecordBatch::try_new(schema.clone(), arrays).map_err(|error| error.to_string())?;
let properties = WriterProperties::builder()
.set_compression(Compression::ZSTD(ZstdLevel::default()))
.build();
let mut output: Vec<u8> = Vec::new();
let mut writer = ArrowWriter::try_new(&mut output, schema, Some(properties))
.map_err(|error| error.to_string())?;
writer.write(&batch).map_err(|error| error.to_string())?;
writer.close().map_err(|error| error.to_string())?;
Ok(output)
}
fn integer(value: Option<&Value>) -> Option<i64> {
match value? {
Value::Number(number) => number.as_i64(),
Value::String(text) => match text.trim().parse::<i64>() {
Ok(number) => Some(number),
Err(_) => None,
},
_ => None,
}
}
fn narrow(number: i64) -> Option<i32> {
match i32::try_from(number) {
Ok(narrowed) => Some(narrowed),
Err(_) => None,
}
}
fn instant_ms(value: &Value) -> Option<i64> {
match value {
Value::String(text) => match chrono::DateTime::parse_from_rfc3339(text) {
Ok(instant) => Some(instant.timestamp_millis()),
Err(_) => None,
},
Value::Number(number) => number.as_i64(),
_ => None,
}
}
fn epoch_days(value: &Value) -> Option<i32> {
use chrono::Datelike;
let text = value.as_str()?;
match chrono::NaiveDate::parse_from_str(text, "%Y-%m-%d") {
Ok(date) => Some(date.num_days_from_ce() - UNIX_EPOCH_DAYS_FROM_CE),
Err(_) => None,
}
}
fn text(value: &Value, json: bool) -> String {
match value {
Value::String(text) if !json => text.clone(),
other => other.to_string(),
}
}