use crate::types::kind::{ARROW_EXTENSION_NAME, GOOGLE_SQL_TYPE};
use crate::{
BigQueryDecimalParams, BigQueryFieldMode, BigQueryFieldSchema, BigQueryFieldType,
BigQueryRangeElementType, BigQueryResult, BigQuerySchemaColumns, BigQueryTableSchema,
};
use arrow_schema::{DataType, Field, Fields, IntervalUnit, Schema, TimeUnit};
use std::collections::HashMap;
const NUMERIC_DEFAULT: BigQueryDecimalParams = BigQueryDecimalParams {
precision: 38,
scale: 9,
};
const BIGNUMERIC_DEFAULT: BigQueryDecimalParams = BigQueryDecimalParams {
precision: 76,
scale: 38,
};
impl BigQuerySchemaColumns {
pub(super) fn table_schema(self) -> BigQueryResult<BigQueryTableSchema> {
Ok(BigQueryTableSchema {
fields: self
.checked()?
.into_iter()
.map(|column| column.field)
.collect(),
})
}
}
impl BigQueryTableSchema {
pub(crate) fn arrow_read_schema(&self) -> Schema {
Schema::new(
self.fields
.iter()
.map(BigQueryFieldSchema::arrow_read_field)
.collect::<Vec<_>>(),
)
}
}
impl BigQueryFieldSchema {
fn arrow_read_field(&self) -> Field {
let (data_type, metadata) = self.field_type.arrow_read_type();
match self.mode {
BigQueryFieldMode::Repeated => Field::new_list(
self.name.clone(),
Field::new("item", data_type, true),
false,
),
BigQueryFieldMode::Nullable => Field::new(self.name.clone(), data_type, true),
BigQueryFieldMode::Required => Field::new(self.name.clone(), data_type, false),
}
.with_metadata(metadata)
}
}
impl BigQueryFieldType {
pub(super) fn arrow_read_type(&self) -> (DataType, HashMap<String, String>) {
let extension =
|name: &str| HashMap::from([(ARROW_EXTENSION_NAME.to_string(), name.to_string())]);
match self {
BigQueryFieldType::Int64 => (DataType::Int64, HashMap::new()),
BigQueryFieldType::Float64 => (DataType::Float64, HashMap::new()),
BigQueryFieldType::Bool => (DataType::Boolean, HashMap::new()),
BigQueryFieldType::String { .. } => (DataType::Utf8, HashMap::new()),
BigQueryFieldType::Bytes { .. } => (DataType::Binary, HashMap::new()),
BigQueryFieldType::Json => (DataType::Utf8, extension("google:sqlType:json")),
BigQueryFieldType::Geography => {
let mut metadata = extension("google:sqlType:geography");
metadata.insert(
"ARROW:extension:metadata".to_string(),
r#"{"encoding": "WKT"}"#.to_string(),
);
(DataType::Utf8, metadata)
}
BigQueryFieldType::Date => (DataType::Date32, HashMap::new()),
BigQueryFieldType::Time => (DataType::Time64(TimeUnit::Microsecond), HashMap::new()),
BigQueryFieldType::DateTime => (
DataType::Timestamp(TimeUnit::Microsecond, None),
extension("google:sqlType:datetime"),
),
BigQueryFieldType::Timestamp => (
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
HashMap::new(),
),
BigQueryFieldType::Numeric(params) => {
let params = params.unwrap_or(NUMERIC_DEFAULT);
let scale = i8::try_from(params.scale).unwrap_or(i8::MAX);
(
DataType::Decimal128(params.precision, scale),
HashMap::new(),
)
}
BigQueryFieldType::BigNumeric(params) => {
let params = params.unwrap_or(BIGNUMERIC_DEFAULT);
let scale = i8::try_from(params.scale).unwrap_or(i8::MAX);
(
DataType::Decimal256(params.precision, scale),
HashMap::new(),
)
}
BigQueryFieldType::Interval => (
DataType::Interval(IntervalUnit::MonthDayNano),
extension("google:sqlType:interval"),
),
BigQueryFieldType::Range(element) => {
let bound = element.arrow_read_type();
(
DataType::Struct(Fields::from(vec![
Field::new("start", bound.clone(), true),
Field::new("end", bound, true),
])),
HashMap::from([(GOOGLE_SQL_TYPE.to_string(), "range".to_string())]),
)
}
BigQueryFieldType::Struct(fields) => (
DataType::Struct(
fields
.iter()
.map(BigQueryFieldSchema::arrow_read_field)
.collect(),
),
HashMap::new(),
),
}
}
}
impl BigQueryRangeElementType {
fn arrow_read_type(self) -> DataType {
match self {
BigQueryRangeElementType::Date => DataType::Date32,
BigQueryRangeElementType::DateTime => DataType::Timestamp(TimeUnit::Microsecond, None),
BigQueryRangeElementType::Timestamp => {
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()))
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::testkit::field;
use proptest::prelude::*;
fn mode() -> impl Strategy<Value = BigQueryFieldMode> {
prop_oneof![
Just(BigQueryFieldMode::Nullable),
Just(BigQueryFieldMode::Required),
Just(BigQueryFieldMode::Repeated),
]
}
fn field_type() -> impl Strategy<Value = BigQueryFieldType> {
let leaf = prop_oneof![
Just(BigQueryFieldType::Int64),
Just(BigQueryFieldType::Float64),
Just(BigQueryFieldType::Bool),
Just(BigQueryFieldType::String { max_length: None }),
Just(BigQueryFieldType::Bytes { max_length: None }),
Just(BigQueryFieldType::Date),
Just(BigQueryFieldType::Time),
Just(BigQueryFieldType::DateTime),
Just(BigQueryFieldType::Timestamp),
Just(BigQueryFieldType::Numeric(None)),
Just(BigQueryFieldType::BigNumeric(None)),
Just(BigQueryFieldType::Geography),
Just(BigQueryFieldType::Json),
Just(BigQueryFieldType::Interval),
Just(BigQueryFieldType::Range(BigQueryRangeElementType::Date)),
Just(BigQueryFieldType::Range(BigQueryRangeElementType::DateTime)),
Just(BigQueryFieldType::Range(
BigQueryRangeElementType::Timestamp
)),
];
leaf.prop_recursive(2, 12, 4, |inner| {
columns(inner).prop_map(BigQueryFieldType::Struct)
})
}
fn columns(
field_type: impl Strategy<Value = BigQueryFieldType>,
) -> impl Strategy<Value = Vec<BigQueryFieldSchema>> {
proptest::collection::vec((field_type, mode()), 1..5).prop_map(|columns| {
columns
.into_iter()
.enumerate()
.map(|(index, (field_type, mode))| {
field(&format!("column_{index}"), field_type, mode)
})
.collect()
})
}
proptest! {
#[test]
fn the_read_layout_reads_back_as_its_schema(fields in columns(field_type())) {
let schema = BigQueryTableSchema { fields };
let read_back = BigQueryTableSchema::from_arrow(&schema.arrow_read_schema());
prop_assert_eq!(read_back.ok(), Some(schema));
}
}
}