use crate::db::proto::millis;
use crate::errors::{
BigQueryError, BigQuerySchemaInferenceError, BigQuerySchemaInferenceErrorKind,
};
use crate::sql::{column_segment_violation, dotted_path};
use crate::BigQueryInstant;
use crate::BigQueryLabels;
use crate::{
BigQueryDecimalParams, BigQueryFieldMode, BigQueryFieldSchema, BigQueryFieldType,
BigQueryRangeElementType, BigQueryResult, BigQueryTableRef,
};
use serde::de::DeserializeOwned;
use std::collections::HashSet;
use std::fmt::{Display, Formatter};
use std::time::Duration;
#[derive(Debug, Clone, PartialEq)]
pub struct BigQuerySchemaColumn {
name: String,
kind: Option<ColumnKind>,
mode: BigQueryFieldMode,
description: Option<String>,
default_value: Option<String>,
renamed_from: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) enum ColumnKind {
Type(BigQueryFieldType),
Record(Vec<BigQuerySchemaColumn>),
Uninferred(BigQuerySchemaInferenceError),
}
impl BigQuerySchemaColumn {
fn new(name: String) -> Self {
Self {
name,
kind: None,
mode: BigQueryFieldMode::Nullable,
description: None,
default_value: None,
renamed_from: None,
}
}
pub(super) fn inferred(name: String, kind: ColumnKind, mode: BigQueryFieldMode) -> Self {
Self {
kind: Some(kind),
mode,
..Self::new(name)
}
}
pub fn of_type(self, field_type: BigQueryFieldType) -> Self {
Self {
kind: Some(ColumnKind::Type(field_type)),
..self
}
}
pub fn int64(self) -> Self {
self.of_type(BigQueryFieldType::Int64)
}
pub fn float64(self) -> Self {
self.of_type(BigQueryFieldType::Float64)
}
pub fn numeric(self) -> Self {
self.of_type(BigQueryFieldType::Numeric(None))
}
pub fn numeric_with(self, precision: u8, scale: u8) -> Self {
self.of_type(BigQueryFieldType::Numeric(Some(BigQueryDecimalParams {
precision,
scale,
})))
}
pub fn bignumeric(self) -> Self {
self.of_type(BigQueryFieldType::BigNumeric(None))
}
pub fn bignumeric_with(self, precision: u8, scale: u8) -> Self {
self.of_type(BigQueryFieldType::BigNumeric(Some(BigQueryDecimalParams {
precision,
scale,
})))
}
pub fn bool(self) -> Self {
self.of_type(BigQueryFieldType::Bool)
}
pub fn string(self) -> Self {
self.of_type(BigQueryFieldType::String { max_length: None })
}
pub fn string_with_max_length(self, max_length: u64) -> Self {
self.of_type(BigQueryFieldType::String {
max_length: Some(max_length),
})
}
pub fn bytes(self) -> Self {
self.of_type(BigQueryFieldType::Bytes { max_length: None })
}
pub fn bytes_with_max_length(self, max_length: u64) -> Self {
self.of_type(BigQueryFieldType::Bytes {
max_length: Some(max_length),
})
}
pub fn date(self) -> Self {
self.of_type(BigQueryFieldType::Date)
}
pub fn time(self) -> Self {
self.of_type(BigQueryFieldType::Time)
}
pub fn datetime(self) -> Self {
self.of_type(BigQueryFieldType::DateTime)
}
pub fn timestamp(self) -> Self {
self.of_type(BigQueryFieldType::Timestamp)
}
pub fn geography(self) -> Self {
self.of_type(BigQueryFieldType::Geography)
}
pub fn json(self) -> Self {
self.of_type(BigQueryFieldType::Json)
}
pub fn interval(self) -> Self {
self.of_type(BigQueryFieldType::Interval)
}
pub fn range(self, element: BigQueryRangeElementType) -> Self {
self.of_type(BigQueryFieldType::Range(element))
}
pub fn record<F>(self, fields: F) -> Self
where
F: FnOnce(BigQuerySchemaColumnsBuilder) -> Vec<BigQuerySchemaColumn>,
{
Self {
kind: Some(ColumnKind::Record(fields(BigQuerySchemaColumnsBuilder))),
..self
}
}
pub fn nullable(self) -> Self {
Self {
mode: BigQueryFieldMode::Nullable,
..self
}
}
pub fn required(self) -> Self {
Self {
mode: BigQueryFieldMode::Required,
..self
}
}
pub fn repeated(self) -> Self {
Self {
mode: BigQueryFieldMode::Repeated,
..self
}
}
pub fn description(self, description: impl Into<String>) -> Self {
Self {
description: Some(description.into()),
..self
}
}
pub fn default_value(self, expression: impl Into<String>) -> Self {
Self {
default_value: Some(expression.into()),
..self
}
}
pub fn renamed_from(self, old_name: impl Into<String>) -> Self {
Self {
renamed_from: Some(old_name.into()),
..self
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct BigQuerySchemaColumnsBuilder;
impl BigQuerySchemaColumnsBuilder {
pub fn field(&self, name: impl Into<String>) -> BigQuerySchemaColumn {
BigQuerySchemaColumn::new(name.into())
}
pub fn fields<I>(&self, columns: I) -> Vec<BigQuerySchemaColumn>
where
I: IntoIterator<Item = BigQuerySchemaColumn>,
{
columns.into_iter().collect()
}
pub fn from_type<T: DeserializeOwned>(&self) -> BigQuerySchemaColumns {
BigQuerySchemaColumns::infer::<T>()
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct BigQuerySchemaColumns {
pub(super) columns: Vec<BigQuerySchemaColumn>,
pub(super) problem: Option<BigQuerySchemaInferenceError>,
}
impl BigQuerySchemaColumns {
pub fn with<F>(mut self, path: impl Into<String>, column: F) -> Self
where
F: FnOnce(BigQuerySchemaColumn) -> BigQuerySchemaColumn,
{
let path = path.into();
match self.column_mut(&path) {
Some(found) => {
let current = std::mem::replace(found, BigQuerySchemaColumn::new(String::new()));
let name = current.name.clone();
*found = BigQuerySchemaColumn {
name,
..column(current)
};
}
None => {
self.problem.get_or_insert(
BigQuerySchemaInferenceError::new(
BigQuerySchemaInferenceErrorKind::UnknownColumn,
)
.with_path(path),
);
}
}
self
}
pub(crate) fn checked(self) -> BigQueryResult<Vec<DeclaredColumn>> {
if let Some(problem) = self.problem {
return Err(BigQueryError::SchemaInferenceError(problem));
}
DeclaredColumn::checked(self.columns, "")
}
fn column_mut(&mut self, path: &str) -> Option<&mut BigQuerySchemaColumn> {
let mut segments = path.split('.');
let top = segments.next()?;
let mut column = self.columns.iter_mut().find(|column| column.name == top)?;
for segment in segments {
column = match &mut column.kind {
Some(ColumnKind::Record(fields)) => {
fields.iter_mut().find(|field| field.name == segment)?
}
_ => return None,
};
}
Some(column)
}
}
impl From<Vec<BigQuerySchemaColumn>> for BigQuerySchemaColumns {
fn from(columns: Vec<BigQuerySchemaColumn>) -> Self {
Self {
columns,
problem: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum BigQueryPartitionUnit {
Hour,
Day,
Month,
Year,
}
impl BigQueryPartitionUnit {
pub(crate) fn name(self) -> &'static str {
match self {
BigQueryPartitionUnit::Hour => "HOUR",
BigQueryPartitionUnit::Day => "DAY",
BigQueryPartitionUnit::Month => "MONTH",
BigQueryPartitionUnit::Year => "YEAR",
}
}
pub(crate) fn parse(name: &str) -> Option<Self> {
match name {
"HOUR" => Some(BigQueryPartitionUnit::Hour),
"DAY" => Some(BigQueryPartitionUnit::Day),
"MONTH" => Some(BigQueryPartitionUnit::Month),
"YEAR" => Some(BigQueryPartitionUnit::Year),
_ => None,
}
}
}
impl Display for BigQueryPartitionUnit {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.write_str(self.name())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum BigQueryPartitioning {
Time {
unit: BigQueryPartitionUnit,
column: Option<String>,
},
Range {
column: String,
start: i64,
end: i64,
interval: i64,
},
}
impl BigQueryPartitioning {
pub(crate) fn column(&self) -> Option<&str> {
match self {
BigQueryPartitioning::Time { column, .. } => column.as_deref(),
BigQueryPartitioning::Range { column, .. } => Some(column),
}
}
pub(crate) fn same_as(&self, other: &BigQueryPartitioning) -> bool {
let same_column = match (self.column(), other.column()) {
(Some(a), Some(b)) => a.eq_ignore_ascii_case(b),
(None, None) => true,
_ => false,
};
same_column
&& match (self, other) {
(
BigQueryPartitioning::Time { unit: a, .. },
BigQueryPartitioning::Time { unit: b, .. },
) => a == b,
(
BigQueryPartitioning::Range {
start: s1,
end: e1,
interval: i1,
..
},
BigQueryPartitioning::Range {
start: s2,
end: e2,
interval: i2,
..
},
) => (s1, e1, i1) == (s2, e2, i2),
_ => false,
}
}
}
impl Display for BigQueryPartitioning {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
BigQueryPartitioning::Time {
unit,
column: Some(column),
} => write!(f, "{unit} on `{column}`"),
BigQueryPartitioning::Time { unit, column: None } => {
write!(f, "{unit} by ingestion time")
}
BigQueryPartitioning::Range {
column,
start,
end,
interval,
} => write!(f, "RANGE on `{column}` from {start} to {end} by {interval}"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BigQueryRecreatePolicy {
IfEmpty,
DangerouslyWithDataLoss,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct DeclaredColumn {
pub field: BigQueryFieldSchema,
pub renamed_from: Option<String>,
}
#[derive(Debug, Clone)]
pub(crate) struct BigQueryTableDeclarationDraft {
pub table: BigQueryTableRef,
pub columns: BigQuerySchemaColumns,
pub primary_key: Option<Vec<String>>,
pub partitioning: Option<BigQueryPartitioning>,
pub partition_expiration: Option<Duration>,
pub clustering: Option<Vec<String>>,
pub description: Option<String>,
pub labels: BigQueryLabels,
pub expiration: Option<BigQueryInstant>,
pub allow_widening: bool,
pub prune: bool,
pub recreate: Option<BigQueryRecreatePolicy>,
pub snapshot_first: bool,
}
impl BigQueryTableDeclarationDraft {
pub(crate) fn new(table: BigQueryTableRef) -> Self {
Self {
table,
columns: Vec::new().into(),
primary_key: None,
partitioning: None,
partition_expiration: None,
clustering: None,
description: None,
labels: BigQueryLabels::new(),
expiration: None,
allow_widening: false,
prune: false,
recreate: None,
snapshot_first: false,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct BigQueryTableDeclaration {
pub(crate) table: BigQueryTableRef,
pub(crate) columns: Vec<DeclaredColumn>,
pub(crate) primary_key: Option<Vec<String>>,
pub(crate) partitioning: Option<BigQueryPartitioning>,
pub(crate) partition_expiration: Option<Duration>,
pub(crate) clustering: Option<Vec<String>>,
pub(crate) description: Option<String>,
pub(crate) labels: BigQueryLabels,
pub(crate) expiration: Option<BigQueryInstant>,
pub(crate) allow_widening: bool,
pub(crate) prune: bool,
pub(crate) recreate: Option<BigQueryRecreatePolicy>,
pub(crate) snapshot_first: bool,
}
impl BigQueryTableDeclaration {
pub fn table(&self) -> &BigQueryTableRef {
&self.table
}
}
fn column_name_violation(name: &str) -> Option<String> {
if name.contains('.') {
Some("contains `.`; declare a nested field inside `record(..)`".into())
} else {
column_segment_violation(name)
}
}
fn check_name(field: &str, name: &str) -> BigQueryResult<()> {
match column_name_violation(name) {
Some(violation) => Err(BigQueryError::invalid_parameters(
field,
format!("the column name {name:?} {violation}"),
)),
None => Ok(()),
}
}
impl DeclaredColumn {
fn checked(columns: Vec<BigQuerySchemaColumn>, path: &str) -> BigQueryResult<Vec<Self>> {
let mut seen = HashSet::new();
let mut out = Vec::with_capacity(columns.len());
for column in columns {
let at = dotted_path(path, &column.name);
check_name("columns", &column.name)?;
if !seen.insert(column.name.to_ascii_lowercase()) {
return Err(BigQueryError::invalid_parameters(
"columns",
format!("the column `{at}` is declared twice (names ignore case)"),
));
}
if let Some(old) = &column.renamed_from {
if !path.is_empty() {
return Err(BigQueryError::invalid_parameters(
"renamed_from",
format!(
"`{at}` is a nested field; BigQuery renames top-level columns only"
),
));
}
check_name("renamed_from", old)?;
}
let field_type = match column.kind {
None => {
return Err(BigQueryError::invalid_parameters(
"columns",
format!("the column `{at}` has no type"),
))
}
Some(ColumnKind::Uninferred(error)) => {
return Err(BigQueryError::SchemaInferenceError(error))
}
Some(ColumnKind::Type(BigQueryFieldType::Struct(fields))) => {
for field in &fields {
check_name("columns", &field.name)?;
}
BigQueryFieldType::Struct(fields)
}
Some(ColumnKind::Type(field_type)) => field_type,
Some(ColumnKind::Record(fields)) => BigQueryFieldType::Struct(
Self::checked(fields, &at)?
.into_iter()
.map(|c| c.field)
.collect(),
),
};
out.push(Self {
field: BigQueryFieldSchema {
name: column.name,
field_type,
mode: column.mode,
description: column.description,
default_value_expression: column.default_value,
},
renamed_from: column.renamed_from,
});
}
Ok(out)
}
}
impl TryFrom<BigQueryTableDeclarationDraft> for BigQueryTableDeclaration {
type Error = BigQueryError;
fn try_from(draft: BigQueryTableDeclarationDraft) -> Result<Self, Self::Error> {
let columns = draft.columns.checked()?;
if let Some(old) = columns
.iter()
.filter_map(|c| c.renamed_from.as_ref())
.find(|old| {
columns
.iter()
.any(|c| c.field.name.eq_ignore_ascii_case(old))
})
{
return Err(BigQueryError::invalid_parameters(
"renamed_from",
format!("`{old}` is renamed from and declared as a column at once"),
));
}
if let Some(column) = draft.partitioning.as_ref().and_then(|p| p.column()) {
if !columns
.iter()
.any(|c| c.field.name.eq_ignore_ascii_case(column))
{
return Err(BigQueryError::invalid_parameters(
"partition_by",
format!("the partitioning column `{column}` is not declared"),
));
}
}
if draft.partition_expiration.is_some() && draft.partitioning.is_none() {
return Err(BigQueryError::invalid_parameters(
"partition_expiration",
"needs a partition_by_* declaration",
));
}
if draft.clustering.as_ref().is_some_and(Vec::is_empty) {
return Err(BigQueryError::invalid_parameters(
"cluster_by",
"needs at least one column; prune_undeclared() removes undeclared clustering",
));
}
if draft.primary_key.as_ref().is_some_and(Vec::is_empty) {
return Err(BigQueryError::invalid_parameters(
"primary_key",
"needs at least one column; prune_undeclared() removes an undeclared key",
));
}
if draft.snapshot_first && draft.recreate.is_none() {
return Err(BigQueryError::invalid_parameters(
"snapshot_first",
"needs recreate_if_empty() or dangerously_recreate_with_data_loss()",
));
}
Ok(Self {
table: draft.table,
columns,
primary_key: draft.primary_key,
partitioning: draft.partitioning,
partition_expiration: draft
.partition_expiration
.map(|duration| {
millis::<i64>("partition_expiration", duration)
.map(|millis| Duration::from_millis(millis.unsigned_abs()))
})
.transpose()?,
clustering: draft.clustering,
description: draft.description,
labels: draft.labels,
expiration: draft
.expiration
.map(|instant| {
BigQueryInstant::from_millisecond(instant.as_millisecond()).map_err(|err| {
BigQueryError::invalid_parameters("expiration", format!("{instant}: {err}"))
})
})
.transpose()?,
allow_widening: draft.allow_widening,
prune: draft.prune,
recreate: draft.recreate,
snapshot_first: draft.snapshot_first,
})
}
}