use crate::db::proto::millis;
use crate::schema::declaration::DeclaredColumn;
use crate::schema::existing::ExistingTable;
use crate::schema::plan::ChangeStep;
use crate::sql::dotted_path;
use crate::BigQueryLabels;
use crate::{
BigQueryDecimalParams, BigQueryFieldMode, BigQueryFieldSchema, BigQueryFieldType,
BigQueryRecreate, BigQueryRecreateMethod, BigQueryRecreatePolicy, BigQueryRefusal,
BigQueryResult, BigQuerySchemaChange, BigQueryTableDeclaration, BigQueryTablePlan,
BigQueryTableRef, BigQueryTableTarget, BigQueryWithheldChange, BigQueryWithheldReason,
};
use gcloud_sdk::google::cloud::bigquery::v2;
use std::collections::HashSet;
impl BigQueryTableDeclaration {
pub(crate) fn plan(
&self,
table: BigQueryTableRef,
existing: Option<&ExistingTable>,
) -> BigQueryTablePlan {
let mut plan = BigQueryTablePlan {
table,
create: None,
changes: Vec::new(),
withheld: Vec::new(),
recreate: None,
impossible: Vec::new(),
refusal: None,
};
let Some(existing) = existing else {
plan.create = Some(self.target(None));
return plan;
};
let mut diff = Diff {
declaration: self,
changes: Vec::new(),
withheld: Vec::new(),
impossible: Vec::new(),
};
diff.columns(&existing.schema.fields);
diff.settings(existing);
diff.changes.sort_by_key(BigQuerySchemaChange::step);
plan.withheld = diff.withheld;
if diff.impossible.is_empty() {
plan.changes = diff.changes;
return plan;
}
let dangerous = match self.recreate {
Some(BigQueryRecreatePolicy::DangerouslyWithDataLoss) => true,
Some(BigQueryRecreatePolicy::IfEmpty) if existing.num_rows == Some(0) => false,
Some(BigQueryRecreatePolicy::IfEmpty) => {
plan.changes = diff.changes;
plan.impossible = diff.impossible;
plan.refusal = Some(BigQueryRefusal::NotEmpty {
num_rows: existing.num_rows,
});
return plan;
}
None => {
plan.changes = diff.changes;
plan.impossible = diff.impossible;
plan.refusal = Some(BigQueryRefusal::NoRecreateOptIn);
return plan;
}
};
let target = self.target(Some(existing));
let same_partitioning = match (&target.partitioning, &existing.partitioning) {
(Some(a), Some(b)) => a.same_as(b),
(None, None) => true,
_ => false,
};
let method = if same_partitioning && same_columns(&target.clustering, &existing.clustering)
{
BigQueryRecreateMethod::CreateOrReplace
} else {
BigQueryRecreateMethod::DropAndCreate
};
plan.recreate = Some(BigQueryRecreate {
method,
reasons: diff.impossible,
target,
num_rows: existing.num_rows,
num_bytes: existing.num_bytes,
row_access_policies: Vec::new(),
dangerous,
snapshot_first: self.snapshot_first,
});
plan
}
}
fn same_name(left: &str, right: &str) -> bool {
left.eq_ignore_ascii_case(right)
}
fn same_columns(left: &[String], right: &[String]) -> bool {
left.len() == right.len()
&& left
.iter()
.zip(right)
.all(|(left, right)| same_name(left, right))
}
fn find<'f>(fields: &'f [BigQueryFieldSchema], name: &str) -> Option<&'f BigQueryFieldSchema> {
fields.iter().find(|field| same_name(&field.name, name))
}
impl DeclaredColumn {
fn matched<'f>(&self, existing: &'f [BigQueryFieldSchema]) -> Option<&'f BigQueryFieldSchema> {
find(existing, &self.field.name).or_else(|| {
self.renamed_from
.as_deref()
.and_then(|old| find(existing, old))
})
}
}
impl BigQueryDecimalParams {
fn widens_to(self, to: Option<BigQueryDecimalParams>) -> bool {
match to {
None => true,
Some(to) => {
to.scale >= self.scale
&& i32::from(to.precision) - i32::from(to.scale)
>= i32::from(self.precision) - i32::from(self.scale)
}
}
}
}
fn length_widens(from: Option<u64>, to: Option<u64>) -> bool {
match (from, to) {
(Some(_), None) => true,
(Some(from), Some(to)) => to > from,
(None, _) => false,
}
}
impl BigQueryFieldType {
fn widens_to(&self, to: &BigQueryFieldType) -> bool {
use BigQueryFieldType::*;
match (self, to) {
(Int64, Numeric(None) | BigNumeric(None)) => true,
(Numeric(None), BigNumeric(None)) => true,
(Numeric(from), Numeric(to)) | (BigNumeric(from), BigNumeric(to)) => {
from.is_some_and(|from| from.widens_to(*to))
}
(String { max_length: from }, String { max_length: to })
| (Bytes { max_length: from }, Bytes { max_length: to }) => length_widens(*from, *to),
_ => false,
}
}
}
struct Diff<'a> {
declaration: &'a BigQueryTableDeclaration,
changes: Vec<BigQuerySchemaChange>,
withheld: Vec<BigQueryWithheldChange>,
impossible: Vec<BigQuerySchemaChange>,
}
impl Diff<'_> {
fn undeclared(&mut self, change: BigQuerySchemaChange) {
if !self.declaration.prune {
self.withheld.push(BigQueryWithheldChange {
change,
reason: BigQueryWithheldReason::PruneUndeclared,
});
} else if change.step() == ChangeStep::Impossible {
self.impossible.push(change);
} else {
self.changes.push(change);
}
}
fn add(&mut self, path: String, field: &BigQueryFieldSchema) {
let field = field.clone();
if field.mode == BigQueryFieldMode::Required {
self.impossible
.push(BigQuerySchemaChange::AddRequiredColumn { path, field });
} else {
self.changes
.push(BigQuerySchemaChange::AddColumn { path, field });
}
}
fn columns(&mut self, existing: &[BigQueryFieldSchema]) {
let declaration = self.declaration;
let mut matched_names = HashSet::new();
for column in &declaration.columns {
let Some(current) = column.matched(existing) else {
self.add(column.field.name.clone(), &column.field);
continue;
};
matched_names.insert(current.name.to_ascii_lowercase());
if !same_name(¤t.name, &column.field.name) {
self.changes.push(BigQuerySchemaChange::RenameColumn {
from: current.name.clone(),
to: column.field.name.clone(),
});
}
self.compare(
¤t.name,
Some(&column.field.name),
&column.field,
current,
);
}
for current in existing {
if !matched_names.contains(¤t.name.to_ascii_lowercase()) {
self.undeclared(BigQuerySchemaChange::DropColumn {
column: current.name.clone(),
field_type: current.field_type.clone(),
});
}
}
}
fn compare(
&mut self,
path: &str,
ddl_name: Option<&str>,
declared: &BigQueryFieldSchema,
current: &BigQueryFieldSchema,
) {
match (&declared.field_type, ¤t.field_type) {
(
BigQueryFieldType::Struct(declared_fields),
BigQueryFieldType::Struct(live_fields),
) => self.nested(path, declared_fields, live_fields),
(to, from) if to == from => {}
(to, from) if ddl_name.is_some() && from.widens_to(to) => {
let change = BigQuerySchemaChange::WidenColumn {
column: ddl_name.unwrap_or(path).to_string(),
from: from.clone(),
to: to.clone(),
};
if self.declaration.allow_widening {
self.changes.push(change);
} else {
self.withheld.push(BigQueryWithheldChange {
change,
reason: BigQueryWithheldReason::AllowWidening,
});
}
}
(to, from) => self
.impossible
.push(BigQuerySchemaChange::ChangeColumnType {
path: path.to_string(),
from: from.clone(),
to: to.clone(),
}),
}
match (declared.mode, current.mode) {
(to, from) if to == from => {}
(BigQueryFieldMode::Nullable, BigQueryFieldMode::Required) => {
self.changes.push(BigQuerySchemaChange::RelaxColumn {
path: path.to_string(),
})
}
(to, from) => self
.impossible
.push(BigQuerySchemaChange::ChangeColumnMode {
path: path.to_string(),
from,
to,
}),
}
if let Some(to) = &declared.description {
if current.description.as_deref().unwrap_or("") != to {
self.changes
.push(BigQuerySchemaChange::SetColumnDescription {
path: path.to_string(),
from: current.description.clone(),
to: to.clone(),
});
}
}
if let Some(to) = &declared.default_value_expression {
if current.default_value_expression.as_ref() != Some(to) {
self.changes.push(BigQuerySchemaChange::SetColumnDefault {
path: path.to_string(),
from: current.default_value_expression.clone(),
to: to.clone(),
});
}
}
}
fn nested(
&mut self,
prefix: &str,
declared: &[BigQueryFieldSchema],
existing: &[BigQueryFieldSchema],
) {
for field in declared {
let path = dotted_path(prefix, &field.name);
match find(existing, &field.name) {
Some(current) => self.compare(&path, None, field, current),
None => self.add(path, field),
}
}
for current in existing {
if find(declared, ¤t.name).is_none() {
self.undeclared(BigQuerySchemaChange::DropNestedField {
path: dotted_path(prefix, ¤t.name),
field_type: current.field_type.clone(),
});
}
}
}
fn settings(&mut self, existing: &ExistingTable) {
let declaration = self.declaration;
if let Some(to) = &declaration.description {
if existing.description.as_deref().unwrap_or("") != to {
self.changes.push(BigQuerySchemaChange::SetDescription {
from: existing.description.clone(),
to: to.clone(),
});
}
}
for (key, to) in declaration.labels.iter() {
let from = existing.labels.get(key);
if from != Some(to) {
self.changes.push(BigQuerySchemaChange::SetLabel {
key: key.to_string(),
from: from.map(str::to_string),
to: to.to_string(),
});
}
}
for (key, value) in existing.labels.iter() {
if declaration.labels.get(key).is_none() {
self.undeclared(BigQuerySchemaChange::RemoveLabel {
key: key.to_string(),
value: value.to_string(),
});
}
}
if let Some(to) = declaration.expiration {
if existing.expiration != Some(to) {
self.changes.push(BigQuerySchemaChange::SetExpiration {
from: existing.expiration,
to,
});
}
}
match (&declaration.partitioning, &existing.partitioning) {
(Some(declared), Some(current)) if declared.same_as(current) => {
if let Some(to) = declaration.partition_expiration {
if existing.partition_expiration != Some(to) {
self.changes
.push(BigQuerySchemaChange::SetPartitionExpiration {
from: existing.partition_expiration,
to,
});
}
}
}
(Some(declared), current) => {
self.impossible
.push(BigQuerySchemaChange::ChangePartitioning {
from: current.clone(),
to: Some(declared.clone()),
})
}
(None, Some(current)) => self.undeclared(BigQuerySchemaChange::ChangePartitioning {
from: Some(current.clone()),
to: None,
}),
(None, None) => {}
}
match &declaration.clustering {
Some(to) if !same_columns(to, &existing.clustering) => {
self.changes.push(BigQuerySchemaChange::SetClustering {
from: existing.clustering.clone(),
to: to.clone(),
})
}
Some(_) => {}
None if !existing.clustering.is_empty() => {
self.undeclared(BigQuerySchemaChange::RemoveClustering {
from: existing.clustering.clone(),
})
}
None => {}
}
match (&declaration.primary_key, &existing.primary_key) {
(Some(to), current)
if current
.as_ref()
.is_none_or(|current| !same_columns(to, current)) =>
{
self.changes.push(BigQuerySchemaChange::SetPrimaryKey {
from: current.clone(),
to: to.clone(),
})
}
(None, Some(current)) => self.undeclared(BigQuerySchemaChange::RemovePrimaryKey {
from: current.clone(),
}),
_ => {}
}
}
}
impl BigQueryFieldSchema {
fn merged_with(
&self,
current: Option<&BigQueryFieldSchema>,
prune: bool,
) -> BigQueryFieldSchema {
let mut field = self.clone();
let Some(current) = current else {
return field;
};
if field.description.is_none() {
field.description.clone_from(¤t.description);
}
if field.default_value_expression.is_none() {
field
.default_value_expression
.clone_from(¤t.default_value_expression);
}
if let (
BigQueryFieldType::Struct(declared_fields),
BigQueryFieldType::Struct(live_fields),
) = (&self.field_type, ¤t.field_type)
{
let mut fields: Vec<BigQueryFieldSchema> = declared_fields
.iter()
.map(|f| f.merged_with(find(live_fields, &f.name), prune))
.collect();
if !prune {
fields.extend(
live_fields
.iter()
.filter(|f| find(declared_fields, &f.name).is_none())
.cloned(),
);
}
field.field_type = BigQueryFieldType::Struct(fields);
}
field
}
}
impl BigQueryTableDeclaration {
fn target(&self, existing: Option<&ExistingTable>) -> BigQueryTableTarget {
let prune = self.prune;
let live_fields = existing.map_or(&[][..], |l| &l.schema.fields[..]);
let mut columns: Vec<BigQueryFieldSchema> = self
.columns
.iter()
.map(|c| c.field.merged_with(c.matched(live_fields), prune))
.collect();
let mut labels = BigQueryLabels::new();
let mut target = BigQueryTableTarget {
columns: Vec::new(),
primary_key: self.primary_key.clone(),
partitioning: self.partitioning.clone(),
partition_expiration: self.partition_expiration,
clustering: self.clustering.clone().unwrap_or_default(),
description: self.description.clone(),
labels: BigQueryLabels::new(),
expiration: self.expiration,
};
if let Some(existing) = existing {
if !prune {
let declared_live: HashSet<String> = self
.columns
.iter()
.filter_map(|c| c.matched(live_fields))
.map(|f| f.name.to_ascii_lowercase())
.collect();
columns.extend(
live_fields
.iter()
.filter(|f| !declared_live.contains(&f.name.to_ascii_lowercase()))
.cloned(),
);
labels.clone_from(&existing.labels);
if target.primary_key.is_none() {
target.primary_key.clone_from(&existing.primary_key);
}
if target.partitioning.is_none() {
target.partitioning.clone_from(&existing.partitioning);
}
if self.clustering.is_none() {
target.clustering.clone_from(&existing.clustering);
}
}
let same_partitioning = match (&target.partitioning, &existing.partitioning) {
(Some(a), Some(b)) => a.same_as(b),
_ => false,
};
if target.partition_expiration.is_none() && same_partitioning {
target.partition_expiration = existing.partition_expiration;
}
if target.description.is_none() {
target.description.clone_from(&existing.description);
}
if target.expiration.is_none() {
target.expiration = existing.expiration;
}
}
for (key, value) in self.labels.iter() {
labels.insert(key, value);
}
target.columns = columns;
target.labels = labels;
target
}
}
fn field_at<'f>(
fields: &'f mut [v2::TableFieldSchema],
path: &str,
) -> Option<&'f mut v2::TableFieldSchema> {
let (head, rest) = match path.split_once('.') {
Some((head, rest)) => (head, Some(rest)),
None => (path, None),
};
let field = fields.iter_mut().find(|f| same_name(&f.name, head))?;
match rest {
Some(rest) => field_at(&mut field.fields, rest),
None => Some(field),
}
}
fn add_field(schema: &mut v2::TableSchema, path: &str, field: v2::TableFieldSchema) {
match path.rsplit_once('.') {
Some((parent, _)) => {
if let Some(parent) = field_at(&mut schema.fields, parent) {
parent.fields.push(field);
}
}
None => schema.fields.push(field),
}
}
impl BigQueryTablePlan {
pub(crate) fn patch_body(&self, existing: &v2::Table) -> BigQueryResult<Option<v2::Table>> {
let changes = &self.changes;
let mut body = v2::Table::default();
let mut schema = existing.schema.clone().unwrap_or_default();
let mut schema_edited = false;
let mut any = false;
let foreign_keys = || {
existing
.table_constraints
.as_ref()
.map(|c| c.foreign_keys.clone())
.unwrap_or_default()
};
for change in changes.iter().filter(|c| c.step() == ChangeStep::Patch) {
any = true;
if matches!(
change,
BigQuerySchemaChange::AddColumn { .. }
| BigQuerySchemaChange::RelaxColumn { .. }
| BigQuerySchemaChange::SetColumnDescription { .. }
| BigQuerySchemaChange::SetColumnDefault { .. }
) {
schema_edited = true;
}
match change {
BigQuerySchemaChange::AddColumn { path, field } => {
let mut field = v2::TableFieldSchema::from(field);
field.default_value_expression = None;
add_field(&mut schema, path, field);
}
BigQuerySchemaChange::RelaxColumn { path } => {
if let Some(field) = field_at(&mut schema.fields, path) {
field.mode = "NULLABLE".into();
}
}
BigQuerySchemaChange::SetColumnDescription { path, to, .. } => {
if let Some(field) = field_at(&mut schema.fields, path) {
field.description = Some(to.clone());
}
}
BigQuerySchemaChange::SetColumnDefault { path, to, .. } => {
if let Some(field) = field_at(&mut schema.fields, path) {
field.default_value_expression = Some(to.clone());
}
}
BigQuerySchemaChange::SetDescription { to, .. } => {
body.description = Some(to.clone())
}
BigQuerySchemaChange::SetLabel { key, to, .. } => {
body.labels.insert(key.clone(), to.clone());
}
BigQuerySchemaChange::SetExpiration { to, .. } => {
body.expiration_time = Some(to.as_millisecond());
}
BigQuerySchemaChange::SetPartitionExpiration { to, .. } => {
body.time_partitioning = Some(v2::TimePartitioning {
expiration_ms: Some(millis("partition_expiration", *to)?),
..existing.time_partitioning.clone().unwrap_or_default()
});
}
BigQuerySchemaChange::SetClustering { to, .. } => {
body.clustering = Some(v2::Clustering { fields: to.clone() });
}
BigQuerySchemaChange::SetPrimaryKey { to, .. } => {
body.table_constraints = Some(v2::TableConstraints {
primary_key: Some(v2::PrimaryKey {
columns: to.clone(),
}),
foreign_keys: foreign_keys(),
});
}
BigQuerySchemaChange::RemovePrimaryKey { .. } => {
body.table_constraints = Some(v2::TableConstraints {
primary_key: None,
foreign_keys: foreign_keys(),
});
}
_ => {}
}
}
body.schema = schema_edited.then_some(schema);
Ok(any.then_some(body))
}
pub(crate) fn defaults_body(&self, latest: &v2::Table) -> Option<v2::Table> {
let changes = &self.changes;
let mut schema = latest.schema.clone().unwrap_or_default();
let mut any = false;
for change in changes {
if let BigQuerySchemaChange::AddColumn { path, field } = change {
if let Some(default) = &field.default_value_expression {
if let Some(field) = field_at(&mut schema.fields, path) {
field.default_value_expression = Some(default.clone());
any = true;
}
}
}
}
any.then(|| v2::Table {
schema: Some(schema),
..Default::default()
})
}
pub(crate) fn update_body(&self, latest: &v2::Table) -> Option<v2::Table> {
let changes = &self.changes;
let mut body = latest.clone();
let mut any = false;
for change in changes {
match change {
BigQuerySchemaChange::RemoveLabel { key, .. } => {
body.labels.remove(key);
any = true;
}
BigQuerySchemaChange::RemoveClustering { .. } => {
body.clustering = None;
any = true;
}
_ => {}
}
}
any.then_some(body)
}
}
#[cfg(test)]
mod tests;