use crate::BigQueryLabels;
use crate::{
BigQueryFieldMode, BigQueryFieldSchema, BigQueryFieldType, BigQueryInstant,
BigQueryPartitioning, BigQueryTableRef,
};
use std::fmt::{self, Display, Formatter};
use std::time::Duration;
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BigQuerySchemaChange {
AddColumn {
path: String,
field: BigQueryFieldSchema,
},
RelaxColumn {
path: String,
},
SetColumnDescription {
path: String,
from: Option<String>,
to: String,
},
SetColumnDefault {
path: String,
from: Option<String>,
to: String,
},
SetDescription {
from: Option<String>,
to: String,
},
SetLabel {
key: String,
from: Option<String>,
to: String,
},
SetExpiration {
from: Option<BigQueryInstant>,
to: BigQueryInstant,
},
SetPartitionExpiration {
from: Option<Duration>,
to: Duration,
},
SetClustering {
from: Vec<String>,
to: Vec<String>,
},
SetPrimaryKey {
from: Option<Vec<String>>,
to: Vec<String>,
},
RemovePrimaryKey {
from: Vec<String>,
},
RemoveLabel {
key: String,
value: String,
},
RemoveClustering {
from: Vec<String>,
},
RenameColumn {
from: String,
to: String,
},
WidenColumn {
column: String,
from: BigQueryFieldType,
to: BigQueryFieldType,
},
DropColumn {
column: String,
field_type: BigQueryFieldType,
},
ChangeColumnType {
path: String,
from: BigQueryFieldType,
to: BigQueryFieldType,
},
ChangeColumnMode {
path: String,
from: BigQueryFieldMode,
to: BigQueryFieldMode,
},
AddRequiredColumn {
path: String,
field: BigQueryFieldSchema,
},
DropNestedField {
path: String,
field_type: BigQueryFieldType,
},
ChangePartitioning {
from: Option<BigQueryPartitioning>,
to: Option<BigQueryPartitioning>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum ChangeStep {
Patch,
Update,
Rename,
Widen,
Drop,
Impossible,
}
impl BigQuerySchemaChange {
pub(crate) fn step(&self) -> ChangeStep {
use BigQuerySchemaChange::*;
match self {
AddColumn { .. }
| RelaxColumn { .. }
| SetColumnDescription { .. }
| SetColumnDefault { .. }
| SetDescription { .. }
| SetLabel { .. }
| SetExpiration { .. }
| SetPartitionExpiration { .. }
| SetClustering { .. }
| SetPrimaryKey { .. }
| RemovePrimaryKey { .. } => ChangeStep::Patch,
RemoveLabel { .. } | RemoveClustering { .. } => ChangeStep::Update,
RenameColumn { .. } => ChangeStep::Rename,
WidenColumn { .. } => ChangeStep::Widen,
DropColumn { .. } => ChangeStep::Drop,
ChangeColumnType { .. }
| ChangeColumnMode { .. }
| AddRequiredColumn { .. }
| DropNestedField { .. }
| ChangePartitioning { .. } => ChangeStep::Impossible,
}
}
}
fn debug_or_none<T: fmt::Debug>(value: &Option<T>) -> String {
value
.as_ref()
.map_or_else(|| "none".to_string(), |v| format!("{v:?}"))
}
fn list(columns: &[String]) -> String {
if columns.is_empty() {
return "none".into();
}
let quoted: Vec<String> = columns.iter().map(|c| format!("`{c}`")).collect();
quoted.join(", ")
}
impl Display for BigQuerySchemaChange {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
use BigQuerySchemaChange::*;
match self {
AddColumn { path, field } => {
write!(
f,
"[patch] add column `{path}` {} {}",
field.field_type, field.mode
)?;
if let Some(default) = &field.default_value_expression {
write!(
f,
", then DEFAULT {default} in a second patch (existing rows stay NULL)"
)?;
}
Ok(())
}
RelaxColumn { path } => write!(f, "[patch] relax `{path}` REQUIRED to NULLABLE"),
SetColumnDescription { path, from, to } => write!(
f,
"[patch] description of `{path}`: {} to {to:?}",
debug_or_none(from)
),
SetColumnDefault { path, from, to } => {
write!(
f,
"[patch] default of `{path}`: {} to {to}",
debug_or_none(from)
)
}
SetDescription { from, to } => {
write!(
f,
"[patch] table description: {} to {to:?}",
debug_or_none(from)
)
}
SetLabel { key, from, to } => {
write!(
f,
"[patch] label {key:?}: {} to {to:?}",
debug_or_none(from)
)
}
SetExpiration { from, to } => write!(
f,
"[patch] table expiration: {} to {to}",
from.map_or_else(|| "none".to_string(), |t| t.to_string()),
),
SetPartitionExpiration { from, to } => write!(
f,
"[patch] partition expiration: {} to {} ms",
from.map_or_else(|| "none".to_string(), |d| format!("{} ms", d.as_millis())),
to.as_millis()
),
SetClustering { from, to } => {
write!(f, "[patch] clustering: {} to {}", list(from), list(to))
}
SetPrimaryKey { from, to } => write!(
f,
"[patch] primary key: {} to {}",
from.as_deref().map_or_else(|| "none".to_string(), list),
list(to)
),
RemovePrimaryKey { from } => {
write!(f, "[patch] remove undeclared primary key {}", list(from))
}
RemoveLabel { key, value } => {
write!(f, "[update] remove undeclared label {key:?} = {value:?}")
}
RemoveClustering { from } => {
write!(f, "[update] remove undeclared clustering {}", list(from))
}
RenameColumn { from, to } => write!(
f,
"[ddl] RENAME COLUMN `{from}` TO `{to}` (writers still sending `{from}` lose it)"
),
WidenColumn { column, from, to } => {
write!(f, "[ddl] widen `{column}` from {from} to {to}")
}
DropColumn { column, field_type } => write!(
f,
"[ddl] DROP COLUMN `{column}` {field_type}: DATA LOSS, every value in it is \
deleted"
),
ChangeColumnType { path, from, to } => {
write!(f, "change the type of `{path}` from {from} to {to}")
}
ChangeColumnMode { path, from, to } => {
write!(f, "change the mode of `{path}` from {} to {}", from, to)
}
AddRequiredColumn { path, field } => write!(
f,
"add REQUIRED column `{path}` {} to an existing table",
field.field_type
),
DropNestedField { path, field_type } => {
write!(f, "drop the nested field `{path}` {field_type}: DATA LOSS")
}
ChangePartitioning { from, to } => write!(
f,
"change the partitioning from {} to {}",
from.as_ref()
.map_or_else(|| "none".to_string(), ToString::to_string),
to.as_ref()
.map_or_else(|| "none".to_string(), ToString::to_string)
),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BigQueryWithheldReason {
PruneUndeclared,
AllowWidening,
}
impl Display for BigQueryWithheldReason {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.write_str(match self {
BigQueryWithheldReason::PruneUndeclared => "needs prune_undeclared()",
BigQueryWithheldReason::AllowWidening => "needs allow_widening()",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryWithheldChange {
pub change: BigQuerySchemaChange,
pub reason: BigQueryWithheldReason,
}
impl Display for BigQueryWithheldChange {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
write!(f, "{} ({})", self.change, self.reason)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryTableTarget {
pub columns: Vec<BigQueryFieldSchema>,
pub primary_key: Option<Vec<String>>,
pub partitioning: Option<BigQueryPartitioning>,
pub partition_expiration: Option<Duration>,
pub clustering: Vec<String>,
pub description: Option<String>,
pub labels: BigQueryLabels,
pub expiration: Option<BigQueryInstant>,
}
impl Display for BigQueryTableTarget {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
let columns: Vec<String> = self
.columns
.iter()
.map(|c| format!("`{}` {} {}", c.name, c.field_type, c.mode))
.collect();
write!(f, "columns {}", columns.join(", "))?;
if let Some(key) = &self.primary_key {
write!(f, "; primary key {}", list(key))?;
}
if let Some(partitioning) = &self.partitioning {
write!(f, "; partitioned {partitioning}")?;
}
if !self.clustering.is_empty() {
write!(f, "; clustered by {}", list(&self.clustering))?;
}
if !self.labels.is_empty() {
write!(f, "; labels {:?}", self.labels)?;
}
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum BigQueryRecreateMethod {
CreateOrReplace,
DropAndCreate,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryRecreate {
pub method: BigQueryRecreateMethod,
pub reasons: Vec<BigQuerySchemaChange>,
pub target: BigQueryTableTarget,
pub num_rows: Option<u64>,
pub num_bytes: Option<i64>,
pub row_access_policies: Vec<String>,
pub dangerous: bool,
pub snapshot_first: bool,
}
impl Display for BigQueryRecreate {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
let method = match self.method {
BigQueryRecreateMethod::CreateOrReplace => "CREATE OR REPLACE TABLE",
BigQueryRecreateMethod::DropAndCreate => "DROP TABLE + CREATE TABLE",
};
writeln!(
f,
" recreate with {method}{}, because:",
if self.dangerous {
" (dangerously_recreate_with_data_loss)"
} else {
" (recreate_if_empty)"
}
)?;
for reason in &self.reasons {
writeln!(f, " {reason}")?;
}
writeln!(
f,
" DATA LOSS: every row, num_rows {} and num_bytes {} per GetTable, which lags \
Storage Write by a minute or more",
self.num_rows
.map_or_else(|| "not reported".to_string(), |n| n.to_string()),
self.num_bytes
.map_or_else(|| "not reported".to_string(), |n| n.to_string()),
)?;
if self.row_access_policies.is_empty() {
writeln!(f, " loses: row access policies (none on the table)")?;
} else {
writeln!(
f,
" loses: row access policies {}",
list(&self.row_access_policies)
)?;
}
match self.method {
BigQueryRecreateMethod::CreateOrReplace => writeln!(f, " keeps: table IAM")?,
BigQueryRecreateMethod::DropAndCreate => writeln!(f, " loses: table IAM")?,
}
if self.snapshot_first {
writeln!(
f,
" snapshot first: CREATE SNAPSHOT TABLE, which misses rows still in the \
streaming buffer"
)?;
}
writeln!(
f,
" stop writers and readers first: open writers keep getting acks for lost rows"
)?;
writeln!(f, " table as recreated: {}", self.target)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum BigQueryRefusal {
NoRecreateOptIn,
NotEmpty {
num_rows: Option<u64>,
},
}
impl Display for BigQueryRefusal {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
BigQueryRefusal::NoRecreateOptIn => f.write_str(
"no recreate opt-in: recreate_if_empty() or dangerously_recreate_with_data_loss()",
),
BigQueryRefusal::NotEmpty { num_rows } => write!(
f,
"recreate_if_empty() needs num_rows == 0, GetTable reports {}",
num_rows.map_or_else(|| "nothing".to_string(), |n| n.to_string())
),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryTablePlan {
pub table: BigQueryTableRef,
pub create: Option<BigQueryTableTarget>,
pub changes: Vec<BigQuerySchemaChange>,
pub withheld: Vec<BigQueryWithheldChange>,
pub recreate: Option<BigQueryRecreate>,
pub impossible: Vec<BigQuerySchemaChange>,
pub refusal: Option<BigQueryRefusal>,
}
impl BigQueryTablePlan {
pub fn is_empty(&self) -> bool {
self.create.is_none()
&& self.changes.is_empty()
&& self.withheld.is_empty()
&& self.recreate.is_none()
&& self.impossible.is_empty()
}
}
fn write_section<T: Display>(f: &mut Formatter<'_>, label: &str, items: &[T]) -> fmt::Result {
if items.is_empty() {
return Ok(());
}
writeln!(f, " {label}: {}", items.len())?;
for item in items {
writeln!(f, " {item}")?;
}
Ok(())
}
impl Display for BigQueryTablePlan {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
if self.is_empty() {
return writeln!(f, "BigQuery table plan for {}: no changes", self.table);
}
writeln!(f, "BigQuery table plan for {}:", self.table)?;
if let Some(target) = &self.create {
writeln!(f, " create: {target}")?;
}
write_section(f, "changes, in write order", &self.changes)?;
write_section(f, "withheld", &self.withheld)?;
if let Some(recreate) = &self.recreate {
write!(f, "{recreate}")?;
}
write_section(f, "impossible in place", &self.impossible)?;
if let Some(refusal) = &self.refusal {
writeln!(f, " refused, sync writes nothing: {refusal}")?;
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum BigQueryDroppedData {
Column {
column: String,
},
Rows {
num_rows: Option<u64>,
num_bytes: Option<i64>,
},
}
impl Display for BigQueryDroppedData {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
BigQueryDroppedData::Column { column } => write!(f, "column `{column}`"),
BigQueryDroppedData::Rows {
num_rows,
num_bytes,
} => write!(
f,
"all rows: num_rows {}, num_bytes {} (GetTable lags Storage Write)",
num_rows.map_or_else(|| "not reported".to_string(), |n| n.to_string()),
num_bytes.map_or_else(|| "not reported".to_string(), |n| n.to_string()),
),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryTableSyncReport {
pub table: BigQueryTableRef,
pub created: Option<BigQueryTableTarget>,
pub applied: Vec<BigQuerySchemaChange>,
pub withheld: Vec<BigQueryWithheldChange>,
pub recreated: Option<BigQueryRecreate>,
pub snapshot: Option<BigQueryTableRef>,
pub dropped_data: Vec<BigQueryDroppedData>,
}
impl BigQueryTableSyncReport {
pub(crate) fn new(table: BigQueryTableRef) -> Self {
Self {
table,
created: None,
applied: Vec::new(),
withheld: Vec::new(),
recreated: None,
snapshot: None,
dropped_data: Vec::new(),
}
}
}
impl Display for BigQueryTableSyncReport {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
writeln!(f, "BigQuery table sync report for {}:", self.table)?;
if let Some(target) = &self.created {
writeln!(f, " created: {target}")?;
}
write_section(f, "applied", &self.applied)?;
write_section(f, "withheld", &self.withheld)?;
if let Some(snapshot) = &self.snapshot {
writeln!(f, " snapshot: {snapshot}")?;
}
if let Some(recreate) = &self.recreated {
write!(f, "{recreate}")?;
}
write_section(f, "dropped_data (DATA LOSS)", &self.dropped_data)?;
if self.created.is_none()
&& self.applied.is_empty()
&& self.withheld.is_empty()
&& self.recreated.is_none()
{
writeln!(f, " no changes")?;
}
Ok(())
}
}