use crate::admin::paged;
use crate::db::proto::timestamp_ms;
use crate::errors::BigQueryError;
use crate::BigQueryInstant;
use crate::{
BigQueryDatasetRef, BigQueryDb, BigQueryPartitioning, BigQueryResult, BigQueryTableRef,
BigQueryTableSchema,
};
use crate::{BigQueryLabels, BigQueryLocation};
use futures::stream::BoxStream;
use gcloud_sdk::google::cloud::bigquery::v2;
use std::fmt::{Display, Formatter};
use tracing::Span;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum BigQueryTableType {
Table,
View,
MaterializedView,
External,
Snapshot,
Other(String),
}
impl BigQueryTableType {
pub fn as_str(&self) -> &str {
match self {
Self::Table => "TABLE",
Self::View => "VIEW",
Self::MaterializedView => "MATERIALIZED_VIEW",
Self::External => "EXTERNAL",
Self::Snapshot => "SNAPSHOT",
Self::Other(name) => name,
}
}
fn known(name: &str) -> Option<Self> {
Some(match name {
"TABLE" => Self::Table,
"VIEW" => Self::View,
"MATERIALIZED_VIEW" => Self::MaterializedView,
"EXTERNAL" => Self::External,
"SNAPSHOT" => Self::Snapshot,
_ => return None,
})
}
}
impl From<&str> for BigQueryTableType {
fn from(name: &str) -> Self {
Self::known(name).unwrap_or_else(|| Self::Other(name.to_string()))
}
}
impl From<String> for BigQueryTableType {
fn from(name: String) -> Self {
Self::known(&name).unwrap_or(Self::Other(name))
}
}
impl Display for BigQueryTableType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct BigQueryTable {
pub reference: BigQueryTableRef,
pub table_type: Option<BigQueryTableType>,
pub schema: BigQueryTableSchema,
pub description: Option<String>,
pub labels: BigQueryLabels,
pub partitioning: Option<BigQueryPartitioning>,
pub clustering: Vec<String>,
pub num_rows: Option<u64>,
pub num_bytes: Option<i64>,
pub location: Option<BigQueryLocation>,
pub creation_time: Option<BigQueryInstant>,
pub last_modified_time: Option<BigQueryInstant>,
pub expiration_time: Option<BigQueryInstant>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryTableSummary {
pub reference: BigQueryTableRef,
pub table_type: Option<BigQueryTableType>,
pub labels: BigQueryLabels,
pub creation_time: Option<BigQueryInstant>,
pub expiration_time: Option<BigQueryInstant>,
}
fn table_reference(reference: Option<v2::TableReference>) -> BigQueryResult<BigQueryTableRef> {
reference
.ok_or_else(|| BigQueryError::unexpected_response("BigQuery returned no table_reference"))?
.try_into()
}
impl TryFrom<v2::Table> for BigQueryTable {
type Error = BigQueryError;
fn try_from(table: v2::Table) -> Result<Self, Self::Error> {
let (partitioning, _) = BigQueryPartitioning::from_table(&table)?;
let schema = table
.schema
.as_ref()
.map(BigQueryTableSchema::try_from)
.transpose()?
.unwrap_or(BigQueryTableSchema { fields: Vec::new() });
let last_modified_time = i64::try_from(table.last_modified_time).unwrap_or(i64::MAX);
Ok(Self {
reference: table_reference(table.table_reference)?,
table_type: Some(table.r#type).filter(|v| !v.is_empty()).map(Into::into),
schema,
description: table.description.filter(|v| !v.is_empty()),
labels: table.labels.into_iter().collect(),
partitioning,
clustering: table.clustering.map(|c| c.fields).unwrap_or_default(),
num_rows: table.num_rows,
num_bytes: table.num_bytes,
location: BigQueryLocation::reported(table.location),
creation_time: timestamp_ms("creation_time", table.creation_time)?,
last_modified_time: timestamp_ms("last_modified_time", last_modified_time)?,
expiration_time: timestamp_ms("expiration_time", table.expiration_time.unwrap_or(0))?,
})
}
}
impl TryFrom<v2::ListFormatTable> for BigQueryTableSummary {
type Error = BigQueryError;
fn try_from(table: v2::ListFormatTable) -> Result<Self, Self::Error> {
Ok(Self {
reference: table_reference(table.table_reference)?,
table_type: Some(table.r#type).filter(|v| !v.is_empty()).map(Into::into),
labels: table.labels.into_iter().collect(),
creation_time: timestamp_ms("creation_time", table.creation_time)?,
expiration_time: timestamp_ms("expiration_time", table.expiration_time)?,
})
}
}
impl BigQueryDb {
pub(crate) async fn get_table(
&self,
table: &BigQueryTableRef,
) -> BigQueryResult<BigQueryTable> {
self.get_table_body(table).await?.try_into()
}
pub(crate) async fn get_table_body(
&self,
table: &BigQueryTableRef,
) -> BigQueryResult<v2::Table> {
let ids = table.ids(&self.options().google_project_id);
let request = v2::GetTableRequest {
project_id: ids.project,
dataset_id: ids.dataset,
table_id: ids.table,
..Default::default()
};
self.retry(&table.admin_span(), "get a table", &request, |r| {
let mut client = self.table_client();
async move { client.get_table(r).await }
})
.await
}
pub(crate) async fn delete_table(&self, table: &BigQueryTableRef) -> BigQueryResult<()> {
let ids = table.ids(&self.options().google_project_id);
let request = v2::DeleteTableRequest {
project_id: ids.project,
dataset_id: ids.dataset,
table_id: ids.table,
};
self.retry(&table.admin_span(), "delete a table", &request, |r| {
let mut client = self.table_client();
async move { client.delete_table(r).await }
})
.await
}
pub(crate) fn list_tables(
&self,
dataset: &BigQueryDatasetRef,
page_size: Option<u32>,
) -> BoxStream<'static, BigQueryResult<BigQueryTableSummary>> {
let db = self.clone();
let project_id = dataset
.project_or(&self.options().google_project_id)
.to_string();
let dataset_id = dataset.dataset().to_string();
let span = tracing::debug_span!("BigQuery tables", "/bigquery/dataset" = %dataset);
paged(move |page_token| {
let db = db.clone();
let span = span.clone();
let request = v2::ListTablesRequest {
project_id: project_id.clone(),
dataset_id: dataset_id.clone(),
max_results: page_size,
page_token,
};
async move {
let page = db
.retry(&span, "list tables", &request, |r| {
let mut client = db.table_client();
async move { client.list_tables(r).await }
})
.await?;
let tables = page
.tables
.into_iter()
.map(TryInto::try_into)
.collect::<BigQueryResult<_>>()?;
Ok((tables, page.next_page_token))
}
})
}
}
impl BigQueryTableRef {
fn admin_span(&self) -> Span {
tracing::debug_span!("BigQuery table", "/bigquery/table" = %self)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn table_types_map_known_names_and_keep_unknown_ones() {
for name in ["TABLE", "VIEW", "MATERIALIZED_VIEW", "EXTERNAL", "SNAPSHOT"] {
let parsed = BigQueryTableType::from(name);
assert!(
!matches!(parsed, BigQueryTableType::Other(_)),
"{name} reads as {parsed:?}"
);
assert_eq!(parsed.as_str(), name);
}
let unknown = BigQueryTableType::from("CLONE".to_string());
assert_eq!(unknown, BigQueryTableType::Other("CLONE".into()));
assert_eq!(unknown.to_string(), "CLONE");
}
}