bigquery 0.10.0

Library provides a simple API for Google BigQuery based on the gRPC Storage Read/Write and v2 APIs
Documentation
//! Tables: get, delete and list, through the v2 `TableService`. Tables are created and changed
//! by the declarative schema API.

use crate::admin::paged;
use crate::db::proto::timestamp_ms;
use crate::errors::BigQueryError;
use crate::BigQueryInstant;
use crate::{
    BigQueryDatasetRef, BigQueryDb, BigQueryPartitioning, BigQueryResult, BigQueryTableRef,
    BigQueryTableSchema,
};
use crate::{BigQueryLabels, BigQueryLocation};
use futures::stream::BoxStream;
use gcloud_sdk::google::cloud::bigquery::v2;
use std::fmt::{Display, Formatter};
use tracing::Span;

/// What kind of table a table is, as BigQuery names it in `Table.type`: `TABLE`, `VIEW`,
/// `MATERIALIZED_VIEW`, `EXTERNAL` or `SNAPSHOT`.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum BigQueryTableType {
    /// A table that stores its rows.
    Table,
    /// A logical view.
    View,
    /// A materialized view.
    MaterializedView,
    /// A table over data outside BigQuery.
    External,
    /// A table snapshot.
    Snapshot,
    /// A type this crate does not know, as BigQuery named it.
    Other(String),
}

impl BigQueryTableType {
    /// The name as BigQuery writes it, such as `MATERIALIZED_VIEW`.
    pub fn as_str(&self) -> &str {
        match self {
            Self::Table => "TABLE",
            Self::View => "VIEW",
            Self::MaterializedView => "MATERIALIZED_VIEW",
            Self::External => "EXTERNAL",
            Self::Snapshot => "SNAPSHOT",
            Self::Other(name) => name,
        }
    }

    fn known(name: &str) -> Option<Self> {
        Some(match name {
            "TABLE" => Self::Table,
            "VIEW" => Self::View,
            "MATERIALIZED_VIEW" => Self::MaterializedView,
            "EXTERNAL" => Self::External,
            "SNAPSHOT" => Self::Snapshot,
            _ => return None,
        })
    }
}

impl From<&str> for BigQueryTableType {
    fn from(name: &str) -> Self {
        Self::known(name).unwrap_or_else(|| Self::Other(name.to_string()))
    }
}

impl From<String> for BigQueryTableType {
    fn from(name: String) -> Self {
        Self::known(&name).unwrap_or(Self::Other(name))
    }
}

impl Display for BigQueryTableType {
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
        f.write_str(self.as_str())
    }
}

/// A table as `GetTable` returns it.
#[derive(Debug, Clone, PartialEq)]
pub struct BigQueryTable {
    /// The table, with its project.
    pub reference: BigQueryTableRef,
    /// The kind of table.
    pub table_type: Option<BigQueryTableType>,
    /// The columns, with the v2 API's legacy type names normalised, so it compares with `==`
    /// against a schema from any other source.
    pub schema: BigQueryTableSchema,
    /// The description.
    pub description: Option<String>,
    /// The labels.
    pub labels: BigQueryLabels,
    /// The partitioning.
    pub partitioning: Option<BigQueryPartitioning>,
    /// The clustering columns, most important first.
    pub clustering: Vec<String>,
    /// The rows, as of BigQuery's last count. It lags Storage Write: rows written through
    /// COMMITTED or PENDING streams show after 60 to 70 seconds, default-stream rows can take
    /// more than 5 minutes.
    pub num_rows: Option<u64>,
    /// The logical bytes, with the same lag as `num_rows`.
    pub num_bytes: Option<i64>,
    /// Where the table is stored.
    pub location: Option<BigQueryLocation>,
    /// When the table was created.
    pub creation_time: Option<BigQueryInstant>,
    /// When the table was last changed.
    pub last_modified_time: Option<BigQueryInstant>,
    /// When BigQuery deletes the table.
    pub expiration_time: Option<BigQueryInstant>,
}

/// A table as `ListTables` returns it, without its schema and sizes: read one with
/// [`BigQueryTableSchemaBuilder::get`](crate::BigQueryTableSchemaBuilder::get) for those.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryTableSummary {
    /// The table, with its project.
    pub reference: BigQueryTableRef,
    /// The kind of table.
    pub table_type: Option<BigQueryTableType>,
    /// The labels.
    pub labels: BigQueryLabels,
    /// When the table was created.
    pub creation_time: Option<BigQueryInstant>,
    /// When BigQuery deletes the table.
    pub expiration_time: Option<BigQueryInstant>,
}

fn table_reference(reference: Option<v2::TableReference>) -> BigQueryResult<BigQueryTableRef> {
    reference
        .ok_or_else(|| BigQueryError::unexpected_response("BigQuery returned no table_reference"))?
        .try_into()
}

/// # Errors
/// [`BigQueryError::SystemError`] for a missing table reference,
/// [`BigQueryError::InvalidParametersError`] for an invalid one or
/// partitioning the crate does not model, and [`BigQueryError::DeserializeError`] for a column
/// type the crate does not handle or a time out of range.
impl TryFrom<v2::Table> for BigQueryTable {
    type Error = BigQueryError;

    fn try_from(table: v2::Table) -> Result<Self, Self::Error> {
        let (partitioning, _) = BigQueryPartitioning::from_table(&table)?;
        let schema = table
            .schema
            .as_ref()
            .map(BigQueryTableSchema::try_from)
            .transpose()?
            .unwrap_or(BigQueryTableSchema { fields: Vec::new() });
        // Past i64::MAX it is out of range for a timestamp too, and fails as one.
        let last_modified_time = i64::try_from(table.last_modified_time).unwrap_or(i64::MAX);
        Ok(Self {
            reference: table_reference(table.table_reference)?,
            table_type: Some(table.r#type).filter(|v| !v.is_empty()).map(Into::into),
            schema,
            description: table.description.filter(|v| !v.is_empty()),
            labels: table.labels.into_iter().collect(),
            partitioning,
            clustering: table.clustering.map(|c| c.fields).unwrap_or_default(),
            num_rows: table.num_rows,
            num_bytes: table.num_bytes,
            location: BigQueryLocation::reported(table.location),
            creation_time: timestamp_ms("creation_time", table.creation_time)?,
            last_modified_time: timestamp_ms("last_modified_time", last_modified_time)?,
            expiration_time: timestamp_ms("expiration_time", table.expiration_time.unwrap_or(0))?,
        })
    }
}

/// # Errors
/// [`BigQueryError::SystemError`] for a missing table reference,
/// [`BigQueryError::InvalidParametersError`] for an invalid one, and
/// [`BigQueryError::DeserializeError`] for a time out of range.
impl TryFrom<v2::ListFormatTable> for BigQueryTableSummary {
    type Error = BigQueryError;

    fn try_from(table: v2::ListFormatTable) -> Result<Self, Self::Error> {
        Ok(Self {
            reference: table_reference(table.table_reference)?,
            table_type: Some(table.r#type).filter(|v| !v.is_empty()).map(Into::into),
            labels: table.labels.into_iter().collect(),
            creation_time: timestamp_ms("creation_time", table.creation_time)?,
            expiration_time: timestamp_ms("expiration_time", table.expiration_time)?,
        })
    }
}

impl BigQueryDb {
    pub(crate) async fn get_table(
        &self,
        table: &BigQueryTableRef,
    ) -> BigQueryResult<BigQueryTable> {
        self.get_table_body(table).await?.try_into()
    }

    pub(crate) async fn get_table_body(
        &self,
        table: &BigQueryTableRef,
    ) -> BigQueryResult<v2::Table> {
        let ids = table.ids(&self.options().google_project_id);
        let request = v2::GetTableRequest {
            project_id: ids.project,
            dataset_id: ids.dataset,
            table_id: ids.table,
            ..Default::default()
        };
        self.retry(&table.admin_span(), "get a table", &request, |r| {
            let mut client = self.table_client();
            async move { client.get_table(r).await }
        })
        .await
    }

    pub(crate) async fn delete_table(&self, table: &BigQueryTableRef) -> BigQueryResult<()> {
        let ids = table.ids(&self.options().google_project_id);
        let request = v2::DeleteTableRequest {
            project_id: ids.project,
            dataset_id: ids.dataset,
            table_id: ids.table,
        };
        self.retry(&table.admin_span(), "delete a table", &request, |r| {
            let mut client = self.table_client();
            async move { client.delete_table(r).await }
        })
        .await
    }

    pub(crate) fn list_tables(
        &self,
        dataset: &BigQueryDatasetRef,
        page_size: Option<u32>,
    ) -> BoxStream<'static, BigQueryResult<BigQueryTableSummary>> {
        let db = self.clone();
        let project_id = dataset
            .project_or(&self.options().google_project_id)
            .to_string();
        let dataset_id = dataset.dataset().to_string();
        let span = tracing::debug_span!("BigQuery tables", "/bigquery/dataset" = %dataset);
        paged(move |page_token| {
            let db = db.clone();
            let span = span.clone();
            let request = v2::ListTablesRequest {
                project_id: project_id.clone(),
                dataset_id: dataset_id.clone(),
                max_results: page_size,
                page_token,
            };
            async move {
                let page = db
                    .retry(&span, "list tables", &request, |r| {
                        let mut client = db.table_client();
                        async move { client.list_tables(r).await }
                    })
                    .await?;
                let tables = page
                    .tables
                    .into_iter()
                    .map(TryInto::try_into)
                    .collect::<BigQueryResult<_>>()?;
                Ok((tables, page.next_page_token))
            }
        })
    }
}

impl BigQueryTableRef {
    /// The span of one table admin call.
    fn admin_span(&self) -> Span {
        tracing::debug_span!("BigQuery table", "/bigquery/table" = %self)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn table_types_map_known_names_and_keep_unknown_ones() {
        for name in ["TABLE", "VIEW", "MATERIALIZED_VIEW", "EXTERNAL", "SNAPSHOT"] {
            let parsed = BigQueryTableType::from(name);
            assert!(
                !matches!(parsed, BigQueryTableType::Other(_)),
                "{name} reads as {parsed:?}"
            );
            assert_eq!(parsed.as_str(), name);
        }
        let unknown = BigQueryTableType::from("CLONE".to_string());
        assert_eq!(unknown, BigQueryTableType::Other("CLONE".into()));
        assert_eq!(unknown.to_string(), "CLONE");
    }
}