use crate::db::fake::FakeCall;
use crate::testing::rules::BigQueryFakeRpc;
use crate::testing::server::{FakeRefusal, FakeShared};
use crate::testing::state::{
fit_to_layout, DatasetKey, FakeDataset, FakeGeneration, FakeState, FakeTable, TableKey,
};
use crate::{BigQueryFieldMode, BigQueryFieldSchema, BigQueryFieldType, BigQueryTableSchema};
use gcloud_sdk::google::cloud::bigquery::v2;
use gcloud_sdk::tonic::Status;
use std::num::NonZeroUsize;
use std::sync::Arc;
type AdminResult<T> = Result<T, FakeRefusal>;
impl FakeShared {
pub(super) async fn serve_admin(&self, call: FakeCall) {
match call.method() {
"GetTable" => self.get_table(call).await,
"InsertTable" => self.insert_table(call).await,
"PatchTable" => {
self.patch_or_update_table(call, BigQueryFakeRpc::PatchTable)
.await
}
"UpdateTable" => {
self.patch_or_update_table(call, BigQueryFakeRpc::UpdateTable)
.await
}
"DeleteTable" => self.delete_table(call).await,
"ListTables" => self.list_tables(call).await,
"GetDataset" => self.get_dataset(call).await,
"InsertDataset" => self.insert_dataset(call).await,
"UpdateDataset" => self.update_dataset(call).await,
"DeleteDataset" => self.delete_dataset(call).await,
"ListDatasets" => self.list_datasets(call).await,
method => {
let described = method.to_string();
self.unmatched(call, &described, &[]);
}
}
}
fn table_key(
&self,
call: FakeCall,
rpc: BigQueryFakeRpc,
project: &str,
dataset: &str,
table: &str,
) -> Option<(FakeCall, TableKey)> {
match TableKey::new(project, dataset, table) {
Ok(key) => Some((call, key)),
Err(err) => {
self.unmatched(call, &format!("{rpc:?} of an invalid table: {err}"), &[]);
None
}
}
}
fn dataset_key(
&self,
call: FakeCall,
rpc: BigQueryFakeRpc,
project: &str,
dataset: &str,
) -> Option<(FakeCall, DatasetKey)> {
match DatasetKey::new(project, dataset) {
Ok(key) => Some((call, key)),
Err(err) => {
self.unmatched(call, &format!("{rpc:?} of an invalid dataset: {err}"), &[]);
None
}
}
}
async fn get_table(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::GetTable;
let Some((call, request)) = self.first_request::<v2::GetTableRequest>(call).await else {
return;
};
let Some((call, key)) = self.table_key(
call,
rpc,
&request.project_id,
&request.dataset_id,
&request.table_id,
) else {
return;
};
let Some(call) = self.unfaulted(call, rpc, Some(&key)).await else {
return;
};
let answer = self.state().table_resource(&key);
self.answer(call, answer);
}
async fn insert_table(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::InsertTable;
let Some((call, request)) = self.first_request::<v2::InsertTableRequest>(call).await else {
return;
};
let body = request.table.unwrap_or_default();
let table_id = body
.table_reference
.as_ref()
.map(|reference| reference.table_id.clone())
.unwrap_or_default();
let Some((call, key)) = self.table_key(
call,
rpc,
&request.project_id,
&request.dataset_id,
&table_id,
) else {
return;
};
let Some(call) = self.unfaulted(call, rpc, Some(&key)).await else {
return;
};
let answer = self.state().insert_table(&key, body);
self.answer(call, answer);
}
async fn patch_or_update_table(&self, call: FakeCall, rpc: BigQueryFakeRpc) {
let Some((call, request)) = self
.first_request::<v2::UpdateOrPatchTableRequest>(call)
.await
else {
return;
};
let Some((call, key)) = self.table_key(
call,
rpc,
&request.project_id,
&request.dataset_id,
&request.table_id,
) else {
return;
};
let Some(call) = self.unfaulted(call, rpc, Some(&key)).await else {
return;
};
let if_match = call.header("if-match");
let body = request.table.unwrap_or_default();
let replace = rpc == BigQueryFakeRpc::UpdateTable;
let answer = self
.state()
.write_table(&key, body, if_match.as_deref(), replace);
self.answer(call, answer);
}
async fn delete_table(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::DeleteTable;
let Some((call, request)) = self.first_request::<v2::DeleteTableRequest>(call).await else {
return;
};
let Some((call, key)) = self.table_key(
call,
rpc,
&request.project_id,
&request.dataset_id,
&request.table_id,
) else {
return;
};
let Some(call) = self.unfaulted(call, rpc, Some(&key)).await else {
return;
};
let answer = match self.state().tables.remove(&key) {
Some(_) => Ok(()),
None => Err(key.not_found().into()),
};
self.answer(call, answer);
}
async fn list_tables(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::ListTables;
let Some((call, request)) = self.first_request::<v2::ListTablesRequest>(call).await else {
return;
};
let Some((call, key)) =
self.dataset_key(call, rpc, &request.project_id, &request.dataset_id)
else {
return;
};
let Some(call) = self.unfaulted(call, rpc, None).await else {
return;
};
let answer = self.state().list_tables(&key, &request);
self.answer(call, answer);
}
async fn get_dataset(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::GetDataset;
let Some((call, request)) = self.first_request::<v2::GetDatasetRequest>(call).await else {
return;
};
let Some((call, key)) =
self.dataset_key(call, rpc, &request.project_id, &request.dataset_id)
else {
return;
};
let Some(call) = self.unfaulted(call, rpc, None).await else {
return;
};
let answer = self
.state()
.dataset(&key)
.map(|dataset| dataset.resource(&key))
.map_err(FakeRefusal::from);
self.answer(call, answer);
}
async fn insert_dataset(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::InsertDataset;
let Some((call, request)) = self.first_request::<v2::InsertDatasetRequest>(call).await
else {
return;
};
let body = request.dataset.unwrap_or_default();
let dataset_id = body
.dataset_reference
.as_ref()
.map(|reference| reference.dataset_id.clone())
.unwrap_or_default();
let Some((call, key)) = self.dataset_key(call, rpc, &request.project_id, &dataset_id)
else {
return;
};
let Some(call) = self.unfaulted(call, rpc, None).await else {
return;
};
let answer = self.state().insert_dataset(&key, body);
self.answer(call, answer);
}
async fn update_dataset(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::UpdateDataset;
let Some((call, request)) = self
.first_request::<v2::UpdateOrPatchDatasetRequest>(call)
.await
else {
return;
};
let Some((call, key)) =
self.dataset_key(call, rpc, &request.project_id, &request.dataset_id)
else {
return;
};
let Some(call) = self.unfaulted(call, rpc, None).await else {
return;
};
let if_match = call.header("if-match");
let body = request.dataset.unwrap_or_default();
let answer = self.state().update_dataset(&key, body, if_match.as_deref());
self.answer(call, answer);
}
async fn delete_dataset(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::DeleteDataset;
let Some((call, request)) = self.first_request::<v2::DeleteDatasetRequest>(call).await
else {
return;
};
let Some((call, key)) =
self.dataset_key(call, rpc, &request.project_id, &request.dataset_id)
else {
return;
};
let Some(call) = self.unfaulted(call, rpc, None).await else {
return;
};
let answer = self.state().delete_dataset(&key, request.delete_contents);
self.answer(call, answer);
}
async fn list_datasets(&self, call: FakeCall) {
let rpc = BigQueryFakeRpc::ListDatasets;
let Some((call, request)) = self.first_request::<v2::ListDatasetsRequest>(call).await
else {
return;
};
let Some(call) = self.unfaulted(call, rpc, None).await else {
return;
};
let answer = self.state().list_datasets(&request);
self.answer(call, answer);
}
}
impl FakeState {
fn dataset(&self, key: &DatasetKey) -> Result<&FakeDataset, Status> {
self.datasets.get(key).ok_or_else(|| key.not_found())
}
fn table_resource(&self, key: &TableKey) -> AdminResult<v2::Table> {
let table = self.tables.get(key).ok_or_else(|| key.not_found())?;
Ok(table.resource(key))
}
fn insert_table(&mut self, key: &TableKey, mut body: v2::Table) -> AdminResult<v2::Table> {
self.dataset(&key.dataset)?;
let schema = table_schema(key, body.schema.take().as_ref())?;
let generation = self.next_generation();
let mut table = FakeTable::new(schema, Vec::new(), NonZeroUsize::MIN, generation);
table.resource = body;
let resource = table.resource(key);
self.create_table(key.clone(), table)
.map_err(|_| key.already_exists())?;
Ok(resource)
}
fn write_table(
&mut self,
key: &TableKey,
mut body: v2::Table,
if_match: Option<&str>,
replace: bool,
) -> AdminResult<v2::Table> {
let generation = self.next_generation();
let table = self.tables.get_mut(key).ok_or_else(|| key.not_found())?;
table.generation.check(if_match)?;
if let Some(schema) = body.schema.take() {
let schema = table_schema(key, Some(&schema))?;
table.change_schema(key, schema)?;
}
if replace {
table.resource = body;
} else {
patch_table(&mut table.resource, body);
}
table.generation = generation;
Ok(table.resource(key))
}
fn list_tables(
&self,
key: &DatasetKey,
request: &v2::ListTablesRequest,
) -> AdminResult<v2::TableList> {
self.dataset(key)?;
let tables: Vec<_> = self
.tables
.iter()
.filter(|(table, _)| table.dataset == *key)
.map(|(table, held)| held.listed(table))
.collect();
let total_items = i32::try_from(tables.len()).ok();
let (tables, next_page_token) = page(tables, request.max_results, &request.page_token)?;
Ok(v2::TableList {
kind: "bigquery#tableList".into(),
next_page_token,
tables,
total_items,
..Default::default()
})
}
fn insert_dataset(&mut self, key: &DatasetKey, body: v2::Dataset) -> AdminResult<v2::Dataset> {
let dataset = self.create_dataset(key.clone())?;
dataset.resource = body;
Ok(dataset.resource(key))
}
fn update_dataset(
&mut self,
key: &DatasetKey,
mut body: v2::Dataset,
if_match: Option<&str>,
) -> AdminResult<v2::Dataset> {
let generation = self.next_generation();
let dataset = self.datasets.get_mut(key).ok_or_else(|| key.not_found())?;
dataset.generation.check(if_match)?;
body.location = dataset.resource.location.clone();
dataset.resource = body;
dataset.generation = generation;
Ok(dataset.resource(key))
}
fn delete_dataset(&mut self, key: &DatasetKey, delete_contents: bool) -> AdminResult<()> {
self.dataset(key)?;
let in_dataset = |table: &TableKey| table.dataset == *key;
if !delete_contents && self.tables.keys().any(in_dataset) {
return Err(Status::failed_precondition(format!(
"Dataset {} is still in use",
key.legacy_id()
))
.into());
}
self.tables.retain(|table, _| !in_dataset(table));
self.datasets.remove(key);
Ok(())
}
fn list_datasets(&self, request: &v2::ListDatasetsRequest) -> AdminResult<v2::DatasetList> {
let datasets: Vec<_> = self
.datasets
.iter()
.filter(|(key, _)| key.project == request.project_id)
.filter(|(key, _)| request.all || !key.dataset.to_string().starts_with('_'))
.map(|(key, dataset)| dataset.listed(key))
.collect();
let (datasets, next_page_token) = page(datasets, request.max_results, &request.page_token)?;
Ok(v2::DatasetList {
kind: "bigquery#datasetList".into(),
next_page_token,
datasets,
..Default::default()
})
}
}
impl FakeGeneration {
fn check(self, if_match: Option<&str>) -> Result<(), Status> {
match if_match {
Some(etag) if etag != self.etag() => {
Err(Status::failed_precondition("Precondition check failed."))
}
_ => Ok(()),
}
}
}
impl FakeTable {
fn listed(&self, key: &TableKey) -> v2::ListFormatTable {
let resource = self.resource(key);
v2::ListFormatTable {
kind: "bigquery#table".into(),
id: resource.id,
table_reference: resource.table_reference,
friendly_name: resource.friendly_name,
r#type: resource.r#type,
time_partitioning: resource.time_partitioning,
range_partitioning: resource.range_partitioning,
clustering: resource.clustering,
labels: resource.labels,
creation_time: resource.creation_time,
expiration_time: resource.expiration_time.unwrap_or_default(),
..Default::default()
}
}
fn change_schema(&mut self, key: &TableKey, schema: BigQueryTableSchema) -> AdminResult<()> {
if let Some(problem) = schema_change_refusal(&self.schema.fields, &schema.fields, "") {
return Err(FakeRefusal::from(Status::invalid_argument(format!(
"Provided Schema does not match Table {}. {problem}",
key.legacy_id()
))));
}
let arrow_schema = Arc::new(schema.arrow_read_schema());
let batches = self
.batches
.iter()
.map(|batch| {
fit_to_layout(batch, &schema, &arrow_schema).map_err(|err| {
FakeRefusal::Internal(format!(
"the rows of {key} do not carry over to its new schema: {err}"
))
})
})
.collect::<AdminResult<Vec<_>>>()?;
self.schema = schema;
self.arrow_schema = arrow_schema;
self.batches = batches;
Ok(())
}
}
fn schema_change_refusal(
old: &[BigQueryFieldSchema],
new: &[BigQueryFieldSchema],
parent: &str,
) -> Option<String> {
let path = |field: &BigQueryFieldSchema| {
if parent.is_empty() {
field.name.clone()
} else {
format!("{parent}.{}", field.name)
}
};
for old_field in old {
let Some(new_field) = new
.iter()
.find(|new_field| new_field.name.eq_ignore_ascii_case(&old_field.name))
else {
return Some(format!(
"Field {} is missing in new schema",
path(old_field)
));
};
let nested = match (&old_field.field_type, &new_field.field_type) {
(BigQueryFieldType::Struct(old_fields), BigQueryFieldType::Struct(new_fields)) => {
schema_change_refusal(old_fields, new_fields, &path(old_field))
}
(old_type, new_type) if old_type != new_type => {
Some(format!("Field {} has changed type", path(old_field)))
}
_ => None,
};
if nested.is_some() {
return nested;
}
let relaxed = old_field.mode == BigQueryFieldMode::Required
&& new_field.mode == BigQueryFieldMode::Nullable;
if old_field.mode != new_field.mode && !relaxed {
return Some(format!("Field {} has changed mode", path(old_field)));
}
}
new.iter()
.find(|new_field| {
new_field.mode == BigQueryFieldMode::Required
&& !old
.iter()
.any(|old_field| old_field.name.eq_ignore_ascii_case(&new_field.name))
})
.map(|added| {
format!(
"Cannot add required fields to an existing schema. (field: {})",
path(added)
)
})
}
impl FakeDataset {
fn listed(&self, key: &DatasetKey) -> v2::ListFormatDataset {
let resource = self.resource(key);
v2::ListFormatDataset {
kind: "bigquery#dataset".into(),
id: resource.id,
dataset_reference: resource.dataset_reference,
labels: resource.labels,
friendly_name: resource.friendly_name,
location: resource.location,
..Default::default()
}
}
}
fn patch_table(settings: &mut v2::Table, body: v2::Table) {
settings.labels.extend(body.labels);
settings.description = body.description.or(settings.description.take());
settings.friendly_name = body.friendly_name.or(settings.friendly_name.take());
settings.expiration_time = body.expiration_time.or(settings.expiration_time.take());
settings.time_partitioning = body.time_partitioning.or(settings.time_partitioning.take());
settings.range_partitioning = body
.range_partitioning
.or(settings.range_partitioning.take());
settings.clustering = body.clustering.or(settings.clustering.take());
settings.table_constraints = body.table_constraints.or(settings.table_constraints.take());
}
fn table_schema(
key: &TableKey,
schema: Option<&v2::TableSchema>,
) -> Result<BigQueryTableSchema, Status> {
let schema = schema.ok_or_else(|| {
Status::invalid_argument(format!("The table {} has no schema", key.legacy_id()))
})?;
BigQueryTableSchema::try_from(schema).map_err(|err| {
Status::invalid_argument(format!(
"The schema of {} is invalid: {err}",
key.legacy_id()
))
})
}
fn page<T>(
items: Vec<T>,
max_results: Option<u32>,
page_token: &str,
) -> Result<(Vec<T>, String), Status> {
let first = match page_token {
"" => 0,
token => token
.parse::<usize>()
.map_err(|_| Status::invalid_argument(format!("Invalid page token {token:?}")))?,
};
let length = max_results
.and_then(|max| usize::try_from(max).ok())
.filter(|max| *max > 0)
.unwrap_or(usize::MAX);
let end = first.saturating_add(length).min(items.len());
let next_page_token = if end < items.len() {
end.to_string()
} else {
String::new()
};
let page = items
.into_iter()
.skip(first)
.take(end.saturating_sub(first))
.collect();
Ok((page, next_page_token))
}
#[cfg(test)]
mod tests {
use crate::db::if_match;
use crate::errors::BigQueryError;
use crate::testing::BigQueryFake;
use crate::{
BigQueryDataset, BigQueryDatasetId, BigQueryResult, BigQuerySchemaChange,
BigQuerySchemaColumn, BigQuerySchemaColumns, BigQuerySchemaColumnsBuilder, BigQueryTableId,
BigQueryTableRef,
};
use futures::TryStreamExt;
use gcloud_sdk::google::cloud::bigquery::v2;
use gcloud_sdk::tonic::Code;
use serde::{Deserialize, Serialize};
use tracing::Span;
const SHOP: BigQueryDatasetId = BigQueryDatasetId::from_static("shop");
const ARCHIVE: BigQueryDatasetId = BigQueryDatasetId::from_static("archive");
const ORDERS: BigQueryTableId = BigQueryTableId::from_static("orders");
const CUSTOMERS: BigQueryTableId = BigQueryTableId::from_static("customers");
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct Order {
id: i64,
customer: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct Payment {
id: i64,
amount: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct NotedPayment {
id: i64,
amount: String,
note: Option<String>,
}
#[tokio::test]
async fn an_undeclared_bignumeric_beyond_ten_to_the_38_survives_a_schema_sync(
) -> BigQueryResult<()> {
const PAYMENTS: BigQueryTableId = BigQueryTableId::from_static("payments");
let payment = Payment {
id: 1,
amount: "200000000000000000000000000000000000000.5".to_string(),
};
let fake = BigQueryFake::start().await?;
fake.table(SHOP.table(PAYMENTS), |columns| {
columns
.from_type::<Payment>()
.with("amount", |amount| amount.bignumeric())
})
.rows([payment.clone()])
.create()?;
fake.db()
.fluent()
.schema()
.table(SHOP.table(PAYMENTS))
.columns(|columns| {
columns
.from_type::<NotedPayment>()
.with("amount", |amount| amount.bignumeric())
})
.sync()
.await?;
let expected = NotedPayment {
id: payment.id,
amount: payment.amount,
note: None,
};
assert_eq!(
fake.rows::<NotedPayment>(SHOP.table(PAYMENTS))?,
vec![expected]
);
Ok(())
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct NotedOrder {
id: i64,
customer: Option<String>,
note: Option<String>,
}
fn order_columns(c: BigQuerySchemaColumnsBuilder) -> Vec<BigQuerySchemaColumn> {
c.fields([
c.field("id").int64().required(),
c.field("customer").string(),
])
}
fn noted_order_columns(c: BigQuerySchemaColumnsBuilder) -> Vec<BigQuerySchemaColumn> {
c.fields([
c.field("id").int64().required(),
c.field("customer").string(),
c.field("note").string(),
])
}
async fn dataset_ids(fake: &BigQueryFake) -> BigQueryResult<Vec<BigQueryDatasetId>> {
fake.db()
.fluent()
.schema()
.datasets()
.page_size(1)
.stream_all_with_errors()
.await?
.map_ok(|dataset| dataset.reference.dataset().clone())
.try_collect()
.await
}
async fn table_ids(
fake: &BigQueryFake,
dataset: BigQueryDatasetId,
) -> BigQueryResult<Vec<BigQueryTableId>> {
fake.db()
.fluent()
.schema()
.dataset(dataset)
.tables()
.stream_all_with_errors()
.await?
.map_ok(|table| table.reference.table().clone())
.try_collect()
.await
}
#[tokio::test]
async fn a_created_dataset_reads_back_and_is_listed() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
let shop = fake.db().fluent().schema().dataset(SHOP);
let created = shop
.create()
.description("The shop")
.labels([("team", "sales")])
.execute()
.await?;
fake.db()
.fluent()
.schema()
.dataset(ARCHIVE)
.create()
.execute()
.await?;
assert_eq!(created.reference.project(), Some("fake-project"));
assert_eq!(created.description.as_deref(), Some("The shop"));
assert!(created.creation_time.is_some());
let read: BigQueryDataset = fake.db().fluent().schema().dataset(SHOP).get().await?;
assert_eq!(read, created);
assert_eq!(dataset_ids(&fake).await?, [ARCHIVE, SHOP]);
Ok(())
}
#[tokio::test]
async fn creating_a_dataset_that_exists_is_a_conflict() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
fake.create_dataset(SHOP)?;
let created = fake
.db()
.fluent()
.schema()
.dataset(SHOP)
.create()
.execute()
.await;
assert!(
matches!(created, Err(BigQueryError::DataConflictError(_))),
"{created:?}"
);
Ok(())
}
#[tokio::test]
async fn a_missing_dataset_is_not_found() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
let shop = || fake.db().fluent().schema().dataset(SHOP);
let read = shop().get().await;
let deleted = shop().delete().await;
let listed = table_ids(&fake, SHOP).await;
assert!(
matches!(read, Err(BigQueryError::DataNotFoundError(_))),
"{read:?}"
);
assert!(
matches!(deleted, Err(BigQueryError::DataNotFoundError(_))),
"{deleted:?}"
);
assert!(
matches!(listed, Err(BigQueryError::DataNotFoundError(_))),
"{listed:?}"
);
Ok(())
}
#[tokio::test]
async fn an_update_changes_the_description_and_labels() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
let shop = || fake.db().fluent().schema().dataset(SHOP);
shop()
.create()
.description("The shop")
.labels([("team", "sales"), ("tier", "gold")])
.execute()
.await?;
let updated = shop()
.update()
.clear_description()
.label("team", "finance")
.remove_label("tier")
.execute()
.await?;
assert_eq!(updated.description, None);
assert_eq!(updated.labels, [("team", "finance")].into());
assert_eq!(shop().get().await?, updated);
Ok(())
}
#[tokio::test]
async fn a_write_under_a_stale_etag_is_a_conflict() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
fake.table(SHOP.table(ORDERS), order_columns).create()?;
let db = fake.db();
let stale = if_match("GetTable", "fake-etag-0")?;
let dataset = v2::UpdateOrPatchDatasetRequest {
project_id: "fake-project".into(),
dataset_id: SHOP.to_string(),
dataset: Some(v2::Dataset::default()),
..Default::default()
};
let dataset = db
.retry_with_metadata(&Span::none(), "update", &dataset, &stale, |r| {
let mut client = db.dataset_client();
async move { client.update_dataset(r).await }
})
.await
.map_err(|err| err.on_stale_etag(String::new));
let table = v2::UpdateOrPatchTableRequest {
project_id: "fake-project".into(),
dataset_id: SHOP.to_string(),
table_id: ORDERS.to_string(),
table: Some(v2::Table::default()),
autodetect_schema: false,
};
let table = db
.retry_with_metadata(&Span::none(), "patch", &table, &stale, |r| {
let mut client = db.table_client();
async move { client.patch_table(r).await }
})
.await
.map_err(|err| err.on_stale_etag(String::new));
assert!(
matches!(dataset, Err(BigQueryError::DataConflictError(_))),
"{dataset:?}"
);
assert!(
matches!(table, Err(BigQueryError::DataConflictError(_))),
"{table:?}"
);
Ok(())
}
#[tokio::test]
async fn deleting_a_dataset_with_tables_needs_its_contents_deleted() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
fake.table(SHOP.table(ORDERS), order_columns).create()?;
let shop = || fake.db().fluent().schema().dataset(SHOP);
let refused = shop().delete().await;
assert!(
matches!(&refused, Err(BigQueryError::DatabaseError(err))
if err.public.code == "FailedPrecondition" && !err.retry_possible),
"{refused:?}"
);
assert_eq!(fake.rows::<Order>(SHOP.table(ORDERS))?, []);
shop().dangerously_delete_with_contents().await?;
let rows = fake.rows::<Order>(SHOP.table(ORDERS));
assert!(
matches!(rows, Err(BigQueryError::DataNotFoundError(_))),
"{rows:?}"
);
assert_eq!(dataset_ids(&fake).await?, []);
Ok(())
}
#[tokio::test]
async fn tables_are_listed_and_deleted() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
fake.table(SHOP.table(ORDERS), order_columns).create()?;
fake.table(SHOP.table(CUSTOMERS), order_columns).create()?;
let orders = || fake.db().fluent().schema().table(SHOP.table(ORDERS));
assert_eq!(table_ids(&fake, SHOP).await?, [CUSTOMERS, ORDERS]);
orders().delete().await?;
assert_eq!(table_ids(&fake, SHOP).await?, [CUSTOMERS]);
let read = orders().get().await;
assert!(
matches!(read, Err(BigQueryError::DataNotFoundError(_))),
"{read:?}"
);
let deleted = orders().delete().await;
assert!(
matches!(deleted, Err(BigQueryError::DataNotFoundError(_))),
"{deleted:?}"
);
Ok(())
}
#[tokio::test]
async fn sync_creates_a_table_then_adds_a_column() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
fake.create_dataset(SHOP)?;
let orders = || fake.db().fluent().schema().table(SHOP.table(ORDERS));
let created = orders().columns(order_columns).sync().await?;
assert!(created.created.is_some(), "{created:?}");
fake.db()
.fluent()
.insert()
.into(SHOP.table(ORDERS))
.objects(&[Order {
id: 1,
customer: Some("Alice".into()),
}])
.execute()
.await?;
let patched = orders().columns(noted_order_columns).sync().await?;
assert!(
matches!(
&patched.applied[..],
[BigQuerySchemaChange::AddColumn { .. }]
),
"{patched:?}"
);
let table = orders().get().await?;
let declared = orders().columns(noted_order_columns).plan().await?;
assert!(declared.changes.is_empty(), "{declared:?}");
assert_eq!(table.num_rows, Some(1));
let rows: Vec<NotedOrder> = fake.rows(SHOP.table(ORDERS))?;
assert_eq!(
rows,
[NotedOrder {
id: 1,
customer: Some("Alice".into()),
note: None
}]
);
Ok(())
}
#[tokio::test]
async fn sync_into_a_missing_dataset_is_not_found() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
let table: BigQueryTableRef = SHOP.table(ORDERS);
let synced = fake
.db()
.fluent()
.schema()
.table(table)
.columns(order_columns)
.sync()
.await;
assert!(
matches!(synced, Err(BigQueryError::DataNotFoundError(_))),
"{synced:?}"
);
Ok(())
}
#[tokio::test]
async fn a_schema_change_matches_columns_ignoring_case() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
let alice = Order {
id: 1,
customer: Some("Alice".into()),
};
fake.table(SHOP.table(ORDERS), order_columns)
.rows([alice])
.create()?;
let columns = BigQuerySchemaColumnsBuilder;
let shouted: BigQuerySchemaColumns = columns
.fields([
columns.field("ID").int64().required(),
columns.field("CUSTOMER").string(),
columns.field("note").string(),
])
.into();
let schema = shouted.table_schema()?;
let patched = fake
.db()
.table_client()
.patch_table(v2::UpdateOrPatchTableRequest {
project_id: "fake-project".to_string(),
dataset_id: SHOP.to_string(),
table_id: ORDERS.to_string(),
table: Some(v2::Table {
schema: Some(v2::TableSchema::from(&schema)),
..Default::default()
}),
..Default::default()
})
.await
.map_err(BigQueryError::from)?
.into_inner();
assert_eq!(patched.num_rows, Some(1));
Ok(())
}
const DELIVERIES: BigQueryTableId = BigQueryTableId::from_static("deliveries");
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct Address {
city: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct Delivery {
id: i64,
addresses: Vec<Address>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct ZipAddress {
city: String,
zip: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct ZipDelivery {
id: i64,
addresses: Vec<ZipAddress>,
}
#[tokio::test]
async fn sync_adds_a_nested_field_to_a_table_with_rows() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
let paris = Address {
city: "Paris".to_string(),
};
fake.table(SHOP.table(DELIVERIES), |columns| {
columns.from_type::<Delivery>()
})
.rows([Delivery {
id: 1,
addresses: vec![paris.clone()],
}])
.create()?;
fake.db()
.fluent()
.schema()
.table(SHOP.table(DELIVERIES))
.columns(|columns| columns.from_type::<ZipDelivery>())
.sync()
.await?;
let rows: Vec<ZipDelivery> = fake.rows(SHOP.table(DELIVERIES))?;
assert_eq!(
rows,
[ZipDelivery {
id: 1,
addresses: vec![ZipAddress {
city: paris.city,
zip: None
}],
}]
);
Ok(())
}
#[tokio::test]
async fn a_schema_change_that_drops_a_nested_field_is_refused() -> BigQueryResult<()> {
let fake = BigQueryFake::start().await?;
fake.table(SHOP.table(DELIVERIES), |columns| {
columns.from_type::<ZipDelivery>()
})
.create()?;
let columns = BigQuerySchemaColumnsBuilder.from_type::<Delivery>();
let schema = columns.table_schema()?;
let patched = fake
.db()
.table_client()
.patch_table(v2::UpdateOrPatchTableRequest {
project_id: "fake-project".to_string(),
dataset_id: SHOP.to_string(),
table_id: DELIVERIES.to_string(),
table: Some(v2::Table {
schema: Some(v2::TableSchema::from(&schema)),
..Default::default()
}),
..Default::default()
})
.await;
assert_eq!(
patched.map(|_| ()).map_err(|status| status.code()),
Err(Code::InvalidArgument)
);
Ok(())
}
}