use crate::fluent_api::tests::mockdb::MockDatabase;
use crate::fluent_api::BigQueryExprBuilder;
use crate::{
BigQueryChange, BigQueryChangeSequenceNumber, BigQueryChangeType, BigQueryDatasetRef,
BigQueryInsertParams, BigQueryResult, BigQueryStreamingWriteOptions, BigQueryWriteMode,
BigQueryWriteSummary, BigQueryWriteSupport,
};
use arrow_array::{Int64Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema};
use async_trait::async_trait;
use serde::Serialize;
use std::cell::RefCell;
use std::sync::Arc;
use crate::db::fake::{ORDERS, SHOP};
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct RecordedInsert {
pub(crate) params: BigQueryInsertParams,
pub(crate) rows: Vec<serde_json::Value>,
pub(crate) changes: Option<Vec<(BigQueryChangeType, Option<String>)>>,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct RecordedBatchInsert {
pub(crate) params: BigQueryInsertParams,
pub(crate) batches: Vec<RecordBatch>,
}
thread_local! {
static INSERTS: RefCell<Vec<RecordedInsert>> = const { RefCell::new(Vec::new()) };
static BATCH_INSERTS: RefCell<Vec<RecordedBatchInsert>> = const { RefCell::new(Vec::new()) };
}
pub(crate) fn take_inserts() -> Vec<RecordedInsert> {
INSERTS.with(|inserts| inserts.take())
}
pub(crate) fn take_batch_inserts() -> Vec<RecordedBatchInsert> {
BATCH_INSERTS.with(|inserts| inserts.take())
}
fn json<T: Serialize>(row: &T) -> serde_json::Value {
serde_json::to_value(row).expect("a test row serializes to JSON")
}
fn summary(rows: usize) -> BigQueryWriteSummary {
BigQueryWriteSummary {
rows_written: rows as u64,
rows_failed: 0,
batches: 1,
bytes_sent: 0,
stream: None,
commit_time: None,
}
}
#[async_trait]
impl BigQueryWriteSupport for MockDatabase {
async fn insert_objects<T, I>(
&self,
params: BigQueryInsertParams,
rows: I,
) -> BigQueryResult<BigQueryWriteSummary>
where
T: Serialize + Send + Sync,
I: IntoIterator<Item = T> + Send,
I::IntoIter: Send,
{
let rows: Vec<serde_json::Value> = rows.into_iter().map(|row| json(&row)).collect();
let count = rows.len();
INSERTS.with(|inserts| {
inserts.borrow_mut().push(RecordedInsert {
params,
rows,
changes: None,
})
});
Ok(summary(count))
}
async fn insert_record_batches<I>(
&self,
params: BigQueryInsertParams,
batches: I,
) -> BigQueryResult<BigQueryWriteSummary>
where
I: IntoIterator<Item = RecordBatch> + Send,
I::IntoIter: Send,
{
let batches: Vec<RecordBatch> = batches.into_iter().collect();
let rows = batches.iter().map(RecordBatch::num_rows).sum();
BATCH_INSERTS.with(|inserts| {
inserts
.borrow_mut()
.push(RecordedBatchInsert { params, batches })
});
Ok(summary(rows))
}
async fn insert_changes<T, I>(
&self,
params: BigQueryInsertParams,
changes: I,
) -> BigQueryResult<BigQueryWriteSummary>
where
T: Serialize + Send + Sync,
I: IntoIterator<Item = BigQueryChange<T>> + Send,
I::IntoIter: Send,
{
let (rows, kinds): (Vec<_>, Vec<_>) = changes
.into_iter()
.map(|change| {
(
json(&change.row),
(
change.change_type,
change.sequence_number.map(|s| s.as_str().to_string()),
),
)
})
.unzip();
let count = rows.len();
INSERTS.with(|inserts| {
inserts.borrow_mut().push(RecordedInsert {
params,
rows,
changes: Some(kinds),
})
});
Ok(summary(count))
}
}
#[derive(Serialize)]
struct Row {
id: i64,
}
#[tokio::test]
async fn insert_chain_passes_mode_and_rows() {
let db = MockDatabase;
let rows = [Row { id: 1 }, Row { id: 2 }];
let ids = |insert: &RecordedInsert| -> Vec<i64> {
insert
.rows
.iter()
.filter_map(|row| row["id"].as_i64())
.collect()
};
take_inserts();
let summary = BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.objects(&rows)
.execute()
.await
.expect("the insert runs");
assert_eq!(summary.rows_written, 2);
BigQueryExprBuilder::new(&db)
.insert()
.into(
BigQueryDatasetRef::new("acme-prod", SHOP)
.expect("valid test input")
.table(ORDERS),
)
.object(&rows[0])
.exactly_once()
.execute()
.await
.expect("the insert runs");
BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.objects(rows.iter())
.options(BigQueryStreamingWriteOptions::new().with_max_batch_rows(7))
.atomic()
.execute()
.await
.expect("the insert runs");
BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.objects(&rows)
.upsert()
.execute()
.await
.expect("the insert runs");
BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.changes(vec![BigQueryChange {
change_type: BigQueryChangeType::Delete,
sequence_number: Some(BigQueryChangeSequenceNumber::from(10)),
row: Row { id: 2 },
}])
.execute()
.await
.expect("the insert runs");
let inserts = take_inserts();
assert_eq!(inserts.len(), 5);
let modes: Vec<BigQueryWriteMode> = inserts.iter().map(|i| i.params.options.mode).collect();
assert_eq!(
modes,
[
BigQueryWriteMode::Default,
BigQueryWriteMode::Committed,
BigQueryWriteMode::Pending,
BigQueryWriteMode::Default,
BigQueryWriteMode::Default
]
);
assert_eq!(inserts[0].params.table, SHOP.table(ORDERS));
assert_eq!(
inserts[1].params.table,
BigQueryDatasetRef::new("acme-prod", SHOP)
.expect("valid test input")
.table(ORDERS)
);
assert_eq!(inserts[2].params.options.max_batch_rows, Some(7));
assert_eq!(
inserts.iter().map(ids).collect::<Vec<_>>(),
[vec![1, 2], vec![1], vec![1, 2], vec![1, 2], vec![2]]
);
assert_eq!(inserts[0].changes, None);
assert_eq!(
inserts[3].changes,
Some(vec![
(BigQueryChangeType::Upsert, None),
(BigQueryChangeType::Upsert, None)
])
);
assert_eq!(
inserts[4].changes,
Some(vec![(BigQueryChangeType::Delete, Some("A".to_string()))])
);
}
#[tokio::test]
async fn record_batch_insert_chain_passes_mode_options_and_batches() {
let db = MockDatabase;
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
let batch = RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1, 2]))])
.expect("the column fits the schema");
take_batch_inserts();
let summary = BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.record_batches([batch.clone()])
.execute()
.await
.expect("the insert runs");
assert_eq!(summary.rows_written, 2);
for insert in [
BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.record_batches([batch.clone()])
.exactly_once(),
BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.record_batches([batch.clone()])
.options(BigQueryStreamingWriteOptions::new().with_max_batch_rows(7))
.atomic(),
BigQueryExprBuilder::new(&db)
.insert()
.into(SHOP.table(ORDERS))
.record_batches([batch.clone()])
.buffered(),
] {
insert.execute().await.expect("the insert runs");
}
let inserts = take_batch_inserts();
let modes: Vec<BigQueryWriteMode> = inserts.iter().map(|i| i.params.options.mode).collect();
assert_eq!(
modes,
[
BigQueryWriteMode::Default,
BigQueryWriteMode::Committed,
BigQueryWriteMode::Pending,
BigQueryWriteMode::Buffered,
]
);
assert_eq!(inserts[2].params.options.max_batch_rows, Some(7));
assert!(inserts
.iter()
.all(|insert| insert.batches == [batch.clone()]));
}