use crate::errors::BigQueryCodecErrorKind;
use crate::types::error::CodecError;
use crate::write::descriptor::WritePlan;
use crate::BigQueryMissingValue;
use gcloud_sdk::google::cloud::bigquery::storage::v1::append_rows_request::{
self, MissingValueInterpretation, ProtoData,
};
use gcloud_sdk::google::cloud::bigquery::storage::v1::{AppendRowsRequest, ProtoRows, ProtoSchema};
use gcloud_sdk::prost::encoding::encoded_len_varint;
use gcloud_sdk::prost::Message;
use std::sync::Arc;
use tokio::time::Instant;
pub(crate) const MAX_REQUEST_BYTES: usize = 19_000_000;
const NESTED_LENGTH_SLACK: usize = 2 * 3;
#[derive(Debug, Clone)]
pub(crate) struct RequestTarget {
pub(crate) write_stream: String,
pub(crate) trace_id: Option<String>,
pub(crate) missing_value: Option<BigQueryMissingValue>,
}
pub(crate) fn append_request(
target: &RequestTarget,
offset: Option<i64>,
plan: Option<&WritePlan>,
rows: Vec<Vec<u8>>,
) -> AppendRowsRequest {
AppendRowsRequest {
write_stream: target.write_stream.clone(),
offset,
trace_id: target.trace_id.clone().unwrap_or_default(),
default_missing_value_interpretation: match target.missing_value {
None => MissingValueInterpretation::Unspecified,
Some(BigQueryMissingValue::Null) => MissingValueInterpretation::NullValue,
Some(BigQueryMissingValue::DefaultValue) => MissingValueInterpretation::DefaultValue,
}
.into(),
rows: Some(append_rows_request::Rows::ProtoRows(ProtoData {
writer_schema: plan.map(|plan| ProtoSchema {
proto_descriptor: Some(plan.descriptor().clone()),
}),
rows: Some(ProtoRows {
serialized_rows: rows,
}),
})),
..Default::default()
}
}
impl WritePlan {
fn request_overhead(&self, target: &RequestTarget) -> usize {
append_request(target, Some(i64::MAX), Some(self), Vec::new()).encoded_len()
+ NESTED_LENGTH_SLACK
}
}
#[derive(Debug)]
pub(crate) struct Batch {
pub(crate) index: u64,
pub(crate) first_row: u64,
pub(crate) rows: Vec<Vec<u8>>,
pub(crate) bytes: usize,
pub(crate) plan: Arc<WritePlan>,
pub(crate) attempts: usize,
}
impl Batch {
pub(crate) fn row_count(&self) -> u64 {
self.rows.len() as u64
}
}
#[derive(Debug)]
struct OpenBatch {
first_row: u64,
rows: Vec<Vec<u8>>,
bytes: usize,
since: Instant,
}
#[derive(Debug)]
pub(crate) struct Batcher {
plan: Arc<WritePlan>,
target: RequestTarget,
max_request_bytes: usize,
max_rows: Option<usize>,
overhead: usize,
open: Option<OpenBatch>,
next_index: u64,
next_row: u64,
}
impl Batcher {
pub(crate) fn new(
plan: Arc<WritePlan>,
target: &RequestTarget,
max_request_bytes: usize,
max_rows: Option<usize>,
) -> Self {
let overhead = plan.request_overhead(target);
Batcher {
plan,
target: target.clone(),
max_request_bytes,
max_rows,
overhead,
open: None,
next_index: 0,
next_row: 0,
}
}
#[cfg(test)]
pub(crate) fn overhead(&self) -> usize {
self.overhead
}
pub(crate) fn capacity(&self) -> usize {
self.max_request_bytes.saturating_sub(self.overhead)
}
pub(crate) fn next_row(&self) -> u64 {
self.next_row
}
pub(crate) fn cost(&self, len: usize) -> Result<usize, CodecError> {
let cost = 1 + encoded_len_varint(len as u64) + len;
if cost > self.capacity() {
return Err(CodecError::new(
BigQueryCodecErrorKind::RowTooLarge,
format!(
"the row takes {cost} bytes and a request has room for {} rows bytes under \
max_request_bytes = {}",
self.capacity(),
self.max_request_bytes
),
));
}
Ok(cost)
}
pub(crate) fn fits(&self, cost: usize) -> bool {
match &self.open {
None => true,
Some(open) => {
open.bytes + cost <= self.capacity()
&& self.max_rows.is_none_or(|max| open.rows.len() < max)
}
}
}
pub(crate) fn is_full(&self) -> bool {
match (&self.open, self.max_rows) {
(Some(open), Some(max)) => open.rows.len() >= max,
_ => false,
}
}
pub(crate) fn push(&mut self, row: Vec<u8>, cost: usize, now: Instant) -> bool {
let opened = self.open.is_none();
let next_row = self.next_row;
let open = self.open.get_or_insert_with(|| OpenBatch {
first_row: next_row,
rows: Vec::new(),
bytes: 0,
since: now,
});
open.rows.push(row);
open.bytes += cost;
self.next_row += 1;
opened
}
pub(crate) fn open_since(&self) -> Option<Instant> {
self.open.as_ref().map(|open| open.since)
}
pub(crate) fn open_bytes(&self) -> Option<usize> {
self.open.as_ref().map(|open| open.bytes)
}
pub(crate) fn seal(&mut self) -> Option<Batch> {
let open = self.open.take()?;
let index = self.next_index;
self.next_index += 1;
Some(Batch {
index,
first_row: open.first_row,
rows: open.rows,
bytes: open.bytes,
plan: self.plan.clone(),
attempts: 0,
})
}
pub(crate) fn set_plan(&mut self, plan: Arc<WritePlan>) {
debug_assert!(self.open.is_none(), "a batch holds rows of one plan only");
self.overhead = plan.request_overhead(&self.target);
self.plan = plan;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::testkit::field;
use crate::{BigQueryFieldMode, BigQueryFieldType, BigQueryTableSchema};
use gcloud_sdk::prost::Message;
use proptest::prelude::*;
use std::sync::Arc;
const STREAM: &str =
"projects/p/datasets/shop/tables/orders/streams/Cic2NjQ2ZjZkNjE2OTZlEhAKCgoKCgoKCgoKCgoK";
fn plan(columns: usize) -> Arc<WritePlan> {
let fields = (0..columns)
.map(|i| {
field(
&format!("column_with_a_long_name_{i}"),
BigQueryFieldType::String { max_length: None },
BigQueryFieldMode::Nullable,
)
})
.collect();
Arc::new(WritePlan::new(&BigQueryTableSchema { fields }, false))
}
fn target() -> RequestTarget {
RequestTarget {
write_stream: STREAM.to_string(),
trace_id: Some("bigquery-rs:test".into()),
missing_value: None,
}
}
fn batch_all(batcher: &mut Batcher, rows: &[usize]) -> Result<Vec<Batch>, CodecError> {
let mut sealed = Vec::new();
for &len in rows {
let cost = batcher.cost(len)?;
if !batcher.fits(cost) {
sealed.extend(batcher.seal());
}
batcher.push(vec![7u8; len], cost, Instant::now());
if batcher.is_full() {
sealed.extend(batcher.seal());
}
}
sealed.extend(batcher.seal());
Ok(sealed)
}
proptest! {
#[test]
fn requests_never_exceed_the_budget(
max_request_bytes in 2_000usize..40_000,
columns in 1usize..40,
rows in proptest::collection::vec(0usize..3_000, 1..200),
max_rows in proptest::option::of(1usize..20),
) {
let plan = plan(columns);
let target = target();
let mut batcher = Batcher::new(plan.clone(), &target, max_request_bytes, max_rows);
let capacity = batcher.capacity();
let fitting: Vec<usize> = rows.iter().copied().filter(|&len| len + 8 <= capacity).collect();
prop_assume!(!fitting.is_empty());
let batches = batch_all(&mut batcher, &fitting).expect("every row fits");
let mut written = Vec::new();
for (k, batch) in batches.iter().enumerate() {
prop_assert_eq!(batch.index, k as u64);
prop_assert_eq!(batch.first_row, written.len() as u64);
if let Some(max_rows) = max_rows {
prop_assert!(batch.rows.len() <= max_rows);
}
let request = append_request(
&target,
Some(i64::MAX),
Some(&plan),
batch.rows.clone(),
);
let size = request.encoded_len();
prop_assert!(size <= max_request_bytes, "{} > {}", size, max_request_bytes);
prop_assert_eq!(size - batch.bytes <= batcher.overhead(), true);
written.extend(batch.rows.iter().map(Vec::len));
}
prop_assert_eq!(written, fitting.clone());
for pair in batches.windows(2) {
let next = pair[1].rows[0].len();
let full_by_rows = max_rows.is_some_and(|m| pair[0].rows.len() >= m);
let cost = 1 + encoded_len_varint(next as u64) + next;
prop_assert!(full_by_rows || pair[0].bytes + cost > capacity);
}
}
}
#[test]
fn a_row_larger_than_the_budget_is_row_too_large() {
let target = target();
let batcher = Batcher::new(plan(3), &target, 1_000, None);
let capacity = batcher.capacity();
let fits = capacity - 1 - encoded_len_varint(capacity as u64);
assert!(batcher.cost(fits).is_ok());
let err = batcher
.cost(fits + 1)
.expect_err("one byte over the budget");
assert_eq!(err.kind(), BigQueryCodecErrorKind::RowTooLarge);
let err = batcher.cost(10_000).expect_err("far over the budget");
assert_eq!(err.kind(), BigQueryCodecErrorKind::RowTooLarge);
}
#[test]
fn descriptor_size_is_reserved_in_every_request() {
let target = target();
let wide = plan(200);
let budget = 30_000;
let mut batcher = Batcher::new(wide.clone(), &target, budget, None);
let descriptor = wide.descriptor().encoded_len();
assert!(
batcher.overhead() > descriptor,
"{} <= {descriptor}",
batcher.overhead()
);
let batches = batch_all(&mut batcher, &[500; 200]).expect("rows fit");
assert!(batches.len() > 2);
for batch in &batches {
let with = append_request(&target, Some(1), Some(&wide), batch.rows.clone());
let without = append_request(&target, Some(1), None, batch.rows.clone());
assert!(with.encoded_len() <= budget, "{}", with.encoded_len());
assert!(without.encoded_len() + descriptor <= budget);
assert!(writer_schema_is_absent(&without));
}
}
fn writer_schema_is_absent(request: &AppendRowsRequest) -> bool {
match &request.rows {
Some(append_rows_request::Rows::ProtoRows(data)) => data.writer_schema.is_none(),
_ => false,
}
}
}