use crate::errors::BigQueryCodecErrorKind;
use crate::types::error::CodecError;
use crate::write::arrow::ArrowWriterSchema;
use crate::write::descriptor::WritePlan;
use crate::BigQueryMissingValue;
use gcloud_sdk::google::cloud::bigquery::storage::v1::append_rows_request::{
self, ArrowData, MissingValueInterpretation, ProtoData,
};
use gcloud_sdk::google::cloud::bigquery::storage::v1::{
AppendRowsRequest, ArrowRecordBatch, ArrowSchema, ProtoRows, ProtoSchema,
};
use gcloud_sdk::prost::encoding::encoded_len_varint;
use gcloud_sdk::prost::Message;
use std::sync::Arc;
use tokio::time::Instant;
pub(crate) const MAX_REQUEST_BYTES: usize = 19_000_000;
const NESTED_LENGTH_SLACK: usize = 2 * 3;
#[derive(Debug, Clone)]
pub(crate) struct RequestTarget {
pub(crate) write_stream: String,
pub(crate) trace_id: Option<String>,
pub(crate) missing_value: Option<BigQueryMissingValue>,
}
pub(crate) fn append_request(
target: &RequestTarget,
offset: Option<i64>,
rows: &BatchRows,
with_schema: bool,
) -> AppendRowsRequest {
let rows = match rows {
BatchRows::Proto { plan, rows } => append_rows_request::Rows::ProtoRows(ProtoData {
writer_schema: with_schema.then(|| ProtoSchema {
proto_descriptor: Some(plan.descriptor().clone()),
}),
rows: Some(ProtoRows {
serialized_rows: rows.clone(),
}),
}),
BatchRows::Arrow {
schema,
record_batch,
..
} => append_rows_request::Rows::ArrowRows(ArrowData {
writer_schema: with_schema.then(|| ArrowSchema {
serialized_schema: schema.serialized().to_vec(),
}),
rows: Some(ArrowRecordBatch {
serialized_record_batch: record_batch.clone(),
..Default::default()
}),
}),
};
AppendRowsRequest {
write_stream: target.write_stream.clone(),
offset,
trace_id: target.trace_id.clone().unwrap_or_default(),
default_missing_value_interpretation: match target.missing_value {
None => MissingValueInterpretation::Unspecified,
Some(BigQueryMissingValue::Null) => MissingValueInterpretation::NullValue,
Some(BigQueryMissingValue::DefaultValue) => MissingValueInterpretation::DefaultValue,
}
.into(),
rows: Some(rows),
..Default::default()
}
}
#[derive(Debug, Clone)]
pub(crate) enum WriterSchema {
Proto(Arc<WritePlan>),
Arrow(Arc<ArrowWriterSchema>),
}
impl WriterSchema {
pub(crate) fn is(&self, other: &WriterSchema) -> bool {
match (self, other) {
(WriterSchema::Proto(one), WriterSchema::Proto(other)) => Arc::ptr_eq(one, other),
(WriterSchema::Arrow(one), WriterSchema::Arrow(other)) => Arc::ptr_eq(one, other),
_ => false,
}
}
fn request_overhead(&self, target: &RequestTarget) -> usize {
let empty = match self {
WriterSchema::Proto(plan) => BatchRows::Proto {
plan: plan.clone(),
rows: Vec::new(),
},
WriterSchema::Arrow(schema) => BatchRows::Arrow {
schema: schema.clone(),
record_batch: Vec::new(),
row_count: 0,
},
};
append_request(target, Some(i64::MAX), &empty, true).encoded_len() + NESTED_LENGTH_SLACK
}
}
#[derive(Debug)]
pub(crate) enum BatchRows {
Proto {
plan: Arc<WritePlan>,
rows: Vec<Vec<u8>>,
},
Arrow {
schema: Arc<ArrowWriterSchema>,
record_batch: Vec<u8>,
row_count: u64,
},
}
impl BatchRows {
pub(crate) fn row_count(&self) -> u64 {
match self {
BatchRows::Proto { rows, .. } => rows.len() as u64,
BatchRows::Arrow { row_count, .. } => *row_count,
}
}
pub(crate) fn schema(&self) -> WriterSchema {
match self {
BatchRows::Proto { plan, .. } => WriterSchema::Proto(plan.clone()),
BatchRows::Arrow { schema, .. } => WriterSchema::Arrow(schema.clone()),
}
}
}
#[derive(Debug)]
pub(crate) struct Batch {
pub(crate) index: u64,
pub(crate) first_row: u64,
pub(crate) rows: BatchRows,
pub(crate) bytes: usize,
pub(crate) attempts: usize,
}
impl Batch {
pub(crate) fn row_count(&self) -> u64 {
self.rows.row_count()
}
}
#[derive(Debug)]
struct OpenBatch {
first_row: u64,
rows: Vec<Vec<u8>>,
bytes: usize,
since: Instant,
}
#[derive(Debug)]
pub(crate) struct Batcher {
plan: Arc<WritePlan>,
target: RequestTarget,
max_request_bytes: usize,
max_rows: Option<usize>,
overhead: usize,
open: Option<OpenBatch>,
next_index: u64,
next_row: u64,
}
impl Batcher {
pub(crate) fn new(
plan: Arc<WritePlan>,
target: &RequestTarget,
max_request_bytes: usize,
max_rows: Option<usize>,
) -> Self {
let overhead = WriterSchema::Proto(plan.clone()).request_overhead(target);
Batcher {
plan,
target: target.clone(),
max_request_bytes,
max_rows,
overhead,
open: None,
next_index: 0,
next_row: 0,
}
}
#[cfg(test)]
pub(crate) fn overhead(&self) -> usize {
self.overhead
}
pub(crate) fn capacity(&self) -> usize {
self.max_request_bytes.saturating_sub(self.overhead)
}
pub(crate) fn arrow_capacity(&self, schema: &Arc<ArrowWriterSchema>) -> usize {
let overhead = WriterSchema::Arrow(schema.clone()).request_overhead(&self.target);
self.max_request_bytes.saturating_sub(overhead)
}
pub(crate) fn max_rows(&self) -> Option<usize> {
self.max_rows
}
pub(crate) fn next_row(&self) -> u64 {
self.next_row
}
pub(crate) fn cost(&self, len: usize) -> Result<usize, CodecError> {
let cost = 1 + encoded_len_varint(len as u64) + len;
if cost > self.capacity() {
return Err(CodecError::new(
BigQueryCodecErrorKind::RowTooLarge,
format!(
"the row takes {cost} bytes and a request has room for {} rows bytes under \
max_request_bytes = {}",
self.capacity(),
self.max_request_bytes
),
));
}
Ok(cost)
}
pub(crate) fn fits(&self, cost: usize) -> bool {
match &self.open {
None => true,
Some(open) => {
open.bytes + cost <= self.capacity()
&& self.max_rows.is_none_or(|max| open.rows.len() < max)
}
}
}
pub(crate) fn is_full(&self) -> bool {
match (&self.open, self.max_rows) {
(Some(open), Some(max)) => open.rows.len() >= max,
_ => false,
}
}
pub(crate) fn push(&mut self, row: Vec<u8>, cost: usize, now: Instant) -> bool {
let opened = self.open.is_none();
let next_row = self.next_row;
let open = self.open.get_or_insert_with(|| OpenBatch {
first_row: next_row,
rows: Vec::new(),
bytes: 0,
since: now,
});
open.rows.push(row);
open.bytes += cost;
self.next_row += 1;
opened
}
pub(crate) fn open_since(&self) -> Option<Instant> {
self.open.as_ref().map(|open| open.since)
}
pub(crate) fn open_bytes(&self) -> Option<usize> {
self.open.as_ref().map(|open| open.bytes)
}
pub(crate) fn seal(&mut self) -> Option<Batch> {
let open = self.open.take()?;
let index = self.next_index;
self.next_index += 1;
Some(Batch {
index,
first_row: open.first_row,
rows: BatchRows::Proto {
plan: self.plan.clone(),
rows: open.rows,
},
bytes: open.bytes,
attempts: 0,
})
}
pub(crate) fn seal_rows(&mut self, rows: BatchRows, bytes: usize) -> Batch {
debug_assert!(self.open.is_none(), "batches stay in write order");
let index = self.next_index;
self.next_index += 1;
let first_row = self.next_row;
self.next_row += rows.row_count();
Batch {
index,
first_row,
rows,
bytes,
attempts: 0,
}
}
pub(crate) fn set_plan(&mut self, plan: Arc<WritePlan>) {
debug_assert!(self.open.is_none(), "a batch holds rows of one plan only");
self.overhead = WriterSchema::Proto(plan.clone()).request_overhead(&self.target);
self.plan = plan;
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::testkit::field;
use crate::{BigQueryFieldMode, BigQueryFieldType, BigQueryTableSchema};
use gcloud_sdk::prost::Message;
use proptest::prelude::*;
use std::sync::Arc;
const STREAM: &str =
"projects/p/datasets/shop/tables/orders/streams/Cic2NjQ2ZjZkNjE2OTZlEhAKCgoKCgoKCgoKCgoK";
fn plan(columns: usize) -> Arc<WritePlan> {
let fields = (0..columns)
.map(|i| {
field(
&format!("column_with_a_long_name_{i}"),
BigQueryFieldType::String { max_length: None },
BigQueryFieldMode::Nullable,
)
})
.collect();
Arc::new(WritePlan::new(&BigQueryTableSchema { fields }, false))
}
fn target() -> RequestTarget {
RequestTarget {
write_stream: STREAM.to_string(),
trace_id: Some("bigquery-rs:test".into()),
missing_value: None,
}
}
fn batch_all(batcher: &mut Batcher, rows: &[usize]) -> Result<Vec<Batch>, CodecError> {
let mut sealed = Vec::new();
for &len in rows {
let cost = batcher.cost(len)?;
if !batcher.fits(cost) {
sealed.extend(batcher.seal());
}
batcher.push(vec![7u8; len], cost, Instant::now());
if batcher.is_full() {
sealed.extend(batcher.seal());
}
}
sealed.extend(batcher.seal());
Ok(sealed)
}
proptest! {
#[test]
fn requests_never_exceed_the_budget(
max_request_bytes in 2_000usize..40_000,
columns in 1usize..40,
rows in proptest::collection::vec(0usize..3_000, 1..200),
max_rows in proptest::option::of(1usize..20),
) {
let plan = plan(columns);
let target = target();
let mut batcher = Batcher::new(plan.clone(), &target, max_request_bytes, max_rows);
let capacity = batcher.capacity();
let fitting: Vec<usize> = rows.iter().copied().filter(|&len| len + 8 <= capacity).collect();
prop_assume!(!fitting.is_empty());
let batches = batch_all(&mut batcher, &fitting).expect("every row fits");
let mut written = Vec::new();
for (k, batch) in batches.iter().enumerate() {
prop_assert_eq!(batch.index, k as u64);
prop_assert_eq!(batch.first_row, written.len() as u64);
if let Some(max_rows) = max_rows {
prop_assert!(batch.row_count() <= max_rows as u64);
}
let request = append_request(&target, Some(i64::MAX), &batch.rows, true);
let size = request.encoded_len();
prop_assert!(size <= max_request_bytes, "{} > {}", size, max_request_bytes);
prop_assert_eq!(size - batch.bytes <= batcher.overhead(), true);
written.extend(proto_rows(batch).iter().map(Vec::len));
}
prop_assert_eq!(written, fitting.clone());
for pair in batches.windows(2) {
let next = proto_rows(&pair[1])[0].len();
let full_by_rows = max_rows.is_some_and(|m| pair[0].row_count() >= m as u64);
let cost = 1 + encoded_len_varint(next as u64) + next;
prop_assert!(full_by_rows || pair[0].bytes + cost > capacity);
}
}
}
#[test]
fn a_row_larger_than_the_budget_is_row_too_large() {
let target = target();
let batcher = Batcher::new(plan(3), &target, 1_000, None);
let capacity = batcher.capacity();
let fits = capacity - 1 - encoded_len_varint(capacity as u64);
assert!(batcher.cost(fits).is_ok());
let err = batcher
.cost(fits + 1)
.expect_err("one byte over the budget");
assert_eq!(err.kind(), BigQueryCodecErrorKind::RowTooLarge);
let err = batcher.cost(10_000).expect_err("far over the budget");
assert_eq!(err.kind(), BigQueryCodecErrorKind::RowTooLarge);
}
#[test]
fn descriptor_size_is_reserved_in_every_request() {
let target = target();
let wide = plan(200);
let budget = 30_000;
let mut batcher = Batcher::new(wide.clone(), &target, budget, None);
let descriptor = wide.descriptor().encoded_len();
assert!(
batcher.overhead() > descriptor,
"{} <= {descriptor}",
batcher.overhead()
);
let batches = batch_all(&mut batcher, &[500; 200]).expect("rows fit");
assert!(batches.len() > 2);
for batch in &batches {
let with = append_request(&target, Some(1), &batch.rows, true);
let without = append_request(&target, Some(1), &batch.rows, false);
assert!(with.encoded_len() <= budget, "{}", with.encoded_len());
assert!(without.encoded_len() + descriptor <= budget);
assert!(writer_schema_is_absent(&without));
}
}
fn proto_rows(batch: &Batch) -> &[Vec<u8>] {
match &batch.rows {
BatchRows::Proto { rows, .. } => rows,
BatchRows::Arrow { .. } => panic!("the batcher seals protobuf rows"),
}
}
fn writer_schema_is_absent(request: &AppendRowsRequest) -> bool {
match &request.rows {
Some(append_rows_request::Rows::ProtoRows(data)) => data.writer_schema.is_none(),
_ => false,
}
}
}