use crate::errors::{BigQueryCodecErrorKind, BigQueryError};
use crate::types::error::CodecError;
use crate::write::arrow::{ArrowWriterSchema, RecordBatchSlices};
use crate::write::batch::{BatchRows, Batcher, RequestTarget, MAX_REQUEST_BYTES};
use crate::write::connection::{
Command, ConnectionTask, FinishKind, Finished, Shared, TaskSettings, TASK_ENDED,
};
use crate::write::descriptor::{relaxes_a_required_field, WritePlan};
use crate::write::encoder::Encoder;
use crate::{
BigQueryDb, BigQueryFinalizedStream, BigQueryResult, BigQueryStreamingWriteOptions,
BigQueryTableRef, BigQueryTableSchema, BigQueryWriteMode, BigQueryWriteResponse,
BigQueryWriteSummary,
};
use crate::{BigQueryInstant, BigQueryWriteStreamName};
use arrow_array::RecordBatch;
use futures::stream::BoxStream;
use futures::StreamExt;
use gcloud_sdk::google::cloud::bigquery::storage::v1::storage_error::StorageErrorCode;
use gcloud_sdk::google::cloud::bigquery::storage::v1::write_stream::Type as WriteStreamType;
use gcloud_sdk::google::cloud::bigquery::storage::v1::{
BatchCommitWriteStreamsRequest, CreateWriteStreamRequest, FinalizeWriteStreamRequest,
FlushRowsRequest, GetWriteStreamRequest, WriteStream, WriteStreamView,
};
use gcloud_sdk::prost_types::Timestamp;
use serde::Serialize;
use std::marker::PhantomData;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinHandle;
use tokio::time::Instant;
use tracing::field::Empty;
use tracing::{debug_span, warn, Span};
impl TryFrom<&WriteStream> for BigQueryTableSchema {
type Error = BigQueryError;
fn try_from(stream: &WriteStream) -> Result<Self, Self::Error> {
let schema = stream.table_schema.as_ref().ok_or_else(|| {
BigQueryError::write_stream(
"NO_TABLE_SCHEMA",
stream.name.clone(),
"the write stream came back without its table schema",
)
})?;
BigQueryTableSchema::try_from(schema)
}
}
impl BigQueryDb {
pub(crate) async fn get_write_stream(
&self,
span: &Span,
name: &BigQueryWriteStreamName,
view: WriteStreamView,
) -> BigQueryResult<WriteStream> {
let request = GetWriteStreamRequest {
name: name.as_str().to_string(),
view: view.into(),
};
self.retry(span, "get the write stream", &request, |request| {
let mut client = self.write_client();
async move { client.get_write_stream(request).await }
})
.await
}
async fn create_write_stream(
&self,
span: &Span,
table_path: &str,
stream_type: WriteStreamType,
) -> BigQueryResult<WriteStream> {
let request = CreateWriteStreamRequest {
parent: table_path.to_string(),
write_stream: Some(WriteStream {
r#type: stream_type.into(),
..Default::default()
}),
};
self.retry(span, "create a write stream", &request, |request| {
let mut client = self.write_client();
async move { client.create_write_stream(request).await }
})
.await
}
pub(crate) async fn finalize_write_stream(
&self,
span: &Span,
name: &BigQueryWriteStreamName,
) -> BigQueryResult<i64> {
let request = FinalizeWriteStreamRequest {
name: name.as_str().to_string(),
};
let response = self
.retry(span, "finalize the write stream", &request, |request| {
let mut client = self.write_client();
async move { client.finalize_write_stream(request).await }
})
.await?;
Ok(response.row_count)
}
pub(crate) async fn flush_rows(
&self,
span: &Span,
name: &BigQueryWriteStreamName,
offset: i64,
) -> BigQueryResult<i64> {
let request = FlushRowsRequest {
write_stream: name.as_str().to_string(),
offset: Some(offset),
};
let response = self
.retry(span, "flush the write stream", &request, |request| {
let mut client = self.write_client();
async move { client.flush_rows(request).await }
})
.await?;
Ok(response.offset)
}
pub(crate) async fn batch_commit(
&self,
span: &Span,
table_path: &str,
streams: &[BigQueryWriteStreamName],
) -> BigQueryResult<BigQueryInstant> {
let request = BatchCommitWriteStreamsRequest {
parent: table_path.to_string(),
write_streams: streams.iter().map(|s| s.as_str().to_string()).collect(),
};
let attempts = AtomicUsize::new(0);
let response = self
.retry(span, "commit the write streams", &request, |request| {
attempts.fetch_add(1, Ordering::Relaxed);
let mut client = self.write_client();
async move { client.batch_commit_write_streams(request).await }
})
.await?;
let commit_time = match response.stream_errors.first() {
None => response.commit_time.ok_or_else(|| {
BigQueryError::write_stream(
"NO_COMMIT_TIME",
request.write_streams.join(", "),
"the commit reported no error and no commit time",
)
})?,
Some(error) => {
let already_committed = response.stream_errors.iter().all(|stream_error| {
stream_error.code() == StorageErrorCode::StreamAlreadyCommitted
});
let earlier_commit = if already_committed && attempts.into_inner() > 1 {
self.common_commit_time(span, streams).await
} else {
None
};
earlier_commit.ok_or_else(|| {
let code = StorageErrorCode::try_from(error.code)
.map(|code| code.as_str_name().to_string())
.unwrap_or_else(|_| error.code.to_string());
BigQueryError::write_stream(
&code,
error.entity.clone(),
error.error_message.clone(),
)
})?
}
};
BigQueryInstant::new(commit_time.seconds, commit_time.nanos).map_err(|err| {
BigQueryError::write_stream(
"INVALID_COMMIT_TIME",
request.write_streams.join(", "),
format!("the commit time {commit_time:?} is not a timestamp: {err}"),
)
})
}
async fn common_commit_time(
&self,
span: &Span,
streams: &[BigQueryWriteStreamName],
) -> Option<Timestamp> {
let mut common: Option<Timestamp> = None;
for stream in streams {
let stream = self
.get_write_stream(span, stream, WriteStreamView::Basic)
.await
.ok()?;
if stream.r#type() != WriteStreamType::Pending {
return None;
}
let commit_time = stream.commit_time?;
if *common.get_or_insert(commit_time) != commit_time {
return None;
}
}
common
}
}
impl BigQueryStreamingWriteOptions {
fn check(&self, cdc: bool) -> BigQueryResult<()> {
if self.max_request_bytes == 0 || self.max_request_bytes > MAX_REQUEST_BYTES {
return Err(BigQueryError::invalid_parameters(
"max_request_bytes",
format!(
"{} is outside 1..={MAX_REQUEST_BYTES}; BigQuery ends the whole connection on a \
request over about 20 MB",
self.max_request_bytes
),
));
}
if self.max_batch_rows == Some(0) {
return Err(BigQueryError::invalid_parameters(
"max_batch_rows",
"a batch holds at least one row",
));
}
if self.max_inflight_requests == 0 {
return Err(BigQueryError::invalid_parameters(
"max_inflight_requests",
"at least one request must be allowed in flight",
));
}
if cdc && self.mode != BigQueryWriteMode::Default {
return Err(BigQueryError::invalid_parameters(
"mode",
format!(
"CDC writes go through the default stream only, not {:?}",
self.mode
),
));
}
Ok(())
}
}
impl BigQueryCodecErrorKind {
fn may_be_a_stale_schema(self) -> bool {
matches!(
self,
BigQueryCodecErrorKind::UnknownField
| BigQueryCodecErrorKind::NullForRequired
| BigQueryCodecErrorKind::MissingRequiredField
)
}
}
pub(crate) struct WriterCore {
db: BigQueryDb,
span: Span,
table: BigQueryTableRef,
stream: BigQueryWriteStreamName,
mode: BigQueryWriteMode,
cdc: bool,
refresh_interval: Duration,
last_refresh: Option<Instant>,
encoder: Encoder,
row_hint: usize,
arrow_schema: Option<(Arc<ArrowWriterSchema>, usize)>,
shared: Arc<Shared>,
commands: mpsc::UnboundedSender<Command>,
task: Option<JoinHandle<()>>,
finished: bool,
}
pub(crate) type ResponseStream<'b> = BoxStream<'b, BigQueryResult<BigQueryWriteResponse>>;
impl WriterCore {
pub(crate) async fn open<'b>(
db: &BigQueryDb,
table: BigQueryTableRef,
options: BigQueryStreamingWriteOptions,
cdc: bool,
) -> BigQueryResult<(Self, ResponseStream<'b>)> {
options.check(cdc)?;
let table_path = table.table_path(&db.options().google_project_id);
let span = debug_span!(
"BigQuery streaming write",
"/bigquery/table" = %table,
"/bigquery/write_mode" = ?options.mode,
"/bigquery/rows_appended" = Empty,
"/bigquery/bytes_sent" = Empty,
"/bigquery/appends" = Empty,
"/bigquery/retries" = Empty,
);
let stream = match options.mode {
BigQueryWriteMode::Default => {
let default =
BigQueryWriteStreamName::reported(format!("{table_path}/streams/_default"));
db.get_write_stream(&span, &default, WriteStreamView::Full)
.await?
}
BigQueryWriteMode::Committed => {
db.create_write_stream(&span, &table_path, WriteStreamType::Committed)
.await?
}
BigQueryWriteMode::Pending => {
db.create_write_stream(&span, &table_path, WriteStreamType::Pending)
.await?
}
BigQueryWriteMode::Buffered => {
db.create_write_stream(&span, &table_path, WriteStreamType::Buffered)
.await?
}
};
let schema = BigQueryTableSchema::try_from(&stream)?;
let stream_name = BigQueryWriteStreamName::reported(stream.name.clone());
let plan = Arc::new(WritePlan::new(&schema, cdc));
let target = RequestTarget {
write_stream: stream.name.clone(),
trace_id: options.trace_id.as_ref().map(ToString::to_string),
missing_value: options.missing_value,
};
let batcher = Batcher::new(
plan.clone(),
&target,
options.max_request_bytes,
options.max_batch_rows,
);
let shared = Arc::new(Shared::new(
batcher,
options.max_inflight_requests,
options.max_inflight_bytes,
));
let (commands, commands_rx) = mpsc::unbounded_channel();
let (responses, responses_rx) = futures::channel::mpsc::unbounded();
let task = ConnectionTask::new(
TaskSettings {
db: db.clone(),
span: span.clone(),
mode: options.mode,
target,
stream: stream_name.clone(),
table: table.clone(),
table_path,
batch_delay: options.max_batch_delay,
},
shared.clone(),
commands_rx,
responses,
);
let task = tokio::spawn(task.run());
let core = WriterCore {
db: db.clone(),
span,
table,
stream: stream_name,
mode: options.mode,
cdc,
refresh_interval: options.schema_refresh_interval,
last_refresh: None,
encoder: Encoder::new(plan),
row_hint: 0,
arrow_schema: None,
shared,
commands,
task: Some(task),
finished: false,
};
Ok((core, responses_rx.boxed()))
}
pub(crate) fn mode(&self) -> BigQueryWriteMode {
self.mode
}
pub(crate) fn stream_name(&self) -> &BigQueryWriteStreamName {
&self.stream
}
fn check_failed(&self) -> BigQueryResult<()> {
match &self.shared.lock().failed {
Some(err) => Err(err.clone()),
None => Ok(()),
}
}
fn command(&self, command: Command) -> BigQueryResult<()> {
self.commands
.send(command)
.map_err(|_| BigQueryError::system("WRITER_TASK_ENDED", TASK_ENDED))
}
pub(crate) async fn write_with<F>(&mut self, mut encode: F) -> BigQueryResult<()>
where
F: FnMut(&mut Encoder, &mut Vec<u8>) -> Result<(), CodecError>,
{
self.check_failed()?;
let pending = self.shared.lock().pending_schema.take();
if let Some(schema) = pending {
self.switch_plan(schema).await?;
}
let index = self.shared.lock().batcher.next_row();
let mut row = Vec::with_capacity(self.row_hint);
if let Err(err) = encode(&mut self.encoder, &mut row) {
if !(err.kind()).may_be_a_stale_schema() || !self.refresh_schema().await? {
return Err(err.with_row(index).into_serialize());
}
row.clear();
encode(&mut self.encoder, &mut row)
.map_err(|err| err.with_row(index).into_serialize())?;
}
self.row_hint = row.len();
let cost = self
.shared
.lock()
.batcher
.cost(row.len())
.map_err(|err| err.with_row(index).into_serialize())?;
if !self.shared.lock().batcher.fits(cost) {
self.seal_open().await?;
}
let (opened, full) = {
let mut state = self.shared.lock();
if let Some(err) = &state.failed {
return Err(err.clone());
}
let opened = state.batcher.push(row, cost, Instant::now());
(opened, state.batcher.is_full())
};
if opened {
self.command(Command::Wake)?;
}
if full {
self.seal_open().await?;
}
Ok(())
}
pub(crate) async fn write_record_batch(&mut self, batch: &RecordBatch) -> BigQueryResult<()> {
self.check_failed()?;
let (schema, capacity) = match &self.arrow_schema {
Some((schema, capacity)) if schema.describes(batch) => (schema.clone(), *capacity),
_ => {
let schema = Arc::new(ArrowWriterSchema::new(batch.schema())?);
let capacity = self.shared.lock().batcher.arrow_capacity(&schema);
self.arrow_schema = Some((schema.clone(), capacity));
(schema, capacity)
}
};
let (first_row, max_rows) = {
let state = self.shared.lock();
(state.batcher.next_row(), state.batcher.max_rows())
};
let slices = RecordBatchSlices::new(batch, &schema, capacity, max_rows, first_row);
for slice in slices {
let slice = slice.map_err(CodecError::into_serialize)?;
let rows = BatchRows::Arrow {
schema: schema.clone(),
record_batch: slice.serialized,
row_count: slice.row_count as u64,
};
self.seal_rows(rows, slice.cost).await?;
}
Ok(())
}
async fn seal_rows(&self, rows: BatchRows, bytes: usize) -> BigQueryResult<()> {
self.seal_open().await?;
loop {
let notified = self.shared.capacity.notified();
tokio::pin!(notified);
notified.as_mut().enable();
{
let mut state = self.shared.lock();
if let Some(err) = &state.failed {
return Err(err.clone());
}
if self.shared.has_room(&state, bytes) {
let batch = state.batcher.seal_rows(rows, bytes);
Shared::queue_locked(&mut state, batch);
drop(state);
return self.command(Command::Wake);
}
}
notified.await;
}
}
async fn refresh_schema(&mut self) -> BigQueryResult<bool> {
let now = Instant::now();
if self
.last_refresh
.is_some_and(|last| now < last + self.refresh_interval)
{
return Ok(false);
}
self.last_refresh = Some(now);
let stream = self
.db
.get_write_stream(&self.span, &self.stream, WriteStreamView::Full)
.await?;
let schema = BigQueryTableSchema::try_from(&stream)?;
self.switch_plan(schema).await
}
async fn switch_plan(&mut self, schema: BigQueryTableSchema) -> BigQueryResult<bool> {
if &schema == self.encoder.plan().schema() {
return Ok(false);
}
self.seal_open().await?;
let relaxed =
relaxes_a_required_field(&self.encoder.plan().schema().fields, &schema.fields);
let plan = Arc::new(WritePlan::new(&schema, self.cdc));
self.shared.lock().batcher.set_plan(plan.clone());
self.encoder = Encoder::new(plan);
if relaxed {
self.command(Command::Reconnect)?;
}
Ok(true)
}
async fn seal_open(&self) -> BigQueryResult<()> {
loop {
let notified = self.shared.capacity.notified();
tokio::pin!(notified);
notified.as_mut().enable();
{
let mut state = self.shared.lock();
if let Some(err) = &state.failed {
return Err(err.clone());
}
if state.batcher.open_bytes().is_none() {
return Ok(());
}
if self.shared.can_seal(&state) {
self.shared.seal_locked(&mut state);
drop(state);
return self.command(Command::Wake);
}
}
notified.await;
}
}
pub(crate) async fn flush(&mut self) -> BigQueryResult<i64> {
self.seal_open().await?;
let (tx, rx) = oneshot::channel();
self.command(Command::Flush(tx))?;
rx.await
.map_err(|_| BigQueryError::system("WRITER_TASK_ENDED", TASK_ENDED))?
}
fn check_mode(&self, mode: BigQueryWriteMode, operation: &str) -> BigQueryResult<()> {
if self.mode == mode {
return Ok(());
}
Err(BigQueryError::invalid_parameters(
"mode",
format!(
"{operation} is for {mode:?} streams, the writer is {:?}",
self.mode
),
))
}
pub(crate) async fn flush_rows(&mut self) -> BigQueryResult<Option<i64>> {
self.check_mode(BigQueryWriteMode::Buffered, "flush_rows()")?;
let written = self.flush().await?;
if written == 0 {
return Ok(None);
}
self.flush_rows_to(written - 1).await.map(Some)
}
pub(crate) async fn flush_rows_to(&mut self, offset: i64) -> BigQueryResult<i64> {
self.check_mode(BigQueryWriteMode::Buffered, "flush_rows_to()")?;
self.db.flush_rows(&self.span, &self.stream, offset).await
}
pub(crate) async fn finish(&mut self, kind: FinishKind) -> BigQueryResult<Finished> {
self.finished = true;
let sealed = self.seal_open().await;
let (tx, rx) = oneshot::channel();
self.command(Command::Finish(kind, tx))?;
let finished = rx
.await
.map_err(|_| BigQueryError::system("WRITER_TASK_ENDED", TASK_ENDED))?;
if let Some(task) = self.task.take() {
let _ = task.await;
}
sealed?;
finished
}
pub(crate) fn abandon(&mut self) {
self.finished = true;
if let Some(task) = self.task.take() {
task.abort();
}
}
pub(crate) fn table(&self) -> &BigQueryTableRef {
&self.table
}
}
impl Drop for WriterCore {
fn drop(&mut self) {
if !self.finished {
warn!(
parent: &self.span,
stream = %self.stream,
"A BigQuery streaming writer was dropped without finish(); rows not yet \
acknowledged may be lost and a pending stream stays uncommitted."
);
}
if let Some(task) = self.task.take() {
task.abort();
}
}
}
pub struct BigQueryStreamingWriter<T> {
core: WriterCore,
_row: PhantomData<fn(&T)>,
}
impl<T: Serialize> BigQueryStreamingWriter<T> {
pub async fn write(&mut self, row: &T) -> BigQueryResult<()> {
self.core
.write_with(|encoder, out| encoder.encode(row, out))
.await
}
pub async fn write_all<'r, I>(&mut self, rows: I) -> BigQueryResult<()>
where
I: IntoIterator<Item = &'r T>,
T: 'r,
{
for row in rows {
self.write(row).await?;
}
Ok(())
}
}
impl BigQueryStreamingWriter<RecordBatch> {
pub async fn write_batch(&mut self, batch: &RecordBatch) -> BigQueryResult<()> {
self.core.write_record_batch(batch).await
}
}
impl<T> BigQueryStreamingWriter<T> {
fn from_core(core: WriterCore) -> Self {
Self {
core,
_row: PhantomData,
}
}
pub async fn flush(&mut self) -> BigQueryResult<()> {
self.core.flush().await.map(|_| ())
}
pub async fn finish(mut self) -> BigQueryResult<BigQueryWriteSummary> {
Ok(self.core.finish(FinishKind::Close).await?.summary)
}
pub async fn finalize(mut self) -> BigQueryResult<BigQueryFinalizedStream> {
if let Err(err) = self
.core
.check_mode(BigQueryWriteMode::Pending, "finalize()")
{
self.core.abandon();
return Err(err);
}
let finished = self.core.finish(FinishKind::Finalize).await?;
Ok(BigQueryFinalizedStream {
table: self.core.table().clone(),
name: self.core.stream_name().clone(),
row_count: finished.finalized_rows.unwrap_or_default(),
})
}
pub async fn flush_rows(&mut self) -> BigQueryResult<Option<i64>> {
self.core.flush_rows().await
}
pub async fn flush_rows_to(&mut self, offset: i64) -> BigQueryResult<i64> {
self.core.flush_rows_to(offset).await
}
pub fn stream_name(&self) -> Option<&BigQueryWriteStreamName> {
(self.core.mode() != BigQueryWriteMode::Default).then(|| self.core.stream_name())
}
}
impl BigQueryDb {
pub async fn create_streaming_writer<'b, T: Serialize>(
&self,
table: impl Into<BigQueryTableRef>,
) -> BigQueryResult<(
BigQueryStreamingWriter<T>,
BoxStream<'b, BigQueryResult<BigQueryWriteResponse>>,
)> {
self.create_streaming_writer_with_options(table, BigQueryStreamingWriteOptions::new())
.await
}
pub async fn create_streaming_writer_with_options<'b, T: Serialize>(
&self,
table: impl Into<BigQueryTableRef>,
options: BigQueryStreamingWriteOptions,
) -> BigQueryResult<(
BigQueryStreamingWriter<T>,
BoxStream<'b, BigQueryResult<BigQueryWriteResponse>>,
)> {
let (core, responses) = WriterCore::open(self, table.into(), options, false).await?;
Ok((BigQueryStreamingWriter::from_core(core), responses))
}
pub async fn create_record_batch_writer<'b>(
&self,
table: impl Into<BigQueryTableRef>,
) -> BigQueryResult<(
BigQueryStreamingWriter<RecordBatch>,
BoxStream<'b, BigQueryResult<BigQueryWriteResponse>>,
)> {
self.create_record_batch_writer_with_options(table, BigQueryStreamingWriteOptions::new())
.await
}
pub async fn create_record_batch_writer_with_options<'b>(
&self,
table: impl Into<BigQueryTableRef>,
options: BigQueryStreamingWriteOptions,
) -> BigQueryResult<(
BigQueryStreamingWriter<RecordBatch>,
BoxStream<'b, BigQueryResult<BigQueryWriteResponse>>,
)> {
let (core, responses) = WriterCore::open(self, table.into(), options, false).await?;
Ok((BigQueryStreamingWriter::from_core(core), responses))
}
pub async fn commit_write_streams(
&self,
streams: Vec<BigQueryFinalizedStream>,
) -> BigQueryResult<BigQueryInstant> {
let Some(first) = streams.first() else {
return Err(BigQueryError::invalid_parameters(
"streams",
"there is no stream to commit",
));
};
let table = first.table.clone();
if let Some(other) = streams.iter().find(|s| s.table != table) {
return Err(BigQueryError::invalid_parameters(
"streams",
format!(
"BigQuery commits streams of one table together, got {table} and {}",
other.table
),
));
}
let table_path = table.table_path(&self.options().google_project_id);
let span = debug_span!("BigQuery commit write streams", "/bigquery/table" = %table);
self.batch_commit(
&span,
&table_path,
&streams.into_iter().map(|s| s.name).collect::<Vec<_>>(),
)
.await
}
}
#[cfg(test)]
mod tests;