use arrow_array::RecordBatch;
use crate::adapters::bigquery::_helpers::write::arrow_rows::{append_requests, write_api_batch};
use crate::adapters::bigquery::constants::{APPEND_REQUEST_BYTES, STREAMS_PER_BATCH};
use crate::adapters::bigquery::models::{BigQueryWriter, OpenStream};
use crate::adapters::contract::models::WriteTarget;
use crate::adapters::contract::types::TableWriter;
use crate::bigquery_api::main::append_answer::append_answer;
use crate::bigquery_api::main::append_rows::append_rows;
use crate::bigquery_api::main::batch_commit_write_streams::batch_commit_write_streams;
use crate::bigquery_api::main::create_write_stream::create_write_stream;
use crate::bigquery_api::main::finalize_write_stream::finalize_write_stream;
use crate::bigquery_api::main::table_path::table_path;
use crate::bigquery_api::models::{AppendRowsRequest, AppendRowsResponse, Int64Value};
use crate::http::errors::ApiError;
use crate::loader::errors::LoaderError;
impl TableWriter for BigQueryWriter {
fn writes_directly(&self) -> bool {
false
}
async fn write_batch(
&mut self,
target: &WriteTarget<'_>,
batch: RecordBatch,
) -> Result<(), LoaderError> {
let table = target.staging.unwrap_or(target.table);
let project = table
.database
.as_deref()
.unwrap_or(&self.destination.project);
self.parent = table_path(project, &table.schema, &table.table);
let columns = match target.builds {
true => target.write_columns.to_vec(),
false => target.columns.to_vec(),
};
let builds = target.builds;
let requests = tokio::task::spawn_blocking(move || {
append_requests(
&write_api_batch(&batch, (&columns, builds))?,
APPEND_REQUEST_BYTES,
)
})
.await
.map_err(|error| {
LoaderError::Invalid(format!("the Arrow encoding thread failed: {error}"))
})?
.map_err(LoaderError::Invalid)?;
for (request, rows) in requests {
self.send(request, rows).await?;
}
Ok(())
}
fn staged_bytes(&self) -> Option<usize> {
None
}
async fn finish_files(&mut self, _target: &WriteTarget<'_>) -> Result<(), LoaderError> {
Ok(())
}
async fn load_batch(&mut self, _target: &WriteTarget<'_>) -> Result<(), LoaderError> {
self.commit_streams().await
}
fn next_batch(&mut self) {
self.streams.clear();
self.requests_sent = 0;
}
async fn settle_rows(&mut self) -> Result<(), LoaderError> {
self.commit_streams().await
}
fn reopen_rows(&mut self) {
self.next_batch();
}
async fn rollback_batch(&mut self) {
for stream in self.streams.drain(..) {
drop(stream.requests);
if let Some(answers) = stream.answers {
answers.abort();
}
}
self.requests_sent = 0;
}
async fn remove_batch_files(&mut self, _target: &WriteTarget<'_>) {}
}
impl BigQueryWriter {
async fn send(
&mut self,
mut request: AppendRowsRequest,
rows: usize,
) -> Result<(), LoaderError> {
let index = self.requests_sent % STREAMS_PER_BATCH;
let rows = i64::try_from(rows).map_err(|error| LoaderError::Invalid(error.to_string()))?;
if index >= self.streams.len() {
let stream = create_write_stream(&self.destination.api, &self.parent).await?;
request.write_stream.clone_from(&stream.name);
request.offset = Some(Int64Value { value: 0 });
let opened = append_rows(&self.destination.api, request).await?;
self.streams.push(OpenStream {
name: stream.name,
offset: rows,
requests: Some(opened.requests),
answers: Some(tokio::spawn(read_answers(opened.responses))),
});
self.requests_sent += 1;
return Ok(());
}
self.requests_sent += 1;
let stream = &mut self.streams[index];
if stream
.answers
.as_ref()
.is_some_and(tokio::task::JoinHandle::is_finished)
{
let answers = stream.answers.take();
return Err(match answers {
Some(answers) => ended(answers.await),
None => ApiError::Protocol("the append stream was already closed".to_owned()),
}
.into());
}
request.write_stream.clone_from(&stream.name);
request.offset = Some(Int64Value {
value: stream.offset,
});
if let Some(arrow) = request.arrow_rows.as_mut() {
arrow.writer_schema = None;
}
stream.offset += rows;
let Some(requests) = &stream.requests else {
return Err(
ApiError::Protocol("the append stream was already closed".to_owned()).into(),
);
};
let Err(tokio::sync::mpsc::error::SendError(_)) = requests.send(request).await else {
return Ok(());
};
let answers = stream.answers.take();
Err(match answers {
Some(answers) => ended(answers.await),
None => ApiError::Protocol("the append stream was already closed".to_owned()),
}
.into())
}
async fn commit_streams(&mut self) -> Result<(), LoaderError> {
if self.streams.is_empty() {
return Ok(());
}
let mut names: Vec<String> = Vec::with_capacity(self.streams.len());
for mut stream in self.streams.drain(..) {
drop(stream.requests.take());
if let Some(answers) = stream.answers.take() {
answers.await.map_err(|error| {
ApiError::Protocol(format!("the append reader failed: {error}"))
})??;
}
let held = finalize_write_stream(&self.destination.api, &stream.name).await?;
if held != stream.offset {
return Err(LoaderError::Invalid(format!(
"BigQuery's stream {} holds {held} rows but {} were sent",
stream.name, stream.offset
)));
}
names.push(stream.name);
}
self.requests_sent = 0;
batch_commit_write_streams(&self.destination.api, &self.parent, &names).await?;
Ok(())
}
}
fn ended(answers: Result<Result<usize, ApiError>, tokio::task::JoinError>) -> ApiError {
match answers {
Ok(Err(error)) => error,
Ok(Ok(answered)) => ApiError::Protocol(format!(
"BigQuery closed the append stream after {answered} answers without an error"
)),
Err(error) => ApiError::Protocol(format!("the append reader failed: {error}")),
}
}
async fn read_answers(
mut responses: tonic::Streaming<AppendRowsResponse>,
) -> Result<usize, ApiError> {
let mut answered = 0;
while let Some(answer) = append_answer(&mut responses).await? {
if let Some(refused) = answer.row_errors.first() {
return Err(ApiError::Status {
status: 400,
code: "rowErrors".to_owned(),
message: format!(
"BigQuery refused {} row(s) of an append, the first at index {}: {}",
answer.row_errors.len(),
refused.index,
refused.message
),
});
}
if let Some(error) = answer.error {
return Err(ApiError::Status {
status: 400,
code: format!("appendError{}", error.code),
message: error.message,
});
}
answered += 1;
}
Ok(answered)
}