use crate::errors::BigQueryError;
use crate::query::params::parameter_mode;
use crate::read::ArrowIpcDecoder;
use crate::{
BigQueryDb, BigQueryDryRunResult, BigQueryJobCreation, BigQueryJobId, BigQueryJobRef,
BigQueryJobStats, BigQueryLocation, BigQueryQueryDestination, BigQueryQueryId,
BigQueryQueryOutcome, BigQueryQueryParams, BigQueryReadCompression, BigQueryResult,
BigQueryStatementType, BigQueryTableRef, BigQueryTableSchema,
};
use arrow_array::RecordBatch;
use gcloud_sdk::google::cloud::bigquery::v2::arrow_serialization_options::CompressionCodec;
use gcloud_sdk::google::cloud::bigquery::v2::query_request::{
JobCreationMode, QueryResultsFormat, ResultsFormatSerializationOptions,
};
use gcloud_sdk::google::cloud::bigquery::v2::query_response::{Results, ResultsSchema};
use gcloud_sdk::google::cloud::bigquery::v2::{
ArrowSerializationOptions, DataFormatOptions, DatasetReference, GetQueryResultsResponse,
InsertJobRequest, Job, JobConfiguration, JobConfigurationQuery, JobReference, PostQueryRequest,
QueryRequest, QueryResponse,
};
use gcloud_sdk::tonic::{Code, Response};
use rand::RngExt;
use std::sync::atomic::{AtomicBool, Ordering};
use tracing::field::Empty;
use tracing::Span;
impl BigQueryQueryParams {
pub(crate) fn span(&self) -> Span {
tracing::debug_span!(
"BigQuery Query",
"/bigquery/sql_len" = self.sql.len(),
"/bigquery/job_id" = Empty,
"/bigquery/query_id" = Empty,
"/bigquery/location" = Empty,
"/bigquery/statement_type" = Empty,
"/bigquery/bytes_processed" = Empty,
"/bigquery/bytes_billed" = Empty,
"/bigquery/slot_ms" = Empty,
"/bigquery/cache_hit" = Empty,
"/bigquery/dml_rows" = Empty,
"/bigquery/total_rows" = Empty,
"/bigquery/route" = Empty,
)
}
}
impl From<BigQueryReadCompression> for CompressionCodec {
fn from(compression: BigQueryReadCompression) -> Self {
match compression {
BigQueryReadCompression::None => CompressionCodec::CompressionUnspecified,
BigQueryReadCompression::Lz4 => CompressionCodec::Lz4Frame,
BigQueryReadCompression::Zstd => CompressionCodec::Zstd,
}
}
}
impl From<BigQueryJobCreation> for JobCreationMode {
fn from(mode: BigQueryJobCreation) -> Self {
match mode {
BigQueryJobCreation::Optional => Self::JobCreationOptional,
BigQueryJobCreation::Required => Self::JobCreationRequired,
}
}
}
impl From<&QueryResponse> for BigQueryJobStats {
fn from(response: &QueryResponse) -> Self {
let job = response.job_reference.clone().map(|reference| {
let mut job = BigQueryJobRef::from(reference);
if job.location.is_none() {
job.location = BigQueryLocation::reported(response.location.clone());
}
job
});
Self {
job,
query_id: Some(response.query_id.clone())
.filter(|v| !v.is_empty())
.map(BigQueryQueryId::reported),
statement_type: Some(response.statement_type.clone())
.filter(|v| !v.is_empty())
.map(Into::into),
total_rows: response.total_rows,
total_bytes_processed: response.total_bytes_processed,
total_bytes_billed: response.total_bytes_billed,
total_slot_ms: response.total_slot_ms,
cache_hit: response.cache_hit,
num_dml_affected_rows: response.num_dml_affected_rows,
dml_stats: response.dml_stats.map(Into::into),
}
}
}
impl BigQueryJobStats {
fn merge_results(&mut self, results: &GetQueryResultsResponse) {
self.total_rows = self.total_rows.or(results.total_rows);
self.total_bytes_processed = self.total_bytes_processed.or(results.total_bytes_processed);
self.cache_hit = self.cache_hit.or(results.cache_hit);
self.num_dml_affected_rows = self.num_dml_affected_rows.or(results.num_dml_affected_rows);
}
fn merge_job(&mut self, job: &Job) {
let Some(statistics) = &job.statistics else {
return;
};
if let Some(query) = &statistics.query {
self.statement_type = self.statement_type.take().or_else(|| {
Some(query.statement_type.clone())
.filter(|v| !v.is_empty())
.map(Into::into)
});
self.total_bytes_processed = self.total_bytes_processed.or(query.total_bytes_processed);
self.total_bytes_billed = self.total_bytes_billed.or(query.total_bytes_billed);
self.total_slot_ms = self.total_slot_ms.or(query.total_slot_ms);
self.cache_hit = self.cache_hit.or(query.cache_hit);
self.num_dml_affected_rows = self.num_dml_affected_rows.or(query.num_dml_affected_rows);
self.dml_stats = self.dml_stats.or(query.dml_stats.map(Into::into));
}
self.total_bytes_processed = self
.total_bytes_processed
.or(statistics.total_bytes_processed);
self.total_slot_ms = self.total_slot_ms.or(statistics.total_slot_ms);
}
fn first_response(response: &QueryResponse, span: &Span) -> Self {
let stats = Self::from(response);
stats.record(span);
if stats.job.is_none() && !response.location.is_empty() {
span.record("/bigquery/location", response.location.as_str());
}
stats
}
fn record(&self, span: &Span) {
if let Some(query_id) = &self.query_id {
span.record("/bigquery/query_id", query_id.as_str());
}
if let Some(job) = &self.job {
span.record("/bigquery/job_id", job.job_id.as_str());
if let Some(location) = &job.location {
span.record("/bigquery/location", location.as_str());
}
}
if let Some(statement_type) = &self.statement_type {
span.record("/bigquery/statement_type", statement_type.as_str());
}
if let Some(bytes) = self.total_bytes_processed {
span.record("/bigquery/bytes_processed", bytes);
}
if let Some(bytes) = self.total_bytes_billed {
span.record("/bigquery/bytes_billed", bytes);
}
if let Some(slot_ms) = self.total_slot_ms {
span.record("/bigquery/slot_ms", slot_ms);
}
if let Some(cache_hit) = self.cache_hit {
span.record("/bigquery/cache_hit", cache_hit);
}
if let Some(rows) = self.num_dml_affected_rows {
span.record("/bigquery/dml_rows", rows);
}
if let Some(rows) = self.total_rows {
span.record("/bigquery/total_rows", rows);
}
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Purpose {
Rows,
Execute,
DryRun,
}
pub(crate) enum Rows {
Inline(RecordBatch),
Table(BigQueryTableRef),
None,
}
fn random_request_id() -> String {
format!("{:032x}", rand::rng().random::<u128>())
}
impl BigQueryQueryParams {
fn default_dataset_reference(&self, project_id: &str) -> Option<DatasetReference> {
self.default_dataset.as_ref().map(|d| DatasetReference {
project_id: d.project_or(project_id).to_string(),
dataset_id: d.dataset().to_string(),
})
}
}
impl BigQueryDb {
fn query_location(&self, params: &BigQueryQueryParams) -> Option<String> {
params
.location
.as_ref()
.or(self.options().location.as_ref())
.map(ToString::to_string)
}
fn query_job_request(
&self,
params: &BigQueryQueryParams,
destination: &BigQueryQueryDestination,
dry_run: bool,
) -> BigQueryResult<InsertJobRequest> {
let project_id = self.options().google_project_id.clone();
let query = JobConfigurationQuery {
query: params.sql.clone(),
destination_table: Some(destination.table.table_reference(&project_id)),
create_disposition: "CREATE_IF_NEEDED".to_string(),
write_disposition: destination.write.disposition().to_string(),
default_dataset: params.default_dataset_reference(&project_id),
use_query_cache: params.use_query_cache,
maximum_bytes_billed: params.maximum_bytes_billed,
use_legacy_sql: Some(false),
parameter_mode: parameter_mode(¶ms.query_parameters)?.to_string(),
query_parameters: params.query_parameters.clone(),
..Default::default()
};
Ok(InsertJobRequest {
project_id: project_id.clone(),
job: Some(Job {
job_reference: Some(JobReference {
project_id,
job_id: BigQueryJobId::random().to_string(),
location: self.query_location(params),
}),
configuration: Some(JobConfiguration {
query: Some(query),
dry_run: Some(dry_run),
job_timeout_ms: params.job_timeout_ms()?,
labels: params.labels.clone().into_iter().collect(),
..Default::default()
}),
..Default::default()
}),
})
}
async fn insert_query_job(
&self,
params: &BigQueryQueryParams,
destination: &BigQueryQueryDestination,
dry_run: bool,
span: &Span,
) -> BigQueryResult<Job> {
let request = self.query_job_request(params, destination, dry_run)?;
let sent = request
.job
.as_ref()
.and_then(|job| job.job_reference.clone());
let retrying = AtomicBool::new(false);
self.retry(span, "insert a query job", &request, |r| {
let retry = retrying.swap(true, Ordering::Relaxed);
let sent = sent.clone();
let mut client = self.job_client();
async move {
match client.insert_job(r).await {
Err(status) if retry && status.code() == Code::AlreadyExists => {
match sent.and_then(|job| earlier_attempt_job(job, status.message())) {
Some(job) => Ok(Response::new(Job {
job_reference: Some(job),
..Default::default()
})),
None => Err(status),
}
}
other => other,
}
}
})
.await
}
async fn run_query_job(
&self,
params: &BigQueryQueryParams,
destination: &BigQueryQueryDestination,
span: &Span,
) -> BigQueryResult<(BigQueryJobStats, BigQueryJobRef, Job)> {
let inserted = self
.insert_query_job(params, destination, false, span)
.await?;
let mut stats = BigQueryJobStats {
job: inserted.job_reference.map(BigQueryJobRef::from),
..Default::default()
};
stats.record(span);
let (job, details) = self.finish_job(&mut stats, true, params, span).await?;
Ok((stats, job, details))
}
fn query_request(
&self,
params: &BigQueryQueryParams,
purpose: Purpose,
) -> BigQueryResult<PostQueryRequest> {
let project_id = self.options().google_project_id.clone();
let default_dataset = params.default_dataset_reference(&project_id);
let max_results = match purpose {
Purpose::Rows => params.inline_rows_limit,
Purpose::Execute => Some(0),
Purpose::DryRun => None,
};
Ok(PostQueryRequest {
project_id,
query_request: Some(QueryRequest {
query: params.sql.clone(),
max_results,
default_dataset,
timeout_ms: Some(params.timeout_ms()?),
job_timeout_ms: params.job_timeout_ms()?,
dry_run: purpose == Purpose::DryRun,
use_query_cache: params.use_query_cache,
use_legacy_sql: Some(false),
parameter_mode: parameter_mode(¶ms.query_parameters)?.to_string(),
query_parameters: params.query_parameters.clone(),
location: self.query_location(params).unwrap_or_default(),
format_options: Some(DataFormatOptions {
use_int64_timestamp: true,
..Default::default()
}),
labels: params.labels.clone().into_iter().collect(),
maximum_bytes_billed: params.maximum_bytes_billed,
job_creation_mode: JobCreationMode::from(params.job_creation).into(),
request_id: params
.request_id
.as_ref()
.map_or_else(random_request_id, ToString::to_string),
query_results_format: QueryResultsFormat::Arrow.into(),
results_format_serialization_options: Some(
ResultsFormatSerializationOptions::ArrowSerializationOptions(
ArrowSerializationOptions {
buffer_compression: CompressionCodec::from(
params.read_options.compression,
)
.into(),
..Default::default()
},
),
),
..Default::default()
}),
})
}
async fn post_query(
&self,
params: &BigQueryQueryParams,
purpose: Purpose,
span: &Span,
) -> BigQueryResult<QueryResponse> {
let request = self.query_request(params, purpose)?;
let retrying = AtomicBool::new(false);
self.retry(span, "run a query", &request, |mut r| {
if retrying.swap(true, Ordering::Relaxed) {
if let Some(query) = r.get_mut().query_request.as_mut() {
query.job_creation_mode = JobCreationMode::JobCreationRequired.into();
}
}
let mut client = self.job_client();
async move { client.query(r).await }
})
.await
}
pub(crate) async fn query_rows(
&self,
params: &BigQueryQueryParams,
span: &Span,
) -> BigQueryResult<(Rows, BigQueryJobStats)> {
if let Some(destination) = ¶ms.destination {
let (stats, job, details) = self.run_query_job(params, destination, span).await?;
return Self::job_rows(stats, &job, &details, span);
}
let response = self.post_query(params, Purpose::Rows, span).await?;
let mut stats = BigQueryJobStats::first_response(&response, span);
let complete = response.job_complete == Some(true);
if complete && response.page_token.is_empty() {
let batch = inline_batch(&response)?;
let inline_rows = batch.as_ref().map_or(0, |b| b.num_rows() as u64);
if response.total_rows.is_none_or(|total| total == inline_rows) {
span.record("/bigquery/route", "inline");
return Ok((batch.map_or(Rows::None, Rows::Inline), stats));
}
}
let (job, details) = self.finish_job(&mut stats, !complete, params, span).await?;
Self::job_rows(stats, &job, &details, span)
}
fn job_rows(
stats: BigQueryJobStats,
job: &BigQueryJobRef,
details: &Job,
span: &Span,
) -> BigQueryResult<(Rows, BigQueryJobStats)> {
let destination = details
.configuration
.as_ref()
.and_then(|c| c.query.as_ref())
.and_then(|q| q.destination_table.clone());
match destination {
Some(table) => {
span.record("/bigquery/route", "storage_read");
let table = BigQueryTableRef::try_from(table).map_err(|err| {
BigQueryError::system(
"INVALID_DESTINATION_TABLE",
format!(
"The query job {} names a destination table this crate cannot read: {err}",
job.job_id
),
)
})?;
Ok((Rows::Table(table), stats))
}
None if stats.statement_type == Some(BigQueryStatementType::Select) => {
Err(BigQueryError::system(
"NO_DESTINATION_TABLE",
format!(
"The query job {} returned rows but has no destination table to read them from",
job.job_id
),
))
}
None => {
span.record("/bigquery/route", "none");
Ok((Rows::None, stats))
}
}
}
pub(crate) async fn execute_statement(
&self,
params: &BigQueryQueryParams,
span: &Span,
) -> BigQueryResult<BigQueryQueryOutcome> {
if let Some(destination) = ¶ms.destination {
return Ok(self.run_query_job(params, destination, span).await?.0);
}
let response = self.post_query(params, Purpose::Execute, span).await?;
let mut stats = BigQueryJobStats::first_response(&response, span);
if response.job_complete == Some(true) {
return Ok(stats);
}
self.finish_job(&mut stats, true, params, span).await?;
Ok(stats)
}
async fn finish_job(
&self,
stats: &mut BigQueryJobStats,
wait: bool,
params: &BigQueryQueryParams,
span: &Span,
) -> BigQueryResult<(BigQueryJobRef, Job)> {
let job = stats
.job
.clone()
.ok_or_else(|| BigQueryError::system("NO_JOB_REFERENCE", MISSING_JOB))?;
if wait {
let results = self.wait_for_job(&job, params.timeout_ms()?, span).await?;
stats.merge_results(&results);
}
let details = self.get_query_job(&job, span).await?;
stats.merge_job(&details);
stats.record(span);
Ok((job, details))
}
pub(crate) async fn dry_run_statement(
&self,
params: &BigQueryQueryParams,
span: &Span,
) -> BigQueryResult<BigQueryDryRunResult> {
if let Some(destination) = ¶ms.destination {
let job = self
.insert_query_job(params, destination, true, span)
.await?;
return BigQueryDryRunResult::try_from(&job);
}
let response = self.post_query(params, Purpose::DryRun, span).await?;
Ok(BigQueryDryRunResult {
total_bytes_processed: response.total_bytes_processed,
schema: response
.schema
.as_ref()
.map(BigQueryTableSchema::try_from)
.transpose()?,
})
}
}
fn earlier_attempt_job(sent: JobReference, already_exists: &str) -> Option<JobReference> {
let qualified_id = format!(".{}", sent.job_id);
let named = already_exists
.split_whitespace()
.find(|word| word.ends_with(&qualified_id))?;
let location = named
.strip_suffix(&qualified_id)?
.rsplit_once(':')
.map(|(_, location)| location.to_string())
.filter(|location| !location.is_empty());
Some(JobReference {
location: location.or(sent.location),
..sent
})
}
impl TryFrom<&Job> for BigQueryDryRunResult {
type Error = BigQueryError;
fn try_from(job: &Job) -> Result<Self, Self::Error> {
let statistics = job.statistics.as_ref();
let query = statistics.and_then(|statistics| statistics.query.as_ref());
Ok(Self {
total_bytes_processed: query
.and_then(|query| query.total_bytes_processed)
.or(statistics.and_then(|statistics| statistics.total_bytes_processed)),
schema: query
.and_then(|query| query.schema.as_ref())
.map(BigQueryTableSchema::try_from)
.transpose()?,
})
}
}
const MISSING_JOB: &str = "BigQuery answered a query that is unfinished or only partly inline \
without a job reference to read the rest from";
fn inline_batch(response: &QueryResponse) -> BigQueryResult<Option<RecordBatch>> {
let (Some(ResultsSchema::ArrowSchema(schema)), Some(Results::ArrowRecordBatch(batch))) =
(&response.results_schema, &response.results)
else {
return Ok(None);
};
if batch.serialized_record_batch.is_empty() {
return Ok(None);
}
ArrowIpcDecoder::new(&schema.serialized_schema)?
.decode(&batch.serialized_record_batch)
.map(Some)
}