use crate::admin::{logging_errors, paged};
use crate::db::proto::timestamp_ms;
use crate::errors::{BigQueryError, BigQueryJobErrorEntry};
use crate::BigQueryInstant;
use crate::{BigQueryDb, BigQueryJobRef, BigQueryResult, BigQueryStatementType};
use crate::{BigQueryJobId, BigQueryLabels};
use futures::stream::BoxStream;
use gcloud_sdk::google::cloud::bigquery::v2;
use rsb_derive::Builder;
use std::fmt::{Display, Formatter};
use tracing::Span;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum BigQueryJobState {
Pending,
Running,
Done,
Other(String),
}
impl BigQueryJobState {
pub fn as_str(&self) -> &str {
match self {
Self::Pending => "PENDING",
Self::Running => "RUNNING",
Self::Done => "DONE",
Self::Other(name) => name,
}
}
fn known(name: &str) -> Option<Self> {
Some(match name {
"PENDING" => Self::Pending,
"RUNNING" => Self::Running,
"DONE" => Self::Done,
_ => return None,
})
}
}
impl From<&str> for BigQueryJobState {
fn from(name: &str) -> Self {
Self::known(name).unwrap_or_else(|| Self::Other(name.to_string()))
}
}
impl From<String> for BigQueryJobState {
fn from(name: String) -> Self {
Self::known(&name).unwrap_or(Self::Other(name))
}
}
impl Display for BigQueryJobState {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum BigQueryJobType {
Query,
Load,
Extract,
Copy,
Unknown,
Other(String),
}
impl BigQueryJobType {
pub fn as_str(&self) -> &str {
match self {
Self::Query => "QUERY",
Self::Load => "LOAD",
Self::Extract => "EXTRACT",
Self::Copy => "COPY",
Self::Unknown => "UNKNOWN",
Self::Other(name) => name,
}
}
fn known(name: &str) -> Option<Self> {
Some(match name {
"QUERY" => Self::Query,
"LOAD" => Self::Load,
"EXTRACT" => Self::Extract,
"COPY" => Self::Copy,
"UNKNOWN" => Self::Unknown,
_ => return None,
})
}
}
impl From<&str> for BigQueryJobType {
fn from(name: &str) -> Self {
Self::known(name).unwrap_or_else(|| Self::Other(name.to_string()))
}
}
impl From<String> for BigQueryJobType {
fn from(name: String) -> Self {
Self::known(&name).unwrap_or(Self::Other(name))
}
}
impl Display for BigQueryJobType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BigQueryJob {
pub reference: BigQueryJobRef,
pub job_type: Option<BigQueryJobType>,
pub state: Option<BigQueryJobState>,
pub error: Option<BigQueryJobErrorEntry>,
pub user_email: Option<String>,
pub labels: BigQueryLabels,
pub statement_type: Option<BigQueryStatementType>,
pub creation_time: Option<BigQueryInstant>,
pub start_time: Option<BigQueryInstant>,
pub end_time: Option<BigQueryInstant>,
pub total_bytes_processed: Option<i64>,
pub total_bytes_billed: Option<i64>,
}
struct JobParts {
reference: Option<v2::JobReference>,
configuration: Option<v2::JobConfiguration>,
statistics: Option<v2::JobStatistics>,
status: Option<v2::JobStatus>,
user_email: String,
}
impl TryFrom<JobParts> for BigQueryJob {
type Error = BigQueryError;
fn try_from(job: JobParts) -> Result<Self, Self::Error> {
let reference = job
.reference
.ok_or_else(|| {
BigQueryError::unexpected_response("BigQuery returned no job_reference")
})?
.into();
let configuration = job.configuration.unwrap_or_default();
let statistics = job.statistics.unwrap_or_default();
let query = statistics.query.clone().unwrap_or_default();
let status = job.status.unwrap_or_default();
Ok(Self {
reference,
job_type: Some(configuration.job_type)
.filter(|v| !v.is_empty())
.map(Into::into),
state: Some(status.state).filter(|v| !v.is_empty()).map(Into::into),
error: status.error_result.map(Into::into),
user_email: Some(job.user_email).filter(|v| !v.is_empty()),
labels: configuration.labels.into_iter().collect(),
statement_type: Some(query.statement_type)
.filter(|v| !v.is_empty())
.map(Into::into),
creation_time: timestamp_ms("creation_time", statistics.creation_time)?,
start_time: timestamp_ms("start_time", statistics.start_time)?,
end_time: timestamp_ms("end_time", statistics.end_time)?,
total_bytes_processed: statistics.total_bytes_processed,
total_bytes_billed: query.total_bytes_billed,
})
}
}
impl TryFrom<v2::Job> for BigQueryJob {
type Error = BigQueryError;
fn try_from(job: v2::Job) -> Result<Self, Self::Error> {
JobParts {
reference: job.job_reference,
configuration: job.configuration,
statistics: job.statistics,
status: job.status,
user_email: job.user_email,
}
.try_into()
}
}
impl TryFrom<v2::ListFormatJob> for BigQueryJob {
type Error = BigQueryError;
fn try_from(job: v2::ListFormatJob) -> Result<Self, Self::Error> {
JobParts {
reference: job.job_reference,
configuration: job.configuration,
statistics: job.statistics,
status: job.status,
user_email: job.user_email,
}
.try_into()
}
}
#[derive(Debug, PartialEq, Clone, Builder)]
pub struct BigQueryListJobsParams {
pub project_id: Option<String>,
#[default = "false"]
pub all_users: bool,
pub min_creation_time: Option<BigQueryInstant>,
pub max_creation_time: Option<BigQueryInstant>,
#[default = "Vec::new()"]
pub states: Vec<BigQueryJobState>,
pub parent_job_id: Option<BigQueryJobId>,
pub page_size: Option<u32>,
}
impl BigQueryDb {
pub async fn get_job(&self, job: &BigQueryJobRef) -> BigQueryResult<BigQueryJob> {
let request = v2::GetJobRequest {
project_id: job.project_id.clone(),
job_id: job.job_id.to_string(),
location: job.location_field(),
};
self.retry(&job.admin_span(), "get a job", &request, |r| {
let mut client = self.job_client();
async move { client.get_job(r).await }
})
.await?
.try_into()
}
pub async fn delete_job(&self, job: &BigQueryJobRef) -> BigQueryResult<()> {
let request = v2::DeleteJobRequest {
project_id: job.project_id.clone(),
job_id: job.job_id.to_string(),
location: job.location_field(),
};
self.retry(&job.admin_span(), "delete a job", &request, |r| {
let mut client = self.job_client();
async move { client.delete_job(r).await }
})
.await
}
pub async fn stream_jobs<'b>(
&self,
params: BigQueryListJobsParams,
) -> BigQueryResult<BoxStream<'b, BigQueryJob>> {
Ok(logging_errors(
self.stream_jobs_with_errors(params).await?,
"jobs",
))
}
pub async fn stream_jobs_with_errors<'b>(
&self,
params: BigQueryListJobsParams,
) -> BigQueryResult<BoxStream<'b, BigQueryResult<BigQueryJob>>> {
let template = params.into_request(&self.options().google_project_id)?;
let db = self.clone();
let span = tracing::debug_span!(
"BigQuery jobs",
"/bigquery/project" = template.project_id.as_str()
);
Ok(paged(move |page_token| {
let db = db.clone();
let span = span.clone();
let request = v2::ListJobsRequest {
page_token,
..template.clone()
};
async move {
let page = db
.retry(&span, "list jobs", &request, |r| {
let mut client = db.job_client();
async move { client.list_jobs(r).await }
})
.await?;
let jobs = page
.jobs
.into_iter()
.map(TryInto::try_into)
.collect::<BigQueryResult<_>>()?;
Ok((jobs, page.next_page_token))
}
}))
}
}
impl BigQueryJobRef {
fn admin_span(&self) -> Span {
tracing::debug_span!("BigQuery job", "/bigquery/job_id" = self.job_id.as_str())
}
}
fn creation_bound(field: &str, at: BigQueryInstant) -> BigQueryResult<u64> {
u64::try_from(at.as_millisecond())
.map_err(|_| BigQueryError::invalid_parameters(field, format!("{at} is before 1970")))
}
impl BigQueryListJobsParams {
fn into_request(self, default_project_id: &str) -> BigQueryResult<v2::ListJobsRequest> {
use v2::list_jobs_request::{Projection, StateFilter};
let state_filter = self
.states
.into_iter()
.map(|state| match state {
BigQueryJobState::Pending => Ok(StateFilter::Pending.into()),
BigQueryJobState::Running => Ok(StateFilter::Running.into()),
BigQueryJobState::Done => Ok(StateFilter::Done.into()),
BigQueryJobState::Other(name) => Err(BigQueryError::invalid_parameters(
"states",
format!("{name:?} is not a state ListJobs filters on"),
)),
})
.collect::<BigQueryResult<_>>()?;
Ok(v2::ListJobsRequest {
project_id: self
.project_id
.unwrap_or_else(|| default_project_id.to_string()),
all_users: self.all_users,
max_results: self.page_size.map(|n| i32::try_from(n).unwrap_or(i32::MAX)),
min_creation_time: self
.min_creation_time
.map(|at| creation_bound("min_creation_time", at))
.transpose()?
.unwrap_or(0),
max_creation_time: self
.max_creation_time
.map(|at| creation_bound("max_creation_time", at))
.transpose()?,
page_token: String::new(),
projection: Projection::Full.into(),
state_filter,
parent_job_id: self
.parent_job_id
.map(|id| id.to_string())
.unwrap_or_default(),
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn job_types_map_known_names_and_keep_unknown_ones() {
for name in ["QUERY", "LOAD", "EXTRACT", "COPY", "UNKNOWN"] {
let parsed = BigQueryJobType::from(name);
assert!(
!matches!(parsed, BigQueryJobType::Other(_)),
"{name} reads as {parsed:?}"
);
assert_eq!(parsed.as_str(), name);
}
let unknown = BigQueryJobType::from("SNAPSHOT".to_string());
assert_eq!(unknown, BigQueryJobType::Other("SNAPSHOT".into()));
assert_eq!(unknown.to_string(), "SNAPSHOT");
}
#[test]
fn job_states_map_known_names_and_keep_unknown_ones() {
for name in ["PENDING", "RUNNING", "DONE"] {
let parsed = BigQueryJobState::from(name);
assert!(
!matches!(parsed, BigQueryJobState::Other(_)),
"{name} reads as {parsed:?}"
);
assert_eq!(parsed.as_str(), name);
}
let unknown = BigQueryJobState::from("PAUSED".to_string());
assert_eq!(unknown, BigQueryJobState::Other("PAUSED".into()));
assert_eq!(unknown.to_string(), "PAUSED");
}
}