use crate::db::fake::wire::IpcCompression;
use crate::errors::BigQueryError;
use crate::testing::rows::rescale_decimal;
use crate::testing::BigQueryFakeJobFailure;
use crate::{
BigQueryChangeSequenceNumber, BigQueryChangeType, BigQueryDatasetId, BigQueryDatasetRef,
BigQueryDecimalParams, BigQueryDmlStats, BigQueryFieldSchema, BigQueryFieldType,
BigQueryInstant, BigQueryJobId, BigQueryResult, BigQueryStatementType, BigQueryTableId,
BigQueryTableRef, BigQueryTableSchema, BigQueryWriteMode, BigQueryWriteStreamName,
};
use arrow_array::cast::AsArray;
use arrow_array::types::{Decimal128Type, Decimal256Type};
use arrow_array::{
new_null_array, Array, ArrayRef, ListArray, RecordBatch, RecordBatchOptions, StructArray,
};
use arrow_buffer::i256;
use arrow_schema::{ArrowError, DataType, Field, Fields, SchemaRef};
use gcloud_sdk::google::cloud::bigquery::v2;
use gcloud_sdk::tonic::Status;
use std::collections::{BTreeMap, HashMap};
use std::fmt::{Display, Formatter};
use std::num::NonZeroUsize;
use std::sync::Arc;
pub(super) const QUERY_RESULTS_DATASET: BigQueryDatasetId =
BigQueryDatasetId::from_static("_fake_query_results");
pub(super) const DEFAULT_LOCATION: &str = "US";
pub(super) fn location_or_default(location: &str) -> String {
if location.is_empty() {
DEFAULT_LOCATION.to_string()
} else {
location.to_string()
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub(super) struct DatasetKey {
pub project: String,
pub dataset: BigQueryDatasetId,
}
impl DatasetKey {
pub(super) fn resolve(dataset: &BigQueryDatasetRef, client_project: &str) -> Self {
Self {
project: dataset.project_or(client_project).to_string(),
dataset: dataset.dataset().clone(),
}
}
pub(super) fn new(project: &str, dataset: &str) -> BigQueryResult<Self> {
let dataset = BigQueryDatasetRef::new(project, BigQueryDatasetId::new(dataset)?)?;
Ok(Self::resolve(&dataset, project))
}
pub(super) fn table(&self, table: BigQueryTableId) -> TableKey {
TableKey {
dataset: self.clone(),
table,
}
}
pub(super) fn reference(&self) -> v2::DatasetReference {
v2::DatasetReference {
project_id: self.project.clone(),
dataset_id: self.dataset.to_string(),
}
}
pub(super) fn legacy_id(&self) -> String {
format!("{}:{}", self.project, self.dataset)
}
pub(super) fn not_found(&self) -> Status {
Status::not_found(format!("Not found: Dataset {}", self.legacy_id()))
}
pub(super) fn already_exists(&self) -> Status {
Status::already_exists(format!("Already Exists: Dataset {}", self.legacy_id()))
}
}
impl Display for DatasetKey {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.project, self.dataset)
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub(super) struct TableKey {
pub dataset: DatasetKey,
pub table: BigQueryTableId,
}
impl TableKey {
pub(super) fn resolve(table: &BigQueryTableRef, client_project: &str) -> Self {
Self {
dataset: DatasetKey {
project: table.project_or(client_project).to_string(),
dataset: table.dataset().clone(),
},
table: table.table().clone(),
}
}
pub(super) fn new(project: &str, dataset: &str, table: &str) -> BigQueryResult<Self> {
Ok(DatasetKey::new(project, dataset)?.table(BigQueryTableId::new(table)?))
}
pub(super) fn from_path(path: &str) -> BigQueryResult<Self> {
let mut segments = path.split('/');
let mut take = |label: &str| match (segments.next(), segments.next()) {
(Some(found), Some(id)) if found == label => Ok(id),
_ => Err(BigQueryError::invalid_parameters(
"table",
format!("{path:?} is not projects/*/datasets/*/tables/*"),
)),
};
let project = take("projects")?;
let dataset = take("datasets")?;
let table = take("tables")?;
Self::new(project, dataset, table)
}
pub(super) fn path(&self) -> String {
BigQueryTableRef::from(self).table_path(&self.dataset.project)
}
pub(super) fn reference(&self) -> v2::TableReference {
BigQueryTableRef::from(self).table_reference(&self.dataset.project)
}
pub(super) fn legacy_id(&self) -> String {
format!(
"{}:{}.{}",
self.dataset.project, self.dataset.dataset, self.table
)
}
pub(super) fn not_found(&self) -> Status {
Status::not_found(format!("Not found: Table {}", self.legacy_id()))
}
pub(super) fn already_exists(&self) -> Status {
Status::already_exists(format!("Already Exists: Table {}", self.legacy_id()))
}
}
impl From<&TableKey> for BigQueryTableRef {
fn from(key: &TableKey) -> Self {
BigQueryTableRef::new(
Some(key.dataset.project.clone()),
key.dataset.dataset.clone(),
key.table.clone(),
)
}
}
impl Display for TableKey {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.dataset, self.table)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct FakeGeneration(u64);
impl FakeGeneration {
pub(super) fn etag(self) -> String {
format!("fake-etag-{}", self.0)
}
}
#[derive(Clone, Debug)]
pub(super) struct FakeDataset {
pub generation: FakeGeneration,
pub created: BigQueryInstant,
pub resource: v2::Dataset,
}
impl FakeDataset {
fn new(generation: FakeGeneration) -> Self {
Self {
generation,
created: BigQueryInstant::now(),
resource: v2::Dataset::default(),
}
}
pub(super) fn resource(&self, key: &DatasetKey) -> v2::Dataset {
let location = location_or_default(&self.resource.location);
v2::Dataset {
kind: "bigquery#dataset".into(),
id: key.legacy_id(),
dataset_reference: Some(key.reference()),
etag: self.generation.etag(),
creation_time: self.created.as_millisecond(),
location,
..self.resource.clone()
}
}
}
#[derive(Clone, Debug)]
pub(super) struct FakeTable {
pub schema: BigQueryTableSchema,
pub arrow_schema: SchemaRef,
pub batches: Vec<RecordBatch>,
pub changes: Vec<FakeChanges>,
pub read_streams: NonZeroUsize,
pub generation: FakeGeneration,
pub created: BigQueryInstant,
pub resource: v2::Table,
}
impl FakeTable {
pub(super) fn new(
schema: BigQueryTableSchema,
batches: Vec<RecordBatch>,
read_streams: NonZeroUsize,
generation: FakeGeneration,
) -> Self {
Self {
arrow_schema: Arc::new(schema.arrow_read_schema()),
schema,
batches,
changes: Vec::new(),
read_streams,
generation,
created: BigQueryInstant::now(),
resource: v2::Table::default(),
}
}
pub(super) fn num_rows(&self) -> usize {
self.batches.iter().map(RecordBatch::num_rows).sum()
}
pub(super) fn resource(&self, key: &TableKey) -> v2::Table {
v2::Table {
table_reference: Some(key.reference()),
id: key.legacy_id(),
schema: Some(v2::TableSchema::from(&self.schema)),
etag: self.generation.etag(),
num_rows: u64::try_from(self.num_rows()).ok(),
creation_time: self.created.as_millisecond(),
r#type: "TABLE".into(),
..self.resource.clone()
}
}
}
#[derive(Clone, Debug)]
pub(super) struct FakeChanges {
pub rows: RecordBatch,
pub changes: Vec<FakeChange>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct FakeChange {
pub change_type: BigQueryChangeType,
pub sequence_number: Option<BigQueryChangeSequenceNumber>,
}
#[derive(Clone, Debug)]
pub(super) struct FakeWriteStream {
pub table: TableKey,
pub mode: BigQueryWriteMode,
pub appended: Vec<RecordBatch>,
pub length: i64,
pub flushed: i64,
pub finalized: bool,
pub committed: bool,
}
impl FakeWriteStream {
pub(super) fn new(table: TableKey, mode: BigQueryWriteMode) -> Self {
Self {
table,
mode,
appended: Vec::new(),
length: 0,
flushed: 0,
finalized: false,
committed: false,
}
}
}
#[derive(Clone, Debug)]
pub(super) struct FakeReadStream {
pub table: TableKey,
pub arrow_schema: SchemaRef,
pub batches: Vec<RecordBatch>,
pub compression: IpcCompression,
}
#[derive(Clone, Debug)]
pub(super) struct FakeJob {
pub location: String,
pub statement_type: BigQueryStatementType,
pub complete: bool,
pub cancelled: bool,
pub destination: Option<TableKey>,
pub total_rows: Option<u64>,
pub bytes_processed: Option<i64>,
pub dml: Option<BigQueryDmlStats>,
pub failure: Option<BigQueryFakeJobFailure>,
}
#[derive(Debug, Default)]
pub(super) struct FakeState {
pub datasets: BTreeMap<DatasetKey, FakeDataset>,
pub tables: BTreeMap<TableKey, FakeTable>,
pub write_streams: HashMap<BigQueryWriteStreamName, FakeWriteStream>,
pub read_streams: BTreeMap<String, FakeReadStream>,
pub jobs: BTreeMap<BigQueryJobId, FakeJob>,
last_id: u64,
}
impl FakeState {
pub(super) fn next_id(&mut self) -> u64 {
self.last_id += 1;
self.last_id
}
pub(super) fn next_generation(&mut self) -> FakeGeneration {
FakeGeneration(self.next_id())
}
pub(super) fn create_dataset(&mut self, key: DatasetKey) -> Result<&mut FakeDataset, Status> {
if self.datasets.contains_key(&key) {
return Err(key.already_exists());
}
let dataset = FakeDataset::new(self.next_generation());
Ok(self.datasets.entry(key).or_insert(dataset))
}
pub(super) fn create_table(&mut self, key: TableKey, table: FakeTable) -> BigQueryResult<()> {
if self.tables.contains_key(&key) {
return Err(BigQueryError::from(key.already_exists()));
}
if !self.datasets.contains_key(&key.dataset) {
self.create_dataset(key.dataset.clone())?;
}
self.tables.insert(key, table);
Ok(())
}
pub(super) fn table(&self, key: &TableKey) -> BigQueryResult<&FakeTable> {
self.tables
.get(key)
.ok_or_else(|| BigQueryError::from(key.not_found()))
}
pub(super) fn add_job(&mut self, job: FakeJob) -> BigQueryJobId {
let job_id = BigQueryJobId::new(format!("fake-job-{}", self.next_id()))
.expect("letters, digits and dashes make a valid job ID");
self.jobs.insert(job_id.clone(), job);
job_id
}
}
pub(super) fn fit_to_layout(
batch: &RecordBatch,
schema: &BigQueryTableSchema,
layout: &SchemaRef,
) -> Result<RecordBatch, String> {
let columns = fit_fields(
batch.schema().fields(),
batch.columns(),
layout.fields(),
&schema.fields,
batch.num_rows(),
)?;
let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
RecordBatch::try_new_with_options(layout.clone(), columns, &options)
.map_err(|err| format!("the Arrow rows do not fit the table: {err}"))
}
fn fit_fields(
fields: &Fields,
columns: &[ArrayRef],
layout: &Fields,
declared: &[BigQueryFieldSchema],
rows: usize,
) -> Result<Vec<ArrayRef>, String> {
if let Some(extra) = fields.iter().find(|field| {
!layout
.iter()
.any(|column| column.name().eq_ignore_ascii_case(field.name()))
}) {
return Err(format!(
"the table has no column for the Arrow field {}",
extra.name()
));
}
layout
.iter()
.zip(declared)
.map(|(column, declared)| {
let found = fields
.iter()
.position(|field| field.name().eq_ignore_ascii_case(column.name()));
match found.and_then(|index| columns.get(index)) {
Some(values) => fit_values(values, column, &declared.field_type),
None => Ok(new_null_array(column.data_type(), rows)),
}
})
.collect()
}
fn fit_values(
values: &ArrayRef,
column: &Field,
field_type: &BigQueryFieldType,
) -> Result<ArrayRef, String> {
let named = |err: ArrowError| format!("the Arrow field {}: {err}", column.name());
let fitted: ArrayRef = match (values.data_type(), column.data_type()) {
(DataType::Struct(fields), DataType::Struct(layout)) => {
let record = values.as_struct();
let declared = field_type.message_fields().ok_or_else(|| {
format!(
"the Arrow field {} is a struct of a {field_type}",
column.name()
)
})?;
let children = fit_fields(fields, record.columns(), layout, &declared, record.len())?;
Arc::new(
StructArray::try_new(layout.clone(), children, record.nulls().cloned())
.map_err(named)?,
)
}
(DataType::List(_), DataType::List(item)) => {
let list = values.as_list::<i32>();
let items = fit_values(list.values(), item, field_type)?;
Arc::new(
ListArray::try_new(
item.clone(),
list.offsets().clone(),
items,
list.nulls().cloned(),
)
.map_err(named)?,
)
}
(DataType::Decimal128(_, from_scale), DataType::Decimal128(precision, scale)) => {
let rescale = decimal_rescale(column, *from_scale, Some(*precision), *scale)?;
Arc::new(
values
.as_primitive::<Decimal128Type>()
.try_unary::<_, Decimal128Type, _>(|value| {
rescale(i256::from_i128(value))?.to_i128().ok_or_else(|| {
format!("a value of {} does not fit 128 bits", column.name())
})
})?
.with_precision_and_scale(*precision, *scale)
.map_err(named)?,
)
}
(DataType::Decimal256(_, from_scale), DataType::Decimal256(precision, scale)) => {
let bound = matches!(field_type, BigQueryFieldType::BigNumeric(Some(_)));
let rescale =
decimal_rescale(column, *from_scale, bound.then_some(*precision), *scale)?;
Arc::new(
values
.as_primitive::<Decimal256Type>()
.try_unary::<_, Decimal256Type, _>(rescale)?
.with_precision_and_scale(*precision, *scale)
.map_err(named)?,
)
}
(found, taken) if found == taken => values.clone(),
(found, taken) => {
return Err(format!(
"the Arrow field {} is {found}, and its column takes {taken}",
column.name()
))
}
};
Ok(fitted)
}
fn decimal_rescale(
column: &Field,
from_scale: i8,
precision: Option<u8>,
scale: i8,
) -> Result<impl Fn(i256) -> Result<i256, String> + '_, String> {
let unsigned = |scale: i8| {
u8::try_from(scale)
.map_err(|_| format!("the Arrow field {} has the scale {scale}", column.name()))
};
let from_scale = u32::from(unsigned(from_scale)?);
let scale = unsigned(scale)?;
Ok(move |value: i256| match precision {
Some(precision) => BigQueryDecimalParams { precision, scale }
.rescale(value, from_scale)
.ok_or_else(|| {
format!(
"a value of {} has more than {precision} digits",
column.name()
)
}),
None => rescale_decimal(value, from_scale, u32::from(scale))
.ok_or_else(|| format!("a value of {} does not fit 256 bits", column.name())),
})
}