use crate::types::kind::FieldKind;
use crate::{
BigQueryFieldMode, BigQueryFieldSchema, BigQueryFieldType, BigQueryRangeElementType,
BigQueryTableSchema,
};
use gcloud_sdk::prost::encoding::{encode_key, key_len, WireType};
use gcloud_sdk::prost_types::field_descriptor_proto::{Label, Type as ProtoType};
use gcloud_sdk::prost_types::{DescriptorProto, FieldDescriptorProto};
use std::collections::HashMap;
pub(crate) const ROOT_MESSAGE: &str = "row";
pub(crate) const CHANGE_TYPE_COLUMN: &str = "_CHANGE_TYPE";
pub(crate) const CHANGE_SEQUENCE_NUMBER_COLUMN: &str = "_CHANGE_SEQUENCE_NUMBER";
#[derive(Clone, Copy, Debug)]
pub(crate) struct Key {
bytes: [u8; 5],
len: u8,
}
impl Key {
fn new(number: u32, wire_type: WireType) -> Key {
let mut bytes = [0u8; 5];
let len = key_len(number);
encode_key(number, wire_type, &mut &mut bytes[..len]);
Key {
bytes,
len: len as u8,
}
}
#[inline]
pub(crate) fn get(&self) -> &[u8] {
&self.bytes[..usize::from(self.len)]
}
}
#[derive(Debug)]
pub(crate) struct FieldPlan {
pub(crate) kind: FieldKind,
pub(crate) required: bool,
pub(crate) repeated: bool,
pub(crate) key: Key,
pub(crate) packed_key: Key,
pub(crate) nested: Option<Box<MessagePlan>>,
}
#[derive(Debug)]
pub(crate) struct MessagePlan {
pub(crate) id: usize,
pub(crate) names: Vec<String>,
pub(crate) fields: Vec<FieldPlan>,
by_name: HashMap<String, usize>,
pub(crate) required: u64,
}
impl MessagePlan {
pub(crate) fn index_of(&self, name: &str) -> Option<usize> {
self.by_name.get(name).copied()
}
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct CdcKeys {
pub(crate) change_type: Key,
pub(crate) sequence_number: Key,
}
#[derive(Debug)]
pub(crate) struct WritePlan {
schema: BigQueryTableSchema,
descriptor: DescriptorProto,
pub(crate) root: MessagePlan,
pub(crate) messages: usize,
pub(crate) cdc: Option<CdcKeys>,
}
impl FieldKind {
fn proto_type(self) -> ProtoType {
match self {
FieldKind::Int64 | FieldKind::Time | FieldKind::DateTime | FieldKind::Timestamp => {
ProtoType::Int64
}
FieldKind::Float64 => ProtoType::Double,
FieldKind::Bool => ProtoType::Bool,
FieldKind::Date => ProtoType::Int32,
FieldKind::Bytes | FieldKind::Numeric | FieldKind::BigNumeric => ProtoType::Bytes,
FieldKind::String | FieldKind::Geography | FieldKind::Json | FieldKind::Interval => {
ProtoType::String
}
FieldKind::Struct | FieldKind::Range => ProtoType::Message,
}
}
fn wire_type(self) -> WireType {
match self.proto_type() {
ProtoType::Int64 | ProtoType::Int32 | ProtoType::Bool => WireType::Varint,
ProtoType::Double => WireType::SixtyFourBit,
_ => WireType::LengthDelimited,
}
}
}
impl BigQueryRangeElementType {
fn field_named(self, name: &str) -> BigQueryFieldSchema {
let field_type = match self {
BigQueryRangeElementType::Date => BigQueryFieldType::Date,
BigQueryRangeElementType::DateTime => BigQueryFieldType::DateTime,
BigQueryRangeElementType::Timestamp => BigQueryFieldType::Timestamp,
};
BigQueryFieldSchema {
name: name.to_string(),
field_type,
mode: BigQueryFieldMode::Nullable,
description: None,
default_value_expression: None,
}
}
}
fn field_number(index: usize) -> u32 {
u32::try_from(index + 1).unwrap_or(u32::MAX)
}
struct Compiler {
nested: Vec<DescriptorProto>,
messages: usize,
}
impl Compiler {
fn compile(
&mut self,
fields: &[BigQueryFieldSchema],
) -> (MessagePlan, Vec<FieldDescriptorProto>) {
let id = self.messages;
self.messages += 1;
let mut plan = MessagePlan {
id,
names: Vec::with_capacity(fields.len()),
fields: Vec::with_capacity(fields.len()),
by_name: HashMap::with_capacity(fields.len()),
required: 0,
};
let mut descriptors = Vec::with_capacity(fields.len());
for (i, field) in fields.iter().enumerate() {
let kind = FieldKind::from(&field.field_type);
let number = field_number(i);
let mut descriptor = FieldDescriptorProto {
name: Some(field.name.clone()),
number: i32::try_from(number).ok(),
label: Some(
match field.mode {
BigQueryFieldMode::Required => Label::Required,
BigQueryFieldMode::Repeated => Label::Repeated,
BigQueryFieldMode::Nullable => Label::Optional,
}
.into(),
),
r#type: Some(kind.proto_type().into()),
..Default::default()
};
let children = match &field.field_type {
BigQueryFieldType::Struct(children) => Some(children.clone()),
BigQueryFieldType::Range(element) => Some(vec![
element.field_named("start"),
element.field_named("end"),
]),
_ => None,
};
let nested = children.map(|children| {
let (nested, nested_fields) = self.compile(&children);
let type_name = format!("{ROOT_MESSAGE}_{}", nested.id);
self.nested.push(DescriptorProto {
name: Some(type_name.clone()),
field: nested_fields,
..Default::default()
});
descriptor.type_name = Some(type_name);
Box::new(nested)
});
let required = field.mode == BigQueryFieldMode::Required;
if required && i < 64 {
plan.required |= 1 << i;
}
plan.by_name.insert(field.name.clone(), i);
plan.names.push(field.name.clone());
plan.fields.push(FieldPlan {
kind,
required,
repeated: field.mode == BigQueryFieldMode::Repeated,
key: Key::new(number, kind.wire_type()),
packed_key: Key::new(number, WireType::LengthDelimited),
nested,
});
descriptors.push(descriptor);
}
(plan, descriptors)
}
}
fn pseudo_column(name: &str, number: u32) -> FieldDescriptorProto {
FieldDescriptorProto {
name: Some(name.to_string()),
number: i32::try_from(number).ok(),
label: Some(Label::Optional.into()),
r#type: Some(ProtoType::String.into()),
..Default::default()
}
}
impl WritePlan {
pub(crate) fn new(schema: &BigQueryTableSchema, cdc: bool) -> WritePlan {
let mut compiler = Compiler {
nested: Vec::new(),
messages: 0,
};
let (root, mut fields) = compiler.compile(&schema.fields);
let cdc = cdc.then(|| {
let change_type = field_number(schema.fields.len());
let sequence_number = change_type.saturating_add(1);
fields.push(pseudo_column(CHANGE_TYPE_COLUMN, change_type));
fields.push(pseudo_column(
CHANGE_SEQUENCE_NUMBER_COLUMN,
sequence_number,
));
CdcKeys {
change_type: Key::new(change_type, WireType::LengthDelimited),
sequence_number: Key::new(sequence_number, WireType::LengthDelimited),
}
});
let descriptor = DescriptorProto {
name: Some(ROOT_MESSAGE.to_string()),
field: fields,
nested_type: compiler.nested,
..Default::default()
};
WritePlan {
schema: schema.clone(),
descriptor,
root,
messages: compiler.messages,
cdc,
}
}
pub(crate) fn descriptor(&self) -> &DescriptorProto {
&self.descriptor
}
pub(crate) fn schema(&self) -> &BigQueryTableSchema {
&self.schema
}
}
pub(crate) fn relaxes_a_required_field(
old: &[BigQueryFieldSchema],
new: &[BigQueryFieldSchema],
) -> bool {
old.iter().any(|before| {
new.iter()
.find(|after| after.name == before.name)
.is_some_and(|after| {
let relaxed = before.mode == BigQueryFieldMode::Required
&& after.mode == BigQueryFieldMode::Nullable;
let nested = match (&before.field_type, &after.field_type) {
(BigQueryFieldType::Struct(before), BigQueryFieldType::Struct(after)) => {
relaxes_a_required_field(before, after)
}
_ => false,
};
relaxed || nested
})
})
}