use crate::errors::BigQueryCodecErrorKind;
use crate::types::civil::{
self, DATE_MAX_DAYS, DATE_MIN_DAYS, MICROS_PER_DAY, TIMESTAMP_MAX_MICROS, TIMESTAMP_MIN_MICROS,
};
use crate::types::decimal::{self, BIGNUMERIC_SCALE, NUMERIC_SCALE, TAG_DECIMAL};
use crate::types::error::CodecError;
use crate::types::interval::{BigQueryInterval, TAG_INTERVAL};
use crate::types::json::TAG_JSON;
use crate::types::kind::FieldKind;
use crate::types::temporal::{capture_integer, temporal_tag_kind};
use crate::write::descriptor::{FieldPlan, MessagePlan, WritePlan};
use crate::{BigQueryChangeSequenceNumber, BigQueryChangeType};
use arrow_buffer::i256;
use gcloud_sdk::prost::encoding::{encode_varint, encoded_len_varint};
use serde::ser::{self, Impossible, Serialize};
use std::fmt::{self, Write as _};
use std::sync::Arc;
impl FieldPlan {
fn mismatch(&self, what: &str) -> CodecError {
CodecError::type_mismatch(format!(
"{what} cannot be written to a {} column",
self.kind.name()
))
}
}
const NOT_A_ROW: &str = "a row must serialize as a struct or a map";
const NOT_A_SEQUENCE: &str = "a REPEATED field takes a sequence";
const NOT_A_KEY: &str = "a map key must be a string";
impl From<serde_json::Error> for CodecError {
fn from(err: serde_json::Error) -> Self {
CodecError::new(
BigQueryCodecErrorKind::Custom,
format!("JSON column: {err}"),
)
}
}
#[derive(Clone, Copy)]
struct CachedKey {
key_address: usize,
key_length: usize,
field_index: u32,
}
type KeyCache = Vec<CachedKey>;
pub(crate) struct Encoder {
plan: Arc<WritePlan>,
caches: Vec<KeyCache>,
scratch: String,
}
impl Encoder {
pub(crate) fn new(plan: Arc<WritePlan>) -> Self {
let caches = vec![Vec::new(); plan.messages];
Encoder {
plan,
caches,
scratch: String::new(),
}
}
pub(crate) fn plan(&self) -> &Arc<WritePlan> {
&self.plan
}
pub(crate) fn encode<T: Serialize + ?Sized>(
&mut self,
row: &T,
out: &mut Vec<u8>,
) -> Result<(), CodecError> {
let start = out.len();
let result = self.encode_row(row, out);
if result.is_err() {
out.truncate(start);
}
result
}
pub(crate) fn encode_change<T: Serialize + ?Sized>(
&mut self,
row: &T,
change_type: BigQueryChangeType,
sequence_number: Option<&BigQueryChangeSequenceNumber>,
out: &mut Vec<u8>,
) -> Result<(), CodecError> {
let Some(cdc) = self.plan.cdc else {
return Err(CodecError::new(
BigQueryCodecErrorKind::Custom,
"the write plan has no CDC pseudo-columns",
));
};
let start = out.len();
if let Err(err) = self.encode_row(row, out) {
out.truncate(start);
return Err(err);
}
let mut output = RowOutput {
out,
caches: &mut self.caches,
scratch: &mut self.scratch,
};
output.put(cdc.change_type.get());
output.length_delimited(change_type.name().as_bytes());
if let Some(sequence_number) = sequence_number {
output.put(cdc.sequence_number.get());
output.length_delimited(sequence_number.as_str().as_bytes());
}
Ok(())
}
fn encode_row<T: Serialize + ?Sized>(
&mut self,
row: &T,
out: &mut Vec<u8>,
) -> Result<(), CodecError> {
let mut output = RowOutput {
out,
caches: &mut self.caches,
scratch: &mut self.scratch,
};
row.serialize(RowSerializer {
output: &mut output,
message: &self.plan.root,
})
}
}
struct RowOutput<'e> {
out: &'e mut Vec<u8>,
caches: &'e mut [KeyCache],
scratch: &'e mut String,
}
impl RowOutput<'_> {
#[inline]
fn put(&mut self, bytes: &[u8]) {
self.out.extend_from_slice(bytes);
}
#[inline]
fn byte(&mut self, byte: u8) {
self.out.push(byte);
}
#[inline]
fn varint(&mut self, value: u64) {
encode_varint(value, self.out);
}
#[inline]
fn length_delimited(&mut self, bytes: &[u8]) {
self.varint(bytes.len() as u64);
self.put(bytes);
}
#[inline]
fn begin(&mut self) -> usize {
self.out.push(0);
self.out.len()
}
#[inline]
fn end(&mut self, mark: usize) {
let len = self.out.len() - mark;
if len < 0x80 {
self.out[mark - 1] = len as u8;
return;
}
let varint_length = encoded_len_varint(len as u64);
let extra = varint_length - 1;
self.out.resize(self.out.len() + extra, 0);
self.out.copy_within(mark..mark + len, mark + extra);
encode_varint(
len as u64,
&mut &mut self.out[mark - 1..mark - 1 + varint_length],
);
}
}
macro_rules! reject {
($err:expr; $($m:ident($($t:ty),*)),* $(,)?) => {
$(fn $m(self, $(_: $t),*) -> Result<Self::Ok, CodecError> { Err($err) })*
};
}
struct RowSerializer<'r, 'e, 'p> {
output: &'r mut RowOutput<'e>,
message: &'p MessagePlan,
}
impl<'r, 'e, 'p> ser::Serializer for RowSerializer<'r, 'e, 'p> {
type Ok = ();
type Error = CodecError;
type SerializeSeq = Impossible<(), CodecError>;
type SerializeTuple = Impossible<(), CodecError>;
type SerializeTupleStruct = Impossible<(), CodecError>;
type SerializeTupleVariant = Impossible<(), CodecError>;
type SerializeMap = MessageSerializer<'r, 'e, 'p>;
type SerializeStruct = MessageSerializer<'r, 'e, 'p>;
type SerializeStructVariant = Impossible<(), CodecError>;
fn serialize_struct(
self,
_name: &'static str,
_len: usize,
) -> Result<Self::SerializeStruct, CodecError> {
Ok(MessageSerializer::new(self.output, self.message, None))
}
fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, CodecError> {
Ok(MessageSerializer::new(self.output, self.message, None))
}
fn serialize_newtype_struct<T: Serialize + ?Sized>(
self,
_name: &'static str,
value: &T,
) -> Result<(), CodecError> {
value.serialize(self)
}
fn serialize_some<T: Serialize + ?Sized>(self, value: &T) -> Result<(), CodecError> {
value.serialize(self)
}
reject!(CodecError::type_mismatch(NOT_A_ROW);
serialize_bool(bool), serialize_i8(i8), serialize_i16(i16), serialize_i32(i32),
serialize_i64(i64), serialize_u8(u8), serialize_u16(u16), serialize_u32(u32),
serialize_u64(u64), serialize_f32(f32), serialize_f64(f64), serialize_char(char),
serialize_str(&str), serialize_bytes(&[u8]), serialize_none(), serialize_unit(),
serialize_unit_struct(&'static str),
serialize_unit_variant(&'static str, u32, &'static str),
);
fn serialize_newtype_variant<T: Serialize + ?Sized>(
self,
_: &'static str,
_: u32,
_: &'static str,
_: &T,
) -> Result<(), CodecError> {
Err(CodecError::type_mismatch(NOT_A_ROW))
}
fn serialize_seq(self, _: Option<usize>) -> Result<Self::SerializeSeq, CodecError> {
Err(CodecError::type_mismatch(NOT_A_ROW))
}
fn serialize_tuple(self, _: usize) -> Result<Self::SerializeTuple, CodecError> {
Err(CodecError::type_mismatch(NOT_A_ROW))
}
fn serialize_tuple_struct(
self,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleStruct, CodecError> {
Err(CodecError::type_mismatch(NOT_A_ROW))
}
fn serialize_tuple_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleVariant, CodecError> {
Err(CodecError::type_mismatch(NOT_A_ROW))
}
fn serialize_struct_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeStructVariant, CodecError> {
Err(CodecError::type_mismatch(NOT_A_ROW))
}
}
impl MessagePlan {
fn field_index(&self, key: &str) -> Result<usize, CodecError> {
self.index_of(key).ok_or_else(|| {
CodecError::new(
BigQueryCodecErrorKind::UnknownField,
"no column of this name in the table schema",
)
.at_field(key)
})
}
}
struct MessageSerializer<'r, 'e, 'p> {
output: &'r mut RowOutput<'e>,
message: &'p MessagePlan,
mark: Option<usize>,
cursor: usize,
seen: u64,
map_key: Option<usize>,
}
impl<'r, 'e, 'p> MessageSerializer<'r, 'e, 'p> {
fn new(output: &'r mut RowOutput<'e>, message: &'p MessagePlan, mark: Option<usize>) -> Self {
MessageSerializer {
output,
message,
mark,
cursor: 0,
seen: 0,
map_key: None,
}
}
#[inline]
fn lookup(&mut self, key: &'static str) -> Result<usize, CodecError> {
let position = self.cursor;
self.cursor += 1;
let key_address = key.as_ptr() as usize;
let cache = &mut self.output.caches[self.message.id];
if let Some(cached) = cache.get(position) {
if cached.key_address == key_address && cached.key_length == key.len() {
return Ok(cached.field_index as usize);
}
}
let field_index = self.message.field_index(key)?;
let entry = CachedKey {
key_address,
key_length: key.len(),
field_index: field_index as u32,
};
match position.cmp(&cache.len()) {
std::cmp::Ordering::Less => cache[position] = entry,
std::cmp::Ordering::Equal => cache.push(entry),
std::cmp::Ordering::Greater => {}
}
Ok(field_index)
}
fn field<T: Serialize + ?Sized>(&mut self, idx: usize, value: &T) -> Result<(), CodecError> {
let field = &self.message.fields[idx];
if idx < 64 {
self.seen |= 1 << idx;
}
let result = if field.repeated {
value.serialize(RepeatedSerializer {
output: self.output,
field,
})
} else {
ValueSerializer {
output: self.output,
field,
packed: false,
in_array: false,
}
.value(value)
};
result.map_err(|error| error.at_field(&self.message.names[idx]))
}
fn finish(self) -> Result<(), CodecError> {
let missing = self.message.required & !self.seen;
if missing != 0 {
let names: Vec<&str> = (0..64)
.filter(|index| missing & (1 << index) != 0)
.map(|index| self.message.names[index].as_str())
.collect();
return Err(CodecError::new(
BigQueryCodecErrorKind::MissingRequiredField,
format!("REQUIRED field(s) not written: {}", names.join(", ")),
)
.at_field(names[0]));
}
if let Some(mark) = self.mark {
self.output.end(mark);
}
Ok(())
}
}
impl ser::SerializeStruct for MessageSerializer<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
#[inline]
fn serialize_field<T: Serialize + ?Sized>(
&mut self,
key: &'static str,
value: &T,
) -> Result<(), CodecError> {
let idx = self.lookup(key)?;
self.field(idx, value)
}
fn end(self) -> Result<(), CodecError> {
self.finish()
}
}
impl ser::SerializeMap for MessageSerializer<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
fn serialize_key<T: Serialize + ?Sized>(&mut self, key: &T) -> Result<(), CodecError> {
let key = key.serialize(KeyCapture)?;
self.map_key = Some(self.message.field_index(&key)?);
Ok(())
}
fn serialize_value<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
let idx = self.map_key.take().ok_or_else(|| {
CodecError::new(BigQueryCodecErrorKind::Custom, "a map value without a key")
})?;
self.field(idx, value)
}
fn end(self) -> Result<(), CodecError> {
self.finish()
}
}
struct RepeatedSerializer<'r, 'e, 'p> {
output: &'r mut RowOutput<'e>,
field: &'p FieldPlan,
}
impl<'r, 'e, 'p> RepeatedSerializer<'r, 'e, 'p> {
fn sequence(self, len: Option<usize>) -> RepeatedSequence<'r, 'e, 'p> {
let mark = if self.field.kind.packable() && len != Some(0) {
self.output.put(self.field.packed_key.get());
Some(self.output.begin())
} else {
None
};
RepeatedSequence {
output: self.output,
field: self.field,
mark,
next_element: 0,
}
}
}
impl<'r, 'e, 'p> ser::Serializer for RepeatedSerializer<'r, 'e, 'p> {
type Ok = ();
type Error = CodecError;
type SerializeSeq = RepeatedSequence<'r, 'e, 'p>;
type SerializeTuple = RepeatedSequence<'r, 'e, 'p>;
type SerializeTupleStruct = Impossible<(), CodecError>;
type SerializeTupleVariant = Impossible<(), CodecError>;
type SerializeMap = Impossible<(), CodecError>;
type SerializeStruct = Impossible<(), CodecError>;
type SerializeStructVariant = Impossible<(), CodecError>;
fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, CodecError> {
Ok(self.sequence(len))
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, CodecError> {
Ok(self.sequence(Some(len)))
}
fn serialize_none(self) -> Result<(), CodecError> {
Ok(())
}
fn serialize_unit(self) -> Result<(), CodecError> {
Ok(())
}
fn serialize_some<T: Serialize + ?Sized>(self, value: &T) -> Result<(), CodecError> {
value.serialize(self)
}
fn serialize_newtype_struct<T: Serialize + ?Sized>(
self,
_: &'static str,
value: &T,
) -> Result<(), CodecError> {
value.serialize(self)
}
reject!(CodecError::type_mismatch(NOT_A_SEQUENCE);
serialize_bool(bool), serialize_i8(i8), serialize_i16(i16), serialize_i32(i32),
serialize_i64(i64), serialize_u8(u8), serialize_u16(u16), serialize_u32(u32),
serialize_u64(u64), serialize_f32(f32), serialize_f64(f64), serialize_char(char),
serialize_str(&str), serialize_bytes(&[u8]), serialize_unit_struct(&'static str),
serialize_unit_variant(&'static str, u32, &'static str),
);
fn serialize_newtype_variant<T: Serialize + ?Sized>(
self,
_: &'static str,
_: u32,
_: &'static str,
_: &T,
) -> Result<(), CodecError> {
Err(CodecError::type_mismatch(NOT_A_SEQUENCE))
}
fn serialize_tuple_struct(
self,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleStruct, CodecError> {
Err(CodecError::type_mismatch(NOT_A_SEQUENCE))
}
fn serialize_tuple_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleVariant, CodecError> {
Err(CodecError::type_mismatch(NOT_A_SEQUENCE))
}
fn serialize_map(self, _: Option<usize>) -> Result<Self::SerializeMap, CodecError> {
Err(CodecError::type_mismatch(NOT_A_SEQUENCE))
}
fn serialize_struct(
self,
_: &'static str,
_: usize,
) -> Result<Self::SerializeStruct, CodecError> {
Err(CodecError::type_mismatch(NOT_A_SEQUENCE))
}
fn serialize_struct_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeStructVariant, CodecError> {
Err(CodecError::type_mismatch(NOT_A_SEQUENCE))
}
}
struct RepeatedSequence<'r, 'e, 'p> {
output: &'r mut RowOutput<'e>,
field: &'p FieldPlan,
mark: Option<usize>,
next_element: usize,
}
impl RepeatedSequence<'_, '_, '_> {
fn element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
let index = self.next_element;
self.next_element += 1;
ValueSerializer {
output: self.output,
field: self.field,
packed: self.mark.is_some(),
in_array: true,
}
.value(value)
.map_err(|error| error.at_index(index))
}
fn done(self) -> Result<(), CodecError> {
if let Some(mark) = self.mark {
self.output.end(mark);
}
Ok(())
}
}
impl ser::SerializeSeq for RepeatedSequence<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
fn serialize_element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.element(value)
}
fn end(self) -> Result<(), CodecError> {
self.done()
}
}
impl ser::SerializeTuple for RepeatedSequence<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
fn serialize_element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.element(value)
}
fn end(self) -> Result<(), CodecError> {
self.done()
}
}
impl FieldKind {
fn takes_integers(self) -> bool {
matches!(
self,
FieldKind::Int64
| FieldKind::Date
| FieldKind::Time
| FieldKind::DateTime
| FieldKind::Timestamp
| FieldKind::Numeric
| FieldKind::BigNumeric
)
}
fn check_signed_year(self, text: &str) -> Result<(), CodecError> {
if text.starts_with('-') {
return Err(CodecError::out_of_range(format!(
"{} `{text}` is before BigQuery's 0001-01-01",
self.name()
)));
}
Ok(())
}
}
struct ValueSerializer<'r, 'e, 'p> {
output: &'r mut RowOutput<'e>,
field: &'p FieldPlan,
packed: bool,
in_array: bool,
}
impl<'r, 'e, 'p> ValueSerializer<'r, 'e, 'p> {
#[inline]
fn value<T: Serialize + ?Sized>(self, value: &T) -> Result<(), CodecError> {
if self.field.kind == FieldKind::Json {
value.serialize(JsonSerializer(self))
} else {
value.serialize(self)
}
}
#[inline]
fn key(&mut self) {
if !self.packed {
self.output.put(self.field.key.get());
}
}
#[inline]
fn length_delimited(mut self, bytes: &[u8]) -> Result<(), CodecError> {
self.key();
self.output.length_delimited(bytes);
Ok(())
}
fn decimal(self, value: i256) -> Result<(), CodecError> {
let (little_endian, length) = decimal::decimal_le_bytes(value);
self.length_delimited(&little_endian[..length])
}
#[inline]
fn varint_field(mut self, value: u64) -> Result<(), CodecError> {
self.key();
self.output.varint(value);
Ok(())
}
fn integer(self, value: i64) -> Result<(), CodecError> {
match self.field.kind {
FieldKind::Int64 => self.varint_field(value as u64),
FieldKind::Date => {
let days = i32::try_from(value)
.ok()
.filter(|days| (DATE_MIN_DAYS..=DATE_MAX_DAYS).contains(days))
.ok_or_else(|| {
CodecError::out_of_range(format!(
"DATE of {value} days is outside BigQuery's range"
))
})?;
self.varint_field(i64::from(days) as u64)
}
FieldKind::Time => {
if !(0..MICROS_PER_DAY).contains(&value) {
return Err(CodecError::out_of_range(format!(
"{value} microseconds is not a time of day"
)));
}
self.varint_field(civil::pack_time(value) as u64)
}
FieldKind::DateTime => {
if !(TIMESTAMP_MIN_MICROS..=TIMESTAMP_MAX_MICROS).contains(&value) {
return Err(CodecError::out_of_range(format!(
"DATETIME of {value} civil microseconds is outside BigQuery's range"
)));
}
self.varint_field(civil::pack_datetime(value) as u64)
}
FieldKind::Timestamp => {
if !(TIMESTAMP_MIN_MICROS..=TIMESTAMP_MAX_MICROS).contains(&value) {
return Err(CodecError::out_of_range(format!(
"TIMESTAMP of {value} microseconds is outside BigQuery's range"
)));
}
self.varint_field(value as u64)
}
FieldKind::Numeric => self.decimal(i256::from_i128(
i128::from(value) * 10i128.pow(NUMERIC_SCALE),
)),
FieldKind::BigNumeric => {
let half = i256::from_i128(10i128.pow(BIGNUMERIC_SCALE / 2));
self.decimal(
i256::from_i128(i128::from(value)).wrapping_mul(half.wrapping_mul(half)),
)
}
_ => Err(self.field.mismatch("an integer")),
}
}
fn wide_integer(self, value: i128) -> Result<(), CodecError> {
if !self.field.kind.takes_integers() {
return Err(self.field.mismatch("an integer"));
}
let value = i64::try_from(value).map_err(|_| {
CodecError::out_of_range(format!(
"{value} does not fit the INT64 range a {} column takes",
self.field.kind.name()
))
})?;
self.integer(value)
}
fn text(self, text: &str) -> Result<(), CodecError> {
let kind = self.field.kind;
match kind {
FieldKind::String | FieldKind::Geography | FieldKind::Json => {
self.length_delimited(text.as_bytes())
}
FieldKind::Date => {
kind.check_signed_year(text)?;
let days = civil::parse_date(text)?;
self.integer(i64::from(days))
}
FieldKind::Time => {
let micros = civil::parse_time(text)?;
self.integer(micros)
}
FieldKind::DateTime => {
kind.check_signed_year(text)?;
let micros = civil::parse_datetime(text)?;
self.integer(micros)
}
FieldKind::Timestamp => {
kind.check_signed_year(text)?;
let micros = civil::parse_timestamp(text)?;
self.integer(micros)
}
FieldKind::Numeric => {
let unscaled = decimal::parse_numeric(text)?;
self.decimal(unscaled)
}
FieldKind::BigNumeric => {
let unscaled = decimal::parse_bignumeric(text)?;
self.decimal(unscaled)
}
FieldKind::Interval => {
let interval = BigQueryInterval::parse_bq(text)?;
self.interval(interval)
}
_ => Err(self.field.mismatch("a string")),
}
}
fn interval(self, interval: BigQueryInterval) -> Result<(), CodecError> {
if interval.nanos % 1000 != 0 {
return Err(CodecError::out_of_range(format!(
"INTERVAL keeps microseconds; {} nanoseconds is not a whole number of them",
interval.nanos
)));
}
let mut text = std::mem::take(self.output.scratch);
text.clear();
interval.write_bq(&mut text);
let ValueSerializer {
output,
field,
packed,
in_array,
} = self;
let result = ValueSerializer {
output: &mut *output,
field,
packed,
in_array,
}
.length_delimited(text.as_bytes());
*output.scratch = text;
result
}
fn null(self) -> Result<(), CodecError> {
if self.in_array {
Err(CodecError::new(
BigQueryCodecErrorKind::NullArrayElement,
"an ARRAY element cannot be NULL",
))
} else if self.field.required {
Err(CodecError::new(
BigQueryCodecErrorKind::NullForRequired,
"NULL for a REQUIRED field",
))
} else {
Ok(())
}
}
fn message(mut self) -> Result<MessageSerializer<'r, 'e, 'p>, CodecError> {
let sub = self
.field
.nested
.as_deref()
.ok_or_else(|| self.field.mismatch("a struct or a map"))?;
self.key();
let mark = self.output.begin();
Ok(MessageSerializer::new(self.output, sub, Some(mark)))
}
}
impl<'r, 'e, 'p> ser::Serializer for ValueSerializer<'r, 'e, 'p> {
type Ok = ();
type Error = CodecError;
type SerializeSeq = Compound<'r, 'e, 'p>;
type SerializeTuple = Compound<'r, 'e, 'p>;
type SerializeTupleStruct = Compound<'r, 'e, 'p>;
type SerializeTupleVariant = Impossible<(), CodecError>;
type SerializeMap = MessageSerializer<'r, 'e, 'p>;
type SerializeStruct = Compound<'r, 'e, 'p>;
type SerializeStructVariant = Impossible<(), CodecError>;
fn serialize_bool(self, value: bool) -> Result<(), CodecError> {
if self.field.kind != FieldKind::Bool {
return Err(self.field.mismatch("a bool"));
}
self.varint_field(u64::from(value))
}
fn serialize_i8(self, value: i8) -> Result<(), CodecError> {
self.serialize_i64(i64::from(value))
}
fn serialize_i16(self, value: i16) -> Result<(), CodecError> {
self.serialize_i64(i64::from(value))
}
fn serialize_i32(self, value: i32) -> Result<(), CodecError> {
self.serialize_i64(i64::from(value))
}
#[inline]
fn serialize_i64(self, value: i64) -> Result<(), CodecError> {
self.integer(value)
}
fn serialize_i128(self, value: i128) -> Result<(), CodecError> {
self.wide_integer(value)
}
fn serialize_u8(self, value: u8) -> Result<(), CodecError> {
self.serialize_i64(i64::from(value))
}
fn serialize_u16(self, value: u16) -> Result<(), CodecError> {
self.serialize_i64(i64::from(value))
}
fn serialize_u32(self, value: u32) -> Result<(), CodecError> {
self.serialize_i64(i64::from(value))
}
fn serialize_u64(self, value: u64) -> Result<(), CodecError> {
self.wide_integer(i128::from(value))
}
fn serialize_u128(self, value: u128) -> Result<(), CodecError> {
match i128::try_from(value) {
Ok(value) => self.wide_integer(value),
Err(_) => self.wide_integer(i128::MAX),
}
}
fn serialize_f32(self, value: f32) -> Result<(), CodecError> {
self.serialize_f64(f64::from(value))
}
#[inline]
fn serialize_f64(mut self, value: f64) -> Result<(), CodecError> {
match self.field.kind {
FieldKind::Float64 => {
self.key();
self.output.put(&value.to_le_bytes());
Ok(())
}
FieldKind::Numeric => {
let unscaled =
decimal::numeric_in_range(decimal::decimal_from_f64(value, NUMERIC_SCALE)?)?;
self.decimal(unscaled)
}
FieldKind::BigNumeric => {
let unscaled = decimal::decimal_from_f64(value, BIGNUMERIC_SCALE)?;
self.decimal(unscaled)
}
_ => Err(self.field.mismatch("a float")),
}
}
fn serialize_char(self, value: char) -> Result<(), CodecError> {
self.text(value.encode_utf8(&mut [0u8; 4]))
}
#[inline]
fn serialize_str(self, value: &str) -> Result<(), CodecError> {
self.text(value)
}
fn collect_str<T: fmt::Display + ?Sized>(self, value: &T) -> Result<(), CodecError> {
let mut text = std::mem::take(self.output.scratch);
text.clear();
let _ = write!(text, "{value}");
let ValueSerializer {
output,
field,
packed,
in_array,
} = self;
let result = ValueSerializer {
output: &mut *output,
field,
packed,
in_array,
}
.text(&text);
*output.scratch = text;
result
}
fn serialize_bytes(self, value: &[u8]) -> Result<(), CodecError> {
match self.field.kind {
FieldKind::Bytes => self.length_delimited(value),
_ => Err(self.field.mismatch("bytes")),
}
}
fn serialize_none(self) -> Result<(), CodecError> {
self.null()
}
fn serialize_some<T: Serialize + ?Sized>(self, value: &T) -> Result<(), CodecError> {
value.serialize(self)
}
fn serialize_unit(self) -> Result<(), CodecError> {
self.null()
}
fn serialize_unit_struct(self, _: &'static str) -> Result<(), CodecError> {
self.null()
}
fn serialize_unit_variant(
self,
_: &'static str,
_: u32,
variant: &'static str,
) -> Result<(), CodecError> {
self.text(variant)
}
fn serialize_newtype_struct<T: Serialize + ?Sized>(
self,
name: &'static str,
value: &T,
) -> Result<(), CodecError> {
if let Some(kind) = temporal_tag_kind(name) {
if kind != self.field.kind {
return Err(self.field.mismatch(name));
}
let raw = capture_integer(value)?;
return self.integer(raw);
}
let tagged_kind_matches = match name {
TAG_DECIMAL => matches!(self.field.kind, FieldKind::Numeric | FieldKind::BigNumeric),
TAG_JSON => self.field.kind == FieldKind::Json,
_ => true,
};
if !tagged_kind_matches {
return Err(self.field.mismatch(name));
}
value.serialize(self)
}
fn serialize_newtype_variant<T: Serialize + ?Sized>(
self,
_: &'static str,
_: u32,
_: &'static str,
_: &T,
) -> Result<(), CodecError> {
Err(self.field.mismatch("an enum variant with data"))
}
fn serialize_seq(mut self, _len: Option<usize>) -> Result<Self::SerializeSeq, CodecError> {
match self.field.kind {
FieldKind::Bytes => {
self.key();
let mark = self.output.begin();
Ok(Compound::Bytes {
output: self.output,
field: self.field,
mark,
})
}
_ if self.in_array => Err(CodecError::new(
BigQueryCodecErrorKind::UnsupportedType,
"an ARRAY of ARRAY is not a BigQuery column type",
)),
_ => Err(self.field.mismatch("a sequence")),
}
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, CodecError> {
self.serialize_seq(Some(len))
}
fn serialize_tuple_struct(
self,
_: &'static str,
len: usize,
) -> Result<Self::SerializeTupleStruct, CodecError> {
self.serialize_seq(Some(len))
}
fn serialize_tuple_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleVariant, CodecError> {
Err(self.field.mismatch("an enum variant with data"))
}
fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, CodecError> {
self.message()
}
fn serialize_struct(
self,
name: &'static str,
_len: usize,
) -> Result<Self::SerializeStruct, CodecError> {
if self.field.kind == FieldKind::Interval {
if name != TAG_INTERVAL {
return Err(self.field.mismatch(name));
}
return Ok(Compound::Interval {
value: self,
parts: [None; 3],
});
}
Ok(Compound::Message(self.message()?))
}
fn serialize_struct_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeStructVariant, CodecError> {
Err(self.field.mismatch("an enum variant with data"))
}
}
use serde_json::value::Serializer as JsonValueSerializer;
type JsonValueSequence = <JsonValueSerializer as ser::Serializer>::SerializeSeq;
type JsonValueTupleVariant = <JsonValueSerializer as ser::Serializer>::SerializeTupleVariant;
type JsonValueMap = <JsonValueSerializer as ser::Serializer>::SerializeMap;
type JsonValueStructVariant = <JsonValueSerializer as ser::Serializer>::SerializeStructVariant;
struct JsonSerializer<'r, 'e, 'p>(ValueSerializer<'r, 'e, 'p>);
impl JsonSerializer<'_, '_, '_> {
fn print(self, value: Result<serde_json::Value, serde_json::Error>) -> Result<(), CodecError> {
let text = value.map_err(CodecError::from)?.to_string();
self.0.text(&text)
}
}
impl<'r, 'e, 'p> ser::Serializer for JsonSerializer<'r, 'e, 'p> {
type Ok = ();
type Error = CodecError;
type SerializeSeq = JsonCompound<'r, 'e, 'p, JsonValueSequence>;
type SerializeTuple = JsonCompound<'r, 'e, 'p, JsonValueSequence>;
type SerializeTupleStruct = JsonCompound<'r, 'e, 'p, JsonValueSequence>;
type SerializeTupleVariant = JsonCompound<'r, 'e, 'p, JsonValueTupleVariant>;
type SerializeMap = JsonCompound<'r, 'e, 'p, JsonValueMap>;
type SerializeStruct = JsonCompound<'r, 'e, 'p, JsonValueMap>;
type SerializeStructVariant = JsonCompound<'r, 'e, 'p, JsonValueStructVariant>;
fn serialize_bool(self, value: bool) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_bool(value))
}
fn serialize_i8(self, value: i8) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_i8(value))
}
fn serialize_i16(self, value: i16) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_i16(value))
}
fn serialize_i32(self, value: i32) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_i32(value))
}
fn serialize_i64(self, value: i64) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_i64(value))
}
fn serialize_i128(self, value: i128) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_i128(value))
}
fn serialize_u8(self, value: u8) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_u8(value))
}
fn serialize_u16(self, value: u16) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_u16(value))
}
fn serialize_u32(self, value: u32) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_u32(value))
}
fn serialize_u64(self, value: u64) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_u64(value))
}
fn serialize_u128(self, value: u128) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_u128(value))
}
fn serialize_f32(self, value: f32) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_f32(value))
}
fn serialize_f64(self, value: f64) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_f64(value))
}
fn serialize_char(self, value: char) -> Result<(), CodecError> {
self.0.serialize_char(value)
}
fn serialize_str(self, value: &str) -> Result<(), CodecError> {
self.0.text(value)
}
fn serialize_bytes(self, value: &[u8]) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_bytes(value))
}
fn serialize_none(self) -> Result<(), CodecError> {
self.0.null()
}
fn serialize_some<T: Serialize + ?Sized>(self, value: &T) -> Result<(), CodecError> {
value.serialize(self)
}
fn serialize_unit(self) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_unit())
}
fn serialize_unit_struct(self, name: &'static str) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_unit_struct(name))
}
fn serialize_unit_variant(
self,
name: &'static str,
index: u32,
variant: &'static str,
) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_unit_variant(name, index, variant))
}
fn serialize_newtype_struct<T: Serialize + ?Sized>(
self,
name: &'static str,
value: &T,
) -> Result<(), CodecError> {
if matches!(name, TAG_JSON | TAG_DECIMAL) || temporal_tag_kind(name).is_some() {
self.0.serialize_newtype_struct(name, value)
} else {
value.serialize(self)
}
}
fn serialize_newtype_variant<T: Serialize + ?Sized>(
self,
name: &'static str,
index: u32,
variant: &'static str,
value: &T,
) -> Result<(), CodecError> {
self.print(JsonValueSerializer.serialize_newtype_variant(name, index, variant, value))
}
fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, CodecError> {
JsonCompound::new(self, JsonValueSerializer.serialize_seq(len))
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, CodecError> {
JsonCompound::new(self, JsonValueSerializer.serialize_tuple(len))
}
fn serialize_tuple_struct(
self,
name: &'static str,
len: usize,
) -> Result<Self::SerializeTupleStruct, CodecError> {
JsonCompound::new(self, JsonValueSerializer.serialize_tuple_struct(name, len))
}
fn serialize_tuple_variant(
self,
name: &'static str,
index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeTupleVariant, CodecError> {
JsonCompound::new(
self,
JsonValueSerializer.serialize_tuple_variant(name, index, variant, len),
)
}
fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, CodecError> {
JsonCompound::new(self, JsonValueSerializer.serialize_map(len))
}
fn serialize_struct(
self,
name: &'static str,
len: usize,
) -> Result<Self::SerializeStruct, CodecError> {
JsonCompound::new(self, JsonValueSerializer.serialize_struct(name, len))
}
fn serialize_struct_variant(
self,
name: &'static str,
index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeStructVariant, CodecError> {
JsonCompound::new(
self,
JsonValueSerializer.serialize_struct_variant(name, index, variant, len),
)
}
}
struct JsonCompound<'r, 'e, 'p, C> {
target: JsonSerializer<'r, 'e, 'p>,
inner: C,
}
impl<'r, 'e, 'p, C> JsonCompound<'r, 'e, 'p, C> {
fn new(
target: JsonSerializer<'r, 'e, 'p>,
inner: Result<C, serde_json::Error>,
) -> Result<Self, CodecError> {
Ok(JsonCompound {
target,
inner: inner.map_err(CodecError::from)?,
})
}
}
impl<C: ser::SerializeSeq<Ok = serde_json::Value, Error = serde_json::Error>> ser::SerializeSeq
for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.inner
.serialize_element(value)
.map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
impl<C: ser::SerializeTuple<Ok = serde_json::Value, Error = serde_json::Error>> ser::SerializeTuple
for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.inner
.serialize_element(value)
.map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
impl<C: ser::SerializeTupleStruct<Ok = serde_json::Value, Error = serde_json::Error>>
ser::SerializeTupleStruct for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_field<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.inner.serialize_field(value).map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
impl<C: ser::SerializeTupleVariant<Ok = serde_json::Value, Error = serde_json::Error>>
ser::SerializeTupleVariant for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_field<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.inner.serialize_field(value).map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
impl<C: ser::SerializeMap<Ok = serde_json::Value, Error = serde_json::Error>> ser::SerializeMap
for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_key<T: Serialize + ?Sized>(&mut self, key: &T) -> Result<(), CodecError> {
self.inner.serialize_key(key).map_err(CodecError::from)
}
fn serialize_value<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.inner.serialize_value(value).map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
impl<C: ser::SerializeStruct<Ok = serde_json::Value, Error = serde_json::Error>>
ser::SerializeStruct for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_field<T: Serialize + ?Sized>(
&mut self,
key: &'static str,
value: &T,
) -> Result<(), CodecError> {
self.inner
.serialize_field(key, value)
.map_err(CodecError::from)
}
fn skip_field(&mut self, key: &'static str) -> Result<(), CodecError> {
self.inner.skip_field(key).map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
impl<C: ser::SerializeStructVariant<Ok = serde_json::Value, Error = serde_json::Error>>
ser::SerializeStructVariant for JsonCompound<'_, '_, '_, C>
{
type Ok = ();
type Error = CodecError;
fn serialize_field<T: Serialize + ?Sized>(
&mut self,
key: &'static str,
value: &T,
) -> Result<(), CodecError> {
self.inner
.serialize_field(key, value)
.map_err(CodecError::from)
}
fn end(self) -> Result<(), CodecError> {
self.target.print(self.inner.end())
}
}
enum Compound<'r, 'e, 'p> {
Message(MessageSerializer<'r, 'e, 'p>),
Bytes {
output: &'r mut RowOutput<'e>,
field: &'p FieldPlan,
mark: usize,
},
Interval {
value: ValueSerializer<'r, 'e, 'p>,
parts: [Option<i64>; 3],
},
}
const INTERVAL_PARTS: [&str; 3] = ["months", "days", "nanos"];
impl Compound<'_, '_, '_> {
fn element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
match self {
Compound::Bytes { output, field, .. } => {
let byte = capture_integer(value)
.ok()
.and_then(|integer| u8::try_from(integer).ok())
.ok_or_else(|| field.mismatch("a sequence of anything but u8"))?;
output.byte(byte);
Ok(())
}
Compound::Interval {
value: serializer, ..
} => Err(serializer.field.mismatch("a sequence")),
Compound::Message(_) => Err(CodecError::new(
BigQueryCodecErrorKind::TypeMismatch,
"a STRUCT takes named fields",
)),
}
}
fn finish(self) -> Result<(), CodecError> {
match self {
Compound::Message(message) => message.finish(),
Compound::Bytes { output, mark, .. } => {
output.end(mark);
Ok(())
}
Compound::Interval { value, parts } => {
let [Some(months), Some(days), Some(nanos)] = parts else {
return Err(value
.field
.mismatch("an INTERVAL without months, days and nanos"));
};
let part = |value: i64, name: &str| {
i32::try_from(value).map_err(|_| {
CodecError::out_of_range(format!("INTERVAL {name} {value} is out of range"))
})
};
let interval = BigQueryInterval {
months: part(months, "months")?,
days: part(days, "days")?,
nanos,
};
value.interval(interval)
}
}
}
}
impl ser::SerializeSeq for Compound<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
fn serialize_element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.element(value)
}
fn end(self) -> Result<(), CodecError> {
self.finish()
}
}
impl ser::SerializeTuple for Compound<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
fn serialize_element<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.element(value)
}
fn end(self) -> Result<(), CodecError> {
self.finish()
}
}
impl ser::SerializeTupleStruct for Compound<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
fn serialize_field<T: Serialize + ?Sized>(&mut self, value: &T) -> Result<(), CodecError> {
self.element(value)
}
fn end(self) -> Result<(), CodecError> {
self.finish()
}
}
impl ser::SerializeStruct for Compound<'_, '_, '_> {
type Ok = ();
type Error = CodecError;
#[inline]
fn serialize_field<T: Serialize + ?Sized>(
&mut self,
key: &'static str,
value: &T,
) -> Result<(), CodecError> {
match self {
Compound::Message(message) => {
ser::SerializeStruct::serialize_field(message, key, value)
}
Compound::Interval { parts, .. } => {
let part = INTERVAL_PARTS
.iter()
.position(|name| *name == key)
.ok_or_else(|| {
CodecError::new(
BigQueryCodecErrorKind::UnknownField,
"an INTERVAL has months, days and nanos",
)
.at_field(key)
})?;
parts[part] = Some(capture_integer(value).map_err(|error| error.at_field(key))?);
Ok(())
}
Compound::Bytes { field, .. } => Err(field.mismatch("a struct")),
}
}
fn end(self) -> Result<(), CodecError> {
self.finish()
}
}
struct KeyCapture;
impl ser::Serializer for KeyCapture {
type Ok = String;
type Error = CodecError;
type SerializeSeq = Impossible<String, CodecError>;
type SerializeTuple = Impossible<String, CodecError>;
type SerializeTupleStruct = Impossible<String, CodecError>;
type SerializeTupleVariant = Impossible<String, CodecError>;
type SerializeMap = Impossible<String, CodecError>;
type SerializeStruct = Impossible<String, CodecError>;
type SerializeStructVariant = Impossible<String, CodecError>;
fn serialize_str(self, value: &str) -> Result<String, CodecError> {
Ok(value.to_string())
}
fn serialize_char(self, value: char) -> Result<String, CodecError> {
Ok(value.to_string())
}
reject!(CodecError::type_mismatch(NOT_A_KEY);
serialize_bool(bool), serialize_i8(i8), serialize_i16(i16), serialize_i32(i32),
serialize_i64(i64), serialize_u8(u8), serialize_u16(u16), serialize_u32(u32),
serialize_u64(u64), serialize_f32(f32), serialize_f64(f64), serialize_bytes(&[u8]),
serialize_none(), serialize_unit(), serialize_unit_struct(&'static str),
serialize_unit_variant(&'static str, u32, &'static str),
);
fn serialize_some<T: Serialize + ?Sized>(self, value: &T) -> Result<String, CodecError> {
value.serialize(self)
}
fn serialize_newtype_struct<T: Serialize + ?Sized>(
self,
_: &'static str,
value: &T,
) -> Result<String, CodecError> {
value.serialize(self)
}
fn serialize_newtype_variant<T: Serialize + ?Sized>(
self,
_: &'static str,
_: u32,
_: &'static str,
_: &T,
) -> Result<String, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_seq(self, _: Option<usize>) -> Result<Self::SerializeSeq, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_tuple(self, _: usize) -> Result<Self::SerializeTuple, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_tuple_struct(
self,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleStruct, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_tuple_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeTupleVariant, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_map(self, _: Option<usize>) -> Result<Self::SerializeMap, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_struct(
self,
_: &'static str,
_: usize,
) -> Result<Self::SerializeStruct, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
fn serialize_struct_variant(
self,
_: &'static str,
_: u32,
_: &'static str,
_: usize,
) -> Result<Self::SerializeStructVariant, CodecError> {
Err(CodecError::type_mismatch(NOT_A_KEY))
}
}
#[cfg(test)]
pub(crate) mod tests;
#[cfg(test)]
mod roundtrip;