use std::{
collections::HashMap,
ops::{Deref, DerefMut},
time::{SystemTime, UNIX_EPOCH},
};
use derive_builder::Builder;
use serde::{Deserialize, Serialize};
use serde_repr::{Deserialize_repr, Serialize_repr};
use uuid::Uuid;
use crate::error::Error;
use super::schema::Schema;
use _serde::ViewMetadataEnum;
pub static REF_PREFIX: &str = "ref-";
pub type ViewMetadata = GeneralViewMetadata<Option<()>>;
pub type ViewMetadataBuilder = GeneralViewMetadataBuilder<Option<()>>;
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Builder)]
#[serde(try_from = "ViewMetadataEnum<T>", into = "ViewMetadataEnum<T>")]
pub struct GeneralViewMetadata<T: Clone + Default> {
#[builder(default = "Uuid::new_v4()")]
pub view_uuid: Uuid,
#[builder(default)]
pub format_version: FormatVersion,
#[builder(setter(into))]
pub location: String,
pub current_version_id: i64,
#[builder(setter(each(name = "with_version")), default)]
pub versions: HashMap<i64, Version>,
#[builder(default)]
pub version_log: Vec<VersionLogStruct>,
#[builder(setter(each(name = "with_schema")), default)]
pub schemas: HashMap<i32, Schema>,
#[builder(default)]
pub properties: ViewProperties<T>,
}
impl<T: Clone + Default> GeneralViewMetadata<T> {
#[inline]
pub fn current_schema(&self, branch: Option<&str>) -> Result<&Schema, Error> {
let id = self.current_version(branch)?.schema_id;
self.schemas
.get(&id)
.ok_or_else(|| Error::InvalidFormat("view metadata".to_string()))
}
#[inline]
pub fn schema(&self, version_id: i64) -> Result<&Schema, Error> {
let id = self
.versions
.get(&version_id)
.ok_or_else(|| Error::NotFound("Version".to_string(), version_id.to_string()))?
.schema_id;
self.schemas
.get(&id)
.ok_or_else(|| Error::InvalidFormat("view metadata".to_string()))
}
#[inline]
pub fn current_version(&self, snapshot_ref: Option<&str>) -> Result<&Version, Error> {
let version_id: i64 = match snapshot_ref {
None => self.current_version_id,
Some(reference) => self
.properties
.get(&(REF_PREFIX.to_string() + reference))
.and_then(|x| x.parse().ok())
.unwrap_or(self.current_version_id),
};
self.versions
.get(&version_id)
.ok_or_else(|| Error::InvalidFormat("view metadata".to_string()))
}
#[inline]
pub fn add_schema(&mut self, schema: Schema) {
self.schemas.insert(*schema.schema_id(), schema);
}
}
mod _serde {
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{
error::Error,
spec::{schema::SchemaV2, table_metadata::VersionNumber},
};
use super::{FormatVersion, GeneralViewMetadata, Version, VersionLogStruct, ViewProperties};
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(untagged)]
pub(super) enum ViewMetadataEnum<T: Clone> {
V1(ViewMetadataV1<T>),
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub struct ViewMetadataV1<T: Clone> {
pub view_uuid: Uuid,
pub format_version: VersionNumber<1>,
pub location: String,
pub current_version_id: i64,
pub versions: Vec<Version>,
pub version_log: Vec<VersionLogStruct>,
pub schemas: Vec<SchemaV2>,
pub properties: Option<ViewProperties<T>>,
}
impl<T: Clone + Default> TryFrom<ViewMetadataEnum<T>> for GeneralViewMetadata<T> {
type Error = Error;
fn try_from(value: ViewMetadataEnum<T>) -> Result<Self, Self::Error> {
match value {
ViewMetadataEnum::V1(metadata) => metadata.try_into(),
}
}
}
impl<T: Clone + Default> From<GeneralViewMetadata<T>> for ViewMetadataEnum<T> {
fn from(value: GeneralViewMetadata<T>) -> Self {
match value.format_version {
FormatVersion::V1 => ViewMetadataEnum::V1(value.into()),
}
}
}
impl<T: Clone + Default> TryFrom<ViewMetadataV1<T>> for GeneralViewMetadata<T> {
type Error = Error;
fn try_from(value: ViewMetadataV1<T>) -> Result<Self, Self::Error> {
Ok(GeneralViewMetadata {
view_uuid: value.view_uuid,
format_version: FormatVersion::V1,
location: value.location,
current_version_id: value.current_version_id,
versions: HashMap::from_iter(value.versions.into_iter().map(|x| (x.version_id, x))),
version_log: value.version_log,
properties: value.properties.unwrap_or_default(),
schemas: HashMap::from_iter(
value
.schemas
.into_iter()
.map(|x| Ok((x.schema_id, x.try_into()?)))
.collect::<Result<Vec<_>, Error>>()?,
),
})
}
}
impl<T: Clone + Default> From<GeneralViewMetadata<T>> for ViewMetadataV1<T> {
fn from(value: GeneralViewMetadata<T>) -> Self {
ViewMetadataV1 {
view_uuid: value.view_uuid,
format_version: VersionNumber::<1>,
location: value.location,
current_version_id: value.current_version_id,
versions: value.versions.into_values().collect(),
version_log: value.version_log,
properties: Some(value.properties),
schemas: value.schemas.into_values().map(Into::into).collect(),
}
}
}
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default)]
pub struct ViewProperties<T: Clone> {
pub storage_table: T,
#[serde(flatten)]
pub other: HashMap<String, String>,
}
impl<T: Clone> Extend<(String, String)> for ViewProperties<T> {
fn extend<S: IntoIterator<Item = (String, String)>>(&mut self, iter: S) {
for (key, value) in iter {
self.other.insert(key, value);
}
}
}
impl<T: Clone> Deref for ViewProperties<T> {
type Target = HashMap<String, String>;
fn deref(&self) -> &Self::Target {
&self.other
}
}
impl<T: Clone> DerefMut for ViewProperties<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.other
}
}
#[derive(Debug, Serialize_repr, Deserialize_repr, PartialEq, Eq, Clone)]
#[repr(u8)]
#[derive(Default)]
pub enum FormatVersion {
#[default]
V1 = b'1',
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Builder)]
#[serde(rename_all = "kebab-case")]
pub struct Version {
pub version_id: i64,
pub schema_id: i32,
#[builder(
default = "SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros() as i64"
)]
pub timestamp_ms: i64,
#[builder(default)]
pub summary: Summary,
#[builder(setter(each(name = "with_representation")), default)]
pub representations: Vec<ViewRepresentation>,
#[builder(default)]
pub default_catalog: Option<String>,
#[builder(default)]
pub default_namespace: Option<Vec<String>>,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct VersionLogStruct {
pub timestamp_ms: i64,
pub version_id: i64,
}
#[derive(Debug, PartialEq, Eq, Clone)]
#[derive(Default)]
pub enum Operation {
#[default]
Create,
Replace,
}
impl Serialize for Operation {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use Operation::*;
match self {
Create => serializer.serialize_str("create"),
Replace => serializer.serialize_str("replace"),
}
}
}
impl<'de> Deserialize<'de> for Operation {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
if s == "create" {
Ok(Operation::Create)
} else if s == "replace" {
Ok(Operation::Replace)
} else {
Err(serde::de::Error::custom("Invalid view operation."))
}
}
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default)]
#[serde(rename_all = "kebab-case")]
pub struct Summary {
pub operation: Operation,
pub engine_name: Option<String>,
pub engine_version: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
#[serde(rename_all = "kebab-case", tag = "type")]
pub enum ViewRepresentation {
#[serde(rename = "sql")]
Sql {
sql: String,
dialect: String,
},
}
impl Representation for ViewRepresentation {}
pub trait Representation: Clone {}
#[cfg(test)]
mod tests {
use crate::{error::Error, spec::view_metadata::ViewMetadata};
#[test]
fn test_deserialize_view_data_v1() -> Result<(), Error> {
let data = r#"
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version" : 1,
"location" : "s3://bucket/warehouse/default.db/event_agg",
"current-version-id" : 1,
"properties" : {
"comment" : "Daily event counts"
},
"versions" : [ {
"version-id" : 1,
"timestamp-ms" : 1573518431292,
"schema-id" : 1,
"default-catalog" : "prod",
"default-namespace" : [ "default" ],
"summary" : {
"operation" : "create",
"engine-name" : "Spark",
"engineVersion" : "3.3.2"
},
"representations" : [ {
"type" : "sql",
"sql" : "SELECT\n COUNT(1), CAST(event_ts AS DATE)\nFROM events\nGROUP BY 2",
"dialect" : "spark"
} ]
} ],
"schemas": [ {
"schema-id": 1,
"type" : "struct",
"fields" : [ {
"id" : 1,
"name" : "event_count",
"required" : false,
"type" : "int",
"doc" : "Count of events"
}, {
"id" : 2,
"name" : "event_date",
"required" : false,
"type" : "date"
} ]
} ],
"version-log" : [ {
"timestamp-ms" : 1573518431292,
"version-id" : 1
} ]
}
"#;
let metadata =
serde_json::from_str::<ViewMetadata>(data).expect("Failed to deserialize json");
let metadata_two: ViewMetadata = serde_json::from_str(
&serde_json::to_string(&metadata).expect("Failed to serialize metadata"),
)
.expect("Failed to serialize json");
assert_eq!(metadata, metadata_two);
Ok(())
}
}