use arrow::datatypes::{DataType, Field, Schema};
use std::collections::HashMap;
use std::sync::Arc;
fn fnv1a_64(data: &[u8]) -> u64 {
const OFFSET_BASIS: u64 = 14_695_981_039_346_656_037;
const FNV_PRIME: u64 = 1_099_511_628_211;
let mut hash = OFFSET_BASIS;
for byte in data {
hash ^= *byte as u64;
hash = hash.wrapping_mul(FNV_PRIME);
}
hash
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
pub struct SchemaVersion {
pub name: String,
pub version: u32,
pub fingerprint: u64,
}
impl SchemaVersion {
pub fn new(name: &str, schema: &Schema) -> Self {
Self {
name: name.to_owned(),
version: 1,
fingerprint: Self::fingerprint_of(schema),
}
}
pub fn fingerprint_of(schema: &Schema) -> u64 {
let mut buf = String::new();
for field in schema.fields() {
buf.push_str(field.name());
buf.push(':');
buf.push_str(&format!("{:?}", field.data_type()));
buf.push(';');
}
fnv1a_64(buf.as_bytes())
}
pub fn is_compatible_with(&self, other: &SchemaVersion) -> bool {
self.name == other.name && self.version <= other.version
}
}
#[derive(Debug, thiserror::Error)]
pub enum SchemaError {
#[error("Schema '{0}' not found")]
NotFound(String),
#[error("Incompatible schema change: {0}")]
Incompatible(String),
#[error("Schema version {0} already registered")]
AlreadyRegistered(u32),
}
#[derive(Debug, Clone)]
pub enum EvolutionStrategy {
AddColumnsOnly,
RenameColumns {
mapping: HashMap<String, String>,
},
FullReplacement,
}
pub struct SchemaRegistry {
schemas: HashMap<SchemaVersion, Arc<Schema>>,
latest: HashMap<String, u32>,
}
impl Default for SchemaRegistry {
fn default() -> Self {
Self::new()
}
}
impl SchemaRegistry {
pub fn new() -> Self {
Self {
schemas: HashMap::new(),
latest: HashMap::new(),
}
}
pub fn register(&mut self, name: &str, schema: Arc<Schema>) -> SchemaVersion {
let version = self.latest.get(name).copied().unwrap_or(0) + 1;
let sv = SchemaVersion {
name: name.to_owned(),
version,
fingerprint: SchemaVersion::fingerprint_of(&schema),
};
self.schemas.insert(sv.clone(), schema);
self.latest.insert(name.to_owned(), version);
sv
}
pub fn get(&self, version: &SchemaVersion) -> Option<Arc<Schema>> {
self.schemas.get(version).cloned()
}
pub fn latest_version(&self, name: &str) -> Option<SchemaVersion> {
let ver = *self.latest.get(name)?;
self.schemas
.keys()
.find(|sv| sv.name == name && sv.version == ver)
.cloned()
}
pub fn upgrade_schema(
&mut self,
name: &str,
new_schema: Arc<Schema>,
) -> Result<SchemaVersion, SchemaError> {
self.upgrade_schema_with_strategy(name, new_schema, &EvolutionStrategy::AddColumnsOnly)
}
pub fn upgrade_schema_with_strategy(
&mut self,
name: &str,
new_schema: Arc<Schema>,
strategy: &EvolutionStrategy,
) -> Result<SchemaVersion, SchemaError> {
let current_sv = self
.latest_version(name)
.ok_or_else(|| SchemaError::NotFound(name.to_owned()))?;
let current_schema = self
.get(¤t_sv)
.ok_or_else(|| SchemaError::NotFound(name.to_owned()))?;
match strategy {
EvolutionStrategy::AddColumnsOnly => {
validate_add_columns_only(¤t_schema, &new_schema)?;
}
EvolutionStrategy::RenameColumns { mapping } => {
validate_rename_columns(¤t_schema, &new_schema, mapping)?;
}
EvolutionStrategy::FullReplacement => {
}
}
Ok(self.register(name, new_schema))
}
pub fn can_read_with(
&self,
writer_version: &SchemaVersion,
reader_version: &SchemaVersion,
) -> bool {
if !writer_version.is_compatible_with(reader_version) {
return false;
}
let (Some(writer_schema), Some(reader_schema)) =
(self.get(writer_version), self.get(reader_version))
else {
return false;
};
for writer_field in writer_schema.fields() {
match reader_schema.field_with_name(writer_field.name()) {
Ok(reader_field) => {
if !types_compatible(writer_field.data_type(), reader_field.data_type()) {
return false;
}
}
Err(_) => {
return false;
}
}
}
true
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SchemaEvolutionFrame {
pub session_id: String,
pub old_version: SchemaVersion,
pub new_version: SchemaVersion,
pub schema_ipc: Vec<u8>,
}
impl SchemaEvolutionFrame {
pub fn new(
session_id: impl Into<String>,
old_version: SchemaVersion,
new_version: SchemaVersion,
new_schema: &Schema,
) -> Self {
let schema_ipc = schema_to_ipc_bytes(new_schema);
Self {
session_id: session_id.into(),
old_version,
new_version,
schema_ipc,
}
}
pub fn to_bytes(&self) -> Result<Vec<u8>, SchemaError> {
serde_json::to_vec(self)
.map_err(|e| SchemaError::Incompatible(format!("serialisation error: {e}")))
}
pub fn from_bytes(bytes: &[u8]) -> Result<Self, SchemaError> {
serde_json::from_slice(bytes)
.map_err(|e| SchemaError::Incompatible(format!("deserialisation error: {e}")))
}
pub fn decode_schema(&self) -> Result<Arc<Schema>, SchemaError> {
ipc_bytes_to_schema(&self.schema_ipc)
}
}
pub fn schema_to_ipc_bytes(schema: &Schema) -> Vec<u8> {
use arrow::ipc::writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions};
let opts = IpcWriteOptions::default();
let gen = IpcDataGenerator {};
let mut tracker = DictionaryTracker::new(false);
let encoded = gen.schema_to_bytes_with_dictionary_tracker(schema, &mut tracker, &opts);
let msg = &encoded.ipc_message;
let len = msg.len() as u32;
let mut out = Vec::with_capacity(8 + msg.len());
out.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]);
out.extend_from_slice(&len.to_le_bytes());
out.extend_from_slice(msg);
out
}
pub fn ipc_bytes_to_schema(bytes: &[u8]) -> Result<Arc<Schema>, SchemaError> {
use arrow::ipc::convert::fb_to_schema;
use arrow::ipc::root_as_message;
let payload = if bytes.len() >= 8 && bytes[0..4] == [0xFF, 0xFF, 0xFF, 0xFF] {
&bytes[8..]
} else {
bytes
};
let message = root_as_message(payload)
.map_err(|e| SchemaError::Incompatible(format!("invalid Arrow IPC message: {e}")))?;
let schema_ref = message.header_as_schema().ok_or_else(|| {
SchemaError::Incompatible("IPC message does not contain a Schema header".to_owned())
})?;
Ok(Arc::new(fb_to_schema(schema_ref)))
}
fn validate_add_columns_only(old: &Schema, new: &Schema) -> Result<(), SchemaError> {
for old_field in old.fields() {
match new.field_with_name(old_field.name()) {
Ok(new_field) => {
if !types_compatible(old_field.data_type(), new_field.data_type()) {
return Err(SchemaError::Incompatible(format!(
"field '{}' changed type from {:?} to {:?}",
old_field.name(),
old_field.data_type(),
new_field.data_type(),
)));
}
}
Err(_) => {
return Err(SchemaError::Incompatible(format!(
"existing field '{}' was removed",
old_field.name()
)));
}
}
}
for new_field in new.fields() {
if old.field_with_name(new_field.name()).is_err() && !new_field.is_nullable() {
return Err(SchemaError::Incompatible(format!(
"new field '{}' must be nullable when using AddColumnsOnly strategy",
new_field.name()
)));
}
}
Ok(())
}
fn validate_rename_columns(
old: &Schema,
new: &Schema,
mapping: &HashMap<String, String>,
) -> Result<(), SchemaError> {
let renamed_fields: Vec<Arc<Field>> = old
.fields()
.iter()
.map(|f| {
let new_name = mapping
.get(f.name())
.cloned()
.unwrap_or_else(|| f.name().clone());
Arc::new(Field::new(new_name, f.data_type().clone(), f.is_nullable()))
})
.collect();
for new_field in new.fields() {
let found = renamed_fields
.iter()
.find(|rf| rf.name() == new_field.name());
match found {
Some(rf) => {
if !types_compatible(rf.data_type(), new_field.data_type()) {
return Err(SchemaError::Incompatible(format!(
"field '{}' changed type after rename",
new_field.name()
)));
}
}
None => {
if !new_field.is_nullable() {
return Err(SchemaError::Incompatible(format!(
"new non-nullable field '{}' not covered by rename mapping",
new_field.name()
)));
}
}
}
}
for rf in &renamed_fields {
if new.field_with_name(rf.name()).is_err() {
return Err(SchemaError::Incompatible(format!(
"renamed field '{}' is absent from the new schema",
rf.name()
)));
}
}
Ok(())
}
fn types_compatible(writer: &DataType, reader: &DataType) -> bool {
writer == reader
}
#[cfg(test)]
mod tests {
use super::*;
use arrow::datatypes::{DataType, Field, Schema};
use std::sync::Arc;
fn make_schema(fields: Vec<(&str, DataType, bool)>) -> Arc<Schema> {
let arrow_fields: Vec<Field> = fields
.into_iter()
.map(|(name, dt, nullable)| Field::new(name, dt, nullable))
.collect();
Arc::new(Schema::new(arrow_fields))
}
#[test]
fn test_schema_version_fingerprint_deterministic() {
let schema = make_schema(vec![
("id", DataType::Int64, false),
("embedding", DataType::Float32, false),
]);
let fp1 = SchemaVersion::fingerprint_of(&schema);
let fp2 = SchemaVersion::fingerprint_of(&schema);
assert_eq!(fp1, fp2, "fingerprint must be deterministic");
}
#[test]
fn test_schema_version_different_schemas() {
let schema_a = make_schema(vec![("id", DataType::Int64, false)]);
let schema_b = make_schema(vec![("id", DataType::Int32, false)]);
let fp_a = SchemaVersion::fingerprint_of(&schema_a);
let fp_b = SchemaVersion::fingerprint_of(&schema_b);
assert_ne!(
fp_a, fp_b,
"different schemas must produce different fingerprints"
);
}
#[test]
fn test_registry_register_and_get() {
let mut reg = SchemaRegistry::new();
let schema = make_schema(vec![("x", DataType::Float32, false)]);
let sv = reg.register("model", schema.clone());
assert_eq!(sv.name, "model");
assert_eq!(sv.version, 1);
let retrieved = reg.get(&sv).expect("schema should be retrievable");
assert_eq!(retrieved.fields().len(), 1);
}
#[test]
fn test_registry_upgrade_add_column() {
let mut reg = SchemaRegistry::new();
let schema_v1 = make_schema(vec![("x", DataType::Float32, false)]);
reg.register("model", schema_v1);
let schema_v2 = make_schema(vec![
("x", DataType::Float32, false),
("bias", DataType::Float32, true), ]);
let sv2 = reg
.upgrade_schema("model", schema_v2)
.expect("adding nullable column must succeed");
assert_eq!(sv2.version, 2);
}
#[test]
fn test_registry_upgrade_incompatible() {
let mut reg = SchemaRegistry::new();
let schema_v1 = make_schema(vec![("x", DataType::Float32, false)]);
reg.register("model", schema_v1);
let schema_bad = make_schema(vec![
("y", DataType::Float32, false), ]);
let result = reg.upgrade_schema("model", schema_bad);
assert!(
matches!(result, Err(SchemaError::Incompatible(_))),
"removing a field must be incompatible"
);
}
#[test]
fn test_can_read_with() {
let mut reg = SchemaRegistry::new();
let schema_v1 = make_schema(vec![("x", DataType::Float32, false)]);
let sv1 = reg.register("model", schema_v1);
let schema_v2 = make_schema(vec![
("x", DataType::Float32, false),
("y", DataType::Float32, true),
]);
let sv2 = reg
.upgrade_schema("model", schema_v2)
.expect("test: upgrade schema");
assert!(
reg.can_read_with(&sv1, &sv2),
"newer reader must be able to read older writer data"
);
assert!(
!reg.can_read_with(&sv2, &sv1),
"older reader must NOT be able to read newer writer data with extra fields"
);
}
#[test]
fn test_schema_evolution_frame_roundtrip() {
let mut reg = SchemaRegistry::new();
let schema_v1 = make_schema(vec![("a", DataType::Int32, false)]);
let sv1 = reg.register("tensors", schema_v1);
let schema_v2 = make_schema(vec![
("a", DataType::Int32, false),
("b", DataType::Float64, true),
]);
let sv2 = reg
.upgrade_schema("tensors", schema_v2.clone())
.expect("test: upgrade schema");
let frame = SchemaEvolutionFrame::new("session-42", sv1.clone(), sv2.clone(), &schema_v2);
let bytes = frame.to_bytes().expect("serialisation must succeed");
let decoded =
SchemaEvolutionFrame::from_bytes(&bytes).expect("deserialisation must succeed");
assert_eq!(decoded.session_id, "session-42");
assert_eq!(decoded.old_version, sv1);
assert_eq!(decoded.new_version, sv2);
let recovered = decoded.decode_schema().expect("IPC decode must succeed");
assert_eq!(recovered.fields().len(), 2);
assert_eq!(recovered.field(0).name(), "a");
assert_eq!(recovered.field(1).name(), "b");
}
#[test]
fn test_latest_version() {
let mut reg = SchemaRegistry::new();
assert!(reg.latest_version("missing").is_none());
let s = make_schema(vec![("v", DataType::Utf8, true)]);
reg.register("ns", s.clone());
let sv = reg.latest_version("ns").expect("test: get latest version");
assert_eq!(sv.version, 1);
let s2 = make_schema(vec![
("v", DataType::Utf8, true),
("w", DataType::Int8, true),
]);
reg.upgrade_schema("ns", s2).expect("test: upgrade schema");
let sv2 = reg.latest_version("ns").expect("test: get latest version");
assert_eq!(sv2.version, 2);
}
#[test]
fn test_upgrade_rename_columns() {
let mut reg = SchemaRegistry::new();
let schema_v1 = make_schema(vec![("old_name", DataType::Float32, false)]);
reg.register("layer", schema_v1);
let schema_v2 = make_schema(vec![("new_name", DataType::Float32, false)]);
let mut map = HashMap::new();
map.insert("old_name".to_owned(), "new_name".to_owned());
let result = reg.upgrade_schema_with_strategy(
"layer",
schema_v2,
&EvolutionStrategy::RenameColumns { mapping: map },
);
assert!(result.is_ok(), "valid rename must succeed: {result:?}");
}
#[test]
fn test_upgrade_full_replacement() {
let mut reg = SchemaRegistry::new();
let schema_v1 = make_schema(vec![("a", DataType::Int64, false)]);
reg.register("data", schema_v1);
let schema_v2 = make_schema(vec![("z", DataType::Boolean, false)]);
let result = reg.upgrade_schema_with_strategy(
"data",
schema_v2,
&EvolutionStrategy::FullReplacement,
);
assert!(result.is_ok(), "FullReplacement must accept any new schema");
}
}