use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Debug, Clone, PartialEq)]
pub struct ColumnChange {
pub name: String,
pub from: Option<Value>,
pub to: Value,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SchemaDiff {
pub additions: Vec<ColumnChange>,
pub widenings: Vec<ColumnChange>,
pub incompatible: Vec<ColumnChange>,
pub droppable_required: Vec<String>,
}
impl SchemaDiff {
pub fn is_empty(&self) -> bool {
self.additions.is_empty()
&& self.widenings.is_empty()
&& self.incompatible.is_empty()
&& self.droppable_required.is_empty()
}
pub fn changed_columns(&self) -> Vec<String> {
self.additions
.iter()
.chain(&self.widenings)
.chain(&self.incompatible)
.map(|c| c.name.clone())
.chain(self.droppable_required.iter().cloned())
.collect()
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct SchemaEvolution {
pub additions: Vec<ColumnChange>,
pub widenings: Vec<ColumnChange>,
pub relax_nullability: Vec<String>,
}
impl SchemaEvolution {
pub fn is_empty(&self) -> bool {
self.additions.is_empty() && self.widenings.is_empty() && self.relax_nullability.is_empty()
}
}
fn type_set(fragment: &Value) -> (Vec<String>, bool) {
let mut names = Vec::new();
let mut nullable = false;
match fragment.get("type") {
Some(Value::String(t)) => {
if t == "null" {
nullable = true
} else {
names.push(t.clone())
}
}
Some(Value::Array(arr)) => {
for v in arr {
if let Some(t) = v.as_str() {
if t == "null" {
nullable = true
} else {
names.push(t.to_string())
}
}
}
}
_ => {}
}
names.sort();
(names, nullable)
}
fn fits(dest: &Value, page: &Value) -> bool {
let (dn, dnull) = type_set(dest);
let (pn, pnull) = type_set(page);
if pnull && !dnull {
return false;
}
pn.iter()
.all(|t| dn.contains(t) || (t == "integer" && dn.iter().any(|d| d == "number")))
}
fn evolvable(dest: &Value, page: &Value) -> bool {
let (dn, _) = type_set(dest);
let (pn, _) = type_set(page);
let mut merged: Vec<String> = dn.into_iter().chain(pn).collect();
merged.sort();
merged.dedup();
if merged.iter().any(|t| t == "number") {
merged.retain(|t| t != "integer");
}
merged.len() == 1
}
pub fn base_widened(from: &Value, to: &Value) -> bool {
let (dn, _) = type_set(from);
let (pn, _) = type_set(to);
let collapse = |names: Vec<String>| -> Vec<String> {
let mut m: Vec<String> = names;
m.sort();
m.dedup();
if m.iter().any(|t| t == "number") {
m.retain(|t| t != "integer");
}
m
};
let dest_family = collapse(dn.clone());
let merged = collapse(dn.into_iter().chain(pn).collect());
merged != dest_family
}
pub fn adds_null(from: &Value, to: &Value) -> bool {
let (_, fnull) = type_set(from);
let (_, tnull) = type_set(to);
tnull && !fnull
}
pub fn diff_schema(destination: &Value, page: &Value, allow_widening: bool) -> SchemaDiff {
let empty = serde_json::Map::new();
let dest_props = destination
.get("properties")
.and_then(|p| p.as_object())
.unwrap_or(&empty);
let page_props = page
.get("properties")
.and_then(|p| p.as_object())
.unwrap_or(&empty);
let mut diff = SchemaDiff::default();
for (name, page_ty) in page_props {
match dest_props.get(name) {
None => diff.additions.push(ColumnChange {
name: name.clone(),
from: None,
to: page_ty.clone(),
}),
Some(dest_ty) => {
if fits(dest_ty, page_ty) {
continue; }
let change = ColumnChange {
name: name.clone(),
from: Some(dest_ty.clone()),
to: page_ty.clone(),
};
if allow_widening && evolvable(dest_ty, page_ty) {
diff.widenings.push(change);
} else {
diff.incompatible.push(change);
}
}
}
}
for (name, dest_ty) in dest_props {
if !page_props.contains_key(name) {
let (_, nullable) = type_set(dest_ty);
if !nullable {
diff.droppable_required.push(name.clone());
}
}
}
diff.additions.sort_by(|a, b| a.name.cmp(&b.name));
diff.widenings.sort_by(|a, b| a.name.cmp(&b.name));
diff.incompatible.sort_by(|a, b| a.name.cmp(&b.name));
diff.droppable_required.sort();
diff
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)]
#[serde(rename_all = "snake_case")]
pub enum OnDrift {
#[default]
Warn,
Evolve,
Ignore,
Quarantine,
Fail,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema, Default)]
#[serde(rename_all = "snake_case")]
pub enum OnIncompatible {
#[default]
Fail,
Quarantine,
}
fn default_true() -> bool {
true
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct SchemaDriftSpec {
#[serde(default)]
pub on_drift: OnDrift,
#[serde(default = "default_true")]
pub allow_type_widening: bool,
#[serde(default)]
pub on_incompatible: OnIncompatible,
#[serde(default)]
pub relax_nullability_on_missing: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SchemaDriftPolicy {
pub on_drift: OnDrift,
pub allow_widening: bool,
pub on_incompatible: OnIncompatible,
pub relax_nullability_on_missing: bool,
}
impl SchemaDriftPolicy {
pub fn compile(spec: &SchemaDriftSpec) -> Self {
Self {
on_drift: spec.on_drift,
allow_widening: spec.allow_type_widening,
on_incompatible: spec.on_incompatible,
relax_nullability_on_missing: spec.relax_nullability_on_missing,
}
}
pub fn requires_dlq(&self) -> bool {
self.on_drift == OnDrift::Quarantine
|| (self.on_drift == OnDrift::Evolve
&& self.on_incompatible == OnIncompatible::Quarantine)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SqlBaseType {
Integer,
Double,
Boolean,
Text,
Json,
}
pub fn json_schema_base_type(fragment: &Value) -> Option<SqlBaseType> {
let (names, _nullable) = type_set(fragment);
if names.iter().any(|t| t == "object" || t == "array") {
return Some(SqlBaseType::Json);
}
if names.iter().any(|t| t == "string") {
return Some(SqlBaseType::Text);
}
if names.iter().any(|t| t == "number") {
return Some(SqlBaseType::Double);
}
if names.iter().any(|t| t == "integer") {
return Some(SqlBaseType::Integer);
}
if names.iter().any(|t| t == "boolean") {
return Some(SqlBaseType::Boolean);
}
None
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn schema(props: Value) -> Value {
json!({ "type": "object", "properties": props })
}
#[test]
fn sql_type_mapping() {
use super::SqlBaseType::*;
assert_eq!(
json_schema_base_type(&json!({"type":"integer"})),
Some(Integer)
);
assert_eq!(
json_schema_base_type(&json!({"type":"number"})),
Some(Double)
);
assert_eq!(
json_schema_base_type(&json!({"type":"boolean"})),
Some(Boolean)
);
assert_eq!(json_schema_base_type(&json!({"type":"string"})), Some(Text));
assert_eq!(
json_schema_base_type(&json!({"type":["string","null"]})),
Some(Text)
);
assert_eq!(json_schema_base_type(&json!({"type":"object"})), Some(Json));
assert_eq!(json_schema_base_type(&json!({"type":"array"})), Some(Json));
assert_eq!(json_schema_base_type(&json!({"type":"null"})), None);
}
#[test]
fn no_drift_when_shapes_match() {
let dest = schema(json!({ "id": {"type": "integer"}, "name": {"type": "string"} }));
let page = schema(json!({ "id": {"type": "integer"}, "name": {"type": "string"} }));
let d = diff_schema(&dest, &page, true);
assert!(d.is_empty(), "got {d:?}");
}
#[test]
fn detects_addition() {
let dest = schema(json!({ "id": {"type": "integer"} }));
let page = schema(json!({ "id": {"type": "integer"}, "email": {"type": "string"} }));
let d = diff_schema(&dest, &page, true);
assert_eq!(d.additions.len(), 1);
assert_eq!(d.additions[0].name, "email");
assert!(d.additions[0].from.is_none());
assert_eq!(d.additions[0].to, json!({"type": "string"}));
assert!(d.widenings.is_empty() && d.incompatible.is_empty());
}
#[test]
fn integer_to_number_is_widening_when_allowed() {
let dest = schema(json!({ "score": {"type": "integer"} }));
let page = schema(json!({ "score": {"type": "number"} }));
let d = diff_schema(&dest, &page, true);
assert_eq!(d.widenings.len(), 1, "got {d:?}");
assert_eq!(d.widenings[0].name, "score");
assert!(d.incompatible.is_empty());
}
#[test]
fn integer_to_number_is_incompatible_when_widening_disallowed() {
let dest = schema(json!({ "score": {"type": "integer"} }));
let page = schema(json!({ "score": {"type": "number"} }));
let d = diff_schema(&dest, &page, false);
assert_eq!(d.incompatible.len(), 1, "got {d:?}");
assert!(d.widenings.is_empty());
}
#[test]
fn gaining_nullability_is_widening() {
let dest = schema(json!({ "name": {"type": "string"} }));
let page = schema(json!({ "name": {"type": ["string", "null"]} }));
let d = diff_schema(&dest, &page, true);
assert_eq!(d.widenings.len(), 1, "got {d:?}");
}
#[test]
fn string_to_integer_is_incompatible() {
let dest = schema(json!({ "id": {"type": "string"} }));
let page = schema(json!({ "id": {"type": "integer"} }));
let d = diff_schema(&dest, &page, true);
assert_eq!(d.incompatible.len(), 1, "got {d:?}");
assert!(d.widenings.is_empty());
}
#[test]
fn required_destination_column_absent_from_page_is_droppable_required() {
let dest = schema(json!({
"id": {"type": "integer"},
"created_at": {"type": "string"}
}));
let page = schema(json!({ "id": {"type": "integer"} }));
let d = diff_schema(&dest, &page, true);
assert_eq!(
d.droppable_required,
vec!["created_at".to_string()],
"got {d:?}"
);
}
#[test]
fn nullable_destination_column_absent_from_page_is_not_drift() {
let dest = schema(json!({
"id": {"type": "integer"},
"note": {"type": ["string", "null"]}
}));
let page = schema(json!({ "id": {"type": "integer"} }));
let d = diff_schema(&dest, &page, true);
assert!(d.is_empty(), "got {d:?}");
}
#[test]
fn nested_object_treated_as_single_column() {
let dest =
schema(json!({ "meta": {"type": "object", "properties": {"a": {"type": "integer"}}} }));
let page = schema(
json!({ "meta": {"type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}} }),
);
let d = diff_schema(&dest, &page, true);
assert!(
d.is_empty(),
"nested changes must not surface as drift; got {d:?}"
);
}
#[derive(Debug, PartialEq)]
enum Bucket {
None,
Widening,
Incompatible,
}
fn classify_one(dest_ty: Value, page_ty: Value, allow_widening: bool) -> Bucket {
let dest = schema(json!({ "col": dest_ty }));
let page = schema(json!({ "col": page_ty }));
let d = diff_schema(&dest, &page, allow_widening);
assert!(d.additions.is_empty(), "unexpected addition: {d:?}");
assert!(
d.droppable_required.is_empty(),
"unexpected droppable: {d:?}"
);
match (d.widenings.len(), d.incompatible.len()) {
(0, 0) => Bucket::None,
(1, 0) => Bucket::Widening,
(0, 1) => Bucket::Incompatible,
_ => panic!("ambiguous classification: {d:?}"),
}
}
#[test]
fn truth_table_allow_widening() {
use Bucket::*;
let cases: &[(Value, Value, Bucket)] = &[
(json!({"type": "integer"}), json!({"type": "integer"}), None),
(json!({"type": "string"}), json!({"type": "string"}), None),
(
json!({"type": ["string", "null"]}),
json!({"type": ["string", "null"]}),
None,
),
(
json!({"type": ["string", "null"]}),
json!({"type": "string"}),
None,
),
(json!({"type": "number"}), json!({"type": "integer"}), None),
(
json!({"type": "integer"}),
json!({"type": "number"}),
Widening,
),
(
json!({"type": "string"}),
json!({"type": ["string", "null"]}),
Widening,
),
(
json!({"type": "integer"}),
json!({"type": ["number", "null"]}),
Widening,
),
(
json!({"type": ["integer", "null"]}),
json!({"type": "number"}),
Widening,
),
(
json!({"type": "string"}),
json!({"type": "integer"}),
Incompatible,
),
(
json!({"type": "integer"}),
json!({"type": "string"}),
Incompatible,
),
(
json!({"type": "boolean"}),
json!({"type": "number"}),
Incompatible,
),
];
for (dest, page, want) in cases {
let got = classify_one(dest.clone(), page.clone(), true);
assert_eq!(
&got, want,
"allow_widening=true: D={dest} P={page} expected {want:?} got {got:?}"
);
}
}
#[test]
fn truth_table_widening_disallowed() {
use Bucket::*;
let cases: &[(Value, Value, Bucket)] = &[
(
json!({"type": "integer"}),
json!({"type": "number"}),
Incompatible,
),
(
json!({"type": ["string", "null"]}),
json!({"type": "string"}),
None,
),
(
json!({"type": "string"}),
json!({"type": ["string", "null"]}),
Incompatible,
),
];
for (dest, page, want) in cases {
let got = classify_one(dest.clone(), page.clone(), false);
assert_eq!(
&got, want,
"allow_widening=false: D={dest} P={page} expected {want:?} got {got:?}"
);
}
}
#[test]
fn base_widened_detects_base_type_change() {
assert!(base_widened(
&json!({"type": "integer"}),
&json!({"type": "number"})
));
assert!(!base_widened(
&json!({"type": "string"}),
&json!({"type": ["string", "null"]})
));
assert!(!base_widened(
&json!({"type": "integer"}),
&json!({"type": "integer"})
));
assert!(base_widened(
&json!({"type": "integer"}),
&json!({"type": ["number", "null"]})
));
assert!(!base_widened(
&json!({"type": "number"}),
&json!({"type": "integer"})
));
}
#[test]
fn adds_null_detects_nullability_relaxation() {
assert!(adds_null(
&json!({"type": "string"}),
&json!({"type": ["string", "null"]})
));
assert!(!adds_null(
&json!({"type": ["string", "null"]}),
&json!({"type": "string"})
));
assert!(!adds_null(
&json!({"type": "string"}),
&json!({"type": "string"})
));
assert!(adds_null(
&json!({"type": "integer"}),
&json!({"type": ["number", "null"]})
));
}
#[test]
fn spec_defaults() {
let spec: SchemaDriftSpec = serde_json::from_str("{}").unwrap();
assert_eq!(spec.on_drift, OnDrift::Warn);
assert!(spec.allow_type_widening);
assert_eq!(spec.on_incompatible, OnIncompatible::Fail);
}
#[test]
fn on_drift_serializes_snake_case() {
assert_eq!(
serde_json::to_string(&OnDrift::Evolve).unwrap(),
"\"evolve\""
);
assert_eq!(
serde_json::to_string(&OnDrift::Quarantine).unwrap(),
"\"quarantine\""
);
}
#[test]
fn policy_compile_carries_flags() {
let spec: SchemaDriftSpec =
serde_json::from_str(r#"{"on_drift":"evolve","allow_type_widening":false}"#).unwrap();
let policy = SchemaDriftPolicy::compile(&spec);
assert_eq!(policy.on_drift, OnDrift::Evolve);
assert!(!policy.allow_widening);
assert_eq!(policy.on_incompatible, OnIncompatible::Fail);
}
#[test]
fn policy_requires_dlq_only_for_quarantine_paths() {
let q: SchemaDriftSpec = serde_json::from_str(r#"{"on_drift":"quarantine"}"#).unwrap();
assert!(SchemaDriftPolicy::compile(&q).requires_dlq());
let evo_q: SchemaDriftSpec =
serde_json::from_str(r#"{"on_drift":"evolve","on_incompatible":"quarantine"}"#)
.unwrap();
assert!(SchemaDriftPolicy::compile(&evo_q).requires_dlq());
let warn: SchemaDriftSpec = serde_json::from_str(r#"{"on_drift":"warn"}"#).unwrap();
assert!(!SchemaDriftPolicy::compile(&warn).requires_dlq());
}
}