use crate::{
debug, debug_verbose,
schema::core::{make_promoted_scalar_key, SchemaInferenceConfig},
};
use rayon::prelude::*;
use serde_json::{json, Map, Value};
use std::borrow::Cow;
const SCALAR_TYPES: [&str; 4] = ["string", "number", "integer", "boolean"];
fn normalise_nullable(mut schema: &Value) -> &Value {
while let Value::Array(pair) = schema {
match pair.as_slice() {
[a, b] if a == "null" => schema = b,
[a, b] if b == "null" => schema = a,
_ => break,
}
}
schema
}
fn base_type(schema: &Value) -> Option<&str> {
match schema.get("type")? {
Value::String(t) => Some(t),
_ => nullable_type(schema)?.as_str(),
}
}
fn get_scalar_type_name(schema: &Value) -> Option<&str> {
base_type(schema).filter(|t| SCALAR_TYPES.contains(t))
}
fn is_scalar_schema(schema: &Value) -> bool {
get_scalar_type_name(normalise_nullable(schema)).is_some()
}
fn is_array_schema(schema: &Value) -> bool {
base_type(normalise_nullable(schema)) == Some("array")
}
fn is_object_schema(schema: &Value) -> bool {
base_type(schema) == Some("object") && schema.get("properties").is_some()
}
fn is_map_schema(schema: &Value) -> bool {
base_type(schema) == Some("object") && schema.get("additionalProperties").is_some()
}
fn is_empty_record_schema(schema: &Value) -> bool {
if base_type(schema) != Some("object") {
return false;
}
let values = schema.get("additionalProperties");
if values.is_some_and(|v| v.is_object() || *v == Value::Bool(true)) {
return false;
}
match schema.get("properties") {
None => true,
Some(properties) => properties.as_object().is_some_and(|p| p.is_empty()),
}
}
fn nullable_type(schema: &Value) -> Option<&Value> {
match schema.get("type")? {
Value::Array(types) if types.len() == 2 && types.iter().any(|t| t == "null") => {
types.iter().find(|t| *t != "null")
}
_ => None,
}
}
pub(super) struct NonNullView<'a> {
schema: &'a Value,
type_override: Option<&'a Value>,
}
pub(super) fn non_null_view(schema: &Value) -> NonNullView<'_> {
if let Value::Array(arr) = schema {
if arr.len() == 2 && arr.iter().any(|v| v == "null") {
if let Some(inner) = arr.iter().find(|v| *v != "null") {
return NonNullView {
schema: inner,
type_override: None,
};
}
}
}
NonNullView {
schema,
type_override: nullable_type(schema),
}
}
impl NonNullView<'_> {
fn field<'b>(&'b self, key: &str, value: &'b Value) -> &'b Value {
match self.type_override {
Some(t) if key == "type" => t,
_ => value,
}
}
pub(super) fn eq(&self, other: &NonNullView) -> bool {
if self.type_override.is_none() && other.type_override.is_none() {
return self.schema == other.schema;
}
let (Value::Object(a), Value::Object(b)) = (self.schema, other.schema) else {
return false;
};
a.len() == b.len()
&& a.iter().all(|(k, v)| {
b.get(k)
.is_some_and(|w| self.field(k, v) == other.field(k, w))
})
}
pub(super) fn to_value(&self) -> Value {
let mut schema = self.schema.clone();
if let (Some(t), Some(obj)) = (self.type_override, schema.as_object_mut()) {
obj.insert("type".to_string(), t.clone());
}
schema
}
}
fn try_make_nullable_union(a: &Value, b: &Value) -> Option<Value> {
if a.get("type")? != "null" {
return None;
}
let other_type = b.get("type").filter(|t| *t != "null")?;
let mut result = b.clone();
result
.as_object_mut()?
.insert("type".to_string(), json!(["null", other_type]));
Some(result)
}
fn schemas_compatible(existing: &Value, new: &Value) -> Option<Value> {
if existing == new {
return Some(existing.clone());
}
if let Some(result) =
try_make_nullable_union(existing, new).or_else(|| try_make_nullable_union(new, existing))
{
return Some(result);
}
let without_null = |schema| NonNullView {
schema,
type_override: nullable_type(schema),
};
let (existing_view, new_view) = (without_null(existing), without_null(new));
if existing_view
.type_override
.or(new_view.type_override)
.is_none()
|| !existing_view.eq(&new_view)
{
return None;
}
let mut nullable = existing_view.to_value();
if let Some(inner_type) = nullable.get("type").cloned() {
nullable["type"] = json!(["null", inner_type]);
}
Some(nullable)
}
fn promotion_field_name(path: &str) -> &str {
path.rsplit('.')
.find(|s| !s.is_empty() && *s != "items" && *s != "additionalProperties")
.unwrap_or("value")
}
fn try_scalar_promotion(
object_schema: &Value,
scalar_schema: &Value,
field_name: &str,
scalar_side: &str,
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
let Some(scalar_type) = get_scalar_type_name(scalar_schema) else {
debug!(config, "Cannot determine scalar type for promotion");
return None;
};
let wrapped_key = make_promoted_scalar_key(field_name, scalar_type);
debug!(
config,
"Promoting scalar on {} side: wrapping {} into object under key `{}`",
scalar_side,
scalar_type,
wrapped_key
);
let promoted = json!({
"type": "object",
"properties": { wrapped_key: scalar_schema }
});
let mut result = check_unifiable_schemas(
&[object_schema, &promoted],
&format!("{path}.{field_name}"),
config,
)?;
if let Some(obj) = result.as_object_mut() {
obj.shift_remove("required");
}
Some(result)
}
fn extract_field_from_nullable_schema<'a>(
schema: &'a Value,
field_name: &str,
) -> Option<&'a Value> {
normalise_nullable(schema).get(field_name)
}
fn unify_array_schemas(
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
debug!(
config,
"{}: Attempting to unify {} array schemas",
path,
schemas.len()
);
if schemas.is_empty() {
return None;
}
let mut items_schemas = Vec::<&Value>::new();
for (i, &schema) in schemas.iter().enumerate() {
let items = extract_field_from_nullable_schema(schema, "items")
.filter(|items| items.as_object().is_none_or(|o| !o.is_empty()));
if let Some(items) = items {
debug_verbose!(
config,
"{}: Array schema[{}] items: {}",
path,
i,
serde_json::to_string(items).unwrap_or_default()
);
items_schemas.push(items);
} else {
debug!(config, "{}: Array schema[{}] has no items (empty)", path, i);
}
}
match items_schemas.as_slice() {
[] => return Some(json!({ "type": "array" })),
[only] => return Some(json!({ "type": "array", "items": only })),
_ => {}
}
if let Some(unified_items) =
check_unifiable_item_schemas(&items_schemas, &format!("{}.items", path), config)
{
debug!(config, "{}: Successfully unified array items", path);
Some(json!({
"type": "array",
"items": unified_items
}))
} else {
debug!(config, "{}: Failed to unify array items", path);
None
}
}
fn unify_scalar_schemas(
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
if schemas.is_empty() {
debug!(config, "Empty schema at {}", path);
return None;
}
let base_types: std::collections::BTreeSet<&str> = schemas
.iter()
.filter_map(|&schema| base_type(schema))
.collect();
if let (1, Some(base_type)) = (base_types.len(), base_types.first()) {
debug!(
config,
"{}: Unified scalar schemas to nullable {}", path, base_type
);
return Some(json!({"type": ["null", base_type]}));
}
debug!(
config,
"{}: Cannot unify incompatible scalar types: {:?}", path, base_types
);
None
}
fn unify_map_schemas(
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
debug!(
config,
"{}: Attempting to unify {} map schemas",
path,
schemas.len()
);
if schemas.is_empty() {
return None;
}
let Some(additional_props_schemas) = schemas
.iter()
.map(|&schema| extract_field_from_nullable_schema(schema, "additionalProperties"))
.collect::<Option<Vec<&Value>>>()
else {
debug!(
config,
"{}: A map schema is missing additionalProperties", path
);
return None;
};
if let Some(unified_additional_props) = check_unifiable_schemas(
&additional_props_schemas,
&format!("{}.additionalProperties", path),
config,
) {
debug!(
config,
"{}: Successfully unified map additionalProperties", path
);
Some(json!({
"type": "object",
"additionalProperties": unified_additional_props
}))
} else {
debug!(config, "{}: Failed to unify map additionalProperties", path);
None
}
}
fn unify_field_schemas_sequential(
field_name: &str,
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
let (&first, rest) = schemas.split_first()?;
if rest.iter().all(|&s| s == first) {
return Some(first.clone());
}
let field_path = format!("{path}.{field_name}");
let mut unified = first.clone();
for &new in rest {
unified = unify_field_pair(unified, new, field_name, &field_path, path, config)?;
}
Some(unified)
}
fn unify_field_pair(
unified: Value,
new: &Value,
field_name: &str,
field_path: &str,
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
if let Some(compatible) = schemas_compatible(&unified, new) {
return Some(compatible);
}
let record_like = |s: &Value| is_object_schema(s) || is_empty_record_schema(s);
if (is_array_schema(&unified) && is_array_schema(new))
|| (record_like(&unified) && record_like(new))
{
return check_unifiable_schemas(&[&unified, new], field_path, config);
}
if !config.wrap_scalars {
return None;
}
if is_object_schema(&unified) && is_scalar_schema(new) {
try_scalar_promotion(&unified, new, field_name, "new", path, config)
} else if is_object_schema(new) && is_scalar_schema(&unified) {
try_scalar_promotion(new, &unified, field_name, "existing", path, config)
} else if is_scalar_schema(&unified) && is_scalar_schema(new) {
try_mixed_scalar_promotion(&unified, new, field_name, path, config)
} else {
None
}
}
fn unify_field_schemas_parallel(
field_name: &str,
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
if schemas.len() < 10 {
return unify_field_schemas_sequential(field_name, schemas, path, config);
}
let (left, right) = schemas.split_at(schemas.len() / 2);
let (left, right) = rayon::join(
|| unify_field_schemas_parallel(field_name, left, path, config),
|| unify_field_schemas_parallel(field_name, right, path, config),
);
check_unifiable_schemas(&[&left?, &right?], &format!("{path}.{field_name}"), config)
}
fn unify_field_schemas(
field_name: &str,
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
let needs_scalar_promo = config.wrap_scalars
&& schemas.iter().any(|&s| is_scalar_schema(s))
&& schemas.iter().any(|&s| is_object_schema(s));
if needs_scalar_promo || schemas.len() < 50 {
unify_field_schemas_sequential(field_name, schemas, path, config)
} else {
unify_field_schemas_parallel(field_name, schemas, path, config)
}
}
fn unify_record_schemas(
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
debug!(
config,
"{}: Attempting to unify {} record schemas",
path,
schemas.len()
);
let mut fields: ordermap::OrderMap<&str, Vec<&Value>> = ordermap::OrderMap::new();
let mut counts: std::collections::HashMap<&str, usize> = std::collections::HashMap::new();
let mut unified_anyofs: Vec<(&str, Value)> = Vec::new();
for (i, &schema) in schemas.iter().enumerate() {
let properties = match extract_field_from_nullable_schema(schema, "properties") {
Some(Value::Object(properties)) => Some(properties),
_ if is_empty_record_schema(schema) => None,
_ => {
debug!(config, "Schema[{i}] has no properties object");
return None;
}
};
for (name, field_schema) in properties.into_iter().flatten() {
*counts.entry(name.as_str()).or_default() += 1;
let field_schema = normalise_nullable(field_schema);
if let Some(Value::Array(any_of)) = field_schema.get("anyOf") {
let any_of: Vec<&Value> = any_of.iter().collect();
if let Some(unified) = unify_anyof_schemas(&any_of, name, config) {
unified_anyofs.push((name.as_str(), unified));
continue;
}
}
fields.entry(name.as_str()).or_default().push(field_schema);
}
}
for (name, unified) in &unified_anyofs {
fields.entry(name).or_default().push(unified);
}
let merge_start = std::time::Instant::now();
let unify = |name: &'_ str, field_schemas: &[&Value]| {
unify_field_schemas(name, field_schemas, path, config)
};
let unified: Option<Vec<Value>> = if fields.len() >= 10 {
fields.par_iter().map(|(name, s)| unify(name, s)).collect()
} else {
fields.iter().map(|(name, s)| unify(name, s)).collect()
};
let Some(unified) = unified else {
debug!(config, "Failed to unify field schemas");
return None;
};
if config.profile && schemas.len() > 50 {
anstream::eprintln!(" Merge loop took {:?}", merge_start.elapsed());
}
let (in_all, in_some): (Vec<_>, Vec<_>) = fields
.keys()
.zip(unified)
.map(|(&name, field_schema)| (name, counts[name], field_schema))
.partition(|(_, count, _)| *count == schemas.len());
let required: Vec<&str> = in_all.iter().map(|(name, _, _)| *name).collect();
let mut properties = Map::new();
for (name, _, field_schema) in in_all {
debug_verbose!(
config,
"Field `{name}` present in all schemas → keeping non-nullable"
);
properties.insert(name.to_string(), field_schema);
}
for (name, count, field_schema) in in_some {
debug_verbose!(
config,
"Field `{name}` missing in {}/{} schemas → making nullable",
schemas.len() - count,
schemas.len()
);
properties.insert(name.to_string(), make_nullable(field_schema));
}
debug!(config, "{}: Record schemas unified successfully", path);
let mut result = json!({
"type": "object",
"properties": properties
});
if !required.is_empty() {
result["required"] = json!(required);
}
Some(result)
}
fn make_nullable(mut schema: Value) -> Value {
match schema.get("type") {
Some(Value::String(t)) if t != "null" => {
let t = t.clone();
schema["type"] = json!(["null", t]);
schema
}
Some(Value::String(_) | Value::Array(_)) => schema,
_ => json!({ "anyOf": [{ "type": "null" }, schema] }),
}
}
fn try_mixed_scalar_promotion(
existing: &Value,
new: &Value,
field_name: &str,
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
let existing_type = get_scalar_type_name(existing)?;
let new_type = get_scalar_type_name(new)?;
if existing_type == new_type {
return None;
}
debug!(
config,
"{}: Promoting mixed scalars {} and {} for field '{}'",
path,
existing_type,
new_type,
field_name
);
Some(json!({
"type": "object",
"properties": {
make_promoted_scalar_key(field_name, existing_type): existing,
make_promoted_scalar_key(field_name, new_type): new,
}
}))
}
pub(crate) fn unify_anyof_schemas(
schemas: &[&Value],
field_name: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
if !config.wrap_scalars {
return None;
}
let has_scalars = schemas.iter().any(|&s| is_scalar_schema(s));
let has_objects = schemas.iter().any(|&s| is_object_schema(s));
if !has_scalars || !has_objects {
return None; }
debug!(
config,
"anyOf unification: promoting scalars for field '{}'", field_name
);
let promoted = schemas
.iter()
.map(|&schema| {
if !is_scalar_schema(schema) {
return Some(Cow::Borrowed(schema));
}
let wrapped_key = make_promoted_scalar_key(field_name, get_scalar_type_name(schema)?);
Some(Cow::Owned(json!({
"type": "object",
"properties": { wrapped_key: schema }
})))
})
.collect::<Option<Vec<Cow<Value>>>>()?;
let promoted: Vec<&Value> = promoted.iter().map(Cow::as_ref).collect();
check_unifiable_schemas(&promoted, field_name, config)
}
pub(crate) fn check_unifiable_schemas(
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
debug_verbose!(
config,
"=== check_unifiable_schemas called with path='{}' and {} schemas:",
path,
schemas.len()
);
for (i, &schema) in schemas.iter().enumerate() {
debug_verbose!(
config,
" Schema[{}]: {}",
i,
serde_json::to_string(schema).unwrap_or_default()
);
}
if schemas.is_empty() {
debug!(config, "{path}: failed (empty schema list)");
return None;
}
if schemas.iter().all(|&s| is_array_schema(s)) {
debug!(
config,
"{}: All schemas are arrays, attempting array unification", path
);
return unify_array_schemas(schemas, path, config);
}
let all_maps_or_empty = schemas
.iter()
.all(|&s| is_map_schema(s) || is_empty_record_schema(s));
if all_maps_or_empty {
let map_schemas: Vec<&Value> = schemas
.iter()
.filter(|&&s| is_map_schema(s))
.copied()
.collect();
if map_schemas.is_empty() {
debug!(
config,
"{}: All schemas are empty records, treating as empty map", path
);
return Some(json!({
"type": "object",
"additionalProperties": {"type": "string"}
}));
}
debug!(
config,
"{}: Unifying {} maps ({} empty records left out)",
path,
map_schemas.len(),
schemas.len() - map_schemas.len()
);
return unify_map_schemas(&map_schemas, path, config);
}
if schemas
.iter()
.all(|&s| is_object_schema(s) || is_empty_record_schema(s))
{
debug!(
config,
"{}: All schemas are records, attempting record unification", path
);
return unify_record_schemas(schemas, path, config);
}
if schemas.iter().all(|&s| is_scalar_schema(s)) {
debug!(
config,
"{}: All schemas are scalars, attempting scalar unification", path
);
return unify_scalar_schemas(schemas, path, config);
}
debug!(
config,
"{}: Mixed schema types not supported for unification", path
);
for (i, &schema) in schemas.iter().enumerate() {
let schema_type = if is_array_schema(schema) {
"array"
} else if is_map_schema(schema) {
"map"
} else if is_object_schema(schema) {
"record"
} else if is_scalar_schema(schema) {
"scalar"
} else {
"unknown"
};
debug!(
config,
" Schema[{}] type: {} - {}",
i,
schema_type,
serde_json::to_string(schema).unwrap_or_default()
);
}
None
}
pub(crate) fn check_unifiable_item_schemas(
schemas: &[&Value],
path: &str,
config: &SchemaInferenceConfig,
) -> Option<Value> {
check_unifiable_schemas(schemas, path, config).or_else(|| {
let records_and_scalars = schemas
.iter()
.all(|&s| is_scalar_schema(s) || is_object_schema(s) || is_empty_record_schema(s));
if !(config.wrap_scalars && records_and_scalars) {
return None;
}
debug!(config, "{}: Promoting scalar items among records", path);
unify_anyof_schemas(schemas, promotion_field_name(path), config)
})
}
#[cfg(test)]
mod tests {
include!("../../tests/unification.rs");
}