faucet_core/

schema.rs

//! JSON Schema inference from record samples.
//!
//! Given a slice of JSON values (records from a REST API), produces a JSON Schema
//! that is valid for all of them.  The algorithm:
//!
//! * Each field type is inferred independently per record then **merged** across records.
//! * A field absent from some records gets `"null"` added to its type.
//! * `"integer"` widens to `"number"` when the same field is an integer in some records
//!   and a float in others.
//! * Nested objects are recursively inferred and merged.

use serde_json::{Map, Value, json};
use std::collections::HashSet;

/// Infer a JSON Schema `object` descriptor from a slice of record values.
///
/// Non-object top-level values are ignored.  Returns an empty-properties
/// object schema when `records` is empty or contains no objects.
pub fn infer_schema(records: &[Value]) -> Value {
    let objects: Vec<&Map<String, Value>> = records.iter().filter_map(|r| r.as_object()).collect();

    if objects.is_empty() {
        return json!({"type": "object", "properties": {}});
    }

    // Collect all field names across all records.
    let all_keys: HashSet<&String> = objects.iter().flat_map(|o| o.keys()).collect();

    let mut properties = Map::new();

    for key in all_keys {
        let values: Vec<&Value> = objects.iter().filter_map(|o| o.get(key)).collect();
        let records_with_key = values.len();

        let mut field_schema = values
            .into_iter()
            .map(infer_value_schema)
            .reduce(merge_schemas)
            .unwrap_or_else(|| json!({}));

        // Fields absent from some records are implicitly nullable.
        if records_with_key < objects.len() {
            add_null_type(&mut field_schema);
        }

        properties.insert(key.clone(), field_schema);
    }

    json!({
        "type": "object",
        "properties": Value::Object(properties)
    })
}

// ── Internal helpers ──────────────────────────────────────────────────────────

fn infer_value_schema(v: &Value) -> Value {
    match v {
        Value::Null => json!({"type": "null"}),
        Value::Bool(_) => json!({"type": "boolean"}),
        Value::Number(n) => {
            if n.is_i64() || n.is_u64() {
                json!({"type": "integer"})
            } else {
                json!({"type": "number"})
            }
        }
        Value::String(_) => json!({"type": "string"}),
        Value::Array(arr) => {
            let items = if arr.is_empty() {
                json!({})
            } else {
                arr.iter()
                    .map(infer_value_schema)
                    .reduce(merge_schemas)
                    .unwrap_or_else(|| json!({}))
            };
            json!({"type": "array", "items": items})
        }
        Value::Object(map) => {
            let props: Map<String, Value> = map
                .iter()
                .map(|(k, v)| (k.clone(), infer_value_schema(v)))
                .collect();
            json!({"type": "object", "properties": Value::Object(props)})
        }
    }
}

/// Merge two schemas into one that is valid for both.
fn merge_schemas(a: Value, b: Value) -> Value {
    let mut types = collect_types(&a)
        .union(&collect_types(&b))
        .cloned()
        .collect::<Vec<_>>();

    // Numeric widening: integer + number → number.
    if types.contains(&"integer".to_string()) && types.contains(&"number".to_string()) {
        types.retain(|t| t != "integer");
    }
    types.sort();
    types.dedup();

    // Build the merged schema, preserving *both* `properties` (when the union
    // includes `object`) and `items` (when it includes `array`). The previous
    // implementation returned early on `object` and dropped any array `items`,
    // so a field that was an array in some records and an object in others
    // lost its element shape (#78/#35).
    // Two untyped fragments carry no information — return the unknown schema
    // `{}` rather than a malformed `{"type": []}` (#78/#35).
    if types.is_empty() {
        return json!({});
    }

    let has_object = types.iter().any(|t| t == "object");
    let has_array = types.iter().any(|t| t == "array");

    let mut result = Map::new();
    result.insert("type".to_string(), make_type_value(types));

    if has_object {
        let props = merge_properties(extract_properties(&a), extract_properties(&b));
        result.insert("properties".to_string(), Value::Object(props));
    }

    if has_array {
        // Merge the element schemas from whichever side(s) carried `items`.
        // Omit `items` entirely when the element type is genuinely unknown
        // (e.g. only empty arrays were seen) rather than emitting `items: {}`.
        let items = match (a.get("items").cloned(), b.get("items").cloned()) {
            (Some(x), Some(y)) => Some(merge_schemas(x, y)),
            (Some(x), None) | (None, Some(x)) => Some(x),
            (None, None) => None,
        };
        if let Some(items) = items
            && !is_unknown_schema(&items)
        {
            result.insert("items".to_string(), items);
        }
    }

    Value::Object(result)
}

/// A schema carrying no information — `{}` or `null`. Used to decide whether
/// an array's `items` is worth emitting.
fn is_unknown_schema(schema: &Value) -> bool {
    match schema {
        Value::Object(m) => m.is_empty(),
        Value::Null => true,
        _ => false,
    }
}

fn merge_properties(a: Map<String, Value>, b: Map<String, Value>) -> Map<String, Value> {
    let keys_a: HashSet<String> = a.keys().cloned().collect();
    let keys_b: HashSet<String> = b.keys().cloned().collect();
    let mut result = Map::new();

    // Keys in both: merge.
    for key in keys_a.intersection(&keys_b) {
        result.insert(key.clone(), merge_schemas(a[key].clone(), b[key].clone()));
    }
    // Keys only in A: field can be absent → nullable.
    for key in keys_a.difference(&keys_b) {
        let mut s = a[key].clone();
        add_null_type(&mut s);
        result.insert(key.clone(), s);
    }
    // Keys only in B: field can be absent → nullable.
    for key in keys_b.difference(&keys_a) {
        let mut s = b[key].clone();
        add_null_type(&mut s);
        result.insert(key.clone(), s);
    }

    result
}

fn collect_types(schema: &Value) -> HashSet<String> {
    match schema.get("type") {
        Some(Value::String(t)) => std::iter::once(t.clone()).collect(),
        Some(Value::Array(arr)) => arr
            .iter()
            .filter_map(|v| v.as_str().map(String::from))
            .collect(),
        _ => HashSet::new(),
    }
}

fn extract_properties(schema: &Value) -> Map<String, Value> {
    schema
        .get("properties")
        .and_then(|p| p.as_object())
        .cloned()
        .unwrap_or_default()
}

/// Add `"null"` to the type of `schema` if not already present.
fn add_null_type(schema: &mut Value) {
    let mut types = collect_types(schema);
    if types.contains("null") {
        return;
    }
    types.insert("null".to_string());
    let new_type = make_type_value(types.into_iter().collect());
    match schema {
        // Insert (not just overwrite) so a type-less `{}` fragment is still
        // marked nullable instead of silently dropping the null.
        Value::Object(map) => {
            map.insert("type".to_string(), new_type);
        }
        // A non-object fragment (e.g. a bare `null`) becomes a minimal
        // nullable object schema.
        _ => *schema = json!({ "type": new_type }),
    }
}

fn make_type_value(mut types: Vec<String>) -> Value {
    types.sort();
    types.dedup();
    if types.len() == 1 {
        Value::String(types.remove(0))
    } else {
        Value::Array(types.into_iter().map(Value::String).collect())
    }
}

// ── Tests ─────────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn add_null_type_marks_typeless_schema_nullable() {
        // A type-less `{}` fragment (an unknown field) must still be marked
        // nullable when absent from some records — previously the missing
        // `type` key meant the null was silently dropped.
        let mut s = json!({});
        add_null_type(&mut s);
        assert_eq!(s, json!({"type": "null"}));
    }

    #[test]
    fn add_null_type_adds_null_to_existing_type() {
        let mut s = json!({"type": "string"});
        add_null_type(&mut s);
        assert_eq!(s["type"], json!(["null", "string"]));
    }

    #[test]
    fn add_null_type_is_idempotent_when_already_nullable() {
        let mut s = json!({"type": ["null", "string"]});
        add_null_type(&mut s);
        assert_eq!(s["type"], json!(["null", "string"]));
    }

    #[test]
    fn test_infer_schema_basic_types() {
        let records = vec![json!({"id": 1, "name": "Alice", "score": 9.5, "active": true})];
        let schema = infer_schema(&records);
        let props = &schema["properties"];
        assert_eq!(props["id"]["type"], "integer");
        assert_eq!(props["name"]["type"], "string");
        assert_eq!(props["score"]["type"], "number");
        assert_eq!(props["active"]["type"], "boolean");
    }

    #[test]
    fn test_infer_schema_nullable_absent_field() {
        let records = vec![json!({"id": 1, "email": "a@example.com"}), json!({"id": 2})];
        let schema = infer_schema(&records);
        let props = &schema["properties"];
        assert_eq!(props["id"]["type"], "integer");
        // email is absent in second record → nullable
        let email_type = &props["email"]["type"];
        assert!(
            email_type == &json!(["null", "string"]) || email_type == &json!(["string", "null"]),
            "expected nullable string, got {email_type}"
        );
    }

    #[test]
    fn test_infer_schema_explicit_null_value() {
        let records = vec![json!({"tag": "foo"}), json!({"tag": null})];
        let schema = infer_schema(&records);
        let tag_type = &schema["properties"]["tag"]["type"];
        assert!(
            tag_type == &json!(["null", "string"]) || tag_type == &json!(["string", "null"]),
            "expected nullable string, got {tag_type}"
        );
    }

    #[test]
    fn test_infer_schema_integer_widens_to_number() {
        let records = vec![json!({"val": 42}), json!({"val": 3.15})];
        let schema = infer_schema(&records);
        assert_eq!(schema["properties"]["val"]["type"], "number");
    }

    #[test]
    fn test_infer_schema_array_field() {
        let records = vec![json!({"tags": ["rust", "api"]})];
        let schema = infer_schema(&records);
        assert_eq!(schema["properties"]["tags"]["type"], "array");
        assert_eq!(schema["properties"]["tags"]["items"]["type"], "string");
    }

    #[test]
    fn test_infer_schema_nested_object() {
        let records = vec![
            json!({"address": {"city": "NYC", "zip": "10001"}}),
            json!({"address": {"city": "LA"}}),
        ];
        let schema = infer_schema(&records);
        let addr = &schema["properties"]["address"];
        assert_eq!(addr["type"], "object");
        assert_eq!(addr["properties"]["city"]["type"], "string");
        // zip absent from second record → nullable
        let zip_type = &addr["properties"]["zip"]["type"];
        assert!(
            zip_type == &json!(["null", "string"]) || zip_type == &json!(["string", "null"]),
            "expected nullable string, got {zip_type}"
        );
    }

    #[test]
    fn test_infer_schema_empty_records() {
        let schema = infer_schema(&[]);
        assert_eq!(schema["type"], "object");
        assert_eq!(schema["properties"], json!({}));
    }

    #[test]
    fn test_infer_schema_skips_non_objects() {
        // Top-level arrays and primitives are ignored.
        let records = vec![json!("string"), json!(42), json!({"id": 1})];
        let schema = infer_schema(&records);
        assert_eq!(schema["properties"]["id"]["type"], "integer");
    }

    #[test]
    fn test_add_null_type_idempotent() {
        let mut s = json!({"type": ["null", "string"]});
        add_null_type(&mut s);
        // Should not duplicate "null".
        assert_eq!(s["type"], json!(["null", "string"]));
    }

    #[test]
    fn test_merge_schemas_object_merges_properties() {
        let a = json!({"type": "object", "properties": {"x": {"type": "integer"}}});
        let b = json!({"type": "object", "properties": {"y": {"type": "string"}}});
        let merged = merge_schemas(a, b);
        assert_eq!(merged["type"], "object");
        // x is absent from b → nullable in merged
        let x_type = &merged["properties"]["x"]["type"];
        assert!(
            x_type == &json!(["integer", "null"]) || x_type == &json!(["null", "integer"]),
            "got {x_type}"
        );
        // y is absent from a → nullable in merged
        let y_type = &merged["properties"]["y"]["type"];
        assert!(
            y_type == &json!(["null", "string"]) || y_type == &json!(["string", "null"]),
            "got {y_type}"
        );
    }

    #[test]
    fn test_merge_schemas_array_items_merged() {
        let a = json!({"type": "array", "items": {"type": "integer"}});
        let b = json!({"type": "array", "items": {"type": "string"}});
        let merged = merge_schemas(a, b);
        assert_eq!(merged["type"], "array");
        let items_type = &merged["items"]["type"];
        assert!(
            items_type == &json!(["integer", "string"])
                || items_type == &json!(["string", "integer"]),
            "got {items_type}"
        );
    }

    #[test]
    fn test_merge_schemas_array_object_union_preserves_items_and_properties() {
        // Regression for #78/#35: a field that is an array in some records and
        // an object in others must keep *both* the array `items` and the
        // object `properties`, not silently drop the array shape.
        let arr = json!({"type": "array", "items": {"type": "integer"}});
        let obj = json!({"type": "object", "properties": {"k": {"type": "string"}}});
        let merged = merge_schemas(arr, obj);
        let types = &merged["type"];
        assert!(
            types == &json!(["array", "object"]) || types == &json!(["object", "array"]),
            "got {types}"
        );
        assert_eq!(merged["items"]["type"], "integer", "array items dropped");
        // `k` is present only on the object variant → nullable in the union.
        let k_type = &merged["properties"]["k"]["type"];
        assert!(
            k_type == &json!(["null", "string"]) || k_type == &json!(["string", "null"]),
            "got {k_type}"
        );
    }

    #[test]
    fn test_merge_schemas_unknown_array_items_omitted() {
        // Two empty arrays carry no element info → omit `items` rather than
        // emitting a meaningless `items: {}` (#78/#35).
        let a = json!({"type": "array", "items": {}});
        let b = json!({"type": "array", "items": {}});
        let merged = merge_schemas(a, b);
        assert_eq!(merged["type"], "array");
        assert!(merged.get("items").is_none(), "got {merged}");
    }
}