sif_parser/

infer.rs

// SIF Core v1 — Type inference and format conversion.
//
// Converts JSON, JSONL, and CSV into SIF Documents.
//
// Reference: SIF-SPEC.md §25.3 (For Converters), §25.4 (Converter Lessons).

use crate::types::*;
use std::collections::{HashMap, HashSet};

// ── Type Inference (§25.4) ──────────────────────────────────────────

/// Infer the SIF type of a raw string value.
pub fn infer_type(value: &str) -> Type {
    if value == "_" || value.is_empty() || value == "null" || value == "None" {
        return Type::Null;
    }

    if value == "T" || value == "F" || value == "true" || value == "false" {
        return Type::Bool;
    }

    if let Ok(_) = value.parse::<i64>() {
        return if value.starts_with('-') {
            Type::Int
        } else {
            Type::Uint
        };
    }

    if let Ok(_) = value.parse::<f64>() {
        return Type::Float;
    }

    if is_datetime(value) {
        return Type::DateTime;
    }

    if is_date(value) {
        return Type::Date;
    }

    if is_duration(value) {
        return Type::Duration;
    }

    if value.starts_with('[') && value.ends_with(']') {
        return Type::Array(Box::new(Type::Str));
    }

    if value.starts_with('{') && value.ends_with('}') {
        return Type::Map;
    }

    Type::Str
}

fn is_date(s: &str) -> bool {
    if s.len() != 10 || !s.is_ascii() {
        return false;
    }
    let b = s.as_bytes();
    b[4] == b'-'
        && b[7] == b'-'
        && b[0..4].iter().all(|c| c.is_ascii_digit())
        && b[5..7].iter().all(|c| c.is_ascii_digit())
        && b[8..10].iter().all(|c| c.is_ascii_digit())
}

fn is_datetime(s: &str) -> bool {
    if !s.is_ascii() || s.len() < 11 {
        return false;
    }
    if let Some(t_pos) = s.find('T') {
        if t_pos == 10 {
            return is_date(&s[..10]);
        }
    }
    false
}

fn is_duration(s: &str) -> bool {
    s.starts_with('P')
        && s.len() >= 2
        && s[1..]
            .chars()
            .all(|c| c.is_ascii_digit() || "YMWDTHS.".contains(c))
}

/// Merge two types into the most general compatible type.
///
/// Rules from §25.4: int+uint=int, int/uint+float=float,
/// T+null=T?, date+datetime=datetime.
pub fn merge_types(a: &Type, b: &Type) -> Type {
    if a == b {
        return a.clone();
    }

    match (a, b) {
        (Type::Null, Type::Nullable(inner)) | (Type::Nullable(inner), Type::Null) => {
            Type::Nullable(inner.clone())
        }
        (Type::Null, other) | (other, Type::Null) => Type::Nullable(Box::new(other.clone())),

        (Type::Nullable(a_inner), Type::Nullable(b_inner)) => {
            Type::Nullable(Box::new(merge_types(a_inner, b_inner)))
        }
        (Type::Nullable(inner), other) | (other, Type::Nullable(inner)) => {
            Type::Nullable(Box::new(merge_types(inner, other)))
        }

        (Type::Int, Type::Uint) | (Type::Uint, Type::Int) => Type::Int,

        (Type::Int | Type::Uint, Type::Float) | (Type::Float, Type::Int | Type::Uint) => {
            Type::Float
        }

        (Type::Date, Type::DateTime) | (Type::DateTime, Type::Date) => Type::DateTime,

        _ => Type::Str,
    }
}

/// Infer a SIF schema from a set of string-keyed rows.
pub fn infer_schema(rows: &[Vec<(&str, &str)>]) -> Schema {
    if rows.is_empty() {
        return Schema {
            fields: Vec::new(),
        };
    }

    let mut keys: Vec<String> = Vec::new();
    let mut seen = HashSet::new();
    for row in rows {
        for (k, _) in row {
            if seen.insert(*k) {
                keys.push(k.to_string());
            }
        }
    }

    let fields: Vec<FieldDef> = keys
        .iter()
        .map(|key| {
            let mut ty: Option<Type> = None;
            let mut all_present = true;

            for row in rows {
                if let Some((_, v)) = row.iter().find(|(k, _)| *k == key.as_str()) {
                    let inferred = infer_type(v);
                    ty = Some(match ty {
                        Some(existing) => merge_types(&existing, &inferred),
                        None => inferred,
                    });
                } else {
                    all_present = false;
                }
            }

            let mut final_ty = ty.unwrap_or(Type::Any);

            if !all_present && !matches!(&final_ty, Type::Nullable(_)) {
                final_ty = Type::Nullable(Box::new(final_ty));
            }

            let semantic = infer_semantic(key);

            FieldDef {
                name: key.clone(),
                field_type: final_ty,
                semantic,
                deprecated: false,
                modifiers: Vec::new(),
            }
        })
        .collect();

    Schema { fields }
}

/// Infer schema from owned string pairs.
pub fn infer_schema_owned(rows: &[Vec<(String, String)>]) -> Schema {
    let borrowed: Vec<Vec<(&str, &str)>> = rows
        .iter()
        .map(|row| {
            row.iter()
                .map(|(k, v)| (k.as_str(), v.as_str()))
                .collect()
        })
        .collect();
    infer_schema(&borrowed)
}

/// Infer semantic annotation from a field name (§25.4).
pub fn infer_semantic(name: &str) -> Option<String> {
    let lower = name.to_lowercase();
    let sem = match lower.as_str() {
        "id" | "uid" | "uuid" | "guid" => "id",
        "url" | "href" | "link" | "uri" => "url",
        "path" | "file" | "filepath" | "file_path" | "filename" => "path",
        "error" | "err" | "error_message" | "err_msg" => "error",
        "warning" | "warn" => "warning",
        "command" | "cmd" => "command",
        "description" | "desc" | "summary" | "body" => "description",
        "name" | "title" | "label" | "display_name" => "label",
        "line" | "line_number" | "lineno" | "line_no" => "line_number",
        "code" | "source_code" | "snippet" => "snippet",
        _ => {
            if lower.ends_with("_id") || lower.ends_with("_uid") {
                return Some("ref".to_string());
            }
            return None;
        }
    };
    Some(sem.to_string())
}

/// Sanitize a field name into a valid SIF field name (§25.4).
pub fn sanitize_field_name(name: &str) -> String {
    let s = name.trim();
    let s = if s.starts_with('"') && s.ends_with('"') && s.len() >= 2 {
        &s[1..s.len() - 1]
    } else {
        s
    };
    let mut out = String::with_capacity(s.len());
    for c in s.chars() {
        if c.is_ascii_alphanumeric() || c == '_' {
            out.push(c.to_ascii_lowercase());
        } else if c == ' ' || c == '-' || c == '/' || c == '.' || c == '$' {
            if !out.ends_with('_') {
                out.push('_');
            }
        }
    }
    while out.ends_with('_') {
        out.pop();
    }
    if out.is_empty() {
        "field".to_string()
    } else {
        out
    }
}

// ── JSON → SIF ──────────────────────────────────────────────────────

/// Convert JSON (array or object) into a SIF Document (§25.3).
///
/// Handles:
/// - `[{...}, {...}]` — array of objects → single section
/// - `[1, "two"]` — array of scalars → single section
/// - `{"items":[{...}]}` — wrapper with array → extracts largest array
/// - `{"paths": {"k": {...}, ...}}` — object-of-objects → each value becomes a record
/// - `{...}` — single object → single record
///
/// For complex objects with multiple large children (like OpenAPI specs),
/// each top-level key that holds significant data becomes its own section
/// per §25.4 guidance on splitting sparse datasets.
pub fn from_json(json: &str) -> Result<Document, String> {
    let json = json.trim();

    if json.starts_with('[') {
        let inner = json[1..].trim_start();
        if inner.starts_with('{') || inner.starts_with(']') {
            from_json_array(json)
        } else {
            from_json_scalar_array(json)
        }
    } else if json.starts_with('{') {
        from_json_object(json)
    } else {
        Err("expected JSON array or object".into())
    }
}

/// Smart conversion of a JSON object to SIF.
///
/// Strategy:
/// 1. Parse the top-level object.
/// 2. Classify each value: scalar, array-of-objects, object-of-objects, or other.
/// 3. If there's a dominant array-of-objects, extract it as the main section.
/// 4. If there are multiple large children, create one section per child.
/// 5. Object-of-objects (like OpenAPI `paths`) get their keys as a column.
fn from_json_object(json: &str) -> Result<Document, String> {
    let obj = parse_json_object(json)?;

    // Classify each top-level field
    struct FieldInfo {
        key: String,
        value: String,
        kind: FieldKind,
        size: usize,
    }

    #[derive(Debug)]
    enum FieldKind {
        Scalar,
        ArrayOfObjects,
        ArrayOfScalars,
        ObjectOfObjects,
        Object,
        EmptyArray,
    }

    let mut fields: Vec<FieldInfo> = Vec::new();
    for (key, value) in &obj {
        let v = value.trim();
        let kind = if v.starts_with('[') && v.ends_with(']') {
            let inner = v[1..].trim_start();
            if inner.starts_with(']') || inner.is_empty() {
                FieldKind::EmptyArray
            } else if inner.starts_with('{') {
                FieldKind::ArrayOfObjects
            } else {
                FieldKind::ArrayOfScalars
            }
        } else if v.starts_with('{') && v.ends_with('}') {
            // Check if it's an object-of-objects (each value is also an object)
            if let Ok(inner_obj) = parse_json_object(v) {
                let obj_children = inner_obj.iter()
                    .filter(|(_, v)| v.trim().starts_with('{'))
                    .count();
                if obj_children > 3 && obj_children as f64 / inner_obj.len().max(1) as f64 > 0.5 {
                    FieldKind::ObjectOfObjects
                } else {
                    FieldKind::Object
                }
            } else {
                FieldKind::Object
            }
        } else {
            FieldKind::Scalar
        };
        fields.push(FieldInfo {
            key: key.clone(),
            value: value.clone(),
            kind,
            size: value.len(),
        });
    }

    // Count structured children (arrays of objects, objects of objects)
    let structured_children: Vec<&FieldInfo> = fields.iter()
        .filter(|f| matches!(f.kind,
            FieldKind::ArrayOfObjects | FieldKind::ObjectOfObjects))
        .collect();

    // If there's exactly one array-of-objects and no object-of-objects, use simple path
    let has_obj_of_obj = structured_children.iter()
        .any(|f| matches!(f.kind, FieldKind::ObjectOfObjects));
    let array_children: Vec<&&FieldInfo> = structured_children.iter()
        .filter(|f| matches!(f.kind, FieldKind::ArrayOfObjects))
        .collect();

    if !has_obj_of_obj && array_children.len() == 1 {
        return from_json_array(&array_children[0].value);
    }

    // Multi-section approach: each large structured child gets its own section.
    // Small scalars go into a metadata section.
    let mut sections = Vec::new();

    // Metadata section: collect all scalar fields
    let scalars: Vec<(&str, &str)> = fields.iter()
        .filter(|f| matches!(f.kind, FieldKind::Scalar))
        .map(|f| (f.key.as_str(), f.value.as_str()))
        .collect();

    if !scalars.is_empty() {
        let schema = Schema {
            fields: vec![
                FieldDef {
                    name: "key".to_string(),
                    field_type: Type::Str,
                    semantic: Some("id".to_string()),
                    deprecated: false,
                    modifiers: Vec::new(),
                },
                FieldDef {
                    name: "value".to_string(),
                    field_type: Type::Str,
                    semantic: None,
                    deprecated: false,
                    modifiers: Vec::new(),
                },
            ],
        };
        let records: Vec<Record> = scalars.iter().map(|(k, v)| {
            Record {
                values: vec![
                    Value::Str(k.to_string()),
                    Value::Str(v.to_string()),
                ],
                cdc_op: CdcOp::Insert,
            }
        }).collect();
        sections.push(Section {
            id: Some("metadata".to_string()),
            directives: vec![Directive::Context("Document metadata".to_string())],
            schema: Some(schema),
            records,
            blocks: Vec::new(),
            templates: Vec::new(),
        });
    }

    // Process each structured child
    for field in &fields {
        match &field.kind {
            FieldKind::ArrayOfObjects => {
                if let Ok(objects) = parse_json_array(&field.value) {
                    if objects.is_empty() { continue; }
                    let flat: Vec<Vec<(String, String)>> = objects.iter()
                        .map(|o| flatten_object(o, ""))
                        .collect();
                    let rows: Vec<Vec<(&str, &str)>> = flat.iter()
                        .map(|r| r.iter().map(|(k,v)| (k.as_str(), v.as_str())).collect())
                        .collect();
                    let schema = infer_schema(&rows);
                    let records = build_records(&flat, &schema);
                    let section_id = sanitize_field_name(&field.key);
                    sections.push(Section {
                        id: Some(section_id.clone()),
                        directives: vec![Directive::Context(
                            format!("{} ({} records)", field.key, records.len())
                        )],
                        schema: Some(schema),
                        records,
                        blocks: Vec::new(),
                        templates: Vec::new(),
                    });
                }
            }
            FieldKind::ObjectOfObjects => {
                // Each key in the object becomes a record, with the key as the first column.
                if let Ok(inner_obj) = parse_json_object(field.value.trim()) {
                    let mut all_flat: Vec<Vec<(String, String)>> = Vec::new();
                    for (k, v) in &inner_obj {
                        let trimmed = v.trim();
                        if trimmed.starts_with('{') {
                            if let Ok(child) = parse_json_object(trimmed) {
                                let mut flat = flatten_object(&child, "");
                                flat.insert(0, ("_key".to_string(), k.clone()));
                                all_flat.push(flat);
                            }
                        } else {
                            // Non-object value in an object-of-objects
                            all_flat.push(vec![
                                ("_key".to_string(), k.clone()),
                                ("value".to_string(), json_value_to_sif_string(trimmed)),
                            ]);
                        }
                    }
                    if all_flat.is_empty() { continue; }
                    let rows: Vec<Vec<(&str, &str)>> = all_flat.iter()
                        .map(|r| r.iter().map(|(k,v)| (k.as_str(), v.as_str())).collect())
                        .collect();
                    let schema = infer_schema(&rows);
                    let records = build_records(&all_flat, &schema);
                    let section_id = sanitize_field_name(&field.key);
                    sections.push(Section {
                        id: Some(section_id.clone()),
                        directives: vec![Directive::Context(
                            format!("{} ({} entries)", field.key, records.len())
                        )],
                        schema: Some(schema),
                        records,
                        blocks: Vec::new(),
                        templates: Vec::new(),
                    });
                }
            }
            FieldKind::Object if field.size > 1024 => {
                // Large nested object — flatten into key/value pairs
                if let Ok(inner) = parse_json_object(field.value.trim()) {
                    let flat = flatten_object(&inner, "");
                    let schema = Schema {
                        fields: vec![
                            FieldDef {
                                name: "key".to_string(),
                                field_type: Type::Str,
                                semantic: Some("id".to_string()),
                                deprecated: false,
                                modifiers: Vec::new(),
                            },
                            FieldDef {
                                name: "value".to_string(),
                                field_type: Type::Str,
                                semantic: None,
                                deprecated: false,
                                modifiers: Vec::new(),
                            },
                        ],
                    };
                    let records: Vec<Record> = flat.iter().map(|(k, v)| {
                        Record {
                            values: vec![
                                Value::Str(k.clone()),
                                Value::Str(v.clone()),
                            ],
                            cdc_op: CdcOp::Insert,
                        }
                    }).collect();
                    let section_id = sanitize_field_name(&field.key);
                    sections.push(Section {
                        id: Some(section_id),
                        directives: vec![Directive::Context(field.key.clone())],
                        schema: Some(schema),
                        records,
                        blocks: Vec::new(),
                        templates: Vec::new(),
                    });
                }
            }
            _ => {} // small objects, empty arrays, scalars already handled
        }
    }

    // If we still have no sections, fall back to single-record
    if sections.is_empty() {
        let wrapped = format!("[{}]", json);
        return from_json_array(&wrapped);
    }

    Ok(Document {
        header: Header {
            version: 1,
            attributes: HashMap::new(),
        },
        sections,
    })
}

/// Convert JSONL (one JSON object per line) into a SIF Document.
pub fn from_jsonl(input: &str, max_records: Option<usize>) -> Result<Document, String> {
    let limit = max_records.unwrap_or(usize::MAX);
    let mut all_flat: Vec<Vec<(String, String)>> = Vec::new();

    let mut buf = String::new();
    let mut depth = 0i32;
    let mut in_string = false;
    let mut escape_next = false;

    for line in input.lines() {
        let trimmed = line.trim();
        if trimmed.is_empty() && depth == 0 {
            continue;
        }
        if all_flat.len() >= limit {
            break;
        }

        if !buf.is_empty() {
            buf.push(' ');
        }
        buf.push_str(line);

        for ch in line.chars() {
            if escape_next {
                escape_next = false;
                continue;
            }
            if ch == '\\' && in_string {
                escape_next = true;
                continue;
            }
            if ch == '"' {
                in_string = !in_string;
                continue;
            }
            if !in_string {
                if ch == '{' {
                    depth += 1;
                }
                if ch == '}' {
                    depth -= 1;
                }
            }
        }

        if depth == 0 && !buf.trim().is_empty() {
            let obj = parse_json_object(buf.trim())?;
            let flat = flatten_object(&obj, "");
            all_flat.push(flat);
            buf.clear();
            in_string = false;
        }
    }

    if all_flat.is_empty() {
        return Ok(empty_doc());
    }

    let sample_size = all_flat.len().min(1000);
    let sample: Vec<Vec<(&str, &str)>> = all_flat[..sample_size]
        .iter()
        .map(|row| {
            row.iter()
                .map(|(k, v)| (k.as_str(), v.as_str()))
                .collect()
        })
        .collect();
    let schema = infer_schema(&sample);

    let records = build_records(&all_flat, &schema);

    Ok(Document {
        header: Header {
            version: 1,
            attributes: HashMap::new(),
        },
        sections: vec![Section {
            id: None,
            directives: Vec::new(),
            schema: Some(schema),
            records,
            blocks: Vec::new(),
            templates: Vec::new(),
        }],
    })
}

/// Convert CSV text into a SIF Document (§25.3).
pub fn from_csv(csv: &str) -> Result<Document, String> {
    let mut lines = csv.lines();

    let header_line = lines.next().ok_or("empty CSV")?;
    let raw_headers = split_csv_line(header_line);
    let headers: Vec<String> = raw_headers
        .iter()
        .map(|h| sanitize_field_name(h))
        .collect();

    let mut raw_rows: Vec<Vec<String>> = Vec::new();
    for line in lines {
        if line.trim().is_empty() {
            continue;
        }
        let values: Vec<String> = split_csv_line(line)
            .into_iter()
            .map(|s| {
                let trimmed = s.trim().to_string();
                if trimmed.is_empty() {
                    "_".to_string()
                } else {
                    trimmed
                }
            })
            .collect();
        raw_rows.push(values);
    }

    let rows: Vec<Vec<(&str, &str)>> = raw_rows
        .iter()
        .map(|row| {
            headers
                .iter()
                .zip(row.iter())
                .map(|(h, v)| (h.as_str(), v.as_str()))
                .collect()
        })
        .collect();

    let schema = infer_schema(&rows);
    let records = build_records_csv(&raw_rows, &schema);

    Ok(Document {
        header: Header {
            version: 1,
            attributes: HashMap::new(),
        },
        sections: vec![Section {
            id: None,
            directives: Vec::new(),
            schema: Some(schema),
            records,
            blocks: Vec::new(),
            templates: Vec::new(),
        }],
    })
}

// ── SIF → JSON ──────────────────────────────────────────────────────

/// Convert a SIF section to JSON Lines (one JSON object per record).
pub fn to_jsonl(schema: &Schema, records: &[Record]) -> String {
    let mut out = String::new();
    for record in records {
        out.push('{');
        for (i, field) in schema.fields.iter().enumerate() {
            if i > 0 {
                out.push(',');
            }
            out.push('"');
            out.push_str(&field.name);
            out.push_str("\":");
            value_to_json(&record.values[i], &mut out);
        }
        out.push_str("}\n");
    }
    out
}

/// Convert a SIF section to a JSON array.
pub fn to_json_array(schema: &Schema, records: &[Record]) -> String {
    let mut out = String::from("[");
    for (ri, record) in records.iter().enumerate() {
        if ri > 0 {
            out.push(',');
        }
        out.push('{');
        for (i, field) in schema.fields.iter().enumerate() {
            if i > 0 {
                out.push(',');
            }
            out.push('"');
            out.push_str(&field.name);
            out.push_str("\":");
            if i < record.values.len() {
                value_to_json(&record.values[i], &mut out);
            } else {
                out.push_str("null");
            }
        }
        out.push('}');
    }
    out.push(']');
    out
}

/// Convert a SIF section to CSV.
pub fn to_csv(schema: &Schema, records: &[Record]) -> String {
    let mut out = String::new();
    for (i, field) in schema.fields.iter().enumerate() {
        if i > 0 {
            out.push(',');
        }
        out.push_str(&field.name);
    }
    out.push('\n');
    for record in records {
        for (i, value) in record.values.iter().enumerate() {
            if i > 0 {
                out.push(',');
            }
            value_to_csv(value, &mut out);
        }
        out.push('\n');
    }
    out
}

fn value_to_json(value: &Value, out: &mut String) {
    match value {
        Value::Null => out.push_str("null"),
        Value::Bool(true) => out.push_str("true"),
        Value::Bool(false) => out.push_str("false"),
        Value::Int(n) => out.push_str(&n.to_string()),
        Value::Uint(n) => out.push_str(&n.to_string()),
        Value::Float(n) => {
            if n.is_finite() {
                let s = n.to_string();
                if s.contains('.') {
                    out.push_str(&s);
                } else {
                    out.push_str(&format!("{}.0", s));
                }
            } else {
                out.push_str("null");
            }
        }
        Value::Str(s)
        | Value::Date(s)
        | Value::DateTime(s)
        | Value::Duration(s)
        | Value::Enum(s) => {
            out.push('"');
            for c in s.chars() {
                match c {
                    '"' => out.push_str("\\\""),
                    '\\' => out.push_str("\\\\"),
                    '\n' => out.push_str("\\n"),
                    '\t' => out.push_str("\\t"),
                    '\r' => out.push_str("\\r"),
                    c if c.is_control() => out.push_str(&format!("\\u{:04x}", c as u32)),
                    c => out.push(c),
                }
            }
            out.push('"');
        }
        Value::Bytes(b) => {
            out.push('"');
            out.push_str(&crate::types::base64_encode(b));
            out.push('"');
        }
        Value::Array(items) => {
            out.push('[');
            for (i, item) in items.iter().enumerate() {
                if i > 0 {
                    out.push(',');
                }
                value_to_json(item, out);
            }
            out.push(']');
        }
        Value::Map(pairs) => {
            out.push('{');
            for (i, (k, v)) in pairs.iter().enumerate() {
                if i > 0 {
                    out.push(',');
                }
                out.push('"');
                out.push_str(k);
                out.push_str("\":");
                value_to_json(v, out);
            }
            out.push('}');
        }
    }
}

fn value_to_csv(value: &Value, out: &mut String) {
    match value {
        Value::Null => {}
        Value::Bool(true) => out.push_str("true"),
        Value::Bool(false) => out.push_str("false"),
        Value::Int(n) => out.push_str(&n.to_string()),
        Value::Uint(n) => out.push_str(&n.to_string()),
        Value::Float(n) => out.push_str(&n.to_string()),
        Value::Str(s) => {
            if s.contains(',') || s.contains('"') || s.contains('\n') || s.contains('\r') {
                out.push('"');
                out.push_str(&s.replace('"', "\"\""));
                out.push('"');
            } else {
                out.push_str(s);
            }
        }
        _ => {
            let mut json = String::new();
            value_to_json(value, &mut json);
            if json.contains(',') || json.contains('"') {
                out.push('"');
                out.push_str(&json.replace('"', "\"\""));
                out.push('"');
            } else {
                out.push_str(&json);
            }
        }
    }
}

// ── Internal JSON Parser ────────────────────────────────────────────

const MAX_FLATTEN_DEPTH: usize = 2;

fn from_json_array(json: &str) -> Result<Document, String> {
    let json = json.trim();
    let objects = parse_json_array(json)?;
    if objects.is_empty() {
        return Ok(empty_doc());
    }

    let flat_objects: Vec<Vec<(String, String)>> =
        objects.iter().map(|obj| flatten_object(obj, "")).collect();

    let rows: Vec<Vec<(&str, &str)>> = flat_objects
        .iter()
        .map(|obj| {
            obj.iter()
                .map(|(k, v)| (k.as_str(), v.as_str()))
                .collect()
        })
        .collect();

    let schema = infer_schema(&rows);
    let records = build_records(&flat_objects, &schema);

    Ok(Document {
        header: Header {
            version: 1,
            attributes: HashMap::new(),
        },
        sections: vec![Section {
            id: None,
            directives: Vec::new(),
            schema: Some(schema),
            records,
            blocks: Vec::new(),
            templates: Vec::new(),
        }],
    })
}

fn from_json_scalar_array(json: &str) -> Result<Document, String> {
    let inner = json[1..json.len() - 1].trim();
    if inner.is_empty() {
        return Ok(empty_doc());
    }

    let parts = split_json_top_level(inner, ',');
    let values: Vec<String> = parts
        .iter()
        .map(|p| json_value_to_sif_string(p.trim()))
        .collect();

    let rows: Vec<Vec<(&str, &str)>> = values
        .iter()
        .map(|v| vec![("value", v.as_str())])
        .collect();
    let schema = infer_schema(&rows);

    let records: Vec<Record> = values
        .iter()
        .map(|v| {
            let val = crate::parse::parse_untyped_value(v);
            Record {
                values: vec![val],
                cdc_op: CdcOp::Insert,
            }
        })
        .collect();

    Ok(Document {
        header: Header {
            version: 1,
            attributes: HashMap::new(),
        },
        sections: vec![Section {
            id: None,
            directives: Vec::new(),
            schema: Some(schema),
            records,
            blocks: Vec::new(),
            templates: Vec::new(),
        }],
    })
}

fn parse_json_array(json: &str) -> Result<Vec<Vec<(String, String)>>, String> {
    let json = json.trim();
    if !json.starts_with('[') || !json.ends_with(']') {
        return Err("expected JSON array".into());
    }
    let inner = json[1..json.len() - 1].trim();
    if inner.is_empty() {
        return Ok(Vec::new());
    }

    let mut objects = Vec::new();
    let parts = split_json_top_level(inner, ',');
    for part in parts {
        let part = part.trim();
        if part.is_empty() {
            continue;
        }
        objects.push(parse_json_object(part)?);
    }
    Ok(objects)
}

fn parse_json_object(json: &str) -> Result<Vec<(String, String)>, String> {
    let json = json.trim();
    if !json.starts_with('{') || !json.ends_with('}') {
        return Err(format!(
            "expected JSON object, got: {}...",
            &json[..json.len().min(40)]
        ));
    }

    let inner = json[1..json.len() - 1].trim();
    if inner.is_empty() {
        return Ok(Vec::new());
    }

    let mut pairs = Vec::new();
    let parts = split_json_top_level(inner, ',');

    for part in parts {
        let part = part.trim();
        if part.is_empty() {
            continue;
        }
        let colon = find_json_colon(part)?;
        let key_raw = part[..colon].trim();
        let val_raw = part[colon + 1..].trim();

        let key = if key_raw.starts_with('"') && key_raw.ends_with('"') {
            json_unescape(&key_raw[1..key_raw.len() - 1])
        } else {
            return Err(format!("expected quoted key, got: {}", key_raw));
        };

        let value = json_value_to_sif_string(val_raw);
        pairs.push((key, value));
    }
    Ok(pairs)
}

fn flatten_object(pairs: &[(String, String)], prefix: &str) -> Vec<(String, String)> {
    flatten_object_depth(pairs, prefix, 0)
}

fn flatten_object_depth(
    pairs: &[(String, String)],
    prefix: &str,
    depth: usize,
) -> Vec<(String, String)> {
    let mut out = Vec::new();
    for (key, value) in pairs {
        let full_key = if prefix.is_empty() {
            sanitize_field_name(key)
        } else {
            format!("{}.{}", prefix, sanitize_field_name(key))
        };

        let trimmed = value.trim();
        if depth < MAX_FLATTEN_DEPTH && trimmed.starts_with('{') && trimmed.ends_with('}') {
            if let Ok(nested) = parse_json_object(trimmed) {
                let flat = flatten_object_depth(&nested, &full_key, depth + 1);
                out.extend(flat);
                continue;
            }
        }

        let sif_value = if trimmed.starts_with('[') && trimmed.ends_with(']') {
            json_array_to_sif(trimmed)
        } else {
            value.clone()
        };

        out.push((full_key, sif_value));
    }
    out
}

fn json_value_to_sif_string(v: &str) -> String {
    let v = v.trim();
    if v == "null" {
        return "_".to_string();
    }
    if v == "true" {
        return "T".to_string();
    }
    if v == "false" {
        return "F".to_string();
    }
    if v.starts_with('"') && v.ends_with('"') {
        return json_unescape(&v[1..v.len() - 1]);
    }
    if v.starts_with('{') || v.starts_with('[') {
        return v.to_string();
    }
    v.to_string()
}

fn json_unescape(s: &str) -> String {
    let mut out = String::new();
    let mut chars = s.chars();
    while let Some(c) = chars.next() {
        if c == '\\' {
            match chars.next() {
                Some('n') => out.push('\n'),
                Some('t') => out.push('\t'),
                Some('r') => out.push('\r'),
                Some('\\') => out.push('\\'),
                Some('"') => out.push('"'),
                Some('/') => out.push('/'),
                Some('u') => {
                    let mut hex = String::with_capacity(4);
                    for _ in 0..4 {
                        if let Some(h) = chars.next() {
                            hex.push(h);
                        }
                    }
                    if let Ok(cp) = u32::from_str_radix(&hex, 16) {
                        if let Some(ch) = char::from_u32(cp) {
                            out.push(ch);
                        }
                    }
                }
                Some(other) => {
                    out.push('\\');
                    out.push(other);
                }
                None => out.push('\\'),
            }
        } else {
            out.push(c);
        }
    }
    out
}

fn json_array_to_sif(v: &str) -> String {
    let inner = v[1..v.len() - 1].trim();
    if inner.is_empty() {
        return "[]".to_string();
    }
    let parts = split_json_top_level(inner, ',');
    let sif_parts: Vec<String> = parts
        .iter()
        .map(|p| json_value_to_sif_string(p.trim()))
        .collect();
    format!("[{}]", sif_parts.join(","))
}

fn split_json_top_level(s: &str, delimiter: char) -> Vec<&str> {
    let mut parts = Vec::new();
    let mut depth = 0;
    let mut in_string = false;
    let mut escape = false;
    let mut start = 0;

    for (i, c) in s.char_indices() {
        if escape {
            escape = false;
            continue;
        }
        if c == '\\' && in_string {
            escape = true;
            continue;
        }
        if c == '"' {
            in_string = !in_string;
            continue;
        }
        if in_string {
            continue;
        }
        match c {
            '[' | '{' => depth += 1,
            ']' | '}' => depth -= 1,
            c2 if c2 == delimiter && depth == 0 => {
                parts.push(&s[start..i]);
                start = i + 1;
            }
            _ => {}
        }
    }
    parts.push(&s[start..]);
    parts
}

fn find_json_colon(s: &str) -> Result<usize, String> {
    let mut in_string = false;
    let mut escape = false;

    for (i, c) in s.char_indices() {
        if escape {
            escape = false;
            continue;
        }
        if c == '\\' && in_string {
            escape = true;
            continue;
        }
        if c == '"' {
            in_string = !in_string;
            continue;
        }
        if c == ':' && !in_string {
            return Ok(i);
        }
    }
    Err(format!("no colon found in: {}", &s[..s.len().min(60)]))
}

fn split_csv_line(line: &str) -> Vec<String> {
    let mut fields = Vec::new();
    let mut current = String::new();
    let mut in_quote = false;
    let mut chars = line.chars().peekable();

    while let Some(c) = chars.next() {
        if in_quote {
            if c == '"' {
                if chars.peek() == Some(&'"') {
                    chars.next();
                    current.push('"');
                } else {
                    in_quote = false;
                }
            } else {
                current.push(c);
            }
        } else {
            match c {
                ',' => {
                    fields.push(current.clone());
                    current.clear();
                }
                '"' => in_quote = true,
                _ => current.push(c),
            }
        }
    }
    fields.push(current);
    fields
}

fn build_records(flat_rows: &[Vec<(String, String)>], schema: &Schema) -> Vec<Record> {
    flat_rows
        .iter()
        .map(|flat_row| {
            let values: Vec<Value> = schema
                .fields
                .iter()
                .map(|field_def| {
                    if let Some((_, v)) = flat_row.iter().find(|(k, _)| k == &field_def.name) {
                        crate::parse::parse_typed_value(v, &field_def.field_type)
                            .unwrap_or_else(|_| crate::parse::parse_untyped_value(v))
                    } else {
                        Value::Null
                    }
                })
                .collect();
            Record {
                values,
                cdc_op: CdcOp::Insert,
            }
        })
        .collect()
}

fn build_records_csv(raw_rows: &[Vec<String>], schema: &Schema) -> Vec<Record> {
    raw_rows
        .iter()
        .map(|raw_row| {
            let values: Vec<Value> = schema
                .fields
                .iter()
                .enumerate()
                .map(|(i, field_def)| {
                    if let Some(v) = raw_row.get(i) {
                        crate::parse::parse_typed_value(v, &field_def.field_type)
                            .unwrap_or_else(|_| crate::parse::parse_untyped_value(v))
                    } else {
                        Value::Null
                    }
                })
                .collect();
            Record {
                values,
                cdc_op: CdcOp::Insert,
            }
        })
        .collect()
}

fn empty_doc() -> Document {
    Document {
        header: Header {
            version: 1,
            attributes: HashMap::new(),
        },
        sections: vec![Section {
            id: None,
            directives: Vec::new(),
            schema: None,
            records: Vec::new(),
            blocks: Vec::new(),
            templates: Vec::new(),
        }],
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_infer_basic_types() {
        assert_eq!(infer_type("42"), Type::Uint);
        assert_eq!(infer_type("-7"), Type::Int);
        assert_eq!(infer_type("3.14"), Type::Float);
        assert_eq!(infer_type("hello"), Type::Str);
        assert_eq!(infer_type("T"), Type::Bool);
        assert_eq!(infer_type("_"), Type::Null);
        assert_eq!(infer_type("2026-03-08"), Type::Date);
        assert_eq!(infer_type("2026-03-08T06:50:51Z"), Type::DateTime);
        assert_eq!(infer_type("PT2H30M"), Type::Duration);
    }

    #[test]
    fn test_merge_types() {
        assert_eq!(merge_types(&Type::Int, &Type::Uint), Type::Int);
        assert_eq!(merge_types(&Type::Int, &Type::Float), Type::Float);
        assert_eq!(
            merge_types(&Type::Int, &Type::Null),
            Type::Nullable(Box::new(Type::Int))
        );
        assert_eq!(merge_types(&Type::Date, &Type::DateTime), Type::DateTime);
    }

    #[test]
    fn test_from_json_basic() {
        let json = r#"[{"id":1,"name":"alice"},{"id":2,"name":"bob"}]"#;
        let doc = from_json(json).unwrap();
        let sec = &doc.sections[0];
        assert_eq!(sec.records.len(), 2);
        assert_eq!(sec.records[0].values[0], Value::Uint(1));
    }

    #[test]
    fn test_from_json_nested() {
        let json = r#"[{"id":1,"actor":{"login":"alice","id":42}}]"#;
        let doc = from_json(json).unwrap();
        let schema = doc.sections[0].schema.as_ref().unwrap();
        let names: Vec<&str> = schema.fields.iter().map(|f| f.name.as_str()).collect();
        assert!(names.contains(&"actor.login"));
        assert!(names.contains(&"actor.id"));
    }

    #[test]
    fn test_from_json_nulls() {
        let json = r#"[{"id":1,"score":42},{"id":2,"score":null}]"#;
        let doc = from_json(json).unwrap();
        let schema = doc.sections[0].schema.as_ref().unwrap();
        assert!(matches!(schema.fields[1].field_type, Type::Nullable(_)));
    }

    #[test]
    fn test_from_csv() {
        let csv = "id,name,active\n1,alice,T\n2,bob,F\n";
        let doc = from_csv(csv).unwrap();
        let sec = &doc.sections[0];
        assert_eq!(sec.records.len(), 2);
        assert_eq!(sec.records[0].values[1], Value::Str("alice".to_string()));
    }

    #[test]
    fn test_from_jsonl() {
        let jsonl = r#"{"id":1,"name":"alice"}
{"id":2,"name":"bob"}
"#;
        let doc = from_jsonl(jsonl, None).unwrap();
        assert_eq!(doc.sections[0].records.len(), 2);
    }

    #[test]
    fn test_sanitize_field_name() {
        assert_eq!(sanitize_field_name("Case Number"), "case_number");
        assert_eq!(sanitize_field_name("$ref"), "_ref");
        assert_eq!(sanitize_field_name("\"STATION\""), "station");
    }

    #[test]
    fn test_to_jsonl() {
        let schema = Schema {
            fields: vec![
                FieldDef {
                    name: "id".to_string(),
                    field_type: Type::Uint,
                    semantic: Some("id".to_string()),
                    deprecated: false,
                    modifiers: vec![],
                },
                FieldDef {
                    name: "name".to_string(),
                    field_type: Type::Str,
                    semantic: None,
                    deprecated: false,
                    modifiers: vec![],
                },
            ],
        };
        let records = vec![Record {
            values: vec![Value::Uint(1), Value::Str("alice".to_string())],
            cdc_op: CdcOp::Insert,
        }];
        let json = to_jsonl(&schema, &records);
        assert!(json.contains("\"id\":1"));
        assert!(json.contains("\"name\":\"alice\""));
    }

    #[test]
    fn test_from_json_object_of_objects() {
        // Simplified OpenAPI-like structure: object-of-objects
        let json = r#"{
            "openapi": "3.0",
            "paths": {
                "/users": {"get": {"summary": "List users", "operationId": "getUsers"}},
                "/users/{id}": {"get": {"summary": "Get user", "operationId": "getUser"}},
                "/posts": {"get": {"summary": "List posts", "operationId": "getPosts"}},
                "/posts/{id}": {"get": {"summary": "Get post", "operationId": "getPost"}}
            }
        }"#;
        let doc = from_json(json).unwrap();
        eprintln!("sections: {}", doc.sections.len());
        for sec in &doc.sections {
            eprintln!("  id={:?} records={} fields={}",
                sec.id, sec.records.len(),
                sec.schema.as_ref().map(|s| s.fields.len()).unwrap_or(0));
        }
        // Should extract paths as a section with 4 records
        let total: usize = doc.sections.iter().map(|s| s.records.len()).sum();
        assert!(total >= 4, "expected >=4 records, got {}", total);
    }
}