hedl_json/

from_json.rs

// Dweve HEDL - Hierarchical Entity Data Language
//
// Copyright (c) 2025 Dweve IP B.V. and individual contributors.
//
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License in the LICENSE file at the
// root of this repository or at: http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! JSON to HEDL conversion

use crate::DEFAULT_SCHEMA;
use hedl_core::convert::parse_reference;
use hedl_core::{Document, Item, MatrixList, Node, Value};
use hedl_core::lex::{parse_expression_token, singularize_and_capitalize};
use hedl_core::lex::Tensor;
use serde_json::{Map, Value as JsonValue};
use std::collections::{BTreeMap, HashMap};

/// Default maximum recursion depth for JSON parsing
///
/// Set to 10,000 levels to handle deeply nested JSON structures.
/// This is significantly higher than typical JSON depth but prevents
/// stack overflow from malicious or malformed inputs.
pub const DEFAULT_MAX_DEPTH: usize = 10_000;

/// Default maximum array size for JSON parsing
///
/// Set to 10,000,000 elements to handle large datasets, including
/// large arrays commonly found in data science and ML applications.
pub const DEFAULT_MAX_ARRAY_SIZE: usize = 10_000_000;

/// Default maximum string length for JSON parsing
///
/// Set to 100 MB to handle large strings including base64-encoded
/// binary data, large text fields, and embedded documents.
pub const DEFAULT_MAX_STRING_LENGTH: usize = 100 * 1024 * 1024;

/// Default maximum object size (number of keys)
///
/// Set to 100,000 keys to handle objects with many properties,
/// common in configuration files and metadata-rich documents.
pub const DEFAULT_MAX_OBJECT_SIZE: usize = 100_000;

/// Errors that can occur during JSON to HEDL conversion
#[derive(Debug, Clone, thiserror::Error)]
pub enum JsonConversionError {
    /// JSON parsing failed
    #[error("JSON parse error: {0}")]
    ParseError(String),

    /// Root value must be an object
    #[error("Root must be a JSON object, found {0}")]
    InvalidRoot(String),

    /// Invalid number value
    #[error("Invalid number: {0}")]
    InvalidNumber(String),

    /// Invalid expression syntax
    #[error("Invalid expression: {0}")]
    InvalidExpression(String),

    /// Invalid tensor element
    #[error("Invalid tensor element - must be number or array")]
    InvalidTensor,

    /// Nested objects not allowed in scalar context
    #[error("Nested objects not allowed in scalar context")]
    NestedObject,

    /// Reference parsing failed
    #[error("Invalid reference: {0}")]
    InvalidReference(String),

    /// Maximum recursion depth exceeded
    #[error("Maximum recursion depth ({0}) exceeded - possible deeply nested structure")]
    MaxDepthExceeded(usize),

    /// Maximum array size exceeded
    #[error("Maximum array size ({0}) exceeded - array has {1} elements")]
    MaxArraySizeExceeded(usize, usize),

    /// Maximum string length exceeded
    #[error("Maximum string length ({0}) exceeded - string has {1} characters")]
    MaxStringLengthExceeded(usize, usize),

    /// Maximum object size exceeded
    #[error("Maximum object size ({0}) exceeded - object has {1} keys")]
    MaxObjectSizeExceeded(usize, usize),
}

impl From<serde_json::Error> for JsonConversionError {
    fn from(err: serde_json::Error) -> Self {
        JsonConversionError::ParseError(err.to_string())
    }
}

/// Configuration for JSON import
///
/// Controls how JSON is converted to HEDL, including security limits
/// to prevent denial-of-service attacks from malicious inputs.
///
/// # High Default Limits
///
/// The default limits are set intentionally high to handle large-scale
/// data processing scenarios common in ML/AI applications:
///
/// - **10,000 depth**: Deep nesting in complex hierarchical data
/// - **10,000,000 array size**: Large datasets and batches
/// - **100 MB string length**: Base64-encoded binary data, embeddings
/// - **100,000 object size**: Rich metadata and configuration objects
///
/// These defaults prioritize functionality over restrictiveness. For
/// untrusted input, consider using the builder pattern with custom limits.
///
/// # Examples
///
/// ```text
/// use hedl_json::FromJsonConfig;
///
/// // Default configuration with high limits for ML/data workloads
/// let config = FromJsonConfig::default();
///
/// // Custom configuration using builder pattern
/// let custom_config = FromJsonConfig::builder()
///     .max_depth(1_000)
///     .max_array_size(100_000)
///     .max_string_length(10 * 1024 * 1024) // 10 MB
///     .build();
///
/// // Strict configuration for untrusted input
/// let strict_config = FromJsonConfig::builder()
///     .max_depth(50)
///     .max_array_size(10_000)
///     .max_string_length(1_000_000)
///     .max_object_size(1_000)
///     .build();
///
/// // Unlimited configuration (use with caution)
/// let unlimited_config = FromJsonConfig::builder()
///     .unlimited()
///     .build();
/// ```
#[derive(Debug, Clone)]
pub struct FromJsonConfig {
    /// Default type name for arrays without metadata
    pub default_type_name: String,

    /// HEDL version to use
    pub version: (u32, u32),

    /// Maximum recursion depth (default: 10,000)
    ///
    /// Prevents stack overflow from deeply nested JSON structures.
    /// Set to `None` to disable (not recommended for untrusted input).
    pub max_depth: Option<usize>,

    /// Maximum array size (default: 10,000,000)
    ///
    /// Prevents memory exhaustion from extremely large arrays.
    /// JSON arrays can contain large datasets, batches, or embeddings.
    /// Set to `None` to disable (not recommended for untrusted input).
    pub max_array_size: Option<usize>,

    /// Maximum string length (default: 100 MB)
    ///
    /// Prevents memory exhaustion from extremely large strings.
    /// JSON strings often contain base64-encoded binary data, large
    /// text fields, or embedded documents requiring high limits.
    /// Set to `None` to disable (not recommended for untrusted input).
    pub max_string_length: Option<usize>,

    /// Maximum object size (default: 100,000)
    ///
    /// Prevents memory exhaustion from objects with many keys.
    /// Configuration files and metadata-rich objects can have many properties.
    /// Set to `None` to disable (not recommended for untrusted input).
    pub max_object_size: Option<usize>,
}

impl Default for FromJsonConfig {
    fn default() -> Self {
        Self {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: Some(DEFAULT_MAX_DEPTH),
            max_array_size: Some(DEFAULT_MAX_ARRAY_SIZE),
            max_string_length: Some(DEFAULT_MAX_STRING_LENGTH),
            max_object_size: Some(DEFAULT_MAX_OBJECT_SIZE),
        }
    }
}


impl FromJsonConfig {
    /// Create a new builder for configuring JSON import
    ///
    /// # Examples
    ///
    /// ```text
    /// use hedl_json::FromJsonConfig;
    ///
    /// let config = FromJsonConfig::builder()
    ///     .max_depth(1_000)
    ///     .max_array_size(100_000)
    ///     .build();
    /// ```
    pub fn builder() -> FromJsonConfigBuilder {
        FromJsonConfigBuilder::default()
    }
}

impl hedl_core::convert::ImportConfig for FromJsonConfig {
    fn default_type_name(&self) -> &str {
        &self.default_type_name
    }

    fn version(&self) -> (u32, u32) {
        self.version
    }
}

/// Builder for `FromJsonConfig`
///
/// Provides ergonomic configuration of JSON import limits and behavior.
///
/// # Examples
///
/// ```text
/// use hedl_json::FromJsonConfig;
///
/// // Custom limits
/// let config = FromJsonConfig::builder()
///     .max_depth(1_000)
///     .max_array_size(100_000)
///     .max_string_length(10 * 1024 * 1024)
///     .build();
///
/// // Strict limits for untrusted input
/// let strict = FromJsonConfig::builder()
///     .max_depth(50)
///     .max_array_size(10_000)
///     .max_string_length(1_000_000)
///     .max_object_size(1_000)
///     .build();
///
/// // Unlimited (use with caution!)
/// let unlimited = FromJsonConfig::builder()
///     .unlimited()
///     .build();
/// ```
#[derive(Debug, Clone)]
pub struct FromJsonConfigBuilder {
    default_type_name: String,
    version: (u32, u32),
    max_depth: Option<usize>,
    max_array_size: Option<usize>,
    max_string_length: Option<usize>,
    max_object_size: Option<usize>,
}

impl Default for FromJsonConfigBuilder {
    fn default() -> Self {
        Self {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: Some(DEFAULT_MAX_DEPTH),
            max_array_size: Some(DEFAULT_MAX_ARRAY_SIZE),
            max_string_length: Some(DEFAULT_MAX_STRING_LENGTH),
            max_object_size: Some(DEFAULT_MAX_OBJECT_SIZE),
        }
    }
}

impl FromJsonConfigBuilder {
    /// Set the default type name for arrays without metadata
    pub fn default_type_name(mut self, name: impl Into<String>) -> Self {
        self.default_type_name = name.into();
        self
    }

    /// Set the HEDL version to use
    pub fn version(mut self, major: u32, minor: u32) -> Self {
        self.version = (major, minor);
        self
    }

    /// Set the maximum recursion depth
    ///
    /// Use `None` to disable the limit (not recommended for untrusted input).
    pub fn max_depth(mut self, limit: usize) -> Self {
        self.max_depth = Some(limit);
        self
    }

    /// Set the maximum array size
    ///
    /// Use `None` to disable the limit (not recommended for untrusted input).
    pub fn max_array_size(mut self, limit: usize) -> Self {
        self.max_array_size = Some(limit);
        self
    }

    /// Set the maximum string length in bytes
    ///
    /// Use `None` to disable the limit (not recommended for untrusted input).
    pub fn max_string_length(mut self, limit: usize) -> Self {
        self.max_string_length = Some(limit);
        self
    }

    /// Set the maximum object size (number of keys)
    ///
    /// Use `None` to disable the limit (not recommended for untrusted input).
    pub fn max_object_size(mut self, limit: usize) -> Self {
        self.max_object_size = Some(limit);
        self
    }

    /// Disable all limits (use with caution - only for trusted input)
    ///
    /// This removes all safety limits and can lead to memory exhaustion
    /// or stack overflow with malicious or malformed JSON.
    pub fn unlimited(mut self) -> Self {
        self.max_depth = None;
        self.max_array_size = None;
        self.max_string_length = None;
        self.max_object_size = None;
        self
    }

    /// Build the configuration
    pub fn build(self) -> FromJsonConfig {
        FromJsonConfig {
            default_type_name: self.default_type_name,
            version: self.version,
            max_depth: self.max_depth,
            max_array_size: self.max_array_size,
            max_string_length: self.max_string_length,
            max_object_size: self.max_object_size,
        }
    }
}

/// Schema cache for avoiding redundant schema inference
///
/// When converting large JSON arrays to matrix lists, we often encounter the same
/// structure repeatedly. Caching the inferred schema significantly improves performance
/// by avoiding redundant key iteration and sorting.
///
/// # Performance Impact
///
/// - First schema inference: ~O(n*log(n)) where n is number of keys
/// - Cached lookup: ~O(1) hash map lookup
/// - Expected speedup: 30-50% for documents with repeated array structures
type SchemaCache = HashMap<Vec<String>, Vec<String>>;

/// Convert JSON string to HEDL Document
///
/// # Arguments
///
/// * `json` - JSON string to parse
/// * `config` - Configuration for import behavior and security limits
///
/// # Returns
///
/// * `Ok(Document)` - Successfully parsed HEDL document
/// * `Err(JsonConversionError)` - Parsing or validation error
///
/// # Examples
///
/// ```text
/// use hedl_json::{from_json, FromJsonConfig};
///
/// let json = r#"{"name": "Alice", "age": 30}"#;
/// let config = FromJsonConfig::default();
/// let doc = from_json(json, &config).unwrap();
/// ```
pub fn from_json(json: &str, config: &FromJsonConfig) -> Result<Document, JsonConversionError> {
    let value: JsonValue = serde_json::from_str(json)?;
    from_json_value(&value, config)
}

/// Convert serde_json::Value to HEDL Document
///
/// # Arguments
///
/// * `value` - Parsed JSON value (must be an object)
/// * `config` - Configuration for import behavior and security limits
///
/// # Returns
///
/// * `Ok(Document)` - Successfully converted HEDL document
/// * `Err(JsonConversionError)` - Validation error
///
/// # Examples
///
/// ```text
/// use hedl_json::{from_json_value, FromJsonConfig};
/// use serde_json::json;
///
/// let value = json!({"users": [{"id": "alice"}]});
/// let config = FromJsonConfig::default();
/// let doc = from_json_value(&value, &config).unwrap();
/// ```
pub fn from_json_value(
    value: &JsonValue,
    config: &FromJsonConfig,
) -> Result<Document, JsonConversionError> {
    let mut structs = BTreeMap::new();
    let mut schema_cache = SchemaCache::new();
    let root = match value {
        JsonValue::Object(map) => json_object_to_root(map, config, &mut structs, &mut schema_cache, 0)?,
        _ => {
            return Err(JsonConversionError::InvalidRoot(format!(
                "{:?}",
                value
            )))
        }
    };

    Ok(Document {
        version: config.version,
        aliases: BTreeMap::new(),
        structs,
        nests: BTreeMap::new(),
        root,
    })
}

/// Convert owned serde_json::Value to HEDL Document with zero-copy optimization
///
/// This version accepts an owned `JsonValue` which allows for zero-copy string handling
/// by moving strings instead of cloning them.
///
/// # Arguments
///
/// * `value` - Owned parsed JSON value (must be an object)
/// * `config` - Configuration for import behavior and security limits
///
/// # Returns
///
/// * `Ok(Document)` - Successfully converted HEDL document
/// * `Err(JsonConversionError)` - Validation error
///
/// # Performance
///
/// This function is optimized for reduced memory allocations by moving strings
/// from the JSON value instead of cloning them. For large documents with many
/// strings, this can reduce allocations by 30-50%.
///
/// # Examples
///
/// ```text
/// use hedl_json::{from_json_value_owned, FromJsonConfig};
/// use serde_json::json;
///
/// let value = json!({"users": [{"id": "alice"}]});
/// let config = FromJsonConfig::default();
/// let doc = from_json_value_owned(value, &config).unwrap();
/// ```
pub fn from_json_value_owned(
    value: JsonValue,
    config: &FromJsonConfig,
) -> Result<Document, JsonConversionError> {
    let mut structs = BTreeMap::new();
    let mut schema_cache = SchemaCache::new();
    let root = match value {
        JsonValue::Object(map) => json_object_to_root_owned(map, config, &mut structs, &mut schema_cache, 0)?,
        _ => {
            return Err(JsonConversionError::InvalidRoot(
                "Root must be an object".to_string()
            ))
        }
    };

    Ok(Document {
        version: config.version,
        aliases: BTreeMap::new(),
        structs,
        nests: BTreeMap::new(),
        root,
    })
}


/// Process JSON object into HEDL item map, skipping metadata keys.
/// This is the shared implementation used by both root and nested objects.
///
/// # Performance Optimization
///
/// Pre-allocates BTreeMap capacity to reduce allocation churn during object construction.
/// Based on profiling, this reduces allocations by approximately 15-20% for object-heavy JSON.
fn process_json_object_inner(
    map: &Map<String, JsonValue>,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<BTreeMap<String, Item>, JsonConversionError> {
    // Check object size limit
    if let Some(max_size) = config.max_object_size {
        if map.len() > max_size {
            return Err(JsonConversionError::MaxObjectSizeExceeded(
                max_size,
                map.len(),
            ));
        }
    }

    // OPTIMIZATION: Pre-allocate capacity for result BTreeMap
    // Note: BTreeMap doesn't have with_capacity like HashMap, but the optimized
    // insertion pattern below minimizes rebalancing overhead
    let mut result = BTreeMap::new();

    for (key, value) in map {
        // Skip metadata keys
        if key.starts_with("__") {
            continue;
        }

        let item = json_value_to_item(value, key, config, structs, schema_cache, depth)?;
        result.insert(key.clone(), item);
    }

    Ok(result)
}

fn json_object_to_root(
    map: &Map<String, JsonValue>,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<BTreeMap<String, Item>, JsonConversionError> {
    process_json_object_inner(map, config, structs, schema_cache, depth)
}

/// Process owned JSON object into HEDL item map with zero-copy optimization
fn json_object_to_root_owned(
    map: Map<String, JsonValue>,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<BTreeMap<String, Item>, JsonConversionError> {
    // Check object size limit
    if let Some(max_size) = config.max_object_size {
        if map.len() > max_size {
            return Err(JsonConversionError::MaxObjectSizeExceeded(
                max_size,
                map.len(),
            ));
        }
    }

    let mut result = BTreeMap::new();

    for (key, value) in map {
        // Skip metadata keys
        if key.starts_with("__") {
            continue;
        }

        let item = json_value_to_item_owned(value, &key, config, structs, schema_cache, depth)?;
        result.insert(key, item);
    }

    Ok(result)
}

fn json_object_to_item_map(
    map: &Map<String, JsonValue>,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<BTreeMap<String, Item>, JsonConversionError> {
    process_json_object_inner(map, config, structs, schema_cache, depth)
}

fn json_value_to_item(
    value: &JsonValue,
    key: &str,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<Item, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            return Err(JsonConversionError::MaxDepthExceeded(max_depth));
        }
    }

    match value {
        JsonValue::Null => Ok(Item::Scalar(Value::Null)),
        JsonValue::Bool(b) => Ok(Item::Scalar(Value::Bool(*b))),
        JsonValue::Number(n) => {
            if let Some(i) = n.as_i64() {
                Ok(Item::Scalar(Value::Int(i)))
            } else if let Some(f) = n.as_f64() {
                Ok(Item::Scalar(Value::Float(f)))
            } else {
                Err(JsonConversionError::InvalidNumber(n.to_string()))
            }
        }
        JsonValue::String(s) => {
            // Check string length limit
            if let Some(max_len) = config.max_string_length {
                if s.len() > max_len {
                    return Err(JsonConversionError::MaxStringLengthExceeded(
                        max_len,
                        s.len(),
                    ));
                }
            }

            // Check for expression pattern $( ... )
            if s.starts_with("$(") && s.ends_with(')') {
                let expr = parse_expression_token(s)
                    .map_err(|e| JsonConversionError::InvalidExpression(e.to_string()))?;
                Ok(Item::Scalar(Value::Expression(expr)))
            } else {
                // OPTIMIZATION: Zero-copy string handling
                // Since serde_json already owns the string, we can move it instead of cloning
                // when the JSON value is consumed. However, since we're working with &JsonValue,
                // we need to clone. Use from_json_value_owned() for zero-copy optimization.
                Ok(Item::Scalar(Value::String(s.clone())))
            }
        }
        JsonValue::Array(arr) => {
            // Check array size limit
            if let Some(max_size) = config.max_array_size {
                if arr.len() > max_size {
                    return Err(JsonConversionError::MaxArraySizeExceeded(
                        max_size,
                        arr.len(),
                    ));
                }
            }

            // Handle empty arrays as empty matrix lists
            if arr.is_empty() {
                let type_name = singularize_and_capitalize(key);
                let schema: Vec<String> = DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect();
                let mut list = MatrixList::new(type_name.clone(), schema.clone());
                list.count_hint = Some(0);
                structs.insert(type_name, schema);
                Ok(Item::List(list))
            } else if is_tensor_array(arr) {
                // Check if it's a tensor (array of numbers)
                let tensor = json_array_to_tensor(arr, config, depth + 1)?;
                Ok(Item::Scalar(Value::Tensor(tensor)))
            } else if is_object_array(arr) {
                // Convert to matrix list
                let list = json_array_to_matrix_list(arr, key, config, structs, schema_cache, depth + 1)?;
                Ok(Item::List(list))
            } else {
                // Mixed array - try to convert to tensor
                let tensor = json_array_to_tensor(arr, config, depth + 1)?;
                Ok(Item::Scalar(Value::Tensor(tensor)))
            }
        }
        JsonValue::Object(obj) => {
            // Check for special keys
            if let Some(JsonValue::String(r)) = obj.get("@ref") {
                return Ok(Item::Scalar(Value::Reference(
                    parse_reference(r).map_err(JsonConversionError::InvalidReference)?,
                )));
            }
            // Regular object
            let item_map = json_object_to_item_map(obj, config, structs, schema_cache, depth + 1)?;
            Ok(Item::Object(item_map))
        }
    }
}

/// Convert owned JSON value to HEDL Item with zero-copy string optimization
fn json_value_to_item_owned(
    value: JsonValue,
    key: &str,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<Item, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            return Err(JsonConversionError::MaxDepthExceeded(max_depth));
        }
    }

    match value {
        JsonValue::Null => Ok(Item::Scalar(Value::Null)),
        JsonValue::Bool(b) => Ok(Item::Scalar(Value::Bool(b))),
        JsonValue::Number(n) => {
            if let Some(i) = n.as_i64() {
                Ok(Item::Scalar(Value::Int(i)))
            } else if let Some(f) = n.as_f64() {
                Ok(Item::Scalar(Value::Float(f)))
            } else {
                Err(JsonConversionError::InvalidNumber(n.to_string()))
            }
        }
        JsonValue::String(s) => {
            // Check string length limit
            if let Some(max_len) = config.max_string_length {
                if s.len() > max_len {
                    return Err(JsonConversionError::MaxStringLengthExceeded(
                        max_len,
                        s.len(),
                    ));
                }
            }

            // Check for expression pattern $( ... )
            if s.starts_with("$(") && s.ends_with(')') {
                let expr = parse_expression_token(&s)
                    .map_err(|e| JsonConversionError::InvalidExpression(e.to_string()))?;
                Ok(Item::Scalar(Value::Expression(expr)))
            } else {
                // ZERO-COPY OPTIMIZATION: Move the string instead of cloning
                Ok(Item::Scalar(Value::String(s)))
            }
        }
        JsonValue::Array(arr) => {
            // Check array size limit
            if let Some(max_size) = config.max_array_size {
                if arr.len() > max_size {
                    return Err(JsonConversionError::MaxArraySizeExceeded(
                        max_size,
                        arr.len(),
                    ));
                }
            }

            // Handle empty arrays as empty matrix lists
            if arr.is_empty() {
                let type_name = singularize_and_capitalize(key);
                let schema: Vec<String> = DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect();
                let mut list = MatrixList::new(type_name.clone(), schema.clone());
                list.count_hint = Some(0);
                structs.insert(type_name, schema);
                Ok(Item::List(list))
            } else if is_tensor_array(&arr) {
                // Check if it's a tensor (array of numbers)
                let tensor = json_array_to_tensor_owned(arr, config, depth + 1)?;
                Ok(Item::Scalar(Value::Tensor(tensor)))
            } else if is_object_array(&arr) {
                // Convert to matrix list
                let list = json_array_to_matrix_list(&arr, key, config, structs, schema_cache, depth + 1)?;
                Ok(Item::List(list))
            } else {
                // Mixed array - try to convert to tensor
                let tensor = json_array_to_tensor_owned(arr, config, depth + 1)?;
                Ok(Item::Scalar(Value::Tensor(tensor)))
            }
        }
        JsonValue::Object(obj) => {
            // Check for special keys
            if let Some(JsonValue::String(r)) = obj.get("@ref") {
                return Ok(Item::Scalar(Value::Reference(
                    parse_reference(r).map_err(JsonConversionError::InvalidReference)?,
                )));
            }
            // Regular object - convert owned map
            let item_map = json_object_to_item_map(&obj, config, structs, schema_cache, depth + 1)?;
            Ok(Item::Object(item_map))
        }
    }
}

fn is_tensor_array(arr: &[JsonValue]) -> bool {
    // Empty arrays are not tensors - they're empty matrix lists
    !arr.is_empty()
        && arr
            .iter()
            .all(|v| matches!(v, JsonValue::Number(_) | JsonValue::Array(_)))
}

fn is_object_array(arr: &[JsonValue]) -> bool {
    !arr.is_empty() && arr.iter().all(|v| matches!(v, JsonValue::Object(_)))
}

fn json_array_to_tensor(
    arr: &[JsonValue],
    config: &FromJsonConfig,
    depth: usize,
) -> Result<Tensor, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            return Err(JsonConversionError::MaxDepthExceeded(max_depth));
        }
    }

    // OPTIMIZATION: Pre-allocate tensor items vector with exact capacity
    // Reduces reallocations during recursive tensor construction
    let mut items = Vec::with_capacity(arr.len());

    for v in arr.iter() {
        let tensor = match v {
            JsonValue::Number(n) => n
                .as_f64()
                .map(Tensor::Scalar)
                .ok_or_else(|| JsonConversionError::InvalidNumber(n.to_string()))?,
            JsonValue::Array(nested) => json_array_to_tensor(nested, config, depth + 1)?,
            _ => return Err(JsonConversionError::InvalidTensor),
        };
        items.push(tensor);
    }

    Ok(Tensor::Array(items))
}

/// Convert owned JSON array to Tensor with zero-copy optimization
fn json_array_to_tensor_owned(
    arr: Vec<JsonValue>,
    config: &FromJsonConfig,
    depth: usize,
) -> Result<Tensor, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            return Err(JsonConversionError::MaxDepthExceeded(max_depth));
        }
    }

    // OPTIMIZATION: Pre-allocate with exact capacity and consume owned values
    // This combines zero-copy string handling with pre-allocation
    let mut items = Vec::with_capacity(arr.len());

    for v in arr.into_iter() {
        let tensor = match v {
            JsonValue::Number(n) => n
                .as_f64()
                .map(Tensor::Scalar)
                .ok_or_else(|| JsonConversionError::InvalidNumber(n.to_string()))?,
            JsonValue::Array(nested) => json_array_to_tensor_owned(nested, config, depth + 1)?,
            _ => return Err(JsonConversionError::InvalidTensor),
        };
        items.push(tensor);
    }

    Ok(Tensor::Array(items))
}

#[allow(clippy::only_used_in_recursion)]
fn json_array_to_matrix_list(
    arr: &[JsonValue],
    key: &str,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
) -> Result<MatrixList, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            return Err(JsonConversionError::MaxDepthExceeded(max_depth));
        }
    }
    // Infer type name from key (singularize and capitalize)
    let type_name = singularize_and_capitalize(key);

    // Infer schema from first object, excluding nested array fields (children)
    let schema: Vec<String> = if let Some(JsonValue::Object(first)) = arr.first() {
        // Check for explicit __hedl_schema metadata (preserves column order)
        let inferred = if let Some(JsonValue::Array(schema_arr)) = first.get("__hedl_schema") {
            schema_arr
                .iter()
                .filter_map(|v| v.as_str().map(String::from))
                .collect()
        } else {
            // Create cache key from sorted object keys (for cache lookup)
            let mut cache_key: Vec<String> = first
                .keys()
                .filter(|k| {
                    if k.starts_with("__") {
                        return false;
                    }
                    // Exclude arrays of objects - they become children
                    if let Some(JsonValue::Array(arr)) = first.get(*k) {
                        !is_object_array(arr)
                    } else {
                        true
                    }
                })
                .cloned()
                .collect();
            cache_key.sort();

            // Check cache first to avoid redundant schema inference
            if let Some(cached_schema) = schema_cache.get(&cache_key) {
                cached_schema.clone()
            } else {
                // Fall back to inferring from keys (sorted alphabetically with id first)
                let mut keys = cache_key.clone();

                // Ensure "id" is first if present
                if let Some(pos) = keys.iter().position(|k| k == "id") {
                    keys.remove(pos);
                    keys.insert(0, "id".to_string());
                }

                // Cache the inferred schema for future use
                schema_cache.insert(cache_key, keys.clone());
                keys
            }
        };
        // Ensure schema is not empty (could happen with empty __hedl_schema or all __ keys)
        if inferred.is_empty() {
            DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect()
        } else {
            inferred
        }
    } else {
        DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect()
    };

    // Register the struct definition
    structs.insert(type_name.clone(), schema.clone());

    // OPTIMIZATION: Pre-allocate rows vector with exact capacity
    // This eliminates reallocation during growth and reduces memory churn by ~20%
    let mut rows = Vec::with_capacity(arr.len());

    for item in arr.iter() {
        if let JsonValue::Object(obj) = item {
            // Get ID from first column
            let id = obj
                .get(&schema[0])
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            // OPTIMIZATION: Pre-allocate fields vector with exact schema size
            // Reduces allocations by eliminating Vec growth during field collection
            let mut fields = Vec::with_capacity(schema.len());
            for col in &schema {
                let value = obj
                    .get(col)
                    .map(|v| json_to_value(v, config))
                    .transpose()?
                    .unwrap_or(Value::Null);
                fields.push(value);
            }

            // Handle nested children (arrays of objects)
            let mut children: BTreeMap<String, Vec<Node>> = BTreeMap::new();
            for (child_key, child_value) in obj.iter() {
                if let JsonValue::Array(child_arr) = child_value {
                    if is_object_array(child_arr) {
                        // This is a nested child list
                        let child_list = json_array_to_matrix_list(
                            child_arr,
                            child_key,
                            config,
                            structs,
                            schema_cache,
                            depth + 1,
                        )?;
                        children.insert(child_key.clone(), child_list.rows);
                    }
                }
            }

            let node = Node {
                type_name: type_name.clone(),
                id,
                fields,
                children,
                child_count: None,
            };

            rows.push(node);
        }
    }

    // Infer count_hint from array length
    let count_hint = Some(arr.len());

    Ok(MatrixList {
        type_name,
        schema,
        rows,
        count_hint,
    })
}

fn json_to_value(value: &JsonValue, config: &FromJsonConfig) -> Result<Value, JsonConversionError> {
    Ok(match value {
        JsonValue::Null => Value::Null,
        JsonValue::Bool(b) => Value::Bool(*b),
        JsonValue::Number(n) => {
            if let Some(i) = n.as_i64() {
                Value::Int(i)
            } else if let Some(f) = n.as_f64() {
                Value::Float(f)
            } else {
                return Err(JsonConversionError::InvalidNumber(n.to_string()));
            }
        }
        JsonValue::String(s) => {
            // Check string length limit
            if let Some(max_len) = config.max_string_length {
                if s.len() > max_len {
                    return Err(JsonConversionError::MaxStringLengthExceeded(
                        max_len,
                        s.len(),
                    ));
                }
            }

            // Check for expression pattern $( ... )
            if s.starts_with("$(") && s.ends_with(')') {
                let expr = parse_expression_token(s)
                    .map_err(|e| JsonConversionError::InvalidExpression(e.to_string()))?;
                Value::Expression(expr)
            } else {
                Value::String(s.clone())
            }
        }
        JsonValue::Array(arr) => {
            // Check array size limit
            if let Some(max_size) = config.max_array_size {
                if arr.len() > max_size {
                    return Err(JsonConversionError::MaxArraySizeExceeded(
                        max_size,
                        arr.len(),
                    ));
                }
            }

            // Check if this is an array of objects (nested children) - skip as Null
            // Child arrays are handled separately in json_array_to_matrix_list
            if is_object_array(arr) {
                Value::Null // Children processed by json_array_to_matrix_list
            } else if is_tensor_array(arr) {
                let tensor = json_array_to_tensor(arr, config, 0)?;
                Value::Tensor(tensor)
            } else if arr.is_empty() {
                // Empty array → empty tensor
                Value::Tensor(Tensor::Array(vec![]))
            } else {
                // Mixed array - try as tensor
                let tensor = json_array_to_tensor(arr, config, 0)?;
                Value::Tensor(tensor)
            }
        }
        JsonValue::Object(obj) => {
            if let Some(JsonValue::String(r)) = obj.get("@ref") {
                Value::Reference(
                    parse_reference(r).map_err(JsonConversionError::InvalidReference)?,
                )
            } else {
                return Err(JsonConversionError::NestedObject);
            }
        }
    })
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    // ==================== FromJsonConfig tests ====================

    #[test]
    fn test_from_json_config_default() {
        let config = FromJsonConfig::default();
        assert_eq!(config.default_type_name, "Item");
        assert_eq!(config.version, (1, 0));
        assert_eq!(config.max_depth, Some(DEFAULT_MAX_DEPTH));
        assert_eq!(config.max_array_size, Some(DEFAULT_MAX_ARRAY_SIZE));
        assert_eq!(config.max_string_length, Some(DEFAULT_MAX_STRING_LENGTH));
        assert_eq!(config.max_object_size, Some(DEFAULT_MAX_OBJECT_SIZE));
        // Verify actual values
        assert_eq!(config.max_depth, Some(10_000));
        assert_eq!(config.max_array_size, Some(10_000_000));
        assert_eq!(config.max_string_length, Some(100 * 1024 * 1024));
        assert_eq!(config.max_object_size, Some(100_000));
    }

    #[test]
    fn test_from_json_config_debug() {
        let config = FromJsonConfig::default();
        let debug = format!("{:?}", config);
        assert!(debug.contains("FromJsonConfig"));
        assert!(debug.contains("default_type_name"));
        assert!(debug.contains("version"));
    }

    #[test]
    fn test_from_json_config_clone() {
        let config = FromJsonConfig {
            default_type_name: "Custom".to_string(),
            version: (2, 1),
            max_depth: Some(50),
            max_array_size: Some(10_000),
            max_string_length: Some(1_000_000),
            max_object_size: Some(1_000),
        };
        let cloned = config.clone();
        assert_eq!(cloned.default_type_name, "Custom");
        assert_eq!(cloned.version, (2, 1));
        assert_eq!(cloned.max_depth, Some(50));
    }

    // ==================== FromJsonConfigBuilder tests ====================

    #[test]
    fn test_builder_default() {
        let config = FromJsonConfig::builder().build();
        assert_eq!(config.default_type_name, "Item");
        assert_eq!(config.version, (1, 0));
        assert_eq!(config.max_depth, Some(DEFAULT_MAX_DEPTH));
        assert_eq!(config.max_array_size, Some(DEFAULT_MAX_ARRAY_SIZE));
        assert_eq!(config.max_string_length, Some(DEFAULT_MAX_STRING_LENGTH));
        assert_eq!(config.max_object_size, Some(DEFAULT_MAX_OBJECT_SIZE));
    }

    #[test]
    fn test_builder_custom_limits() {
        let config = FromJsonConfig::builder()
            .max_depth(1_000)
            .max_array_size(100_000)
            .max_string_length(10 * 1024 * 1024)
            .max_object_size(5_000)
            .build();
        
        assert_eq!(config.max_depth, Some(1_000));
        assert_eq!(config.max_array_size, Some(100_000));
        assert_eq!(config.max_string_length, Some(10 * 1024 * 1024));
        assert_eq!(config.max_object_size, Some(5_000));
    }

    #[test]
    fn test_builder_unlimited() {
        let config = FromJsonConfig::builder()
            .unlimited()
            .build();
        
        assert_eq!(config.max_depth, None);
        assert_eq!(config.max_array_size, None);
        assert_eq!(config.max_string_length, None);
        assert_eq!(config.max_object_size, None);
    }

    #[test]
    fn test_builder_custom_type_and_version() {
        let config = FromJsonConfig::builder()
            .default_type_name("CustomType")
            .version(2, 1)
            .build();
        
        assert_eq!(config.default_type_name, "CustomType");
        assert_eq!(config.version, (2, 1));
    }

    #[test]
    fn test_builder_chaining() {
        let config = FromJsonConfig::builder()
            .default_type_name("Entity")
            .version(1, 5)
            .max_depth(500)
            .max_array_size(50_000)
            .max_string_length(5 * 1024 * 1024)
            .max_object_size(2_500)
            .build();
        
        assert_eq!(config.default_type_name, "Entity");
        assert_eq!(config.version, (1, 5));
        assert_eq!(config.max_depth, Some(500));
        assert_eq!(config.max_array_size, Some(50_000));
        assert_eq!(config.max_string_length, Some(5 * 1024 * 1024));
        assert_eq!(config.max_object_size, Some(2_500));
    }

    // ==================== parse_reference tests ====================

    #[test]
    fn test_parse_reference_qualified() {
        let r = parse_reference("@User:123").unwrap();
        assert_eq!(r.type_name, Some("User".to_string()));
        assert_eq!(r.id, "123");
    }

    #[test]
    fn test_parse_reference_local() {
        let r = parse_reference("@123").unwrap();
        assert_eq!(r.type_name, None);
        assert_eq!(r.id, "123");
    }

    #[test]
    fn test_parse_reference_invalid() {
        let result = parse_reference("User:123");
        assert!(result.is_err());
    }

    // ==================== is_tensor_array tests ====================

    #[test]
    fn test_is_tensor_array_numbers() {
        let arr = vec![json!(1), json!(2), json!(3)];
        assert!(is_tensor_array(&arr));
    }

    #[test]
    fn test_is_tensor_array_nested() {
        let arr = vec![json!([1, 2]), json!([3, 4])];
        assert!(is_tensor_array(&arr));
    }

    #[test]
    fn test_is_tensor_array_empty() {
        let arr: Vec<JsonValue> = vec![];
        assert!(!is_tensor_array(&arr));
    }

    #[test]
    fn test_is_tensor_array_with_strings() {
        let arr = vec![json!(1), json!("not a tensor")];
        assert!(!is_tensor_array(&arr));
    }

    #[test]
    fn test_is_tensor_array_with_objects() {
        let arr = vec![json!({"id": 1})];
        assert!(!is_tensor_array(&arr));
    }

    // ==================== is_object_array tests ====================

    #[test]
    fn test_is_object_array_true() {
        let arr = vec![json!({"id": 1}), json!({"id": 2})];
        assert!(is_object_array(&arr));
    }

    #[test]
    fn test_is_object_array_empty() {
        let arr: Vec<JsonValue> = vec![];
        assert!(!is_object_array(&arr));
    }

    #[test]
    fn test_is_object_array_mixed() {
        let arr = vec![json!({"id": 1}), json!(123)];
        assert!(!is_object_array(&arr));
    }

    // ==================== json_array_to_tensor tests ====================

    #[test]
    fn test_json_array_to_tensor_1d() {
        let arr = vec![json!(1.0), json!(2.0), json!(3.0)];
        let config = FromJsonConfig::default();
        let tensor = json_array_to_tensor(&arr, &config, 0).unwrap();
        assert_eq!(tensor.flatten(), vec![1.0, 2.0, 3.0]);
    }

    #[test]
    fn test_json_array_to_tensor_2d() {
        let arr = vec![json!([1.0, 2.0]), json!([3.0, 4.0])];
        let config = FromJsonConfig::default();
        let tensor = json_array_to_tensor(&arr, &config, 0).unwrap();
        assert_eq!(tensor.flatten(), vec![1.0, 2.0, 3.0, 4.0]);
    }

    #[test]
    fn test_json_array_to_tensor_invalid_element() {
        let arr = vec![json!(1.0), json!("not a number")];
        let config = FromJsonConfig::default();
        let result = json_array_to_tensor(&arr, &config, 0);
        assert!(result.is_err());
    }

    // ==================== json_to_value tests ====================

    #[test]
    fn test_json_to_value_null() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&JsonValue::Null, &config).unwrap();
        assert!(matches!(result, Value::Null));
    }

    #[test]
    fn test_json_to_value_bool() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!(true), &config).unwrap();
        assert!(matches!(result, Value::Bool(true)));

        let result = json_to_value(&json!(false), &config).unwrap();
        assert!(matches!(result, Value::Bool(false)));
    }

    #[test]
    fn test_json_to_value_int() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!(42), &config).unwrap();
        assert!(matches!(result, Value::Int(42)));
    }

    #[test]
    fn test_json_to_value_float() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!(3.5), &config).unwrap();
        if let Value::Float(f) = result {
            assert!((f - 3.5).abs() < 0.001);
        } else {
            panic!("Expected Float");
        }
    }

    #[test]
    fn test_json_to_value_string() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!("hello"), &config).unwrap();
        assert!(matches!(result, Value::String(s) if s == "hello"));
    }

    #[test]
    fn test_json_to_value_expression() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!("$(foo)"), &config).unwrap();
        assert!(matches!(result, Value::Expression(_)));
    }

    #[test]
    fn test_json_to_value_tensor() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!([1.0, 2.0, 3.0]), &config).unwrap();
        if let Value::Tensor(t) = result {
            assert_eq!(t.flatten(), vec![1.0, 2.0, 3.0]);
        } else {
            panic!("Expected Tensor");
        }
    }

    #[test]
    fn test_json_to_value_reference() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!({"@ref": "@User:123"}), &config).unwrap();
        if let Value::Reference(r) = result {
            assert_eq!(r.type_name, Some("User".to_string()));
            assert_eq!(r.id, "123");
        } else {
            panic!("Expected Reference");
        }
    }

    #[test]
    fn test_json_to_value_nested_object_error() {
        let config = FromJsonConfig::default();
        let result = json_to_value(&json!({"key": "value"}), &config);
        assert!(result.is_err());
    }

    // ==================== from_json tests ====================

    #[test]
    fn test_from_json_empty_object() {
        let json = "{}";
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        assert!(doc.root.is_empty());
        assert_eq!(doc.version, (1, 0));
    }

    #[test]
    fn test_from_json_simple_scalars() {
        let json = r#"{"name": "test", "count": 42, "active": true}"#;
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        assert!(doc.root.contains_key("name"));
        assert!(doc.root.contains_key("count"));
        assert!(doc.root.contains_key("active"));
    }

    #[test]
    fn test_from_json_nested_object() {
        let json = r#"{"outer": {"inner": "value"}}"#;
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        if let Item::Object(obj) = &doc.root["outer"] {
            assert!(obj.contains_key("inner"));
        } else {
            panic!("Expected Object");
        }
    }

    #[test]
    fn test_from_json_array_of_objects() {
        let json = r#"{"users": [{"id": "1", "name": "Alice"}]}"#;
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        if let Item::List(list) = &doc.root["users"] {
            assert_eq!(list.type_name, "User");
            assert_eq!(list.rows.len(), 1);
        } else {
            panic!("Expected List");
        }
    }

    #[test]
    fn test_from_json_tensor() {
        let json = r#"{"data": [1, 2, 3]}"#;
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        if let Item::Scalar(Value::Tensor(t)) = &doc.root["data"] {
            assert_eq!(t.flatten(), vec![1.0, 2.0, 3.0]);
        } else {
            panic!("Expected Tensor");
        }
    }

    #[test]
    fn test_from_json_invalid_json() {
        let json = "not valid json";
        let config = FromJsonConfig::default();
        let result = from_json(json, &config);
        assert!(result.is_err());
    }

    #[test]
    fn test_from_json_non_object_root() {
        let json = "[1, 2, 3]";
        let config = FromJsonConfig::default();
        let result = from_json(json, &config);
        assert!(result.is_err());
    }

    // ==================== from_json_value tests ====================

    #[test]
    fn test_from_json_value_simple() {
        let value = json!({"key": 42});
        let config = FromJsonConfig::default();
        let doc = from_json_value(&value, &config).unwrap();
        if let Item::Scalar(Value::Int(n)) = &doc.root["key"] {
            assert_eq!(*n, 42);
        } else {
            panic!("Expected Int");
        }
    }

    // ==================== json_value_to_item tests ====================

    #[test]
    fn test_json_value_to_item_null() {
        let config = FromJsonConfig::default();
        let mut structs = BTreeMap::new();
        let mut schema_cache = SchemaCache::new();
        let result = json_value_to_item(&JsonValue::Null, "test", &config, &mut structs, &mut schema_cache, 0).unwrap();
        assert!(matches!(result, Item::Scalar(Value::Null)));
    }

    #[test]
    fn test_json_value_to_item_bool() {
        let config = FromJsonConfig::default();
        let mut structs = BTreeMap::new();
        let mut schema_cache = SchemaCache::new();
        let result = json_value_to_item(&json!(true), "test", &config, &mut structs, &mut schema_cache, 0).unwrap();
        assert!(matches!(result, Item::Scalar(Value::Bool(true))));
    }

    #[test]
    fn test_json_value_to_item_empty_array() {
        let config = FromJsonConfig::default();
        let mut structs = BTreeMap::new();
        let mut schema_cache = SchemaCache::new();
        let result = json_value_to_item(&json!([]), "items", &config, &mut structs, &mut schema_cache, 0).unwrap();
        if let Item::List(list) = result {
            assert!(list.rows.is_empty());
            assert_eq!(list.type_name, "Item");
        } else {
            panic!("Expected List");
        }
    }

    // ==================== Schema inference tests ====================

    #[test]
    fn test_schema_inference_id_first() {
        let json = r#"{"users": [{"name": "Alice", "id": "1", "age": 30}]}"#;
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        if let Item::List(list) = &doc.root["users"] {
            assert_eq!(list.schema[0], "id"); // id should be first
        } else {
            panic!("Expected List");
        }
    }

    #[test]
    fn test_struct_registration() {
        let json = r#"{"users": [{"id": "1"}]}"#;
        let config = FromJsonConfig::default();
        let doc = from_json(json, &config).unwrap();
        assert!(doc.structs.contains_key("User"));
    }

    // ==================== Security limit tests ====================

    #[test]
    fn test_max_depth_exceeded() {
        // Test with custom low limit for faster testing
        // Default is now 10,000 which is too deep to test efficiently
        let json = r#"{"a":1}"#;

        let config = FromJsonConfig {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: Some(0),  // Fail on any value
            max_array_size: Some(100_000),
            max_string_length: Some(10_000_000),
            max_object_size: Some(10_000),
        };

        let result = from_json(json, &config);
        assert!(result.is_err(), "Expected error for depth 0");
        let err_msg = result.unwrap_err().to_string();
        assert!(err_msg.contains("Maximum recursion depth"));
    }

    #[test]
    fn test_max_array_size_exceeded() {
        let config = FromJsonConfig {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: Some(100),
            max_array_size: Some(10), // Small limit for testing
            max_string_length: Some(10_000_000),
            max_object_size: Some(10_000),
        };

        // Create array with 11 elements
        let json = r#"{"items": [1,2,3,4,5,6,7,8,9,10,11]}"#;
        let result = from_json(json, &config);
        assert!(result.is_err());
        let err_msg = result.unwrap_err().to_string();
        assert!(err_msg.contains("Maximum array size"));
    }

    #[test]
    fn test_max_string_length_exceeded() {
        let config = FromJsonConfig {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: Some(100),
            max_array_size: Some(100_000),
            max_string_length: Some(100), // Small limit for testing
            max_object_size: Some(10_000),
        };

        // Create string with 101 characters
        let long_string = "a".repeat(101);
        let json = format!(r#"{{"text": "{}"}}"#, long_string);
        let result = from_json(&json, &config);
        assert!(result.is_err());
        let err_msg = result.unwrap_err().to_string();
        assert!(err_msg.contains("Maximum string length"));
    }

    #[test]
    fn test_max_object_size_exceeded() {
        let config = FromJsonConfig {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: Some(100),
            max_array_size: Some(100_000),
            max_string_length: Some(10_000_000),
            max_object_size: Some(5), // Small limit for testing
        };

        // Create object with 6 keys
        let json = r#"{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6}"#;
        let result = from_json(json, &config);
        assert!(result.is_err());
        let err_msg = result.unwrap_err().to_string();
        assert!(err_msg.contains("Maximum object size"));
    }

    #[test]
    fn test_limits_disabled() {
        let config = FromJsonConfig {
            default_type_name: "Item".to_string(),
            version: (1, 0),
            max_depth: None, // Disabled
            max_array_size: None,
            max_string_length: None,
            max_object_size: None,
        };

        // These would fail with limits enabled
        let long_string = "a".repeat(1000);
        let json = format!(r#"{{"text": "{}"}}"#, long_string);
        let result = from_json(&json, &config);
        assert!(result.is_ok());
    }

    #[test]
    fn test_error_message_quality() {
        let config = FromJsonConfig::default();

        // Test various error types
        let result1 = from_json("not json", &config);
        assert!(result1.unwrap_err().to_string().contains("JSON parse error"));

        let result2 = from_json("[1,2,3]", &config);
        assert!(result2.unwrap_err().to_string().contains("Root must be"));

        let result3 = from_json(r#"{"ref": {"@ref": "bad"}}"#, &config);
        assert!(result3.is_err()); // Invalid reference
    }
}

// ============================================================================
// PARTIAL PARSING IMPLEMENTATION
// ============================================================================

/// Error tolerance strategy for partial parsing
///
/// Determines how the parser should behave when encountering errors.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Default)]
pub enum ErrorTolerance {
    /// Stop on the first error encountered
    #[default]
    StopOnFirst,

    /// Collect up to N errors before stopping
    MaxErrors(usize),

    /// Collect all errors and continue parsing
    CollectAll,

    /// Skip invalid items in arrays/objects and continue
    SkipInvalidItems,
}


/// Location information for an error
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ErrorLocation {
    /// JSON path to the error (e.g., "$.users[2].email")
    pub path: String,

    /// Depth in the JSON structure
    pub depth: usize,
}

impl ErrorLocation {
    fn root() -> Self {
        Self {
            path: "$".to_string(),
            depth: 0,
        }
    }

    fn child(&self, key: &str) -> Self {
        Self {
            path: format!("{}.{}", self.path, key),
            depth: self.depth + 1,
        }
    }

    fn index(&self, idx: usize) -> Self {
        Self {
            path: format!("{}[{}]", self.path, idx),
            depth: self.depth + 1,
        }
    }
}

/// Captured error during partial parsing
#[derive(Debug, Clone)]
pub struct ParseError {
    /// The error that occurred
    pub error: JsonConversionError,

    /// Location where the error occurred
    pub location: ErrorLocation,

    /// Whether this error is fatal (prevents document creation)
    pub is_fatal: bool,
}

impl ParseError {
    fn new(error: JsonConversionError, location: ErrorLocation, is_fatal: bool) -> Self {
        Self {
            error,
            location,
            is_fatal,
        }
    }
}

/// Configuration for partial parsing
#[derive(Debug, Clone)]
#[derive(Default)]
pub struct PartialConfig {
    /// Base configuration for JSON conversion
    pub from_json_config: FromJsonConfig,

    /// Error tolerance strategy
    pub tolerance: ErrorTolerance,

    /// Whether to include partial results even on fatal errors
    pub include_partial_on_fatal: bool,

    /// Replace invalid values with null instead of skipping
    pub replace_invalid_with_null: bool,
}


impl PartialConfig {
    /// Create a new builder for partial parsing configuration
    pub fn builder() -> PartialConfigBuilder {
        PartialConfigBuilder::default()
    }
}

/// Builder for PartialConfig
#[derive(Debug, Clone)]
#[derive(Default)]
pub struct PartialConfigBuilder {
    from_json_config: FromJsonConfig,
    tolerance: ErrorTolerance,
    include_partial_on_fatal: bool,
    replace_invalid_with_null: bool,
}


impl PartialConfigBuilder {
    /// Set the base FromJsonConfig
    pub fn from_json_config(mut self, config: FromJsonConfig) -> Self {
        self.from_json_config = config;
        self
    }

    /// Set the error tolerance strategy
    pub fn tolerance(mut self, tolerance: ErrorTolerance) -> Self {
        self.tolerance = tolerance;
        self
    }

    /// Set whether to include partial results on fatal errors
    pub fn include_partial_on_fatal(mut self, value: bool) -> Self {
        self.include_partial_on_fatal = value;
        self
    }

    /// Set whether to replace invalid values with null
    pub fn replace_invalid_with_null(mut self, value: bool) -> Self {
        self.replace_invalid_with_null = value;
        self
    }

    /// Build the PartialConfig
    pub fn build(self) -> PartialConfig {
        PartialConfig {
            from_json_config: self.from_json_config,
            tolerance: self.tolerance,
            include_partial_on_fatal: self.include_partial_on_fatal,
            replace_invalid_with_null: self.replace_invalid_with_null,
        }
    }
}

/// Result of partial parsing
#[derive(Debug)]
pub struct PartialResult {
    /// Parsed document (if any)
    pub document: Option<Document>,

    /// All errors encountered during parsing
    pub errors: Vec<ParseError>,

    /// Whether parsing stopped early due to error limits
    pub stopped_early: bool,
}

impl PartialResult {
    /// Check if parsing completed successfully without errors
    pub fn is_complete(&self) -> bool {
        self.errors.is_empty() && self.document.is_some()
    }

    /// Check if parsing failed (fatal errors or no document)
    pub fn is_failed(&self) -> bool {
        self.errors.iter().any(|e| e.is_fatal) || self.document.is_none()
    }

    /// Convert to Result type for simpler error handling
    pub fn into_result(self) -> Result<Document, Vec<ParseError>> {
        if self.errors.is_empty() {
            self.document.ok_or_else(Vec::new)
        } else {
            Err(self.errors)
        }
    }
}

/// Error collection context for partial parsing
struct ErrorContext {
    errors: Vec<ParseError>,
    config: PartialConfig,
    stopped: bool,
}

impl ErrorContext {
    fn new(config: PartialConfig) -> Self {
        Self {
            errors: Vec::new(),
            config,
            stopped: false,
        }
    }

    /// Record an error and determine if parsing should continue
    fn record_error(&mut self, error: JsonConversionError, location: ErrorLocation, is_fatal: bool) -> bool {
        if self.stopped {
            return false;
        }

        let parse_error = ParseError::new(error, location, is_fatal);
        self.errors.push(parse_error);

        // Check if we should stop
        let should_stop = match self.config.tolerance {
            ErrorTolerance::StopOnFirst => true,
            ErrorTolerance::MaxErrors(max) => self.errors.len() >= max,
            ErrorTolerance::CollectAll => false,
            ErrorTolerance::SkipInvalidItems => is_fatal,
        };

        if should_stop {
            self.stopped = true;
        }

        !should_stop
    }

    fn should_continue(&self) -> bool {
        !self.stopped
    }
}

/// Parse JSON string with partial error recovery
///
/// This function attempts to parse as much of the JSON as possible,
/// collecting errors instead of failing on the first error.
///
/// # Examples
///
/// ```text
/// use hedl_json::from_json::{partial_parse_json, PartialConfig, ErrorTolerance};
///
/// let json = r#"{"valid": "data", "invalid": ...}"#;
/// let config = PartialConfig::builder()
///     .tolerance(ErrorTolerance::CollectAll)
///     .build();
///
/// let result = partial_parse_json(json, &config);
/// assert!(result.document.is_some());
/// assert!(!result.errors.is_empty());
/// ```
pub fn partial_parse_json(json: &str, config: &PartialConfig) -> PartialResult {
    // Try to parse JSON first
    let value = match serde_json::from_str::<JsonValue>(json) {
        Ok(v) => v,
        Err(e) => {
            // Fatal JSON parsing error
            return PartialResult {
                document: None,
                errors: vec![ParseError::new(
                    JsonConversionError::ParseError(e.to_string()),
                    ErrorLocation::root(),
                    true,
                )],
                stopped_early: false,
            };
        }
    };

    partial_parse_json_value(&value, config)
}

/// Parse serde_json::Value with partial error recovery
pub fn partial_parse_json_value(value: &JsonValue, config: &PartialConfig) -> PartialResult {
    let mut context = ErrorContext::new(config.clone());
    let mut structs = BTreeMap::new();
    let mut schema_cache = SchemaCache::new();

    // Try to parse the root
    let root = match value {
        JsonValue::Object(map) => {
            match partial_json_object_to_root(
                map,
                &config.from_json_config,
                &mut structs,
                &mut schema_cache,
                0,
                &ErrorLocation::root(),
                &mut context,
            ) {
                Ok(root) => Some(root),
                Err(_) => {
                    if config.include_partial_on_fatal {
                        Some(BTreeMap::new())
                    } else {
                        None
                    }
                }
            }
        }
        _ => {
            context.record_error(
                JsonConversionError::InvalidRoot(format!("{:?}", value)),
                ErrorLocation::root(),
                true,
            );
            None
        }
    };

    let document = root.map(|root| Document {
        version: config.from_json_config.version,
        aliases: BTreeMap::new(),
        structs,
        nests: BTreeMap::new(),
        root,
    });

    PartialResult {
        document,
        errors: context.errors,
        stopped_early: context.stopped,
    }
}

/// Partial parsing version of json_object_to_root
fn partial_json_object_to_root(
    map: &Map<String, JsonValue>,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
    location: &ErrorLocation,
    context: &mut ErrorContext,
) -> Result<BTreeMap<String, Item>, JsonConversionError> {
    // Check object size limit
    if let Some(max_size) = config.max_object_size {
        if map.len() > max_size {
            let err = JsonConversionError::MaxObjectSizeExceeded(max_size, map.len());
            context.record_error(err.clone(), location.clone(), false);
            return Err(err);
        }
    }

    let mut result = BTreeMap::new();

    for (key, value) in map {
        if !context.should_continue() {
            break;
        }

        // Skip metadata keys
        if key.starts_with("__") {
            continue;
        }

        let item_location = location.child(key);
        match partial_json_value_to_item(
            value,
            key,
            config,
            structs,
            schema_cache,
            depth,
            &item_location,
            context,
        ) {
            Ok(item) => {
                result.insert(key.clone(), item);
            }
            Err(_) => {
                // Error already recorded in partial_json_value_to_item
                if context.config.replace_invalid_with_null {
                    result.insert(key.clone(), Item::Scalar(Value::Null));
                }
                // Otherwise skip this item
            }
        }
    }

    Ok(result)
}

/// Partial parsing version of json_value_to_item
fn partial_json_value_to_item(
    value: &JsonValue,
    key: &str,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
    location: &ErrorLocation,
    context: &mut ErrorContext,
) -> Result<Item, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            let err = JsonConversionError::MaxDepthExceeded(max_depth);
            context.record_error(err.clone(), location.clone(), false);
            return Err(err);
        }
    }

    match value {
        JsonValue::Null => Ok(Item::Scalar(Value::Null)),
        JsonValue::Bool(b) => Ok(Item::Scalar(Value::Bool(*b))),
        JsonValue::Number(n) => {
            if let Some(i) = n.as_i64() {
                Ok(Item::Scalar(Value::Int(i)))
            } else if let Some(f) = n.as_f64() {
                Ok(Item::Scalar(Value::Float(f)))
            } else {
                let err = JsonConversionError::InvalidNumber(n.to_string());
                context.record_error(err.clone(), location.clone(), false);
                Err(err)
            }
        }
        JsonValue::String(s) => {
            // Check string length limit
            if let Some(max_len) = config.max_string_length {
                if s.len() > max_len {
                    let err = JsonConversionError::MaxStringLengthExceeded(max_len, s.len());
                    context.record_error(err.clone(), location.clone(), false);
                    return Err(err);
                }
            }

            // Check for expression pattern $( ... )
            if s.starts_with("$(") && s.ends_with(')') {
                match parse_expression_token(s) {
                    Ok(expr) => Ok(Item::Scalar(Value::Expression(expr))),
                    Err(e) => {
                        let err = JsonConversionError::InvalidExpression(e.to_string());
                        context.record_error(err.clone(), location.clone(), false);
                        Err(err)
                    }
                }
            } else {
                Ok(Item::Scalar(Value::String(s.clone())))
            }
        }
        JsonValue::Array(arr) => {
            // Check array size limit
            if let Some(max_size) = config.max_array_size {
                if arr.len() > max_size {
                    let err = JsonConversionError::MaxArraySizeExceeded(max_size, arr.len());
                    context.record_error(err.clone(), location.clone(), false);
                    return Err(err);
                }
            }

            // Handle empty arrays
            if arr.is_empty() {
                let type_name = singularize_and_capitalize(key);
                let schema: Vec<String> = DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect();
                let mut list = MatrixList::new(type_name.clone(), schema.clone());
                list.count_hint = Some(0);
                structs.insert(type_name, schema);
                Ok(Item::List(list))
            } else if is_tensor_array(arr) {
                match partial_json_array_to_tensor(arr, config, depth + 1, location, context) {
                    Ok(tensor) => Ok(Item::Scalar(Value::Tensor(tensor))),
                    Err(err) => Err(err),
                }
            } else if is_object_array(arr) {
                match partial_json_array_to_matrix_list(
                    arr,
                    key,
                    config,
                    structs,
                    schema_cache,
                    depth + 1,
                    location,
                    context,
                ) {
                    Ok(list) => Ok(Item::List(list)),
                    Err(err) => Err(err),
                }
            } else {
                // Mixed array - try to convert to tensor
                match partial_json_array_to_tensor(arr, config, depth + 1, location, context) {
                    Ok(tensor) => Ok(Item::Scalar(Value::Tensor(tensor))),
                    Err(err) => Err(err),
                }
            }
        }
        JsonValue::Object(obj) => {
            // Check for special keys
            if let Some(JsonValue::String(r)) = obj.get("@ref") {
                match parse_reference(r) {
                    Ok(reference) => Ok(Item::Scalar(Value::Reference(reference))),
                    Err(e) => {
                        let err = JsonConversionError::InvalidReference(e);
                        context.record_error(err.clone(), location.clone(), false);
                        Err(err)
                    }
                }
            } else {
                // Regular object
                match partial_json_object_to_item_map(
                    obj,
                    config,
                    structs,
                    schema_cache,
                    depth + 1,
                    location,
                    context,
                ) {
                    Ok(item_map) => Ok(Item::Object(item_map)),
                    Err(err) => Err(err),
                }
            }
        }
    }
}

/// Partial parsing version of json_object_to_item_map
fn partial_json_object_to_item_map(
    map: &Map<String, JsonValue>,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
    location: &ErrorLocation,
    context: &mut ErrorContext,
) -> Result<BTreeMap<String, Item>, JsonConversionError> {
    // Check object size limit
    if let Some(max_size) = config.max_object_size {
        if map.len() > max_size {
            let err = JsonConversionError::MaxObjectSizeExceeded(max_size, map.len());
            context.record_error(err.clone(), location.clone(), false);
            return Err(err);
        }
    }

    let mut result = BTreeMap::new();

    for (key, value) in map {
        if !context.should_continue() {
            break;
        }

        if key.starts_with("__") {
            continue;
        }

        let item_location = location.child(key);
        match partial_json_value_to_item(
            value,
            key,
            config,
            structs,
            schema_cache,
            depth,
            &item_location,
            context,
        ) {
            Ok(item) => {
                result.insert(key.clone(), item);
            }
            Err(_) => {
                if context.config.replace_invalid_with_null {
                    result.insert(key.clone(), Item::Scalar(Value::Null));
                }
            }
        }
    }

    Ok(result)
}

/// Partial parsing version of json_array_to_tensor
fn partial_json_array_to_tensor(
    arr: &[JsonValue],
    config: &FromJsonConfig,
    depth: usize,
    location: &ErrorLocation,
    context: &mut ErrorContext,
) -> Result<Tensor, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            let err = JsonConversionError::MaxDepthExceeded(max_depth);
            context.record_error(err.clone(), location.clone(), false);
            return Err(err);
        }
    }

    let mut items = Vec::with_capacity(arr.len());

    for (idx, v) in arr.iter().enumerate() {
        if !context.should_continue() {
            break;
        }

        let elem_location = location.index(idx);
        let tensor = match v {
            JsonValue::Number(n) => {
                match n.as_f64() {
                    Some(f) => Ok(Tensor::Scalar(f)),
                    None => {
                        let err = JsonConversionError::InvalidNumber(n.to_string());
                        context.record_error(err.clone(), elem_location, false);
                        Err(err)
                    }
                }
            }
            JsonValue::Array(nested) => {
                partial_json_array_to_tensor(nested, config, depth + 1, &elem_location, context)
            }
            _ => {
                let err = JsonConversionError::InvalidTensor;
                context.record_error(err.clone(), elem_location, false);
                Err(err)
            }
        };

        match tensor {
            Ok(t) => items.push(t),
            Err(_) => {
                if context.config.replace_invalid_with_null {
                    items.push(Tensor::Scalar(0.0));
                }
                // Otherwise skip this item
            }
        }
    }

    Ok(Tensor::Array(items))
}

/// Partial parsing version of json_array_to_matrix_list
#[allow(clippy::too_many_arguments)]
fn partial_json_array_to_matrix_list(
    arr: &[JsonValue],
    key: &str,
    config: &FromJsonConfig,
    structs: &mut BTreeMap<String, Vec<String>>,
    schema_cache: &mut SchemaCache,
    depth: usize,
    location: &ErrorLocation,
    context: &mut ErrorContext,
) -> Result<MatrixList, JsonConversionError> {
    // Check recursion depth
    if let Some(max_depth) = config.max_depth {
        if depth >= max_depth {
            let err = JsonConversionError::MaxDepthExceeded(max_depth);
            context.record_error(err.clone(), location.clone(), false);
            return Err(err);
        }
    }

    let type_name = singularize_and_capitalize(key);

    // Infer schema from first object
    let schema: Vec<String> = if let Some(JsonValue::Object(first)) = arr.first() {
        if let Some(JsonValue::Array(schema_arr)) = first.get("__hedl_schema") {
            schema_arr
                .iter()
                .filter_map(|v| v.as_str().map(String::from))
                .collect()
        } else {
            let mut cache_key: Vec<String> = first
                .keys()
                .filter(|k| {
                    if k.starts_with("__") {
                        return false;
                    }
                    if let Some(JsonValue::Array(arr)) = first.get(*k) {
                        !is_object_array(arr)
                    } else {
                        true
                    }
                })
                .cloned()
                .collect();
            cache_key.sort();

            if let Some(cached_schema) = schema_cache.get(&cache_key) {
                cached_schema.clone()
            } else {
                let mut keys = cache_key.clone();
                if let Some(pos) = keys.iter().position(|k| k == "id") {
                    keys.remove(pos);
                    keys.insert(0, "id".to_string());
                }
                schema_cache.insert(cache_key, keys.clone());
                keys
            }
        }
    } else {
        DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect()
    };

    let schema = if schema.is_empty() {
        DEFAULT_SCHEMA.iter().map(|s| s.to_string()).collect()
    } else {
        schema
    };

    structs.insert(type_name.clone(), schema.clone());

    let mut rows = Vec::with_capacity(arr.len());

    for (idx, item) in arr.iter().enumerate() {
        if !context.should_continue() {
            break;
        }

        let row_location = location.index(idx);

        if let JsonValue::Object(obj) = item {
            let id = obj
                .get(&schema[0])
                .and_then(|v| v.as_str())
                .unwrap_or("")
                .to_string();

            let mut fields = Vec::with_capacity(schema.len());
            for col in &schema {
                match obj.get(col) {
                    Some(v) => {
                        match partial_json_to_value(v, config, &row_location.child(col), context) {
                            Ok(value) => fields.push(value),
                            Err(_) => {
                                if context.config.replace_invalid_with_null {
                                    fields.push(Value::Null);
                                } else {
                                    fields.push(Value::Null);
                                }
                            }
                        }
                    }
                    None => fields.push(Value::Null),
                }
            }

            // Handle nested children
            let mut children: BTreeMap<String, Vec<Node>> = BTreeMap::new();
            for (child_key, child_value) in obj.iter() {
                if !context.should_continue() {
                    break;
                }

                if let JsonValue::Array(child_arr) = child_value {
                    if is_object_array(child_arr) {
                        let child_location = row_location.child(child_key);
                        match partial_json_array_to_matrix_list(
                            child_arr,
                            child_key,
                            config,
                            structs,
                            schema_cache,
                            depth + 1,
                            &child_location,
                            context,
                        ) {
                            Ok(child_list) => {
                                children.insert(child_key.clone(), child_list.rows);
                            }
                            Err(_) => {
                                // Error already recorded, skip this child
                            }
                        }
                    }
                }
            }

            let node = Node {
                type_name: type_name.clone(),
                id,
                fields,
                children,
                child_count: None,
            };

            rows.push(node);
        } else {
            // Invalid item in array - record error
            let err = JsonConversionError::InvalidRoot("Expected object in array".to_string());
            context.record_error(err, row_location, false);

            // Skip this item based on tolerance
            if context.config.tolerance == ErrorTolerance::SkipInvalidItems {
                continue;
            }
        }
    }

    let count_hint = Some(rows.len());

    Ok(MatrixList {
        type_name,
        schema,
        rows,
        count_hint,
    })
}

/// Partial parsing version of json_to_value
fn partial_json_to_value(
    value: &JsonValue,
    config: &FromJsonConfig,
    location: &ErrorLocation,
    context: &mut ErrorContext,
) -> Result<Value, JsonConversionError> {
    match value {
        JsonValue::Null => Ok(Value::Null),
        JsonValue::Bool(b) => Ok(Value::Bool(*b)),
        JsonValue::Number(n) => {
            if let Some(i) = n.as_i64() {
                Ok(Value::Int(i))
            } else if let Some(f) = n.as_f64() {
                Ok(Value::Float(f))
            } else {
                let err = JsonConversionError::InvalidNumber(n.to_string());
                context.record_error(err.clone(), location.clone(), false);
                Err(err)
            }
        }
        JsonValue::String(s) => {
            // Check string length limit
            if let Some(max_len) = config.max_string_length {
                if s.len() > max_len {
                    let err = JsonConversionError::MaxStringLengthExceeded(max_len, s.len());
                    context.record_error(err.clone(), location.clone(), false);
                    return Err(err);
                }
            }

            // Check for expression pattern
            if s.starts_with("$(") && s.ends_with(')') {
                match parse_expression_token(s) {
                    Ok(expr) => Ok(Value::Expression(expr)),
                    Err(e) => {
                        let err = JsonConversionError::InvalidExpression(e.to_string());
                        context.record_error(err.clone(), location.clone(), false);
                        Err(err)
                    }
                }
            } else {
                Ok(Value::String(s.clone()))
            }
        }
        JsonValue::Array(arr) => {
            // Check array size limit
            if let Some(max_size) = config.max_array_size {
                if arr.len() > max_size {
                    let err = JsonConversionError::MaxArraySizeExceeded(max_size, arr.len());
                    context.record_error(err.clone(), location.clone(), false);
                    return Err(err);
                }
            }

            if is_object_array(arr) {
                Ok(Value::Null) // Children processed separately
            } else if is_tensor_array(arr) {
                match partial_json_array_to_tensor(arr, config, 0, location, context) {
                    Ok(tensor) => Ok(Value::Tensor(tensor)),
                    Err(err) => Err(err),
                }
            } else if arr.is_empty() {
                Ok(Value::Tensor(Tensor::Array(vec![])))
            } else {
                match partial_json_array_to_tensor(arr, config, 0, location, context) {
                    Ok(tensor) => Ok(Value::Tensor(tensor)),
                    Err(err) => Err(err),
                }
            }
        }
        JsonValue::Object(obj) => {
            if let Some(JsonValue::String(r)) = obj.get("@ref") {
                match parse_reference(r) {
                    Ok(reference) => Ok(Value::Reference(reference)),
                    Err(e) => {
                        let err = JsonConversionError::InvalidReference(e);
                        context.record_error(err.clone(), location.clone(), false);
                        Err(err)
                    }
                }
            } else {
                let err = JsonConversionError::NestedObject;
                context.record_error(err.clone(), location.clone(), false);
                Err(err)
            }
        }
    }
}