json_eval_rs/jsoneval/

getters.rs

use super::JSONEval;
use crate::jsoneval::path_utils;
use crate::jsoneval::types::{ResolvedLayoutResult, ReturnFormat};
use crate::time_block;
use crate::utils::clean_float_noise_scalar;
use serde_json::Value;

impl JSONEval {
    /// Check if a field is effectively hidden by checking its condition and all parents
    /// Also checks for $layout.hideLayout.all on parents
    /// Check if a field is effectively hidden by checking its condition and all parents
    /// Also checks for $layout.hideLayout.all on parents
    pub(crate) fn is_effective_hidden(&self, schema_pointer: &str) -> bool {
        self.ensure_layout_resolved();
        let schema_pointer = schema_pointer.trim_start_matches('#');
        if let Ok(state) = self.layout_state.read() {
            if state.layout_hidden_refs.iter().any(|hidden_ref| {
                schema_pointer == hidden_ref
                    || schema_pointer
                        .strip_prefix(hidden_ref)
                        .is_some_and(|suffix| {
                            suffix.starts_with("/properties/") || suffix.starts_with("/items/")
                        })
            }) {
                return true;
            }
        }

        self.is_schema_effective_hidden(schema_pointer)
    }

    /// Check if a field is effectively hidden using pre-acquired layout hidden refs and a local cache.
    pub(crate) fn is_effective_hidden_with_cache(
        &self,
        schema_pointer: &str,
        layout_hidden_refs: &indexmap::IndexSet<String>,
        cache: &mut std::collections::HashMap<String, bool>,
    ) -> bool {
        let schema_pointer = schema_pointer.trim_start_matches('#');
        if let Some(&is_hidden) = cache.get(schema_pointer) {
            return is_hidden;
        }

        if layout_hidden_refs.iter().any(|hidden_ref| {
            schema_pointer == hidden_ref
                || schema_pointer
                    .strip_prefix(hidden_ref)
                    .is_some_and(|suffix| {
                        suffix.starts_with("/properties/") || suffix.starts_with("/items/")
                    })
        }) {
            cache.insert(schema_pointer.to_string(), true);
            return true;
        }

        let is_hidden = self.is_schema_effective_hidden_cached(schema_pointer, cache);
        cache.insert(schema_pointer.to_string(), is_hidden);
        is_hidden
    }

    /// Check if a field is effectively hidden in the schema hierarchy with ancestor caching.
    pub(crate) fn is_schema_effective_hidden_cached(
        &self,
        schema_pointer: &str,
        cache: &mut std::collections::HashMap<String, bool>,
    ) -> bool {
        let schema_pointer = schema_pointer.trim_start_matches('#');
        let mut end = schema_pointer.len();

        loop {
            let current_path = &schema_pointer[..end];

            if let Some(&ancestor_hidden) = cache.get(current_path) {
                if ancestor_hidden {
                    return true;
                }
                break;
            }

            if let Some(schema_node) = self.evaluated_schema.pointer(current_path) {
                if let Value::Object(map) = schema_node {
                    if let Some(Value::Object(condition)) = map.get("condition") {
                        if let Some(Value::Bool(true)) = condition.get("hidden") {
                            cache.insert(current_path.to_string(), true);
                            return true;
                        }
                    }

                    if let Some(Value::Object(layout)) = map.get("$layout") {
                        if let Some(Value::Object(hide_layout)) = layout.get("hideLayout") {
                            if let Some(Value::Bool(true)) = hide_layout.get("all") {
                                cache.insert(current_path.to_string(), true);
                                return true;
                            }
                        }
                    }
                }
            }

            if end == 0 {
                break;
            }

            match schema_pointer[..end].rfind('/') {
                Some(0) | None => {
                    end = 0;
                }
                Some(last_slash) => {
                    end = last_slash;
                    let parent = &schema_pointer[..end];
                    if parent.ends_with("/properties") {
                        end -= "/properties".len();
                    } else if parent.ends_with("/items") {
                        end -= "/items".len();
                    }
                }
            }
        }

        false
    }

    /// Check if a field is effectively hidden in the schema hierarchy (condition.hidden or $layout.hideLayout.all)
    /// without requiring layout resolution.
    pub(crate) fn is_schema_effective_hidden(&self, schema_pointer: &str) -> bool {
        let schema_pointer = schema_pointer.trim_start_matches('#');
        let mut end = schema_pointer.len();

        loop {
            let current_path = &schema_pointer[..end];

            if let Some(schema_node) = self.evaluated_schema.pointer(current_path) {
                if let Value::Object(map) = schema_node {
                    if let Some(Value::Object(condition)) = map.get("condition") {
                        if let Some(Value::Bool(true)) = condition.get("hidden") {
                            return true;
                        }
                    }

                    if let Some(Value::Object(layout)) = map.get("$layout") {
                        if let Some(Value::Object(hide_layout)) = layout.get("hideLayout") {
                            if let Some(Value::Bool(true)) = hide_layout.get("all") {
                                return true;
                            }
                        }
                    }
                }
            }

            if end == 0 {
                break;
            }

            // Move to parent: find last '/' and strip /properties or /items suffixes
            match schema_pointer[..end].rfind('/') {
                Some(0) | None => {
                    end = 0;
                }
                Some(last_slash) => {
                    end = last_slash;
                    let parent = &schema_pointer[..end];
                    if parent.ends_with("/properties") {
                        end -= "/properties".len();
                    } else if parent.ends_with("/items") {
                        end -= "/items".len();
                    }
                }
            }
        }

        false
    }

    /// Check if a field is effectively readonly/disabled using pre-acquired layout disabled refs and a local cache.
    pub(crate) fn is_effective_readonly_with_cache(
        &self,
        schema_pointer: &str,
        layout_disabled_refs: &indexmap::IndexSet<String>,
        cache: &mut std::collections::HashMap<String, bool>,
    ) -> bool {
        let schema_pointer = schema_pointer.trim_start_matches('#');
        if let Some(&is_readonly) = cache.get(schema_pointer) {
            return is_readonly;
        }

        if layout_disabled_refs.iter().any(|disabled_ref| {
            schema_pointer == disabled_ref
                || schema_pointer
                    .strip_prefix(disabled_ref)
                    .is_some_and(|suffix| {
                        suffix.starts_with("/properties/") || suffix.starts_with("/items/")
                    })
        }) {
            cache.insert(schema_pointer.to_string(), true);
            return true;
        }

        let is_readonly = self.is_schema_effective_readonly_cached(schema_pointer, cache);
        cache.insert(schema_pointer.to_string(), is_readonly);
        is_readonly
    }

    /// Check if a field is effectively readonly/disabled in the schema hierarchy with ancestor caching.
    pub(crate) fn is_schema_effective_readonly_cached(
        &self,
        schema_pointer: &str,
        cache: &mut std::collections::HashMap<String, bool>,
    ) -> bool {
        let schema_pointer = schema_pointer.trim_start_matches('#');
        let mut end = schema_pointer.len();

        loop {
            let current_path = &schema_pointer[..end];

            if let Some(&ancestor_readonly) = cache.get(current_path) {
                if ancestor_readonly {
                    return true;
                }
                break;
            }

            if let Some(schema_node) = self.evaluated_schema.pointer(current_path) {
                if let Value::Object(map) = schema_node {
                    if map.get("disabled").and_then(Value::as_bool) == Some(true)
                        || map.get("readonly").and_then(Value::as_bool) == Some(true)
                        || map.get("readOnly").and_then(Value::as_bool) == Some(true)
                        || map.get("$readonly").and_then(Value::as_bool) == Some(true)
                    {
                        cache.insert(current_path.to_string(), true);
                        return true;
                    }

                    if let Some(Value::Object(condition)) = map.get("condition") {
                        if condition.get("disabled").and_then(Value::as_bool) == Some(true)
                            || condition.get("readonly").and_then(Value::as_bool) == Some(true)
                            || condition.get("readOnly").and_then(Value::as_bool) == Some(true)
                        {
                            cache.insert(current_path.to_string(), true);
                            return true;
                        }
                    }
                }
            }

            if end == 0 {
                break;
            }

            match schema_pointer[..end].rfind('/') {
                Some(0) | None => {
                    end = 0;
                }
                Some(last_slash) => {
                    end = last_slash;
                    let parent = &schema_pointer[..end];
                    if parent.ends_with("/properties") {
                        end -= "/properties".len();
                    } else if parent.ends_with("/items") {
                        end -= "/items".len();
                    }
                }
            }
        }

        false
    }

    /// Return whether a schema field appears in any `$layout` element.
    /// Field references are collected once while parsing schema, so this is O(1).
    fn is_mapped_in_any_layout(&self, schema_path: &str) -> bool {
        self.layout_field_refs
            .contains(schema_path.trim_start_matches('#'))
    }

    /// Prune hidden values from data object recursively
    fn prune_hidden_values(&self, data: &mut Value, current_path: &str) {
        if let Value::Object(map) = data {
            // Collect keys to remove to avoid borrow checker issues
            let mut keys_to_remove = Vec::new();

            for (key, value) in map.iter_mut() {
                // Skip special keys
                if key == "$params" || key == "$context" {
                    continue;
                }

                // Construct schema path for this key
                // For root fields: /properties/key (or /key if schema root directly holds keys as in subforms)
                // For nested fields: current_path/properties/key
                let schema_path = if current_path.is_empty() {
                    if self
                        .evaluated_schema
                        .pointer(&format!("/properties/{}", key))
                        .is_some()
                    {
                        format!("/properties/{}", key)
                    } else if self
                        .evaluated_schema
                        .pointer(&format!("/{}", key))
                        .is_some()
                    {
                        format!("/{}", key)
                    } else {
                        format!("/properties/{}", key)
                    }
                } else {
                    format!("{}/properties/{}", current_path, key)
                };

                if self.is_effective_hidden(&schema_path) {
                    keys_to_remove.push(key.clone());
                } else {
                    // Recurse if object
                    if value.is_object() {
                        self.prune_hidden_values(value, &schema_path);
                    }
                }
            }

            // Remove hidden keys
            for key in keys_to_remove {
                map.remove(&key);
            }
        }
    }

    /// Replace any `{"$static_array": "/$table/..."}` and `{"$static_array": "/$params/..."}` markers in `schema_output`
    /// with the actual evaluated array data from `eval_data`.
    ///
    /// By iterating only over tracked `static_arrays`, we replace markers in O(markers) time
    /// instead of requiring an expensive O(schema_nodes) recursive tree walk.
    fn resolve_static_markers_in_value(&self, schema_output: &mut Value) {
        for (static_key, array_arc) in self.static_arrays.iter() {
            // Determine the schema pointer path where this marker was placed
            let schema_path = if static_key.starts_with("/$table") {
                &static_key["/$table".len()..] // e.g. /properties/product_benefit/...
            } else {
                static_key.as_str() // e.g. /$params/references/...
            };

            // Only attempt replacement if the exact path exists in the cloned schema output
            if let Some(target_val) = schema_output.pointer_mut(schema_path) {
                // The actual evaluated array is seamlessly stored right in the map's value
                *target_val = (**array_arc).clone();
            }
        }
    }

    /// Get the evaluated schema (compact — $ref intact, no layout expansion).
    ///
    /// # Returns
    ///
    /// The evaluated schema as a JSON value, with all `$static_array` markers resolved
    /// to their actual evaluated data.
    pub fn get_evaluated_schema(&mut self) -> Value {
        time_block!("get_evaluated_schema()", {
            let mut schema = self.evaluated_schema.clone();
            self.resolve_static_markers_in_value(&mut schema);
            schema
        })
    }

    /// Get layout overlay entries — the delta properties per layout element.
    /// Consumer merges these into compact schema to get fully resolved layout.
    pub fn get_resolved_layout(&mut self) -> ResolvedLayoutResult {
        time_block!("get_resolved_layout()", {
            self.ensure_layout_resolved();
            self.layout_state
                .read()
                .unwrap()
                .cache
                .as_ref()
                .map(|c| (**c).clone())
                .unwrap_or_default()
        })
    }

    /// Get evaluated schema with layout overlays already applied.
    /// Convenience: returns compact schema + overlays merged.
    ///
    /// Two-pass approach to handle nested elements:
    /// 1. First pass: resolve $ref and apply overlay for entries whose target
    ///    path exists in the compact schema (top-level elements).
    /// 2. Second pass: apply overlay-only for entries whose path appears
    ///    after parent $ref resolution (nested elements).
    pub fn get_evaluated_schema_resolved(&mut self) -> Value {
        time_block!("get_evaluated_schema_resolved()", {
            let mut schema = self.get_evaluated_schema_without_params();
            let overlays = self.get_resolved_layout();

            struct ResolveEntry {
                layout_path: String,
                element_idx: usize,
                overlay: indexmap::IndexMap<String, Value>,
            }

            let mut entries: Vec<ResolveEntry> = overlays
                .iter()
                .map(|entry| {
                    let layout_path =
                        path_utils::normalize_to_json_pointer(&entry.layout_path).into_owned();
                    ResolveEntry {
                        layout_path,
                        element_idx: entry.element_idx,
                        overlay: entry.overlay.clone(),
                    }
                })
                .collect();
            drop(overlays);

            // Sort entries shallow-first so parent elements are expanded before their children.
            // Child entries (e.g. layout_path = ".../elements/1/elements") depend on the parent
            // ("…/elements") being resolved first so the nested `elements` array exists in `schema`.
            entries.sort_by(|a, b| {
                let depth_a = a.layout_path.matches('/').count();
                let depth_b = b.layout_path.matches('/').count();
                depth_a
                    .cmp(&depth_b)
                    .then_with(|| a.element_idx.cmp(&b.element_idx))
            });

            // ── Phase 2 (mutable): resolve $ref + apply overlays (parent-first order) ──
            // Entries are sorted shallowest layout_path first, so parent elements are
            // expanded before any child entries that path through them.
            for entry in entries {
                // Resolve $ref from the current (already partially mutated) schema so that
                // parent expansions are visible when we process child entries.
                let resolved_value: Option<Value> = (|| -> Option<Value> {
                    let arr = schema.pointer(&entry.layout_path)?.as_array()?;
                    let element = arr.get(entry.element_idx)?;
                    let ref_str = element.get("$ref")?.as_str()?;

                    let ref_pointer = if ref_str.starts_with('#') || ref_str.starts_with('/') {
                        path_utils::normalize_to_json_pointer(ref_str).into_owned()
                    } else {
                        let schema_pointer = path_utils::dot_notation_to_schema_pointer(ref_str);
                        let normalized =
                            path_utils::normalize_to_json_pointer(&schema_pointer).into_owned();
                        if schema.pointer(&normalized).is_some() {
                            normalized
                        } else {
                            format!("/properties/{}", ref_str.replace('.', "/properties/"))
                        }
                    };

                    let mut resolved = schema.pointer(&ref_pointer)?.clone();

                    // Flatten $layout into top level
                    if let Value::Object(ref mut resolved_map) = resolved {
                        if let Some(Value::Object(layout_obj)) = resolved_map.remove("$layout") {
                            let mut result = layout_obj;
                            for (key, value) in resolved_map.clone().into_iter() {
                                if key != "type" || !result.contains_key("type") {
                                    result.insert(key, value);
                                }
                            }
                            resolved = Value::Object(result);
                        }
                    }

                    Some(resolved)
                })();

                if let Some(Value::Array(arr)) = schema.pointer_mut(&entry.layout_path) {
                    if entry.element_idx < arr.len() {
                        let element = &mut arr[entry.element_idx];

                        // Apply $ref resolution
                        if let Some(resolved) = resolved_value {
                            if let Value::Object(mut resolved_map) = resolved {
                                if let Value::Object(mut map) = element.take() {
                                    map.remove("$ref");
                                    for (key, value) in map {
                                        resolved_map.insert(key, value);
                                    }
                                }
                                *element = Value::Object(resolved_map);
                            } else {
                                *element = resolved;
                            }
                        }

                        // Apply overlay on top
                        if let Value::Object(ref mut map) = element {
                            for (k, v) in &entry.overlay {
                                map.insert(k.clone(), v.clone());
                            }
                        }
                    }
                }
            }

            Self::stamp_property_metadata(&mut schema);
            schema
        })
    }

    /// Stamp every schema property with raw pointer-style dotted metadata.
    fn stamp_property_metadata(schema: &mut Value) {
        fn walk(value: &mut Value, path: &str, parent_hidden: bool) {
            let Some(map) = value.as_object_mut() else {
                return;
            };

            let hidden = parent_hidden
                || map
                    .get("condition")
                    .and_then(Value::as_object)
                    .and_then(|condition| condition.get("hidden"))
                    .is_some_and(|hidden| hidden == &Value::Bool(true));

            if let Some(Value::Object(properties)) = map.get_mut("properties") {
                for (name, property) in properties {
                    let property_path = if path.is_empty() {
                        format!("properties.{}", name)
                    } else {
                        format!("{}.properties.{}", path, name)
                    };
                    if let Value::Object(property_map) = property {
                        property_map.insert(
                            "$fullpath".to_string(),
                            Value::String(property_path.clone()),
                        );
                        property_map.insert("$path".to_string(), Value::String(name.clone()));
                        property_map.insert("$parentHide".to_string(), Value::Bool(hidden));
                    }
                    walk(property, &property_path, hidden);
                }
            }

            for (name, child) in map {
                if name != "properties" && !name.starts_with('$') && child.is_object() {
                    let child_path = if path.is_empty() {
                        name.clone()
                    } else {
                        format!("{}.{}", path, name)
                    };
                    walk(child, &child_path, hidden);
                }
            }
        }

        walk(schema, "", false);
    }

    /// Resolve `$static_array` markers within the subtree rooted at `schema_prefix`.
    ///
    /// Clones only the node at `schema_prefix` from `evaluated_schema`, then iterates
    /// the tracked `static_arrays` list filtering to entries whose schema path is at or
    /// under `schema_prefix`. Only those markers are replaced inside the cloned subtree;
    /// unrelated entries are skipped entirely.
    ///
    /// # Examples
    /// - `schema_prefix = "/$params/references"` → resolves only arrays nested under that key
    /// - `schema_prefix = "/properties/foo/value"` → resolves a single marker if the field itself is one
    pub(crate) fn resolve_static_markers_at_path(&self, schema_prefix: &str) -> Option<Value> {
        // Resolve indexed static-array paths directly.
        for (static_key, array_arc) in self.static_arrays.iter() {
            let schema_path: &str = if static_key.starts_with("/$table") {
                &static_key["/$table".len()..]
            } else {
                static_key.as_str()
            };

            if let Some(relative) = schema_prefix
                .strip_prefix(schema_path)
                .and_then(|relative| relative.strip_prefix('/'))
            {
                return array_arc.pointer(&format!("/{}", relative)).cloned();
            }
        }

        let mut subtree = self.evaluated_schema.pointer(schema_prefix)?.clone();

        // Pre-build "prefix/" once for the starts_with check in the loop
        let prefix_slash = format!("{}/", schema_prefix);

        for (static_key, array_arc) in self.static_arrays.iter() {
            // Derive the absolute schema path the same way resolve_static_markers_in_value does
            let schema_path: &str = if static_key.starts_with("/$table") {
                &static_key["/$table".len()..]
            } else {
                static_key.as_str()
            };

            // Compute the path relative to the subtree root
            let relative: &str = if schema_path == schema_prefix {
                // The subtree root itself is the marker — replace the whole subtree
                ""
            } else if schema_path.starts_with(&prefix_slash) {
                // Strip the prefix: remainder is the sub-path within the cloned subtree
                &schema_path[schema_prefix.len()..]
            } else {
                continue; // Not under the requested path — skip
            };

            if relative.is_empty() {
                subtree = (**array_arc).clone();
            } else if let Some(target) = subtree.pointer_mut(relative) {
                *target = (**array_arc).clone();
            }
        }

        Some(subtree)
    }

    /// Get specific schema value by path, resolving any `$static_array` markers at or
    /// under that path.
    pub fn get_schema_value_by_path(&self, path: &str) -> Option<Value> {
        let pointer_path = path_utils::dot_notation_to_schema_pointer(path);
        self.resolve_static_markers_at_path(pointer_path.trim_start_matches('#'))
    }

    /// Get all schema values (data view).
    ///
    /// Builds a consumer view from current data and evaluated values without mutating
    /// evaluator state. Indexed subforms carry active-item wrappers at their root;
    /// persisting this view would append that wrapper into later form evaluations.
    pub fn get_schema_value(&mut self, include_subforms: Option<bool>) -> Value {
        self.ensure_layout_resolved();
        // Start with current authoritative data from eval_data
        let mut current_data = self.eval_data.data().clone();

        // Ensure it's an object
        if !current_data.is_object() {
            current_data = Value::Object(serde_json::Map::new());
        }

        // Strip $params and $context from data
        if let Some(obj) = current_data.as_object_mut() {
            obj.remove("$params");
            obj.remove("$context");
        }

        // Prune hidden values from current_data (to remove user input in hidden fields)
        self.prune_hidden_values(&mut current_data, "");

        // Override data with values from value evaluations
        // We use value_evaluations which stores the paths of fields with .value
        for eval_key in self.value_evaluations.iter() {
            let clean_key = eval_key.strip_prefix('#').unwrap_or(eval_key);

            // Exclude rules.*.value, options.*.value, and $params
            if clean_key.starts_with("/$params")
                || (clean_key.ends_with("/value")
                    && (clean_key.contains("/rules/") || clean_key.contains("/options/")))
            {
                continue;
            }

            let path = clean_key.replace("/properties", "").replace("/value", "");

            // Check if field is effectively hidden
            // Schema path is clean_key without /value
            let schema_path = clean_key.strip_suffix("/value").unwrap_or(&clean_key);
            if self.is_effective_hidden(schema_path) {
                continue;
            }

            // Resolve static markers at this specific pointer (handles markers at or under this path)
            let value = match self.resolve_static_markers_at_path(clean_key) {
                Some(v) => v,
                None => continue,
            };

            // Parse the path and create nested structure as needed
            let path_parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();

            if path_parts.is_empty() {
                continue;
            }

            // Navigate/create nested structure
            let mut current = &mut current_data;
            for (i, part) in path_parts.iter().enumerate() {
                let is_last = i == path_parts.len() - 1;

                if is_last {
                    // Only disabled calculated fields are library-owned. Editable fields
                    // preserve non-null caller input even when their schema has `$evaluation`.
                    let schema_value = self.schema.pointer(clean_key);
                    let computed_value = schema_value
                        .and_then(Value::as_object)
                        .is_some_and(|value| value.contains_key("$evaluation"));
                    let disabled = self
                        .evaluated_schema
                        .pointer(schema_path)
                        .and_then(Value::as_object)
                        .and_then(|field| field.get("condition"))
                        .and_then(Value::as_object)
                        .and_then(|condition| condition.get("disabled"))
                        .is_some_and(|disabled| disabled == &Value::Bool(true));
                    let computed_disabled =
                        computed_value && (disabled || !self.is_mapped_in_any_layout(schema_path));
                    if let Some(obj) = current.as_object_mut() {
                        let should_update = computed_disabled
                            || match obj.get(*part) {
                                Some(v) => v.is_null(),
                                None => true,
                            };
                        if should_update {
                            obj.insert(
                                (*part).to_string(),
                                crate::utils::clean_float_noise(value.clone()),
                            );
                        }
                    }
                } else {
                    // Ensure current is an object, then navigate/create intermediate objects
                    if let Some(obj) = current.as_object_mut() {
                        if !obj.contains_key(*part) {
                            obj.insert((*part).to_string(), Value::Object(serde_json::Map::new()));
                        }

                        current = obj.get_mut(*part).unwrap();
                    } else {
                        // Skip this path if current is not an object and can't be made into one
                        break;
                    }
                }
            }
        }

        if include_subforms.unwrap_or(false) {
            let subform_keys: Vec<String> = self.subforms.keys().cloned().collect();

            // Sync parent params and static arrays to subforms once before the item loop
            for subform_path in &subform_keys {
                if let Some(subform) = self.subforms.get_mut(subform_path) {
                    if let Some(params) = self.evaluated_schema.pointer("/$params") {
                        if let Some(sub_params) = subform.evaluated_schema.pointer_mut("/$params") {
                            *sub_params = params.clone();
                        }
                    }
                    subform.static_arrays = std::sync::Arc::clone(&self.static_arrays);
                    subform
                        .engine
                        .set_static_arrays(std::sync::Arc::clone(&subform.static_arrays));
                }
            }

            for subform_path in subform_keys {
                let data_ptr = path_utils::schema_path_to_data_pointer(&subform_path);

                let schema_pointer = if subform_path.starts_with("#/") {
                    &subform_path[1..]
                } else if subform_path.starts_with('#') {
                    &subform_path[1..]
                } else {
                    &subform_path
                };

                let original_field_key = subform_path
                    .split('/')
                    .filter(|seg| !seg.is_empty() && *seg != "properties")
                    .last()
                    .unwrap_or(&subform_path)
                    .to_string();

                let root_key = path_utils::get_value_by_pointer(&self.schema, schema_pointer)
                    .and_then(|node| node.get("itemsRootKey"))
                    .and_then(|v| v.as_str())
                    .unwrap_or(&original_field_key)
                    .to_string();

                let item_count = current_data
                    .pointer(&data_ptr)
                    .and_then(Value::as_array)
                    .map(|a| a.len())
                    .unwrap_or(0);

                if item_count > 0 {
                    let full_data = self.eval_data.snapshot_data_clone();
                    let context_value = self
                        .eval_data
                        .data()
                        .get("$context")
                        .cloned()
                        .unwrap_or_else(|| Value::Object(serde_json::Map::new()));

                    let existing_items = current_data
                        .pointer(&data_ptr)
                        .and_then(Value::as_array)
                        .cloned()
                        .unwrap_or_default();

                    let mut new_items = Vec::with_capacity(item_count);

                    for (idx, raw_item) in existing_items.into_iter().enumerate() {
                        let evaluated_item_res = self.with_item_cache_swap(
                            &subform_path,
                            idx,
                            full_data.clone(),
                            context_value.clone(),
                            false,
                            |sf| {
                                sf.evaluate_internal_pre_diffed(None, None)?;
                                if sf.apply_visible_static_defaults_with_dependents(None)? {
                                    sf.evaluate_internal_pre_diffed(None, None)?;
                                }
                                Ok(sf.get_schema_value(Some(true)))
                            },
                        );

                        match evaluated_item_res {
                            Ok(mut val) => {
                                let item_val = val
                                    .as_object_mut()
                                    .and_then(|obj| obj.remove(&root_key))
                                    .unwrap_or(raw_item);
                                new_items.push(item_val);
                            }
                            Err(_) => {
                                new_items.push(raw_item);
                            }
                        }
                    }

                    if let Some(target) = current_data.pointer_mut(&data_ptr) {
                        *target = Value::Array(new_items);
                    }
                }
            }
        }

        crate::utils::clean_float_noise(current_data)
    }

    /// Get all schema values as array of path-value pairs
    /// Returns [{path: "", value: ""}, ...]
    ///
    /// # Returns
    ///
    /// Array of objects containing path (dotted notation) and value pairs from value evaluations
    pub fn get_schema_value_array(&self) -> Value {
        self.ensure_layout_resolved();
        let mut result = Vec::new();

        for eval_key in self.value_evaluations.iter() {
            let clean_key = eval_key.strip_prefix('#').unwrap_or(eval_key);

            // Exclude rules.*.value, options.*.value, and $params
            if clean_key.starts_with("/$params")
                || (clean_key.ends_with("/value")
                    && (clean_key.contains("/rules/") || clean_key.contains("/options/")))
            {
                continue;
            }

            // Check if field is effectively hidden
            let schema_path = clean_key.strip_suffix("/value").unwrap_or(&clean_key);
            if self.is_effective_hidden(schema_path) {
                continue;
            }

            // Convert JSON pointer to dotted notation
            let dotted_path = clean_key
                .replace("/properties", "")
                .replace("/value", "")
                .trim_start_matches('/')
                .replace('/', ".");

            if dotted_path.is_empty() {
                continue;
            }

            // Resolve static markers at this specific pointer (handles markers at or under this path)
            let value = match self.resolve_static_markers_at_path(clean_key) {
                Some(v) => crate::utils::clean_float_noise(v),
                None => continue,
            };

            // Create {path, value} object
            let mut item = serde_json::Map::new();
            item.insert("path".to_string(), Value::String(dotted_path));
            item.insert("value".to_string(), value);
            result.push(Value::Object(item));
        }

        Value::Array(result)
    }

    /// Get all schema values as object with dotted path keys
    /// Returns {path: value, ...}
    ///
    /// # Returns
    ///
    /// Flat object with dotted notation paths as keys and evaluated values
    pub fn get_schema_value_object(&self) -> Value {
        self.ensure_layout_resolved();
        let mut result = serde_json::Map::new();

        for eval_key in self.value_evaluations.iter() {
            let clean_key = eval_key.strip_prefix('#').unwrap_or(eval_key);

            // Exclude rules.*.value, options.*.value, and $params
            if clean_key.starts_with("/$params")
                || (clean_key.ends_with("/value")
                    && (clean_key.contains("/rules/") || clean_key.contains("/options/")))
            {
                continue;
            }

            // Check if field is effectively hidden
            let schema_path = clean_key.strip_suffix("/value").unwrap_or(&clean_key);
            if self.is_effective_hidden(schema_path) {
                continue;
            }

            // Convert JSON pointer to dotted notation
            let dotted_path = clean_key
                .replace("/properties", "")
                .replace("/value", "")
                .trim_start_matches('/')
                .replace('/', ".");

            if dotted_path.is_empty() {
                continue;
            }

            // Resolve static markers at this specific pointer (handles markers at or under this path)
            let value = match self.resolve_static_markers_at_path(clean_key) {
                Some(v) => crate::utils::clean_float_noise(v),
                None => continue,
            };

            result.insert(dotted_path, value);
        }

        Value::Object(result)
    }

    /// Get evaluated schema without $params
    pub fn get_evaluated_schema_without_params(&mut self) -> Value {
        time_block!("get_evaluated_schema_without_params()", {
            let mut schema = if let Value::Object(map) = &self.evaluated_schema {
                let mut filtered = serde_json::Map::with_capacity(map.len().saturating_sub(1));
                for (k, v) in map {
                    if k != "$params" {
                        filtered.insert(k.clone(), v.clone());
                    }
                }
                Value::Object(filtered)
            } else {
                self.evaluated_schema.clone()
            };
            self.resolve_static_markers_in_value(&mut schema);
            schema
        })
    }

    fn process_params_static_arrays(
        &self,
        params_val: &Value,
        with_static_array: bool,
        resolve_tables: bool,
    ) -> Value {
        let mut params = params_val.clone();

        // Evaluated $table arrays in $params are dynamic results and must always be resolved in evaluated params
        if resolve_tables {
            for (static_key, array_arc) in self.static_arrays.iter() {
                if let Some(rel_path) = static_key.strip_prefix("/$table/$params") {
                    if let Some(target) = params.pointer_mut(rel_path) {
                        *target = (**array_arc).clone();
                    }
                }
            }
        }

        if with_static_array {
            for (static_key, array_arc) in self.static_arrays.iter() {
                if let Some(rel_path) = static_key.strip_prefix("/$params") {
                    if let Some(target) = params.pointer_mut(rel_path) {
                        *target = (**array_arc).clone();
                    }
                }
            }
        } else {
            Self::strip_static_array_markers(&mut params);
        }
        params
    }

    fn strip_static_array_markers(val: &mut Value) {
        match val {
            Value::Object(map) => {
                map.retain(|_, v| {
                    if let Value::Object(child_map) = v {
                        if let Some(Value::String(marker_path)) = child_map.get("$static_array") {
                            !marker_path.starts_with("/$params")
                        } else {
                            !child_map.contains_key("$static_array")
                        }
                    } else {
                        true
                    }
                });
                for v in map.values_mut() {
                    Self::strip_static_array_markers(v);
                }
            }
            Value::Array(arr) => {
                arr.retain(|v| {
                    if let Value::Object(child_map) = v {
                        if let Some(Value::String(marker_path)) = child_map.get("$static_array") {
                            !marker_path.starts_with("/$params")
                        } else {
                            !child_map.contains_key("$static_array")
                        }
                    } else {
                        true
                    }
                });
                for v in arr.iter_mut() {
                    Self::strip_static_array_markers(v);
                }
            }
            _ => {}
        }
    }

    /// Get plain `$params` from the original schema (without static array data).
    pub fn get_plain_params(&self) -> Option<Value> {
        let raw_params = self.schema.get("$params")?;
        Some(self.process_params_static_arrays(raw_params, false, false))
    }

    /// Get evaluated `$params` from `evaluated_schema`.
    ///
    /// # Arguments
    /// * `with_static_array` - If true, static arrays extracted to `static_arrays` are resolved
    ///   back into `$params`. If false, static array keys are omitted/stripped.
    pub fn get_evaluated_params(&mut self, with_static_array: bool) -> Option<Value> {
        let raw_params = self.evaluated_schema.get("$params")?;
        Some(self.process_params_static_arrays(raw_params, with_static_array, true))
    }

    /// Get evaluated schema as MessagePack bytes (compact, without $layout resolution)
    pub fn get_evaluated_schema_msgpack(&mut self) -> Result<Vec<u8>, String> {
        let schema = self.get_evaluated_schema();
        rmp_serde::to_vec(&schema).map_err(|e| format!("MessagePack serialization failed: {}", e))
    }

    /// Get layout-resolved evaluated schema as MessagePack bytes.
    ///
    /// Reuses `get_evaluated_schema_resolved`, which omits `$params` and merges
    /// resolved `$layout` overlays.
    pub fn get_evaluated_schema_resolved_msgpack(&mut self) -> Result<Vec<u8>, String> {
        let schema = self.get_evaluated_schema_resolved();
        rmp_serde::to_vec(&schema).map_err(|e| format!("MessagePack serialization failed: {}", e))
    }

    /// Get value from evaluated schema by path
    pub fn get_evaluated_schema_by_path(&mut self, path: &str) -> Option<Value> {
        self.get_schema_value_by_path(path)
    }

    /// Get evaluated schema parts by multiple paths
    pub fn get_evaluated_schema_by_paths(
        &mut self,
        paths: &[String],
        format: Option<ReturnFormat>,
    ) -> Value {
        match format.unwrap_or(ReturnFormat::Nested) {
            ReturnFormat::Nested => {
                let mut result = Value::Object(serde_json::Map::new());
                for path in paths {
                    if let Some(val) = self.get_schema_value_by_path(path) {
                        // Insert into result object at proper path nesting
                        Self::insert_at_path(&mut result, path, val);
                    }
                }
                result
            }
            ReturnFormat::Flat => {
                let mut result = serde_json::Map::new();
                for path in paths {
                    if let Some(val) = self.get_schema_value_by_path(path) {
                        result.insert(path.clone(), val);
                    }
                }
                Value::Object(result)
            }
            ReturnFormat::Array => {
                let mut result = Vec::new();
                for path in paths {
                    if let Some(val) = self.get_schema_value_by_path(path) {
                        result.push(val);
                    } else {
                        result.push(Value::Null);
                    }
                }
                Value::Array(result)
            }
        }
    }

    /// Get original (unevaluated) schema by path
    pub fn get_schema_by_path(&self, path: &str) -> Option<Value> {
        let pointer_path = path_utils::dot_notation_to_schema_pointer(path);
        self.schema
            .pointer(&pointer_path.trim_start_matches('#'))
            .cloned()
    }

    /// Get original schema by multiple paths
    pub fn get_schema_by_paths(&self, paths: &[String], format: Option<ReturnFormat>) -> Value {
        match format.unwrap_or(ReturnFormat::Nested) {
            ReturnFormat::Nested => {
                let mut result = Value::Object(serde_json::Map::new());
                for path in paths {
                    if let Some(val) = self.get_schema_by_path(path) {
                        Self::insert_at_path(&mut result, path, val);
                    }
                }
                result
            }
            ReturnFormat::Flat => {
                let mut result = serde_json::Map::new();
                for path in paths {
                    if let Some(val) = self.get_schema_by_path(path) {
                        result.insert(path.clone(), val);
                    }
                }
                Value::Object(result)
            }
            ReturnFormat::Array => {
                let mut result = Vec::new();
                for path in paths {
                    if let Some(val) = self.get_schema_by_path(path) {
                        result.push(val);
                    } else {
                        result.push(Value::Null);
                    }
                }
                Value::Array(result)
            }
        }
    }

    /// Helper to insert value into nested object at dotted path
    pub(crate) fn insert_at_path(root: &mut Value, path: &str, value: Value) {
        let parts: Vec<&str> = path.split('.').collect();
        let mut current = root;

        for (i, part) in parts.iter().enumerate() {
            if i == parts.len() - 1 {
                // Last part - set value
                if let Value::Object(map) = current {
                    map.insert(part.to_string(), value);
                    return; // Done
                }
            } else {
                // Intermediate part - traverse or create
                // We need to temporarily take the value or use raw pointer manipulation?
                // serde_json pointer is read-only or requires mutable reference

                if !current.is_object() {
                    *current = Value::Object(serde_json::Map::new());
                }

                if let Value::Object(map) = current {
                    if !map.contains_key(*part) {
                        map.insert(part.to_string(), Value::Object(serde_json::Map::new()));
                    }
                    current = map.get_mut(*part).unwrap();
                }
            }
        }
    }

    /// Flatten a nested object key-value pair to dotted keys
    pub fn flatten_object(
        prefix: &str,
        value: &Value,
        result: &mut serde_json::Map<String, Value>,
    ) {
        match value {
            Value::Object(map) => {
                for (k, v) in map {
                    let new_key = if prefix.is_empty() {
                        k.clone()
                    } else {
                        format!("{}.{}", prefix, k)
                    };
                    Self::flatten_object(&new_key, v, result);
                }
            }
            _ => {
                result.insert(prefix.to_string(), value.clone());
            }
        }
    }

    pub fn convert_to_format(value: Value, format: ReturnFormat) -> Value {
        match format {
            ReturnFormat::Nested => value,
            ReturnFormat::Flat => {
                let mut result = serde_json::Map::new();
                Self::flatten_object("", &value, &mut result);
                Value::Object(result)
            }
            ReturnFormat::Array => {
                if let Value::Object(map) = value {
                    Value::Array(map.values().cloned().collect())
                } else if let Value::Array(arr) = value {
                    Value::Array(arr)
                } else {
                    Value::Array(vec![value])
                }
            }
        }
    }

    /// Evaluate and return the options for a specific field on demand.
    ///
    /// Accepts dotted notation (`form.occupation`), JSON pointer
    /// (`/properties/form/properties/occupation`), or schema ref
    /// (`#/properties/form/properties/occupation`).
    ///
    /// Returns `None` when the field does not have an `options` key.
    /// Returns the resolved options value (array, URL string, or null) otherwise.
    pub fn get_field_options(&mut self, field_path: &str) -> Option<Value> {
        // Normalize the input to a schema pointer (e.g. #/properties/form/properties/occupation)
        let schema_ptr = if field_path.starts_with('#') || field_path.starts_with('/') {
            path_utils::normalize_to_json_pointer(field_path).into_owned()
        } else {
            path_utils::dot_notation_to_schema_pointer(field_path)
        };

        // Build the JSON pointer path to the /options node (strip leading # for serde pointer())
        let options_schema_key = format!("{}/options", schema_ptr);
        let options_pointer =
            path_utils::normalize_to_json_pointer(&options_schema_key).into_owned();

        // Check if the options node exists in the evaluated schema
        let options_node = self.evaluated_schema.pointer(&options_pointer)?.clone();

        // If the options node is an object with $evaluation, evaluate it now (deferred)
        if let Value::Object(ref map) = options_node {
            if map.contains_key("$evaluation") {
                let eval_key = options_schema_key.clone();

                if let Some(logic_id) = self.evaluations.get(&eval_key).copied() {
                    let snap = self.eval_data.snapshot_data();
                    if let Ok(result) = self.engine.run(&logic_id, &*snap) {
                        let cleaned = clean_float_noise_scalar(result);
                        if let Some(node) = self.evaluated_schema.pointer_mut(&options_pointer) {
                            *node = cleaned.clone();
                        }
                        return Some(cleaned);
                    }
                }
                // No compiled logic found — options cannot be resolved
                return None;
            }
        }

        // Check options_templates for a URL template at this field's options/url path
        let url_pointer =
            path_utils::normalize_to_json_pointer(&format!("{}/options/url", schema_ptr))
                .into_owned();

        let templates = self.options_templates.clone();
        for (tmpl_url_path, tmpl_str, tmpl_params_path) in templates.iter() {
            if *tmpl_url_path == url_pointer {
                if let Some(params) = self.evaluated_schema.pointer(tmpl_params_path) {
                    let params = params.clone();
                    if let Ok(resolved_url) = self.evaluate_template(tmpl_str, &params) {
                        if let Some(target) = self.evaluated_schema.pointer_mut(&url_pointer) {
                            *target = Value::String(resolved_url);
                        }
                        return self.evaluated_schema.pointer(&options_pointer).cloned();
                    }
                }
                break;
            }
        }

        // Static options (already-evaluated array or plain value)
        Some(options_node)
    }
}