datafold 0.1.55

//! Structure analysis utilities for JSON ingestion
//!
//! This module provides utilities to analyze JSON structure and create supersets
//! that capture all possible fields across multiple objects.

use serde_json::Value;
use std::collections::HashMap;

/// Analyzes JSON structure and creates a superset representation
/// that includes all fields found across all top-level elements
pub struct StructureAnalyzer;

impl StructureAnalyzer {
    /// Analyze JSON data and create a superset structure
    ///
    /// For arrays, this loops through all elements and creates a superset
    /// that includes all fields found across all objects.
    /// For single objects, it returns the object as-is.
    ///
    /// # Arguments
    /// * `json_data` - The JSON data to analyze
    ///
    /// # Returns
    /// * `Value` - A superset structure representing all possible fields
    ///
    /// # Examples
    ///
    /// ```rust
    /// use serde_json::json;
    /// use datafold::ingestion::structure_analyzer::StructureAnalyzer;
    ///
    /// let data = json!([
    ///     {"name": "Alice", "age": 30},
    ///     {"name": "Bob", "email": "bob@example.com"}
    /// ]);
    ///
    /// let superset = StructureAnalyzer::create_superset_structure(&data);
    /// // Result: {"name": "string", "age": "number", "email": "string"}
    /// ```
    pub fn create_superset_structure(json_data: &Value) -> Value {
        match json_data {
            Value::Array(array) => {
                if array.is_empty() {
                    return Value::Object(serde_json::Map::new());
                }

                // Create flattened path-based structure
                let mut path_types: HashMap<String, Vec<String>> = HashMap::new();

                for item in array {
                    if let Some(obj) = item.as_object() {
                        Self::flatten_object(obj, &mut path_types, String::new());
                    }
                }

                // Create superset with flattened paths
                let mut superset = serde_json::Map::new();
                for (path, types) in path_types {
                    let representative_type = Self::get_representative_type(&types);
                    superset.insert(path, Value::String(representative_type));
                }

                Value::Object(superset)
            }
            Value::Object(obj) => {
                // For single objects, use flattened path structure
                let mut path_types: HashMap<String, Vec<String>> = HashMap::new();
                Self::flatten_object(obj, &mut path_types, String::new());

                // Create superset with flattened paths
                let mut superset = serde_json::Map::new();
                for (path, types) in path_types {
                    let representative_type = Self::get_representative_type(&types);
                    superset.insert(path, Value::String(representative_type));
                }

                Value::Object(superset)
            }
            _ => {
                // For primitive values, return a simple type representation
                Value::Object({
                    let mut map = serde_json::Map::new();
                    map.insert(
                        "value".to_string(),
                        Value::String(Self::get_value_type(json_data)),
                    );
                    map
                })
            }
        }
    }

    /// Flatten an object into dot-separated paths
    fn flatten_object(
        obj: &serde_json::Map<String, Value>,
        path_types: &mut HashMap<String, Vec<String>>,
        current_path: String,
    ) {
        for (key, value) in obj {
            let field_path = if current_path.is_empty() {
                key.clone()
            } else {
                format!("{}.{}", current_path, key)
            };

            match value {
                Value::Object(nested_obj) => {
                    // Recursively flatten nested objects
                    Self::flatten_object(nested_obj, path_types, field_path);
                }
                Value::Array(array) => {
                    // For arrays, analyze the structure of elements
                    if !array.is_empty() {
                        // Check if array contains objects
                        if array.iter().all(|item| item.is_object()) {
                            // Array of objects - flatten each object and add array notation
                            for (i, item) in array.iter().enumerate() {
                                if let Some(obj) = item.as_object() {
                                    let array_path = format!("{}[{}]", field_path, i);
                                    Self::flatten_object(obj, path_types, array_path);
                                }
                            }
                        } else {
                            // Array of primitives - just record as array type
                            let element_type = Self::get_value_type(&array[0]);
                            path_types
                                .entry(field_path)
                                .or_default()
                                .push(format!("array[{}]", element_type));
                        }
                    } else {
                        // Empty array
                        path_types
                            .entry(field_path)
                            .or_default()
                            .push("array".to_string());
                    }
                }
                _ => {
                    // Primitive value
                    let type_name = Self::get_value_type(value);
                    path_types.entry(field_path).or_default().push(type_name);
                }
            }
        }
    }

    /// Get the JSON type of a value
    fn get_value_type(value: &Value) -> String {
        match value {
            Value::Null => "null".to_string(),
            Value::Bool(_) => "boolean".to_string(),
            Value::Number(_) => "number".to_string(),
            Value::String(_) => "string".to_string(),
            Value::Array(_) => "array".to_string(),
            Value::Object(_) => "object".to_string(),
        }
    }

    /// Get the most representative type from a list of types
    ///
    /// This handles cases where a field has different types across objects.
    /// The priority is: object > array > string > number > boolean > null
    fn get_representative_type(types: &[String]) -> String {
        if types.is_empty() {
            return "null".to_string();
        }

        // Count occurrences of each type
        let mut type_counts: HashMap<String, usize> = HashMap::new();
        for type_name in types {
            *type_counts.entry(type_name.clone()).or_insert(0) += 1;
        }

        // Priority order for type selection
        let priority_order = vec!["object", "array", "string", "number", "boolean", "null"];

        // Find the highest priority type that exists
        for priority_type in priority_order {
            if type_counts.contains_key(priority_type) {
                return priority_type.to_string();
            }
        }

        // Fallback to the most common type
        type_counts
            .into_iter()
            .max_by_key(|(_, count)| *count)
            .map(|(type_name, _)| type_name)
            .unwrap_or_else(|| "null".to_string())
    }

    /// Get statistics about the structure analysis
    ///
    /// Returns information about the number of elements analyzed,
    /// unique fields found, and type variations.
    pub fn get_analysis_stats(json_data: &Value) -> StructureStats {
        match json_data {
            Value::Array(array) => {
                let mut field_counts: HashMap<String, usize> = HashMap::new();
                let mut type_variations: HashMap<String, HashMap<String, usize>> = HashMap::new();

                for item in array {
                    if let Some(obj) = item.as_object() {
                        for (key, value) in obj {
                            let type_name = Self::get_value_type(value);

                            // Count field occurrences
                            *field_counts.entry(key.clone()).or_insert(0) += 1;

                            // Track type variations per field
                            type_variations
                                .entry(key.clone())
                                .or_default()
                                .entry(type_name)
                                .and_modify(|count| *count += 1)
                                .or_insert(1);
                        }
                    }
                }

                StructureStats {
                    total_elements: array.len(),
                    unique_fields: field_counts.len(),
                    field_counts,
                    type_variations,
                }
            }
            Value::Object(obj) => {
                let mut field_counts: HashMap<String, usize> = HashMap::new();
                let mut type_variations: HashMap<String, HashMap<String, usize>> = HashMap::new();

                for (key, value) in obj {
                    let type_name = Self::get_value_type(value);
                    field_counts.insert(key.clone(), 1);
                    type_variations.insert(key.clone(), {
                        let mut map = HashMap::new();
                        map.insert(type_name, 1);
                        map
                    });
                }

                StructureStats {
                    total_elements: 1,
                    unique_fields: obj.len(),
                    field_counts,
                    type_variations,
                }
            }
            _ => StructureStats {
                total_elements: 1,
                unique_fields: 1,
                field_counts: {
                    let mut map = HashMap::new();
                    map.insert("value".to_string(), 1);
                    map
                },
                type_variations: {
                    let mut map = HashMap::new();
                    map.insert("value".to_string(), {
                        let mut type_map = HashMap::new();
                        type_map.insert(Self::get_value_type(json_data), 1);
                        type_map
                    });
                    map
                },
            },
        }
    }
}

/// Statistics about structure analysis
#[derive(Debug, Clone)]
pub struct StructureStats {
    /// Total number of elements analyzed
    pub total_elements: usize,
    /// Number of unique fields found
    pub unique_fields: usize,
    /// Count of occurrences for each field
    pub field_counts: HashMap<String, usize>,
    /// Type variations for each field
    pub type_variations: HashMap<String, HashMap<String, usize>>,
}

impl StructureStats {
    /// Get fields that appear in all elements (100% coverage)
    pub fn get_common_fields(&self) -> Vec<String> {
        self.field_counts
            .iter()
            .filter(|(_, &count)| count == self.total_elements)
            .map(|(field, _)| field.clone())
            .collect()
    }

    /// Get fields that appear in some but not all elements (partial coverage)
    pub fn get_partial_fields(&self) -> Vec<String> {
        self.field_counts
            .iter()
            .filter(|(_, &count)| count > 0 && count < self.total_elements)
            .map(|(field, _)| field.clone())
            .collect()
    }

    /// Get fields with type variations
    pub fn get_fields_with_type_variations(&self) -> Vec<String> {
        self.type_variations
            .iter()
            .filter(|(_, variations)| variations.len() > 1)
            .map(|(field, _)| field.clone())
            .collect()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn test_create_superset_structure_array() {
        let data = json!([
            {"name": "Alice", "age": 30},
            {"name": "Bob", "email": "bob@example.com"},
            {"name": "Charlie", "age": 25, "email": "charlie@example.com"}
        ]);

        let superset = StructureAnalyzer::create_superset_structure(&data);

        assert!(superset.is_object());
        let obj = superset.as_object().unwrap();
        assert!(obj.contains_key("name"));
        assert!(obj.contains_key("age"));
        assert!(obj.contains_key("email"));
        assert_eq!(obj["name"], "string");
        assert_eq!(obj["age"], "number");
        assert_eq!(obj["email"], "string");
    }

    #[test]
    fn test_create_superset_structure_single_object() {
        let data = json!({"name": "Alice", "age": 30});

        let superset = StructureAnalyzer::create_superset_structure(&data);

        assert!(superset.is_object());
        let obj = superset.as_object().unwrap();
        assert_eq!(obj.len(), 2);
        assert_eq!(obj["name"], "string");
        assert_eq!(obj["age"], "number");
    }

    #[test]
    fn test_create_superset_structure_empty_array() {
        let data = json!([]);

        let superset = StructureAnalyzer::create_superset_structure(&data);

        assert!(superset.is_object());
        assert!(superset.as_object().unwrap().is_empty());
    }

    #[test]
    fn test_type_variations() {
        let data = json!([
            {"id": 1, "name": "Alice"},
            {"id": "2", "name": "Bob"},
            {"id": 3, "name": "Charlie"}
        ]);

        let superset = StructureAnalyzer::create_superset_structure(&data);
        let obj = superset.as_object().unwrap();

        // Should choose "string" as representative type for "id" field (higher priority than number)
        assert_eq!(obj["id"], "string");
        assert_eq!(obj["name"], "string");
    }

    #[test]
    fn test_nested_structure_detection() {
        let data = json!([
            {"name": "Alice", "profile": {"age": 30, "department": "Engineering"}, "tags": ["senior", "backend"]},
            {"name": "Bob", "profile": {"email": "bob@example.com", "role": "Manager"}, "tags": ["lead"]},
            {"name": "Charlie", "profile": {"age": 25, "email": "charlie@example.com", "department": "Marketing"}, "tags": ["junior", "frontend"]}
        ]);

        let superset = StructureAnalyzer::create_superset_structure(&data);
        let obj = superset.as_object().unwrap();

        // Debug: print the actual structure
        println!(
            "Actual superset structure: {}",
            serde_json::to_string_pretty(&superset).unwrap()
        );

        // Check top-level fields
        assert_eq!(obj["name"], "string");

        // Check flattened path structure (new approach)
        assert!(obj.contains_key("profile.age"));
        assert!(obj.contains_key("profile.department"));
        assert!(obj.contains_key("profile.email"));
        assert!(obj.contains_key("profile.role"));
        assert_eq!(obj["profile.age"], "number");
        assert_eq!(obj["profile.department"], "string");
        assert_eq!(obj["profile.email"], "string");
        assert_eq!(obj["profile.role"], "string");

        // Check array structure
        assert_eq!(obj["tags"], "array[string]");
    }
}