webpage_quality_analyzer 1.0.2

//! Optimized JSON Serialization Module
//!
//! This module provides high-performance JSON serialization capabilities with streaming
//! support for large result sets, memory-efficient serialization, and optimized formats
//! for different use cases.

use crate::models::models::PageQualityReport;
use serde::Serialize;
use serde_json::{self, Value};
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{BufWriter, Write};

/// Optimized serialization options for different use cases
#[derive(Debug, Clone)]
pub struct SerializationOptions {
    /// Use compact format (no pretty printing)
    pub compact: bool,
    /// Omit null/empty fields to reduce size
    pub skip_empty_fields: bool,
    /// Include only essential metrics for minimal output
    pub minimal_output: bool,
    /// Use streaming for large datasets
    pub streaming: bool,
    /// Buffer size for streaming (bytes)
    pub buffer_size: usize,
}

impl Default for SerializationOptions {
    fn default() -> Self {
        Self {
            compact: false,
            skip_empty_fields: false,
            minimal_output: false,
            streaming: false,
            buffer_size: 8192, // 8KB default buffer
        }
    }
}

impl SerializationOptions {
    /// Create options for batch processing (optimized for performance)
    pub fn batch_processing() -> Self {
        Self {
            compact: true,
            skip_empty_fields: true,
            minimal_output: false,
            streaming: true,
            buffer_size: 32768, // 32KB buffer for batch
        }
    }

    /// Create options for minimal output (only essential data)
    pub fn minimal() -> Self {
        Self {
            compact: true,
            skip_empty_fields: true,
            minimal_output: true,
            streaming: false,
            buffer_size: 4096,
        }
    }

    /// Create options for ultra-high-throughput processing (100+ pages/sec)
    pub fn ultra_high_throughput() -> Self {
        Self {
            compact: true,
            skip_empty_fields: true,
            minimal_output: true,
            streaming: true,
            buffer_size: 65536, // 64KB buffer for maximum throughput
        }
    }
}

/// Minimal version of PageQualityReport for reduced serialization size
#[derive(Debug, Serialize)]
pub struct MinimalPageQualityReport {
    pub url: String,
    pub score: f32,
    pub verdict: String,
    /// Only core metrics
    pub metrics: MinimalPageMetrics,
}

#[derive(Debug, Serialize)]
pub struct MinimalPageMetrics {
    pub word_count: usize,
    pub readability_score: f32,
    pub technical_score: f32,
    pub seo_score: f32,
    pub accessibility_score: f32,
}

/// Advanced field selection for customizing report output
///
/// This struct allows fine-grained control over which fields are included or excluded
/// from the serialized report, supporting both individual fields and entire sections.
///
/// # Field Path Format
///
/// Field paths use dot notation to specify nested fields:
/// - `"url"` - top-level field
/// - `"metrics.content.word_count"` - nested field
/// - `"processed_document.headings"` - array field
///
/// # Priority Rules
///
/// 1. `exclude_fields` takes precedence over `include_fields` for specific paths
/// 2. `exclude_sections` takes precedence over `include_sections` for section-level filtering
/// 3. Field-level rules take precedence over section-level rules
///
/// # Examples
///
/// ```rust
/// use webpage_quality_analyzer::utils::json_optimizer::FieldSelector;
///
/// // Include only specific fields
/// let selector = FieldSelector::builder()
///     .include_fields(["url", "score", "metrics.content.word_count"])
///     .build();
///
/// // Exclude sensitive or large data
/// let selector = FieldSelector::builder()
///     .exclude_fields(["metadata.og_tags", "processed_document.content_chunks"])
///     .build();
///
/// // Mix section and field-level control
/// let selector = FieldSelector::builder()
///     .include_sections(["metrics"])
///     .exclude_fields(["metrics.html_analysis.accessibility"])
///     .build();
/// ```
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FieldSelector {
    include_fields: Option<HashSet<String>>,
    exclude_fields: Option<HashSet<String>>,
    include_sections: Option<HashSet<String>>,
    exclude_sections: Option<HashSet<String>>,
}

impl Default for FieldSelector {
    fn default() -> Self {
        Self {
            include_fields: None,
            exclude_fields: None,
            include_sections: None,
            exclude_sections: None,
        }
    }
}

impl FieldSelector {
    /// Create a new empty field selector (includes all fields by default)
    pub fn new() -> Self {
        Self::default()
    }

    /// Create a builder for fluent configuration
    pub fn builder() -> FieldSelectorBuilder {
        FieldSelectorBuilder::new()
    }

    /// Check if a field path should be included based on the selection rules
    ///
    /// # Arguments
    ///
    /// * `path` - The dot-notation path to the field (e.g., "metrics.content.word_count")
    ///
    /// # Returns
    ///
    /// `true` if the field should be included, `false` otherwise
    pub fn should_include_field(&self, path: &str) -> bool {
        // Extract section (first part of path)
        let section = path.split('.').next().unwrap_or(path);

        // Rule 1: Check field-level exclusions first (highest priority)
        if let Some(ref exclude_fields) = self.exclude_fields {
            if exclude_fields.contains(path) {
                return false;
            }
        }

        // Rule 2: Check section-level exclusions
        if let Some(ref exclude_sections) = self.exclude_sections {
            if exclude_sections.contains(section) {
                return false;
            }
        }

        // Rule 3: Check field-level inclusions
        if let Some(ref include_fields) = self.include_fields {
            if include_fields.contains(path) {
                return true;
            }
        }

        // Rule 4: Check section-level inclusions
        if let Some(ref include_sections) = self.include_sections {
            if include_sections.contains(section) {
                return true;
            }
        }

        // Rule 5: Default behavior
        // If no include rules are specified, include by default
        // If include rules are specified, exclude by default unless explicitly included
        let has_include_rules = self.include_fields.is_some() || self.include_sections.is_some();
        !has_include_rules
    }

    /// Get a summary of the selection rules for debugging
    pub fn summary(&self) -> String {
        let mut parts = Vec::new();

        if let Some(ref fields) = self.include_fields {
            parts.push(format!("Include fields: {}", fields.len()));
        }

        if let Some(ref fields) = self.exclude_fields {
            parts.push(format!("Exclude fields: {}", fields.len()));
        }

        if let Some(ref sections) = self.include_sections {
            parts.push(format!("Include sections: {}", sections.len()));
        }

        if let Some(ref sections) = self.exclude_sections {
            parts.push(format!("Exclude sections: {}", sections.len()));
        }

        if parts.is_empty() {
            "No filtering rules (include all)".to_string()
        } else {
            parts.join(", ")
        }
    }
}

/// Builder for creating `FieldSelector` instances with fluent API
#[derive(Debug, Default)]
pub struct FieldSelectorBuilder {
    include_fields: HashSet<String>,
    exclude_fields: HashSet<String>,
    include_sections: HashSet<String>,
    exclude_sections: HashSet<String>,
}

impl FieldSelectorBuilder {
    /// Create a new builder
    pub fn new() -> Self {
        Self::default()
    }

    /// Add fields to include (supports multiple calls)
    pub fn include_fields<I, S>(mut self, fields: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.include_fields
            .extend(fields.into_iter().map(Into::into));
        self
    }

    /// Add a single field to include
    pub fn include_field<S: Into<String>>(mut self, field: S) -> Self {
        self.include_fields.insert(field.into());
        self
    }

    /// Add fields to exclude (supports multiple calls)
    pub fn exclude_fields<I, S>(mut self, fields: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.exclude_fields
            .extend(fields.into_iter().map(Into::into));
        self
    }

    /// Add a single field to exclude
    pub fn exclude_field<S: Into<String>>(mut self, field: S) -> Self {
        self.exclude_fields.insert(field.into());
        self
    }

    /// Add sections to include (supports multiple calls)
    pub fn include_sections<I, S>(mut self, sections: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.include_sections
            .extend(sections.into_iter().map(Into::into));
        self
    }

    /// Add a single section to include
    pub fn include_section<S: Into<String>>(mut self, section: S) -> Self {
        self.include_sections.insert(section.into());
        self
    }

    /// Add sections to exclude (supports multiple calls)
    pub fn exclude_sections<I, S>(mut self, sections: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: Into<String>,
    {
        self.exclude_sections
            .extend(sections.into_iter().map(Into::into));
        self
    }

    /// Add a single section to exclude
    pub fn exclude_section<S: Into<String>>(mut self, section: S) -> Self {
        self.exclude_sections.insert(section.into());
        self
    }

    /// Build the final `FieldSelector`
    pub fn build(self) -> FieldSelector {
        FieldSelector {
            include_fields: if self.include_fields.is_empty() {
                None
            } else {
                Some(self.include_fields)
            },
            exclude_fields: if self.exclude_fields.is_empty() {
                None
            } else {
                Some(self.exclude_fields)
            },
            include_sections: if self.include_sections.is_empty() {
                None
            } else {
                Some(self.include_sections)
            },
            exclude_sections: if self.exclude_sections.is_empty() {
                None
            } else {
                Some(self.exclude_sections)
            },
        }
    }
}

/// High-performance JSON serializer with optimization options
pub struct OptimizedSerializer;

impl OptimizedSerializer {
    /// Serialize a single report with options
    pub fn serialize_report(
        report: &PageQualityReport,
        options: &SerializationOptions,
    ) -> Result<String, serde_json::Error> {
        if options.minimal_output {
            let minimal = Self::to_minimal_report(report);
            Self::serialize_with_options(&minimal, options)
        } else {
            Self::serialize_with_options(report, options)
        }
    }

    /// Serialize multiple reports efficiently
    pub fn serialize_reports_batch(
        reports: &[PageQualityReport],
        options: &SerializationOptions,
    ) -> Result<String, serde_json::Error> {
        if options.streaming && reports.len() > 50 {
            Self::serialize_streaming_ultra_fast(reports, options)
        } else if options.streaming && reports.len() > 10 {
            Self::serialize_streaming(reports, options)
        } else {
            Self::serialize_with_options(reports, options)
        }
    }

    /// Ultra-fast streaming serialization for very large batches
    pub fn serialize_streaming_ultra_fast(
        reports: &[PageQualityReport],
        options: &SerializationOptions,
    ) -> Result<String, serde_json::Error> {
        use std::fmt::Write;

        // Pre-allocate with better size estimation
        let estimated_size = if options.minimal_output {
            reports.len() * 512 // ~512 bytes per minimal report
        } else {
            reports.len() * 2048 // ~2KB per full report
        };

        let mut result = String::with_capacity(estimated_size);
        result.push('[');

        // Process in chunks to avoid memory pressure
        let chunk_size = 100;
        for (chunk_idx, chunk) in reports.chunks(chunk_size).enumerate() {
            if chunk_idx > 0 && !options.compact {
                result.push('\n');
            }

            for (i, report) in chunk.iter().enumerate() {
                let global_idx = chunk_idx * chunk_size + i;
                if global_idx > 0 {
                    result.push(',');
                    if !options.compact {
                        result.push('\n');
                    }
                }

                if options.minimal_output {
                    // Inline minimal serialization for speed
                    let _ = write!(
                        result,
                        r#"{{"url":"{}","score":{:.2},"verdict":"{}","metrics":{{"word_count":{},"readability_score":{:.2},"technical_score":{:.2},"accessibility_score":{:.2}}}}}"#,
                        report.url,
                        report.score,
                        format!("{:?}", report.verdict),
                        report.metrics.html_analysis.content.word_count,
                        report
                            .metrics
                            .html_analysis
                            .content
                            .readability_fk
                            .unwrap_or(0.0),
                        report.metrics.html_analysis.technical.technical_score,
                        report
                            .metrics
                            .html_analysis
                            .accessibility
                            .wcag_aa_compliance_score
                    );
                } else {
                    let serialized = Self::serialize_with_options(report, options)?;
                    result.push_str(&serialized);
                }
            }
        }

        result.push(']');
        Ok(result)
    }

    /// Write reports to file using streaming for memory efficiency
    pub fn write_reports_to_file<P: AsRef<std::path::Path>>(
        reports: &[PageQualityReport],
        path: P,
        options: &SerializationOptions,
    ) -> Result<(), Box<dyn std::error::Error>> {
        let file = File::create(path)?;
        let mut writer = BufWriter::with_capacity(options.buffer_size, file);

        if options.minimal_output {
            let minimal_reports: Vec<_> = reports.iter().map(Self::to_minimal_report).collect();

            if options.compact {
                serde_json::to_writer(&mut writer, &minimal_reports)?;
            } else {
                serde_json::to_writer_pretty(&mut writer, &minimal_reports)?;
            }
        } else {
            if options.compact {
                serde_json::to_writer(&mut writer, reports)?;
            } else {
                serde_json::to_writer_pretty(&mut writer, reports)?;
            }
        }

        writer.flush()?;
        Ok(())
    }

    /// Serialize with custom field filtering
    pub fn serialize_with_field_filter<F>(
        report: &PageQualityReport,
        field_filter: F,
    ) -> Result<String, serde_json::Error>
    where
        F: Fn(&str) -> bool,
    {
        // Convert to Value for manipulation
        let mut value = serde_json::to_value(report)?;
        Self::filter_fields(&mut value, &field_filter);

        serde_json::to_string(&value)
    }

    /// Serialize a single report with advanced field selection
    ///
    /// # Arguments
    ///
    /// * `report` - The report to serialize
    /// * `selector` - Field selection rules
    /// * `options` - Optional serialization options (compact, pretty, etc.)
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use webpage_quality_analyzer::utils::json_optimizer::{OptimizedSerializer, FieldSelector};
    ///
    /// # fn example() -> Result<(), Box<dyn std::error::Error>> {
    /// let selector = FieldSelector::builder()
    ///     .include_fields(["url", "score", "metrics.content.word_count"])
    ///     .exclude_sections(["processed_document"])
    ///     .build();
    ///
    /// // let json = OptimizedSerializer::serialize_with_selector(&report, &selector, None)?;
    /// # Ok(())
    /// # }
    /// ```
    pub fn serialize_with_selector(
        report: &PageQualityReport,
        selector: &FieldSelector,
        options: Option<&SerializationOptions>,
    ) -> Result<String, serde_json::Error> {
        // Convert to Value for manipulation
        let mut value = serde_json::to_value(report)?;

        // Apply field selection rules
        Self::filter_fields_with_selector(&mut value, selector, "");

        // Apply serialization options
        if let Some(opts) = options {
            if opts.compact {
                serde_json::to_string(&value)
            } else {
                serde_json::to_string_pretty(&value)
            }
        } else {
            serde_json::to_string_pretty(&value)
        }
    }

    /// Serialize multiple reports with advanced field selection
    ///
    /// # Arguments
    ///
    /// * `reports` - The reports to serialize
    /// * `selector` - Field selection rules applied to each report
    /// * `options` - Optional serialization options
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use webpage_quality_analyzer::utils::json_optimizer::{OptimizedSerializer, FieldSelector, SerializationOptions};
    ///
    /// # fn example() -> Result<(), Box<dyn std::error::Error>> {
    /// let selector = FieldSelector::builder()
    ///     .include_sections(["url", "score", "metrics"])
    ///     .exclude_fields(["metrics.html_analysis.accessibility"])
    ///     .build();
    ///
    /// let opts = SerializationOptions::batch_processing();
    /// // let json = OptimizedSerializer::serialize_batch_with_selector(&reports, &selector, Some(&opts))?;
    /// # Ok(())
    /// # }
    /// ```
    pub fn serialize_batch_with_selector(
        reports: &[PageQualityReport],
        selector: &FieldSelector,
        options: Option<&SerializationOptions>,
    ) -> Result<String, serde_json::Error> {
        let default_opts = SerializationOptions::default();
        let opts = options.unwrap_or(&default_opts);

        // Pre-allocate based on expected size after filtering
        let base_size = if selector.summary().contains("Include sections")
            || selector.summary().contains("Include fields")
        {
            1024 // Smaller expected size for filtered output
        } else {
            2048 // Full size estimate
        };

        let estimated_size = reports.len() * base_size;
        let mut result = String::with_capacity(estimated_size);

        if opts.compact {
            result.push('[');
        } else {
            result.push_str("[\n");
        }

        for (i, report) in reports.iter().enumerate() {
            if i > 0 {
                if opts.compact {
                    result.push(',');
                } else {
                    result.push_str(",\n");
                }
            }

            // Serialize individual report with selector
            let report_json = Self::serialize_with_selector(report, selector, Some(opts))?;

            if opts.compact {
                result.push_str(&report_json);
            } else {
                // Add indentation for pretty printing
                for line in report_json.lines() {
                    result.push_str("  ");
                    result.push_str(line);
                    result.push('\n');
                }
            }
        }

        if opts.compact {
            result.push(']');
        } else {
            result.push_str("\n]");
        }

        Ok(result)
    }

    /// Create a custom serialization for analytics/reporting
    pub fn serialize_for_analytics(
        reports: &[PageQualityReport],
    ) -> Result<String, serde_json::Error> {
        let analytics_data = AnalyticsData::from_reports(reports);
        serde_json::to_string_pretty(&analytics_data)
    }

    // Private helper methods

    fn serialize_with_options<T: Serialize + ?Sized>(
        data: &T,
        options: &SerializationOptions,
    ) -> Result<String, serde_json::Error> {
        if options.compact {
            serde_json::to_string(data)
        } else {
            serde_json::to_string_pretty(data)
        }
    }

    fn serialize_streaming(
        reports: &[PageQualityReport],
        options: &SerializationOptions,
    ) -> Result<String, serde_json::Error> {
        // For very large datasets, serialize in chunks
        let mut result = String::with_capacity(reports.len() * 1024); // Rough estimate
        result.push('[');

        for (i, report) in reports.iter().enumerate() {
            if i > 0 {
                result.push(',');
                if !options.compact {
                    result.push('\n');
                }
            }

            let serialized = if options.minimal_output {
                let minimal = Self::to_minimal_report(report);
                Self::serialize_with_options(&minimal, options)?
            } else {
                Self::serialize_with_options(report, options)?
            };

            result.push_str(&serialized);
        }

        result.push(']');
        Ok(result)
    }

    fn to_minimal_report(report: &PageQualityReport) -> MinimalPageQualityReport {
        MinimalPageQualityReport {
            url: report.url.clone(),
            score: report.score,
            verdict: format!("{:?}", report.verdict),
            metrics: MinimalPageMetrics {
                word_count: report.metrics.html_analysis.content.word_count,
                readability_score: report
                    .metrics
                    .html_analysis
                    .content
                    .readability_fk
                    .unwrap_or(0.0),
                technical_score: report.metrics.html_analysis.technical.technical_score,
                seo_score: 0.0, // Would need to be calculated from SEO metrics
                accessibility_score: report
                    .metrics
                    .html_analysis
                    .accessibility
                    .wcag_aa_compliance_score,
            },
        }
    }

    fn filter_fields<F>(value: &mut Value, filter: &F)
    where
        F: Fn(&str) -> bool,
    {
        match value {
            Value::Object(map) => {
                map.retain(|key, val| {
                    if filter(key) {
                        Self::filter_fields(val, filter);
                        true
                    } else {
                        false
                    }
                });
            }
            Value::Array(arr) => {
                for item in arr {
                    Self::filter_fields(item, filter);
                }
            }
            _ => {}
        }
    }

    /// Apply field selection rules to a JSON value recursively
    ///
    /// # Arguments
    ///
    /// * `value` - The JSON value to filter (modified in place)
    /// * `selector` - The field selection rules
    /// * `current_path` - The current path in dot notation (used for recursion)
    fn filter_fields_with_selector(
        value: &mut Value,
        selector: &FieldSelector,
        current_path: &str,
    ) {
        match value {
            Value::Object(map) => {
                map.retain(|key, val| {
                    // Build the full path for this field
                    let field_path = if current_path.is_empty() {
                        key.clone()
                    } else {
                        format!("{}.{}", current_path, key)
                    };

                    // Check if this field should be included
                    let should_include = selector.should_include_field(&field_path);

                    if should_include {
                        // Recursively filter nested objects
                        Self::filter_fields_with_selector(val, selector, &field_path);
                        true
                    } else {
                        false
                    }
                });
            }
            Value::Array(arr) => {
                // For arrays, apply filtering to each element
                for item in arr {
                    Self::filter_fields_with_selector(item, selector, current_path);
                }
            }
            _ => {
                // Primitive values don't need filtering
            }
        }
    }
}

/// Analytics-focused data structure for reporting and dashboards
#[derive(Debug, Serialize)]
pub struct AnalyticsData {
    pub summary: AnalyticsSummary,
    pub score_distribution: HashMap<String, usize>,
    pub top_issues: Vec<String>,
    pub metrics_averages: MetricsAverages,
}

#[derive(Debug, Serialize)]
pub struct AnalyticsSummary {
    pub total_pages: usize,
    pub average_score: f32,
    pub median_score: f32,
    pub score_std_dev: f32,
    pub min_score: f32,
    pub max_score: f32,
}

#[derive(Debug, Serialize)]
pub struct MetricsAverages {
    pub avg_word_count: f32,
    pub avg_readability: f32,
    pub avg_technical_score: f32,
    pub avg_accessibility_score: f32,
    pub avg_load_time: Option<f32>,
}

impl AnalyticsData {
    pub fn from_reports(reports: &[PageQualityReport]) -> Self {
        let scores: Vec<f32> = reports.iter().map(|r| r.score).collect();
        let total_pages = reports.len();

        let summary = AnalyticsSummary {
            total_pages,
            average_score: scores.iter().sum::<f32>() / total_pages as f32,
            median_score: Self::calculate_median(&scores),
            score_std_dev: Self::calculate_std_dev(&scores),
            min_score: scores.iter().fold(f32::INFINITY, |a, &b| a.min(b)),
            max_score: scores.iter().fold(f32::NEG_INFINITY, |a, &b| a.max(b)),
        };

        let score_distribution = Self::calculate_score_distribution(&scores);
        let metrics_averages = Self::calculate_metrics_averages(reports);
        let top_issues = Self::extract_top_issues(reports);

        Self {
            summary,
            score_distribution,
            top_issues,
            metrics_averages,
        }
    }

    fn calculate_median(scores: &[f32]) -> f32 {
        let mut sorted = scores.to_vec();
        sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
        let len = sorted.len();

        if len % 2 == 0 {
            (sorted[len / 2 - 1] + sorted[len / 2]) / 2.0
        } else {
            sorted[len / 2]
        }
    }

    fn calculate_std_dev(scores: &[f32]) -> f32 {
        let mean = scores.iter().sum::<f32>() / scores.len() as f32;
        let variance = scores
            .iter()
            .map(|score| (score - mean).powi(2))
            .sum::<f32>()
            / scores.len() as f32;
        variance.sqrt()
    }

    fn calculate_score_distribution(scores: &[f32]) -> HashMap<String, usize> {
        let mut distribution = HashMap::new();

        for &score in scores {
            let band = match score as u32 {
                90..=100 => "Excellent (90-100)",
                75..=89 => "Good (75-89)",
                50..=74 => "Fair (50-74)",
                25..=49 => "Poor (25-49)",
                _ => "Very Poor (0-24)",
            };
            *distribution.entry(band.to_string()).or_insert(0) += 1;
        }

        distribution
    }

    fn calculate_metrics_averages(reports: &[PageQualityReport]) -> MetricsAverages {
        let total = reports.len() as f32;

        MetricsAverages {
            avg_word_count: reports
                .iter()
                .map(|r| r.metrics.html_analysis.content.word_count as f32)
                .sum::<f32>()
                / total,
            avg_readability: reports
                .iter()
                .map(|r| {
                    r.metrics
                        .html_analysis
                        .content
                        .readability_fk
                        .unwrap_or(0.0)
                })
                .sum::<f32>()
                / total,
            avg_technical_score: reports
                .iter()
                .map(|r| r.metrics.html_analysis.technical.technical_score)
                .sum::<f32>()
                / total,
            avg_accessibility_score: reports
                .iter()
                .map(|r| {
                    r.metrics
                        .html_analysis
                        .accessibility
                        .wcag_aa_compliance_score
                })
                .sum::<f32>()
                / total,
            avg_load_time: None, // Would need performance metrics
        }
    }

    fn extract_top_issues(reports: &[PageQualityReport]) -> Vec<String> {
        // Extract common issues from notes across all reports
        let mut issue_counts: HashMap<String, usize> = HashMap::new();

        for report in reports {
            for note in &report.notes {
                if note.contains("missing") || note.contains("error") || note.contains("warning") {
                    let simplified_issue = Self::simplify_issue_message(note);
                    *issue_counts.entry(simplified_issue).or_insert(0) += 1;
                }
            }
        }

        let mut issues: Vec<_> = issue_counts
            .into_iter()
            .map(|(issue, count)| format!("{} ({} occurrences)", issue, count))
            .collect();

        issues.sort_by(|a, b| {
            let count_a = Self::extract_count_from_issue(a);
            let count_b = Self::extract_count_from_issue(b);
            count_b.cmp(&count_a)
        });

        issues.into_iter().take(10).collect()
    }

    fn simplify_issue_message(note: &str) -> String {
        // Simplify issue messages for grouping
        if note.contains("missing alt") {
            "Missing alt text".to_string()
        } else if note.contains("missing title") {
            "Missing title".to_string()
        } else if note.contains("missing meta description") {
            "Missing meta description".to_string()
        } else if note.contains("accessibility") {
            "Accessibility issue".to_string()
        } else if note.contains("performance") {
            "Performance issue".to_string()
        } else {
            // Take first 50 chars as generic issue
            note.chars().take(50).collect::<String>()
        }
    }

    fn extract_count_from_issue(issue: &str) -> usize {
        // Extract number from "(X occurrences)"
        issue
            .split('(')
            .nth(1)
            .and_then(|s| s.split(' ').next())
            .and_then(|s| s.parse().ok())
            .unwrap_or(0)
    }
}

/// Streaming JSON writer for large datasets
pub struct StreamingJsonWriter<W: Write> {
    writer: BufWriter<W>,
    options: SerializationOptions,
    first_item: bool,
}

impl<W: Write> StreamingJsonWriter<W> {
    pub fn new(writer: W, options: SerializationOptions) -> Result<Self, std::io::Error> {
        let mut buffered = BufWriter::with_capacity(options.buffer_size, writer);
        buffered.write_all(b"[")?;

        Ok(Self {
            writer: buffered,
            options,
            first_item: true,
        })
    }

    pub fn write_report(
        &mut self,
        report: &PageQualityReport,
    ) -> Result<(), Box<dyn std::error::Error>> {
        if !self.first_item {
            self.writer.write_all(b",")?;
            if !self.options.compact {
                self.writer.write_all(b"\n")?;
            }
        }
        self.first_item = false;

        let serialized = OptimizedSerializer::serialize_report(report, &self.options)?;
        self.writer.write_all(serialized.as_bytes())?;

        Ok(())
    }

    pub fn finish(mut self) -> Result<(), std::io::Error> {
        self.writer.write_all(b"]")?;
        self.writer.flush()?;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::models::models::{
        AccessibilityMetrics, ContentMetrics, ExtractedMetadata, PageMetrics, PageMetricsFromHTML,
        ProcessedDocument, QualityBand, TechnicalMetrics,
    };
    use chrono::Utc;

    fn create_test_report() -> PageQualityReport {
        PageQualityReport {
            url: "https://example.com".to_string(),
            fetched_at: Some(Utc::now()),
            score: 85.5,
            verdict: QualityBand::Good,
            metadata: ExtractedMetadata::default(),
            processed_document: ProcessedDocument::default(),
            notes: vec!["Test note".to_string()],
            version: "1.0.0".to_string(),
            phase3_scoring: None, // Add missing field
            metrics: PageMetrics {
                html_analysis: PageMetricsFromHTML {
                    content: ContentMetrics {
                        word_count: 500,
                        readability_fk: Some(65.0),
                        ..Default::default()
                    },
                    technical: TechnicalMetrics {
                        technical_score: 90.0,
                        ..Default::default()
                    },
                    accessibility: AccessibilityMetrics {
                        wcag_aa_compliance_score: 80.0,
                        ..Default::default()
                    },
                    ..Default::default()
                },
                ..Default::default()
            },
        }
    }

    #[test]
    fn test_serialize_report_compact() {
        let report = create_test_report();
        let options = SerializationOptions {
            compact: true,
            ..Default::default()
        };

        let result = OptimizedSerializer::serialize_report(&report, &options);
        assert!(result.is_ok());
        let json = result.unwrap();

        // Compact should not contain newlines or extra spaces
        assert!(!json.contains("\n"));
        assert!(json.contains("\"score\":85.5"));
    }

    #[test]
    fn test_serialize_minimal() {
        let report = create_test_report();
        let options = SerializationOptions {
            minimal_output: true,
            ..Default::default()
        };

        let result = OptimizedSerializer::serialize_report(&report, &options);
        assert!(result.is_ok());
        let json = result.unwrap();

        // Should contain minimal fields
        assert!(json.contains("\"url\""));
        assert!(json.contains("\"score\""));
        assert!(json.contains("\"verdict\""));

        // Should be smaller than full serialization
        let full_options = SerializationOptions::default();
        let full_result = OptimizedSerializer::serialize_report(&report, &full_options).unwrap();
        assert!(json.len() < full_result.len());
    }

    #[test]
    fn test_analytics_data() {
        let reports = vec![create_test_report(), create_test_report()];
        let analytics = AnalyticsData::from_reports(&reports);

        assert_eq!(analytics.summary.total_pages, 2);
        assert_eq!(analytics.summary.average_score, 85.5);
        assert!(analytics.score_distribution.contains_key("Good (75-89)"));
    }

    #[test]
    fn test_batch_serialization() {
        let reports = (0..5).map(|_| create_test_report()).collect::<Vec<_>>();
        let options = SerializationOptions::batch_processing();

        let result = OptimizedSerializer::serialize_reports_batch(&reports, &options);
        assert!(result.is_ok());
        let json = result.unwrap();

        // Should be valid JSON array
        assert!(json.starts_with('['));
        assert!(json.ends_with(']'));

        // Should contain all reports
        let parsed: Value = serde_json::from_str(&json).unwrap();
        assert!(parsed.is_array());
        assert_eq!(parsed.as_array().unwrap().len(), 5);
    }
}