webpage_quality_analyzer 1.0.2

/// Example: Customizing JSON Output with Field Selection
///
/// This example demonstrates how to customize the JSON output to include only specific fields
/// like url, score, metadata, and processed_document.
///
/// This is especially useful for:
/// - Reducing bandwidth in production environments processing 10,000+ pages/day
/// - Minimizing storage costs for batch processing
/// - Creating focused API responses with only necessary data
/// - Improving serialization performance for large batches
use webpage_quality_analyzer::utils::json_optimizer::{
    FieldSelector, OptimizedSerializer, SerializationOptions,
};
use webpage_quality_analyzer::{analyze, analyze_batch_high_performance, AnalyzeError};

#[tokio::main]
async fn main() -> Result<(), AnalyzeError> {
    println!("=== Custom Output Fields Example ===\n");

    // Example 1: Include ONLY specific top-level fields
    println!("Example 1: Include ONLY url, score, metadata, processed_document");
    println!("-------------------------------------------------------------");

    let url = "https://example.com";
    let report = analyze(url, None).await?;

    // Build a field selector for your exact requirements
    let selector = FieldSelector::builder()
        .include_fields(["url", "score", "metadata", "processed_document"])
        .build();

    // Serialize with the selector
    let custom_json = OptimizedSerializer::serialize_with_selector(&report, &selector, None)?;

    println!("Custom JSON output (pretty-printed):");
    println!("{}\n", custom_json);

    // Example 2: Compact output for production (no pretty printing)
    println!("\nExample 2: Compact output for production bandwidth savings");
    println!("----------------------------------------------------------");

    let compact_opts = SerializationOptions {
        compact: true, // No whitespace
        skip_empty_fields: true,
        minimal_output: false,
        streaming: false,
        buffer_size: 8192,
    };

    let compact_json =
        OptimizedSerializer::serialize_with_selector(&report, &selector, Some(&compact_opts))?;

    println!("Compact JSON (first 200 chars):");
    println!("{}...\n", &compact_json[..200.min(compact_json.len())]);
    println!("Size reduction: {} bytes", compact_json.len());

    // Example 3: Batch processing with field selection
    println!("\nExample 3: Batch processing with custom fields");
    println!("----------------------------------------------");

    let urls = vec![
        "https://example.com",
        "https://www.wikipedia.org",
        "https://github.com",
    ];

    // Use high-performance batch function
    let batch_json_str = analyze_batch_high_performance(&urls, None, 3, None).await?;

    // Parse the JSON to get reports for re-serialization with field selector
    let batch_reports: Vec<webpage_quality_analyzer::PageQualityReport> =
        serde_json::from_str(&batch_json_str)?;

    // Use batch serialization with field selector
    let batch_json = OptimizedSerializer::serialize_batch_with_selector(
        &batch_reports,
        &selector,
        Some(&compact_opts),
    )?;

    println!("Batch JSON with {} reports", batch_reports.len());
    println!("Total size: {} bytes", batch_json.len());
    println!(
        "Average per report: {} bytes",
        batch_json.len() / batch_reports.len()
    );

    // Example 4: Include sections with field exclusions
    println!("\nExample 4: Include metadata section but exclude specific fields");
    println!("---------------------------------------------------------------");

    let advanced_selector = FieldSelector::builder()
        .include_fields(["url", "score", "processed_document"])
        .include_sections(["metadata"]) // Include entire metadata section
        .exclude_fields(["metadata.og_tags", "metadata.twitter_tags"]) // But exclude large tag data
        .build();

    let advanced_json =
        OptimizedSerializer::serialize_with_selector(&report, &advanced_selector, None)?;

    println!("Advanced filtered JSON (first 500 chars):");
    println!("{}...\n", &advanced_json[..500.min(advanced_json.len())]);

    // Example 5: Ultra-minimal output for monitoring/dashboards
    println!("\nExample 5: Ultra-minimal output (url + score only)");
    println!("-------------------------------------------------");

    let minimal_selector = FieldSelector::builder()
        .include_fields(["url", "score", "verdict"])
        .build();

    let minimal_json = OptimizedSerializer::serialize_with_selector(
        &report,
        &minimal_selector,
        Some(&compact_opts),
    )?;

    println!("Minimal JSON: {}", minimal_json);
    println!(
        "Size: {} bytes (perfect for dashboards/monitoring)\n",
        minimal_json.len()
    );

    // Example 6: Real-world production scenario
    println!("\nExample 6: Production batch processing (100 pages/5 seconds target)");
    println!("-------------------------------------------------------------------");

    // Simulate batch processing with your exact requirements
    let production_selector = FieldSelector::builder()
        .include_fields([
            "url",
            "score",
            "metadata",
            "processed_document", // Note: You wrote "procesdoc" but the field is "processed_document"
        ])
        .build();

    let production_opts = SerializationOptions {
        compact: true,           // Minimize bandwidth
        skip_empty_fields: true, // Skip nulls/empty arrays
        minimal_output: false,
        streaming: true,    // Use streaming for large batches
        buffer_size: 16384, // Larger buffer for production
    };

    let production_json = OptimizedSerializer::serialize_batch_with_selector(
        &batch_reports,
        &production_selector,
        Some(&production_opts),
    )?;

    println!("Production batch JSON:");
    println!("  Reports: {}", batch_reports.len());
    println!("  Total size: {} bytes", production_json.len());
    if batch_reports.len() > 0 {
        println!(
            "  Per-report avg: {} bytes",
            production_json.len() / batch_reports.len()
        );
        println!(
            "  Estimated 100 pages: ~{} KB",
            (production_json.len() * 100 / batch_reports.len()) / 1024
        );
    }

    // Example 7: Using field selector summary for debugging
    println!("\nExample 7: Field selector debugging");
    println!("-----------------------------------");

    println!(
        "Production selector summary: {}",
        production_selector.summary()
    );
    println!("Minimal selector summary: {}", minimal_selector.summary());

    // Performance comparison
    println!("\n=== Performance Comparison ===");

    // Full report
    let full_json = serde_json::to_string(&report)?;
    println!("Full report size: {} bytes", full_json.len());

    // Your custom fields
    let custom_json = OptimizedSerializer::serialize_with_selector(
        &report,
        &production_selector,
        Some(&production_opts),
    )?;
    println!("Custom fields size: {} bytes", custom_json.len());

    let reduction_pct =
        ((full_json.len() - custom_json.len()) as f32 / full_json.len() as f32) * 100.0;
    println!("Size reduction: {:.1}%", reduction_pct);
    println!(
        "Savings on 10,000 pages: ~{:.2} MB",
        (full_json.len() - custom_json.len()) as f32 * 10000.0 / 1024.0 / 1024.0
    );

    Ok(())
}

// Additional helper functions for production use

/// Example: Process batch with custom output and save to file
#[allow(dead_code)]
async fn process_and_save_batch(
    urls: &[&str],
    output_path: &str,
) -> Result<(), Box<dyn std::error::Error>> {
    use std::fs::File;
    use std::io::Write;

    // Analyze batch with high-performance function
    let batch_json_str = analyze_batch_high_performance(urls, None, 20, None).await?;

    // Parse to get reports
    let reports: Vec<webpage_quality_analyzer::PageQualityReport> =
        serde_json::from_str(&batch_json_str)?;

    // Create your custom selector
    let selector = FieldSelector::builder()
        .include_fields(["url", "score", "metadata", "processed_document"])
        .build();

    // Optimize for production
    let opts = SerializationOptions {
        compact: true,
        skip_empty_fields: true,
        minimal_output: false,
        streaming: true,
        buffer_size: 16384,
    };

    // Serialize with field selection
    let json =
        OptimizedSerializer::serialize_batch_with_selector(&reports, &selector, Some(&opts))?;

    // Save to file
    let mut file = File::create(output_path)?;
    file.write_all(json.as_bytes())?;

    println!(
        "Saved {} reports ({} bytes) to {}",
        reports.len(),
        json.len(),
        output_path
    );

    Ok(())
}

/// Example: Stream processing for very large batches (1000+ pages)
#[allow(dead_code)]
async fn stream_large_batch(
    urls: &[&str],
    batch_size: usize,
) -> Result<Vec<String>, Box<dyn std::error::Error>> {
    let selector = FieldSelector::builder()
        .include_fields(["url", "score", "metadata", "processed_document"])
        .build();

    let opts = SerializationOptions::batch_processing();

    let mut results = Vec::new();

    // Process in chunks to avoid memory pressure
    for chunk in urls.chunks(batch_size) {
        let batch_json_str = analyze_batch_high_performance(chunk, None, 20, None).await?;
        let reports: Vec<webpage_quality_analyzer::PageQualityReport> =
            serde_json::from_str(&batch_json_str)?;

        let json =
            OptimizedSerializer::serialize_batch_with_selector(&reports, &selector, Some(&opts))?;

        results.push(json.clone());

        println!(
            "Processed chunk: {} pages, {} bytes",
            reports.len(),
            json.len()
        );
    }

    Ok(results)
}