webpage_quality_analyzer 1.0.2

use serde::{Deserialize, Serialize};
use std::fs;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Semaphore;
use tokio::time::sleep;
use webpage_quality_analyzer::{analyze_with_profile, Analyzer};

/// Structure representing each webpage entry in the input JSON
#[derive(Debug, Deserialize, Clone)]
struct WebpageEntry {
    url: String,
    content: String,
    // We'll ignore other fields that might be present
}

/// Structure for the output JSON result
#[derive(Debug, Serialize)]
struct ScoringResult {
    url: String,
    score: f32,
    processing_time_ms: u64,
    success: bool,
    error: Option<String>,
}

/// Summary statistics for the batch processing
#[derive(Debug, Serialize)]
struct BatchProcessingSummary {
    total_pages: usize,
    successful_analyses: usize,
    failed_analyses: usize,
    total_processing_time_ms: u64,
    average_processing_time_ms: f64,
    min_processing_time_ms: u64,
    max_processing_time_ms: u64,
    average_score: f32,
    min_score: f32,
    max_score: f32,
}

/// Output structure combining results and summary
#[derive(Debug, Serialize)]
struct BatchOutput {
    summary: BatchProcessingSummary,
    results: Vec<ScoringResult>,
}

/// Fast parallel batch processing function
async fn analyze_batch_parallel(
    entries: &[WebpageEntry],
    profile: &str,
    concurrency: usize,
) -> Vec<ScoringResult> {
    println!(
        "🚀 Starting parallel processing with {} concurrent tasks...",
        concurrency
    );

    // Create a single analyzer instance to reuse
    let analyzer: Analyzer = Analyzer::builder()
        .with_profile_name(profile)
        .expect("Failed to create analyzer")
        .build()
        .expect("Failed to build analyzer");

    let analyzer = Arc::new(analyzer);

    // Control concurrency with semaphore
    let semaphore = Arc::new(Semaphore::new(concurrency));

    // Create tasks for parallel processing - clone data to avoid lifetime issues
    let tasks = entries.iter().enumerate().map(|(index, entry)| {
        let analyzer = Arc::clone(&analyzer);
        let permit = Arc::clone(&semaphore);
        let url = entry.url.clone();
        let content = entry.content.clone();
        let total_entries = entries.len();

        async move {
            let _permit = permit.acquire().await.unwrap();

            if (index + 1) % 10 == 0 {
                println!("Processing {}/{}: {}", index + 1, total_entries, url);
            }

            let start = Instant::now();

            let result = analyzer.run(&url, Some(&content)).await;
            let processing_time = start.elapsed();

            match result {
                Ok(report) => ScoringResult {
                    url,
                    score: report.score,
                    processing_time_ms: processing_time.as_millis() as u64,
                    success: true,
                    error: None,
                },
                Err(e) => ScoringResult {
                    url,
                    score: 0.0,
                    processing_time_ms: processing_time.as_millis() as u64,
                    success: false,
                    error: Some(e.to_string()),
                },
            }
        }
    });

    // Execute all tasks concurrently
    futures::future::join_all(tasks).await
}

#[tokio::test]
async fn test_batch_html_scoring() {
    println!("🚀 Starting batch HTML scoring test...");

    // Read the JSON file
    let json_path = "/home/gyashu/projects/webpage-quality-analyser/batch_20250903_151527_59.json";
    let json_content = fs::read_to_string(json_path).expect("Failed to read JSON file");

    let all_entries: Vec<WebpageEntry> =
        serde_json::from_str(&json_content).expect("Failed to parse JSON");

    // In debug mode, test with 10 pages; in release mode, test all 100
    let webpage_entries: Vec<_> = if cfg!(debug_assertions) {
        println!("📊 Debug mode: Testing with first 10 pages (run in release for full 100)");
        all_entries.into_iter().take(10).collect()
    } else {
        println!("📊 Release mode: Testing all {} pages", all_entries.len());
        all_entries
    };

    println!("📊 Found {} webpages to process", webpage_entries.len());

    let mut results = Vec::new();
    let mut processing_times = Vec::new();
    let mut successful_analyses = 0;
    let mut failed_analyses = 0;
    let mut scores = Vec::new();

    let total_start_time = Instant::now();

    for (index, entry) in webpage_entries.iter().enumerate() {
        println!(
            "Processing {}/{}: {}",
            index + 1,
            webpage_entries.len(),
            entry.url
        );

        let start_time = Instant::now();
        let mut result = ScoringResult {
            url: entry.url.clone(),
            score: 0.0,
            processing_time_ms: 0,
            success: false,
            error: None,
        };

        // Analyze the HTML content using content_article profile
        match analyze_with_profile(&entry.url, Some(&entry.content), "content_article").await {
            Ok(report) => {
                let processing_time = start_time.elapsed();
                result.score = report.score;
                result.processing_time_ms = processing_time.as_millis() as u64;
                result.success = true;

                successful_analyses += 1;
                processing_times.push(processing_time.as_millis() as u64);
                scores.push(report.score);

                println!(
                    "  ✅ Score: {:.2} ({}ms)",
                    report.score, result.processing_time_ms
                );
            }
            Err(e) => {
                let processing_time = start_time.elapsed();
                result.processing_time_ms = processing_time.as_millis() as u64;
                result.error = Some(e.to_string());
                failed_analyses += 1;

                println!("  ❌ Error: {} ({}ms)", e, result.processing_time_ms);
            }
        }

        results.push(result);

        // Small delay to prevent overwhelming the system
        sleep(Duration::from_millis(10)).await;
    }

    let total_processing_time = total_start_time.elapsed();

    // Calculate summary statistics
    let total_pages = webpage_entries.len();
    let total_processing_time_ms = total_processing_time.as_millis() as u64;
    let average_processing_time_ms = if !processing_times.is_empty() {
        processing_times.iter().sum::<u64>() as f64 / processing_times.len() as f64
    } else {
        0.0
    };
    let min_processing_time_ms = processing_times.iter().min().copied().unwrap_or(0);
    let max_processing_time_ms = processing_times.iter().max().copied().unwrap_or(0);

    let average_score = if !scores.is_empty() {
        scores.iter().sum::<f32>() / scores.len() as f32
    } else {
        0.0
    };
    let min_score = scores
        .iter()
        .min_by(|a, b| a.partial_cmp(b).unwrap())
        .copied()
        .unwrap_or(0.0);
    let max_score = scores
        .iter()
        .max_by(|a, b| a.partial_cmp(b).unwrap())
        .copied()
        .unwrap_or(0.0);

    let summary = BatchProcessingSummary {
        total_pages,
        successful_analyses,
        failed_analyses,
        total_processing_time_ms,
        average_processing_time_ms,
        min_processing_time_ms,
        max_processing_time_ms,
        average_score,
        min_score,
        max_score,
    };

    let batch_output = BatchOutput { summary, results };

    // Write results to JSON file
    let output_path = "/home/gyashu/projects/webpage-quality-analyser/batch_scoring_results.json";
    let output_json =
        serde_json::to_string_pretty(&batch_output).expect("Failed to serialize results to JSON");

    fs::write(output_path, output_json).expect("Failed to write results file");

    // Print summary
    println!("\n📈 === Batch Processing Summary ===");
    println!("📊 Total pages processed: {}", total_pages);
    println!("✅ Successful analyses: {}", successful_analyses);
    println!("❌ Failed analyses: {}", failed_analyses);
    println!(
        "🎯 Success rate: {:.1}%",
        (successful_analyses as f64 / total_pages as f64) * 100.0
    );
    println!(
        "⏱️  Total processing time: {:.2}s",
        total_processing_time_ms as f64 / 1000.0
    );
    println!(
        "⏱️  Average processing time per page: {:.1}ms",
        average_processing_time_ms
    );
    println!(
        "⏱️  Min/Max processing time: {}ms / {}ms",
        min_processing_time_ms, max_processing_time_ms
    );

    if !scores.is_empty() {
        println!("📊 Average score: {:.2}", average_score);
        println!("📊 Score range: {:.2} - {:.2}", min_score, max_score);
    }

    println!("💾 Results saved to: {}", output_path);

    // The test passes if we processed at least some pages successfully
    assert!(
        successful_analyses > 0,
        "At least some analyses should succeed"
    );
    println!("✅ Batch processing test completed successfully!");
}

#[tokio::test]
async fn test_batch_html_scoring_sample() {
    println!("🔬 Running sample batch test with first 3 entries...");

    // Read the JSON file
    let json_path = "/home/gyashu/projects/webpage-quality-analyser/batch_20250903_151527_59.json";
    let json_content = fs::read_to_string(json_path).expect("Failed to read JSON file");

    let all_entries: Vec<WebpageEntry> =
        serde_json::from_str(&json_content).expect("Failed to parse JSON");

    // Take only first 3 entries for quick testing
    let sample_entries: Vec<_> = all_entries.into_iter().take(3).collect();

    println!("📊 Processing {} sample webpages", sample_entries.len());

    let total_start_time = Instant::now();

    for (index, entry) in sample_entries.iter().enumerate() {
        println!(
            "\n🔍 Sample {}/{}: {}",
            index + 1,
            sample_entries.len(),
            entry.url
        );

        let start_time = Instant::now();

        match analyze_with_profile(&entry.url, Some(&entry.content), "content_article").await {
            Ok(report) => {
                let processing_time = start_time.elapsed();
                println!("  📊 Score: {:.2}", report.score);
                println!("  ⏱️  Processing time: {}ms", processing_time.as_millis());
                println!("  🏆 Quality band: {:?}", report.verdict);

                // Print some key metrics
                let metrics = &report.metrics;
                println!(
                    "  📝 Content: {} words, {} paragraphs",
                    metrics.html_analysis.content.word_count,
                    metrics.html_analysis.structure.paragraph_count
                );
                println!(
                    "  🖼️  Images: {} ({:.0}% with alt text)",
                    metrics.html_analysis.media.images_count,
                    metrics.html_analysis.media.image_alt_coverage
                );
                println!(
                    "  🔗 Links: {} total",
                    metrics.html_analysis.links.total_links
                );
            }
            Err(e) => {
                let processing_time = start_time.elapsed();
                println!("  ❌ Error: {} ({}ms)", e, processing_time.as_millis());
            }
        }
    }

    let total_time = total_start_time.elapsed();
    println!(
        "\n⏱️  Total sample processing time: {:.2}s",
        total_time.as_millis() as f64 / 1000.0
    );
    println!(
        "⏱️  Average per page: {:.1}ms",
        total_time.as_millis() as f64 / sample_entries.len() as f64
    );

    println!("✅ Sample batch test completed!");
}

/// Test to analyze the distribution of content types in the batch
#[tokio::test]
async fn test_analyze_batch_content_distribution() {
    println!("📊 Analyzing content distribution in batch...");

    // Read the JSON file
    let json_path = "/home/gyashu/projects/webpage-quality-analyser/batch_20250903_151527_59.json";
    let json_content = fs::read_to_string(json_path).expect("Failed to read JSON file");

    let webpage_entries: Vec<WebpageEntry> =
        serde_json::from_str(&json_content).expect("Failed to parse JSON");

    println!("📋 URLs in the batch:");
    for (index, entry) in webpage_entries.iter().enumerate() {
        println!("  {}: {}", index + 1, entry.url);
    }

    println!("\n📈 Total entries: {}", webpage_entries.len());

    // Analyze URL patterns
    let mut domain_counts = std::collections::HashMap::new();
    for entry in &webpage_entries {
        if let Ok(url) = url::Url::parse(&entry.url) {
            if let Some(domain) = url.host_str() {
                *domain_counts.entry(domain.to_string()).or_insert(0) += 1;
            }
        }
    }

    println!("\n🌐 Domain distribution:");
    let mut sorted_domains: Vec<_> = domain_counts.iter().collect();
    sorted_domains.sort_by(|a, b| b.1.cmp(a.1));
    for (domain, count) in sorted_domains.iter().take(10) {
        println!("  {}: {} pages", domain, count);
    }

    println!("✅ Content distribution analysis completed!");
}

/// Optimized parallel batch processing test - Full 100 pages with high concurrency
/// NOTE: This test requires release mode or increased stack size for debug builds
/// Run with: cargo test --release test_batch_html_scoring_parallel
/// Or in debug mode: RUST_MIN_STACK=16777216 cargo test test_batch_html_scoring_parallel
#[tokio::test]
#[cfg_attr(
    debug_assertions,
    ignore = "Run in release mode: cargo test --release test_batch_html_scoring_parallel"
)]
async fn test_batch_html_scoring_parallel() {
    println!("🚀 Starting PARALLEL batch HTML scoring test with 100 pages...");

    // Read the JSON file
    let json_path = "/home/gyashu/projects/webpage-quality-analyser/batch_20250903_151527_59.json";
    let json_content = fs::read_to_string(json_path).expect("Failed to read JSON file");

    let webpage_entries: Vec<WebpageEntry> =
        serde_json::from_str(&json_content).expect("Failed to parse JSON");

    println!(
        "📊 Found {} webpages to process with parallel optimization",
        webpage_entries.len()
    );

    let total_start_time = Instant::now();

    // Use parallel processing with 20 concurrent tasks for production-level performance
    let results = analyze_batch_parallel(&webpage_entries, "content_article", 20).await;

    let total_processing_time = total_start_time.elapsed();

    // Calculate summary statistics
    let successful_analyses = results.iter().filter(|r| r.success).count();
    let failed_analyses = results.len() - successful_analyses;

    let processing_times: Vec<u64> = results
        .iter()
        .filter(|r| r.success)
        .map(|r| r.processing_time_ms)
        .collect();

    let scores: Vec<f32> = results
        .iter()
        .filter(|r| r.success)
        .map(|r| r.score)
        .collect();

    let total_processing_time_ms = total_processing_time.as_millis() as u64;
    let average_processing_time_ms = if !processing_times.is_empty() {
        processing_times.iter().sum::<u64>() as f64 / processing_times.len() as f64
    } else {
        0.0
    };
    let min_processing_time_ms = processing_times.iter().min().copied().unwrap_or(0);
    let max_processing_time_ms = processing_times.iter().max().copied().unwrap_or(0);

    let average_score = if !scores.is_empty() {
        scores.iter().sum::<f32>() / scores.len() as f32
    } else {
        0.0
    };
    let min_score = scores
        .iter()
        .min_by(|a, b| a.partial_cmp(b).unwrap())
        .copied()
        .unwrap_or(0.0);
    let max_score = scores
        .iter()
        .max_by(|a, b| a.partial_cmp(b).unwrap())
        .copied()
        .unwrap_or(0.0);

    let summary = BatchProcessingSummary {
        total_pages: webpage_entries.len(),
        successful_analyses,
        failed_analyses,
        total_processing_time_ms,
        average_processing_time_ms,
        min_processing_time_ms,
        max_processing_time_ms,
        average_score,
        min_score,
        max_score,
    };

    let batch_output = BatchOutput { summary, results };

    // Write results to JSON file
    let output_path =
        "/home/gyashu/projects/webpage-quality-analyser/batch_scoring_results_parallel.json";
    let output_json =
        serde_json::to_string_pretty(&batch_output).expect("Failed to serialize results to JSON");

    fs::write(output_path, output_json).expect("Failed to write results file");

    // Print summary
    println!("\n📈 === PARALLEL Batch Processing Summary ===");
    println!("📊 Total pages processed: {}", webpage_entries.len());
    println!("✅ Successful analyses: {}", successful_analyses);
    println!("❌ Failed analyses: {}", failed_analyses);
    println!(
        "🎯 Success rate: {:.1}%",
        (successful_analyses as f64 / webpage_entries.len() as f64) * 100.0
    );
    println!(
        "⏱️  Total processing time: {:.2}s",
        total_processing_time_ms as f64 / 1000.0
    );
    println!(
        "⏱️  Average processing time per page: {:.1}ms",
        average_processing_time_ms
    );
    println!(
        "⏱️  Min/Max processing time: {}ms / {}ms",
        min_processing_time_ms, max_processing_time_ms
    );

    if !scores.is_empty() {
        println!("📊 Average score: {:.2}", average_score);
        println!("📊 Score range: {:.2} - {:.2}", min_score, max_score);
    }

    println!("💾 Results saved to: {}", output_path);

    // Performance assertions for production-level processing
    assert_eq!(
        successful_analyses,
        webpage_entries.len(),
        "All analyses should succeed - got {}/{}",
        successful_analyses,
        webpage_entries.len()
    );

    // Production target: 100 pages should complete in under 15 seconds (6-7 pages/sec)
    // This is realistic for comprehensive 115-metric analysis including:
    // - Large pages (Wikipedia: 194KB, 42K words)
    // - Complex DOM traversal (4273+ links)
    // - Content extraction with readability
    // Average: ~113ms per page with 20 concurrent tasks
    let duration_secs = total_processing_time.as_secs();
    assert!(
        duration_secs < 15,
        "Parallel processing should complete in under 15 seconds for 100 pages, got {}s (avg {}ms/page)",
        duration_secs,
        average_processing_time_ms as u64
    );

    println!("✅ PARALLEL batch processing test completed successfully!");

    // Calculate speedup compared to expected sequential time (19s)
    let expected_sequential_time = 19.0;
    let actual_parallel_time = total_processing_time_ms as f64 / 1000.0;
    let speedup = expected_sequential_time / actual_parallel_time;

    println!(
        "🚀 Performance improvement: {:.1}x faster than sequential processing!",
        speedup
    );
}