webpage_quality_analyzer 1.0.2

//! Debug test for content extraction to understand why Wikipedia main page extraction fails

use webpage_quality_analyzer::content_extraction::{
    create_multi_strategy_extractor, ContentExtractor,
};
use webpage_quality_analyzer::*;

#[tokio::test]
async fn debug_wikipedia_content_extraction() {
    println!("🔍 Debugging content extraction for Wikipedia main page...");

    let url = "https://en.wikipedia.org/wiki/Main_Page";
    println!("📥 Fetching URL: {}", url);

    // Fetch the HTML
    let fetcher = WebFetcher::new(FetchOptions::default()).expect("Failed to create fetcher");
    let html_content = fetcher.fetch_html(url).await.expect("Failed to fetch HTML");

    // Parse the DOM
    let dom = parse_html(&html_content).expect("Failed to parse HTML");
    println!("📊 DOM parsed, total children: {}", dom.children().len());

    // Try different selectors to see what Wikipedia has
    let test_selectors = vec![
        "main",
        "article",
        "[role='main']",
        ".content",
        "#content",
        ".post",
        ".entry",
        "#mw-content-text",
        ".mw-parser-output",
        "#bodyContent",
        "#mw-page-base",
        ".mw-body",
        "#content",
        ".vector-body",
        "#mp-upper",
        "#mp-left",
        "#mp-right",
    ];

    for selector in &test_selectors {
        if let Some(node) = dom
            .query_selector(selector)
            .and_then(|mut iter| iter.next())
        {
            if let Some(found_node) = node.get(dom.parser()) {
                if let Some(tag) = found_node.as_tag() {
                    let inner_html = tag.inner_html(dom.parser());
                    println!("✅ Found '{}': length {} chars", selector, inner_html.len());

                    if let Some(id) = tag.attributes().get("id").flatten() {
                        println!("   🆔 ID: {}", id.as_utf8_str());
                    }
                } else {
                    println!("✅ Found '{}': non-tag node", selector);
                }
            }
        } else {
            println!("❌ Not found: '{}'", selector);
        }
    }

    // Test the heuristic extractor directly to see what it selects
    println!("\n🧪 Testing HeuristicExtractor directly...");
    let heuristic = webpage_quality_analyzer::content_extraction::HeuristicExtractor::new();
    match heuristic.extract_content(&dom) {
        Ok(content_node) => {
            println!("🎯 HeuristicExtractor selected: {:?}", content_node);

            if let Some(node) = content_node.get(dom.parser()) {
                if let Some(tag) = node.as_tag() {
                    let tag_name = tag.name().as_utf8_str();
                    println!("🏷️  Selected node is a '{}' tag", tag_name);

                    if let Some(id) = tag.attributes().get("id").flatten() {
                        println!("🆔 Node ID: {}", id.as_utf8_str());
                    }
                    if let Some(class) = tag.attributes().get("class").flatten() {
                        println!("📝 Node class: {}", class.as_utf8_str());
                    }
                } else {
                    println!("❌ Selected node is not a tag - this is the problem!");
                }
            } else {
                println!("❌ Cannot access selected node");
            }
        }
        Err(e) => {
            println!("❌ HeuristicExtractor failed: {}", e);
        }
    }

    // Test each extractor in the multi-strategy chain individually
    println!("\n🔬 Testing individual extractors in MultiStrategyExtractor chain...");

    println!("1️⃣ Testing ReadabilityExtractor...");
    let readability = webpage_quality_analyzer::content_extraction::ReadabilityExtractor::new();
    match readability.extract_content(&dom) {
        Ok(node) => {
            println!("✅ ReadabilityExtractor returned: {:?}", node);
            if let Some(actual_node) = node.get(dom.parser()) {
                if let Some(tag) = actual_node.as_tag() {
                    println!(
                        "   🏷️ Tag: {} with ID: {:?}",
                        tag.name().as_utf8_str(),
                        tag.attributes()
                            .get("id")
                            .flatten()
                            .map(|id| id.as_utf8_str())
                    );
                } else {
                    println!("   ❌ Not a tag node");
                }
            }
        }
        Err(e) => println!("❌ ReadabilityExtractor failed: {}", e),
    }

    println!("2️⃣ Testing HeuristicExtractor (again)...");
    match heuristic.extract_content(&dom) {
        Ok(node) => println!("✅ HeuristicExtractor returned: {:?}", node),
        Err(e) => println!("❌ HeuristicExtractor failed: {}", e),
    }

    println!("3️⃣ Testing NegativeFilteringExtractor...");
    let negative = webpage_quality_analyzer::content_extraction::NegativeFilteringExtractor::new();
    match negative.extract_content(&dom) {
        Ok(node) => {
            println!("✅ NegativeFilteringExtractor returned: {:?}", node);
            if let Some(actual_node) = node.get(dom.parser()) {
                if let Some(tag) = actual_node.as_tag() {
                    println!(
                        "   🏷️ Tag: {} with ID: {:?}",
                        tag.name().as_utf8_str(),
                        tag.attributes()
                            .get("id")
                            .flatten()
                            .map(|id| id.as_utf8_str())
                    );
                } else {
                    println!("   ❌ Not a tag node");
                }
            }
        }
        Err(e) => println!("❌ NegativeFilteringExtractor failed: {}", e),
    }

    // Test our multi-strategy content extractor
    println!("\n🔄 Testing MultiStrategyExtractor...");
    let content_extractor = create_multi_strategy_extractor();
    match content_extractor.extract_with_fallback(&dom) {
        Ok(content_node) => {
            println!("✅ Content node found: {:?}", content_node);

            // Try to get the node and see what it contains
            if let Some(node) = content_node.get(dom.parser()) {
                if let Some(tag) = node.as_tag() {
                    let tag_name = tag.name().as_utf8_str();
                    println!("🏷️  Content node is a '{}' tag", tag_name);

                    // Get some attributes
                    if let Some(id) = tag.attributes().get("id").flatten() {
                        println!("🆔 Node ID: {}", id.as_utf8_str());
                    }
                    if let Some(class) = tag.attributes().get("class").flatten() {
                        println!("📝 Node class: {}", class.as_utf8_str());
                    }

                    // Get inner HTML length
                    let inner_html = tag.inner_html(dom.parser());
                    println!("📏 Inner HTML length: {} characters", inner_html.len());
                    if inner_html.len() > 0 {
                        println!(
                            "📄 First 200 chars: {}",
                            &inner_html[..inner_html.len().min(200)]
                        );
                    }
                } else {
                    println!("❌ Content node is not a tag");
                }
            } else {
                println!("❌ Cannot access content node");
            }
        }
        Err(e) => {
            println!("❌ Content extraction failed: {}", e);
        }
    }
}