webpage_quality_analyzer 1.0.2

//! Positive Content Selection System
//!
//! This module implements a modern content extraction approach that uses positive
//! selection instead of negative filtering. It identifies and extracts the main
//! content area using heuristic scoring, readability algorithms, and semantic analysis.
//!
//! ## Architecture
//!
//! 1. **Library-First Approach**: Uses readability-rust for Mozilla Readability algorithm
//! 2. **Heuristic Fallback**: Custom scoring based on text density, paragraph count, etc.
//! 3. **Multi-Strategy Pattern**: Combines multiple extraction approaches with graceful fallback
//!
//! ## Content Scoring Algorithm
//!
//! The ContentScore struct evaluates potential content sections based on:
//! - Text density (characters per element)
//! - Paragraph density (number of paragraphs)
//! - Link density penalty (high link ratios reduce score)
//! - Semantic bonuses (content-related IDs/classes)
//!
//! ## Two-Stage Processing
//!
//! 1. **Global Metrics**: Calculated from full DOM for comprehensive analysis
//! 2. **Content Metrics**: Calculated from extracted main content area for focused quality assessment

use crate::models::models::AnalyzeError;
use tl::{Node, VDom};

#[cfg(feature = "readability")]
use readability::extractor::extract;

/// Strategy for content extraction
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExtractionStrategy {
    /// Use readability-rust library (requires readability feature)
    Readability,
    /// Use heuristic scoring algorithm
    Heuristic,
    /// Use negative filtering as fallback
    NegativeFiltering,
    /// Try multiple strategies in sequence
    Multi,
}

/// Content quality score with detailed metrics
#[derive(Debug, Clone, PartialEq)]
pub struct ContentScore {
    /// Overall content quality score (0.0 - 1.0)
    pub total_score: f32,
    /// Text density: characters per DOM element
    pub text_density: f32,
    /// Paragraph density: paragraph count relative to total elements
    pub paragraph_density: f32,
    /// Link density: ratio of link text to total text (penalty factor)
    pub link_density: f32,
    /// Semantic bonus: bonus points for content-related attributes
    pub semantic_bonus: f32,
    /// Number of text nodes analyzed
    pub text_nodes: usize,
    /// Number of paragraphs found
    pub paragraphs: usize,
    /// Total character count
    pub character_count: usize,
}

impl ContentScore {
    /// Create a new ContentScore with default values
    pub fn new() -> Self {
        Self {
            total_score: 0.0,
            text_density: 0.0,
            paragraph_density: 0.0,
            link_density: 0.0,
            semantic_bonus: 0.0,
            text_nodes: 0,
            paragraphs: 0,
            character_count: 0,
        }
    }

    /// Calculate the total score from individual components
    pub fn calculate_total(&mut self) {
        // Base score from text and paragraph density
        let base_score = (self.text_density * 0.4 + self.paragraph_density * 0.3).min(1.0);

        // Apply link density penalty (high link density reduces score)
        let link_penalty = (1.0 - self.link_density).max(0.0);

        // Combine all factors
        self.total_score = (base_score * link_penalty + self.semantic_bonus).min(1.0);
    }
}

impl Default for ContentScore {
    fn default() -> Self {
        Self::new()
    }
}

/// Trait for content extraction strategies
pub trait ContentExtractor: Send + Sync {
    /// Extract the main content node from a DOM
    fn extract_content(&self, dom: &VDom) -> Result<tl::NodeHandle, AnalyzeError>;

    /// Optional: Extract content with detailed scoring
    fn extract_with_score(
        &self,
        dom: &VDom,
    ) -> Result<(tl::NodeHandle, ContentScore), AnalyzeError> {
        let node = self.extract_content(dom)?;
        let score = ContentScore::default(); // Default implementation provides no scoring
        Ok((node, score))
    }
}

/// Readability-based extractor using readability-rust library
pub struct ReadabilityExtractor {
    // Configuration could be added here in the future
}

impl ReadabilityExtractor {
    pub fn new() -> Self {
        Self {}
    }

    /// Find the node in original DOM that best matches readability's extracted content
    fn find_matching_content_node(
        &self,
        dom: &VDom,
        content_preview: &str,
    ) -> Option<tl::NodeHandle> {
        // Try different strategies to find matching content

        // Strategy 1: Look for nodes containing substantial portions of the preview text
        let content_words: Vec<&str> = content_preview.split_whitespace().take(10).collect();

        // Check main content containers first
        let candidates = vec![
            ".block--messages", // Forum thread container (PRIORITY - contains all posts)
            ".thread-content",  // Forum thread content
            ".forum-thread",    // Alternative forum structure
            ".message-content", // Individual forum messages (fallback)
            ".post-content",
            ".forum-post",
            ".comment-content",
            "main",
            "article",
            ".content",
            ".post",
            ".entry",
            "#content",
            "#main",
            ".main-content",
            "body",
        ];

        for selector in candidates {
            if let Some(mut nodes) = dom.query_selector(selector) {
                while let Some(node_handle) = nodes.next() {
                    if let Some(_node) = node_handle.get(dom.parser()) {
                        let node_text = self.extract_node_text(dom, node_handle);

                        // Count how many preview words appear in this node
                        let mut matches = 0;
                        for word in &content_words {
                            if node_text.contains(word) {
                                matches += 1;
                            }
                        }

                        // If we find a good match (>50% of preview words), use this node
                        if matches > content_words.len() / 2 {
                            return Some(node_handle);
                        }
                    }
                }
            }
        }

        // Strategy 2: Find the node with the highest text content
        let mut best_node = None;
        let mut best_text_length = 0;

        if let Some(mut body_nodes) = dom.query_selector("body") {
            if let Some(body) = body_nodes.next() {
                self.find_best_content_node_recursive(
                    dom,
                    body,
                    &mut best_node,
                    &mut best_text_length,
                );
            }
        }

        best_node
    }

    /// Find the node with the most text content - ITERATIVE to prevent stack overflow
    fn find_best_content_node_recursive(
        &self,
        dom: &VDom,
        root_handle: tl::NodeHandle,
        best_node: &mut Option<tl::NodeHandle>,
        best_length: &mut usize,
    ) {
        // Use iterative traversal with explicit stack instead of recursion
        let mut stack = vec![root_handle];

        while let Some(node_handle) = stack.pop() {
            if let Some(node) = node_handle.get(dom.parser()) {
                if let Some(tag) = node.as_tag() {
                    // Skip navigation, header, footer, sidebar elements
                    let tag_name = tag.name().as_utf8_str().to_lowercase();
                    if matches!(
                        tag_name.as_str(),
                        "nav" | "header" | "footer" | "aside" | "script" | "style"
                    ) {
                        continue;
                    }

                    // Check if this node has substantial text content
                    let text = self.extract_node_text(dom, node_handle);
                    let text_length = text.trim().len();

                    if text_length > *best_length && text_length > 200 {
                        *best_length = text_length;
                        *best_node = Some(node_handle);
                    }

                    // Add children to stack for processing
                    for child in tag.children().top().iter() {
                        stack.push(*child);
                    }
                }
            }
        }
    }

    /// Extract text content from a node (similar to HeuristicExtractor)
    fn extract_node_text(&self, dom: &VDom, node_handle: tl::NodeHandle) -> String {
        use crate::content::{clean_text, is_navigation_content};

        let mut text = String::new();
        self.collect_text_recursive(dom, node_handle, &mut text);

        // Clean the extracted text
        let cleaned = clean_text(&text);

        // Filter out navigation content
        if is_navigation_content(&cleaned) {
            return String::new();
        }

        cleaned
    }

    /// Collect text from a node tree - ITERATIVE to prevent stack overflow
    fn collect_text_recursive(&self, dom: &VDom, root_handle: tl::NodeHandle, text: &mut String) {
        // Use iterative traversal with explicit stack instead of recursion
        let mut stack = vec![root_handle];

        while let Some(node_handle) = stack.pop() {
            if let Some(node) = node_handle.get(dom.parser()) {
                match node {
                    tl::Node::Raw(raw) => {
                        let raw_text = raw.as_utf8_str();
                        // Skip very short text fragments that are likely noise
                        if raw_text.trim().len() > 2 {
                            text.push_str(&raw_text);
                            text.push(' ');
                        }
                    }
                    tl::Node::Tag(tag) => {
                        // Skip noise tags and common navigation elements
                        let tag_name = tag.name().as_utf8_str().to_lowercase();
                        if matches!(
                            tag_name.as_str(),
                            "script"
                                | "style"
                                | "noscript"
                                | "iframe"
                                | "embed"
                                | "object"
                                | "button"
                                | "input"
                                | "select"
                                | "textarea"
                                | "form"
                                | "nav"
                                | "header"
                                | "footer"
                                | "aside"
                        ) {
                            continue;
                        }

                        // Check for navigation-like classes or IDs
                        if let Some(Some(class)) = tag.attributes().get("class") {
                            let class_str = class.as_utf8_str().to_lowercase();
                            if class_str.contains("nav")
                                || class_str.contains("menu")
                                || class_str.contains("sidebar")
                                || class_str.contains("breadcrumb")
                                || class_str.contains("toolbar")
                                || class_str.contains("footer")
                                || class_str.contains("header")
                            {
                                continue;
                            }
                        }

                        if let Some(Some(id)) = tag.attributes().get("id") {
                            let id_str = id.as_utf8_str().to_lowercase();
                            if id_str.contains("nav")
                                || id_str.contains("menu")
                                || id_str.contains("sidebar")
                                || id_str.contains("breadcrumb")
                                || id_str.contains("toolbar")
                                || id_str.contains("footer")
                                || id_str.contains("header")
                            {
                                continue;
                            }
                        }

                        // Add children to stack for processing
                        // Note: We push in normal order since stack is LIFO (last in, first out)
                        // This maintains left-to-right traversal order
                        for child in tag.children().top().iter() {
                            stack.push(*child);
                        }
                    }
                    _ => {}
                }
            }
        }
    }
}

impl ContentExtractor for ReadabilityExtractor {
    fn extract_content(&self, dom: &VDom) -> Result<tl::NodeHandle, AnalyzeError> {
        #[cfg(feature = "readability")]
        {
            let html = dom.outer_html();
            match extract(
                &mut html.as_bytes(),
                &url::Url::parse("http://example.com").unwrap(),
            ) {
                Ok(product) => {
                    // Debug: Check what readability extracted
                    eprintln!(
                        "🔍 Readability extracted {} chars: {}",
                        product.content.len(),
                        &product.content[..product.content.len().min(100)]
                    );

                    // Instead of trying to parse extracted content back to DOM,
                    // use heuristic extraction guided by readability's findings
                    eprintln!(
                        "✅ Using readability-guided heuristic extraction for better accuracy"
                    );

                    // Find the best matching node in original DOM based on readability's content
                    let content_preview = &product.content[..product.content.len().min(500)];

                    // Try to find nodes containing similar content in the original DOM
                    if let Some(matching_node) =
                        self.find_matching_content_node(dom, content_preview)
                    {
                        Ok(matching_node)
                    } else {
                        eprintln!("⚠️  Readability content not found in original DOM, using heuristic fallback");
                        let heuristic = HeuristicExtractor::new();
                        heuristic.extract_content(dom)
                    }
                }
                Err(e) => {
                    eprintln!("❌ Readability library failed: {:?}", e);
                    // Fallback to heuristic extraction
                    let heuristic = HeuristicExtractor::new();
                    heuristic.extract_content(dom)
                }
            }
        }

        #[cfg(not(feature = "readability"))]
        {
            // Fallback to heuristic extraction when readability feature is not enabled
            let heuristic = HeuristicExtractor::new();
            heuristic.extract_content(dom)
        }
    }
}

/// Heuristic content extractor using custom scoring algorithm
pub struct HeuristicExtractor {
    // Configuration could be added here
}

impl HeuristicExtractor {
    pub fn new() -> Self {
        Self {}
    }

    /// Analyze a node and calculate its content score
    fn score_node(&self, dom: &VDom, node_handle: tl::NodeHandle) -> ContentScore {
        let mut score = ContentScore::new();

        // Get the node from DOM using proper API
        if let Some(_node) = node_handle.get(dom.parser()) {
            // Count text content
            let text_content = self.extract_text_content(dom, node_handle);
            score.character_count = text_content.len();

            // Count elements and text nodes
            let (elements, text_nodes, paragraphs) = self.count_elements(dom, node_handle);
            score.text_nodes = text_nodes;
            score.paragraphs = paragraphs;

            // Calculate densities
            if elements > 0 {
                score.text_density = score.character_count as f32 / elements as f32;
                score.paragraph_density = paragraphs as f32 / elements as f32;
            }

            // Calculate link density
            score.link_density = self.calculate_link_density(dom, node_handle);

            // Calculate semantic bonus
            score.semantic_bonus = self.calculate_semantic_bonus(dom, node_handle);

            // Calculate total score
            score.calculate_total();
        }

        score
    }

    /// Calculate semantic bonus based on element attributes
    fn calculate_semantic_bonus(&self, dom: &VDom, node_handle: tl::NodeHandle) -> f32 {
        if let Some(node) = node_handle.get(dom.parser()) {
            let mut bonus: f32 = 0.0;

            if let Some(tag) = node.as_tag() {
                // Check ID attribute
                if let Some(id) = tag.attributes().get("id").flatten() {
                    if is_content_id(&id.as_utf8_str()) {
                        bonus += 0.3;
                    }
                }

                // Check class attribute
                if let Some(class) = tag.attributes().get("class").flatten() {
                    if is_content_class(&class.as_utf8_str()) {
                        bonus += 0.2;
                    }
                }
            }

            bonus.min(1.0)
        } else {
            0.0
        }
    }

    /// Calculate link density (ratio of link text to total text)
    fn calculate_link_density(&self, dom: &VDom, node_handle: tl::NodeHandle) -> f32 {
        let total_text = self.extract_text_content(dom, node_handle).len();
        if total_text == 0 {
            return 0.0;
        }

        let mut link_text_length = 0;
        self.traverse_links(dom, node_handle, &mut |link_text| {
            link_text_length += link_text.len();
        });

        link_text_length as f32 / total_text as f32
    }

    /// Count elements, text nodes, and paragraphs
    fn count_elements(&self, dom: &VDom, node_handle: tl::NodeHandle) -> (usize, usize, usize) {
        let mut elements = 0;
        let mut text_nodes = 0;
        let mut paragraphs = 0;

        self.traverse_nodes(dom, node_handle, &mut |node| {
            match node {
                Node::Tag(tag) => {
                    elements += 1;
                    if tag.name().as_utf8_str().eq_ignore_ascii_case("p") {
                        paragraphs += 1;
                    }
                }
                Node::Raw(_) => {
                    text_nodes += 1;
                }
                Node::Comment(_) => {
                    // Comments don't count as content elements
                }
            }
        });

        (elements, text_nodes, paragraphs)
    }

    /// Extract all text content from a node tree
    fn extract_text_content(&self, dom: &VDom, node_handle: tl::NodeHandle) -> String {
        use crate::content::{clean_text, is_navigation_content};

        let mut text = String::new();

        self.traverse_nodes(dom, node_handle, &mut |node| {
            if let Node::Raw(raw) = node {
                let raw_text = raw.as_utf8_str();
                // Skip very short fragments that are likely noise
                if raw_text.trim().len() > 2 {
                    text.push_str(&raw_text);
                }
            }
        });

        // Clean the extracted text
        let cleaned = clean_text(&text);

        // Filter out navigation content
        if is_navigation_content(&cleaned) {
            return String::new();
        }

        cleaned
    }

    /// Traverse all nodes in a subtree and apply a function
    fn traverse_nodes<F>(&self, dom: &VDom, node_handle: tl::NodeHandle, func: &mut F)
    where
        F: FnMut(&Node),
    {
        if let Some(node) = node_handle.get(dom.parser()) {
            // Skip script, style, and other non-content tags
            if let Some(tag) = node.as_tag() {
                let tag_name = tag.name().as_utf8_str().to_lowercase();

                // List of tags to exclude from content extraction
                const EXCLUDED_TAGS: &[&str] = &[
                    "script", "style", "noscript", "iframe", "object", "embed", "svg", "canvas",
                    "code", "pre", // May contain code snippets with CSS/JS
                ];

                if EXCLUDED_TAGS.contains(&tag_name.as_str()) {
                    // Skip this tag and all its children
                    return;
                }
            }

            func(node);

            if let Some(tag) = node.as_tag() {
                for child in tag.children().top().iter() {
                    self.traverse_nodes(dom, *child, func);
                }
            }
        }
    }

    /// Traverse all link nodes and apply a function to their text content
    fn traverse_links<F>(&self, dom: &VDom, node_handle: tl::NodeHandle, func: &mut F)
    where
        F: FnMut(&str),
    {
        if let Some(node) = node_handle.get(dom.parser()) {
            if let Some(tag) = node.as_tag() {
                let tag_name = tag.name().as_utf8_str().to_lowercase();

                // Skip non-content tags when traversing links
                const EXCLUDED_TAGS: &[&str] = &[
                    "script", "style", "noscript", "iframe", "object", "embed", "svg", "canvas",
                ];

                if EXCLUDED_TAGS.contains(&tag_name.as_str()) {
                    return;
                }

                if tag.name().as_utf8_str().eq_ignore_ascii_case("a") {
                    let link_text = self.extract_text_content(dom, node_handle);
                    func(&link_text);
                } else {
                    for child in tag.children().top().iter() {
                        self.traverse_links(dom, *child, func);
                    }
                }
            }
        }
    }
}

impl ContentExtractor for HeuristicExtractor {
    fn extract_content(&self, dom: &VDom) -> Result<tl::NodeHandle, AnalyzeError> {
        let mut best_node: Option<tl::NodeHandle> = None;
        let mut best_score = ContentScore::new();

        // Priority system: First check for Wikipedia main page specific selectors
        // These get manual priority to avoid issues with content density scoring
        let wikipedia_priority_selectors = ["#mp-upper"];
        for selector in wikipedia_priority_selectors {
            if let Some(node) = dom
                .query_selector(selector)
                .and_then(|mut iter| iter.next())
            {
                let score = self.score_node(dom, node);
                // Give Wikipedia main page selectors a priority boost if they meet minimum quality
                if score.total_score > 0.5 {
                    // Using Wikipedia priority selector with adequate quality score
                    return Ok(node);
                }
            }
        }

        // Fallback to standard scoring-based selection
        let candidates = vec![
            // Forum and discussion thread selectors (added for forum support)
            ".block--messages", // XenForo/Ars Technica forum structure
            ".thread-content",
            ".forum-thread",
            ".message-content",
            ".post-content",
            ".forum-post",
            ".comment-content",
            "[class*='message']",
            // Wikipedia main page specific selectors (exclude footer/sister projects)
            "#mp-left",
            "#mp-right",
            // General Wikipedia content selectors
            "#mw-content-text",
            ".mw-parser-output",
            // Standard semantic selectors
            "article",
            "[role='main']",
            // More specific content selectors
            ".entry-content",
            ".article-content",
            "#main-content",
            ".page-content",
            ".content-area",
            ".content",
            ".post",
            ".entry",
            // Generic selectors (lower priority to avoid navigation-heavy containers)
            "main",
            "#content",
            "#contents",
            ".mw-body",
            "#bodyContent",
        ];

        for selector in candidates {
            if let Some(node) = dom
                .query_selector(selector)
                .and_then(|mut iter| iter.next())
            {
                let score = self.score_node(dom, node);
                if score.total_score > best_score.total_score {
                    best_score = score;
                    best_node = Some(node);
                }
            }
        }

        // If we found a good candidate, return it
        if let Some(node) = best_node {
            if best_score.total_score > 0.0 {
                return Ok(node);
            }
        }

        // If no good candidate found, try body as fallback
        if let Some(body) = dom.query_selector("body").and_then(|mut iter| iter.next()) {
            Ok(body)
        } else {
            Err(AnalyzeError::ParseError(
                "No suitable content node found".to_string(),
            ))
        }
    }

    fn extract_with_score(
        &self,
        dom: &VDom,
    ) -> Result<(tl::NodeHandle, ContentScore), AnalyzeError> {
        let node = self.extract_content(dom)?;
        let score = self.score_node(dom, node);
        Ok((node, score))
    }
}

/// Negative filtering extractor - simple fallback implementation
pub struct NegativeFilteringExtractor {
    // Basic fallback extractor
}

impl NegativeFilteringExtractor {
    pub fn new() -> Self {
        Self {}
    }
}

impl ContentExtractor for NegativeFilteringExtractor {
    fn extract_content(&self, dom: &VDom) -> Result<tl::NodeHandle, AnalyzeError> {
        // Simple fallback - returns the body element
        if let Some(body) = dom.query_selector("body").and_then(|mut iter| iter.next()) {
            Ok(body)
        } else if let Some(first_child) = dom.children().iter().next().copied() {
            Ok(first_child)
        } else {
            // As a last resort, try to find the root html element
            if let Some(html) = dom.query_selector("html").and_then(|mut iter| iter.next()) {
                Ok(html)
            } else {
                // For completely empty DOM, we'll need to create a minimal structure to extract from
                // This is a fallback for truly empty input
                Err(AnalyzeError::ParseError(
                    "No content found in document".to_string(),
                ))
            }
        }
    }
}

/// Multi-strategy extractor that tries multiple approaches
pub struct MultiStrategyExtractor {
    extractors: Vec<Box<dyn ContentExtractor>>,
}

impl MultiStrategyExtractor {
    pub fn new() -> Self {
        Self {
            extractors: Vec::new(),
        }
    }

    pub fn add_extractor(&mut self, extractor: Box<dyn ContentExtractor>) {
        self.extractors.push(extractor);
    }

    /// Extract content with fallback chain
    pub fn extract_with_fallback(&self, dom: &VDom) -> Result<tl::NodeHandle, AnalyzeError> {
        let mut last_error = None;

        for extractor in &self.extractors {
            match extractor.extract_content(dom) {
                Ok(node) => return Ok(node),
                Err(e) => last_error = Some(e),
            }
        }

        // If all extractors failed, return the last error or a default error
        Err(last_error.unwrap_or_else(|| {
            AnalyzeError::ParseError("All content extraction strategies failed".to_string())
        }))
    }
}

impl ContentExtractor for MultiStrategyExtractor {
    fn extract_content(&self, dom: &VDom) -> Result<tl::NodeHandle, AnalyzeError> {
        self.extract_with_fallback(dom)
    }
}

/// Check if an ID suggests content area
fn is_content_id(id: &str) -> bool {
    let content_patterns = [
        "content",
        "main",
        "article",
        "post",
        "entry",
        "body-content",
        "page-content",
        "main-content",
        "primary",
        "wrapper",
        "message",
        "comment",
        "thread",
        "discussion",
    ];

    let id_lower = id.to_lowercase();
    content_patterns
        .iter()
        .any(|pattern| id_lower.contains(pattern))
}

/// Check if a class suggests content area
fn is_content_class(class: &str) -> bool {
    let content_patterns = [
        "content",
        "main",
        "article",
        "post",
        "entry",
        "body",
        "page",
        "primary",
        "container",
        "wrapper",
        "message",
        "comment",
        "thread",
        "discussion",
        "forum",
    ];

    let class_lower = class.to_lowercase();
    content_patterns
        .iter()
        .any(|pattern| class_lower.contains(pattern))
}

/// Create a default content extractor instance
pub fn create_content_extractor() -> MultiStrategyExtractor {
    MultiStrategyExtractor::new()
}

/// Create a heuristic-only content extractor (for testing/fallback)
pub fn create_heuristic_extractor() -> Box<dyn ContentExtractor> {
    Box::new(HeuristicExtractor::new())
}

/// Create a readability-only content extractor (when available)
pub fn create_readability_extractor() -> Box<dyn ContentExtractor> {
    Box::new(ReadabilityExtractor::new())
}

/// Create a negative filtering content extractor (fallback)
pub fn create_negative_filtering_extractor() -> Box<dyn ContentExtractor> {
    Box::new(NegativeFilteringExtractor::new())
}

/// Create a multi-strategy content extractor with all fallbacks
pub fn create_multi_strategy_extractor() -> MultiStrategyExtractor {
    let mut extractor = MultiStrategyExtractor::new();
    extractor.add_extractor(Box::new(ReadabilityExtractor::new()));
    extractor.add_extractor(Box::new(HeuristicExtractor::new()));
    extractor.add_extractor(Box::new(NegativeFilteringExtractor::new()));
    extractor
}

#[cfg(test)]
mod tests {
    use super::*;
    use tl::ParserOptions;

    fn create_test_dom(html: &str) -> VDom {
        tl::parse(html, ParserOptions::default()).unwrap()
    }

    #[test]
    fn test_content_score_calculation() {
        let mut score = ContentScore::new();
        score.text_density = 0.8;
        score.paragraph_density = 0.6;
        score.link_density = 0.2;
        score.semantic_bonus = 0.1;

        score.calculate_total();

        assert!(score.total_score > 0.0);
        assert!(score.total_score <= 1.0);
    }

    #[test]
    fn test_heuristic_extractor() {
        let html = r#"
            <html>
                <body>
                    <nav>Navigation</nav>
                    <main id="content">
                        <article>
                            <h1>Main Article</h1>
                            <p>This is the main content of the article.</p>
                            <p>Another paragraph with more content.</p>
                        </article>
                    </main>
                    <footer>Footer content</footer>
                </body>
            </html>
        "#;

        let dom = create_test_dom(html);
        let extractor = HeuristicExtractor::new();

        let result = extractor.extract_content(&dom);
        assert!(result.is_ok());
    }

    #[test]
    fn test_multi_strategy_extractor() {
        let html = r#"
            <html>
                <body>
                    <div class="content">
                        <p>Main content here</p>
                    </div>
                </body>
            </html>
        "#;

        let dom = create_test_dom(html);
        let mut extractor = MultiStrategyExtractor::new();
        extractor.add_extractor(Box::new(HeuristicExtractor::new()));

        let result = extractor.extract_content(&dom);
        assert!(result.is_ok());
    }

    #[test]
    fn test_content_patterns() {
        assert!(is_content_id("main-content"));
        assert!(is_content_id("article-body"));
        assert!(!is_content_id("sidebar"));

        assert!(is_content_class("content main"));
        assert!(is_content_class("post-content"));
        assert!(!is_content_class("navigation"));
    }
}