halldyll_parser/

text.rs

//! Text extraction and processing for halldyll-parser
//!
//! This module handles:
//! - Text extraction from HTML documents
//! - Boilerplate removal (nav, footer, ads, etc.)
//! - Text cleaning and normalization
//! - Readability scoring
//! - Language detection (basic)

use scraper::{Html, ElementRef, Node};
use std::collections::HashSet;

use crate::selector::{SELECTORS, try_parse_selector, BOILERPLATE_SELECTORS, CONTENT_SELECTORS};
use crate::types::{TextContent, ParserConfig, ParserResult};

// ============================================================================
// MAIN TEXT EXTRACTION
// ============================================================================

/// Extract main text content from HTML document
pub fn extract_text(document: &Html, config: &ParserConfig) -> ParserResult<TextContent> {
    // First, try to find main content area
    let main_text = extract_main_content(document, config);
    
    // If we got substantial content, use it
    if !main_text.trim().is_empty() && main_text.split_whitespace().count() > 20 {
        return Ok(TextContent::from_raw(&main_text));
    }
    
    // Fallback to body with boilerplate removal
    let body_text = extract_body_text(document, config);
    Ok(TextContent::from_raw(&body_text))
}

/// Extract text from main content area (article, main, etc.)
fn extract_main_content(document: &Html, config: &ParserConfig) -> String {
    // Try configured content selectors first
    for selector_str in &config.content_selectors {
        if let Some(sel) = try_parse_selector(selector_str) {
            if let Some(element) = document.select(&sel).next() {
                let text = extract_element_text(&element, config);
                if !text.trim().is_empty() {
                    return text;
                }
            }
        }
    }
    
    // Try default content selectors
    for selector_str in CONTENT_SELECTORS {
        if let Some(sel) = try_parse_selector(selector_str) {
            if let Some(element) = document.select(&sel).next() {
                let text = extract_element_text(&element, config);
                if !text.trim().is_empty() {
                    return text;
                }
            }
        }
    }
    
    String::new()
}

/// Extract text from body with boilerplate removal
fn extract_body_text(document: &Html, config: &ParserConfig) -> String {
    if let Some(body) = document.select(&SELECTORS.body).next() {
        extract_element_text_filtered(&body, config)
    } else {
        String::new()
    }
}

/// Extract text from an element, preserving structure
fn extract_element_text(element: &ElementRef, config: &ParserConfig) -> String {
    let mut text = String::new();
    
    for node in element.descendants() {
        match node.value() {
            Node::Text(t) => {
                let content = t.text.trim();
                if !content.is_empty() {
                    if !text.is_empty() && !text.ends_with(' ') && !text.ends_with('\n') {
                        text.push(' ');
                    }
                    text.push_str(content);
                }
            }
            Node::Element(el) => {
                // Add line breaks for block elements
                let tag_name = el.name();
                if is_block_element(tag_name) && !text.is_empty() && !text.ends_with('\n') {
                    text.push('\n');
                }
            }
            _ => {}
        }
    }
    
    if config.preserve_whitespace {
        text
    } else {
        normalize_text(&text)
    }
}

/// Extract text from element, filtering out boilerplate
fn extract_element_text_filtered(element: &ElementRef, config: &ParserConfig) -> String {
    // Collect IDs/classes of elements to skip
    let skip_selectors: Vec<_> = config.remove_selectors.iter()
        .chain(BOILERPLATE_SELECTORS.iter().map(|s| s.to_string()).collect::<Vec<_>>().iter())
        .filter_map(|s| try_parse_selector(s))
        .collect();
    
    let mut text = String::new();
    extract_text_recursive(element, &skip_selectors, &mut text, config);
    
    if config.preserve_whitespace {
        text
    } else {
        normalize_text(&text)
    }
}

/// Recursively extract text, skipping boilerplate elements
fn extract_text_recursive(
    element: &ElementRef,
    skip_selectors: &[scraper::Selector],
    text: &mut String,
    _config: &ParserConfig,
) {
    // Check if this element should be skipped
    for sel in skip_selectors {
        if element.select(sel).next().map(|e| e.id() == element.id()).unwrap_or(false) {
            return;
        }
    }
    
    // Check element name
    let tag_name = element.value().name();
    if should_skip_element(tag_name) {
        return;
    }
    
    // Add block element spacing
    if is_block_element(tag_name) && !text.is_empty() && !text.ends_with('\n') {
        text.push('\n');
    }
    
    for child in element.children() {
        match child.value() {
            Node::Text(t) => {
                let content = t.text.trim();
                if !content.is_empty() {
                    if !text.is_empty() && !text.ends_with(' ') && !text.ends_with('\n') {
                        text.push(' ');
                    }
                    text.push_str(content);
                }
            }
            Node::Element(_) => {
                if let Some(child_el) = ElementRef::wrap(child) {
                    extract_text_recursive(&child_el, skip_selectors, text, _config);
                }
            }
            _ => {}
        }
    }
}

// ============================================================================
// TEXT PROCESSING
// ============================================================================

/// Normalize text (collapse whitespace, trim)
pub fn normalize_text(text: &str) -> String {
    let mut result = String::with_capacity(text.len());
    let mut prev_whitespace = false;
    let mut in_line_start = true;
    
    for c in text.chars() {
        if c == '\n' {
            // Preserve single newlines, collapse multiple
            if !result.ends_with('\n') {
                result.push('\n');
            }
            prev_whitespace = false;
            in_line_start = true;
        } else if c.is_whitespace() {
            if !prev_whitespace && !in_line_start {
                result.push(' ');
                prev_whitespace = true;
            }
        } else {
            result.push(c);
            prev_whitespace = false;
            in_line_start = false;
        }
    }
    
    // Trim and collapse multiple newlines
    let trimmed = result.trim();
    collapse_newlines(trimmed)
}

/// Collapse multiple consecutive newlines to at most 2
fn collapse_newlines(text: &str) -> String {
    let mut result = String::with_capacity(text.len());
    let mut newline_count = 0;
    
    for c in text.chars() {
        if c == '\n' {
            newline_count += 1;
            if newline_count <= 2 {
                result.push(c);
            }
        } else {
            newline_count = 0;
            result.push(c);
        }
    }
    
    result
}

/// Clean text by removing control characters
pub fn clean_text(text: &str) -> String {
    text.chars()
        .filter(|c| !c.is_control() || *c == '\n' || *c == '\t')
        .collect()
}

/// Strip HTML tags from text (for cases where we have HTML strings)
pub fn strip_html_tags(html: &str) -> String {
    let doc = Html::parse_fragment(html);
    let mut text = String::new();
    
    for node in doc.tree.nodes() {
        if let Some(t) = node.value().as_text() {
            text.push_str(&t.text);
        }
    }
    
    normalize_text(&text)
}

// ============================================================================
// ELEMENT CLASSIFICATION
// ============================================================================

/// Check if element should be completely skipped
fn should_skip_element(tag_name: &str) -> bool {
    matches!(tag_name, 
        "script" | "style" | "noscript" | "iframe" | "object" | 
        "embed" | "applet" | "svg" | "canvas" | "map" | "template"
    )
}

/// Check if element is a block element (needs line break)
fn is_block_element(tag_name: &str) -> bool {
    matches!(tag_name,
        "p" | "div" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" |
        "blockquote" | "pre" | "ul" | "ol" | "li" | "dl" | "dt" | "dd" |
        "table" | "tr" | "article" | "section" | "aside" |
        "header" | "footer" | "nav" | "main" | "figure" | "figcaption" |
        "address" | "hr" | "br" | "form" | "fieldset"
    )
}

/// Check if element is inline
pub fn is_inline_element(tag_name: &str) -> bool {
    matches!(tag_name,
        "a" | "span" | "em" | "strong" | "b" | "i" | "u" | "s" |
        "mark" | "small" | "sub" | "sup" | "code" | "kbd" | "samp" | "var" |
        "abbr" | "cite" | "dfn" | "time" | "q" | "label"
    )
}

// ============================================================================
// READABILITY SCORING
// ============================================================================

/// Calculate Flesch-Kincaid Reading Ease score
/// Higher score = easier to read (0-100+)
pub fn flesch_reading_ease(text: &str) -> f64 {
    let words = count_words(text);
    let sentences = count_sentences(text);
    let syllables = count_syllables(text);
    
    if words == 0 || sentences == 0 {
        return 0.0;
    }
    
    let words_f = words as f64;
    let sentences_f = sentences as f64;
    let syllables_f = syllables as f64;
    
    206.835 - 1.015 * (words_f / sentences_f) - 84.6 * (syllables_f / words_f)
}

/// Calculate Flesch-Kincaid Grade Level
/// Returns US school grade level needed to understand text
pub fn flesch_kincaid_grade(text: &str) -> f64 {
    let words = count_words(text);
    let sentences = count_sentences(text);
    let syllables = count_syllables(text);
    
    if words == 0 || sentences == 0 {
        return 0.0;
    }
    
    let words_f = words as f64;
    let sentences_f = sentences as f64;
    let syllables_f = syllables as f64;
    
    0.39 * (words_f / sentences_f) + 11.8 * (syllables_f / words_f) - 15.59
}

/// Count words in text
pub fn count_words(text: &str) -> usize {
    text.split_whitespace().count()
}

/// Count sentences in text
pub fn count_sentences(text: &str) -> usize {
    text.chars()
        .filter(|c| *c == '.' || *c == '!' || *c == '?')
        .count()
        .max(1)
}

/// Estimate syllable count (English approximation)
fn count_syllables(text: &str) -> usize {
    text.split_whitespace()
        .map(count_word_syllables)
        .sum()
}

/// Count syllables in a single word (rough estimate)
fn count_word_syllables(word: &str) -> usize {
    let word = word.to_lowercase();
    let word = word.trim_matches(|c: char| !c.is_alphabetic());
    
    if word.is_empty() {
        return 0;
    }
    
    if word.len() <= 3 {
        return 1;
    }
    
    let vowels: HashSet<char> = ['a', 'e', 'i', 'o', 'u', 'y'].into_iter().collect();
    let mut count = 0;
    let mut prev_vowel = false;
    
    for c in word.chars() {
        let is_vowel = vowels.contains(&c);
        if is_vowel && !prev_vowel {
            count += 1;
        }
        prev_vowel = is_vowel;
    }
    
    // Adjust for silent e
    if word.ends_with('e') && count > 1 {
        count -= 1;
    }
    
    count.max(1)
}

// ============================================================================
// LANGUAGE DETECTION (BASIC)
// ============================================================================

/// Simple language detection based on common words
/// Returns ISO 639-1 language code or None
pub fn detect_language(text: &str) -> Option<String> {
    let lowercase_words: Vec<String> = text.split_whitespace()
        .take(100) // Sample first 100 words
        .map(|w| w.to_lowercase())
        .collect();
    
    let words: Vec<&str> = lowercase_words.iter().map(|s| s.as_str()).collect();
    
    if words.is_empty() {
        return None;
    }
    
    // Common words by language
    let english = ["the", "a", "an", "is", "are", "was", "were", "be", "been", "being", 
                   "have", "has", "had", "do", "does", "did", "will", "would", "could",
                   "should", "may", "might", "must", "shall", "can", "of", "to", "in",
                   "for", "on", "with", "at", "by", "from", "and", "or", "but", "not"];
    
    let french = ["le", "la", "les", "un", "une", "des", "de", "du", "est", "sont",
                  "était", "étaient", "être", "avoir", "a", "ont", "fait", "faire",
                  "dit", "dire", "que", "qui", "quoi", "où", "quand", "comment",
                  "pour", "sur", "avec", "dans", "par", "et", "ou", "mais", "ne", "pas"];
    
    let german = ["der", "die", "das", "ein", "eine", "ist", "sind", "war", "waren",
                  "sein", "haben", "hat", "hatte", "hatten", "werden", "wird", "wurde",
                  "und", "oder", "aber", "nicht", "für", "auf", "mit", "in", "an", "von",
                  "zu", "bei", "nach", "aus", "über", "durch", "wenn", "als", "ob"];
    
    let spanish = ["el", "la", "los", "las", "un", "una", "unos", "unas", "es", "son",
                   "era", "eran", "ser", "estar", "tener", "tiene", "hacer", "hecho",
                   "que", "qué", "quien", "quién", "donde", "dónde", "cuando", "cuándo",
                   "para", "por", "con", "en", "de", "y", "o", "pero", "no", "si"];
    
    // Simple word count matching
    let words_text: String = words.iter().map(|w| w.to_string()).collect::<Vec<_>>().join(" ");
    
    let en_count = english.iter().filter(|w| words_text.contains(*w)).count();
    let fr_count = french.iter().filter(|w| words_text.contains(*w)).count();
    let de_count = german.iter().filter(|w| words_text.contains(*w)).count();
    let es_count = spanish.iter().filter(|w| words_text.contains(*w)).count();
    
    let max_count = en_count.max(fr_count).max(de_count).max(es_count);
    
    if max_count < 3 {
        return None; // Not enough confidence
    }
    
    if en_count == max_count {
        Some("en".to_string())
    } else if fr_count == max_count {
        Some("fr".to_string())
    } else if de_count == max_count {
        Some("de".to_string())
    } else if es_count == max_count {
        Some("es".to_string())
    } else {
        None
    }
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_text_simple() {
        let doc = parse_html("<html><body><p>Hello world</p></body></html>");
        let config = ParserConfig::default();
        let text = extract_text(&doc, &config).unwrap();
        assert!(text.cleaned_text.contains("Hello world"));
    }

    #[test]
    fn test_extract_text_from_article() {
        let doc = parse_html(r#"
            <html>
            <body>
                <nav>Navigation here</nav>
                <article>
                    <h1>Title</h1>
                    <p>This is the main content of the article.</p>
                    <p>Another paragraph with more content.</p>
                </article>
                <footer>Footer here</footer>
            </body>
            </html>
        "#);
        let config = ParserConfig::default();
        let text = extract_text(&doc, &config).unwrap();
        assert!(text.cleaned_text.contains("main content"));
    }

    #[test]
    fn test_extract_text_skips_script() {
        let doc = parse_html(r#"
            <html>
            <body>
                <p>Visible text</p>
                <script>var x = "invisible";</script>
                <p>More visible text</p>
            </body>
            </html>
        "#);
        let config = ParserConfig::default();
        let text = extract_text(&doc, &config).unwrap();
        assert!(text.cleaned_text.contains("Visible text"));
        assert!(!text.cleaned_text.contains("invisible"));
    }

    #[test]
    fn test_normalize_text() {
        let input = "  Hello   world  \n\n\n  multiple   spaces  ";
        let result = normalize_text(input);
        // Normalizes: trims, collapses spaces, preserves up to 2 newlines
        assert_eq!(result, "Hello world \nmultiple spaces");
    }

    #[test]
    fn test_clean_text() {
        let input = "Hello\x00World\x01Test\nNewline";
        let cleaned = clean_text(input);
        assert_eq!(cleaned, "HelloWorldTest\nNewline");
    }

    #[test]
    fn test_strip_html_tags() {
        let html = "<p>Hello <strong>world</strong></p>";
        let text = strip_html_tags(html);
        assert_eq!(text, "Hello world");
    }

    #[test]
    fn test_count_words() {
        assert_eq!(count_words("Hello world test"), 3);
        assert_eq!(count_words("One"), 1);
        assert_eq!(count_words("   "), 0);
    }

    #[test]
    fn test_count_sentences() {
        assert_eq!(count_sentences("Hello. World! How?"), 3);
        assert_eq!(count_sentences("No punctuation"), 1);
    }

    #[test]
    fn test_flesch_reading_ease() {
        // Simple text should have high score (easy to read)
        let simple = "The cat sat on the mat. The dog ran fast.";
        let score = flesch_reading_ease(simple);
        assert!(score > 60.0, "Simple text should be easy to read: {}", score);
    }

    #[test]
    fn test_flesch_kincaid_grade() {
        let simple = "The cat sat. The dog ran.";
        let grade = flesch_kincaid_grade(simple);
        assert!(grade < 6.0, "Simple text should be low grade level: {}", grade);
    }

    #[test]
    fn test_count_word_syllables() {
        assert_eq!(count_word_syllables("cat"), 1);
        assert_eq!(count_word_syllables("hello"), 2);
        assert_eq!(count_word_syllables("beautiful"), 3); // beau-ti-ful
        assert_eq!(count_word_syllables("extraordinary"), 5); // ex-tra-or-di-na-ry (algorithm may count differently)
    }

    #[test]
    fn test_detect_language_english() {
        let text = "The quick brown fox jumps over the lazy dog. This is a test of the English language detection system.";
        assert_eq!(detect_language(text), Some("en".to_string()));
    }

    #[test]
    fn test_detect_language_french() {
        let text = "Le chat est sur la table. C'est un beau jour pour une promenade dans le parc.";
        assert_eq!(detect_language(text), Some("fr".to_string()));
    }

    #[test]
    fn test_detect_language_german() {
        let text = "Der Hund ist auf dem Tisch. Das ist ein schöner Tag für einen Spaziergang im Park.";
        assert_eq!(detect_language(text), Some("de".to_string()));
    }

    #[test]
    fn test_detect_language_spanish() {
        let text = "El gato está en la mesa. Es un buen día para un paseo en el parque.";
        assert_eq!(detect_language(text), Some("es".to_string()));
    }

    #[test]
    fn test_detect_language_insufficient() {
        let text = "xyz abc 123";
        assert_eq!(detect_language(text), None);
    }

    #[test]
    fn test_is_block_element() {
        assert!(is_block_element("p"));
        assert!(is_block_element("div"));
        assert!(is_block_element("h1"));
        assert!(!is_block_element("span"));
        assert!(!is_block_element("a"));
    }

    #[test]
    fn test_is_inline_element() {
        assert!(is_inline_element("span"));
        assert!(is_inline_element("a"));
        assert!(is_inline_element("strong"));
        assert!(!is_inline_element("div"));
        assert!(!is_inline_element("p"));
    }

    #[test]
    fn test_should_skip_element() {
        assert!(should_skip_element("script"));
        assert!(should_skip_element("style"));
        assert!(should_skip_element("noscript"));
        assert!(!should_skip_element("p"));
        assert!(!should_skip_element("div"));
    }

    #[test]
    fn test_text_content_reading_time() {
        // 225 words = ~1 minute
        let words = "word ".repeat(225);
        let content = TextContent::from_raw(&words);
        let time = content.reading_time_minutes.unwrap();
        assert!((time - 1.0).abs() < 0.1);
    }

    #[test]
    fn test_text_content_word_count() {
        let content = TextContent::from_raw("Hello world test");
        assert_eq!(content.word_count, 3);
    }
}