halldyll-parser 0.1.0

//! Type definitions for halldyll-parser
//!
//! This module contains all public types used throughout the parser:
//! - Error types
//! - Content types (text, headings, lists, tables, etc.)
//! - Metadata types (OpenGraph, Twitter Cards, etc.)
//! - Structured data types (JSON-LD, Microdata, RDFa)

use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use thiserror::Error;

// ============================================================================
// ERROR TYPES
// ============================================================================

/// Errors that can occur during HTML parsing
#[derive(Debug, Error)]
pub enum ParserError {
    /// HTML parsing failed
    #[error("Failed to parse HTML: {0}")]
    ParseError(String),

    /// Invalid selector syntax
    #[error("Invalid CSS selector: {0}")]
    SelectorError(String),

    /// URL parsing/resolution error
    #[error("URL error: {0}")]
    UrlError(#[from] url::ParseError),

    /// IO error (reading files, etc.)
    #[error("IO error: {0}")]
    IoError(#[from] std::io::Error),

    /// Encoding error
    #[error("Encoding error: {0}")]
    EncodingError(String),

    /// Configuration error
    #[error("Configuration error: {0}")]
    ConfigError(String),
}

/// Result type for parser operations
pub type ParserResult<T> = Result<T, ParserError>;

// ============================================================================
// TEXT CONTENT
// ============================================================================

/// Extracted text content with metadata
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct TextContent {
    /// Raw extracted text
    pub raw_text: String,

    /// Cleaned text (whitespace normalized)
    pub cleaned_text: String,

    /// Word count
    pub word_count: usize,

    /// Character count
    pub char_count: usize,

    /// Detected language (ISO 639-1 code)
    pub language: Option<String>,

    /// Readability score (Flesch-Kincaid or similar)
    pub readability_score: Option<f64>,

    /// Estimated reading time in minutes
    pub reading_time_minutes: Option<f64>,
}

impl TextContent {
    /// Create new text content from raw text
    pub fn from_raw(raw: &str) -> Self {
        let cleaned = normalize_whitespace(raw);
        let word_count = cleaned.split_whitespace().count();
        let char_count = cleaned.chars().count();
        
        // Average reading speed: 200-250 WPM, we use 225
        let reading_time = if word_count > 0 {
            Some(word_count as f64 / 225.0)
        } else {
            None
        };

        Self {
            raw_text: raw.to_string(),
            cleaned_text: cleaned,
            word_count,
            char_count,
            language: None,
            readability_score: None,
            reading_time_minutes: reading_time,
        }
    }

    /// Check if content is empty or minimal
    pub fn is_empty(&self) -> bool {
        self.word_count == 0
    }

    /// Check if content is substantial (more than just a few words)
    pub fn is_substantial(&self) -> bool {
        self.word_count >= 50
    }
}

// ============================================================================
// HEADINGS
// ============================================================================

/// A heading element (h1-h6)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Heading {
    /// Heading level (1-6)
    pub level: u8,

    /// Heading text content
    pub text: String,

    /// ID attribute if present
    pub id: Option<String>,

    /// Class names if present
    pub classes: Vec<String>,
}

impl Heading {
    /// Create a new heading
    pub fn new(level: u8, text: impl Into<String>) -> Self {
        Self {
            level: level.clamp(1, 6),
            text: text.into(),
            id: None,
            classes: Vec::new(),
        }
    }

    /// Create heading with ID
    pub fn with_id(mut self, id: impl Into<String>) -> Self {
        self.id = Some(id.into());
        self
    }
}

// ============================================================================
// LINKS
// ============================================================================

/// Relationship types for links
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LinkRel {
    /// Standard follow link
    Follow,
    /// nofollow link
    NoFollow,
    /// ugc (user generated content)
    Ugc,
    /// sponsored link
    Sponsored,
    /// external link
    External,
    /// noopener
    NoOpener,
    /// noreferrer
    NoReferrer,
    /// Other rel value
    Other,
}

/// Type of link (internal vs external)
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LinkType {
    /// Link to same domain
    Internal,
    /// Link to different domain
    External,
    /// Cannot determine (no base URL)
    Unknown,
}

/// An extracted link
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Link {
    /// Original href value
    pub href: String,

    /// Resolved absolute URL (if possible)
    pub url: Option<String>,

    /// Anchor text
    pub text: String,

    /// Title attribute
    pub title: Option<String>,

    /// Relationship attributes
    pub rel: Vec<LinkRel>,

    /// Link type (internal/external)
    pub link_type: LinkType,

    /// Whether link is nofollow
    pub is_nofollow: bool,

    /// Target attribute (_blank, _self, etc.)
    pub target: Option<String>,

    /// hreflang attribute
    pub hreflang: Option<String>,
}

impl Link {
    /// Create a new link
    pub fn new(href: impl Into<String>, text: impl Into<String>) -> Self {
        Self {
            href: href.into(),
            url: None,
            text: text.into(),
            title: None,
            rel: Vec::new(),
            link_type: LinkType::Unknown,
            is_nofollow: false,
            target: None,
            hreflang: None,
        }
    }

    /// Check if link should be followed by crawlers
    pub fn should_follow(&self) -> bool {
        !self.is_nofollow && !self.rel.contains(&LinkRel::Sponsored) && !self.rel.contains(&LinkRel::Ugc)
    }

    /// Check if link opens in new tab
    pub fn opens_new_tab(&self) -> bool {
        self.target.as_deref() == Some("_blank")
    }
}

// ============================================================================
// IMAGES
// ============================================================================

/// Image loading strategy
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[derive(Default)]
pub enum ImageLoading {
    /// Eager loading (default)
    #[default]
    Eager,
    /// Lazy loading
    Lazy,
}


/// An extracted image
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Image {
    /// Original src attribute
    pub src: String,

    /// Resolved absolute URL
    pub url: Option<String>,

    /// Alt text
    pub alt: String,

    /// Title attribute
    pub title: Option<String>,

    /// Width if specified
    pub width: Option<u32>,

    /// Height if specified
    pub height: Option<u32>,

    /// srcset for responsive images
    pub srcset: Option<String>,

    /// sizes attribute
    pub sizes: Option<String>,

    /// Loading strategy (lazy/eager)
    pub loading: ImageLoading,

    /// Whether image is decorative (empty alt)
    pub is_decorative: bool,
}

impl Image {
    /// Create a new image
    pub fn new(src: impl Into<String>, alt: impl Into<String>) -> Self {
        let alt_str = alt.into();
        let is_decorative = alt_str.is_empty();
        Self {
            src: src.into(),
            url: None,
            alt: alt_str,
            title: None,
            width: None,
            height: None,
            srcset: None,
            sizes: None,
            loading: ImageLoading::default(),
            is_decorative,
        }
    }

    /// Check if image has responsive srcset
    pub fn is_responsive(&self) -> bool {
        self.srcset.is_some()
    }
}

// ============================================================================
// LISTS
// ============================================================================

/// Type of list
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ListType {
    /// Ordered list (ol)
    Ordered,
    /// Unordered list (ul)
    Unordered,
    /// Definition list (dl)
    Definition,
}

/// A list item (may contain nested content)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListItem {
    /// Item text content
    pub text: String,

    /// Nested list if any
    pub nested: Option<Box<ListContent>>,
}

impl ListItem {
    /// Create a simple list item
    pub fn new(text: impl Into<String>) -> Self {
        Self {
            text: text.into(),
            nested: None,
        }
    }

    /// Create item with nested list
    pub fn with_nested(text: impl Into<String>, nested: ListContent) -> Self {
        Self {
            text: text.into(),
            nested: Some(Box::new(nested)),
        }
    }
}

/// An extracted list
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ListContent {
    /// Type of list
    pub list_type: ListType,

    /// List items
    pub items: Vec<ListItem>,

    /// Total item count (including nested)
    pub total_items: usize,
}

impl ListContent {
    /// Create a new list
    pub fn new(list_type: ListType) -> Self {
        Self {
            list_type,
            items: Vec::new(),
            total_items: 0,
        }
    }

    /// Add an item
    pub fn add_item(&mut self, item: ListItem) {
        self.total_items += 1;
        if let Some(ref nested) = item.nested {
            self.total_items += nested.total_items;
        }
        self.items.push(item);
    }

    /// Check if list is empty
    pub fn is_empty(&self) -> bool {
        self.items.is_empty()
    }
}

// ============================================================================
// TABLES
// ============================================================================

/// A table cell
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TableCell {
    /// Cell content
    pub content: String,

    /// Is header cell (th)
    pub is_header: bool,

    /// Column span
    pub colspan: u32,

    /// Row span
    pub rowspan: u32,
}

impl TableCell {
    /// Create a data cell
    pub fn data(content: impl Into<String>) -> Self {
        Self {
            content: content.into(),
            is_header: false,
            colspan: 1,
            rowspan: 1,
        }
    }

    /// Create a header cell
    pub fn header(content: impl Into<String>) -> Self {
        Self {
            content: content.into(),
            is_header: true,
            colspan: 1,
            rowspan: 1,
        }
    }
}

/// A table row
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TableRow {
    /// Cells in this row
    pub cells: Vec<TableCell>,

    /// Is this a header row
    pub is_header_row: bool,
}

impl TableRow {
    /// Create a new row
    pub fn new(cells: Vec<TableCell>) -> Self {
        let is_header = cells.iter().all(|c| c.is_header);
        Self {
            cells,
            is_header_row: is_header,
        }
    }
}

/// An extracted table
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TableContent {
    /// Table caption
    pub caption: Option<String>,

    /// Header rows
    pub headers: Vec<TableRow>,

    /// Body rows
    pub rows: Vec<TableRow>,

    /// Number of columns
    pub column_count: usize,

    /// Table summary (if provided)
    pub summary: Option<String>,
}

impl TableContent {
    /// Create a new empty table
    pub fn new() -> Self {
        Self {
            caption: None,
            headers: Vec::new(),
            rows: Vec::new(),
            column_count: 0,
            summary: None,
        }
    }

    /// Check if table is empty
    pub fn is_empty(&self) -> bool {
        self.headers.is_empty() && self.rows.is_empty()
    }

    /// Get total row count
    pub fn row_count(&self) -> usize {
        self.headers.len() + self.rows.len()
    }
}

impl Default for TableContent {
    fn default() -> Self {
        Self::new()
    }
}

// ============================================================================
// CODE BLOCKS
// ============================================================================

/// An extracted code block
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CodeBlock {
    /// The code content
    pub code: String,

    /// Programming language (if detected)
    pub language: Option<String>,

    /// Line count
    pub line_count: usize,

    /// Whether it's inline code
    pub is_inline: bool,

    /// Filename if specified (e.g., in markdown)
    pub filename: Option<String>,
}

impl CodeBlock {
    /// Create a new code block
    pub fn new(code: impl Into<String>) -> Self {
        let code_str = code.into();
        let line_count = code_str.lines().count();
        Self {
            code: code_str,
            language: None,
            line_count,
            is_inline: false,
            filename: None,
        }
    }

    /// Create with language
    pub fn with_language(mut self, lang: impl Into<String>) -> Self {
        self.language = Some(lang.into());
        self
    }

    /// Mark as inline
    pub fn inline(mut self) -> Self {
        self.is_inline = true;
        self
    }
}

// ============================================================================
// QUOTES
// ============================================================================

/// An extracted blockquote
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Quote {
    /// Quote text content
    pub text: String,

    /// Citation/source
    pub cite: Option<String>,

    /// Citation URL
    pub cite_url: Option<String>,
}

impl Quote {
    /// Create a new quote
    pub fn new(text: impl Into<String>) -> Self {
        Self {
            text: text.into(),
            cite: None,
            cite_url: None,
        }
    }

    /// Add citation
    pub fn with_cite(mut self, cite: impl Into<String>) -> Self {
        self.cite = Some(cite.into());
        self
    }
}

// ============================================================================
// METADATA TYPES
// ============================================================================

/// OpenGraph metadata
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct OpenGraph {
    /// og:title
    pub title: Option<String>,

    /// og:type
    pub og_type: Option<String>,

    /// og:url
    pub url: Option<String>,

    /// og:image
    pub image: Option<String>,

    /// og:description
    pub description: Option<String>,

    /// og:site_name
    pub site_name: Option<String>,

    /// og:locale
    pub locale: Option<String>,

    /// og:video
    pub video: Option<String>,

    /// og:audio
    pub audio: Option<String>,

    /// Additional properties
    pub extra: HashMap<String, String>,
}

impl OpenGraph {
    /// Check if OG data is present
    pub fn is_present(&self) -> bool {
        self.title.is_some() || self.og_type.is_some() || self.url.is_some()
    }
}

/// Twitter Card metadata
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct TwitterCard {
    /// twitter:card
    pub card: Option<String>,

    /// twitter:site
    pub site: Option<String>,

    /// twitter:creator
    pub creator: Option<String>,

    /// twitter:title
    pub title: Option<String>,

    /// twitter:description
    pub description: Option<String>,

    /// twitter:image
    pub image: Option<String>,

    /// Additional properties
    pub extra: HashMap<String, String>,
}

impl TwitterCard {
    /// Check if Twitter Card data is present
    pub fn is_present(&self) -> bool {
        self.card.is_some() || self.site.is_some()
    }
}

/// Robots meta directives
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RobotsMeta {
    /// Can be indexed
    pub index: bool,

    /// Links can be followed
    pub follow: bool,

    /// Can be archived
    pub archive: bool,

    /// Can be cached
    pub cache: bool,

    /// Can show snippet
    pub snippet: bool,

    /// Max snippet length (-1 = unlimited)
    pub max_snippet: i32,

    /// Max image preview (none, standard, large)
    pub max_image_preview: Option<String>,

    /// Max video preview seconds
    pub max_video_preview: i32,

    /// Raw robots content
    pub raw: Option<String>,
}

impl RobotsMeta {
    /// Create default (all allowed)
    pub fn allowed() -> Self {
        Self {
            index: true,
            follow: true,
            archive: true,
            cache: true,
            snippet: true,
            max_snippet: -1,
            max_image_preview: Some("large".to_string()),
            max_video_preview: -1,
            raw: None,
        }
    }

    /// Create noindex nofollow
    pub fn noindex_nofollow() -> Self {
        Self {
            index: false,
            follow: false,
            ..Self::allowed()
        }
    }
}

/// Alternate language version (hreflang)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlternateLink {
    /// Language code (e.g., "en", "fr", "x-default")
    pub hreflang: String,

    /// URL of alternate version
    pub href: String,
}

/// Complete page metadata
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PageMetadata {
    /// Page title
    pub title: Option<String>,

    /// Meta description
    pub description: Option<String>,

    /// Meta keywords
    pub keywords: Vec<String>,

    /// Author
    pub author: Option<String>,

    /// Generator (CMS/framework)
    pub generator: Option<String>,

    /// Canonical URL
    pub canonical: Option<String>,

    /// Base URL from <base> tag
    pub base_url: Option<String>,

    /// Language (html lang attribute)
    pub language: Option<String>,

    /// Character encoding
    pub charset: Option<String>,

    /// Viewport
    pub viewport: Option<String>,

    /// Robots directives
    pub robots: RobotsMeta,

    /// OpenGraph data
    pub opengraph: OpenGraph,

    /// Twitter Card data
    pub twitter: TwitterCard,

    /// Alternate language versions
    pub alternates: Vec<AlternateLink>,

    /// Favicon URL
    pub favicon: Option<String>,

    /// Apple touch icon
    pub apple_touch_icon: Option<String>,

    /// Theme color
    pub theme_color: Option<String>,

    /// Published date
    pub published_date: Option<String>,

    /// Modified date
    pub modified_date: Option<String>,

    /// Schema.org type (if detected)
    pub schema_type: Option<String>,

    /// Custom meta tags (name -> content)
    pub custom: HashMap<String, String>,
}

impl PageMetadata {
    /// Get effective title (OG > Twitter > title tag)
    pub fn effective_title(&self) -> Option<&str> {
        self.opengraph.title.as_deref()
            .or(self.twitter.title.as_deref())
            .or(self.title.as_deref())
    }

    /// Get effective description
    pub fn effective_description(&self) -> Option<&str> {
        self.opengraph.description.as_deref()
            .or(self.twitter.description.as_deref())
            .or(self.description.as_deref())
    }

    /// Get effective image
    pub fn effective_image(&self) -> Option<&str> {
        self.opengraph.image.as_deref()
            .or(self.twitter.image.as_deref())
    }

    /// Check if page should be indexed
    pub fn should_index(&self) -> bool {
        self.robots.index
    }

    /// Check if links should be followed
    pub fn should_follow(&self) -> bool {
        self.robots.follow
    }
}

// ============================================================================
// STRUCTURED DATA
// ============================================================================

/// Type of structured data
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StructuredDataFormat {
    /// JSON-LD (recommended)
    JsonLd,
    /// Microdata
    Microdata,
    /// RDFa
    Rdfa,
}

/// Extracted structured data item
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StructuredData {
    /// Format (JSON-LD, Microdata, RDFa)
    pub format: StructuredDataFormat,

    /// Schema.org type (e.g., "Article", "Product", "Organization")
    pub schema_type: Option<String>,

    /// Raw JSON content (for JSON-LD)
    pub raw_json: Option<String>,

    /// Parsed properties
    pub properties: HashMap<String, serde_json::Value>,
}

impl StructuredData {
    /// Create JSON-LD data
    pub fn json_ld(raw: impl Into<String>) -> Self {
        Self {
            format: StructuredDataFormat::JsonLd,
            schema_type: None,
            raw_json: Some(raw.into()),
            properties: HashMap::new(),
        }
    }

    /// Create Microdata
    pub fn microdata(schema_type: impl Into<String>) -> Self {
        Self {
            format: StructuredDataFormat::Microdata,
            schema_type: Some(schema_type.into()),
            raw_json: None,
            properties: HashMap::new(),
        }
    }
}

// ============================================================================
// PARSED CONTENT (COMPLETE RESULT)
// ============================================================================

/// Complete parsed content from an HTML document
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ParsedContent {
    /// Page metadata
    pub metadata: PageMetadata,

    /// Extracted text content
    pub text: TextContent,

    /// All headings
    pub headings: Vec<Heading>,

    /// All paragraphs
    pub paragraphs: Vec<String>,

    /// All links
    pub links: Vec<Link>,

    /// All images
    pub images: Vec<Image>,

    /// All lists
    pub lists: Vec<ListContent>,

    /// All tables
    pub tables: Vec<TableContent>,

    /// All code blocks
    pub code_blocks: Vec<CodeBlock>,

    /// All quotes
    pub quotes: Vec<Quote>,

    /// Structured data (JSON-LD, Microdata, RDFa)
    pub structured_data: Vec<StructuredData>,

    /// Parsing statistics
    pub stats: ParseStats,
}

impl ParsedContent {
    /// Get internal links only
    pub fn internal_links(&self) -> Vec<&Link> {
        self.links.iter().filter(|l| l.link_type == LinkType::Internal).collect()
    }

    /// Get external links only
    pub fn external_links(&self) -> Vec<&Link> {
        self.links.iter().filter(|l| l.link_type == LinkType::External).collect()
    }

    /// Get followable links only
    pub fn followable_links(&self) -> Vec<&Link> {
        self.links.iter().filter(|l| l.should_follow()).collect()
    }

    /// Get the document outline (headings hierarchy)
    pub fn outline(&self) -> Vec<&Heading> {
        self.headings.iter().collect()
    }

    /// Check if page has structured data
    pub fn has_structured_data(&self) -> bool {
        !self.structured_data.is_empty()
    }
}

/// Parsing statistics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ParseStats {
    /// HTML size in bytes
    pub html_size: usize,

    /// Parse time in microseconds
    pub parse_time_us: u64,

    /// Number of DOM nodes
    pub node_count: usize,

    /// Number of elements
    pub element_count: usize,

    /// Number of text nodes
    pub text_node_count: usize,

    /// Number of comments
    pub comment_count: usize,

    /// Errors encountered during parsing
    pub errors: Vec<String>,

    /// Warnings
    pub warnings: Vec<String>,
}

impl ParseStats {
    /// Check if parsing had errors
    pub fn has_errors(&self) -> bool {
        !self.errors.is_empty()
    }

    /// Check if parsing had warnings
    pub fn has_warnings(&self) -> bool {
        !self.warnings.is_empty()
    }
}

// ============================================================================
// PARSER CONFIGURATION
// ============================================================================

/// Configuration for the HTML parser
#[derive(Debug, Clone)]
pub struct ParserConfig {
    /// Base URL for resolving relative URLs
    pub base_url: Option<url::Url>,

    /// Maximum text length to extract
    pub max_text_length: usize,

    /// Whether to extract images
    pub extract_images: bool,

    /// Whether to extract links
    pub extract_links: bool,

    /// Whether to extract tables
    pub extract_tables: bool,

    /// Whether to extract code blocks
    pub extract_code_blocks: bool,

    /// Whether to extract structured data
    pub extract_structured_data: bool,

    /// Whether to compute readability scores
    pub compute_readability: bool,

    /// Minimum paragraph length to include
    pub min_paragraph_length: usize,

    /// Content selectors (CSS selectors for main content)
    pub content_selectors: Vec<String>,

    /// Selectors for elements to remove (ads, nav, footer, etc.)
    pub remove_selectors: Vec<String>,

    /// Whether to preserve whitespace
    pub preserve_whitespace: bool,
}

impl Default for ParserConfig {
    fn default() -> Self {
        Self {
            base_url: None,
            max_text_length: 1_000_000, // 1MB
            extract_images: true,
            extract_links: true,
            extract_tables: true,
            extract_code_blocks: true,
            extract_structured_data: true,
            compute_readability: false,
            min_paragraph_length: 20,
            content_selectors: vec![
                "article".to_string(),
                "main".to_string(),
                "[role=main]".to_string(),
                ".content".to_string(),
                ".post-content".to_string(),
                ".entry-content".to_string(),
            ],
            remove_selectors: vec![
                "script".to_string(),
                "style".to_string(),
                "noscript".to_string(),
                "nav".to_string(),
                "header".to_string(),
                "footer".to_string(),
                "aside".to_string(),
                ".sidebar".to_string(),
                ".advertisement".to_string(),
                ".ad".to_string(),
                ".ads".to_string(),
                "[role=navigation]".to_string(),
                "[role=banner]".to_string(),
                "[role=contentinfo]".to_string(),
            ],
            preserve_whitespace: false,
        }
    }
}

impl ParserConfig {
    /// Create a new config with base URL
    pub fn with_base_url(url: impl AsRef<str>) -> Result<Self, url::ParseError> {
        Ok(Self {
            base_url: Some(url::Url::parse(url.as_ref())?),
            ..Default::default()
        })
    }

    /// Create minimal config (faster, less extraction)
    pub fn minimal() -> Self {
        Self {
            extract_images: false,
            extract_tables: false,
            extract_code_blocks: false,
            extract_structured_data: false,
            compute_readability: false,
            ..Default::default()
        }
    }

    /// Create config for full extraction
    pub fn full() -> Self {
        Self {
            compute_readability: true,
            ..Default::default()
        }
    }

    /// Set base URL
    pub fn base_url(mut self, url: url::Url) -> Self {
        self.base_url = Some(url);
        self
    }

    /// Add content selector
    pub fn add_content_selector(mut self, selector: impl Into<String>) -> Self {
        self.content_selectors.push(selector.into());
        self
    }

    /// Add remove selector
    pub fn add_remove_selector(mut self, selector: impl Into<String>) -> Self {
        self.remove_selectors.push(selector.into());
        self
    }
}

// ============================================================================
// HELPER FUNCTIONS
// ============================================================================

/// Normalize whitespace in text
pub fn normalize_whitespace(text: &str) -> String {
    // Replace multiple whitespace with single space
    let mut result = String::with_capacity(text.len());
    let mut prev_ws = false;
    
    for c in text.chars() {
        if c.is_whitespace() {
            if !prev_ws {
                result.push(' ');
                prev_ws = true;
            }
        } else {
            result.push(c);
            prev_ws = false;
        }
    }
    
    result.trim().to_string()
}

/// Clean text by removing control characters
pub fn clean_text(text: &str) -> String {
    text.chars()
        .filter(|c| !c.is_control() || c.is_whitespace())
        .collect::<String>()
}

/// Truncate text to max length with ellipsis
pub fn truncate_text(text: &str, max_len: usize) -> String {
    if text.len() <= max_len {
        text.to_string()
    } else {
        let mut truncated = text.chars().take(max_len - 3).collect::<String>();
        truncated.push_str("...");
        truncated
    }
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_text_content_creation() {
        let text = TextContent::from_raw("Hello   world,   this is   a test.");
        assert_eq!(text.cleaned_text, "Hello world, this is a test.");
        assert_eq!(text.word_count, 6);
        assert!(!text.is_empty());
    }

    #[test]
    fn test_heading_creation() {
        let h1 = Heading::new(1, "Main Title").with_id("main");
        assert_eq!(h1.level, 1);
        assert_eq!(h1.id, Some("main".to_string()));
    }

    #[test]
    fn test_heading_level_clamping() {
        let h = Heading::new(10, "Test");
        assert_eq!(h.level, 6); // Clamped to max
    }

    #[test]
    fn test_link_creation() {
        let link = Link::new("https://example.com", "Example");
        assert!(!link.is_nofollow);
        assert!(link.should_follow());
    }

    #[test]
    fn test_link_nofollow() {
        let mut link = Link::new("/page", "Page");
        link.is_nofollow = true;
        assert!(!link.should_follow());
    }

    #[test]
    fn test_image_creation() {
        let img = Image::new("/img/photo.jpg", "A photo");
        assert!(!img.is_decorative);
        
        let decorative = Image::new("/img/spacer.gif", "");
        assert!(decorative.is_decorative);
    }

    #[test]
    fn test_list_content() {
        let mut list = ListContent::new(ListType::Unordered);
        list.add_item(ListItem::new("Item 1"));
        list.add_item(ListItem::new("Item 2"));
        assert_eq!(list.total_items, 2);
        assert!(!list.is_empty());
    }

    #[test]
    fn test_table_content() {
        let table = TableContent::new();
        assert!(table.is_empty());
        assert_eq!(table.row_count(), 0);
    }

    #[test]
    fn test_code_block() {
        let code = CodeBlock::new("fn main() {\n    println!(\"Hello\");\n}").with_language("rust");
        assert_eq!(code.language, Some("rust".to_string()));
        assert_eq!(code.line_count, 3);
        assert!(!code.is_inline);
    }

    #[test]
    fn test_opengraph() {
        let og = OpenGraph::default();
        assert!(!og.is_present());
        
        let og2 = OpenGraph {
            title: Some("Test".to_string()),
            ..Default::default()
        };
        assert!(og2.is_present());
    }

    #[test]
    fn test_robots_meta() {
        let allowed = RobotsMeta::allowed();
        assert!(allowed.index);
        assert!(allowed.follow);
        
        let noindex = RobotsMeta::noindex_nofollow();
        assert!(!noindex.index);
        assert!(!noindex.follow);
    }

    #[test]
    fn test_page_metadata_effective() {
        let mut meta = PageMetadata::default();
        meta.title = Some("Page Title".to_string());
        meta.opengraph.title = Some("OG Title".to_string());
        
        // OG takes precedence
        assert_eq!(meta.effective_title(), Some("OG Title"));
    }

    #[test]
    fn test_parser_config() {
        let config = ParserConfig::default();
        assert!(config.extract_images);
        assert!(config.extract_links);
        
        let minimal = ParserConfig::minimal();
        assert!(!minimal.extract_images);
    }

    #[test]
    fn test_normalize_whitespace() {
        assert_eq!(normalize_whitespace("  hello   world  "), "hello world");
        assert_eq!(normalize_whitespace("a\n\n\nb"), "a b");
        assert_eq!(normalize_whitespace("  "), "");
    }

    #[test]
    fn test_clean_text() {
        let text = "Hello\x00World\x01Test";
        let cleaned = clean_text(text);
        assert_eq!(cleaned, "HelloWorldTest");
    }

    #[test]
    fn test_truncate_text() {
        assert_eq!(truncate_text("Hello", 10), "Hello");
        assert_eq!(truncate_text("Hello World", 8), "Hello...");
    }

    #[test]
    fn test_parsed_content_links() {
        let mut content = ParsedContent::default();
        content.links.push(Link {
            link_type: LinkType::Internal,
            ..Link::new("/page", "Page")
        });
        content.links.push(Link {
            link_type: LinkType::External,
            ..Link::new("https://ext.com", "Ext")
        });
        
        assert_eq!(content.internal_links().len(), 1);
        assert_eq!(content.external_links().len(), 1);
    }

    #[test]
    fn test_reading_time() {
        // 225 WPM average
        let text = TextContent::from_raw(&"word ".repeat(450));
        assert!(text.reading_time_minutes.is_some());
        let time = text.reading_time_minutes.unwrap();
        assert!((time - 2.0).abs() < 0.1); // ~2 minutes
    }
}