halldyll_parser/

metadata.rs

//! Metadata extraction for halldyll-parser
//!
//! This module handles extraction of:
//! - Title and meta tags
//! - OpenGraph metadata
//! - Twitter Card metadata
//! - Robots directives
//! - Canonical URLs
//! - Hreflang alternates
//! - Structured data (JSON-LD, Microdata)
//! - Favicon and icons

use scraper::{Html, ElementRef};
use std::collections::HashMap;
use url::Url;

use crate::selector::{SELECTORS, try_parse_selector};
use crate::types::{
    PageMetadata, OpenGraph, TwitterCard, RobotsMeta,
    AlternateLink, StructuredData,
    ParserResult,
};

// ============================================================================
// MAIN EXTRACTION FUNCTION
// ============================================================================

/// Extract all metadata from an HTML document
pub fn extract_metadata(document: &Html, base_url: Option<&Url>) -> ParserResult<PageMetadata> {
    let metadata = PageMetadata {
        title: extract_title(document),
        charset: extract_charset(document),
        language: extract_language(document),
        base_url: extract_base_url(document),
        viewport: extract_meta_content(document, "viewport"),
        description: extract_meta_content(document, "description"),
        keywords: extract_keywords(document),
        author: extract_meta_content(document, "author"),
        generator: extract_meta_content(document, "generator"),
        published_date: extract_meta_content(document, "article:published_time")
            .or_else(|| extract_meta_content(document, "datePublished"))
            .or_else(|| extract_meta_content(document, "date")),
        modified_date: extract_meta_content(document, "article:modified_time")
            .or_else(|| extract_meta_content(document, "dateModified")),
        canonical: extract_canonical(document, base_url),
        favicon: extract_favicon(document, base_url),
        apple_touch_icon: extract_apple_touch_icon(document, base_url),
        theme_color: extract_meta_content(document, "theme-color"),
        robots: extract_robots(document),
        opengraph: extract_opengraph(document),
        twitter: extract_twitter_card(document),
        alternates: extract_alternates(document, base_url),
        schema_type: None,
        custom: extract_custom_meta(document),
    };
    
    Ok(metadata)
}

// ============================================================================
// BASIC METADATA
// ============================================================================

/// Extract page title
pub fn extract_title(document: &Html) -> Option<String> {
    document
        .select(&SELECTORS.title)
        .next()
        .map(|el| el.text().collect::<String>().trim().to_string())
        .filter(|s| !s.is_empty())
}

/// Extract charset from meta tag
pub fn extract_charset(document: &Html) -> Option<String> {
    // Try <meta charset="...">
    for meta in document.select(&SELECTORS.meta) {
        if let Some(charset) = meta.value().attr("charset") {
            return Some(charset.to_uppercase());
        }
    }
    
    // Try <meta http-equiv="Content-Type" content="...">
    if let Some(sel) = try_parse_selector("meta[http-equiv='Content-Type']") {
        if let Some(meta) = document.select(&sel).next() {
            if let Some(content) = meta.value().attr("content") {
                // Parse "text/html; charset=utf-8"
                if let Some(charset_part) = content.split(';')
                    .find(|p| p.trim().to_lowercase().starts_with("charset"))
                {
                    if let Some(charset) = charset_part.split('=').nth(1) {
                        return Some(charset.trim().to_uppercase());
                    }
                }
            }
        }
    }
    
    None
}

/// Extract language from html lang attribute
pub fn extract_language(document: &Html) -> Option<String> {
    document
        .select(&SELECTORS.html)
        .next()
        .and_then(|el| el.value().attr("lang"))
        .map(|s| s.to_string())
}

/// Extract base URL from <base> tag
pub fn extract_base_url(document: &Html) -> Option<String> {
    document
        .select(&SELECTORS.base)
        .next()
        .and_then(|el| el.value().attr("href"))
        .map(|s| s.to_string())
}

/// Extract content from a meta tag by name
pub fn extract_meta_content(document: &Html, name: &str) -> Option<String> {
    // Try name attribute first
    let name_selector = format!("meta[name='{}' i]", name);
    if let Some(sel) = try_parse_selector(&name_selector) {
        if let Some(meta) = document.select(&sel).next() {
            if let Some(content) = meta.value().attr("content") {
                let trimmed = content.trim();
                if !trimmed.is_empty() {
                    return Some(trimmed.to_string());
                }
            }
        }
    }
    
    // Try property attribute (for OG tags)
    let prop_selector = format!("meta[property='{}']", name);
    if let Some(sel) = try_parse_selector(&prop_selector) {
        if let Some(meta) = document.select(&sel).next() {
            if let Some(content) = meta.value().attr("content") {
                let trimmed = content.trim();
                if !trimmed.is_empty() {
                    return Some(trimmed.to_string());
                }
            }
        }
    }
    
    None
}

/// Extract keywords as a list
pub fn extract_keywords(document: &Html) -> Vec<String> {
    extract_meta_content(document, "keywords")
        .map(|s| {
            s.split(',')
                .map(|k| k.trim().to_string())
                .filter(|k| !k.is_empty())
                .collect()
        })
        .unwrap_or_default()
}

// ============================================================================
// LINK EXTRACTION
// ============================================================================

/// Extract canonical URL
pub fn extract_canonical(document: &Html, base_url: Option<&Url>) -> Option<String> {
    if let Some(sel) = try_parse_selector("link[rel='canonical']") {
        if let Some(link) = document.select(&sel).next() {
            if let Some(href) = link.value().attr("href") {
                return resolve_url(href, base_url);
            }
        }
    }
    None
}

/// Extract favicon URL
pub fn extract_favicon(document: &Html, base_url: Option<&Url>) -> Option<String> {
    // Try various favicon link types
    let selectors = [
        "link[rel='icon']",
        "link[rel='shortcut icon']",
        "link[rel='icon shortcut']",
    ];
    
    for sel_str in selectors {
        if let Some(sel) = try_parse_selector(sel_str) {
            if let Some(link) = document.select(&sel).next() {
                if let Some(href) = link.value().attr("href") {
                    return resolve_url(href, base_url);
                }
            }
        }
    }
    
    // Default to /favicon.ico
    base_url.map(|u| {
        let mut favicon_url = u.clone();
        favicon_url.set_path("/favicon.ico");
        favicon_url.set_query(None);
        favicon_url.to_string()
    })
}

/// Extract Apple touch icon
pub fn extract_apple_touch_icon(document: &Html, base_url: Option<&Url>) -> Option<String> {
    let selectors = [
        "link[rel='apple-touch-icon']",
        "link[rel='apple-touch-icon-precomposed']",
    ];
    
    for sel_str in selectors {
        if let Some(sel) = try_parse_selector(sel_str) {
            // Get the largest icon if multiple exist
            let icons: Vec<_> = document.select(&sel).collect();
            if let Some(icon) = find_largest_icon(&icons) {
                if let Some(href) = icon.value().attr("href") {
                    return resolve_url(href, base_url);
                }
            }
        }
    }
    
    None
}

/// Find the largest icon from a list of link elements
fn find_largest_icon<'a>(icons: &'a [ElementRef<'a>]) -> Option<&'a ElementRef<'a>> {
    if icons.is_empty() {
        return None;
    }
    
    // Try to find one with largest sizes attribute
    let mut best: Option<(&ElementRef, u32)> = None;
    
    for icon in icons {
        let size = icon.value().attr("sizes")
            .and_then(|s| {
                // Parse "180x180" format
                let parts: Vec<_> = s.split('x').collect();
                if parts.len() == 2 {
                    parts[0].parse::<u32>().ok()
                } else {
                    None
                }
            })
            .unwrap_or(0);
        
        if best.is_none() || size > best.unwrap().1 {
            best = Some((icon, size));
        }
    }
    
    best.map(|(el, _)| el)
}

/// Extract alternate language links (hreflang)
pub fn extract_alternates(document: &Html, base_url: Option<&Url>) -> Vec<AlternateLink> {
    let mut alternates = Vec::new();
    
    if let Some(sel) = try_parse_selector("link[rel='alternate'][hreflang]") {
        for link in document.select(&sel) {
            if let (Some(hreflang), Some(href)) = (
                link.value().attr("hreflang"),
                link.value().attr("href"),
            ) {
                if let Some(resolved) = resolve_url(href, base_url) {
                    alternates.push(AlternateLink {
                        hreflang: hreflang.to_string(),
                        href: resolved,
                    });
                }
            }
        }
    }
    
    alternates
}

// ============================================================================
// ROBOTS META
// ============================================================================

/// Extract robots meta directives
pub fn extract_robots(document: &Html) -> RobotsMeta {
    let mut robots = RobotsMeta::allowed();
    
    // Get robots meta content
    let content = extract_meta_content(document, "robots")
        .or_else(|| extract_meta_content(document, "googlebot"));
    
    if let Some(content) = content {
        robots.raw = Some(content.clone());
        
        let directives: Vec<_> = content
            .to_lowercase()
            .split(',')
            .map(|s| s.trim().to_string())
            .collect();
        
        for directive in &directives {
            match directive.as_str() {
                "noindex" => robots.index = false,
                "nofollow" => robots.follow = false,
                "none" => {
                    robots.index = false;
                    robots.follow = false;
                }
                "noarchive" => robots.archive = false,
                "nocache" => robots.cache = false,
                "nosnippet" => robots.snippet = false,
                _ => {
                    // Handle max-snippet:N, max-image-preview:SIZE, max-video-preview:N
                    if let Some((key, value)) = directive.split_once(':') {
                        match key {
                            "max-snippet" => {
                                robots.max_snippet = value.parse().unwrap_or(-1);
                            }
                            "max-image-preview" => {
                                robots.max_image_preview = Some(value.to_string());
                            }
                            "max-video-preview" => {
                                robots.max_video_preview = value.parse().unwrap_or(-1);
                            }
                            _ => {}
                        }
                    }
                }
            }
        }
    }
    
    robots
}

// ============================================================================
// OPENGRAPH
// ============================================================================

/// Extract OpenGraph metadata
pub fn extract_opengraph(document: &Html) -> OpenGraph {
    OpenGraph {
        title: extract_og_property(document, "og:title"),
        og_type: extract_og_property(document, "og:type"),
        url: extract_og_property(document, "og:url"),
        image: extract_og_property(document, "og:image"),
        description: extract_og_property(document, "og:description"),
        site_name: extract_og_property(document, "og:site_name"),
        locale: extract_og_property(document, "og:locale"),
        video: extract_og_property(document, "og:video"),
        audio: extract_og_property(document, "og:audio"),
        extra: extract_all_og_properties(document),
    }
}

/// Extract a single OG property
fn extract_og_property(document: &Html, property: &str) -> Option<String> {
    let selector = format!("meta[property='{}']", property);
    if let Some(sel) = try_parse_selector(&selector) {
        if let Some(meta) = document.select(&sel).next() {
            return meta.value().attr("content")
                .map(|s| s.trim().to_string())
                .filter(|s| !s.is_empty());
        }
    }
    None
}

/// Extract all og: properties as a map
fn extract_all_og_properties(document: &Html) -> HashMap<String, String> {
    let mut props = HashMap::new();
    
    // Standard properties to exclude from extra
    let standard = ["og:title", "og:type", "og:url", "og:image", 
                   "og:description", "og:site_name", "og:locale", 
                   "og:video", "og:audio"];
    
    for meta in document.select(&SELECTORS.meta) {
        if let Some(property) = meta.value().attr("property") {
            if property.starts_with("og:") && !standard.contains(&property) {
                if let Some(content) = meta.value().attr("content") {
                    props.insert(property.to_string(), content.to_string());
                }
            }
        }
    }
    
    props
}

// ============================================================================
// TWITTER CARD
// ============================================================================

/// Extract Twitter Card metadata
pub fn extract_twitter_card(document: &Html) -> TwitterCard {
    TwitterCard {
        card: extract_twitter_property(document, "twitter:card"),
        site: extract_twitter_property(document, "twitter:site"),
        creator: extract_twitter_property(document, "twitter:creator"),
        title: extract_twitter_property(document, "twitter:title"),
        description: extract_twitter_property(document, "twitter:description"),
        image: extract_twitter_property(document, "twitter:image"),
        extra: extract_all_twitter_properties(document),
    }
}

/// Extract a single Twitter property
fn extract_twitter_property(document: &Html, name: &str) -> Option<String> {
    // Try property attribute first (some sites use this)
    let prop_selector = format!("meta[property='{}']", name);
    if let Some(sel) = try_parse_selector(&prop_selector) {
        if let Some(meta) = document.select(&sel).next() {
            if let Some(content) = meta.value().attr("content") {
                let trimmed = content.trim();
                if !trimmed.is_empty() {
                    return Some(trimmed.to_string());
                }
            }
        }
    }
    
    // Then try name attribute
    let name_selector = format!("meta[name='{}']", name);
    if let Some(sel) = try_parse_selector(&name_selector) {
        if let Some(meta) = document.select(&sel).next() {
            if let Some(content) = meta.value().attr("content") {
                let trimmed = content.trim();
                if !trimmed.is_empty() {
                    return Some(trimmed.to_string());
                }
            }
        }
    }
    
    None
}

/// Extract all twitter: properties as a map
fn extract_all_twitter_properties(document: &Html) -> HashMap<String, String> {
    let mut props = HashMap::new();
    
    let standard = ["twitter:card", "twitter:site", "twitter:creator",
                   "twitter:title", "twitter:description", "twitter:image"];
    
    for meta in document.select(&SELECTORS.meta) {
        let key = meta.value().attr("property")
            .or_else(|| meta.value().attr("name"));
        
        if let Some(key) = key {
            if key.starts_with("twitter:") && !standard.contains(&key) {
                if let Some(content) = meta.value().attr("content") {
                    props.insert(key.to_string(), content.to_string());
                }
            }
        }
    }
    
    props
}

// ============================================================================
// STRUCTURED DATA
// ============================================================================

/// Extract all structured data from the document
pub fn extract_structured_data(document: &Html) -> Vec<StructuredData> {
    let mut data = Vec::new();
    
    // Extract JSON-LD
    data.extend(extract_json_ld(document));
    
    // Extract Microdata
    data.extend(extract_microdata(document));
    
    data
}

/// Extract JSON-LD structured data
pub fn extract_json_ld(document: &Html) -> Vec<StructuredData> {
    let mut data = Vec::new();
    
    for script in document.select(&SELECTORS.json_ld) {
        let raw_json = script.text().collect::<String>();
        let trimmed = raw_json.trim();
        
        if trimmed.is_empty() {
            continue;
        }
        
        let mut item = StructuredData::json_ld(trimmed);
        
        // Try to parse and extract @type
        if let Ok(json) = serde_json::from_str::<serde_json::Value>(trimmed) {
            if let Some(schema_type) = json.get("@type").and_then(|v| v.as_str()) {
                item.schema_type = Some(schema_type.to_string());
            }
            
            // Store parsed properties
            if let serde_json::Value::Object(map) = json {
                for (key, value) in map {
                    item.properties.insert(key, value);
                }
            }
        }
        
        data.push(item);
    }
    
    data
}

/// Extract Microdata structured data
pub fn extract_microdata(document: &Html) -> Vec<StructuredData> {
    let mut data = Vec::new();
    
    for item in document.select(&SELECTORS.microdata) {
        if let Some(itemtype) = item.value().attr("itemtype") {
            // Extract schema type from URL (e.g., "https://schema.org/Article" -> "Article")
            let schema_type = itemtype
                .rsplit('/')
                .next()
                .unwrap_or(itemtype)
                .to_string();
            
            let mut structured = StructuredData::microdata(&schema_type);
            
            // Extract itemprop values
            if let Some(sel) = try_parse_selector("[itemprop]") {
                for prop in item.select(&sel) {
                    if let Some(prop_name) = prop.value().attr("itemprop") {
                        // Get value from content attribute or text
                        let value = prop.value().attr("content")
                            .or_else(|| prop.value().attr("href"))
                            .or_else(|| prop.value().attr("src"))
                            .map(|s| s.to_string())
                            .unwrap_or_else(|| prop.text().collect::<String>().trim().to_string());
                        
                        structured.properties.insert(
                            prop_name.to_string(),
                            serde_json::Value::String(value),
                        );
                    }
                }
            }
            
            data.push(structured);
        }
    }
    
    data
}

// ============================================================================
// CUSTOM META
// ============================================================================

/// Extract custom/application-specific meta tags
fn extract_custom_meta(document: &Html) -> HashMap<String, String> {
    let mut custom = HashMap::new();
    
    // Skip standard meta names
    let standard = [
        "description", "keywords", "author", "viewport", "robots",
        "generator", "theme-color", "msapplication-TileColor",
    ];
    
    for meta in document.select(&SELECTORS.meta) {
        if let Some(name) = meta.value().attr("name") {
            // Skip standard, OG, and Twitter tags
            if !standard.contains(&name) 
               && !name.starts_with("og:")
               && !name.starts_with("twitter:")
               && !name.starts_with("article:")
            {
                if let Some(content) = meta.value().attr("content") {
                    custom.insert(name.to_string(), content.to_string());
                }
            }
        }
    }
    
    custom
}

// ============================================================================
// URL UTILITIES
// ============================================================================

/// Resolve a relative URL to absolute
fn resolve_url(href: &str, base_url: Option<&Url>) -> Option<String> {
    let trimmed = href.trim();
    
    if trimmed.is_empty() {
        return None;
    }
    
    // Already absolute
    if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
        return Some(trimmed.to_string());
    }
    
    // Protocol-relative
    if trimmed.starts_with("//") {
        return Some(format!("https:{}", trimmed));
    }
    
    // Resolve relative to base
    base_url
        .and_then(|base| base.join(trimmed).ok())
        .map(|u| u.to_string())
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;
    use crate::types::StructuredDataFormat;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_title() {
        let doc = parse_html("<html><head><title>Test Page</title></head></html>");
        assert_eq!(extract_title(&doc), Some("Test Page".to_string()));
    }

    #[test]
    fn test_extract_title_with_whitespace() {
        let doc = parse_html("<html><head><title>  Test Page  </title></head></html>");
        assert_eq!(extract_title(&doc), Some("Test Page".to_string()));
    }

    #[test]
    fn test_extract_title_empty() {
        let doc = parse_html("<html><head><title></title></head></html>");
        assert_eq!(extract_title(&doc), None);
    }

    #[test]
    fn test_extract_charset_meta() {
        let doc = parse_html("<html><head><meta charset='UTF-8'></head></html>");
        assert_eq!(extract_charset(&doc), Some("UTF-8".to_string()));
    }

    #[test]
    fn test_extract_charset_content_type() {
        let doc = parse_html(
            "<html><head><meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'></head></html>"
        );
        assert_eq!(extract_charset(&doc), Some("ISO-8859-1".to_string()));
    }

    #[test]
    fn test_extract_language() {
        let doc = parse_html("<html lang='en-US'><head></head></html>");
        assert_eq!(extract_language(&doc), Some("en-US".to_string()));
    }

    #[test]
    fn test_extract_meta_content() {
        let doc = parse_html(
            "<html><head><meta name='description' content='Test description'></head></html>"
        );
        assert_eq!(
            extract_meta_content(&doc, "description"),
            Some("Test description".to_string())
        );
    }

    #[test]
    fn test_extract_keywords() {
        let doc = parse_html(
            "<html><head><meta name='keywords' content='rust, web, scraping'></head></html>"
        );
        let keywords = extract_keywords(&doc);
        assert_eq!(keywords, vec!["rust", "web", "scraping"]);
    }

    #[test]
    fn test_extract_canonical() {
        let doc = parse_html(
            "<html><head><link rel='canonical' href='https://example.com/page'></head></html>"
        );
        assert_eq!(
            extract_canonical(&doc, None),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_extract_canonical_relative() {
        let doc = parse_html(
            "<html><head><link rel='canonical' href='/page'></head></html>"
        );
        let base = Url::parse("https://example.com").unwrap();
        assert_eq!(
            extract_canonical(&doc, Some(&base)),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_extract_robots_default() {
        let doc = parse_html("<html><head></head></html>");
        let robots = extract_robots(&doc);
        assert!(robots.index);
        assert!(robots.follow);
    }

    #[test]
    fn test_extract_robots_noindex() {
        let doc = parse_html(
            "<html><head><meta name='robots' content='noindex, nofollow'></head></html>"
        );
        let robots = extract_robots(&doc);
        assert!(!robots.index);
        assert!(!robots.follow);
    }

    #[test]
    fn test_extract_robots_advanced() {
        let doc = parse_html(
            "<html><head><meta name='robots' content='noarchive, max-snippet:150, max-image-preview:large'></head></html>"
        );
        let robots = extract_robots(&doc);
        assert!(robots.index);
        assert!(!robots.archive);
        assert_eq!(robots.max_snippet, 150);
        assert_eq!(robots.max_image_preview, Some("large".to_string()));
    }

    #[test]
    fn test_extract_opengraph() {
        let doc = parse_html(r#"
            <html><head>
                <meta property="og:title" content="OG Title">
                <meta property="og:type" content="article">
                <meta property="og:url" content="https://example.com/article">
                <meta property="og:image" content="https://example.com/image.jpg">
                <meta property="og:description" content="OG Description">
            </head></html>
        "#);
        
        let og = extract_opengraph(&doc);
        assert!(og.is_present());
        assert_eq!(og.title, Some("OG Title".to_string()));
        assert_eq!(og.og_type, Some("article".to_string()));
        assert_eq!(og.url, Some("https://example.com/article".to_string()));
    }

    #[test]
    fn test_extract_twitter_card() {
        let doc = parse_html(r#"
            <html><head>
                <meta name="twitter:card" content="summary_large_image">
                <meta name="twitter:site" content="@example">
                <meta name="twitter:title" content="Twitter Title">
            </head></html>
        "#);
        
        let twitter = extract_twitter_card(&doc);
        assert!(twitter.is_present());
        assert_eq!(twitter.card, Some("summary_large_image".to_string()));
        assert_eq!(twitter.site, Some("@example".to_string()));
    }

    #[test]
    fn test_extract_alternates() {
        let doc = parse_html(r#"
            <html><head>
                <link rel="alternate" hreflang="en" href="https://example.com/en/">
                <link rel="alternate" hreflang="fr" href="https://example.com/fr/">
                <link rel="alternate" hreflang="x-default" href="https://example.com/">
            </head></html>
        "#);
        
        let alternates = extract_alternates(&doc, None);
        assert_eq!(alternates.len(), 3);
        assert!(alternates.iter().any(|a| a.hreflang == "en"));
        assert!(alternates.iter().any(|a| a.hreflang == "fr"));
        assert!(alternates.iter().any(|a| a.hreflang == "x-default"));
    }

    #[test]
    fn test_extract_json_ld() {
        let doc = parse_html(r#"
            <html><head>
                <script type="application/ld+json">
                {
                    "@context": "https://schema.org",
                    "@type": "Article",
                    "headline": "Test Article"
                }
                </script>
            </head></html>
        "#);
        
        let data = extract_json_ld(&doc);
        assert_eq!(data.len(), 1);
        assert_eq!(data[0].format, StructuredDataFormat::JsonLd);
        assert_eq!(data[0].schema_type, Some("Article".to_string()));
    }

    #[test]
    fn test_extract_microdata() {
        let doc = parse_html(r#"
            <div itemscope itemtype="https://schema.org/Person">
                <span itemprop="name">John Doe</span>
                <span itemprop="jobTitle">Software Engineer</span>
            </div>
        "#);
        
        let data = extract_microdata(&doc);
        assert_eq!(data.len(), 1);
        assert_eq!(data[0].format, StructuredDataFormat::Microdata);
        assert_eq!(data[0].schema_type, Some("Person".to_string()));
    }

    #[test]
    fn test_extract_metadata_full() {
        let doc = parse_html(r#"
            <!DOCTYPE html>
            <html lang="en">
            <head>
                <meta charset="UTF-8">
                <title>Full Test Page</title>
                <meta name="description" content="A complete test page">
                <meta name="keywords" content="test, page, rust">
                <meta name="author" content="Test Author">
                <meta name="robots" content="index, follow">
                <link rel="canonical" href="https://example.com/page">
                <meta property="og:title" content="OG Title">
                <meta name="twitter:card" content="summary">
            </head>
            <body></body>
            </html>
        "#);
        
        let metadata = extract_metadata(&doc, None).unwrap();
        
        assert_eq!(metadata.title, Some("Full Test Page".to_string()));
        assert_eq!(metadata.description, Some("A complete test page".to_string()));
        assert_eq!(metadata.language, Some("en".to_string()));
        assert_eq!(metadata.charset, Some("UTF-8".to_string()));
        assert!(metadata.robots.index);
        assert!(metadata.opengraph.is_present());
        assert!(metadata.twitter.is_present());
    }

    #[test]
    fn test_resolve_url_absolute() {
        assert_eq!(
            resolve_url("https://example.com/page", None),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_resolve_url_protocol_relative() {
        assert_eq!(
            resolve_url("//example.com/page", None),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_resolve_url_relative() {
        let base = Url::parse("https://example.com/dir/").unwrap();
        assert_eq!(
            resolve_url("page.html", Some(&base)),
            Some("https://example.com/dir/page.html".to_string())
        );
    }

    #[test]
    fn test_favicon_default() {
        let doc = parse_html("<html><head></head></html>");
        let base = Url::parse("https://example.com/page").unwrap();
        let favicon = extract_favicon(&doc, Some(&base));
        assert_eq!(favicon, Some("https://example.com/favicon.ico".to_string()));
    }

    #[test]
    fn test_favicon_explicit() {
        let doc = parse_html(
            "<html><head><link rel='icon' href='/icons/favicon.png'></head></html>"
        );
        let base = Url::parse("https://example.com").unwrap();
        let favicon = extract_favicon(&doc, Some(&base));
        assert_eq!(favicon, Some("https://example.com/icons/favicon.png".to_string()));
    }
}