halldyll-parser 0.1.0

//! Link extraction for halldyll-parser
//!
//! This module handles:
//! - Link extraction from anchor tags
//! - URL normalization and resolution
//! - Rel attribute parsing (nofollow, ugc, sponsored, etc.)
//! - Internal/external link classification
//! - Link deduplication

use scraper::{Html, ElementRef};
use std::collections::HashSet;
use url::Url;

use crate::selector::SELECTORS;
use crate::types::{Link, LinkRel, LinkType, ParserConfig, ParserResult};

// ============================================================================
// MAIN EXTRACTION
// ============================================================================

/// Extract all links from an HTML document
pub fn extract_links(
    document: &Html,
    config: &ParserConfig,
) -> ParserResult<Vec<Link>> {
    let mut links = Vec::new();
    let mut seen_hrefs: HashSet<String> = HashSet::new();
    
    for anchor in document.select(&SELECTORS.a) {
        if let Some(link) = extract_link(&anchor, config.base_url.as_ref()) {
            // Deduplicate by href
            if !seen_hrefs.contains(&link.href) {
                seen_hrefs.insert(link.href.clone());
                links.push(link);
            }
        }
    }
    
    Ok(links)
}

/// Extract a single link from an anchor element
pub fn extract_link(element: &ElementRef, base_url: Option<&Url>) -> Option<Link> {
    let href = element.value().attr("href")?;
    let href = href.trim();
    
    // Skip empty, javascript, and mailto links
    if href.is_empty() 
        || href.starts_with("javascript:") 
        || href.starts_with("mailto:")
        || href.starts_with("tel:")
        || href.starts_with("data:")
        || href == "#"
    {
        return None;
    }
    
    // Get anchor text
    let text = element.text().collect::<String>().trim().to_string();
    
    // Create link
    let mut link = Link::new(href, &text);
    
    // Resolve URL
    link.url = resolve_url(href, base_url);
    
    // Parse rel attributes
    if let Some(rel) = element.value().attr("rel") {
        link.rel = parse_rel_attribute(rel);
        link.is_nofollow = link.rel.contains(&LinkRel::NoFollow);
    }
    
    // Get other attributes
    link.title = element.value().attr("title").map(|s| s.to_string());
    link.target = element.value().attr("target").map(|s| s.to_string());
    link.hreflang = element.value().attr("hreflang").map(|s| s.to_string());
    
    // Determine link type (internal/external)
    link.link_type = determine_link_type(&link.url, base_url);
    
    Some(link)
}

// ============================================================================
// URL HANDLING
// ============================================================================

/// Resolve a relative URL to absolute
pub fn resolve_url(href: &str, base_url: Option<&Url>) -> Option<String> {
    let trimmed = href.trim();
    
    if trimmed.is_empty() {
        return None;
    }
    
    // Already absolute
    if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
        return normalize_url(trimmed);
    }
    
    // Protocol-relative
    if trimmed.starts_with("//") {
        return normalize_url(&format!("https:{}", trimmed));
    }
    
    // Resolve relative to base
    base_url
        .and_then(|base| base.join(trimmed).ok())
        .and_then(|u| normalize_url(u.as_str()))
}

/// Normalize a URL (remove fragments, trailing slashes for paths)
pub fn normalize_url(url: &str) -> Option<String> {
    Url::parse(url).ok().map(|mut u| {
        // Remove fragment
        u.set_fragment(None);
        
        // Normalize path (remove trailing slash except for root)
        let path = u.path().to_string();
        if path.len() > 1 && path.ends_with('/') {
            u.set_path(path.trim_end_matches('/'));
        }
        
        u.to_string()
    })
}

/// Determine if a link is internal or external
fn determine_link_type(resolved_url: &Option<String>, base_url: Option<&Url>) -> LinkType {
    let (Some(url_str), Some(base)) = (resolved_url, base_url) else {
        return LinkType::Unknown;
    };
    
    let Ok(url) = Url::parse(url_str) else {
        return LinkType::Unknown;
    };
    
    // Compare hosts
    match (url.host_str(), base.host_str()) {
        (Some(url_host), Some(base_host)) => {
            // Check if same domain or subdomain
            if url_host == base_host {
                LinkType::Internal
            } else if url_host.ends_with(&format!(".{}", base_host)) 
                   || base_host.ends_with(&format!(".{}", url_host)) {
                // Subdomain relationship
                LinkType::Internal
            } else {
                LinkType::External
            }
        }
        _ => LinkType::Unknown,
    }
}

// ============================================================================
// REL ATTRIBUTE PARSING
// ============================================================================

/// Parse rel attribute into LinkRel values
pub fn parse_rel_attribute(rel: &str) -> Vec<LinkRel> {
    rel.split_whitespace()
        .map(|r| match r.to_lowercase().as_str() {
            "nofollow" => LinkRel::NoFollow,
            "ugc" => LinkRel::Ugc,
            "sponsored" => LinkRel::Sponsored,
            "external" => LinkRel::External,
            "noopener" => LinkRel::NoOpener,
            "noreferrer" => LinkRel::NoReferrer,
            _ => LinkRel::Other,
        })
        .collect()
}

/// Check if rel indicates nofollow
pub fn is_nofollow(rel: &str) -> bool {
    rel.to_lowercase()
        .split_whitespace()
        .any(|r| r == "nofollow")
}

/// Check if rel indicates sponsored
pub fn is_sponsored(rel: &str) -> bool {
    rel.to_lowercase()
        .split_whitespace()
        .any(|r| r == "sponsored")
}

/// Check if rel indicates user-generated content
pub fn is_ugc(rel: &str) -> bool {
    rel.to_lowercase()
        .split_whitespace()
        .any(|r| r == "ugc")
}

// ============================================================================
// LINK ANALYSIS
// ============================================================================

/// Get all internal links
pub fn filter_internal_links(links: &[Link]) -> Vec<&Link> {
    links.iter()
        .filter(|l| l.link_type == LinkType::Internal)
        .collect()
}

/// Get all external links
pub fn filter_external_links(links: &[Link]) -> Vec<&Link> {
    links.iter()
        .filter(|l| l.link_type == LinkType::External)
        .collect()
}

/// Get all followable links (not nofollow, not sponsored, not ugc)
pub fn filter_followable_links(links: &[Link]) -> Vec<&Link> {
    links.iter()
        .filter(|l| l.should_follow())
        .collect()
}

/// Get unique domains from external links
pub fn get_external_domains(links: &[Link]) -> HashSet<String> {
    links.iter()
        .filter(|l| l.link_type == LinkType::External)
        .filter_map(|l| l.url.as_ref())
        .filter_map(|url| Url::parse(url).ok())
        .filter_map(|url| url.host_str().map(|h| h.to_string()))
        .collect()
}

/// Count links by type
pub struct LinkStats {
    pub total: usize,
    pub internal: usize,
    pub external: usize,
    pub nofollow: usize,
    pub sponsored: usize,
    pub ugc: usize,
    pub with_title: usize,
    pub opens_new_tab: usize,
}

/// Calculate link statistics
pub fn calculate_link_stats(links: &[Link]) -> LinkStats {
    LinkStats {
        total: links.len(),
        internal: links.iter().filter(|l| l.link_type == LinkType::Internal).count(),
        external: links.iter().filter(|l| l.link_type == LinkType::External).count(),
        nofollow: links.iter().filter(|l| l.is_nofollow).count(),
        sponsored: links.iter().filter(|l| l.rel.contains(&LinkRel::Sponsored)).count(),
        ugc: links.iter().filter(|l| l.rel.contains(&LinkRel::Ugc)).count(),
        with_title: links.iter().filter(|l| l.title.is_some()).count(),
        opens_new_tab: links.iter().filter(|l| l.opens_new_tab()).count(),
    }
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_links_basic() {
        let doc = parse_html(r#"
            <html><body>
                <a href="https://example.com">Example</a>
                <a href="/page">Internal</a>
            </body></html>
        "#);
        let config = ParserConfig::default();
        let links = extract_links(&doc, &config).unwrap();
        assert_eq!(links.len(), 2);
    }

    #[test]
    fn test_extract_link_with_attributes() {
        let doc = parse_html(r#"
            <a href="https://example.com" 
               title="Example Site" 
               rel="nofollow external" 
               target="_blank">Link</a>
        "#);
        let anchor = doc.select(&SELECTORS.a).next().unwrap();
        let link = extract_link(&anchor, None).unwrap();
        
        assert_eq!(link.href, "https://example.com");
        assert_eq!(link.text, "Link");
        assert_eq!(link.title, Some("Example Site".to_string()));
        assert!(link.is_nofollow);
        assert!(link.opens_new_tab());
        assert!(link.rel.contains(&LinkRel::NoFollow));
        assert!(link.rel.contains(&LinkRel::External));
    }

    #[test]
    fn test_extract_link_skips_javascript() {
        let doc = parse_html(r#"<a href="javascript:void(0)">Click</a>"#);
        let anchor = doc.select(&SELECTORS.a).next().unwrap();
        assert!(extract_link(&anchor, None).is_none());
    }

    #[test]
    fn test_extract_link_skips_mailto() {
        let doc = parse_html(r#"<a href="mailto:test@example.com">Email</a>"#);
        let anchor = doc.select(&SELECTORS.a).next().unwrap();
        assert!(extract_link(&anchor, None).is_none());
    }

    #[test]
    fn test_extract_link_skips_hash() {
        let doc = parse_html("<a href=\"#\">Top</a>");
        let anchor = doc.select(&SELECTORS.a).next().unwrap();
        assert!(extract_link(&anchor, None).is_none());
    }

    #[test]
    fn test_resolve_url_absolute() {
        assert_eq!(
            resolve_url("https://example.com/page", None),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_resolve_url_protocol_relative() {
        assert_eq!(
            resolve_url("//example.com/page", None),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_resolve_url_relative() {
        let base = Url::parse("https://example.com/dir/").unwrap();
        assert_eq!(
            resolve_url("page.html", Some(&base)),
            Some("https://example.com/dir/page.html".to_string())
        );
    }

    #[test]
    fn test_resolve_url_root_relative() {
        let base = Url::parse("https://example.com/dir/page").unwrap();
        assert_eq!(
            resolve_url("/other", Some(&base)),
            Some("https://example.com/other".to_string())
        );
    }

    #[test]
    fn test_normalize_url_removes_fragment() {
        assert_eq!(
            normalize_url("https://example.com/page#section"),
            Some("https://example.com/page".to_string())
        );
    }

    #[test]
    fn test_determine_link_type_internal() {
        let base = Url::parse("https://example.com").unwrap();
        let url = Some("https://example.com/page".to_string());
        assert_eq!(determine_link_type(&url, Some(&base)), LinkType::Internal);
    }

    #[test]
    fn test_determine_link_type_subdomain() {
        let base = Url::parse("https://example.com").unwrap();
        let url = Some("https://blog.example.com/page".to_string());
        assert_eq!(determine_link_type(&url, Some(&base)), LinkType::Internal);
    }

    #[test]
    fn test_determine_link_type_external() {
        let base = Url::parse("https://example.com").unwrap();
        let url = Some("https://other.com/page".to_string());
        assert_eq!(determine_link_type(&url, Some(&base)), LinkType::External);
    }

    #[test]
    fn test_parse_rel_attribute() {
        let rels = parse_rel_attribute("nofollow ugc sponsored");
        assert!(rels.contains(&LinkRel::NoFollow));
        assert!(rels.contains(&LinkRel::Ugc));
        assert!(rels.contains(&LinkRel::Sponsored));
    }

    #[test]
    fn test_is_nofollow() {
        assert!(is_nofollow("nofollow"));
        assert!(is_nofollow("nofollow external"));
        assert!(is_nofollow("external nofollow"));
        assert!(!is_nofollow("external"));
    }

    #[test]
    fn test_filter_internal_links() {
        let links = vec![
            Link { link_type: LinkType::Internal, ..Link::new("/a", "A") },
            Link { link_type: LinkType::External, ..Link::new("https://ext.com", "B") },
            Link { link_type: LinkType::Internal, ..Link::new("/b", "C") },
        ];
        let internal = filter_internal_links(&links);
        assert_eq!(internal.len(), 2);
    }

    #[test]
    fn test_filter_followable_links() {
        let mut nofollow = Link::new("/page", "Page");
        nofollow.is_nofollow = true;
        
        let links = vec![
            Link::new("/a", "A"),
            nofollow,
            Link::new("/b", "B"),
        ];
        let followable = filter_followable_links(&links);
        assert_eq!(followable.len(), 2);
    }

    #[test]
    fn test_get_external_domains() {
        let links = vec![
            Link { 
                link_type: LinkType::External, 
                url: Some("https://example.com/page".to_string()),
                ..Link::new("https://example.com/page", "A") 
            },
            Link { 
                link_type: LinkType::External, 
                url: Some("https://other.com/page".to_string()),
                ..Link::new("https://other.com/page", "B") 
            },
            Link { 
                link_type: LinkType::External, 
                url: Some("https://example.com/other".to_string()),
                ..Link::new("https://example.com/other", "C") 
            },
        ];
        let domains = get_external_domains(&links);
        assert_eq!(domains.len(), 2);
        assert!(domains.contains("example.com"));
        assert!(domains.contains("other.com"));
    }

    #[test]
    fn test_calculate_link_stats() {
        let mut nofollow = Link::new("/page", "Page");
        nofollow.is_nofollow = true;
        nofollow.link_type = LinkType::Internal;
        
        let mut sponsored = Link::new("https://ad.com", "Ad");
        sponsored.rel = vec![LinkRel::Sponsored];
        sponsored.link_type = LinkType::External;
        
        let links = vec![
            Link { link_type: LinkType::Internal, ..Link::new("/a", "A") },
            nofollow,
            sponsored,
        ];
        
        let stats = calculate_link_stats(&links);
        assert_eq!(stats.total, 3);
        assert_eq!(stats.internal, 2);
        assert_eq!(stats.external, 1);
        assert_eq!(stats.nofollow, 1);
        assert_eq!(stats.sponsored, 1);
    }

    #[test]
    fn test_deduplicate_links() {
        let doc = parse_html(r#"
            <html><body>
                <a href="https://example.com">First</a>
                <a href="https://example.com">Duplicate</a>
                <a href="https://other.com">Other</a>
            </body></html>
        "#);
        let config = ParserConfig::default();
        let links = extract_links(&doc, &config).unwrap();
        assert_eq!(links.len(), 2);
    }
}