halldyll_parser/

fingerprint.rs

//! Content fingerprinting and change detection for halldyll-parser
//!
//! This module handles:
//! - Content hashing for change detection
//! - Structural fingerprinting
//! - AMP page detection
//! - Content comparison
//! - Cache control hints

use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};

use crate::selector::SELECTORS;
use crate::types::ParserResult;

// ============================================================================
// TYPES
// ============================================================================

/// Content fingerprint for change detection
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct ContentFingerprint {
    /// Hash of the full HTML content
    pub full_hash: u64,
    /// Hash of just the main content (excluding nav, footer, etc.)
    pub content_hash: u64,
    /// Hash of the text content only (no tags)
    pub text_hash: u64,
    /// Hash of the document structure (tag hierarchy)
    pub structure_hash: u64,
    /// Number of elements in the document
    pub element_count: usize,
    /// Number of text nodes
    pub text_node_count: usize,
    /// Content length in bytes
    pub content_length: usize,
    /// Main content length
    pub main_content_length: usize,
}

impl ContentFingerprint {
    /// Check if content has changed compared to another fingerprint
    pub fn has_changed(&self, other: &ContentFingerprint) -> bool {
        self.content_hash != other.content_hash
    }

    /// Check if only minor changes occurred (same structure, different content)
    pub fn has_minor_changes(&self, other: &ContentFingerprint) -> bool {
        self.structure_hash == other.structure_hash && 
        self.content_hash != other.content_hash
    }

    /// Check if structure changed (major change)
    pub fn has_structural_changes(&self, other: &ContentFingerprint) -> bool {
        self.structure_hash != other.structure_hash
    }

    /// Get similarity percentage (0.0 to 1.0)
    pub fn similarity(&self, other: &ContentFingerprint) -> f64 {
        let mut matches = 0.0;
        let total = 4.0;

        if self.content_hash == other.content_hash { matches += 1.0; }
        if self.text_hash == other.text_hash { matches += 1.0; }
        if self.structure_hash == other.structure_hash { matches += 1.0; }
        
        // Element count similarity
        let count_diff = (self.element_count as i64 - other.element_count as i64).abs();
        let max_count = self.element_count.max(other.element_count) as f64;
        if max_count > 0.0 {
            matches += 1.0 - (count_diff as f64 / max_count);
        } else {
            matches += 1.0;
        }

        matches / total
    }
}

/// AMP (Accelerated Mobile Pages) information
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub struct AmpInfo {
    /// Whether this page is an AMP page
    pub is_amp: bool,
    /// Whether this is an AMP HTML page (⚡ or amp attribute)
    pub is_amp_html: bool,
    /// URL to the AMP version of this page (if not AMP)
    pub amp_url: Option<String>,
    /// URL to the canonical (non-AMP) version (if this is AMP)
    pub canonical_url: Option<String>,
    /// AMP version detected
    pub amp_version: Option<String>,
    /// Whether AMP runtime is included
    pub has_amp_runtime: bool,
    /// AMP components used
    pub components: Vec<String>,
}

impl AmpInfo {
    pub fn new() -> Self {
        Self::default()
    }

    /// Check if page has AMP version available
    pub fn has_amp_version(&self) -> bool {
        self.amp_url.is_some()
    }
}

/// Cache hints extracted from the page
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub struct CacheHints {
    /// ETag value if found
    pub etag: Option<String>,
    /// Last-Modified header value
    pub last_modified: Option<String>,
    /// Cache-Control directives
    pub cache_control: Option<String>,
    /// Whether page indicates it shouldn't be cached
    pub no_cache: bool,
    /// Max-age value in seconds
    pub max_age: Option<u32>,
}

// ============================================================================
// FINGERPRINTING FUNCTIONS
// ============================================================================

/// Generate content fingerprint from HTML
pub fn generate_fingerprint(html: &str) -> ParserResult<ContentFingerprint> {
    let document = Html::parse_document(html);
    
    // Main content hash (excluding boilerplate)
    let main_content = extract_main_content(&document);
    
    // Text-only hash
    let text_content = extract_text_only(&document);
    
    // Structure hash
    let structure = extract_structure(&document);
    
    let fingerprint = ContentFingerprint {
        full_hash: hash_string(html),
        content_length: html.len(),
        content_hash: hash_string(&main_content),
        main_content_length: main_content.len(),
        text_hash: hash_string(&text_content),
        structure_hash: hash_string(&structure),
        element_count: count_elements(&document),
        text_node_count: count_text_nodes(&document),
    };
    
    Ok(fingerprint)
}

/// Generate fingerprint from parsed document
pub fn fingerprint_document(document: &Html) -> ContentFingerprint {
    let html = document.html();
    generate_fingerprint(&html).unwrap_or_default()
}

/// Hash a string using DefaultHasher
fn hash_string(s: &str) -> u64 {
    let mut hasher = DefaultHasher::new();
    s.hash(&mut hasher);
    hasher.finish()
}

/// Extract main content area (excluding nav, header, footer, sidebar)
fn extract_main_content(document: &Html) -> String {
    // Try to find main content area
    let main_selectors = [
        "main",
        "article",
        "[role='main']",
        ".content",
        "#content",
        ".post-content",
        ".article-content",
        ".entry-content",
    ];

    for selector_str in main_selectors {
        if let Ok(sel) = Selector::parse(selector_str) {
            let content: String = document.select(&sel)
                .map(|el| el.html())
                .collect();
            
            if !content.is_empty() {
                return content;
            }
        }
    }

    // Fallback to body without boilerplate
    if let Some(body) = document.select(&SELECTORS.body).next() {
        let mut content = body.html();
        
        // Remove common boilerplate elements
        let boilerplate = ["<nav", "<header", "<footer", "<aside", "<script", "<style"];
        for bp in boilerplate {
            if let Some(start) = content.find(bp) {
                if let Some(end) = content[start..].find('>') {
                    // Find closing tag - simplified removal
                    let tag_end = start + end + 1;
                    content = format!("{}{}", &content[..start], &content[tag_end..]);
                }
            }
        }
        
        return content;
    }

    document.html()
}

/// Extract text content only (no HTML tags)
fn extract_text_only(document: &Html) -> String {
    document.root_element()
        .text()
        .collect::<String>()
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

/// Extract document structure (tag hierarchy)
fn extract_structure(document: &Html) -> String {
    let mut structure = String::new();
    extract_structure_recursive(document.root_element(), &mut structure, 0);
    structure
}

/// Recursively extract structure
fn extract_structure_recursive(
    element: scraper::ElementRef,
    structure: &mut String,
    depth: usize,
) {
    // Add tag name with depth indicator
    structure.push_str(&format!("{}:{}", depth, element.value().name()));
    
    // Add significant attributes
    for attr in ["id", "class", "role"] {
        if let Some(val) = element.value().attr(attr) {
            // Only include first class and id for fingerprinting
            let short_val: String = val.split_whitespace().take(1).collect();
            if !short_val.is_empty() {
                structure.push_str(&format!("[{}={}]", attr, short_val));
            }
        }
    }
    
    structure.push(';');
    
    // Recurse into children (limit depth for performance)
    if depth < 10 {
        for child in element.children() {
            if let Some(el) = scraper::ElementRef::wrap(child) {
                // Skip script and style
                let name = el.value().name();
                if name != "script" && name != "style" && name != "noscript" {
                    extract_structure_recursive(el, structure, depth + 1);
                }
            }
        }
    }
}

/// Count total elements in document
fn count_elements(document: &Html) -> usize {
    if let Ok(sel) = Selector::parse("*") {
        document.select(&sel).count()
    } else {
        0
    }
}

/// Count text nodes in document
fn count_text_nodes(document: &Html) -> usize {
    document.root_element()
        .text()
        .filter(|t| !t.trim().is_empty())
        .count()
}

// ============================================================================
// AMP DETECTION
// ============================================================================

/// Extract AMP information from document
pub fn extract_amp_info(document: &Html, base_url: Option<&url::Url>) -> ParserResult<AmpInfo> {
    let mut info = AmpInfo::new();

    // Check if this is an AMP page
    info.is_amp_html = detect_is_amp_page(document);
    info.is_amp = info.is_amp_html;

    // Get AMP URL (if this is not an AMP page)
    if !info.is_amp {
        info.amp_url = extract_amp_link(document, base_url);
        if info.amp_url.is_some() {
            info.is_amp = true; // Has AMP version
        }
    }

    // Get canonical URL (if this is an AMP page)
    if info.is_amp_html {
        info.canonical_url = extract_canonical_link(document, base_url);
    }

    // Check for AMP runtime
    info.has_amp_runtime = detect_amp_runtime(document);

    // Extract AMP components
    info.components = extract_amp_components(document);

    // Try to detect AMP version
    info.amp_version = detect_amp_version(document);

    Ok(info)
}

/// Check if document is an AMP HTML page
fn detect_is_amp_page(document: &Html) -> bool {
    // Check for ⚡ or amp attribute on html element
    if let Some(html) = document.select(&SELECTORS.html).next() {
        // Check for amp or ⚡ attribute
        if html.value().attr("amp").is_some() || html.value().attr("⚡").is_some() {
            return true;
        }
        
        // Check class
        if html.value().classes().any(|c| c == "amp" || c == "⚡") {
            return true;
        }
    }

    // Check for AMP boilerplate in head
    let html_str = document.html();
    html_str.contains("amp-boilerplate") || 
    html_str.contains("cdn.ampproject.org")
}

/// Extract link to AMP version
fn extract_amp_link(document: &Html, base_url: Option<&url::Url>) -> Option<String> {
    if let Ok(sel) = Selector::parse("link[rel='amphtml']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(href) = el.value().attr("href") {
                return resolve_url(href, base_url);
            }
        }
    }
    None
}

/// Extract canonical link
fn extract_canonical_link(document: &Html, base_url: Option<&url::Url>) -> Option<String> {
    if let Ok(sel) = Selector::parse("link[rel='canonical']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(href) = el.value().attr("href") {
                return resolve_url(href, base_url);
            }
        }
    }
    None
}

/// Check for AMP runtime script
fn detect_amp_runtime(document: &Html) -> bool {
    if let Ok(sel) = Selector::parse("script[src*='cdn.ampproject.org']") {
        return document.select(&sel).next().is_some();
    }
    false
}

/// Extract AMP component names
fn extract_amp_components(document: &Html) -> Vec<String> {
    let mut components = Vec::new();

    // Find custom element scripts
    if let Ok(sel) = Selector::parse("script[custom-element]") {
        for el in document.select(&sel) {
            if let Some(name) = el.value().attr("custom-element") {
                if !components.contains(&name.to_string()) {
                    components.push(name.to_string());
                }
            }
        }
    }

    // Also check for amp-* tags in the document
    let html = document.html().to_lowercase();
    let amp_tags = [
        "amp-img", "amp-video", "amp-audio", "amp-carousel",
        "amp-accordion", "amp-sidebar", "amp-lightbox",
        "amp-analytics", "amp-ad", "amp-social-share",
        "amp-form", "amp-list", "amp-bind", "amp-state",
    ];

    for tag in amp_tags {
        if html.contains(&format!("<{}", tag)) {
            let tag_str = tag.to_string();
            if !components.contains(&tag_str) {
                components.push(tag_str);
            }
        }
    }

    components
}

/// Detect AMP version from runtime URL
fn detect_amp_version(document: &Html) -> Option<String> {
    if let Ok(sel) = Selector::parse("script[src*='cdn.ampproject.org']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(src) = el.value().attr("src") {
                // Try to extract version from URL like /v0.js or /v0/amp-component-0.1.js
                if src.contains("/v0") {
                    return Some("v0".to_string());
                }
                // Could parse more specific versions here
            }
        }
    }
    None
}

/// Resolve URL helper
fn resolve_url(href: &str, base_url: Option<&url::Url>) -> Option<String> {
    if href.starts_with("http://") || href.starts_with("https://") {
        return Some(href.to_string());
    }

    if let Some(base) = base_url {
        return base.join(href).ok().map(|u| u.to_string());
    }

    None
}

// ============================================================================
// CACHE HINTS EXTRACTION
// ============================================================================

/// Extract cache hints from meta tags
pub fn extract_cache_hints(document: &Html) -> CacheHints {
    let mut hints = CacheHints::default();

    // Check for no-cache meta tag
    if let Ok(sel) = Selector::parse("meta[http-equiv='Cache-Control']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(content) = el.value().attr("content") {
                hints.cache_control = Some(content.to_string());
                hints.no_cache = content.to_lowercase().contains("no-cache") ||
                                 content.to_lowercase().contains("no-store");
                
                // Try to extract max-age
                if let Some(pos) = content.to_lowercase().find("max-age=") {
                    let start = pos + 8;
                    let num: String = content[start..]
                        .chars()
                        .take_while(|c| c.is_ascii_digit())
                        .collect();
                    hints.max_age = num.parse().ok();
                }
            }
        }
    }

    // Check for Pragma: no-cache
    if let Ok(sel) = Selector::parse("meta[http-equiv='Pragma']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(content) = el.value().attr("content") {
                if content.to_lowercase().contains("no-cache") {
                    hints.no_cache = true;
                }
            }
        }
    }

    hints
}

// ============================================================================
// CONVENIENCE FUNCTIONS
// ============================================================================

/// Check if content has changed between two HTML strings
pub fn has_content_changed(old_html: &str, new_html: &str) -> bool {
    let old_fp = generate_fingerprint(old_html).unwrap_or_default();
    let new_fp = generate_fingerprint(new_html).unwrap_or_default();
    old_fp.has_changed(&new_fp)
}

/// Get content similarity between two HTML strings (0.0 to 1.0)
pub fn content_similarity(html1: &str, html2: &str) -> f64 {
    let fp1 = generate_fingerprint(html1).unwrap_or_default();
    let fp2 = generate_fingerprint(html2).unwrap_or_default();
    fp1.similarity(&fp2)
}

/// Check if page is an AMP page
pub fn is_amp_page(document: &Html) -> bool {
    detect_is_amp_page(document)
}

/// Get AMP URL if available
pub fn get_amp_url(document: &Html) -> Option<String> {
    extract_amp_link(document, None)
}

/// Quick hash of HTML content
pub fn quick_hash(html: &str) -> u64 {
    hash_string(html)
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_generate_fingerprint() {
        let html = "<html><body><p>Hello world</p></body></html>";
        let fp = generate_fingerprint(html).unwrap();

        assert!(fp.full_hash != 0);
        assert!(fp.content_hash != 0);
        assert!(fp.text_hash != 0);
        assert!(fp.structure_hash != 0);
        assert!(fp.element_count > 0);
        assert!(fp.content_length > 0);
    }

    #[test]
    fn test_fingerprint_same_content() {
        let html1 = "<html><body><p>Hello world</p></body></html>";
        let html2 = "<html><body><p>Hello world</p></body></html>";

        let fp1 = generate_fingerprint(html1).unwrap();
        let fp2 = generate_fingerprint(html2).unwrap();

        assert!(!fp1.has_changed(&fp2));
        assert_eq!(fp1.similarity(&fp2), 1.0);
    }

    #[test]
    fn test_fingerprint_different_content() {
        let html1 = "<html><body><p>Hello world</p></body></html>";
        let html2 = "<html><body><p>Goodbye world</p></body></html>";

        let fp1 = generate_fingerprint(html1).unwrap();
        let fp2 = generate_fingerprint(html2).unwrap();

        assert!(fp1.has_changed(&fp2));
        // Structure should be the same
        assert!(!fp1.has_structural_changes(&fp2));
        assert!(fp1.has_minor_changes(&fp2));
    }

    #[test]
    fn test_fingerprint_structural_change() {
        let html1 = "<html><body><p>Hello</p></body></html>";
        let html2 = "<html><body><div><p>Hello</p></div></body></html>";

        let fp1 = generate_fingerprint(html1).unwrap();
        let fp2 = generate_fingerprint(html2).unwrap();

        assert!(fp1.has_structural_changes(&fp2));
    }

    #[test]
    fn test_detect_amp_page() {
        let amp_html = r#"
            <!DOCTYPE html>
            <html amp>
            <head>
                <script async src="https://cdn.ampproject.org/v0.js"></script>
            </head>
            <body></body>
            </html>
        "#;

        let doc = parse_html(amp_html);
        assert!(detect_is_amp_page(&doc));
    }

    #[test]
    fn test_detect_amp_page_lightning() {
        let amp_html = r#"
            <!DOCTYPE html>
            <html ⚡>
            <head></head>
            <body></body>
            </html>
        "#;

        let doc = parse_html(amp_html);
        assert!(detect_is_amp_page(&doc));
    }

    #[test]
    fn test_not_amp_page() {
        let html = "<html><body><p>Regular page</p></body></html>";
        let doc = parse_html(html);
        assert!(!detect_is_amp_page(&doc));
    }

    #[test]
    fn test_extract_amp_link() {
        let html = r#"
            <html>
            <head>
                <link rel="amphtml" href="https://example.com/page.amp">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let amp_url = extract_amp_link(&doc, None);
        assert_eq!(amp_url, Some("https://example.com/page.amp".to_string()));
    }

    #[test]
    fn test_extract_amp_components() {
        let html = r#"
            <html amp>
            <head>
                <script custom-element="amp-carousel" src="..."></script>
                <script custom-element="amp-analytics" src="..."></script>
            </head>
            <body>
                <amp-img src="test.jpg"></amp-img>
            </body>
            </html>
        "#;

        let doc = parse_html(html);
        let components = extract_amp_components(&doc);

        assert!(components.contains(&"amp-carousel".to_string()));
        assert!(components.contains(&"amp-analytics".to_string()));
        assert!(components.contains(&"amp-img".to_string()));
    }

    #[test]
    fn test_extract_amp_info() {
        let html = r#"
            <html>
            <head>
                <link rel="amphtml" href="/amp/page">
                <link rel="canonical" href="/page">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let base = url::Url::parse("https://example.com/").unwrap();
        let info = extract_amp_info(&doc, Some(&base)).unwrap();

        assert!(info.has_amp_version());
        assert_eq!(info.amp_url, Some("https://example.com/amp/page".to_string()));
    }

    #[test]
    fn test_extract_cache_hints() {
        let html = r#"
            <html>
            <head>
                <meta http-equiv="Cache-Control" content="max-age=3600, public">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let hints = extract_cache_hints(&doc);

        assert!(!hints.no_cache);
        assert_eq!(hints.max_age, Some(3600));
    }

    #[test]
    fn test_cache_no_cache() {
        let html = r#"
            <html>
            <head>
                <meta http-equiv="Cache-Control" content="no-cache, no-store">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let hints = extract_cache_hints(&doc);

        assert!(hints.no_cache);
    }

    #[test]
    fn test_has_content_changed() {
        let html1 = "<html><body><p>Version 1</p></body></html>";
        let html2 = "<html><body><p>Version 2</p></body></html>";

        assert!(has_content_changed(html1, html2));
        assert!(!has_content_changed(html1, html1));
    }

    #[test]
    fn test_content_similarity() {
        let html1 = "<html><body><p>Hello world</p></body></html>";
        let html2 = "<html><body><p>Hello world</p></body></html>";

        assert_eq!(content_similarity(html1, html2), 1.0);

        let html3 = "<html><body><p>Different content entirely</p></body></html>";
        let sim = content_similarity(html1, html3);
        assert!(sim < 1.0);
        assert!(sim > 0.0);
    }

    #[test]
    fn test_quick_hash() {
        let html1 = "<html><body>Test</body></html>";
        let html2 = "<html><body>Test</body></html>";
        let html3 = "<html><body>Different</body></html>";

        assert_eq!(quick_hash(html1), quick_hash(html2));
        assert_ne!(quick_hash(html1), quick_hash(html3));
    }

    #[test]
    fn test_fingerprint_similarity_range() {
        let html1 = "<html><body><div><p>Test</p></div></body></html>";
        let html2 = "<html><body><span><p>Test</p></span></body></html>";

        let fp1 = generate_fingerprint(html1).unwrap();
        let fp2 = generate_fingerprint(html2).unwrap();

        let sim = fp1.similarity(&fp2);
        assert!(sim >= 0.0 && sim <= 1.0);
    }
}