halldyll_parser/

feeds.rs

//! Feed and sitemap detection for halldyll-parser
//!
//! This module handles detection and extraction of:
//! - RSS feeds
//! - Atom feeds
//! - Sitemap XML
//! - Sitemap index
//! - JSON Feed

use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use url::Url;

use crate::types::ParserResult;

// ============================================================================
// TYPES
// ============================================================================

/// All feeds and sitemaps found on a page
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub struct FeedInfo {
    /// RSS feeds found
    pub rss_feeds: Vec<Feed>,
    /// Atom feeds found
    pub atom_feeds: Vec<Feed>,
    /// JSON feeds found
    pub json_feeds: Vec<Feed>,
    /// Sitemap URLs found
    pub sitemaps: Vec<Sitemap>,
    /// Whether page has any feeds
    pub has_feeds: bool,
    /// Whether page has sitemaps
    pub has_sitemaps: bool,
}

impl FeedInfo {
    pub fn new() -> Self {
        Self::default()
    }

    /// Get all feeds (RSS + Atom + JSON)
    pub fn all_feeds(&self) -> Vec<&Feed> {
        self.rss_feeds.iter()
            .chain(self.atom_feeds.iter())
            .chain(self.json_feeds.iter())
            .collect()
    }

    /// Get primary feed (prefer Atom, then RSS, then JSON)
    pub fn primary_feed(&self) -> Option<&Feed> {
        self.atom_feeds.first()
            .or_else(|| self.rss_feeds.first())
            .or_else(|| self.json_feeds.first())
    }

    /// Get all feed URLs
    pub fn feed_urls(&self) -> Vec<&str> {
        self.all_feeds().iter().map(|f| f.url.as_str()).collect()
    }

    /// Get all sitemap URLs
    pub fn sitemap_urls(&self) -> Vec<&str> {
        self.sitemaps.iter().map(|s| s.url.as_str()).collect()
    }
}

/// A web feed (RSS, Atom, or JSON)
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Feed {
    /// Feed URL
    pub url: String,
    /// Feed title (if available)
    pub title: Option<String>,
    /// Feed type
    pub feed_type: FeedType,
    /// MIME type from link element
    pub mime_type: Option<String>,
    /// Language hint
    pub language: Option<String>,
}

impl Feed {
    pub fn new(url: String, feed_type: FeedType) -> Self {
        Self {
            url,
            title: None,
            feed_type,
            mime_type: None,
            language: None,
        }
    }

    /// Check if this is an RSS feed
    pub fn is_rss(&self) -> bool {
        matches!(self.feed_type, FeedType::Rss | FeedType::Rss2)
    }

    /// Check if this is an Atom feed
    pub fn is_atom(&self) -> bool {
        matches!(self.feed_type, FeedType::Atom)
    }
}

/// Type of web feed
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum FeedType {
    /// RSS 1.0
    Rss,
    /// RSS 2.0
    #[default]
    Rss2,
    /// Atom
    Atom,
    /// JSON Feed
    Json,
    /// Unknown/Generic
    Unknown,
}

/// A sitemap reference
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Sitemap {
    /// Sitemap URL
    pub url: String,
    /// Sitemap type
    pub sitemap_type: SitemapType,
    /// Source of discovery
    pub source: SitemapSource,
}

impl Sitemap {
    pub fn new(url: String, sitemap_type: SitemapType) -> Self {
        Self {
            url,
            sitemap_type,
            source: SitemapSource::LinkTag,
        }
    }
}

/// Type of sitemap
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum SitemapType {
    /// Standard XML sitemap
    #[default]
    Xml,
    /// Sitemap index (contains other sitemaps)
    Index,
    /// News sitemap
    News,
    /// Image sitemap
    Image,
    /// Video sitemap
    Video,
    /// Text sitemap
    Text,
    /// Gzip compressed sitemap
    Gzip,
}

/// How sitemap was discovered
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum SitemapSource {
    /// From <link> tag
    #[default]
    LinkTag,
    /// From robots.txt
    RobotsTxt,
    /// From well-known path
    WellKnown,
    /// From sitemap index
    SitemapIndex,
}

// ============================================================================
// EXTRACTION FUNCTIONS
// ============================================================================

/// Extract all feed and sitemap information from HTML document
pub fn extract_feed_info(document: &Html, base_url: Option<&Url>) -> ParserResult<FeedInfo> {
    let mut info = FeedInfo::new();

    // Extract feeds from <link> elements
    extract_link_feeds(document, &mut info, base_url);

    // Extract sitemaps
    extract_sitemaps(document, &mut info, base_url);

    // Update flags
    info.has_feeds = !info.rss_feeds.is_empty() || 
                     !info.atom_feeds.is_empty() || 
                     !info.json_feeds.is_empty();
    info.has_sitemaps = !info.sitemaps.is_empty();

    Ok(info)
}

/// Extract feeds from <link rel="alternate"> elements
fn extract_link_feeds(document: &Html, info: &mut FeedInfo, base_url: Option<&Url>) {
    // RSS and Atom feeds
    let feed_selector = Selector::parse(
        r#"link[rel="alternate"][type="application/rss+xml"],
           link[rel="alternate"][type="application/atom+xml"],
           link[rel="alternate"][type="application/feed+json"],
           link[rel="alternate"][type="application/json"]"#
    ).unwrap();

    for el in document.select(&feed_selector) {
        let href = match el.value().attr("href") {
            Some(h) => h,
            None => continue,
        };

        let url = resolve_url(href, base_url).unwrap_or_else(|| href.to_string());
        let mime_type = el.value().attr("type").map(|s| s.to_string());
        let title = el.value().attr("title").map(|s| s.to_string());
        let hreflang = el.value().attr("hreflang").map(|s| s.to_string());

        let feed_type = detect_feed_type(&mime_type, &url);

        let mut feed = Feed::new(url, feed_type);
        feed.title = title;
        feed.mime_type = mime_type;
        feed.language = hreflang;

        match feed_type {
            FeedType::Atom => info.atom_feeds.push(feed),
            FeedType::Json => info.json_feeds.push(feed),
            _ => info.rss_feeds.push(feed),
        }
    }

    // Also check for feed links in <a> elements (common pattern)
    if let Ok(sel) = Selector::parse("a[href*='feed'], a[href*='rss'], a[href*='atom']") {
        for el in document.select(&sel) {
            if let Some(href) = el.value().attr("href") {
                let url = resolve_url(href, base_url).unwrap_or_else(|| href.to_string());
                
                // Skip if already found
                if info.all_feeds().iter().any(|f| f.url == url) {
                    continue;
                }

                // Detect type from URL
                let feed_type = detect_feed_type_from_url(&url);
                if feed_type == FeedType::Unknown {
                    continue;
                }

                let mut feed = Feed::new(url, feed_type);
                feed.title = Some(el.text().collect::<String>().trim().to_string());

                match feed_type {
                    FeedType::Atom => info.atom_feeds.push(feed),
                    FeedType::Json => info.json_feeds.push(feed),
                    _ => info.rss_feeds.push(feed),
                }
            }
        }
    }
}

/// Detect feed type from MIME type and URL
fn detect_feed_type(mime_type: &Option<String>, url: &str) -> FeedType {
    if let Some(ref mime) = mime_type {
        match mime.as_str() {
            "application/atom+xml" => return FeedType::Atom,
            "application/rss+xml" => return FeedType::Rss2,
            "application/feed+json" | "application/json" => {
                if url.contains("feed") {
                    return FeedType::Json;
                }
            }
            _ => {}
        }
    }

    detect_feed_type_from_url(url)
}

/// Detect feed type from URL patterns
fn detect_feed_type_from_url(url: &str) -> FeedType {
    let url_lower = url.to_lowercase();
    
    if url_lower.contains("atom") {
        FeedType::Atom
    } else if url_lower.contains("rss") || url_lower.contains("feed.xml") {
        FeedType::Rss2
    } else if url_lower.ends_with("feed.json") || url_lower.contains("feed/json") {
        FeedType::Json
    } else if url_lower.contains("feed") || url_lower.ends_with(".xml") {
        FeedType::Rss2
    } else {
        FeedType::Unknown
    }
}

/// Extract sitemap references
fn extract_sitemaps(document: &Html, info: &mut FeedInfo, base_url: Option<&Url>) {
    // From <link rel="sitemap"> (less common but valid)
    if let Ok(sel) = Selector::parse("link[rel='sitemap']") {
        for el in document.select(&sel) {
            if let Some(href) = el.value().attr("href") {
                let url = resolve_url(href, base_url).unwrap_or_else(|| href.to_string());
                let sitemap_type = detect_sitemap_type(&url);
                
                let mut sitemap = Sitemap::new(url, sitemap_type);
                sitemap.source = SitemapSource::LinkTag;
                info.sitemaps.push(sitemap);
            }
        }
    }

    // Look for sitemap links in footer/nav
    if let Ok(sel) = Selector::parse("a[href*='sitemap']") {
        for el in document.select(&sel) {
            if let Some(href) = el.value().attr("href") {
                let url = resolve_url(href, base_url).unwrap_or_else(|| href.to_string());
                
                // Skip if already found
                if info.sitemaps.iter().any(|s| s.url == url) {
                    continue;
                }

                let sitemap_type = detect_sitemap_type(&url);
                let mut sitemap = Sitemap::new(url, sitemap_type);
                sitemap.source = SitemapSource::LinkTag;
                info.sitemaps.push(sitemap);
            }
        }
    }
}

/// Detect sitemap type from URL
fn detect_sitemap_type(url: &str) -> SitemapType {
    let url_lower = url.to_lowercase();
    
    if url_lower.ends_with(".gz") {
        SitemapType::Gzip
    } else if url_lower.contains("sitemap_index") || url_lower.contains("sitemap-index") {
        SitemapType::Index
    } else if url_lower.contains("news") {
        SitemapType::News
    } else if url_lower.contains("image") {
        SitemapType::Image
    } else if url_lower.contains("video") {
        SitemapType::Video
    } else if url_lower.ends_with(".txt") {
        SitemapType::Text
    } else {
        SitemapType::Xml
    }
}

/// Resolve URL relative to base
fn resolve_url(href: &str, base_url: Option<&Url>) -> Option<String> {
    if href.starts_with("http://") || href.starts_with("https://") {
        return Some(href.to_string());
    }

    if href.starts_with("//") {
        return Some(format!("https:{}", href));
    }

    if let Some(base) = base_url {
        return base.join(href).ok().map(|u| u.to_string());
    }

    None
}

// ============================================================================
// WELL-KNOWN FEED/SITEMAP PATHS
// ============================================================================

/// Common feed paths to check
pub const COMMON_FEED_PATHS: &[&str] = &[
    "/feed",
    "/feed/",
    "/feed.xml",
    "/feed.rss",
    "/rss",
    "/rss/",
    "/rss.xml",
    "/atom.xml",
    "/atom",
    "/feed.atom",
    "/feeds/posts/default",
    "/blog/feed",
    "/blog/rss",
    "/index.xml",
    "/.rss",
    "/feed.json",
];

/// Common sitemap paths to check
pub const COMMON_SITEMAP_PATHS: &[&str] = &[
    "/sitemap.xml",
    "/sitemap_index.xml",
    "/sitemap",
    "/sitemaps.xml",
    "/sitemap1.xml",
    "/sitemap-index.xml",
    "/post-sitemap.xml",
    "/page-sitemap.xml",
    "/news-sitemap.xml",
    "/sitemap.xml.gz",
];

/// Generate potential feed URLs for a domain
pub fn generate_feed_urls(base_url: &Url) -> Vec<String> {
    COMMON_FEED_PATHS.iter()
        .filter_map(|path| base_url.join(path).ok())
        .map(|u| u.to_string())
        .collect()
}

/// Generate potential sitemap URLs for a domain
pub fn generate_sitemap_urls(base_url: &Url) -> Vec<String> {
    COMMON_SITEMAP_PATHS.iter()
        .filter_map(|path| base_url.join(path).ok())
        .map(|u| u.to_string())
        .collect()
}

// ============================================================================
// CONVENIENCE FUNCTIONS
// ============================================================================

/// Check if document has any feeds
pub fn has_feeds(document: &Html) -> bool {
    extract_feed_info(document, None)
        .map(|i| i.has_feeds)
        .unwrap_or(false)
}

/// Get RSS feed URL if exists
pub fn get_rss_feed(document: &Html, base_url: Option<&Url>) -> Option<String> {
    extract_feed_info(document, base_url)
        .ok()
        .and_then(|i| i.rss_feeds.first().map(|f| f.url.clone()))
}

/// Get Atom feed URL if exists
pub fn get_atom_feed(document: &Html, base_url: Option<&Url>) -> Option<String> {
    extract_feed_info(document, base_url)
        .ok()
        .and_then(|i| i.atom_feeds.first().map(|f| f.url.clone()))
}

/// Get any feed URL (prefers Atom over RSS)
pub fn get_feed(document: &Html, base_url: Option<&Url>) -> Option<String> {
    extract_feed_info(document, base_url)
        .ok()
        .and_then(|i| i.primary_feed().map(|f| f.url.clone()))
}

/// Get sitemap URL if found in document
pub fn get_sitemap(document: &Html, base_url: Option<&Url>) -> Option<String> {
    extract_feed_info(document, base_url)
        .ok()
        .and_then(|i| i.sitemaps.first().map(|s| s.url.clone()))
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_rss_feed() {
        let html = r#"
            <html>
            <head>
                <link rel="alternate" type="application/rss+xml" 
                      title="RSS Feed" href="/feed.xml">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let base = Url::parse("https://example.com/").unwrap();
        let info = extract_feed_info(&doc, Some(&base)).unwrap();

        assert!(info.has_feeds);
        assert_eq!(info.rss_feeds.len(), 1);
        assert_eq!(info.rss_feeds[0].url, "https://example.com/feed.xml");
        assert_eq!(info.rss_feeds[0].title, Some("RSS Feed".to_string()));
    }

    #[test]
    fn test_extract_atom_feed() {
        let html = r#"
            <html>
            <head>
                <link rel="alternate" type="application/atom+xml" 
                      title="Atom Feed" href="/atom.xml">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let base = Url::parse("https://example.com/").unwrap();
        let info = extract_feed_info(&doc, Some(&base)).unwrap();

        assert!(info.has_feeds);
        assert_eq!(info.atom_feeds.len(), 1);
        assert_eq!(info.atom_feeds[0].feed_type, FeedType::Atom);
    }

    #[test]
    fn test_extract_json_feed() {
        let html = r#"
            <html>
            <head>
                <link rel="alternate" type="application/feed+json" 
                      title="JSON Feed" href="/feed.json">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let info = extract_feed_info(&doc, None).unwrap();

        assert!(info.has_feeds);
        assert_eq!(info.json_feeds.len(), 1);
        assert_eq!(info.json_feeds[0].feed_type, FeedType::Json);
    }

    #[test]
    fn test_extract_multiple_feeds() {
        let html = r#"
            <html>
            <head>
                <link rel="alternate" type="application/rss+xml" href="/rss.xml">
                <link rel="alternate" type="application/atom+xml" href="/atom.xml">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let info = extract_feed_info(&doc, None).unwrap();

        assert_eq!(info.all_feeds().len(), 2);
        // Primary should be Atom
        assert_eq!(info.primary_feed().unwrap().feed_type, FeedType::Atom);
    }

    #[test]
    fn test_extract_sitemap_link() {
        let html = r#"
            <html>
            <body>
                <footer>
                    <a href="/sitemap.xml">Sitemap</a>
                </footer>
            </body>
            </html>
        "#;

        let doc = parse_html(html);
        let base = Url::parse("https://example.com/").unwrap();
        let info = extract_feed_info(&doc, Some(&base)).unwrap();

        assert!(info.has_sitemaps);
        assert_eq!(info.sitemaps[0].url, "https://example.com/sitemap.xml");
    }

    #[test]
    fn test_detect_sitemap_types() {
        assert_eq!(detect_sitemap_type("/sitemap.xml"), SitemapType::Xml);
        assert_eq!(detect_sitemap_type("/sitemap.xml.gz"), SitemapType::Gzip);
        assert_eq!(detect_sitemap_type("/sitemap_index.xml"), SitemapType::Index);
        assert_eq!(detect_sitemap_type("/news-sitemap.xml"), SitemapType::News);
        assert_eq!(detect_sitemap_type("/image-sitemap.xml"), SitemapType::Image);
        assert_eq!(detect_sitemap_type("/sitemap.txt"), SitemapType::Text);
    }

    #[test]
    fn test_detect_feed_type_from_url() {
        assert_eq!(detect_feed_type_from_url("/atom.xml"), FeedType::Atom);
        assert_eq!(detect_feed_type_from_url("/rss.xml"), FeedType::Rss2);
        assert_eq!(detect_feed_type_from_url("/feed.json"), FeedType::Json);
        assert_eq!(detect_feed_type_from_url("/feed"), FeedType::Rss2);
    }

    #[test]
    fn test_generate_feed_urls() {
        let base = Url::parse("https://example.com/").unwrap();
        let urls = generate_feed_urls(&base);

        assert!(urls.contains(&"https://example.com/feed".to_string()));
        assert!(urls.contains(&"https://example.com/rss.xml".to_string()));
        assert!(urls.contains(&"https://example.com/atom.xml".to_string()));
    }

    #[test]
    fn test_generate_sitemap_urls() {
        let base = Url::parse("https://example.com/").unwrap();
        let urls = generate_sitemap_urls(&base);

        assert!(urls.contains(&"https://example.com/sitemap.xml".to_string()));
        assert!(urls.contains(&"https://example.com/sitemap_index.xml".to_string()));
    }

    #[test]
    fn test_feed_info_methods() {
        let mut info = FeedInfo::new();
        info.rss_feeds.push(Feed::new("/rss".to_string(), FeedType::Rss2));
        info.atom_feeds.push(Feed::new("/atom".to_string(), FeedType::Atom));

        assert_eq!(info.all_feeds().len(), 2);
        assert_eq!(info.feed_urls(), vec!["/rss", "/atom"]);
        assert_eq!(info.primary_feed().unwrap().feed_type, FeedType::Atom);
    }

    #[test]
    fn test_feed_is_rss_atom() {
        let rss = Feed::new("/feed".to_string(), FeedType::Rss2);
        let atom = Feed::new("/atom".to_string(), FeedType::Atom);

        assert!(rss.is_rss());
        assert!(!rss.is_atom());
        assert!(atom.is_atom());
        assert!(!atom.is_rss());
    }

    #[test]
    fn test_no_feeds() {
        let html = "<html><body><p>No feeds here</p></body></html>";
        let doc = parse_html(html);
        let info = extract_feed_info(&doc, None).unwrap();

        assert!(!info.has_feeds);
        assert!(!info.has_sitemaps);
    }

    #[test]
    fn test_feed_with_hreflang() {
        let html = r#"
            <html>
            <head>
                <link rel="alternate" type="application/rss+xml" 
                      hreflang="en" href="/feed-en.xml">
                <link rel="alternate" type="application/rss+xml" 
                      hreflang="fr" href="/feed-fr.xml">
            </head>
            </html>
        "#;

        let doc = parse_html(html);
        let info = extract_feed_info(&doc, None).unwrap();

        assert_eq!(info.rss_feeds.len(), 2);
        assert_eq!(info.rss_feeds[0].language, Some("en".to_string()));
        assert_eq!(info.rss_feeds[1].language, Some("fr".to_string()));
    }
}