halldyll-parser 0.1.0

//! Pagination detection for halldyll-parser
//!
//! This module handles:
//! - rel="next"/rel="prev" link detection
//! - Pagination URL patterns
//! - Page number extraction
//! - Infinite scroll detection
//! - Load more button detection

use regex::Regex;
use scraper::{Html, Selector};
use serde::{Deserialize, Serialize};
use lazy_static::lazy_static;
use url::Url;

use crate::types::ParserResult;

// ============================================================================
// TYPES
// ============================================================================

/// Pagination information for a page
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
pub struct Pagination {
    /// Current page number (if detected)
    pub current_page: Option<u32>,
    /// Total pages (if detected)
    pub total_pages: Option<u32>,
    /// Previous page URL (rel="prev")
    pub prev_url: Option<String>,
    /// Next page URL (rel="next")
    pub next_url: Option<String>,
    /// First page URL
    pub first_url: Option<String>,
    /// Last page URL
    pub last_url: Option<String>,
    /// All detected page URLs with their numbers
    pub page_urls: Vec<PageUrl>,
    /// Pagination type detected
    pub pagination_type: PaginationType,
    /// Whether infinite scroll is detected
    pub has_infinite_scroll: bool,
    /// Whether "load more" button is detected
    pub has_load_more: bool,
    /// Items per page (if detected)
    pub items_per_page: Option<u32>,
    /// Total items (if detected)
    pub total_items: Option<u32>,
}

impl Pagination {
    pub fn new() -> Self {
        Self::default()
    }

    /// Check if pagination exists
    pub fn has_pagination(&self) -> bool {
        self.prev_url.is_some() || 
        self.next_url.is_some() || 
        !self.page_urls.is_empty() ||
        self.has_infinite_scroll ||
        self.has_load_more
    }

    /// Check if there's a next page
    pub fn has_next(&self) -> bool {
        self.next_url.is_some()
    }

    /// Check if there's a previous page
    pub fn has_prev(&self) -> bool {
        self.prev_url.is_some()
    }

    /// Check if this is the first page
    pub fn is_first_page(&self) -> bool {
        self.prev_url.is_none() && self.current_page.map(|p| p <= 1).unwrap_or(true)
    }

    /// Check if this is the last page
    pub fn is_last_page(&self) -> bool {
        self.next_url.is_none() && 
        self.current_page.is_some() && 
        self.total_pages.is_some() &&
        self.current_page == self.total_pages
    }

    /// Get all URLs to crawl for complete pagination
    pub fn all_page_urls(&self) -> Vec<String> {
        let mut urls: Vec<String> = self.page_urls.iter()
            .map(|p| p.url.clone())
            .collect();
        
        if let Some(ref url) = self.first_url {
            if !urls.contains(url) {
                urls.insert(0, url.clone());
            }
        }
        if let Some(ref url) = self.last_url {
            if !urls.contains(url) {
                urls.push(url.clone());
            }
        }
        
        urls
    }
}

/// A page URL with its page number
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PageUrl {
    pub url: String,
    pub page_number: Option<u32>,
    pub is_current: bool,
}

/// Type of pagination detected
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)]
pub enum PaginationType {
    /// Standard numbered pagination (1, 2, 3...)
    Numbered,
    /// Next/Previous only
    NextPrev,
    /// Infinite scroll
    InfiniteScroll,
    /// Load more button
    LoadMore,
    /// Cursor-based pagination
    Cursor,
    /// Offset-based pagination
    Offset,
    /// Unknown or no pagination
    #[default]
    None,
}

// ============================================================================
// LAZY STATIC PATTERNS
// ============================================================================

lazy_static! {
    /// Pattern for page number in URL query string
    static ref PAGE_QUERY_PATTERN: Regex = Regex::new(
        r"(?i)[?&](page|p|pg|pn|pagenum|pagenumber|offset|start|from)=(\d+)"
    ).unwrap();
    
    /// Pattern for page number in URL path
    static ref PAGE_PATH_PATTERN: Regex = Regex::new(
        r"(?i)/(?:page|p|pg)/(\d+)"
    ).unwrap();
    
    /// Pattern for page number at end of path
    static ref PAGE_END_PATTERN: Regex = Regex::new(
        r"/(\d+)/?$"
    ).unwrap();
    
    /// Pattern for "showing X of Y" text
    static ref SHOWING_PATTERN: Regex = Regex::new(
        r"(?i)(?:showing|displaying)\s+(?:\d+[-–]\d+|\d+)\s+(?:of|from)\s+(\d+)"
    ).unwrap();
    
    /// Pattern for "page X of Y" text
    static ref PAGE_OF_PATTERN: Regex = Regex::new(
        r"(?i)page\s+(\d+)\s+(?:of|/)\s+(\d+)"
    ).unwrap();
    
    /// Pattern for items per page
    static ref ITEMS_PER_PAGE_PATTERN: Regex = Regex::new(
        r"(?i)(\d+)\s+(?:per\s+page|results|items)"
    ).unwrap();
}

// ============================================================================
// EXTRACTION FUNCTIONS
// ============================================================================

/// Extract pagination information from HTML document
pub fn extract_pagination(document: &Html, base_url: Option<&Url>) -> ParserResult<Pagination> {
    let mut pagination = Pagination::new();

    // Extract rel="next" and rel="prev" links
    extract_rel_links(document, &mut pagination, base_url);

    // Extract pagination from link elements
    extract_page_links(document, &mut pagination, base_url);

    // Detect current page and total pages from text
    extract_page_info_from_text(document, &mut pagination);

    // Detect infinite scroll
    pagination.has_infinite_scroll = detect_infinite_scroll(document);

    // Detect load more button
    pagination.has_load_more = detect_load_more(document);

    // Determine pagination type
    pagination.pagination_type = determine_pagination_type(&pagination);

    Ok(pagination)
}

/// Extract rel="next" and rel="prev" links from <link> elements
fn extract_rel_links(document: &Html, pagination: &mut Pagination, base_url: Option<&Url>) {
    // rel="next"
    if let Ok(sel) = Selector::parse("link[rel='next'], a[rel='next']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(href) = el.value().attr("href") {
                pagination.next_url = resolve_url(href, base_url);
            }
        }
    }

    // rel="prev" or rel="previous"
    if let Ok(sel) = Selector::parse("link[rel='prev'], link[rel='previous'], a[rel='prev'], a[rel='previous']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(href) = el.value().attr("href") {
                pagination.prev_url = resolve_url(href, base_url);
            }
        }
    }

    // rel="first"
    if let Ok(sel) = Selector::parse("link[rel='first'], a[rel='first']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(href) = el.value().attr("href") {
                pagination.first_url = resolve_url(href, base_url);
            }
        }
    }

    // rel="last"
    if let Ok(sel) = Selector::parse("link[rel='last'], a[rel='last']") {
        if let Some(el) = document.select(&sel).next() {
            if let Some(href) = el.value().attr("href") {
                pagination.last_url = resolve_url(href, base_url);
            }
        }
    }
}

/// Extract page links from common pagination patterns
fn extract_page_links(document: &Html, pagination: &mut Pagination, base_url: Option<&Url>) {
    // Common pagination selectors
    let pagination_selectors = [
        ".pagination a",
        ".pager a",
        ".page-numbers a",
        ".pages a",
        "nav.pagination a",
        "[class*='pagination'] a",
        "[class*='pager'] a",
        "[aria-label='pagination'] a",
        "[role='navigation'] a[href*='page']",
    ];

    let mut seen_urls = std::collections::HashSet::new();

    for selector_str in pagination_selectors {
        if let Ok(sel) = Selector::parse(selector_str) {
            for el in document.select(&sel) {
                if let Some(href) = el.value().attr("href") {
                    let resolved = resolve_url(href, base_url);
                    
                    if let Some(ref url) = resolved {
                        if seen_urls.contains(url) {
                            continue;
                        }
                        seen_urls.insert(url.clone());

                        // Try to extract page number
                        let page_number = extract_page_number_from_url(url)
                            .or_else(|| {
                                // Try from link text
                                let text = el.text().collect::<String>().trim().to_string();
                                text.parse::<u32>().ok()
                            });

                        // Check if this is current page
                        let is_current = el.value().classes().any(|c| 
                            c.contains("current") || c.contains("active") || c.contains("selected")

                        ) || el.value().attr("aria-current").is_some();

                        let page_url = PageUrl {
                            url: url.clone(),
                            page_number,
                            is_current,
                        };

                        // Update current page if found
                        if is_current {
                            pagination.current_page = page_number;
                        }

                        pagination.page_urls.push(page_url);
                    }
                }
            }
        }
    }

    // Sort by page number
    pagination.page_urls.sort_by(|a, b| {
        match (a.page_number, b.page_number) {
            (Some(a), Some(b)) => a.cmp(&b),
            (Some(_), None) => std::cmp::Ordering::Less,
            (None, Some(_)) => std::cmp::Ordering::Greater,
            (None, None) => std::cmp::Ordering::Equal,
        }
    });

    // Deduplicate
    pagination.page_urls.dedup_by(|a, b| a.url == b.url);

    // Try to determine total pages from max page number
    if let Some(max) = pagination.page_urls.iter().filter_map(|p| p.page_number).max() {
        if pagination.total_pages.is_none() {
            pagination.total_pages = Some(max);
        }
    }
}

/// Extract page number from URL
pub fn extract_page_number_from_url(url: &str) -> Option<u32> {
    // Try query parameter patterns
    if let Some(caps) = PAGE_QUERY_PATTERN.captures(url) {
        if let Some(num) = caps.get(2) {
            return num.as_str().parse().ok();
        }
    }

    // Try path patterns like /page/2
    if let Some(caps) = PAGE_PATH_PATTERN.captures(url) {
        if let Some(num) = caps.get(1) {
            return num.as_str().parse().ok();
        }
    }

    // Try number at end of path
    if let Some(caps) = PAGE_END_PATTERN.captures(url) {
        if let Some(num) = caps.get(1) {
            return num.as_str().parse().ok();
        }
    }

    None
}

/// Extract page info from text content
fn extract_page_info_from_text(document: &Html, pagination: &mut Pagination) {
    let body_text = document.root_element().text().collect::<String>();

    // Try "page X of Y" pattern
    if let Some(caps) = PAGE_OF_PATTERN.captures(&body_text) {
        if let (Some(current), Some(total)) = (caps.get(1), caps.get(2)) {
            if pagination.current_page.is_none() {
                pagination.current_page = current.as_str().parse().ok();
            }
            if pagination.total_pages.is_none() {
                pagination.total_pages = total.as_str().parse().ok();
            }
        }
    }

    // Try "showing X of Y" pattern for total items
    if let Some(caps) = SHOWING_PATTERN.captures(&body_text) {
        if let Some(total) = caps.get(1) {
            pagination.total_items = total.as_str().parse().ok();
        }
    }

    // Try items per page pattern
    if let Some(caps) = ITEMS_PER_PAGE_PATTERN.captures(&body_text) {
        if let Some(per_page) = caps.get(1) {
            pagination.items_per_page = per_page.as_str().parse().ok();
        }
    }
}

/// Detect infinite scroll
fn detect_infinite_scroll(document: &Html) -> bool {
    let html = document.html().to_lowercase();

    // Check for common infinite scroll libraries/patterns
    html.contains("infinite-scroll") ||
    html.contains("infinitescroll") ||
    html.contains("infinite_scroll") ||
    html.contains("data-infinite") ||
    html.contains("waypoint") ||
    html.contains("scroll-trigger") ||
    html.contains("lazy-load") && html.contains("scroll")
}

/// Detect load more button
fn detect_load_more(document: &Html) -> bool {
    let load_more_selectors = [
        "button[class*='load-more']",
        "button[class*='loadmore']",
        "a[class*='load-more']",
        "a[class*='loadmore']",
        "[class*='show-more']",
        "[class*='showmore']",
        "[data-action='load-more']",
    ];

    for selector_str in load_more_selectors {
        if let Ok(sel) = Selector::parse(selector_str) {
            if document.select(&sel).next().is_some() {
                return true;
            }
        }
    }

    // Also check for common text patterns in buttons
    if let Ok(sel) = Selector::parse("button, a.btn, a.button") {
        for el in document.select(&sel) {
            let text = el.text().collect::<String>().to_lowercase();
            if text.contains("load more") || 
               text.contains("show more") ||
               text.contains("view more") ||
               text.contains("see more") {
                return true;
            }
        }
    }

    false
}

/// Determine pagination type
fn determine_pagination_type(pagination: &Pagination) -> PaginationType {
    if pagination.has_infinite_scroll {
        return PaginationType::InfiniteScroll;
    }

    if pagination.has_load_more {
        return PaginationType::LoadMore;
    }

    if !pagination.page_urls.is_empty() {
        return PaginationType::Numbered;
    }

    if pagination.next_url.is_some() || pagination.prev_url.is_some() {
        return PaginationType::NextPrev;
    }

    PaginationType::None
}

/// Resolve URL relative to base
fn resolve_url(href: &str, base_url: Option<&Url>) -> Option<String> {
    if href.starts_with("http://") || href.starts_with("https://") {
        return Some(href.to_string());
    }

    if href.starts_with("//") {
        return Some(format!("https:{}", href));
    }

    if let Some(base) = base_url {
        return base.join(href).ok().map(|u| u.to_string());
    }

    None
}

// ============================================================================
// CONVENIENCE FUNCTIONS
// ============================================================================

/// Check if document has pagination
pub fn has_pagination(document: &Html) -> bool {
    extract_pagination(document, None)
        .map(|p| p.has_pagination())
        .unwrap_or(false)
}

/// Get next page URL if exists
pub fn get_next_page(document: &Html, base_url: Option<&Url>) -> Option<String> {
    extract_pagination(document, base_url)
        .ok()
        .and_then(|p| p.next_url)
}

/// Get previous page URL if exists
pub fn get_prev_page(document: &Html, base_url: Option<&Url>) -> Option<String> {
    extract_pagination(document, base_url)
        .ok()
        .and_then(|p| p.prev_url)
}

/// Generate pagination URL for a specific page number
pub fn generate_page_url(base_url: &str, page_number: u32, pattern: &str) -> String {
    pattern.replace("{page}", &page_number.to_string())
        .replace("{url}", base_url)
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_rel_next_prev() {
        let html = r#"
            <html>
            <head>
                <link rel="prev" href="/page/1">
                <link rel="next" href="/page/3">
            </head>
            <body></body>
            </html>
        "#;

        let doc = parse_html(html);
        let base = Url::parse("https://example.com/page/2").unwrap();
        let pagination = extract_pagination(&doc, Some(&base)).unwrap();

        assert_eq!(pagination.prev_url, Some("https://example.com/page/1".to_string()));
        assert_eq!(pagination.next_url, Some("https://example.com/page/3".to_string()));
        assert!(pagination.has_pagination());
    }

    #[test]
    fn test_extract_numbered_pagination() {
        let html = r#"
            <div class="pagination">
                <a href="/page/1">1</a>
                <a href="/page/2" class="active">2</a>
                <a href="/page/3">3</a>
                <a href="/page/4">4</a>
            </div>
        "#;

        let doc = parse_html(html);
        let base = Url::parse("https://example.com/page/2").unwrap();
        let pagination = extract_pagination(&doc, Some(&base)).unwrap();

        assert_eq!(pagination.page_urls.len(), 4);
        assert_eq!(pagination.current_page, Some(2));
        assert_eq!(pagination.total_pages, Some(4));
        assert_eq!(pagination.pagination_type, PaginationType::Numbered);
    }

    #[test]
    fn test_detect_infinite_scroll() {
        let html = r#"
            <html>
            <body>
                <div class="infinite-scroll" data-infinite="true">
                    Content here
                </div>
            </body>
            </html>
        "#;

        let doc = parse_html(html);
        let pagination = extract_pagination(&doc, None).unwrap();

        assert!(pagination.has_infinite_scroll);
        assert_eq!(pagination.pagination_type, PaginationType::InfiniteScroll);
    }

    #[test]
    fn test_detect_load_more() {
        let html = r#"
            <html>
            <body>
                <div class="items">Items...</div>
                <button class="load-more">Load More</button>
            </body>
            </html>
        "#;

        let doc = parse_html(html);
        let pagination = extract_pagination(&doc, None).unwrap();

        assert!(pagination.has_load_more);
    }

    #[test]
    fn test_extract_page_number_from_url() {
        assert_eq!(extract_page_number_from_url("/articles?page=5"), Some(5));
        assert_eq!(extract_page_number_from_url("/blog/page/3"), Some(3));
        assert_eq!(extract_page_number_from_url("/posts?p=10"), Some(10));
        assert_eq!(extract_page_number_from_url("/items?offset=20"), Some(20));
        assert_eq!(extract_page_number_from_url("/no-page-here"), None);
    }

    #[test]
    fn test_page_of_text_detection() {
        let html = r#"
            <html>
            <body>
                <p>Page 3 of 10</p>
                <p>Showing 21-30 of 100 results</p>
            </body>
            </html>
        "#;

        let doc = parse_html(html);
        let pagination = extract_pagination(&doc, None).unwrap();

        assert_eq!(pagination.current_page, Some(3));
        assert_eq!(pagination.total_pages, Some(10));
        assert_eq!(pagination.total_items, Some(100));
    }

    #[test]
    fn test_is_first_last_page() {
        let mut pagination = Pagination::new();
        pagination.current_page = Some(1);
        pagination.total_pages = Some(5);
        
        assert!(pagination.is_first_page());
        assert!(!pagination.is_last_page());

        pagination.current_page = Some(5);
        assert!(!pagination.is_first_page());
        assert!(pagination.is_last_page());
    }

    #[test]
    fn test_all_page_urls() {
        let mut pagination = Pagination::new();
        pagination.first_url = Some("/page/1".to_string());
        pagination.last_url = Some("/page/5".to_string());
        pagination.page_urls = vec![
            PageUrl { url: "/page/2".to_string(), page_number: Some(2), is_current: false },
            PageUrl { url: "/page/3".to_string(), page_number: Some(3), is_current: true },
        ];

        let all_urls = pagination.all_page_urls();
        assert_eq!(all_urls.len(), 4);
        assert_eq!(all_urls[0], "/page/1");
        assert_eq!(all_urls[3], "/page/5");
    }

    #[test]
    fn test_generate_page_url() {
        let url = generate_page_url("https://example.com", 5, "{url}/page/{page}");
        assert_eq!(url, "https://example.com/page/5");
    }

    #[test]
    fn test_no_pagination() {
        let html = "<html><body><p>Just content, no pagination</p></body></html>";
        let doc = parse_html(html);
        let pagination = extract_pagination(&doc, None).unwrap();

        assert!(!pagination.has_pagination());
        assert_eq!(pagination.pagination_type, PaginationType::None);
    }

    #[test]
    fn test_load_more_text_button() {
        let html = r#"
            <html>
            <body>
                <button class="btn">Load more items</button>
            </body>
            </html>
        "#;

        let doc = parse_html(html);
        let pagination = extract_pagination(&doc, None).unwrap();

        assert!(pagination.has_load_more);
    }

    #[test]
    fn test_aria_pagination() {
        let html = r#"
            <nav aria-label="pagination">
                <a href="/page/1">1</a>
                <a href="/page/2" aria-current="page">2</a>
                <a href="/page/3">3</a>
            </nav>
        "#;

        let doc = parse_html(html);
        let base = Url::parse("https://example.com/").unwrap();
        let pagination = extract_pagination(&doc, Some(&base)).unwrap();

        assert!(!pagination.page_urls.is_empty());
    }
}