halldyll-parser 0.1.0

//! Content extraction for halldyll-parser
//!
//! This module handles extraction of structured content:
//! - Headings (h1-h6)
//! - Paragraphs
//! - Lists (ordered, unordered, definition)
//! - Tables
//! - Code blocks
//! - Blockquotes
//! - Images

use scraper::{Html, ElementRef};
use url::Url;

use crate::selector::{SELECTORS, try_parse_selector, heading_selector};
use crate::types::{
    Heading, Image, ImageLoading, ListContent, ListType, ListItem,
    TableContent, TableRow, TableCell, CodeBlock, Quote,
    ParserConfig, ParserResult,
};

// ============================================================================
// HEADINGS
// ============================================================================

/// Extract all headings from the document
pub fn extract_headings(document: &Html) -> ParserResult<Vec<Heading>> {
    let mut headings = Vec::new();
    
    for level in 1..=6 {
        let selector = heading_selector(level);
        
        for element in document.select(selector) {
            let text = element.text().collect::<String>().trim().to_string();
            
            if text.is_empty() {
                continue;
            }
            
            let mut heading = Heading::new(level, &text);
            
            // Get ID if present
            if let Some(id) = element.value().attr("id") {
                heading.id = Some(id.to_string());
            }
            
            // Get classes
            heading.classes = element.value().classes()
                .map(|c| c.to_string())
                .collect();
            
            headings.push(heading);
        }
    }
    
    Ok(headings)
}

/// Get the main heading (first h1)
pub fn get_main_heading(document: &Html) -> Option<String> {
    document.select(&SELECTORS.h1)
        .next()
        .map(|el| el.text().collect::<String>().trim().to_string())
        .filter(|s| !s.is_empty())
}

/// Build document outline from headings
pub fn build_outline(headings: &[Heading]) -> Vec<OutlineItem> {
    let mut outline = Vec::new();
    let mut stack: Vec<(u8, usize)> = Vec::new(); // (level, index)
    
    for heading in headings {
        let item = OutlineItem {
            level: heading.level,
            text: heading.text.clone(),
            id: heading.id.clone(),
            children: Vec::new(),
        };
        
        // Pop items with same or higher level
        while let Some((level, _)) = stack.last() {
            if *level >= heading.level {
                stack.pop();
            } else {
                break;
            }
        }
        
        outline.push(item);
        stack.push((heading.level, outline.len() - 1));
    }
    
    outline
}

/// Document outline item
#[derive(Debug, Clone)]
pub struct OutlineItem {
    pub level: u8,
    pub text: String,
    pub id: Option<String>,
    pub children: Vec<OutlineItem>,
}

// ============================================================================
// PARAGRAPHS
// ============================================================================

/// Extract all paragraphs from the document
pub fn extract_paragraphs(document: &Html, config: &ParserConfig) -> ParserResult<Vec<String>> {
    let mut paragraphs = Vec::new();
    
    for element in document.select(&SELECTORS.p) {
        let text = element.text().collect::<String>().trim().to_string();
        
        // Filter by minimum length
        if text.len() >= config.min_paragraph_length {
            paragraphs.push(text);
        }
    }
    
    Ok(paragraphs)
}

// ============================================================================
// LISTS
// ============================================================================

/// Extract all lists from the document
pub fn extract_lists(document: &Html) -> ParserResult<Vec<ListContent>> {
    let mut lists = Vec::new();
    
    // Ordered lists
    for ol in document.select(&SELECTORS.ol) {
        if let Some(list) = extract_list(&ol, ListType::Ordered) {
            lists.push(list);
        }
    }
    
    // Unordered lists
    for ul in document.select(&SELECTORS.ul) {
        if let Some(list) = extract_list(&ul, ListType::Unordered) {
            lists.push(list);
        }
    }
    
    // Definition lists
    for dl in document.select(&SELECTORS.dl) {
        if let Some(list) = extract_definition_list(&dl) {
            lists.push(list);
        }
    }
    
    Ok(lists)
}

/// Extract a single list
fn extract_list(element: &ElementRef, list_type: ListType) -> Option<ListContent> {
    let mut list = ListContent::new(list_type);
    
    // Only process direct li children
    for child in element.children() {
        if let Some(li) = ElementRef::wrap(child) {
            if li.value().name() == "li" {
                let item = extract_list_item(&li);
                list.add_item(item);
            }
        }
    }
    
    if list.is_empty() {
        None
    } else {
        Some(list)
    }
}

/// Extract a list item (with potential nested list)
fn extract_list_item(element: &ElementRef) -> ListItem {
    // Get text content (excluding nested lists)
    let mut text = String::new();
    let mut nested: Option<ListContent> = None;
    
    for child in element.children() {
        match child.value() {
            scraper::Node::Text(t) => {
                text.push_str(t.text.trim());
            }
            scraper::Node::Element(el) => {
                if let Some(child_el) = ElementRef::wrap(child) {
                    match el.name() {
                        "ul" => {
                            nested = extract_list(&child_el, ListType::Unordered);
                        }
                        "ol" => {
                            nested = extract_list(&child_el, ListType::Ordered);
                        }
                        _ => {
                            // Get text from inline elements
                            text.push_str(&child_el.text().collect::<String>());
                        }
                    }
                }
            }
            _ => {}
        }
    }
    
    if let Some(nested_list) = nested {
        ListItem::with_nested(text.trim(), nested_list)
    } else {
        ListItem::new(text.trim())
    }
}

/// Extract a definition list
fn extract_definition_list(element: &ElementRef) -> Option<ListContent> {
    let mut list = ListContent::new(ListType::Definition);
    
    let mut current_term: Option<String> = None;
    
    for child in element.children() {
        if let Some(el) = ElementRef::wrap(child) {
            match el.value().name() {
                "dt" => {
                    current_term = Some(el.text().collect::<String>().trim().to_string());
                }
                "dd" => {
                    let definition = el.text().collect::<String>().trim().to_string();
                    let item_text = if let Some(term) = current_term.take() {
                        format!("{}: {}", term, definition)
                    } else {
                        definition
                    };
                    list.add_item(ListItem::new(item_text));
                }
                _ => {}
            }
        }
    }
    
    if list.is_empty() {
        None
    } else {
        Some(list)
    }
}

// ============================================================================
// TABLES
// ============================================================================

/// Extract all tables from the document
pub fn extract_tables(document: &Html) -> ParserResult<Vec<TableContent>> {
    let mut tables = Vec::new();
    
    for table_el in document.select(&SELECTORS.table) {
        if let Some(table) = extract_table(&table_el) {
            tables.push(table);
        }
    }
    
    Ok(tables)
}

/// Extract a single table
fn extract_table(element: &ElementRef) -> Option<TableContent> {
    let mut table = TableContent::new();
    
    // Caption
    if let Some(caption) = element.select(&SELECTORS.caption).next() {
        table.caption = Some(caption.text().collect::<String>().trim().to_string());
    }
    
    // Summary attribute
    table.summary = element.value().attr("summary").map(|s| s.to_string());
    
    // Headers (from thead or th elements)
    if let Some(thead) = element.select(&SELECTORS.thead).next() {
        for tr in thead.select(&SELECTORS.tr) {
            let row = extract_table_row(&tr, true);
            if !row.cells.is_empty() {
                table.headers.push(row);
            }
        }
    } else {
        // Look for th in first row
        if let Some(first_tr) = element.select(&SELECTORS.tr).next() {
            let cells: Vec<_> = first_tr.select(&SELECTORS.th).collect();
            if !cells.is_empty() {
                let row = extract_table_row(&first_tr, true);
                table.headers.push(row);
            }
        }
    }
    
    // Body rows
    let tbody_selector = &SELECTORS.tbody;
    let rows_to_process: Vec<ElementRef> = if let Some(tbody) = element.select(tbody_selector).next() {
        tbody.select(&SELECTORS.tr).collect()
    } else {
        // Skip header row if we extracted it
        let all_rows: Vec<_> = element.select(&SELECTORS.tr).collect();
        if !table.headers.is_empty() && !all_rows.is_empty() {
            all_rows.into_iter().skip(1).collect()
        } else {
            all_rows
        }
    };
    
    for tr in rows_to_process {
        let row = extract_table_row(&tr, false);
        if !row.cells.is_empty() {
            // Update column count
            if row.cells.len() > table.column_count {
                table.column_count = row.cells.len();
            }
            table.rows.push(row);
        }
    }
    
    if table.is_empty() {
        None
    } else {
        Some(table)
    }
}

/// Extract a table row
fn extract_table_row(element: &ElementRef, is_header: bool) -> TableRow {
    let mut cells = Vec::new();
    
    // Get both th and td cells
    for child in element.children() {
        if let Some(cell_el) = ElementRef::wrap(child) {
            let tag = cell_el.value().name();
            if tag == "th" || tag == "td" {
                let cell = extract_table_cell(&cell_el, tag == "th");
                cells.push(cell);
            }
        }
    }
    
    TableRow {
        cells,
        is_header_row: is_header,
    }
}

/// Extract a table cell
fn extract_table_cell(element: &ElementRef, is_header: bool) -> TableCell {
    let content = element.text().collect::<String>().trim().to_string();
    
    let colspan = element.value().attr("colspan")
        .and_then(|s| s.parse().ok())
        .unwrap_or(1);
    
    let rowspan = element.value().attr("rowspan")
        .and_then(|s| s.parse().ok())
        .unwrap_or(1);
    
    TableCell {
        content,
        is_header,
        colspan,
        rowspan,
    }
}

// ============================================================================
// CODE BLOCKS
// ============================================================================

/// Extract all code blocks from the document
pub fn extract_code_blocks(document: &Html) -> ParserResult<Vec<CodeBlock>> {
    let mut code_blocks = Vec::new();
    let mut seen_codes: std::collections::HashSet<String> = std::collections::HashSet::new();
    
    // Pre > code (most common)
    for pre in document.select(&SELECTORS.pre) {
        let code = if let Some(code_el) = pre.select(&SELECTORS.code).next() {
            extract_code_block(&code_el, false)
        } else {
            extract_code_block(&pre, false)
        };
        
        // Deduplicate
        if !code.code.trim().is_empty() && !seen_codes.contains(&code.code) {
            seen_codes.insert(code.code.clone());
            code_blocks.push(code);
        }
    }
    
    // Standalone code elements (inline code)
    for code_el in document.select(&SELECTORS.code) {
        // Skip if inside pre (already handled)
        let in_pre = code_el.ancestors()
            .any(|ancestor| {
                ancestor.value().as_element()
                    .map(|e| e.name() == "pre")
                    .unwrap_or(false)
            });
        
        if !in_pre {
            let code = extract_code_block(&code_el, true);
            if !code.code.trim().is_empty() && !seen_codes.contains(&code.code) {
                seen_codes.insert(code.code.clone());
                code_blocks.push(code);
            }
        }
    }
    
    Ok(code_blocks)
}

/// Extract a single code block
fn extract_code_block(element: &ElementRef, is_inline: bool) -> CodeBlock {
    let code = element.text().collect::<String>();
    
    // Detect language from class
    let language = element.value().classes()
        .find(|c| {
            c.starts_with("language-") || 
            c.starts_with("lang-") || 
            c.starts_with("hljs-") ||
            is_known_language(c)
        })
        .map(|c| {
            c.trim_start_matches("language-")
             .trim_start_matches("lang-")
             .trim_start_matches("hljs-")
             .to_string()
        });
    
    // Check for data-language attribute
    let language = language.or_else(|| {
        element.value().attr("data-language")
            .or_else(|| element.value().attr("data-lang"))
            .map(|s| s.to_string())
    });
    
    let mut block = CodeBlock::new(&code);
    block.language = language;
    block.is_inline = is_inline;
    
    block
}

/// Check if a class name is a known programming language
fn is_known_language(class: &str) -> bool {
    let known = [
        "rust", "python", "javascript", "typescript", "java", "c", "cpp", 
        "csharp", "go", "ruby", "php", "swift", "kotlin", "scala", "html",
        "css", "sql", "bash", "shell", "json", "yaml", "xml", "markdown",
    ];
    known.contains(&class.to_lowercase().as_str())
}

// ============================================================================
// QUOTES
// ============================================================================

/// Extract all blockquotes from the document
pub fn extract_quotes(document: &Html) -> ParserResult<Vec<Quote>> {
    let mut quotes = Vec::new();
    
    for blockquote in document.select(&SELECTORS.blockquote) {
        let text = blockquote.text().collect::<String>().trim().to_string();
        
        if text.is_empty() {
            continue;
        }
        
        let mut quote = Quote::new(&text);
        
        // Look for cite attribute
        quote.cite_url = blockquote.value().attr("cite").map(|s| s.to_string());
        
        // Look for footer/cite element for attribution
        if let Some(sel) = try_parse_selector("footer, cite") {
            if let Some(cite_el) = blockquote.select(&sel).next() {
                quote.cite = Some(cite_el.text().collect::<String>().trim().to_string());
            }
        }
        
        quotes.push(quote);
    }
    
    Ok(quotes)
}

// ============================================================================
// IMAGES
// ============================================================================

/// Extract all images from the document
pub fn extract_images(document: &Html, base_url: Option<&Url>) -> ParserResult<Vec<Image>> {
    let mut images = Vec::new();
    
    for img in document.select(&SELECTORS.img) {
        if let Some(image) = extract_image(&img, base_url) {
            images.push(image);
        }
    }
    
    Ok(images)
}

/// Extract a single image
fn extract_image(element: &ElementRef, base_url: Option<&Url>) -> Option<Image> {
    let src = element.value().attr("src")
        .or_else(|| element.value().attr("data-src"))
        .or_else(|| element.value().attr("data-lazy-src"))?;
    
    let alt = element.value().attr("alt").unwrap_or("").to_string();
    
    let mut image = Image::new(src, &alt);
    
    // Resolve URL
    image.url = resolve_image_url(src, base_url);
    
    // Get dimensions
    image.width = element.value().attr("width")
        .and_then(|s| s.trim_end_matches("px").parse().ok());
    image.height = element.value().attr("height")
        .and_then(|s| s.trim_end_matches("px").parse().ok());
    
    // Responsive images
    image.srcset = element.value().attr("srcset").map(|s| s.to_string());
    image.sizes = element.value().attr("sizes").map(|s| s.to_string());
    
    // Loading attribute
    image.loading = match element.value().attr("loading") {
        Some("lazy") => ImageLoading::Lazy,
        _ => ImageLoading::Eager,
    };
    
    // Title
    image.title = element.value().attr("title").map(|s| s.to_string());
    
    Some(image)
}

/// Resolve image URL
fn resolve_image_url(src: &str, base_url: Option<&Url>) -> Option<String> {
    let trimmed = src.trim();
    
    if trimmed.is_empty() || trimmed.starts_with("data:") {
        return None;
    }
    
    if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
        return Some(trimmed.to_string());
    }
    
    if trimmed.starts_with("//") {
        return Some(format!("https:{}", trimmed));
    }
    
    base_url
        .and_then(|base| base.join(trimmed).ok())
        .map(|u| u.to_string())
}

// ============================================================================
// TESTS
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    fn parse_html(html: &str) -> Html {
        Html::parse_document(html)
    }

    #[test]
    fn test_extract_headings() {
        let doc = parse_html(r#"
            <html><body>
                <h1 id="main">Main Title</h1>
                <h2>Section 1</h2>
                <h2>Section 2</h2>
                <h3>Subsection</h3>
            </body></html>
        "#);
        
        let headings = extract_headings(&doc).unwrap();
        assert_eq!(headings.len(), 4);
        assert_eq!(headings[0].level, 1);
        assert_eq!(headings[0].text, "Main Title");
        assert_eq!(headings[0].id, Some("main".to_string()));
    }

    #[test]
    fn test_get_main_heading() {
        let doc = parse_html("<html><body><h1>Main Title</h1></body></html>");
        assert_eq!(get_main_heading(&doc), Some("Main Title".to_string()));
    }

    #[test]
    fn test_extract_paragraphs() {
        let doc = parse_html(r#"
            <html><body>
                <p>This is a long enough paragraph to be included.</p>
                <p>Short</p>
                <p>Another paragraph that should be extracted.</p>
            </body></html>
        "#);
        
        let config = ParserConfig::default();
        let paragraphs = extract_paragraphs(&doc, &config).unwrap();
        assert_eq!(paragraphs.len(), 2);
    }

    #[test]
    fn test_extract_ordered_list() {
        let doc = parse_html(r#"
            <ol>
                <li>First item</li>
                <li>Second item</li>
                <li>Third item</li>
            </ol>
        "#);
        
        let lists = extract_lists(&doc).unwrap();
        assert_eq!(lists.len(), 1);
        assert_eq!(lists[0].list_type, ListType::Ordered);
        assert_eq!(lists[0].items.len(), 3);
    }

    #[test]
    fn test_extract_nested_list() {
        let doc = parse_html(r#"
            <ul>
                <li>Item 1
                    <ul>
                        <li>Nested 1</li>
                        <li>Nested 2</li>
                    </ul>
                </li>
                <li>Item 2</li>
            </ul>
        "#);
        
        let lists = extract_lists(&doc).unwrap();
        assert!(!lists.is_empty());
        // First item should have nested list
        assert!(lists[0].items[0].nested.is_some());
    }

    #[test]
    fn test_extract_table() {
        let doc = parse_html(r#"
            <table>
                <caption>Test Table</caption>
                <thead>
                    <tr><th>Header 1</th><th>Header 2</th></tr>
                </thead>
                <tbody>
                    <tr><td>Cell 1</td><td>Cell 2</td></tr>
                    <tr><td>Cell 3</td><td>Cell 4</td></tr>
                </tbody>
            </table>
        "#);
        
        let tables = extract_tables(&doc).unwrap();
        assert_eq!(tables.len(), 1);
        assert_eq!(tables[0].caption, Some("Test Table".to_string()));
        assert_eq!(tables[0].headers.len(), 1);
        assert_eq!(tables[0].rows.len(), 2);
        assert_eq!(tables[0].column_count, 2);
    }

    #[test]
    fn test_extract_code_block() {
        let doc = parse_html(r#"
            <pre><code class="language-rust">
                fn main() {
                    println!("Hello");
                }
            </code></pre>
        "#);
        
        let code_blocks = extract_code_blocks(&doc).unwrap();
        assert_eq!(code_blocks.len(), 1);
        assert_eq!(code_blocks[0].language, Some("rust".to_string()));
        assert!(!code_blocks[0].is_inline);
    }

    #[test]
    fn test_extract_inline_code() {
        let doc = parse_html(r#"<p>Use the <code>println!</code> macro.</p>"#);
        
        let code_blocks = extract_code_blocks(&doc).unwrap();
        assert_eq!(code_blocks.len(), 1);
        assert!(code_blocks[0].is_inline);
    }

    #[test]
    fn test_extract_quotes() {
        let doc = parse_html(r#"
            <blockquote cite="https://example.com">
                <p>This is a quote.</p>
                <footer>— Author Name</footer>
            </blockquote>
        "#);
        
        let quotes = extract_quotes(&doc).unwrap();
        assert_eq!(quotes.len(), 1);
        assert!(quotes[0].text.contains("This is a quote"));
        assert_eq!(quotes[0].cite_url, Some("https://example.com".to_string()));
    }

    #[test]
    fn test_extract_images() {
        let doc = parse_html(r#"
            <img src="/images/photo.jpg" 
                 alt="A photo" 
                 title="Photo title"
                 width="800" 
                 height="600"
                 loading="lazy">
        "#);
        
        let base = Url::parse("https://example.com").unwrap();
        let images = extract_images(&doc, Some(&base)).unwrap();
        
        assert_eq!(images.len(), 1);
        assert_eq!(images[0].alt, "A photo");
        assert_eq!(images[0].title, Some("Photo title".to_string()));
        assert_eq!(images[0].width, Some(800));
        assert_eq!(images[0].height, Some(600));
        assert_eq!(images[0].loading, ImageLoading::Lazy);
        assert_eq!(images[0].url, Some("https://example.com/images/photo.jpg".to_string()));
    }

    #[test]
    fn test_image_decorative() {
        let doc = parse_html(r#"<img src="/spacer.gif" alt="">"#);
        let images = extract_images(&doc, None).unwrap();
        assert!(images[0].is_decorative);
    }

    #[test]
    fn test_table_with_colspan() {
        let doc = parse_html(r#"
            <table>
                <tr><td colspan="2">Spanning cell</td></tr>
                <tr><td>Cell 1</td><td>Cell 2</td></tr>
            </table>
        "#);
        
        let tables = extract_tables(&doc).unwrap();
        assert_eq!(tables[0].rows[0].cells[0].colspan, 2);
    }

    #[test]
    fn test_definition_list() {
        let doc = parse_html(r#"
            <dl>
                <dt>Term 1</dt>
                <dd>Definition 1</dd>
                <dt>Term 2</dt>
                <dd>Definition 2</dd>
            </dl>
        "#);
        
        let lists = extract_lists(&doc).unwrap();
        assert_eq!(lists.len(), 1);
        assert_eq!(lists[0].list_type, ListType::Definition);
        assert_eq!(lists[0].items.len(), 2);
    }

    #[test]
    fn test_build_outline() {
        let headings = vec![
            Heading::new(1, "Main"),
            Heading::new(2, "Section 1"),
            Heading::new(3, "Subsection 1.1"),
            Heading::new(2, "Section 2"),
        ];
        
        let outline = build_outline(&headings);
        assert_eq!(outline.len(), 4);
    }

    #[test]
    fn test_is_known_language() {
        assert!(is_known_language("rust"));
        assert!(is_known_language("Python"));
        assert!(is_known_language("JAVASCRIPT"));
        assert!(!is_known_language("unknown-lang"));
    }

    #[test]
    fn test_responsive_image() {
        let doc = parse_html(r#"
            <img src="/img.jpg" 
                 srcset="/img-320.jpg 320w, /img-640.jpg 640w"
                 sizes="(max-width: 320px) 280px, 640px"
                 alt="Responsive">
        "#);
        
        let images = extract_images(&doc, None).unwrap();
        assert!(images[0].is_responsive());
        assert!(images[0].srcset.is_some());
        assert!(images[0].sizes.is_some());
    }
}