kardo_core/parser/

markdown.rs

//! Markdown parsing via `pulldown-cmark`.
//!
//! Extracts headings, links, code blocks, word count, and list count from Markdown content.

use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
use serde::Serialize;

#[derive(Debug, Clone, Serialize)]
pub struct Heading {
    pub level: u8,
    pub text: String,
    /// Slug-style anchor derived from heading text (e.g. "my-heading").
    pub anchor: String,
}

#[derive(Debug, Clone, Serialize)]
pub struct Link {
    pub text: String,
    pub url: String,
    /// `true` if the link points to a relative file path (not http/https, not anchor).
    pub is_internal: bool,
    /// Whether the target file actually exists on disk. Set to `None` by the parser;
    /// resolved later by the integrity checker.
    pub target_exists: Option<bool>,
}

#[derive(Debug, Clone, Serialize)]
pub struct CodeBlock {
    pub language: Option<String>,
    pub line_count: usize,
}

#[derive(Debug, Clone, Serialize)]
pub struct ParsedDocument {
    pub headings: Vec<Heading>,
    pub links: Vec<Link>,
    pub code_blocks: Vec<CodeBlock>,
    pub list_count: usize,
    pub word_count: usize,
    pub has_frontmatter: bool,
    pub frontmatter_keys: Vec<String>,
}

/// Parse Markdown content into structured data.
///
/// Extracts headings (with hierarchy level and anchor), links (internal/external),
/// code blocks (with language and line count), list count, and word count.
/// Frontmatter keys are extracted separately via `parse_frontmatter`.
pub fn parse_markdown(content: &str) -> ParsedDocument {
    let has_frontmatter = content.starts_with("---\n")
        || content.starts_with("---\r\n");

    let frontmatter_keys = if has_frontmatter {
        extract_frontmatter_keys(content)
    } else {
        vec![]
    };

    // Strip frontmatter before parsing markdown
    let md_content = strip_frontmatter(content);

    let mut options = Options::empty();
    options.insert(Options::ENABLE_TABLES);
    options.insert(Options::ENABLE_STRIKETHROUGH);

    let parser = Parser::new_ext(md_content, options);

    let mut headings = Vec::new();
    let mut links = Vec::new();
    let mut code_blocks = Vec::new();
    let mut word_count = 0usize;
    let mut list_count = 0usize;

    let mut in_heading = false;
    let mut current_heading_level: u8 = 0;
    let mut current_heading_text = String::new();

    let mut in_link = false;
    let mut current_link_url = String::new();
    let mut current_link_text = String::new();

    let mut in_code_block = false;
    let mut current_code_lang: Option<String> = None;
    let mut current_code_content = String::new();

    for event in parser {
        match event {
            Event::Start(Tag::Heading { level, .. }) => {
                in_heading = true;
                current_heading_level = level as u8;
                current_heading_text.clear();
            }
            Event::End(TagEnd::Heading(_)) => {
                in_heading = false;
                let text = current_heading_text.trim().to_string();
                let anchor = slugify(&text);
                headings.push(Heading {
                    level: current_heading_level,
                    text,
                    anchor,
                });
                current_heading_text.clear();
            }
            Event::Start(Tag::Link { dest_url, .. }) => {
                in_link = true;
                current_link_url = dest_url.to_string();
                current_link_text.clear();
            }
            Event::End(TagEnd::Link) => {
                in_link = false;
                let is_internal = is_internal_link(&current_link_url);
                links.push(Link {
                    url: current_link_url.clone(),
                    text: current_link_text.trim().to_string(),
                    is_internal,
                    target_exists: None,
                });
                current_link_url.clear();
                current_link_text.clear();
            }
            Event::Start(Tag::List(_)) => {
                list_count += 1;
            }
            Event::Start(Tag::CodeBlock(cb_kind)) => {
                in_code_block = true;
                current_code_content.clear();
                current_code_lang = match cb_kind {
                    pulldown_cmark::CodeBlockKind::Fenced(lang) => {
                        let lang = lang.trim().to_string();
                        if lang.is_empty() { None } else { Some(lang) }
                    }
                    pulldown_cmark::CodeBlockKind::Indented => None,
                };
            }
            Event::End(TagEnd::CodeBlock) => {
                in_code_block = false;
                let line_count = current_code_content.lines().count();
                code_blocks.push(CodeBlock {
                    language: current_code_lang.take(),
                    line_count,
                });
                current_code_content.clear();
            }
            Event::Text(text) => {
                if in_heading {
                    current_heading_text.push_str(&text);
                }
                if in_link {
                    current_link_text.push_str(&text);
                }
                if in_code_block {
                    current_code_content.push_str(&text);
                } else {
                    word_count += count_words(&text);
                }
            }
            Event::Code(code) => {
                word_count += count_words(&code);
                if in_heading {
                    current_heading_text.push_str(&code);
                }
                if in_link {
                    current_link_text.push_str(&code);
                }
            }
            _ => {}
        }
    }

    ParsedDocument {
        headings,
        links,
        code_blocks,
        list_count,
        word_count,
        has_frontmatter,
        frontmatter_keys,
    }
}

/// Determine if a URL is an internal reference.
/// Internal: relative paths (./docs/prd.md, ../README.md) and fragment-only (#heading).
/// External: http://, https://, mailto:.
fn is_internal_link(url: &str) -> bool {
    if url.starts_with("http://")
        || url.starts_with("https://")
        || url.starts_with("mailto:")
    {
        return false;
    }
    true
}

/// Generate a slug-style anchor from heading text.
/// Lowercases, replaces non-alphanumeric with hyphens, trims hyphens.
fn slugify(text: &str) -> String {
    let slug: String = text
        .to_lowercase()
        .chars()
        .map(|c| if c.is_alphanumeric() { c } else { '-' })
        .collect();
    // Collapse multiple hyphens and trim
    let mut result = String::new();
    let mut prev_hyphen = true; // trim leading
    for c in slug.chars() {
        if c == '-' {
            if !prev_hyphen {
                result.push('-');
            }
            prev_hyphen = true;
        } else {
            result.push(c);
            prev_hyphen = false;
        }
    }
    // Trim trailing hyphen
    if result.ends_with('-') {
        result.pop();
    }
    result
}

/// Strip YAML frontmatter (between --- markers) from content.
fn strip_frontmatter(content: &str) -> &str {
    if !content.starts_with("---\n") && !content.starts_with("---\r\n") {
        return content;
    }

    let after_first = if let Some(stripped) = content.strip_prefix("---\r\n") {
        stripped
    } else {
        &content[4..]
    };

    if let Some(end_pos) = after_first.find("\n---") {
        let skip = end_pos + 4; // "\n---"
        let rest = &after_first[skip..];
        if let Some(stripped) = rest.strip_prefix('\n') {
            stripped
        } else if let Some(stripped) = rest.strip_prefix("\r\n") {
            stripped
        } else {
            rest
        }
    } else {
        content
    }
}

/// Extract frontmatter keys (simple YAML key: value lines).
fn extract_frontmatter_keys(content: &str) -> Vec<String> {
    let after_first = if let Some(stripped) = content.strip_prefix("---\r\n") {
        stripped
    } else if let Some(stripped) = content.strip_prefix("---\n") {
        stripped
    } else {
        return vec![];
    };

    let end_pos = match after_first.find("\n---") {
        Some(pos) => pos,
        None => return vec![],
    };

    let fm_str = &after_first[..end_pos];
    let mut keys = Vec::new();

    for line in fm_str.lines() {
        let line = line.trim();
        if line.is_empty() || line.starts_with('#') {
            continue;
        }
        if let Some((key, _)) = line.split_once(':') {
            let key = key.trim();
            if !key.is_empty() {
                keys.push(key.to_string());
            }
        }
    }

    keys
}

/// Count words in a text fragment.
fn count_words(text: &str) -> usize {
    text.split_whitespace().count()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_headings() {
        let md = "# Title\n\n## Section 1\n\n### Subsection\n\n## Section 2\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.headings.len(), 4);
        assert_eq!(doc.headings[0].level, 1);
        assert_eq!(doc.headings[0].text, "Title");
        assert_eq!(doc.headings[0].anchor, "title");
        assert_eq!(doc.headings[1].level, 2);
        assert_eq!(doc.headings[1].text, "Section 1");
        assert_eq!(doc.headings[1].anchor, "section-1");
        assert_eq!(doc.headings[2].level, 3);
        assert_eq!(doc.headings[2].anchor, "subsection");
    }

    #[test]
    fn test_parse_links_external() {
        let md = "Check [Google](https://google.com) and [Docs](http://docs.rs).\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.links.len(), 2);
        assert_eq!(doc.links[0].url, "https://google.com");
        assert_eq!(doc.links[0].text, "Google");
        assert!(!doc.links[0].is_internal);
        assert!(!doc.links[1].is_internal);
        assert!(doc.links[0].target_exists.is_none());
    }

    #[test]
    fn test_parse_links_internal() {
        let md = "See [setup](./docs/setup.md) and [config](../config.yaml).\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.links.len(), 2);
        assert!(doc.links[0].is_internal);
        assert_eq!(doc.links[0].url, "./docs/setup.md");
        assert!(doc.links[1].is_internal);
        assert!(doc.links[0].target_exists.is_none());
    }

    #[test]
    fn test_parse_links_anchor() {
        let md = "Jump to [section](#overview).\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.links.len(), 1);
        assert!(doc.links[0].is_internal); // fragment-only is internal
        assert_eq!(doc.links[0].url, "#overview");
    }

    #[test]
    fn test_parse_code_blocks() {
        let md = "# Demo\n\n```rust\nfn main() {\n    println!(\"hi\");\n}\n```\n\n```\nplain\n```\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.code_blocks.len(), 2);
        assert_eq!(doc.code_blocks[0].language.as_deref(), Some("rust"));
        assert_eq!(doc.code_blocks[0].line_count, 3);
        assert!(doc.code_blocks[1].language.is_none());
        assert_eq!(doc.code_blocks[1].line_count, 1);
    }

    #[test]
    fn test_word_count() {
        let md = "Hello world. This is a test.\n\nAnother paragraph here.\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.word_count, 9);
    }

    #[test]
    fn test_word_count_excludes_code_blocks() {
        let md = "One two three.\n\n```rust\nfn main() {}\n```\n\nFour five.\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.word_count, 5);
    }

    #[test]
    fn test_list_count() {
        let md = "# Lists\n\n- item a\n- item b\n\n1. first\n2. second\n\nParagraph.\n\n- another list\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.list_count, 3);
    }

    #[test]
    fn test_frontmatter_detected() {
        let md = "---\ntitle: Hello\nauthor: Alice\n---\n\n# Hello\n";
        let doc = parse_markdown(md);
        assert!(doc.has_frontmatter);
        assert_eq!(doc.frontmatter_keys, vec!["title", "author"]);
        assert_eq!(doc.headings.len(), 1);
        assert_eq!(doc.headings[0].text, "Hello");
    }

    #[test]
    fn test_no_frontmatter() {
        let md = "# Just a heading\n\nSome text.\n";
        let doc = parse_markdown(md);
        assert!(!doc.has_frontmatter);
        assert!(doc.frontmatter_keys.is_empty());
    }

    #[test]
    fn test_empty_document() {
        let doc = parse_markdown("");
        assert!(doc.headings.is_empty());
        assert!(doc.links.is_empty());
        assert!(doc.code_blocks.is_empty());
        assert_eq!(doc.word_count, 0);
        assert_eq!(doc.list_count, 0);
        assert!(!doc.has_frontmatter);
        assert!(doc.frontmatter_keys.is_empty());
    }

    #[test]
    fn test_mixed_link_types() {
        let md = "[ext](https://example.com) [int](./file.md) [anc](#top)\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.links.len(), 3);
        assert!(!doc.links[0].is_internal); // external
        assert!(doc.links[1].is_internal);  // relative path
        assert!(doc.links[2].is_internal);  // fragment-only anchor
    }

    #[test]
    fn test_slugify() {
        assert_eq!(slugify("Hello World"), "hello-world");
        assert_eq!(slugify("API Reference (v2)"), "api-reference-v2");
        assert_eq!(slugify("  Leading Spaces  "), "leading-spaces");
    }

    #[test]
    fn test_internal_reference_detection() {
        let md = "See [PRD](./docs/prd.md) and [README](../README.md) for details.\n";
        let doc = parse_markdown(md);
        assert_eq!(doc.links.len(), 2);
        assert!(doc.links[0].is_internal);
        assert_eq!(doc.links[0].url, "./docs/prd.md");
        assert!(doc.links[1].is_internal);
        assert_eq!(doc.links[1].url, "../README.md");
    }
}