quillmark_core/

parse.rs

//! # Parsing Module
//!
//! Parsing functionality for markdown documents with YAML frontmatter.
//!
//! ## Overview
//!
//! The `parse` module provides the [`decompose`] function for parsing markdown documents
//! and the [`ParsedDocument`] type for accessing parsed content.
//!
//! ## Key Types
//!
//! - [`ParsedDocument`]: Container for parsed frontmatter fields and body content
//! - [`BODY_FIELD`]: Constant for the field name storing document body
//!
//! ## Examples
//!
//! ### Basic Parsing
//!
//! ```
//! use quillmark_core::decompose;
//!
//! let markdown = r#"---
//! title: My Document
//! author: John Doe
//! ---
//!
//! # Introduction
//!
//! Document content here.
//! "#;
//!
//! let doc = decompose(markdown).unwrap();
//! let title = doc.get_field("title")
//!     .and_then(|v| v.as_str())
//!     .unwrap_or("Untitled");
//! ```
//!
//! ### Extended Metadata with Tags
//!
//! ```
//! use quillmark_core::decompose;
//!
//! let markdown = r#"---
//! catalog_title: Product Catalog
//! ---
//!
//! # Products
//!
//! ---
//! !products
//! name: Widget
//! price: 19.99
//! ---
//!
//! A versatile widget for all occasions.
//! "#;
//!
//! let doc = decompose(markdown).unwrap();
//!
//! // Access tagged collections
//! if let Some(products) = doc.get_field("products")
//!     .and_then(|v| v.as_sequence())
//! {
//!     for product in products {
//!         let name = product.get("name").and_then(|v| v.as_str()).unwrap();
//!         let price = product.get("price").and_then(|v| v.as_f64()).unwrap();
//!         println!("{}: ${}", name, price);
//!     }
//! }
//! ```
//!
//! ## Error Handling
//!
//! The [`decompose`] function returns errors for:
//! - Malformed YAML syntax
//! - Unclosed frontmatter blocks
//! - Multiple global frontmatter blocks
//! - Invalid tag directive syntax
//! - Reserved field name usage
//! - Name collisions
//!
//! See [PARSE.md](https://github.com/nibsbin/quillmark/blob/main/quillmark-core/docs/designs/PARSE.md) for comprehensive documentation of the Extended YAML Metadata Standard.

use std::collections::HashMap;

/// The field name used to store the document body
pub const BODY_FIELD: &str = "body";

/// A parsed markdown document with frontmatter
#[derive(Debug, Clone)]
pub struct ParsedDocument {
    fields: HashMap<String, serde_yaml::Value>,
}

impl ParsedDocument {
    /// Create a new ParsedDocument with the given fields
    pub fn new(fields: HashMap<String, serde_yaml::Value>) -> Self {
        Self { fields }
    }

    /// Get the document body
    pub fn body(&self) -> Option<&str> {
        self.fields.get(BODY_FIELD).and_then(|v| v.as_str())
    }

    /// Get a specific field
    pub fn get_field(&self, name: &str) -> Option<&serde_yaml::Value> {
        self.fields.get(name)
    }

    /// Get all fields (including body)
    pub fn fields(&self) -> &HashMap<String, serde_yaml::Value> {
        &self.fields
    }
}

#[derive(Debug)]
struct MetadataBlock {
    start: usize, // Position of opening "---"
    end: usize,   // Position after closing "---\n"
    yaml_content: String,
    tag: Option<String>, // Tag directive if present
}

/// Validate tag name follows pattern [a-z_][a-z0-9_]*
fn is_valid_tag_name(name: &str) -> bool {
    if name.is_empty() {
        return false;
    }

    let mut chars = name.chars();
    let first = chars.next().unwrap();

    if !first.is_ascii_lowercase() && first != '_' {
        return false;
    }

    for ch in chars {
        if !ch.is_ascii_lowercase() && !ch.is_ascii_digit() && ch != '_' {
            return false;
        }
    }

    true
}

/// Find all metadata blocks in the document
fn find_metadata_blocks(
    markdown: &str,
) -> Result<Vec<MetadataBlock>, Box<dyn std::error::Error + Send + Sync>> {
    let mut blocks = Vec::new();
    let mut pos = 0;

    while pos < markdown.len() {
        // Look for opening "---\n" or "---\r\n"
        let search_str = &markdown[pos..];
        let delimiter_result = if let Some(p) = search_str.find("---\n") {
            Some((p, 4, "\n"))
        } else if let Some(p) = search_str.find("---\r\n") {
            Some((p, 5, "\r\n"))
        } else {
            None
        };

        if let Some((delimiter_pos, delimiter_len, _line_ending)) = delimiter_result {
            let abs_pos = pos + delimiter_pos;
            let content_start = abs_pos + delimiter_len; // After "---\n" or "---\r\n"

            // Check if opening --- is followed by a blank line (horizontal rule, not metadata)
            let followed_by_blank = if content_start < markdown.len() {
                markdown[content_start..].starts_with('\n')
                    || markdown[content_start..].starts_with("\r\n")
            } else {
                false
            };

            if followed_by_blank {
                // This is a horizontal rule in the body, skip it
                pos = abs_pos + 3; // Skip past "---"
                continue;
            }

            // Found potential metadata block opening
            // Look for closing "\n---\n" or "\r\n---\r\n" etc., OR "\n---" / "\r\n---" at end of document
            let rest = &markdown[content_start..];

            // First try to find delimiters with trailing newlines
            let closing_patterns = ["\n---\n", "\r\n---\r\n", "\n---\r\n", "\r\n---\n"];
            let closing_with_newline = closing_patterns
                .iter()
                .filter_map(|delim| rest.find(delim).map(|p| (p, delim.len())))
                .min_by_key(|(p, _)| *p);

            // Also check for closing at end of document (no trailing newline)
            let closing_at_eof = ["\n---", "\r\n---"]
                .iter()
                .filter_map(|delim| {
                    rest.find(delim).and_then(|p| {
                        if p + delim.len() == rest.len() {
                            Some((p, delim.len()))
                        } else {
                            None
                        }
                    })
                })
                .min_by_key(|(p, _)| *p);

            let closing_result = match (closing_with_newline, closing_at_eof) {
                (Some((p1, _l1)), Some((p2, _))) if p2 < p1 => closing_at_eof,
                (Some(_), Some(_)) => closing_with_newline,
                (Some(_), None) => closing_with_newline,
                (None, Some(_)) => closing_at_eof,
                (None, None) => None,
            };

            if let Some((closing_pos, closing_len)) = closing_result {
                let abs_closing_pos = content_start + closing_pos;
                let content = &markdown[content_start..abs_closing_pos];

                // Check if the block is contiguous (no blank lines in the YAML content)
                if content.contains("\n\n") || content.contains("\r\n\r\n") {
                    // Not a contiguous block
                    if abs_pos == 0 {
                        // Started at beginning but has blank lines - this is an error
                        return Err("Frontmatter started but not closed with ---".into());
                    }
                    // Otherwise treat as horizontal rule in body
                    pos = abs_pos + 3;
                    continue;
                }

                // Extract tag directive if present
                let (tag, yaml_content) = if content.starts_with('!') {
                    if let Some(newline_pos) = content.find(|c| c == '\n' || c == '\r') {
                        let tag_line = &content[1..newline_pos];
                        // Skip newline(s) after tag
                        let yaml_start = if content[newline_pos..].starts_with("\r\n") {
                            newline_pos + 2
                        } else {
                            newline_pos + 1
                        };
                        let yaml = if yaml_start < content.len() {
                            &content[yaml_start..]
                        } else {
                            ""
                        };
                        (Some(tag_line.trim().to_string()), yaml.to_string())
                    } else {
                        // Tag directive with no YAML content (entire content is just tag)
                        (Some(content[1..].trim().to_string()), String::new())
                    }
                } else {
                    (None, content.to_string())
                };

                // Validate tag name if present
                if let Some(ref tag_name) = tag {
                    if !is_valid_tag_name(tag_name) {
                        return Err(format!(
                            "Invalid tag name '{}': must match pattern [a-z_][a-z0-9_]*",
                            tag_name
                        )
                        .into());
                    }
                    if tag_name == BODY_FIELD {
                        return Err(format!(
                            "Cannot use reserved field name '{}' as tag directive",
                            BODY_FIELD
                        )
                        .into());
                    }
                }

                blocks.push(MetadataBlock {
                    start: abs_pos,
                    end: abs_closing_pos + closing_len, // After closing delimiter
                    yaml_content,
                    tag,
                });

                pos = abs_closing_pos + closing_len;
            } else if abs_pos == 0 {
                // Frontmatter started but not closed
                return Err("Frontmatter started but not closed with ---".into());
            } else {
                // Not a valid metadata block, skip this position
                pos = abs_pos + 3;
            }
        } else {
            break;
        }
    }

    Ok(blocks)
}

/// Decompose markdown into frontmatter fields and body
pub fn decompose(
    markdown: &str,
) -> Result<ParsedDocument, Box<dyn std::error::Error + Send + Sync>> {
    let mut fields = HashMap::new();

    // Find all metadata blocks
    let blocks = find_metadata_blocks(markdown)?;

    if blocks.is_empty() {
        // No metadata blocks, entire content is body
        fields.insert(
            BODY_FIELD.to_string(),
            serde_yaml::Value::String(markdown.to_string()),
        );
        return Ok(ParsedDocument::new(fields));
    }

    // Track which attributes are used for tagged blocks
    let mut tagged_attributes: HashMap<String, Vec<serde_yaml::Value>> = HashMap::new();
    let mut has_global_frontmatter = false;
    let mut global_frontmatter_index: Option<usize> = None;

    // First pass: identify global frontmatter and validate
    for (idx, block) in blocks.iter().enumerate() {
        if block.tag.is_none() {
            if has_global_frontmatter {
                return Err(
                    "Multiple global frontmatter blocks found: only one untagged block allowed"
                        .into(),
                );
            }
            has_global_frontmatter = true;
            global_frontmatter_index = Some(idx);
        }
    }

    // Parse global frontmatter if present
    if let Some(idx) = global_frontmatter_index {
        let block = &blocks[idx];

        // Parse YAML frontmatter
        let yaml_fields: HashMap<String, serde_yaml::Value> = if block.yaml_content.is_empty() {
            HashMap::new()
        } else {
            serde_yaml::from_str(&block.yaml_content)
                .map_err(|e| format!("Invalid YAML frontmatter: {}", e))?
        };

        // Check that all tagged blocks don't conflict with global fields
        for other_block in &blocks {
            if let Some(ref tag) = other_block.tag {
                if yaml_fields.contains_key(tag) {
                    return Err(format!(
                        "Name collision: global field '{}' conflicts with tagged attribute",
                        tag
                    )
                    .into());
                }
            }
        }

        fields.extend(yaml_fields);
    }

    // Parse tagged blocks
    for (idx, block) in blocks.iter().enumerate() {
        if let Some(ref tag_name) = block.tag {
            // Check if this conflicts with global fields
            if fields.contains_key(tag_name) {
                return Err(format!(
                    "Name collision: tagged attribute '{}' conflicts with global field",
                    tag_name
                )
                .into());
            }

            // Parse YAML metadata
            let mut item_fields: HashMap<String, serde_yaml::Value> =
                if block.yaml_content.is_empty() {
                    HashMap::new()
                } else {
                    serde_yaml::from_str(&block.yaml_content).map_err(|e| {
                        format!("Invalid YAML in tagged block '{}': {}", tag_name, e)
                    })?
                };

            // Extract body for this tagged block
            let body_start = block.end;
            let body_end = if idx + 1 < blocks.len() {
                blocks[idx + 1].start
            } else {
                markdown.len()
            };
            let body = &markdown[body_start..body_end];

            // Add body to item fields
            item_fields.insert(
                BODY_FIELD.to_string(),
                serde_yaml::Value::String(body.to_string()),
            );

            // Convert HashMap to serde_yaml::Value::Mapping
            let item_value = serde_yaml::to_value(item_fields)?;

            // Add to collection
            tagged_attributes
                .entry(tag_name.clone())
                .or_insert_with(Vec::new)
                .push(item_value);
        }
    }

    // Extract global body
    let (body_start, body_end) = if let Some(idx) = global_frontmatter_index {
        // Global body starts after frontmatter
        let start = blocks[idx].end;

        // Global body ends at the first tagged block after the frontmatter, or EOF
        let end = blocks
            .iter()
            .skip(idx + 1)
            .find(|b| b.tag.is_some())
            .map(|b| b.start)
            .unwrap_or(markdown.len());

        (start, end)
    } else {
        // No global frontmatter - body is everything before the first tagged block
        let end = blocks
            .iter()
            .find(|b| b.tag.is_some())
            .map(|b| b.start)
            .unwrap_or(0);

        (0, end)
    };

    let global_body = &markdown[body_start..body_end];

    fields.insert(
        BODY_FIELD.to_string(),
        serde_yaml::Value::String(global_body.to_string()),
    );

    // Add all tagged collections to fields
    for (tag_name, items) in tagged_attributes {
        fields.insert(tag_name, serde_yaml::Value::Sequence(items));
    }

    Ok(ParsedDocument::new(fields))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_no_frontmatter() {
        let markdown = "# Hello World\n\nThis is a test.";
        let doc = decompose(markdown).unwrap();

        assert_eq!(doc.body(), Some(markdown));
        assert_eq!(doc.fields().len(), 1);
    }

    #[test]
    fn test_with_frontmatter() {
        let markdown = r#"---
title: Test Document
author: Test Author
---

# Hello World

This is the body."#;

        let doc = decompose(markdown).unwrap();

        assert_eq!(doc.body(), Some("\n# Hello World\n\nThis is the body."));
        assert_eq!(
            doc.get_field("title").unwrap().as_str().unwrap(),
            "Test Document"
        );
        assert_eq!(
            doc.get_field("author").unwrap().as_str().unwrap(),
            "Test Author"
        );
        assert_eq!(doc.fields().len(), 3); // title, author, body
    }

    #[test]
    fn test_complex_yaml_frontmatter() {
        let markdown = r#"---
title: Complex Document
tags:
  - test
  - yaml
metadata:
  version: 1.0
  nested:
    field: value
---

Content here."#;

        let doc = decompose(markdown).unwrap();

        assert_eq!(doc.body(), Some("\nContent here."));
        assert_eq!(
            doc.get_field("title").unwrap().as_str().unwrap(),
            "Complex Document"
        );

        let tags = doc.get_field("tags").unwrap().as_sequence().unwrap();
        assert_eq!(tags.len(), 2);
        assert_eq!(tags[0].as_str().unwrap(), "test");
        assert_eq!(tags[1].as_str().unwrap(), "yaml");
    }

    #[test]
    fn test_invalid_yaml() {
        let markdown = r#"---
title: [invalid yaml
author: missing close bracket
---

Content here."#;

        let result = decompose(markdown);
        assert!(result.is_err());
        assert!(result
            .unwrap_err()
            .to_string()
            .contains("Invalid YAML frontmatter"));
    }

    #[test]
    fn test_unclosed_frontmatter() {
        let markdown = r#"---
title: Test
author: Test Author

Content without closing ---"#;

        let result = decompose(markdown);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("not closed"));
    }

    // Extended metadata tests

    #[test]
    fn test_basic_tagged_block() {
        let markdown = r#"---
title: Main Document
---

Main body content.

---
!items
name: Item 1
---

Body of item 1."#;

        let doc = decompose(markdown).unwrap();

        assert_eq!(doc.body(), Some("\nMain body content.\n\n"));
        assert_eq!(
            doc.get_field("title").unwrap().as_str().unwrap(),
            "Main Document"
        );

        let items = doc.get_field("items").unwrap().as_sequence().unwrap();
        assert_eq!(items.len(), 1);

        let item = items[0].as_mapping().unwrap();
        assert_eq!(
            item.get(&serde_yaml::Value::String("name".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "Item 1"
        );
        assert_eq!(
            item.get(&serde_yaml::Value::String("body".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "\nBody of item 1."
        );
    }

    #[test]
    fn test_multiple_tagged_blocks() {
        let markdown = r#"---
!items
name: Item 1
tags: [a, b]
---

First item body.

---
!items
name: Item 2
tags: [c, d]
---

Second item body."#;

        let doc = decompose(markdown).unwrap();

        let items = doc.get_field("items").unwrap().as_sequence().unwrap();
        assert_eq!(items.len(), 2);

        let item1 = items[0].as_mapping().unwrap();
        assert_eq!(
            item1
                .get(&serde_yaml::Value::String("name".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "Item 1"
        );

        let item2 = items[1].as_mapping().unwrap();
        assert_eq!(
            item2
                .get(&serde_yaml::Value::String("name".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "Item 2"
        );
    }

    #[test]
    fn test_mixed_global_and_tagged() {
        let markdown = r#"---
title: Global
author: John Doe
---

Global body.

---
!sections
title: Section 1
---

Section 1 content.

---
!sections
title: Section 2
---

Section 2 content."#;

        let doc = decompose(markdown).unwrap();

        assert_eq!(doc.get_field("title").unwrap().as_str().unwrap(), "Global");
        assert_eq!(doc.body(), Some("\nGlobal body.\n\n"));

        let sections = doc.get_field("sections").unwrap().as_sequence().unwrap();
        assert_eq!(sections.len(), 2);
    }

    #[test]
    fn test_empty_tagged_metadata() {
        let markdown = r#"---
!items
---

Body without metadata."#;

        let doc = decompose(markdown).unwrap();

        let items = doc.get_field("items").unwrap().as_sequence().unwrap();
        assert_eq!(items.len(), 1);

        let item = items[0].as_mapping().unwrap();
        assert_eq!(
            item.get(&serde_yaml::Value::String("body".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "\nBody without metadata."
        );
    }

    #[test]
    fn test_tagged_block_without_body() {
        let markdown = r#"---
!items
name: Item
---"#;

        let doc = decompose(markdown).unwrap();

        let items = doc.get_field("items").unwrap().as_sequence().unwrap();
        assert_eq!(items.len(), 1);

        let item = items[0].as_mapping().unwrap();
        assert_eq!(
            item.get(&serde_yaml::Value::String("body".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            ""
        );
    }

    #[test]
    fn test_name_collision_global_and_tagged() {
        let markdown = r#"---
items: "global value"
---

Body

---
!items
name: Item
---

Item body"#;

        let result = decompose(markdown);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("collision"));
    }

    #[test]
    fn test_reserved_field_name() {
        let markdown = r#"---
!body
content: Test
---"#;

        let result = decompose(markdown);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("reserved"));
    }

    #[test]
    fn test_invalid_tag_syntax() {
        let markdown = r#"---
!Invalid-Name
title: Test
---"#;

        let result = decompose(markdown);
        assert!(result.is_err());
        assert!(result.unwrap_err().to_string().contains("Invalid tag name"));
    }

    #[test]
    fn test_multiple_global_frontmatter_blocks() {
        let markdown = r#"---
title: First
---

Body

---
author: Second
---

More body"#;

        let result = decompose(markdown);
        assert!(result.is_err());
        assert!(result
            .unwrap_err()
            .to_string()
            .contains("Multiple global frontmatter"));
    }

    #[test]
    fn test_adjacent_blocks_different_tags() {
        let markdown = r#"---
!items
name: Item 1
---

Item 1 body

---
!sections
title: Section 1
---

Section 1 body"#;

        let doc = decompose(markdown).unwrap();

        assert!(doc.get_field("items").is_some());
        assert!(doc.get_field("sections").is_some());

        let items = doc.get_field("items").unwrap().as_sequence().unwrap();
        assert_eq!(items.len(), 1);

        let sections = doc.get_field("sections").unwrap().as_sequence().unwrap();
        assert_eq!(sections.len(), 1);
    }

    #[test]
    fn test_order_preservation() {
        let markdown = r#"---
!items
id: 1
---

First

---
!items
id: 2
---

Second

---
!items
id: 3
---

Third"#;

        let doc = decompose(markdown).unwrap();

        let items = doc.get_field("items").unwrap().as_sequence().unwrap();
        assert_eq!(items.len(), 3);

        for (i, item) in items.iter().enumerate() {
            let mapping = item.as_mapping().unwrap();
            let id = mapping
                .get(&serde_yaml::Value::String("id".to_string()))
                .unwrap()
                .as_i64()
                .unwrap();
            assert_eq!(id, (i + 1) as i64);
        }
    }

    #[test]
    fn test_product_catalog_integration() {
        let markdown = r#"---
title: Product Catalog
author: John Doe
date: 2024-01-01
---

This is the main catalog description.

---
!products
name: Widget A
price: 19.99
sku: WID-001
---

The **Widget A** is our most popular product.

---
!products
name: Gadget B
price: 29.99
sku: GAD-002
---

The **Gadget B** is perfect for professionals.

---
!reviews
product: Widget A
rating: 5
---

"Excellent product! Highly recommended."

---
!reviews
product: Gadget B
rating: 4
---

"Very good, but a bit pricey.""#;

        let doc = decompose(markdown).unwrap();

        // Verify global fields
        assert_eq!(
            doc.get_field("title").unwrap().as_str().unwrap(),
            "Product Catalog"
        );
        assert_eq!(
            doc.get_field("author").unwrap().as_str().unwrap(),
            "John Doe"
        );
        assert_eq!(
            doc.get_field("date").unwrap().as_str().unwrap(),
            "2024-01-01"
        );

        // Verify global body
        assert!(doc.body().unwrap().contains("main catalog description"));

        // Verify products collection
        let products = doc.get_field("products").unwrap().as_sequence().unwrap();
        assert_eq!(products.len(), 2);

        let product1 = products[0].as_mapping().unwrap();
        assert_eq!(
            product1
                .get(&serde_yaml::Value::String("name".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "Widget A"
        );
        assert_eq!(
            product1
                .get(&serde_yaml::Value::String("price".to_string()))
                .unwrap()
                .as_f64()
                .unwrap(),
            19.99
        );

        // Verify reviews collection
        let reviews = doc.get_field("reviews").unwrap().as_sequence().unwrap();
        assert_eq!(reviews.len(), 2);

        let review1 = reviews[0].as_mapping().unwrap();
        assert_eq!(
            review1
                .get(&serde_yaml::Value::String("product".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "Widget A"
        );
        assert_eq!(
            review1
                .get(&serde_yaml::Value::String("rating".to_string()))
                .unwrap()
                .as_i64()
                .unwrap(),
            5
        );

        // Total fields: title, author, date, body, products, reviews = 6
        assert_eq!(doc.fields().len(), 6);
    }
}
#[cfg(test)]
mod demo_file_test {
    use super::*;

    #[test]
    fn test_extended_metadata_demo_file() {
        let markdown = include_str!("../../quillmark-fixtures/resources/extended_metadata_demo.md");
        let doc = decompose(markdown).unwrap();

        // Verify global fields
        assert_eq!(
            doc.get_field("title").unwrap().as_str().unwrap(),
            "Extended Metadata Demo"
        );
        assert_eq!(
            doc.get_field("author").unwrap().as_str().unwrap(),
            "Quillmark Team"
        );
        // version is parsed as a number by YAML
        assert_eq!(doc.get_field("version").unwrap().as_f64().unwrap(), 1.0);

        // Verify body
        assert!(doc
            .body()
            .unwrap()
            .contains("extended YAML metadata standard"));

        // Verify features collection
        let features = doc.get_field("features").unwrap().as_sequence().unwrap();
        assert_eq!(features.len(), 3);

        // Verify use_cases collection
        let use_cases = doc.get_field("use_cases").unwrap().as_sequence().unwrap();
        assert_eq!(use_cases.len(), 2);

        // Check first feature
        let feature1 = features[0].as_mapping().unwrap();
        assert_eq!(
            feature1
                .get(&serde_yaml::Value::String("name".to_string()))
                .unwrap()
                .as_str()
                .unwrap(),
            "Tag Directives"
        );
    }
}