lex-babel 0.18.1

//! Markdown serialization (Lex → Markdown export)
//!
//! Converts Lex documents to CommonMark Markdown.
//! Pipeline: Lex AST → IR → Events → Comrak AST → Markdown string

use crate::common::nested_to_flat::tree_to_events;
use crate::common::splice::SpliceState;
use crate::error::FormatError;
use crate::ir::events::Event;
use crate::ir::nodes::{Annotation as IrAnnotation, DocNode, InlineContent, TableCellAlignment};
use crate::render_dispatch::{dispatch_render, RenderedNode};
use comrak::nodes::{
    Ast, AstNode, ListDelimType, ListType, NodeDescriptionItem, NodeTable, NodeValue,
    TableAlignment,
};
use comrak::{format_commonmark, Arena, ComrakOptions};
use lex_core::lex::ast::Document;
use lex_extension_host::Registry;
use std::cell::RefCell;

/// Result of [`serialize_to_markdown_with_registry`]: the rendered
/// Markdown plus any handler-emitted diagnostic messages (renderer
/// errors, format-shape mismatches, namespace disabled).
#[derive(Debug, Clone, PartialEq)]
pub struct MarkdownExportOutcome {
    pub markdown: String,
    pub diagnostics: Vec<String>,
}

/// Serialize a Lex document to Markdown, dispatching `on_render`
/// handlers through [`crate::default_registry()`]. Diagnostics
/// produced by handlers are discarded; for diagnostic-aware
/// serialization use [`serialize_to_markdown_with_registry`].
pub fn serialize_to_markdown(doc: &Document) -> Result<String, FormatError> {
    let outcome = serialize_to_markdown_with_registry(doc, crate::default_registry())?;
    Ok(outcome.markdown)
}

/// Serialize with an extension-system [`Registry`] in scope: every
/// labelled annotation whose schema declares `hooks.render: ["markdown"]`
/// is dispatched to its handler. Handlers returning `RenderOut::String`
/// are spliced into the output in place of the annotation's default
/// rendering. For doc-scope annotations (the `document_annotations` IR
/// slot), handler output is consumed as a YAML frontmatter line; the
/// `lex.metadata.*` family without registered handlers falls back to
/// in-place synthesis.
pub fn serialize_to_markdown_with_registry(
    doc: &Document,
    registry: &Registry,
) -> Result<MarkdownExportOutcome, FormatError> {
    let ir_doc = crate::to_ir_with_registry(doc, registry);
    let plan = dispatch_render(&ir_doc, registry, "markdown");

    // Extract title from IR
    let document_title = ir_doc.title.as_ref().map(|title_inlines| {
        let title_text = inlines_to_text(title_inlines);
        let subtitle_text = ir_doc.subtitle.as_ref().map(|sub| inlines_to_text(sub));
        (title_text, subtitle_text)
    });

    // Phase 3b (#614): YAML frontmatter is synthesized directly from
    // `document_annotations` rather than via a `frontmatter` event
    // that `tree_to_events` used to inject. The IR slot is the single
    // source of truth on the lex → markdown path. For doc-scope
    // annotations with a registered render handler (e.g. `doc.*`), the
    // handler's YAML line replaces the default synthesis; everything
    // else (`lex.metadata.*` shortcuts) falls back to the legacy
    // synthesis path.
    let doc_scope_plan = &plan.nodes[..plan.doc_scope_count];
    let body_plan = &plan.nodes[plan.doc_scope_count..];
    let frontmatter_yaml =
        render_document_annotations_as_yaml(&ir_doc.document_annotations, doc_scope_plan);

    // Step 2: IR → Events
    let events = tree_to_events(&DocNode::Document(ir_doc));

    // Step 3: Events → Comrak AST. SpliceState consumes the body plan
    // to replace handler-rendered annotations with their raw markdown
    // output (emitted as a `NodeValue::HtmlBlock` literal so Comrak
    // passes it through unchanged).
    let arena = Arena::new();
    let root = build_comrak_ast(&arena, &events, body_plan)?;

    // Step 4: Comrak AST → Markdown string (using comrak's serializer)
    let mut output = Vec::new();
    let options = default_comrak_options();
    format_commonmark(root, &options, &mut output).map_err(|e| {
        FormatError::SerializationError(format!("Comrak serialization failed: {e}"))
    })?;

    let markdown = String::from_utf8(output)
        .map_err(|e| FormatError::SerializationError(format!("UTF-8 conversion failed: {e}")))?;

    // Remove Comrak's "end list" HTML comments which appear between consecutive lists
    let cleaned = markdown.replace("<!-- end list -->\n\n", "");

    // #606: Strip Comrak's backslash-escape on a leading digit-dot inside an
    // ATX heading. Comrak escapes `1.` at heading start to disambiguate from
    // an ordered-list marker, but a `#`-headed line can't open a list — the
    // escape is visually noisy. Only applies at the start of a heading line.
    let cleaned = strip_heading_digit_dot_escape(&cleaned);

    // Prepend document title as H1 heading if present
    let with_title = prepend_title_as_h1(&cleaned, document_title);

    // Prepend YAML frontmatter from document_annotations, if any.
    // Markdown imports already place a `frontmatter` annotation in
    // `children[0]` (handled inside `build_comrak_ast`), so this
    // branch fires only when the IR was built from a lex source —
    // the two paths don't double-write.
    let with_frontmatter = match frontmatter_yaml {
        Some(yaml) => format!("{yaml}{with_title}"),
        None => with_title,
    };

    let mut diagnostics: Vec<String> = plan
        .nodes
        .iter()
        .filter_map(|n| n.diagnostic.clone())
        .collect();
    diagnostics.extend(plan.root_diagnostics);

    Ok(MarkdownExportOutcome {
        markdown: with_frontmatter,
        diagnostics,
    })
}

/// Build the YAML frontmatter block (`---\n…\n---\n\n`) from an IR
/// document's `document_annotations`, or `None` if the slot would
/// produce an empty block.
///
/// `doc_scope_plan` is the doc-scope prefix of the registry's render
/// plan (the first `plan.doc_scope_count` entries). Annotations whose
/// handler returned a `RenderOut::String` consume the next plan entry
/// and emit the handler's text as the YAML line — this is how the
/// `doc.*` namespace (registered via Sub B) routes through the
/// markdown pipeline. Annotations without a registered handler — most
/// notably the `lex.metadata.*` shortcut family — fall back to the
/// legacy in-place synthesis that mirrors the key-flattening logic the
/// retired `emit_frontmatter_event` used: `lex.metadata.<key>` →
/// `<key>`; annotations with a paragraph body produce `<key>: <body>`;
/// annotations with structured params produce `<key>.<sub>: <value>`;
/// marker-form annotations contribute nothing.
fn render_document_annotations_as_yaml(
    annotations: &[IrAnnotation],
    doc_scope_plan: &[RenderedNode],
) -> Option<String> {
    if annotations.is_empty() {
        return None;
    }
    let mut lines = String::new();
    let mut plan_iter = doc_scope_plan.iter().peekable();
    for ann in annotations {
        // Lockstep with the plan: `dispatch_render` visits
        // `document_annotations` in order and emits one plan entry
        // per registered annotation, so the next plan entry whose
        // label matches this annotation belongs to it.
        let plan_entry = match plan_iter.peek() {
            Some(entry) if entry.label == ann.label => plan_iter.next(),
            _ => None,
        };
        if let Some(entry) = plan_entry {
            if let Some(out) = &entry.output {
                lines.push_str(out);
                continue;
            }
            // Registered but no output (handler returned None or
            // errored) — fall through to legacy synthesis.
        }
        synthesize_yaml_line_from_annotation(&mut lines, ann);
    }
    if lines.is_empty() {
        return None;
    }
    Some(format!("---\n{lines}---\n\n"))
}

/// Legacy in-place synthesis: append one or more YAML lines for `ann`
/// to `lines`. Used for annotations whose label isn't registered with a
/// markdown render handler — primarily the `lex.metadata.*` shortcut
/// family preserved while the shortcut table still maps `:: title ::`
/// onto `lex.metadata.title` rather than `doc.title`.
fn synthesize_yaml_line_from_annotation(lines: &mut String, ann: &IrAnnotation) {
    let key = ann
        .label
        .strip_prefix("lex.metadata.")
        .unwrap_or(ann.label.as_str())
        .to_string();
    // Trim whitespace lex picks up after the closing `::` separator
    // (e.g. `:: title :: My Doc` produces a paragraph whose first
    // inline is `" My Doc"`). Collapse internal newlines to spaces —
    // multi-line annotation bodies emit `InlineContent::Text("\n")`
    // between lines (see `from_lex_paragraph`), and a literal `\n`
    // inside a YAML scalar would orphan the trailing lines from the
    // key.
    let body_text = flatten_annotation_body_text(&ann.content)
        .replace('\n', " ")
        .trim()
        .to_string();
    if !body_text.is_empty() {
        lines.push_str(&format!("{key}: {body_text}\n"));
    } else if !ann.parameters.is_empty() {
        for (k, v) in &ann.parameters {
            lines.push_str(&format!("{key}.{k}: {v}\n"));
        }
    }
}

/// Flatten the text content of an annotation's paragraph children
/// into a single string. Used for YAML frontmatter synthesis. Covers
/// every inline shape that can legitimately appear inside metadata
/// bodies — `Text`, `Code`, `Math`, `Reference`, `Link` (regression
/// coverage from #596 / #597). Bold / Italic / Image are skipped on
/// purpose: YAML values are leaf strings, not rich content.
fn flatten_annotation_body_text(content: &[DocNode]) -> String {
    let mut text = String::new();
    for child in content {
        if let DocNode::Paragraph(p) = child {
            for inline in &p.content {
                match inline {
                    InlineContent::Text(t) | InlineContent::Code(t) | InlineContent::Math(t) => {
                        text.push_str(t)
                    }
                    InlineContent::Reference { raw, .. } => text.push_str(raw),
                    InlineContent::Link { text: t, .. } => text.push_str(t),
                    _ => {}
                }
            }
        }
    }
    text
}

/// Prepend document title as an H1 heading, optionally followed by subtitle as H2
///
/// If the document has a title, prepend `# Title` at the beginning.
/// If it also has a subtitle, append `## Subtitle` below.
fn prepend_title_as_h1(markdown: &str, title: Option<(String, Option<String>)>) -> String {
    match title {
        Some((t, Some(sub))) => format!("# {t}\n\n## {sub}\n\n{markdown}"),
        Some((t, None)) => format!("# {t}\n\n{markdown}"),
        None => markdown.to_string(),
    }
}

/// Convert IR inline content to plain text for title rendering
fn inlines_to_text(content: &[InlineContent]) -> String {
    content
        .iter()
        .map(|inline| match inline {
            InlineContent::Text(t) => t.clone(),
            InlineContent::Bold(c) => inlines_to_text(c),
            InlineContent::Italic(c) => inlines_to_text(c),
            InlineContent::Code(c) => c.clone(),
            InlineContent::Math(m) => m.clone(),
            InlineContent::Reference { raw, .. } => raw.clone(),
            InlineContent::Link { text, .. } => text.clone(),
            InlineContent::Image(img) => img.alt.clone(),
        })
        .collect()
}

/// Strip Comrak's `\.` escape after a leading digit run inside an ATX
/// heading. Comrak escapes `^(#+ \d+)\.` → `^(#+ \d+)\\.` to keep
/// pure CommonMark renderers from misreading the line as an ordered list,
/// but a `#`-prefixed line can't open a list, so the escape is just noise.
/// Applies per line, headings only — paragraph-leading `1\.` is left
/// alone since Comrak's protection there is meaningful (#606).
fn strip_heading_digit_dot_escape(markdown: &str) -> String {
    static HEADING_DIGIT_DOT: once_cell::sync::Lazy<regex::Regex> =
        once_cell::sync::Lazy::new(|| {
            regex::Regex::new(r"(?m)^(#+ \d+(?:\.\d+)*)\\\.").expect("compile heading digit-dot")
        });
    HEADING_DIGIT_DOT.replace_all(markdown, "$1.").into_owned()
}

fn default_comrak_options() -> ComrakOptions<'static> {
    let mut options = ComrakOptions::default();
    options.extension.table = true;
    options.extension.strikethrough = true;
    options.extension.autolink = true;
    options.extension.tasklist = true;
    options.extension.superscript = true;
    options.extension.front_matter_delimiter = Some("---".to_string());
    // Pandoc-flavored definition lists — let Comrak emit `Term\n\n: details`
    // for `DescriptionList` / `DescriptionItem` / `DescriptionTerm` /
    // `DescriptionDetails` nodes (see `cm.rs::format_description_details`).
    // Lex emits these instead of the legacy `**Term**:` fallback so the
    // `<dl>` structure round-trips through Pandoc-aware tools (#605).
    options.extension.description_lists = true;
    // Allow HTML output for annotations (rendered as HTML comments)
    options.render.unsafe_ = true;
    options
}

/// Build a Comrak AST from IR events, optionally splicing
/// handler-rendered Markdown in place of default annotation rendering.
///
/// `body_plan` is the body slice of the registry's render plan
/// (entries past the doc-scope prefix). When a `StartAnnotation`
/// matches a plan entry with output, the handler's raw markdown is
/// emitted as a `NodeValue::HtmlBlock` literal so Comrak passes it
/// through unchanged, and the annotation's children are skipped via
/// [`SpliceState`].
fn build_comrak_ast<'a>(
    arena: &'a Arena<AstNode<'a>>,
    events: &[Event],
    body_plan: &[RenderedNode],
) -> Result<&'a AstNode<'a>, FormatError> {
    // Create document root
    let root = arena.alloc(AstNode::new(RefCell::new(Ast::new(
        NodeValue::Document,
        (0, 0).into(),
    ))));

    let mut current_parent: &'a AstNode<'a> = root;
    let mut parent_stack: Vec<&'a AstNode<'a>> = vec![];

    // State for collecting verbatim content
    let mut in_verbatim = false;
    let mut verbatim_content = String::new();
    let mut verbatim_language = None;

    // State for handling headings (which can only contain inline content).
    // Once we start a block after the heading, we clear this so later inline
    // events do not get appended to the heading text (a prior bug).
    let mut current_heading: Option<&'a AstNode<'a>> = None;

    // State for handling list items
    let mut in_list_item = false;
    let mut list_item_paragraph: Option<&'a AstNode<'a>> = None;

    // State for handling table cells (flatten paragraphs inside cells)
    let mut in_table_cell = false;

    // Splice state for handler-rendered annotations. `body_plan` is
    // empty for serializers that don't dispatch through a registry,
    // in which case `SpliceState::advance_at_start` always returns
    // None and every annotation gets its default comment-pair
    // rendering.
    let splice_plan = if body_plan.is_empty() {
        None
    } else {
        Some(body_plan)
    };
    let mut splice = SpliceState::new(splice_plan);

    for event in events {
        // Inside a handler-owned splice region, only Start/EndAnnotation
        // events are inspected — for nesting depth and counter
        // bookkeeping. Every other event is suppressed so the
        // handler's output replaces the annotation's subtree entirely.
        if splice.should_skip()
            && !matches!(
                event,
                Event::StartAnnotation { .. } | Event::EndAnnotation { .. }
            )
        {
            continue;
        }
        match event {
            Event::StartDocument => {
                // Already created root
            }

            Event::EndDocument => {
                // Done
            }

            Event::StartHeading(level) => {
                // Headings can only contain inline content, not block elements
                // Create heading and set it as target for inline content
                let heading_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Heading(comrak::nodes::NodeHeading {
                        level: (*level as u8).min(6),
                        setext: false,
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(heading_node);
                current_heading = Some(heading_node);
                // Note: We do NOT change current_parent or push to parent_stack
                // Block content after this heading will be siblings at document level
            }

            Event::EndHeading(_) => {
                // Close heading - block content goes back to document level
                current_heading = None;
            }

            Event::StartContent => {
                // Content markers are for HTML indentation - no-op in Markdown
            }

            Event::EndContent => {
                // Content markers are for HTML indentation - no-op in Markdown
            }

            Event::StartParagraph => {
                // Block after a heading – inline content should no longer
                // target the heading title.
                current_heading = None;

                if in_table_cell {
                    // Don't create paragraph node inside table cell
                    // Just let inline content be added to current_parent (which is TableCell)
                } else {
                    let para_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::Paragraph,
                        (0, 0).into(),
                    ))));
                    current_parent.append(para_node);
                    parent_stack.push(current_parent);
                    current_parent = para_node;
                    // If we're in a list item, this explicit paragraph replaces any auto-created one
                    if in_list_item {
                        list_item_paragraph = None;
                    }
                }
            }

            Event::EndParagraph => {
                if !in_table_cell {
                    current_parent = parent_stack.pop().ok_or_else(|| {
                        FormatError::SerializationError("Unbalanced paragraph end".to_string())
                    })?;
                }
            }

            Event::StartList { ordered, .. } => {
                current_heading = None;

                let list_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::List(comrak::nodes::NodeList {
                        list_type: if *ordered {
                            ListType::Ordered
                        } else {
                            ListType::Bullet
                        },
                        marker_offset: 0,
                        padding: 0,
                        start: 1,
                        delimiter: ListDelimType::Period,
                        bullet_char: b'-',
                        tight: true, // Use tight lists to avoid blank lines between items
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(list_node);
                parent_stack.push(current_parent);
                current_parent = list_node;
            }

            Event::EndList => {
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced list end".to_string())
                })?;
            }

            Event::StartListItem => {
                current_heading = None;

                let item_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Item(comrak::nodes::NodeList {
                        list_type: ListType::Bullet,
                        marker_offset: 0,
                        padding: 0,
                        start: 1,
                        delimiter: ListDelimType::Period,
                        bullet_char: b'-',
                        tight: true, // Tight items don't add extra spacing
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(item_node);
                parent_stack.push(current_parent);
                current_parent = item_node;
                in_list_item = true;
                list_item_paragraph = None;
            }

            Event::EndListItem => {
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced list item end".to_string())
                })?;
                in_list_item = false;
                list_item_paragraph = None;
            }

            Event::StartVerbatim {
                language,
                subject,
                subject_href,
                parameters: _,
            } => {
                current_heading = None;

                // Render subject as bold text before the code block. A reference
                // line can anchor the subject (references-general.lex §2.3.2):
                // when `subject_href` is set, the bold caption wraps a link
                // (`**[subject](href)**`).
                if let Some(subj) = subject {
                    let para = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::Paragraph,
                        (0, 0).into(),
                    ))));
                    current_parent.append(para);
                    let strong = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::Strong,
                        (0, 0).into(),
                    ))));
                    para.append(strong);
                    let text = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::Text(subj.clone()),
                        (0, 0).into(),
                    ))));
                    if let Some(href) = subject_href {
                        let link = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                            NodeValue::Link(comrak::nodes::NodeLink {
                                url: href.clone(),
                                title: String::new(),
                            }),
                            (0, 0).into(),
                        ))));
                        strong.append(link);
                        link.append(text);
                    } else {
                        strong.append(text);
                    }
                }

                // Check for special metadata comment format
                if let Some(lang) = &language {
                    if let Some(label) = lang.strip_prefix("lex-metadata:") {
                        // This is a metadata annotation to be rendered as an HTML comment
                        // The content will follow as Inline(Text)
                        // We need to capture it and wrap it in <!-- lex:label ... -->

                        let node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                            NodeValue::HtmlBlock(comrak::nodes::NodeHtmlBlock {
                                block_type: 0,
                                literal: String::new(), // Will be filled by content
                            }),
                            (0, 0).into(),
                        ))));
                        current_parent.append(node);
                        parent_stack.push(current_parent); // Push the old parent
                        current_parent = node; // Set new parent to the HtmlBlock

                        // Prepend the start tag now.
                        let start_tag = format!("<!-- lex:{label}");
                        let mut data = node.data.borrow_mut();
                        if let NodeValue::HtmlBlock(ref mut html) = data.value {
                            html.literal.push_str(&start_tag);
                        }

                        // Set in_verbatim to true to indicate we are accumulating content
                        // for this special HtmlBlock.
                        in_verbatim = true;
                        verbatim_language = language.clone(); // Store for EndVerbatim check
                        verbatim_content.clear(); // Clear any previous content

                        continue; // Skip the rest of the StartVerbatim logic
                    }
                }

                // Original verbatim block handling
                in_verbatim = true;
                verbatim_language = language.clone();
                verbatim_content.clear();
            }

            Event::EndVerbatim => {
                // If we were processing a metadata comment (HtmlBlock), we need to close it
                let is_html_block =
                    matches!(current_parent.data.borrow().value, NodeValue::HtmlBlock(_));

                if is_html_block {
                    // Append closing tag
                    let mut data = current_parent.data.borrow_mut();
                    if let NodeValue::HtmlBlock(ref mut html) = data.value {
                        html.literal.push_str("\n-->");
                    }
                    // Pop the HtmlBlock node from the stack
                    current_parent = parent_stack.pop().ok_or_else(|| {
                        FormatError::SerializationError("Unbalanced HTML block end".to_string())
                    })?;
                } else {
                    // Original verbatim block handling: Create code block with accumulated content
                    let code_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::CodeBlock(comrak::nodes::NodeCodeBlock {
                            fenced: true,
                            fence_char: b'`',
                            fence_length: 3,
                            fence_offset: 0,
                            info: verbatim_language.take().unwrap_or_default(),
                            literal: verbatim_content.clone(),
                        }),
                        (0, 0).into(),
                    ))));
                    current_parent.append(code_node);
                }
                in_verbatim = false;
                verbatim_content.clear();
            }

            Event::Inline(inline_content) => {
                let inline_to_emit = inline_content.clone();

                if in_verbatim {
                    // If we are in a special lex-metadata verbatim block (which is an HtmlBlock)
                    // or a regular verbatim block, accumulate content.
                    if let InlineContent::Text(text) = &inline_to_emit {
                        if matches!(current_parent.data.borrow().value, NodeValue::HtmlBlock(_)) {
                            // Append to the HtmlBlock's literal directly
                            let mut data = current_parent.data.borrow_mut();
                            if let NodeValue::HtmlBlock(ref mut html) = data.value {
                                html.literal.push_str(text);
                            }
                        } else {
                            // Accumulate for a regular CodeBlock (will be created in EndVerbatim)
                            verbatim_content.push_str(text);
                        }
                    }
                } else if matches!(current_parent.data.borrow().value, NodeValue::HtmlBlock(_)) {
                    // If we are inside an HtmlBlock (metadata comment), append text to literal
                    let mut data = current_parent.data.borrow_mut();
                    if let NodeValue::HtmlBlock(ref mut html) = data.value {
                        if let InlineContent::Text(text) = inline_to_emit {
                            html.literal.push_str(&text);
                        }
                    }
                } else if let Some(heading) = current_heading {
                    // Add to heading (headings can have inline content directly)
                    // Session numbering is preserved — it's an author's communication choice
                    add_inline_to_node(arena, heading, &inline_to_emit)?;
                } else if in_list_item {
                    // If we're already inside an explicit paragraph, write directly to it.
                    if matches!(current_parent.data.borrow().value, NodeValue::Paragraph) {
                        add_inline_to_node(arena, current_parent, &inline_to_emit)?;
                    } else {
                        // Auto-wrap inline content in a paragraph. List items need block content.
                        // Using tight lists prevents extra blank lines.
                        if list_item_paragraph.is_none() {
                            let para = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                                NodeValue::Paragraph,
                                (0, 0).into(),
                            ))));
                            current_parent.append(para);
                            list_item_paragraph = Some(para);
                        }
                        add_inline_to_node(arena, list_item_paragraph.unwrap(), &inline_to_emit)?;
                    }
                } else {
                    // Regular inline content added to current_parent
                    add_inline_to_node(arena, current_parent, &inline_to_emit)?;
                }
            }

            Event::StartAnnotation {
                label, parameters, ..
            } if label == "frontmatter" => {
                // Serialize as YAML frontmatter
                let mut yaml = String::from("---\n");
                for (key, value) in parameters {
                    // Simple YAML serialization
                    // If value contains special chars, we might need quoting, but for now simple string
                    yaml.push_str(&format!("{key}: {value}\n"));
                }
                yaml.push_str("---\n\n");

                let frontmatter_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::FrontMatter(yaml),
                    (0, 0).into(),
                ))));
                current_parent.append(frontmatter_node);
                // No need to push to stack or change current_parent as FrontMatter is a leaf block
            }

            Event::StartAnnotation {
                label, parameters, ..
            } => {
                current_heading = None;

                if let Some(rendered_markdown) = splice.advance_at_start(label) {
                    // Handler-rendered passthrough — emit as an
                    // HtmlBlock literal so Comrak hands it through to
                    // the output without re-escaping. `SpliceState`
                    // suppresses the annotation's body events until
                    // the matching `EndAnnotation`.
                    let html_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::HtmlBlock(comrak::nodes::NodeHtmlBlock {
                            block_type: 0,
                            literal: rendered_markdown.to_string(),
                        }),
                        (0, 0).into(),
                    ))));
                    current_parent.append(html_node);
                } else if !splice.should_skip() {
                    // No handler — emit the default lex:label start
                    // comment. The annotation's body events render with
                    // their default rendering; the matching
                    // `EndAnnotation` arm emits the closing comment.
                    // Suppress when we're inside a handler-owned splice
                    // region so a nested annotation doesn't leak a
                    // stray `<!-- lex:inner -->` into the handler's
                    // output (Copilot review on PR #625).
                    let mut comment = format!("<!-- lex:{label}");
                    for (key, value) in parameters {
                        comment.push_str(&format!(" {key}={value}"));
                    }
                    comment.push_str(" -->");

                    let html_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::HtmlBlock(comrak::nodes::NodeHtmlBlock {
                            block_type: 0,
                            literal: comment,
                        }),
                        (0, 0).into(),
                    ))));
                    current_parent.append(html_node);
                }
            }

            Event::EndAnnotation { label } if label == "frontmatter" => {
                // Nothing to do, FrontMatter node is self-contained
            }

            Event::EndAnnotation { label } => {
                if splice.should_skip() {
                    splice.advance_at_end();
                    continue;
                }
                // Not in splice — emit the default lex:label end
                // comment as before.
                let closing_tag = format!("<!-- /lex:{label} -->");
                let html_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::HtmlBlock(comrak::nodes::NodeHtmlBlock {
                        block_type: 0,
                        literal: closing_tag,
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(html_node);
            }

            Event::StartDefinition => {
                // #605: Emit Pandoc-flavored definition lists via Comrak's
                // native DescriptionList / DescriptionItem / DescriptionTerm /
                // DescriptionDetails nodes. Comrak's CommonMark formatter
                // writes `: ` in front of the description; the term renders
                // as a plain paragraph. The `description_lists` option is
                // enabled in `default_comrak_options`.
                current_heading = None;
                let dl_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::DescriptionList,
                    (0, 0).into(),
                ))));
                current_parent.append(dl_node);
                parent_stack.push(current_parent);
                current_parent = dl_node;

                let item_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::DescriptionItem(NodeDescriptionItem {
                        marker_offset: 0,
                        padding: 2,
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(item_node);
                parent_stack.push(current_parent);
                current_parent = item_node;
            }

            Event::EndDefinition => {
                // Pop DescriptionItem (discard intermediate), then DescriptionList.
                let _ = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced definition item end".to_string())
                })?;
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced definition list end".to_string())
                })?;
            }

            Event::StartDefinitionTerm => {
                current_heading = None;
                let term_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::DescriptionTerm,
                    (0, 0).into(),
                ))));
                current_parent.append(term_node);
                parent_stack.push(current_parent);
                current_parent = term_node;

                // Comrak's CM formatter requires the term content to live
                // inside a Paragraph so it renders with the right line
                // break. Without it, the term inlines and the description's
                // `: ` prefix render on the same line.
                let para_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Paragraph,
                    (0, 0).into(),
                ))));
                current_parent.append(para_node);
                parent_stack.push(current_parent);
                current_parent = para_node;
            }

            Event::EndDefinitionTerm => {
                // Close paragraph (discard intermediate), then close DescriptionTerm.
                let _ = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError(
                        "Unbalanced definition term paragraph end".to_string(),
                    )
                })?;
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced definition term end".to_string())
                })?;
            }

            Event::StartDefinitionDescription => {
                let details_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::DescriptionDetails,
                    (0, 0).into(),
                ))));
                current_parent.append(details_node);
                parent_stack.push(current_parent);
                current_parent = details_node;
            }

            Event::EndDefinitionDescription => {
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError(
                        "Unbalanced definition description end".to_string(),
                    )
                })?;
            }

            Event::StartTable { caption, .. } => {
                current_heading = None;

                // Render caption as bold paragraph before the table
                if let Some(caption_inlines) = caption {
                    let para = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::Paragraph,
                        (0, 0).into(),
                    ))));
                    current_parent.append(para);
                    let strong = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                        NodeValue::Strong,
                        (0, 0).into(),
                    ))));
                    para.append(strong);
                    for inline in caption_inlines {
                        add_inline_to_node(arena, strong, inline)?;
                    }
                }

                let table_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Table(NodeTable {
                        alignments: vec![],
                        num_columns: 0,
                        num_rows: 0,
                        num_nonempty_cells: 0,
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(table_node);
                parent_stack.push(current_parent);
                current_parent = table_node;
            }

            Event::EndTable => {
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced table end".to_string())
                })?;
            }

            Event::StartTableRow { header } => {
                let row_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::TableRow(*header),
                    (0, 0).into(),
                ))));
                current_parent.append(row_node);
                parent_stack.push(current_parent);
                current_parent = row_node;
            }

            Event::EndTableRow => {
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced table row end".to_string())
                })?;
            }

            Event::StartTableFootnotes => {
                // Markdown doesn't have table footnotes; render as content after the table
            }

            Event::EndTableFootnotes => {
                // No-op in Markdown
            }

            Event::StartTableCell {
                header: _, align, ..
            } => {
                let cell_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::TableCell,
                    (0, 0).into(),
                ))));
                current_parent.append(cell_node);
                parent_stack.push(current_parent);

                // Update table alignments if we are in the first row
                if parent_stack.len() >= 2 {
                    let table_node = parent_stack[parent_stack.len() - 2];
                    let row_node = parent_stack[parent_stack.len() - 1];

                    let mut table_data = table_node.data.borrow_mut();
                    if let NodeValue::Table(ref mut table) = table_data.value {
                        let is_first_row = table_node
                            .first_child()
                            .is_some_and(|first| std::ptr::eq(first, row_node));

                        if is_first_row {
                            let align_enum = match align {
                                TableCellAlignment::Left => TableAlignment::Left,
                                TableCellAlignment::Right => TableAlignment::Right,
                                TableCellAlignment::Center => TableAlignment::Center,
                                TableCellAlignment::None => TableAlignment::None,
                            };
                            table.alignments.push(align_enum);
                        }
                    }
                }

                current_parent = cell_node;
                in_table_cell = true;
            }

            Event::EndTableCell => {
                in_table_cell = false;
                current_parent = parent_stack.pop().ok_or_else(|| {
                    FormatError::SerializationError("Unbalanced table cell end".to_string())
                })?;
            }

            Event::Image(image) => {
                // Render as paragraph with image
                let para_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Paragraph,
                    (0, 0).into(),
                ))));
                current_parent.append(para_node);

                let image_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Image(comrak::nodes::NodeLink {
                        url: image.src.clone(),
                        title: image.title.clone().unwrap_or_default(),
                    }),
                    (0, 0).into(),
                ))));
                para_node.append(image_node);

                let text_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Text(image.alt.clone()),
                    (0, 0).into(),
                ))));
                image_node.append(text_node);
            }

            Event::Video(video) => {
                // Render as HTML <video>
                let mut html = format!(r#"<video src="{}""#, video.src);
                if let Some(poster) = &video.poster {
                    html.push_str(&format!(r#" poster="{poster}""#));
                }
                if let Some(title) = &video.title {
                    html.push_str(&format!(r#" title="{title}""#));
                }
                html.push_str(" controls></video>");

                let html_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::HtmlBlock(comrak::nodes::NodeHtmlBlock {
                        block_type: 0,
                        literal: html,
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(html_node);
            }

            Event::Audio(audio) => {
                // Render as HTML <audio>
                let mut html = format!(r#"<audio src="{}""#, audio.src);
                if let Some(title) = &audio.title {
                    html.push_str(&format!(r#" title="{title}""#));
                }
                html.push_str(" controls></audio>");

                let html_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::HtmlBlock(comrak::nodes::NodeHtmlBlock {
                        block_type: 0,
                        literal: html,
                    }),
                    (0, 0).into(),
                ))));
                current_parent.append(html_node);
            }
        }
    }

    Ok(root)
}

/// Add inline content to a comrak node
fn add_inline_to_node<'a>(
    arena: &'a Arena<AstNode<'a>>,
    parent: &'a AstNode<'a>,
    inline: &crate::ir::nodes::InlineContent,
) -> Result<(), FormatError> {
    use crate::ir::nodes::InlineContent;

    match inline {
        InlineContent::Text(text) => {
            let sanitized = text.replace('\n', " ");

            let text_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Text(sanitized),
                (0, 0).into(),
            ))));
            parent.append(text_node);
        }

        InlineContent::Bold(children) => {
            let strong_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Strong,
                (0, 0).into(),
            ))));
            parent.append(strong_node);
            for child in children {
                add_inline_to_node(arena, strong_node, child)?;
            }
        }

        InlineContent::Italic(children) => {
            let emph_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Emph,
                (0, 0).into(),
            ))));
            parent.append(emph_node);
            for child in children {
                add_inline_to_node(arena, emph_node, child)?;
            }
        }

        InlineContent::Code(code_text) => {
            let code_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Code(comrak::nodes::NodeCode {
                    num_backticks: 1,
                    literal: code_text.clone(),
                }),
                (0, 0).into(),
            ))));
            parent.append(code_node);
        }

        InlineContent::Reference {
            raw: ref_text,
            kind,
        } => {
            // Unresolved reference (non-linkable types: citations,
            // footnotes, general, etc.). Dispatch on the typed kind
            // preserved from lex-core; fall back to raw-string sniffing
            // only for `NotSure` (markdown re-import paths that didn't
            // re-classify against lex's reference grammar).
            use crate::ir::nodes::ReferenceType;
            let url = match kind {
                ReferenceType::Citation(data) => {
                    // Anchor on the citation KEY only (`#ref-spec2025`). The raw
                    // literal includes the locator (`@spec2025, pp. 45-46`);
                    // slugifying it baked "pp. 45-46" into the href, which never
                    // resolved (MD051). `keys` already holds the bare key(s);
                    // multi-key citations anchor on the first. Display text
                    // (ref_text, below) keeps the full literal.
                    let key = data.keys.first().cloned().unwrap_or_else(|| {
                        ref_text.strip_prefix('@').unwrap_or(ref_text).to_string()
                    });
                    Some(format!("#ref-{key}"))
                }
                ReferenceType::NotSure => ref_text
                    .strip_prefix('@')
                    .map(|citation| format!("#ref-{citation}")),
                _ => None,
            };

            if let Some(url) = url {
                let link_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Link(comrak::nodes::NodeLink {
                        url,
                        title: String::new(),
                    }),
                    (0, 0).into(),
                ))));
                parent.append(link_node);

                let text_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Text(ref_text.clone()),
                    (0, 0).into(),
                ))));
                link_node.append(text_node);
            } else {
                // Render as plain text with brackets: [reference]
                let text_with_brackets = format!("[{ref_text}]");
                let text_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                    NodeValue::Text(text_with_brackets),
                    (0, 0).into(),
                ))));
                parent.append(text_node);
            }
        }

        InlineContent::Link { text, href } => {
            let link_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Link(comrak::nodes::NodeLink {
                    url: href.clone(),
                    title: String::new(),
                }),
                (0, 0).into(),
            ))));
            parent.append(link_node);

            let text_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Text(text.clone()),
                (0, 0).into(),
            ))));
            link_node.append(text_node);
        }

        InlineContent::Math(math_text) => {
            // Math is emitted as raw `$expr$` via HtmlInline so Comrak's
            // CommonMark serializer doesn't escape `\` and `_` inside the
            // expression (which breaks KaTeX/MathJax: `\\` means newline).
            let math_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::HtmlInline(format!("${math_text}$")),
                (0, 0).into(),
            ))));
            parent.append(math_node);
        }

        InlineContent::Image(image) => {
            let image_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Image(comrak::nodes::NodeLink {
                    url: image.src.clone(),
                    title: image.title.clone().unwrap_or_default(),
                }),
                (0, 0).into(),
            ))));
            parent.append(image_node);

            let text_node = arena.alloc(AstNode::new(RefCell::new(Ast::new(
                NodeValue::Text(image.alt.clone()),
                (0, 0).into(),
            ))));
            image_node.append(text_node);
        }
    }

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use comrak::{parse_document, ComrakOptions};
    use lex_core::lex::transforms::standard::STRING_TO_AST;

    #[test]
    fn test_simple_paragraph_ast() {
        let lex_src = "This is a simple paragraph.\n";
        let lex_doc = STRING_TO_AST.run(lex_src.to_string()).unwrap();

        // Convert to markdown
        let md = serialize_to_markdown(&lex_doc).unwrap();

        // Parse back to comrak AST to verify structure
        let arena = Arena::new();
        let options = ComrakOptions::default();
        let root = parse_document(&arena, &md, &options);

        // Verify we have a paragraph
        let mut found_paragraph = false;
        for child in root.children() {
            if matches!(child.data.borrow().value, NodeValue::Paragraph) {
                found_paragraph = true;

                // Check inline text content
                for _inline in child.children() {
                    if let NodeValue::Text(ref text) = child.data.borrow().value {
                        assert!(text.contains("simple paragraph"));
                    }
                }
            }
        }
        assert!(found_paragraph, "Should have a paragraph node");
    }

    #[test]
    fn test_heading_ast() {
        let lex_src = "1. Introduction\n\n    Content here.\n";
        let lex_doc = STRING_TO_AST.run(lex_src.to_string()).unwrap();

        let md = serialize_to_markdown(&lex_doc).unwrap();

        // Parse and verify AST structure
        let arena = Arena::new();
        let options = ComrakOptions::default();
        let root = parse_document(&arena, &md, &options);

        let mut found_heading = false;
        for child in root.children() {
            if let NodeValue::Heading(ref heading) = child.data.borrow().value {
                assert_eq!(heading.level, 2);
                found_heading = true;
            }
        }
        assert!(found_heading, "Should have a heading node");
    }

    /// Phase 3b (#614): a lex source with `:: lex.metadata.* ::`
    /// document-scope annotations must surface in the markdown
    /// output as a YAML frontmatter block. Pre-Phase-3b this worked
    /// via the `frontmatter` event synthesis in `tree_to_events`;
    /// after the flip the markdown serializer synthesizes YAML
    /// directly from `document_annotations`.
    #[test]
    fn lex_metadata_annotations_emit_yaml_frontmatter() {
        let lex_src = ":: title :: My Doc\n\n:: author :: Alice\n\nBody.\n";
        let lex_doc = STRING_TO_AST.run(lex_src.to_string()).unwrap();
        let md = serialize_to_markdown(&lex_doc).unwrap();

        assert!(
            md.starts_with("---\n"),
            "expected YAML frontmatter at start of markdown, got:\n{md}"
        );
        assert!(md.contains("title: My Doc"), "missing title key in:\n{md}");
        assert!(md.contains("author: Alice"), "missing author key in:\n{md}");
    }

    /// Phase 3b coverage retained from #596 / #597: the YAML body
    /// flatten must read `Reference` and `Link` inlines (not only
    /// `Text`/`Code`/`Math`) so a `:: author :: Alice [https://…]`
    /// doesn't silently drop the link text in the YAML preamble.
    #[test]
    fn yaml_synthesis_includes_link_and_reference_inlines() {
        use crate::ir::nodes::{Annotation as IrAnn, DocNode, InlineContent, LabelForm, Paragraph};

        let yaml = render_document_annotations_as_yaml(
            &[IrAnn {
                label: "lex.metadata.author".to_string(),
                parameters: vec![],
                content: vec![DocNode::Paragraph(Paragraph {
                    content: vec![
                        InlineContent::Text("Alice ".to_string()),
                        InlineContent::Link {
                            text: "https://alice.example".to_string(),
                            href: "https://alice.example".to_string(),
                        },
                    ],
                })],
                form: LabelForm::Canonical,
            }],
            &[],
        )
        .expect("yaml block synthesized");
        assert!(
            yaml.contains("author: Alice https://alice.example"),
            "{yaml}"
        );

        let yaml = render_document_annotations_as_yaml(
            &[IrAnn {
                label: "lex.metadata.tags".to_string(),
                parameters: vec![],
                content: vec![DocNode::Paragraph(Paragraph {
                    content: vec![
                        InlineContent::Text("rust ".to_string()),
                        InlineContent::Reference {
                            raw: "@manning".to_string(),
                            kind: crate::ir::nodes::ReferenceType::NotSure,
                        },
                    ],
                })],
                form: LabelForm::Canonical,
            }],
            &[],
        )
        .expect("yaml block synthesized");
        assert!(yaml.contains("tags: rust @manning"), "{yaml}");
    }

    /// Gemini review on PR #621: a multi-line annotation body emits
    /// `InlineContent::Text("\n")` separators between lines (see
    /// `from_lex_paragraph`). A literal `\n` inside a YAML scalar
    /// orphans the trailing lines — the YAML parser reads them as
    /// keyless content. Collapse internal newlines to spaces so the
    /// preamble stays valid.
    #[test]
    fn yaml_synthesis_collapses_internal_newlines_to_spaces() {
        use crate::ir::nodes::{Annotation as IrAnn, DocNode, InlineContent, LabelForm, Paragraph};

        let yaml = render_document_annotations_as_yaml(
            &[IrAnn {
                label: "lex.metadata.note".to_string(),
                parameters: vec![],
                content: vec![DocNode::Paragraph(Paragraph {
                    content: vec![
                        InlineContent::Text("Line one.".to_string()),
                        InlineContent::Text("\n".to_string()),
                        InlineContent::Text("Line two.".to_string()),
                    ],
                })],
                form: LabelForm::Canonical,
            }],
            &[],
        )
        .expect("yaml block synthesized");

        assert!(
            yaml.contains("note: Line one. Line two.\n"),
            "internal newlines must collapse to spaces; got:\n{yaml}"
        );
        // Defensive: the value line itself must not contain a raw
        // newline character that would break YAML scalar parsing.
        let value_line = yaml.lines().find(|l| l.starts_with("note:")).unwrap();
        assert!(
            !value_line.contains('\n'),
            "value line must be single-line in YAML"
        );
    }

    /// Gemini review on PR #597: the body flatten must also cover
    /// `Code` and `Math` inline content so a metadata body like
    /// `Doc with \`snippet\` and #E=mc^2#` doesn't silently drop the
    /// code/math text in the YAML preamble.
    #[test]
    fn yaml_synthesis_includes_code_and_math_inlines() {
        use crate::ir::nodes::{Annotation as IrAnn, DocNode, InlineContent, LabelForm, Paragraph};

        let yaml = render_document_annotations_as_yaml(
            &[IrAnn {
                label: "lex.metadata.title".to_string(),
                parameters: vec![],
                content: vec![DocNode::Paragraph(Paragraph {
                    content: vec![
                        InlineContent::Text("Doc with ".to_string()),
                        InlineContent::Code("snippet".to_string()),
                        InlineContent::Text(" and ".to_string()),
                        InlineContent::Math("E=mc^2".to_string()),
                    ],
                })],
                form: LabelForm::Canonical,
            }],
            &[],
        )
        .expect("yaml block synthesized");
        assert!(
            yaml.contains("title: Doc with snippet and E=mc^2"),
            "{yaml}"
        );
    }
}