use comrak::nodes::{AstNode, ListType, NodeValue};
use comrak::{Arena, Options};
use crate::dialect::{extract_trailing_anchor, is_standalone_anchor};
use crate::finding::SourcePos;
use crate::frontmatter::yaml_to_json;
use crate::js;
use crate::tree::{BlockNode, DocTree, Frontmatter, ListItem, SectionNode};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FrontmatterSplit {
pub raw: Option<String>,
pub body: String,
}
fn is_fence(line: &str) -> bool {
let line = line.strip_suffix('\r').unwrap_or(line);
match line.strip_prefix("---") {
Some(rest) => rest.chars().all(|c| c == ' ' || c == '\t'),
None => false,
}
}
fn opening_fence_end(md: &str) -> Option<usize> {
let eol = md.find('\n')?;
is_fence(&md[..eol]).then_some(eol + 1)
}
pub fn split_frontmatter(source: &str) -> FrontmatterSplit {
let whole = || FrontmatterSplit {
raw: None,
body: source.to_string(),
};
let md = source.strip_prefix('\u{feff}').unwrap_or(source);
let Some(open_end) = opening_fence_end(md) else {
return whole();
};
let mut idx = open_end;
while idx <= md.len() {
let (line, line_term_end) = match md[idx..].find('\n') {
Some(i) => (&md[idx..idx + i], idx + i + 1),
None => (&md[idx..], md.len()),
};
if is_fence(line) {
let mut raw_end = idx;
if raw_end > open_end && md.as_bytes()[raw_end - 1] == b'\n' {
raw_end -= 1;
if raw_end > open_end && md.as_bytes()[raw_end - 1] == b'\r' {
raw_end -= 1;
}
}
return FrontmatterSplit {
raw: Some(md[open_end..raw_end].to_string()),
body: md[line_term_end..].to_string(),
};
}
if line_term_end >= md.len() {
break; }
idx = line_term_end;
}
whole()
}
pub fn parse_document(source: &str) -> DocTree {
let split = split_frontmatter(source);
let prefix_len = source.len() - split.body.len();
let line_offset = source[..prefix_len].bytes().filter(|&b| b == b'\n').count() as u32;
let frontmatter = split.raw.map(|raw| {
let data = serde_yaml::from_str::<serde_yaml::Value>(&raw)
.ok()
.map(yaml_to_json);
Frontmatter {
data,
raw,
pos: SourcePos::at(1, 1),
}
});
let body = split.body.strip_prefix('\u{feff}').unwrap_or(&split.body);
let base = split.body.len() - body.len();
let root = project_body(body, line_offset, base);
DocTree {
frontmatter,
body: split.body,
root,
}
}
fn gfm_options() -> Options<'static> {
let mut options = Options::default();
options.extension.table = true;
options.extension.tasklist = true;
options.extension.strikethrough = true;
options.extension.autolink = true;
options.extension.footnotes = true;
options
}
struct Cx<'s> {
off: u32,
lines: Vec<&'s str>,
starts: Vec<usize>,
base: usize,
len: usize,
}
impl Cx<'_> {
fn offset(&self, line: usize, column: usize) -> usize {
let start = self
.starts
.get(line.saturating_sub(1))
.copied()
.unwrap_or(self.len);
(start + column.saturating_sub(1)).min(self.len)
}
fn range_of<'a>(&self, node: &'a AstNode<'a>) -> std::ops::Range<usize> {
let sp = node.data.borrow().sourcepos;
let start = self.offset(sp.start.line, sp.start.column);
let end = (self.offset(sp.end.line, sp.end.column) + 1)
.min(self.len)
.max(start);
self.base + start..self.base + end
}
}
fn project_body(body: &str, line_offset: u32, base: usize) -> SectionNode {
let arena = Arena::new();
let root = comrak::parse_document(&arena, body, &gfm_options());
let mut starts = vec![0];
starts.extend(body.match_indices('\n').map(|(i, _)| i + 1));
let cx = Cx {
off: line_offset,
lines: body.lines().collect(),
starts,
base,
len: body.len(),
};
build_tree(root, &cx)
}
fn pos_of<'a>(node: &'a AstNode<'a>, off: u32) -> SourcePos {
let sp = node.data.borrow().sourcepos;
SourcePos::at(sp.start.line as u32 + off, sp.start.column as u32)
}
fn flatten_inline<'a>(nodes: impl Iterator<Item = &'a AstNode<'a>>, out: &mut String) {
for n in nodes {
match &n.data.borrow().value {
NodeValue::Text(t) => out.push_str(t),
NodeValue::Code(c) => out.push_str(&c.literal),
NodeValue::SoftBreak => out.push('\n'),
NodeValue::LineBreak => out.push(' '),
NodeValue::HtmlInline(h) => out.push_str(h),
NodeValue::Image(_) => {} _ => flatten_inline(n.children(), out),
}
}
}
fn flatten_to_string<'a>(nodes: impl Iterator<Item = &'a AstNode<'a>>) -> String {
let mut out = String::new();
flatten_inline(nodes, &mut out);
out
}
fn flatten_list_item<'a>(item: &'a AstNode<'a>) -> String {
let parts: Vec<String> = item
.children()
.filter(|c| matches!(c.data.borrow().value, NodeValue::Paragraph))
.map(|c| flatten_to_string(c.children()))
.collect();
js::trim(&parts.join(" ")).to_string()
}
fn project_list<'a>(node: &'a AstNode<'a>, ordered: bool, cx: &Cx) -> BlockNode {
let items = node
.children()
.map(|li| {
let checked = match &li.data.borrow().value {
NodeValue::TaskItem(t) => Some(t.symbol.is_some()),
_ => None,
};
ListItem {
text: flatten_list_item(li),
checked,
pos: pos_of(li, cx.off),
}
})
.collect();
BlockNode::List {
ordered,
items,
anchor: None,
pos: pos_of(node, cx.off),
range: cx.range_of(node),
}
}
fn project_code<'a>(node: &'a AstNode<'a>, cx: &Cx) -> BlockNode {
let (lang, value, fenced) = match &node.data.borrow().value {
NodeValue::CodeBlock(cb) => (
cb.info.split_whitespace().next().map(str::to_string),
cb.literal
.strip_suffix('\n')
.map(|v| v.strip_suffix('\r').unwrap_or(v))
.unwrap_or(&cb.literal)
.to_string(),
cb.fenced,
),
_ => (None, String::new(), true),
};
BlockNode::Code {
lang,
value,
fenced,
anchor: None,
pos: pos_of(node, cx.off),
range: cx.range_of(node),
}
}
fn project_paragraph<'a>(node: &'a AstNode<'a>, cx: &Cx) -> BlockNode {
BlockNode::Paragraph {
text: js::trim(&flatten_to_string(node.children())).to_string(),
anchor: None,
pos: pos_of(node, cx.off),
range: cx.range_of(node),
}
}
fn project_table<'a>(node: &'a AstNode<'a>, cx: &Cx) -> BlockNode {
let mut columns = Vec::new();
let mut rows: Vec<Vec<String>> = Vec::new();
let mut row_pos = Vec::new();
let mut last_row_line = 0usize;
for row in node.children() {
let is_header = matches!(row.data.borrow().value, NodeValue::TableRow(true));
let cells: Vec<String> = row
.children()
.map(|c| js::trim(&flatten_to_string(c.children())).to_string())
.collect();
if is_header && columns.is_empty() {
columns = cells;
} else {
last_row_line = row.data.borrow().sourcepos.start.line;
row_pos.push(pos_of(row, cx.off));
rows.push(cells);
}
}
let mut anchor = None;
if !rows.is_empty()
&& let Some(id) = cx
.lines
.get(last_row_line.wrapping_sub(1))
.and_then(|line| single_cell_anchor(line))
{
anchor = Some(id.to_string());
rows.pop();
row_pos.pop();
}
BlockNode::Table {
columns,
rows,
row_pos,
anchor,
pos: pos_of(node, cx.off),
range: cx.range_of(node),
}
}
fn single_cell_anchor(line: &str) -> Option<&str> {
let cell = js::trim(line);
let cell = cell.strip_prefix('|').unwrap_or(cell);
let cell = cell.strip_suffix('|').unwrap_or(cell);
if cell.contains('|') {
return None;
}
is_standalone_anchor(cell)
}
fn project_block<'a>(node: &'a AstNode<'a>, cx: &Cx) -> Option<BlockNode> {
let value = &node.data.borrow().value;
match value {
NodeValue::Paragraph => Some(project_paragraph(node, cx)),
NodeValue::CodeBlock(_) => Some(project_code(node, cx)),
NodeValue::List(l) => Some(project_list(node, l.list_type == ListType::Ordered, cx)),
NodeValue::Table(_) => Some(project_table(node, cx)),
_ => None,
}
}
fn bind_trailing_anchor(block: &mut BlockNode) {
match block {
BlockNode::Paragraph { text, anchor, .. } => {
if let Some(t) = extract_trailing_anchor(text) {
*text = t.rest;
*anchor = Some(t.id);
}
}
BlockNode::List { items, anchor, .. } => {
for item in items {
if let Some(t) = extract_trailing_anchor(&item.text) {
item.text = t.rest;
*anchor = Some(t.id);
break;
}
}
}
_ => {}
}
}
fn bind_anchor(sec: &mut SectionNode, id: &str) {
match sec.blocks.last_mut() {
Some(last) if last.anchor().is_none() => *last.anchor_mut() = Some(id.to_string()),
_ => sec.anchors.push(id.to_string()),
}
}
fn attach_content_node<'a>(sec: &mut SectionNode, node: &'a AstNode<'a>, cx: &Cx) {
let Some(mut block) = project_block(node, cx) else {
return; };
if let BlockNode::Paragraph { text, .. } = &block
&& let Some(id) = is_standalone_anchor(text)
{
let id = id.to_string();
bind_anchor(sec, &id);
return;
}
bind_trailing_anchor(&mut block);
sec.blocks.push(block);
}
fn pop_to_depth(stack: &mut Vec<SectionNode>, depth: u8, end: usize) {
while stack.len() > 1 && stack.last().expect("non-empty stack").depth >= depth {
let mut done = stack.pop().expect("len > 1");
done.range.end = end;
stack.last_mut().expect("root remains").sections.push(done);
}
}
fn build_tree<'a>(root: &'a AstNode<'a>, cx: &Cx) -> SectionNode {
let doc_root = SectionNode::new("", 1, SourcePos::line(0));
let mut title: Option<String> = None;
let mut stack: Vec<SectionNode> = vec![doc_root];
for node in root.children() {
let heading = match &node.data.borrow().value {
NodeValue::Heading(h) => Some(h.level),
_ => None,
};
if let Some(level) = heading {
let name = js::trim(&flatten_to_string(node.children())).to_string();
if level == 1 && title.is_none() {
title = Some(name);
continue;
}
let start = cx.range_of(node).start;
pop_to_depth(&mut stack, level, start);
let mut section = SectionNode::new(name, level, pos_of(node, cx.off));
section.range = start..start;
stack.push(section);
continue;
}
attach_content_node(stack.last_mut().expect("non-empty stack"), node, cx);
}
let end = cx.base + cx.len;
pop_to_depth(&mut stack, 1, end); let mut doc_root = stack.pop().expect("root");
doc_root.range = 0..end;
doc_root.name = title.unwrap_or_default();
doc_root
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn one_section_one_paragraph() {
let tree = parse_document("## Overview\n\nPlain prose.\n");
assert!(tree.frontmatter.is_none());
assert_eq!(tree.root.name, "");
assert_eq!(tree.root.sections.len(), 1);
let sec = &tree.root.sections[0];
assert_eq!(sec.name, "Overview");
assert_eq!(sec.depth, 2);
assert_eq!(sec.pos, SourcePos::at(1, 1));
assert_eq!(
sec.blocks,
vec![BlockNode::Paragraph {
text: "Plain prose.".into(),
anchor: None,
pos: SourcePos::at(3, 1),
range: 13..25,
}]
);
assert_eq!(sec.range, 0..26);
assert_eq!(tree.root.range, 0..26);
}
#[test]
fn leading_h1_is_the_title_not_a_section() {
let tree = parse_document("# Widget notes\n\n## Background\n\nWidgets are small.");
assert_eq!(tree.root.name, "Widget notes");
assert_eq!(tree.root.sections.len(), 1);
assert_eq!(tree.root.sections[0].name, "Background");
assert_eq!(tree.root.sections[0].pos.line, 3);
}
#[test]
fn frontmatter_splits_raw_and_verbatim_body() {
let s = split_frontmatter("---\na: 1\n---\nbody\n");
assert_eq!(s.raw.as_deref(), Some("a: 1"));
assert_eq!(s.body, "body\n");
}
#[test]
fn unclosed_fence_is_not_frontmatter() {
let s = split_frontmatter("---\na: 1\nbody\n");
assert_eq!(s.raw, None);
assert_eq!(s.body, "---\na: 1\nbody\n");
}
#[test]
fn fence_must_be_exactly_three_dashes_at_column_one() {
assert_eq!(split_frontmatter("----\na: 1\n----\nbody\n").raw, None);
assert_eq!(split_frontmatter(" ---\na: 1\n---\nbody\n").raw, None);
assert_eq!(split_frontmatter("\n---\na: 1\n---\nbody\n").raw, None);
assert_eq!(
split_frontmatter("--- \na: 1\n--- \nbody\n").raw.as_deref(),
Some("a: 1")
);
}
#[test]
fn empty_frontmatter_and_eof_closing_fence() {
let s = split_frontmatter("---\n---\nbody\n");
assert_eq!(s.raw.as_deref(), Some(""));
assert_eq!(s.body, "body\n");
let s = split_frontmatter("---\na: 1\n---");
assert_eq!(s.raw.as_deref(), Some("a: 1"));
assert_eq!(s.body, "");
}
#[test]
fn crlf_frontmatter_keeps_inner_line_endings_and_strips_the_fence_terminator() {
let s = split_frontmatter("---\r\na: 1\r\nb: 2\r\n---\r\nbody\r\n");
assert_eq!(s.raw.as_deref(), Some("a: 1\r\nb: 2"));
assert_eq!(s.body, "body\r\n");
}
#[test]
fn dots_do_not_close_a_frontmatter_block() {
assert_eq!(split_frontmatter("---\na: 1\n...\nbody\n").raw, None);
}
#[test]
fn body_positions_are_absolute_document_lines() {
let src = "---\nid: D-0099\nstatus: open/draft\ntitle: Adopt the widget protocol\n---\n\n# Adopt the widget protocol\n\n## Summary\n\nWe will adopt the widget protocol across all services.";
let tree = parse_document(src);
let fm = tree.frontmatter.as_ref().expect("frontmatter present");
assert_eq!(
fm.raw,
"id: D-0099\nstatus: open/draft\ntitle: Adopt the widget protocol"
);
assert_eq!(fm.pos, SourcePos::at(1, 1));
assert_eq!(tree.root.name, "Adopt the widget protocol");
let sec = &tree.root.sections[0];
assert_eq!(sec.name, "Summary");
assert_eq!(sec.pos, SourcePos::at(9, 1));
assert_eq!(sec.blocks[0].pos(), SourcePos::at(11, 1));
}
#[test]
fn table_columns_rows_and_row_lines() {
let src = "## Decision\n\n| # | Component | Resolution |\n| - | --------- | ---------- |\n| 1 | engine | markdown-contract |";
let tree = parse_document(src);
let BlockNode::Table {
columns,
rows,
row_pos,
anchor,
pos,
..
} = &tree.root.sections[0].blocks[0]
else {
panic!("expected a table");
};
assert_eq!(columns, &["#", "Component", "Resolution"]);
assert_eq!(
rows,
&[vec![
"1".to_string(),
"engine".into(),
"markdown-contract".into()
]]
);
assert_eq!(pos.line, 3);
assert_eq!(row_pos[0].line, 5); assert_eq!(*anchor, None);
}
#[test]
fn task_list_items_carry_checked_state_and_lines() {
let src = "## Acceptance criteria\n\n- [ ] first thing\n- [x] second thing\n- plain item\n";
let tree = parse_document(src);
let BlockNode::List { ordered, items, .. } = &tree.root.sections[0].blocks[0] else {
panic!("expected a list");
};
assert!(!ordered);
assert_eq!(
items
.iter()
.map(|i| (i.text.as_str(), i.checked, i.pos.line))
.collect::<Vec<_>>(),
vec![
("first thing", Some(false), 3),
("second thing", Some(true), 4),
("plain item", None, 5)
]
);
}
#[test]
fn fenced_code_is_opaque_and_keeps_its_lang() {
let src = "## Example\n\n```ts\nconst x = 1;\n## not a heading\n```\n";
let tree = parse_document(src);
let sec = &tree.root.sections[0];
assert_eq!(sec.sections.len(), 0); let BlockNode::Code {
lang,
value,
fenced,
pos,
..
} = &sec.blocks[0]
else {
panic!("expected code");
};
assert_eq!(lang.as_deref(), Some("ts"));
assert_eq!(value, "const x = 1;\n## not a heading");
assert!(fenced);
assert_eq!(pos.line, 3);
}
#[test]
fn indented_code_is_not_fenced() {
let tree = parse_document(
"## Example
let x = 1;
",
);
let BlockNode::Code { fenced, pos, .. } = &tree.root.sections[0].blocks[0] else {
panic!("expected code");
};
assert!(!fenced);
assert_eq!(pos.line, 3);
}
#[test]
fn a_second_h1_starts_a_new_top_level_section() {
let tree = parse_document("# T1\n\n## A\n\n# T2\n\n## B\n");
let names: Vec<_> = tree.root.sections.iter().map(|s| s.name.as_str()).collect();
assert_eq!(names, vec!["A", "T2"]);
assert_eq!(tree.root.sections[1].sections[0].name, "B");
}
#[test]
fn a_leading_bom_is_skipped() {
let tree = parse_document("\u{FEFF}---\nstatus: open\n---\n\n## A\n");
let fm = tree.frontmatter.as_ref().expect("frontmatter");
assert_eq!(fm.data, Some(serde_json::json!({"status": "open"})));
assert_eq!(tree.root.sections[0].name, "A");
assert_eq!(tree.root.sections[0].pos.line, 5);
}
#[test]
fn a_two_cell_row_with_an_anchor_is_a_data_row() {
let table = |last: &str| {
let src = format!("## T\n\n| A | B |\n| - | - |\n| 1 | 2 |\n{last}\n");
let tree = parse_document(&src);
let BlockNode::Table { rows, anchor, .. } = &tree.root.sections[0].blocks[0] else {
panic!("expected a table");
};
(rows.len(), anchor.clone())
};
assert_eq!(table("| ^x | |"), (2, None));
assert_eq!(table("| ^x |"), (1, Some("x".to_string())));
}
#[test]
fn paragraph_trailing_anchor_line_binds_to_the_paragraph() {
let src = "## Summary\n\nProse line one.\n^summary\n";
let tree = parse_document(src);
let block = &tree.root.sections[0].blocks[0];
assert_eq!(block.anchor(), Some("summary"));
let BlockNode::Paragraph { text, .. } = block else {
panic!()
};
assert_eq!(text, "Prose line one.");
}
#[test]
fn standalone_anchor_paragraph_binds_to_the_preceding_block() {
let src = "## Decision\n\n| A | B |\n| - | - |\n| 1 | 2 |\n\n^components\n";
let tree = parse_document(src);
assert_eq!(tree.root.sections[0].blocks[0].anchor(), Some("components"));
}
#[test]
fn table_absorbed_anchor_row_is_lifted_out() {
let src = "## Decision\n\n| # | Component | Resolution |\n| - | --------- | ---------- |\n| 1 | Projection | mdast → DocTree |\n^components\n";
let tree = parse_document(src);
let BlockNode::Table { rows, anchor, .. } = &tree.root.sections[0].blocks[0] else {
panic!("expected a table");
};
assert_eq!(anchor.as_deref(), Some("components"));
assert_eq!(rows.len(), 1); }
#[test]
fn standalone_anchor_with_no_block_is_section_level() {
let src = "## Summary\n\n^orphan\n";
let tree = parse_document(src);
assert_eq!(tree.root.sections[0].anchors, vec!["orphan".to_string()]);
assert!(tree.root.sections[0].blocks.is_empty());
}
#[test]
fn skipped_level_attaches_to_nearest_ancestor_with_true_depth() {
let src = "## Decision\n\nProse.\n\n#### Components\n\nDeep.\n";
let tree = parse_document(src);
let decision = &tree.root.sections[0];
assert_eq!(decision.sections.len(), 1);
assert_eq!(decision.sections[0].depth, 4); assert_eq!(decision.sections[0].pos.line, 5);
}
#[test]
fn blockquote_content_is_not_hoisted() {
let src = "## Notes\n\n> | A |\n> | - |\n> | 1 |\n";
let tree = parse_document(src);
assert!(tree.root.sections[0].blocks.is_empty()); }
#[test]
fn multiline_paragraph_flattens_with_soft_breaks_as_newlines() {
let src = "## Summary\n\nline one\nline two\n";
let tree = parse_document(src);
let BlockNode::Paragraph { text, .. } = &tree.root.sections[0].blocks[0] else {
panic!()
};
assert_eq!(text, "line one\nline two");
}
#[test]
fn wikilinks_survive_projection_intact() {
let src = "## Refs\n\nsee [[D-0002]] and ![[Diagram]]\n";
let tree = parse_document(src);
let BlockNode::Paragraph { text, .. } = &tree.root.sections[0].blocks[0] else {
panic!()
};
assert_eq!(text, "see [[D-0002]] and ![[Diagram]]");
}
#[test]
fn section_and_block_ranges_are_body_bytes() {
let doc = "---\na: 1\n---\n# T\n\n## A\n\npara\n\n### A1\n\n- x\n- y\n\n## B\n\n```\n## not\n```\n";
let tree = parse_document(doc);
let body = tree.body.as_str();
let a = &tree.root.sections[0];
assert_eq!(
&body[a.range.clone()],
"## A\n\npara\n\n### A1\n\n- x\n- y\n\n"
);
assert_eq!(&body[a.blocks[0].range()], "para");
let a1 = &a.sections[0];
assert_eq!(&body[a1.range.clone()], "### A1\n\n- x\n- y\n\n");
assert_eq!(&body[a1.blocks[0].range()], "- x\n- y");
let b = &tree.root.sections[1];
assert_eq!(&body[b.range.clone()], "## B\n\n```\n## not\n```\n");
assert_eq!(&body[b.blocks[0].range()], "```\n## not\n```");
assert_eq!(tree.root.range, 0..body.len());
}
#[test]
fn ranges_count_a_leading_bom() {
let tree = parse_document("\u{feff}## A\n\npara\n");
let sec = &tree.root.sections[0];
assert_eq!(&tree.body[sec.range.clone()], "## A\n\npara\n");
assert_eq!(&tree.body[sec.blocks[0].range()], "para");
}
#[test]
fn block_by_id_finds_a_block_in_any_section() {
let tree = parse_document("## A\n\n### B\n\nfound ^here\n");
let block = tree.block_by_id("here").expect("bound block");
assert_eq!(&tree.body[block.range()], "found ^here");
assert!(tree.block_by_id("nowhere").is_none());
assert_eq!(&tree.body[tree.block_range_by_id("here").unwrap()], "found");
}
#[test]
fn heading_selection_on_the_tree_counts_the_title_h1() {
let tree = parse_document("# T\n\n## A\n\na\n\n## B\n");
let r = tree.section_by_heading_path(&["T", "A"]).unwrap();
assert_eq!(&tree.body[r], "## A\n\na");
let r = tree.section_by_selector("T#B").unwrap();
assert_eq!(&tree.body[r], "## B");
assert!(tree.section_by_heading_path(&["A", "B"]).is_none());
}
}