use std::fmt::Write as _;
use std::ops::Range;
use std::path::{Path, PathBuf};
use pulldown_cmark::{
Alignment, BlockQuoteKind, CodeBlockKind, Event, Parser, Tag, TagEnd, TextMergeWithOffset,
};
use serde::Serialize;
use crate::markdown::{LineIndex, markdown_options, section_rule_attrs, strip_bom};
const UPDATE_ENV: &str = "MBR_UPDATE_OUTLINES";
#[derive(Debug, Clone, PartialEq, Serialize)]
struct OutlineNode {
depth: usize,
kind: &'static str,
lines: [u32; 2],
#[serde(skip_serializing_if = "Option::is_none")]
level: Option<u8>,
#[serde(skip_serializing_if = "Option::is_none")]
id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
classes: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
alert: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
ordered: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
start: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
tight: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
task: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
fenced: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
lang: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
align: Option<Vec<&'static str>>,
#[serde(skip_serializing_if = "Option::is_none")]
rows: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
label: Option<String>,
}
impl OutlineNode {
fn new(depth: usize, kind: &'static str) -> Self {
Self {
depth,
kind,
lines: [0, 0],
level: None,
id: None,
classes: None,
alert: None,
ordered: None,
start: None,
tight: None,
task: None,
fenced: None,
lang: None,
align: None,
rows: None,
label: None,
}
}
}
struct Open {
node: Option<usize>,
start: usize,
wraps_inline: bool,
}
fn outline(source: &str) -> Vec<OutlineNode> {
let body = strip_bom(source);
let shift = source.len() - body.len();
let lines = LineIndex::build(source);
let span = |range: Range<usize>| -> [u32; 2] { trimmed_lines(source, &lines, range) };
let mut nodes: Vec<OutlineNode> = Vec::new();
let mut stack: Vec<Open> = Vec::new();
let mut synthetic: Option<(usize, Range<usize>)> = None;
let mut paragraph: Option<(usize, usize, bool)> = None;
let mut at_item_start = false;
let parser =
TextMergeWithOffset::new(Parser::new_ext(body, markdown_options()).into_offset_iter());
for (event, range) in parser {
let range = range.start + shift..range.end + shift;
let depth = stack.iter().filter(|open| open.node.is_some()).count();
let was_at_item_start = at_item_start;
at_item_start = matches!(event, Event::Start(Tag::Item))
|| (at_item_start && matches!(event, Event::Start(Tag::Paragraph)));
let is_block = match &event {
Event::Start(tag) => is_block_tag(tag),
Event::End(tag) => is_block_tag_end(tag),
Event::Rule => true,
_ => false,
};
if !is_block {
let task = match &event {
Event::TaskListMarker(done) => Some(if *done { "x" } else { " " }),
Event::Text(text) if was_at_item_start => extended_marker(text),
_ => None,
};
if let Some(task) = task
&& let Some(item) = innermost(&nodes, &stack, "item")
{
nodes[item].task = Some(task);
}
if let Some((_, count, matched)) = paragraph.as_mut() {
if *count == 0 {
*matched =
matches!(&event, Event::Text(text) if section_rule_attrs(text).is_some());
}
*count += 1;
}
if stack.last().is_some_and(|open| open.wraps_inline) {
match synthetic.as_mut() {
Some((_, covered)) => covered.end = range.end,
None => {
nodes.push(OutlineNode::new(depth, "paragraph"));
synthetic = Some((nodes.len() - 1, range));
}
}
}
continue;
}
if let Some((index, covered)) = synthetic.take() {
nodes[index].lines = span(covered);
}
match event {
Event::Start(tag) => {
if matches!(tag, Tag::Paragraph) {
mark_loose(&mut nodes, &stack);
}
if matches!(tag, Tag::TableHead | Tag::TableRow)
&& let Some(table) = innermost(&nodes, &stack, "table")
&& let Some(rows) = nodes[table].rows.as_mut()
{
*rows += 1;
}
let index = start_node(&tag, depth).map(|node| {
nodes.push(node);
nodes.len() - 1
});
if matches!(tag, Tag::Paragraph) {
paragraph = index.map(|index| (index, 0, false));
}
stack.push(Open {
node: index,
start: range.start,
wraps_inline: matches!(tag, Tag::Item | Tag::DefinitionListDefinition),
});
}
Event::End(tag) => {
let Some(open) = stack.pop() else { continue };
if let Some(index) = open.node {
nodes[index].lines = span(open.start..range.end);
}
if matches!(tag, TagEnd::Paragraph)
&& let Some((index, 1, true)) = paragraph
{
nodes[index].kind = "section_break";
}
paragraph = None;
}
Event::Rule => {
let mut node = OutlineNode::new(depth, "rule");
node.lines = span(range);
nodes.push(node);
}
_ => {}
}
}
nodes
}
fn innermost(nodes: &[OutlineNode], stack: &[Open], kind: &str) -> Option<usize> {
stack
.iter()
.rev()
.filter_map(|open| open.node)
.find(|&index| nodes[index].kind == kind)
}
fn trimmed_lines(source: &str, lines: &LineIndex, range: Range<usize>) -> [u32; 2] {
let text = &source[range.clone()];
let lead = text.len() - text.trim_start().len();
let content = text.trim();
if content.is_empty() {
let line = lines.line_of(range.start);
return [line, line];
}
let first = range.start + lead;
[
lines.line_of(first),
lines.line_of(first + content.len() - 1),
]
}
fn extended_marker(text: &str) -> Option<&'static str> {
crate::markdown::split_extended_marker(text)?;
Some(if text.starts_with("[-]") { "-" } else { ">" })
}
fn mark_loose(nodes: &mut [OutlineNode], stack: &[Open]) {
let mut nodes_up = stack.iter().rev().filter_map(|open| open.node);
if let Some(parent) = nodes_up.next()
&& matches!(nodes[parent].kind, "item" | "dd")
&& let Some(list) = nodes_up.next()
{
nodes[list].tight = Some(false);
}
}
fn is_block_tag(tag: &Tag<'_>) -> bool {
!matches!(
tag,
Tag::Emphasis
| Tag::Strong
| Tag::Strikethrough
| Tag::Superscript
| Tag::Subscript
| Tag::Link { .. }
| Tag::Image { .. }
)
}
fn is_block_tag_end(tag: &TagEnd) -> bool {
!matches!(
tag,
TagEnd::Emphasis
| TagEnd::Strong
| TagEnd::Strikethrough
| TagEnd::Superscript
| TagEnd::Subscript
| TagEnd::Link
| TagEnd::Image
)
}
fn start_node(tag: &Tag<'_>, depth: usize) -> Option<OutlineNode> {
let node = |kind| OutlineNode::new(depth, kind);
Some(match tag {
Tag::Paragraph => node("paragraph"),
Tag::Heading {
level, id, classes, ..
} => OutlineNode {
level: Some(*level as u8),
id: id.as_ref().map(|id| id.to_string()),
classes: (!classes.is_empty()).then(|| classes.iter().map(|c| c.to_string()).collect()),
..node("heading")
},
Tag::BlockQuote(kind) => OutlineNode {
alert: kind.map(alert_name),
..node("blockquote")
},
Tag::CodeBlock(kind) => OutlineNode {
fenced: Some(matches!(kind, CodeBlockKind::Fenced(_))),
lang: match kind {
CodeBlockKind::Fenced(info) => info.split_whitespace().next().map(str::to_string),
CodeBlockKind::Indented => None,
},
..node("code")
},
Tag::HtmlBlock => node("html"),
Tag::List(start) => OutlineNode {
ordered: Some(start.is_some()),
start: *start,
tight: Some(true),
..node("list")
},
Tag::Item => node("item"),
Tag::FootnoteDefinition(label) => OutlineNode {
label: Some(label.to_string()),
..node("footnote")
},
Tag::DefinitionList => OutlineNode {
tight: Some(true),
..node("dl")
},
Tag::DefinitionListTitle => node("dt"),
Tag::DefinitionListDefinition => node("dd"),
Tag::Table(alignments) => OutlineNode {
align: Some(alignments.iter().map(alignment_name).collect()),
rows: Some(0),
..node("table")
},
Tag::TableHead
| Tag::TableRow
| Tag::TableCell
| Tag::MetadataBlock(_)
| Tag::Emphasis
| Tag::Strong
| Tag::Strikethrough
| Tag::Superscript
| Tag::Subscript
| Tag::Link { .. }
| Tag::Image { .. } => return None,
})
}
fn alert_name(kind: BlockQuoteKind) -> &'static str {
match kind {
BlockQuoteKind::Note => "note",
BlockQuoteKind::Tip => "tip",
BlockQuoteKind::Important => "important",
BlockQuoteKind::Warning => "warning",
BlockQuoteKind::Caution => "caution",
}
}
fn alignment_name(alignment: &Alignment) -> &'static str {
match alignment {
Alignment::None => "none",
Alignment::Left => "left",
Alignment::Center => "center",
Alignment::Right => "right",
}
}
fn render_outline(nodes: &[OutlineNode]) -> String {
let mut out = String::from("[\n");
for (i, node) in nodes.iter().enumerate() {
let json = serde_json::to_string(node).expect("outline nodes always serialize");
let sep = if i + 1 < nodes.len() { "," } else { "" };
let _ = writeln!(out, " {json}{sep}");
}
out.push_str("]\n");
out
}
fn fixtures_dir() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/fixtures/editor-roundtrip")
}
#[test]
fn renderer_outlines_match_goldens() {
let dir = fixtures_dir();
let updating = std::env::var_os(UPDATE_ENV).is_some();
let mut fixtures: Vec<PathBuf> = std::fs::read_dir(&dir)
.unwrap_or_else(|e| panic!("cannot read {}: {e}", dir.display()))
.filter_map(|entry| entry.ok().map(|e| e.path()))
.filter(|path| path.extension().is_some_and(|ext| ext == "md"))
.collect();
fixtures.sort();
assert!(!fixtures.is_empty(), "no fixtures in {}", dir.display());
let mut failures = Vec::new();
for fixture in &fixtures {
let source = std::fs::read_to_string(fixture).expect("fixture is UTF-8");
let actual = render_outline(&outline(&source));
let golden = fixture.with_extension("outline.json");
if updating {
std::fs::write(&golden, &actual).expect("write golden");
continue;
}
match std::fs::read_to_string(&golden) {
Ok(expected) if expected == actual => {}
Ok(expected) => failures.push(format!(
"{}:\n--- golden\n{expected}--- actual\n{actual}",
fixture.display()
)),
Err(e) => failures.push(format!(
"missing golden {} ({e}); regenerate with {UPDATE_ENV}=1",
golden.display()
)),
}
}
let orphans: Vec<String> = std::fs::read_dir(&dir)
.expect("fixtures dir")
.filter_map(|entry| entry.ok().map(|e| e.path()))
.filter(|path| path.to_string_lossy().ends_with(".outline.json"))
.filter(|path| {
let name = path.to_string_lossy();
!Path::new(&name.replace(".outline.json", ".md")).exists()
})
.map(|path| path.display().to_string())
.collect();
assert!(
orphans.is_empty(),
"outline goldens without a fixture: {orphans:?}"
);
assert!(
failures.is_empty(),
"{} outline(s) differ from their goldens. If the renderer change is \
intended, regenerate with `{UPDATE_ENV}=1 cargo test -p mbr-core editor_parity` \
and review the diff.\n\n{}",
failures.len(),
failures.join("\n")
);
}
fn shape(source: &str) -> Vec<(usize, &'static str, u32, u32)> {
outline(source)
.into_iter()
.map(|node| (node.depth, node.kind, node.lines[0], node.lines[1]))
.collect()
}
#[test]
fn tight_items_get_a_synthetic_paragraph_and_loose_ones_a_real_one() {
let tight = outline("- a\n- b\n");
let loose = outline("- a\n\n- b\n");
assert_eq!(tight[0].tight, Some(true));
assert_eq!(loose[0].tight, Some(false));
assert_eq!(
shape("- a\n- b\n"),
[
(0, "list", 1, 2),
(1, "item", 1, 1),
(2, "paragraph", 1, 1),
(1, "item", 2, 2),
(2, "paragraph", 2, 2),
]
);
assert_eq!(shape("- a\n\n- b\n")[3], (1, "item", 3, 3));
}
#[test]
fn every_task_marker_is_reported_in_tight_and_loose_items() {
let markers = |source: &str| -> Vec<Option<&'static str>> {
outline(source)
.into_iter()
.filter(|node| node.kind == "item")
.map(|node| node.task)
.collect()
};
let tight = "- [ ] a\n- [x] b\n- [X] c\n- [-] d\n- [>] e\n- [-]x f\n- g\n";
let expected = [
Some(" "),
Some("x"),
Some("x"),
Some("-"),
Some(">"),
None,
None,
];
assert_eq!(markers(tight), expected);
assert_eq!(markers(&tight.replace('\n', "\n\n")), expected);
}
#[test]
fn section_rule_needs_a_paragraph_of_its_own() {
assert_eq!(shape("a\n\n--- {#x}\n\nb\n")[1], (0, "section_break", 3, 3));
assert_eq!(shape("a\n--- {#x}\n"), [(0, "paragraph", 1, 2)]);
assert_eq!(shape("a\n\n---\n")[1], (0, "rule", 3, 3));
assert_eq!(shape("--- {#x} tail\n")[0].1, "paragraph");
}
#[test]
fn section_rule_spellings_the_editor_mirrors() {
let kind = |source: &str| shape(source)[0].1;
for source in [
"--- {.a}\n",
"\u{2014} {.a}\n",
"— {.a}\n",
"--- {}\n",
"--- {data-background-iframe=https://example.com/x}\n",
] {
assert_eq!(kind(source), "section_break", "{source:?}");
}
for source in [
"---{x}\n",
"--- {x}\n",
"---- {x}\n",
"-- {x}\n",
"\\--- {x}\n",
"--- {a *b*}\n",
"--- {a `b`}\n",
] {
assert_eq!(kind(source), "paragraph", "{source:?}");
}
assert_eq!(shape("> a\n>\n> --- {.x}\n")[2], (1, "section_break", 3, 3));
}
#[test]
fn wikilink_edges_the_editor_mirrors() {
use pulldown_cmark::LinkType;
let links = |source: &str| -> Vec<(bool, String)> {
Parser::new_ext(source, markdown_options())
.filter_map(|event| match event {
Event::Start(Tag::Link {
link_type: LinkType::WikiLink { .. },
dest_url,
..
}) => Some((false, dest_url.to_string())),
Event::Start(Tag::Image {
link_type: LinkType::WikiLink { .. },
dest_url,
..
}) => Some((true, dest_url.to_string())),
_ => None,
})
.collect()
};
let one = |embed: bool, dest: &str| vec![(embed, dest.to_string())];
assert_eq!(links("[[Note]]"), one(false, "Note"));
assert_eq!(links("[[Note|the alias]]"), one(false, "Note"));
assert_eq!(links("[[Note#Heading]]"), one(false, "Note#Heading"));
assert_eq!(links("[[#Local]]"), one(false, "#Local"));
assert_eq!(links("![[diagram.png]]"), one(true, "diagram.png"));
assert_eq!(links("[[a#b|c|d]]"), one(false, "a#b"));
assert_eq!(links("[[a\\|b]]"), one(false, "a\\"));
assert_eq!(links("[[[a]]"), one(false, "a"));
assert_eq!(links("[[x|]]"), one(false, "x"));
assert_eq!(links("[[a]b]]"), one(false, "a]b"));
assert_eq!(links("[[a\nb]]"), one(false, "a\nb"));
assert_eq!(links("[[a[b]]"), one(false, "a[b"));
assert_eq!(links("Wow\\![[x]]"), one(false, "x"));
for source in ["[[]]", "[[|x]]", "`[[x]]`"] {
assert_eq!(links(source), [], "{source:?}");
}
}
#[test]
fn heading_attribute_edges_the_editor_mirrors() {
let attrs = |source: &str| {
let node = outline(source).remove(0);
(node.id, node.classes)
};
assert_eq!(
attrs("# Nest {#a {.b}\n"),
(None, Some(vec!["b".to_string()]))
);
assert_eq!(
attrs("# Both {#a .b .c #d}\n"),
(
Some("d".to_string()),
Some(vec!["b".to_string(), "c".to_string()])
)
);
assert_eq!(
attrs("# Pairs {key=v # . .ok}\n").1,
Some(vec!["ok".to_string()])
);
assert_eq!(attrs("# Esc {#a\\b}\n"), (None, None));
assert_eq!(attrs("Setext {#s}\n===\n").0, Some("s".to_string()));
}
#[test]
fn definition_list_nesting_and_frontmatter_lines() {
let source = "---\ntitle: x\n---\nTerm\n: one\n * aid\n: two\n";
assert_eq!(
shape(source),
[
(0, "dl", 4, 7),
(1, "dt", 4, 4),
(1, "dd", 5, 6),
(2, "paragraph", 5, 5),
(2, "list", 6, 6),
(3, "item", 6, 6),
(4, "paragraph", 6, 6),
(1, "dd", 7, 7),
(2, "paragraph", 7, 7),
]
);
}
#[test]
fn bom_and_crlf_do_not_shift_lines() {
assert_eq!(
shape("\u{feff}# A\r\n\r\nb\r\nc\r\n"),
[(0, "heading", 1, 1), (0, "paragraph", 3, 4),]
);
}
#[test]
fn golden_text_is_one_node_per_line() {
let text = render_outline(&outline("# A {#a}\n\n| x |\n|:-:|\n| 1 |\n"));
assert_eq!(
text,
"[\n {\"depth\":0,\"kind\":\"heading\",\"lines\":[1,1],\"level\":1,\"id\":\"a\"},\n \
{\"depth\":0,\"kind\":\"table\",\"lines\":[3,5],\"align\":[\"center\"],\"rows\":2}\n]\n"
);
}