#![allow(dead_code)]
use std::ops::Range;
use pulldown_cmark::{Alignment, CodeBlockKind, Event, Options, Parser, Tag, TagEnd};
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Doc<'a> {
pub events: Vec<(Event<'a>, Range<usize>)>,
pub blocks: Vec<Block>,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Block {
pub kind: BlockKind,
pub src: Range<usize>,
pub children: Vec<Block>,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum BlockKind {
Heading {
level: u8,
inline: Range<usize>,
id: Option<String>,
classes: Vec<String>,
attrs: Vec<(String, Option<String>)>,
},
Paragraph {
inline: Range<usize>,
},
CodeBlock {
lang: Option<String>,
fenced: bool,
body_spans: Vec<Range<usize>>,
},
List {
ordered: bool,
start: Option<u64>,
},
ListItem { task: Option<Task> },
Quote {
alert: Option<AlertKind>,
alert_title: String,
},
Table {
aligns: Vec<Alignment>,
rows: Vec<Vec<Range<usize>>>,
},
Html { tag: Option<String> },
Details {
open_attr: bool,
summary: String,
glued_body: Option<Range<usize>>,
},
ThematicBreak,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Task {
pub state: char,
pub state_at: usize,
}
pub(crate) use super::AlertKind;
struct Walker<'a> {
iter: std::iter::Peekable<pulldown_cmark::OffsetIter<'a>>,
events: Vec<(Event<'a>, Range<usize>)>,
}
impl<'a> Walker<'a> {
fn peek(&mut self) -> Option<&(Event<'a>, Range<usize>)> {
self.iter.peek()
}
fn next(&mut self) -> Option<(Event<'a>, Range<usize>)> {
let item = self.iter.next()?;
self.events.push(item.clone());
Some(item)
}
fn pos(&self) -> usize {
self.events.len()
}
}
impl<'a> Doc<'a> {
pub(crate) fn parse(src: &'a str) -> Doc<'a> {
let mut w = Walker {
iter: Parser::new_ext(src, parse_options())
.into_offset_iter()
.peekable(),
events: Vec::new(),
};
let blocks = parse_blocks(&mut w, src);
Doc {
events: w.events,
blocks,
}
}
}
pub(crate) fn parse_options() -> Options {
let mut o = super::tui_markdown_parse_options();
o.insert(Options::ENABLE_TABLES);
o
}
fn parse_blocks(events: &mut Walker, src: &str) -> Vec<Block> {
fold_details(parse_blocks_raw(events, src), src)
}
fn parse_blocks_raw(events: &mut Walker, src: &str) -> Vec<Block> {
let mut out = Vec::new();
while let Some((ev, range)) = events.next() {
match ev {
Event::End(_) => return out,
Event::Start(tag) if is_inline_start_tag(&tag) => {
let inline_start = events.pos() - 1;
let run = collect_stray_inline_run((Event::Start(tag), range), events);
let inline = inline_start..events.pos();
out.push(leaf(BlockKind::Paragraph { inline }, run));
}
Event::Start(tag) => {
if let Some(block) = parse_container(tag, range, events, src) {
out.push(block);
}
}
Event::Rule => out.push(Block {
kind: BlockKind::ThematicBreak,
src: range,
children: Vec::new(),
}),
other if is_stray_inline_leaf(&other) => {
let inline_start = events.pos() - 1;
let run = collect_stray_inline_run((other, range), events);
let inline = inline_start..events.pos();
out.push(leaf(BlockKind::Paragraph { inline }, run));
}
_ => {}
}
}
out
}
fn fold_details(blocks: Vec<Block>, src: &str) -> Vec<Block> {
let mut out = Vec::with_capacity(blocks.len());
let mut rest: std::collections::VecDeque<Block> = blocks.into();
while let Some(b) = rest.pop_front() {
let open_attr = match &b.kind {
BlockKind::Html { tag: Some(t) } if t == "details" => {
super::details_open_tag(first_source_line(src, &b.src))
}
_ => None,
};
let Some(open_attr) = open_attr else {
out.push(b);
continue;
};
if rest.is_empty() || html_leaf_contains_its_own_close(src, &b) {
if let Some((summary, glued_body)) = glued_details_fold(src, &b) {
out.push(Block {
kind: BlockKind::Details {
open_attr,
summary,
glued_body: Some(glued_body),
},
src: b.src.clone(),
children: Vec::new(),
});
continue;
}
out.push(b);
continue;
}
let close_pos = rest.iter().position(|c| is_details_close_block(c, src));
let body_start = line_after(src, b.src.start);
let (mut children, block_end, summary_end) = match close_pos {
Some(pos) => {
let mut children = Vec::with_capacity(pos);
for _ in 0..pos {
children.push(rest.pop_front().expect("position() found this many ahead"));
}
let close = rest
.pop_front()
.expect("position() found a close at this offset");
(children, close.src.end, close.src.start)
}
None => {
let children: Vec<Block> = rest.drain(..).collect();
let end = children.last().map_or(b.src.end, |c| c.src.end);
(children, end, end)
}
};
let inner_start = body_start.min(summary_end);
let inner = &src[inner_start..summary_end];
let (summary, _body) = super::extract_summary_body(inner);
let real_body_start = match super::summary_tag_end(inner) {
Some(off) => line_after(src, inner_start + off),
None => inner_start,
}
.min(b.src.end);
if real_body_start < b.src.end && !src[real_body_start..b.src.end].trim().is_empty() {
children.insert(
0,
Block {
kind: BlockKind::Html { tag: None },
src: real_body_start..b.src.end,
children: Vec::new(),
},
);
}
out.push(Block {
kind: BlockKind::Details {
open_attr,
summary,
glued_body: None,
},
src: b.src.start..block_end,
children,
});
}
out
}
fn first_source_line<'a>(src: &'a str, r: &Range<usize>) -> &'a str {
src[r.clone()].lines().next().unwrap_or("").trim_end()
}
fn is_details_close_block(c: &Block, src: &str) -> bool {
matches!(&c.kind, BlockKind::Html { tag: Some(t) } if t == "details")
&& super::is_details_close(first_source_line(src, &c.src))
}
fn html_leaf_contains_its_own_close(src: &str, b: &Block) -> bool {
let text = &src[b.src.clone()];
text.lines()
.skip(1)
.any(|line| super::is_details_close(line.trim_end()))
}
fn glued_details_fold(src: &str, b: &Block) -> Option<(String, Range<usize>)> {
let body_start = line_after(src, b.src.start);
let scan = &src[body_start..b.src.end];
let mut close: Option<(usize, usize)> = None; let mut nested = false;
let mut pos = 0usize; loop {
let rel_end = scan[pos..].find('\n').map_or(scan.len(), |i| pos + i);
let line = &scan[pos..rel_end];
if super::is_details_close(line) {
close = Some((body_start + pos, body_start + rel_end));
break;
}
if super::details_open_tag(line).is_some() {
nested = true;
}
if rel_end >= scan.len() {
break; }
pos = rel_end + 1;
}
let (close_start, close_end) = close?;
if nested {
return None;
}
if !src[close_end..b.src.end].trim().is_empty() {
return None;
}
let inner = &src[body_start..close_start];
let (summary, _body) = super::extract_summary_body(inner);
let body_off = super::summary_tag_end(inner).unwrap_or(0);
Some((summary, (body_start + body_off)..close_start))
}
fn line_after(src: &str, start: usize) -> usize {
match src[start..].find('\n') {
Some(off) => start + off + 1,
None => src.len(),
}
}
fn parse_container(tag: Tag, range: Range<usize>, events: &mut Walker, src: &str) -> Option<Block> {
match tag {
Tag::Paragraph => {
let inline_start = events.pos();
skip_inline_to(events, TagEnd::Paragraph);
let inline = inline_start..events.pos() - 1;
Some(leaf(BlockKind::Paragraph { inline }, range))
}
Tag::Heading {
level,
id,
classes,
attrs,
} => {
let inline_start = events.pos();
skip_inline_to(events, TagEnd::Heading(level));
let inline = inline_start..events.pos() - 1;
Some(leaf(
BlockKind::Heading {
level: level as u8,
inline,
id: id.map(|c| c.into_string()),
classes: classes.into_iter().map(|c| c.into_string()).collect(),
attrs: attrs
.into_iter()
.map(|(k, v)| (k.into_string(), v.map(|v| v.into_string())))
.collect(),
},
range,
))
}
Tag::CodeBlock(kind) => {
let (fenced, lang) = match kind {
CodeBlockKind::Fenced(info) => (
true,
if info.trim().is_empty() {
None
} else {
Some(info.into_string())
},
),
CodeBlockKind::Indented => (false, None),
};
let body_spans = collect_code_body_spans(events);
Some(leaf(
BlockKind::CodeBlock {
lang,
fenced,
body_spans,
},
range,
))
}
Tag::List(start) => {
let children = parse_blocks(events, src); Some(Block {
kind: BlockKind::List {
ordered: start.is_some(),
start,
},
src: range,
children,
})
}
Tag::Item => {
let (task, children) = parse_item_task_and_children(events, src);
Some(Block {
kind: BlockKind::ListItem { task },
src: range,
children,
})
}
Tag::BlockQuote(_) => {
let children = parse_blocks(events, src); let (alert, alert_title) = alert_kind_of(src, &range);
Some(Block {
kind: BlockKind::Quote { alert, alert_title },
src: range,
children,
})
}
Tag::Table(aligns) => {
let rows = collect_table_rows(events);
Some(leaf(BlockKind::Table { aligns, rows }, range))
}
Tag::HtmlBlock => {
let tag = collect_html_tag(events, &range, src);
Some(leaf(BlockKind::Html { tag }, range))
}
other => {
debug_assert!(
is_unmodeled_container_tag(&other),
"parse_container's own exhaustive match and is_unmodeled_container_tag must \
agree on which Tag variants land here — the latter is a second, independent \
reading of the exact same set for md_model_snapshot_tests's completeness cross-\
check (see its own doc comment); a new Tag variant landing in *only one* of the \
two would let that check pass without actually exercising the gap it exists to \
catch."
);
skip_inline_to(events, other.to_end());
None
}
}
}
fn leaf(kind: BlockKind, src: Range<usize>) -> Block {
Block {
kind,
src,
children: Vec::new(),
}
}
pub(crate) fn is_unmodeled_container_tag(tag: &Tag) -> bool {
matches!(
tag,
Tag::FootnoteDefinition(_)
| Tag::DefinitionList
| Tag::DefinitionListTitle
| Tag::DefinitionListDefinition
| Tag::MetadataBlock(_)
)
}
fn is_inline_start_tag(tag: &Tag) -> bool {
matches!(
tag,
Tag::Emphasis
| Tag::Strong
| Tag::Strikethrough
| Tag::Superscript
| Tag::Subscript
| Tag::Link { .. }
| Tag::Image { .. }
)
}
pub(crate) fn is_stray_inline_leaf(ev: &Event) -> bool {
matches!(
ev,
Event::Text(_)
| Event::Code(_)
| Event::InlineMath(_)
| Event::DisplayMath(_)
| Event::Html(_)
| Event::InlineHtml(_)
| Event::FootnoteReference(_)
| Event::SoftBreak
| Event::HardBreak
)
}
fn collect_stray_inline_run(first: (Event<'_>, Range<usize>), events: &mut Walker) -> Range<usize> {
let (first_ev, first_range) = first;
let start = first_range.start;
let mut end = first_range.end;
if let Event::Start(tag) = first_ev {
skip_inline_to(events, tag.to_end());
}
loop {
let continues = match events.peek() {
Some((Event::Start(tag), _)) => is_inline_start_tag(tag),
Some((ev, _)) => is_stray_inline_leaf(ev),
None => false,
};
if !continues {
return start..end;
}
let (ev, range) = events.next().expect("peeked Some above");
end = range.end;
if let Event::Start(tag) = ev {
skip_inline_to(events, tag.to_end());
}
}
}
fn skip_inline_to(events: &mut Walker, end: TagEnd) {
while let Some((ev, _)) = events.next() {
match ev {
Event::Start(t) => skip_inline_to(events, t.to_end()),
Event::End(e) if e == end => return,
_ => {}
}
}
}
fn parse_item_task_and_children(events: &mut Walker, src: &str) -> (Option<Task>, Vec<Block>) {
if matches!(events.peek(), Some((Event::Start(Tag::Paragraph), _))) {
let (_, para_range) = events.next().expect("peeked Some above");
let task = take_task_marker(events, src);
let inline_start = events.pos();
skip_inline_to(events, TagEnd::Paragraph);
let inline = inline_start..events.pos() - 1;
let mut children = vec![leaf(BlockKind::Paragraph { inline }, para_range)];
children.extend(parse_blocks(events, src)); return (task, children);
}
let task = take_task_marker(events, src);
let children = parse_blocks(events, src); (task, children)
}
fn take_task_marker(events: &mut Walker, src: &str) -> Option<Task> {
if !matches!(events.peek(), Some((Event::TaskListMarker(_), _))) {
return None;
}
let (_, range) = events.next().expect("peeked Some above");
let state_at = range.start + 1;
let state = src[state_at..]
.chars()
.next()
.expect("a TaskListMarker range always has a state byte following '['");
Some(Task { state, state_at })
}
fn collect_code_body_spans(events: &mut Walker) -> Vec<Range<usize>> {
let mut spans = Vec::new();
while let Some((ev, range)) = events.next() {
match ev {
Event::Text(_) => spans.push(range),
Event::End(TagEnd::CodeBlock) => break,
Event::Start(t) => skip_inline_to(events, t.to_end()),
_ => {}
}
}
spans
}
pub(crate) fn code_body_text(body_spans: &[Range<usize>], src: &str) -> String {
let mut s = String::new();
for r in body_spans {
s.push_str(&src[r.clone()]);
}
match s.strip_suffix('\n') {
Some(t) => t.to_string(),
None => s,
}
}
fn collect_table_rows(events: &mut Walker) -> Vec<Vec<Range<usize>>> {
let mut rows = Vec::new();
while let Some((ev, _)) = events.next() {
match ev {
Event::Start(Tag::TableHead) => rows.push(collect_row_cells(events, TagEnd::TableHead)),
Event::Start(Tag::TableRow) => rows.push(collect_row_cells(events, TagEnd::TableRow)),
Event::End(TagEnd::Table) => break,
Event::Start(t) => skip_inline_to(events, t.to_end()), _ => {}
}
}
rows
}
fn collect_row_cells(events: &mut Walker, end: TagEnd) -> Vec<Range<usize>> {
let mut cells = Vec::new();
while let Some((ev, range)) = events.next() {
match ev {
Event::Start(Tag::TableCell) => {
skip_inline_to(events, TagEnd::TableCell);
cells.push(range);
}
Event::End(e) if e == end => break,
Event::Start(t) => skip_inline_to(events, t.to_end()), _ => {}
}
}
cells
}
fn collect_html_tag(events: &mut Walker, block_range: &Range<usize>, src: &str) -> Option<String> {
while let Some((ev, _)) = events.next() {
match ev {
Event::End(TagEnd::HtmlBlock) => break,
Event::Start(t) => skip_inline_to(events, t.to_end()), _ => {}
}
}
let first_line = src[block_range.clone()]
.lines()
.next()
.unwrap_or("")
.trim_end();
if super::details_open_tag(first_line).is_some() || super::is_details_close(first_line) {
return Some("details".to_string());
}
html_tag_name(first_line)
}
fn html_tag_name(line: &str) -> Option<String> {
let rest = line.trim_start().strip_prefix('<')?;
let rest = rest.strip_prefix('/').unwrap_or(rest);
if rest.starts_with(['!', '?']) {
return None;
}
let name: String = rest
.chars()
.take_while(|c| c.is_ascii_alphanumeric() || *c == '-')
.collect();
if name.is_empty() {
None
} else {
Some(name.to_ascii_lowercase())
}
}
fn alert_kind_of(src: &str, range: &Range<usize>) -> (Option<AlertKind>, String) {
let first_line = src[range.clone()].lines().next().unwrap_or("");
match super::parse_alert_header(first_line) {
Some((kind, title)) => (Some(kind), title),
None => (None, String::new()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn quoted_paragraph_src_range_excludes_the_first_lines_marker_but_not_a_continuation_lines() {
let src = "> hello $x$ world\n> line2\n";
let doc = Doc::parse(src);
let BlockKind::Quote { .. } = &doc.blocks[0].kind else {
panic!("expected a top-level quote: {:?}", doc.blocks[0].kind);
};
assert_eq!(doc.blocks[0].children.len(), 1);
let child = &doc.blocks[0].children[0];
assert!(matches!(&child.kind, BlockKind::Paragraph { .. }));
assert_eq!(&src[child.src.clone()], "hello $x$ world\n> line2\n");
}
#[test]
fn tight_list_items_synthetic_first_paragraph_never_includes_the_bullet_marker() {
let src = "- item $x$ here\n- second\n";
let doc = Doc::parse(src);
let BlockKind::List { .. } = &doc.blocks[0].kind else {
panic!("expected a top-level list: {:?}", doc.blocks[0].kind);
};
assert_eq!(doc.blocks[0].children.len(), 2);
let BlockKind::ListItem { .. } = &doc.blocks[0].children[0].kind else {
panic!("expected the first list item");
};
let first_child = &doc.blocks[0].children[0].children[0];
assert!(matches!(&first_child.kind, BlockKind::Paragraph { .. }));
assert_eq!(&src[first_child.src.clone()], "item $x$ here");
}
fn inline_plain_text(doc: &Doc<'_>, inline: &Range<usize>) -> String {
let mut s = String::new();
for (ev, _) in &doc.events[inline.clone()] {
match ev {
Event::Text(t) => s.push_str(t),
Event::Code(c) => s.push_str(c),
Event::SoftBreak => s.push(' '),
Event::HardBreak => s.push('\n'),
_ => {}
}
}
s
}
#[test]
fn heading_and_paragraph_are_leaves_with_no_children() {
let doc = Doc::parse("# Title\n\nbody\n");
assert_eq!(doc.blocks.len(), 2);
let BlockKind::Heading {
level: 1, inline, ..
} = &doc.blocks[0].kind
else {
panic!("expected a level-1 heading")
};
assert_eq!(inline_plain_text(&doc, inline), "Title");
assert!(doc.blocks[0].children.is_empty());
assert_eq!(doc.blocks[0].src, 0..8);
let BlockKind::Paragraph { inline } = &doc.blocks[1].kind else {
panic!("expected a paragraph")
};
assert_eq!(inline_plain_text(&doc, inline), "body");
assert_eq!(doc.blocks[1].src, 9..14);
}
#[test]
fn heading_level_matches_the_number_of_hashes() {
for (n, marker) in [
(1, "#"),
(2, "##"),
(3, "###"),
(4, "####"),
(5, "#####"),
(6, "######"),
] {
let src = format!("{marker} h\n");
let doc = Doc::parse(&src);
let BlockKind::Heading { level, inline, .. } = &doc.blocks[0].kind else {
panic!("expected a heading for {marker:?}")
};
assert_eq!(*level, n, "marker {marker:?}");
assert_eq!(inline_plain_text(&doc, inline), "h");
}
}
#[test]
fn heading_attributes_are_captured_as_owned_data() {
let doc = Doc::parse("## Custom Heading {#custom-id .note lang=en}\n");
let BlockKind::Heading {
id,
classes,
attrs,
inline,
..
} = &doc.blocks[0].kind
else {
panic!("expected a heading")
};
assert_eq!(id.as_deref(), Some("custom-id"));
assert_eq!(classes, &["note".to_string()]);
assert_eq!(attrs, &[("lang".to_string(), Some("en".to_string()))]);
assert_eq!(inline_plain_text(&doc, inline), "Custom Heading");
}
#[test]
fn heading_with_no_attribute_syntax_has_none_and_empty() {
let doc = Doc::parse("## Plain\n");
let BlockKind::Heading {
id, classes, attrs, ..
} = &doc.blocks[0].kind
else {
panic!("expected a heading")
};
assert_eq!(*id, None);
assert!(classes.is_empty());
assert!(attrs.is_empty());
}
#[test]
fn reference_link_with_definition_in_a_different_block_resolves_across_the_whole_document() {
let doc = Doc::parse("See [the docs][ref] here.\n\n[ref]: https://example.com/docs\n");
let BlockKind::Paragraph { inline } = &doc.blocks[0].kind else {
panic!("expected a paragraph")
};
assert_eq!(inline_plain_text(&doc, inline), "See the docs here.");
let dangling = Doc::parse("See [the docs][missing] here.\n");
let BlockKind::Paragraph { inline } = &dangling.blocks[0].kind else {
panic!("expected a paragraph")
};
assert_eq!(
inline_plain_text(&dangling, inline),
"See [the docs][missing] here."
);
}
#[test]
fn fenced_code_block_body_excludes_fence_and_info_string() {
let src = "```rust\nfn a() {}\nfn b() {}\n```\n";
let doc = Doc::parse(src);
let BlockKind::CodeBlock {
lang,
fenced,
body_spans,
} = &doc.blocks[0].kind
else {
panic!("expected a code block")
};
assert_eq!(lang.as_deref(), Some("rust"));
assert!(fenced);
assert_eq!(
code_body_text(body_spans, src),
"fn a() {}\nfn b() {}",
"content, with the one trailing newline stripped"
);
}
#[test]
fn fenced_code_block_with_no_info_string_has_no_lang() {
let doc = Doc::parse("```\nplain\n```\n");
let BlockKind::CodeBlock { lang, fenced, .. } = &doc.blocks[0].kind else {
panic!("expected a code block")
};
assert_eq!(*lang, None);
assert!(fenced);
}
#[test]
fn indented_code_block_is_not_fenced_and_has_no_lang() {
let doc = Doc::parse("para\n\n line one\n line two\n");
let BlockKind::CodeBlock { lang, fenced, .. } = &doc.blocks[1].kind else {
panic!("expected a code block")
};
assert_eq!(*lang, None);
assert!(!fenced);
}
#[test]
fn empty_fenced_code_block_has_no_body_spans() {
let src = "```\n```\n";
let doc = Doc::parse(src);
let BlockKind::CodeBlock { body_spans, .. } = &doc.blocks[0].kind else {
panic!("expected a code block")
};
assert_eq!(
body_spans,
&Vec::<Range<usize>>::new(),
"an empty code block reports no Event::Text spans at all"
);
assert_eq!(code_body_text(body_spans, src), "");
}
#[test]
fn list_children_are_items_and_ordered_lists_report_their_start() {
let doc = Doc::parse("5. a\n6. b\n");
assert_eq!(doc.blocks.len(), 1);
let BlockKind::List { ordered, start } = doc.blocks[0].kind else {
panic!("expected a list")
};
assert!(ordered);
assert_eq!(start, Some(5));
assert_eq!(doc.blocks[0].children.len(), 2);
for item in &doc.blocks[0].children {
assert!(matches!(item.kind, BlockKind::ListItem { task: None }));
}
}
#[test]
fn bullet_list_is_unordered_with_no_start() {
let doc = Doc::parse("- a\n- b\n");
let BlockKind::List { ordered, start } = doc.blocks[0].kind else {
panic!("expected a list")
};
assert!(!ordered);
assert_eq!(start, None);
}
#[test]
fn tight_list_item_gets_a_synthetic_paragraph_child_loose_item_gets_a_real_one() {
let src = "- a\n- b\n";
let tight = Doc::parse(src);
assert_eq!(tight.blocks[0].children[0].children.len(), 1);
let synthetic = &tight.blocks[0].children[0].children[0];
let BlockKind::Paragraph { inline } = &synthetic.kind else {
panic!("expected a synthetic paragraph")
};
assert_eq!(inline_plain_text(&tight, inline), "a");
assert_eq!(&src[synthetic.src.clone()], "a");
let loose_src = "- a\n\n- b\n";
let loose = Doc::parse(loose_src);
assert_eq!(loose.blocks[0].children[0].children.len(), 1);
let real = &loose.blocks[0].children[0].children[0];
let BlockKind::Paragraph { inline } = &real.kind else {
panic!("expected a real paragraph")
};
assert_eq!(inline_plain_text(&loose, inline), "a");
assert_eq!(&loose_src[real.src.clone()], "a\n");
}
#[test]
fn task_marker_state_and_position_are_exact() {
let src = "- [ ] a\n- [x] b\n- [X] c\n";
let doc = Doc::parse(src);
let states: Vec<(char, usize)> = doc.blocks[0]
.children
.iter()
.map(|item| {
let BlockKind::ListItem { task } = &item.kind else {
panic!("expected a list item")
};
let t = task.expect("expected a task marker");
(t.state, t.state_at)
})
.collect();
assert_eq!(states, vec![(' ', 3), ('x', 11), ('X', 19)]);
for (_, at) in &states {
assert_eq!(src.as_bytes()[at - 1], b'[');
assert_eq!(src.as_bytes()[at + 1], b']');
}
}
#[test]
fn loose_task_item_task_marker_is_still_captured() {
let src = "- [ ] outer\n\n - [ ] nested at matching indent\n";
let doc = Doc::parse(src);
let outer = &doc.blocks[0].children[0];
let BlockKind::ListItem { task } = &outer.kind else {
panic!("expected a list item")
};
let t = task.expect("expected the outer (loose) item's task marker to be captured");
assert_eq!((t.state, t.state_at), (' ', 3));
let BlockKind::Paragraph { inline } = &outer.children[0].kind else {
panic!("expected the loose item's own paragraph")
};
assert_eq!(inline_plain_text(&doc, inline), "outer");
let nested_list = &outer.children[1];
let nested_item = &nested_list.children[0];
let BlockKind::ListItem {
task: nested_task, ..
} = &nested_item.kind
else {
panic!("expected the nested list item")
};
let nested = nested_task.expect("expected the nested item's own marker to be captured");
assert_eq!(nested.state, ' ');
}
#[test]
fn tight_list_item_starting_with_inline_formatting_is_still_captured() {
let src = "- **bold** and *em* and `code` end\n";
let doc = Doc::parse(src);
let item = &doc.blocks[0].children[0];
assert_eq!(item.children.len(), 1);
let synthetic = &item.children[0];
let BlockKind::Paragraph { inline } = &synthetic.kind else {
panic!("expected a synthetic paragraph")
};
assert_eq!(
&src[synthetic.src.clone()],
"**bold** and *em* and `code` end"
);
assert_eq!(inline_plain_text(&doc, inline), "bold and em and code end");
}
#[test]
fn non_task_list_item_has_no_task() {
let doc = Doc::parse("- plain\n");
let BlockKind::ListItem { task } = doc.blocks[0].children[0].kind else {
panic!("expected a list item")
};
assert_eq!(task, None);
}
#[test]
fn custom_task_state_char_is_not_recognized_as_a_task_by_pulldown_cmark() {
let doc = Doc::parse("- [/] in progress\n");
let BlockKind::ListItem { task } = doc.blocks[0].children[0].kind else {
panic!("expected a list item")
};
assert_eq!(
task, None,
"pulldown-cmark does not emit TaskListMarker for a custom state"
);
}
#[test]
fn block_quote_alert_kind_uses_the_real_alert_header_parser() {
let doc = Doc::parse("> [!WARNING] Heads up\n> body\n");
let BlockKind::Quote { alert, alert_title } = doc.blocks[0].kind.clone() else {
panic!("expected a quote")
};
assert_eq!(alert, Some(AlertKind::Warning));
assert_eq!(
alert_title, "Heads up",
"the header's own Obsidian-style trailing title travels alongside alert"
);
}
#[test]
fn plain_block_quote_has_no_alert_kind() {
let doc = Doc::parse("> just a quote\n");
let BlockKind::Quote { alert, alert_title } = doc.blocks[0].kind.clone() else {
panic!("expected a quote")
};
assert_eq!(alert, None);
assert_eq!(alert_title, "", "no alert header means no title either");
}
#[test]
fn table_rows_have_header_first_then_body_with_correct_alignment() {
let src = "| a | b |\n|:--|--:|\n| 1 | 2 |\n| 3 | 4 |\n";
let doc = Doc::parse(src);
let BlockKind::Table { aligns, rows } = &doc.blocks[0].kind else {
panic!("expected a table")
};
assert_eq!(aligns, &[Alignment::Left, Alignment::Right]);
assert_eq!(rows.len(), 3, "1 header row + 2 body rows");
assert_eq!(&src[rows[0][0].clone()], " a ");
assert_eq!(&src[rows[0][1].clone()], " b ");
assert_eq!(&src[rows[1][0].clone()], " 1 ");
assert_eq!(&src[rows[2][1].clone()], " 4 ");
}
#[test]
fn table_with_no_alignment_colons_reports_none_not_left() {
let doc = Doc::parse("| a |\n|---|\n| 1 |\n");
let BlockKind::Table { aligns, .. } = &doc.blocks[0].kind else {
panic!("expected a table")
};
assert_eq!(aligns, &[Alignment::None]);
}
#[test]
fn html_block_tag_name_is_read_from_the_opening_line() {
let doc = Doc::parse("<div class=\"x\">\nhello\n</div>\n");
let BlockKind::Html { tag } = &doc.blocks[0].kind else {
panic!("expected an html block")
};
assert_eq!(tag.as_deref(), Some("div"));
}
#[test]
fn html_comment_block_has_no_tag_name() {
let doc = Doc::parse("<!-- a comment -->\n");
let BlockKind::Html { tag } = &doc.blocks[0].kind else {
panic!("expected an html block")
};
assert_eq!(*tag, None);
}
#[test]
fn well_formed_details_folds_into_one_container_with_summary_and_children() {
let src = "<details>\n<summary>S</summary>\n\nbody\n\n</details>\n";
let doc = Doc::parse(src);
assert_eq!(doc.blocks.len(), 1, "folded into a single Details block");
let BlockKind::Details {
open_attr, summary, ..
} = &doc.blocks[0].kind
else {
panic!("expected a Details block: {:?}", doc.blocks[0].kind)
};
assert!(!open_attr, "no `open` attribute on the tag");
assert_eq!(summary, "S");
assert_eq!(
&src[doc.blocks[0].src.clone()],
src,
"spans the whole construct"
);
assert_eq!(doc.blocks[0].children.len(), 1, "just the body paragraph");
let BlockKind::Paragraph { inline } = &doc.blocks[0].children[0].kind else {
panic!("expected the body paragraph")
};
assert_eq!(inline_plain_text(&doc, inline), "body");
}
#[test]
fn details_open_attribute_is_captured() {
let doc = Doc::parse("<details open>\n<summary>S</summary>\n\nbody\n\n</details>\n");
let BlockKind::Details { open_attr, .. } = &doc.blocks[0].kind else {
panic!("expected a Details block")
};
assert!(open_attr);
}
#[test]
fn details_with_no_summary_tag_reports_an_empty_summary() {
let doc = Doc::parse("<details>\n\nbody\n\n</details>\n");
let BlockKind::Details { summary, .. } = &doc.blocks[0].kind else {
panic!("expected a Details block")
};
assert_eq!(summary, "");
}
#[test]
fn unclosed_details_folds_every_remaining_sibling_as_its_body() {
let src = "<details>\n<summary>S</summary>\n\nbody one\n\nbody two\n";
let doc = Doc::parse(src);
assert_eq!(doc.blocks.len(), 1);
let BlockKind::Details {
open_attr, summary, ..
} = &doc.blocks[0].kind
else {
panic!("expected a Details block")
};
assert!(!open_attr);
assert_eq!(summary, "S");
assert_eq!(
doc.blocks[0].children.len(),
2,
"both trailing paragraphs, with no closing tag to stop at"
);
assert_eq!(
doc.blocks[0].src.end,
src.len(),
"an unclosed block runs to the end of input"
);
}
#[test]
fn nested_details_swallows_the_inner_close_first_matching_split_details() {
let src = "<details>\n<summary>Outer</summary>\n\n\
<details open>\n<summary>Inner</summary>\n\n\
inner body\n\n\
</details>\n\n\
outer body after inner\n\n\
</details>\n";
let doc = Doc::parse(src);
assert_eq!(
doc.blocks.len(),
3,
"outer Details, the leftover paragraph, and the leftover close tag: {:?}",
doc.blocks
);
let BlockKind::Details {
open_attr, summary, ..
} = &doc.blocks[0].kind
else {
panic!("expected the outer Details block")
};
assert!(!open_attr);
assert_eq!(summary, "Outer");
assert_eq!(doc.blocks[0].children.len(), 2);
assert!(matches!(
doc.blocks[0].children[0].kind,
BlockKind::Html { tag: Some(ref t) } if t == "details"
));
let BlockKind::Paragraph { inline } = &doc.blocks[0].children[1].kind else {
panic!("expected the inner body paragraph")
};
assert_eq!(inline_plain_text(&doc, inline), "inner body");
assert!(matches!(doc.blocks[1].kind, BlockKind::Paragraph { .. }));
assert!(matches!(
doc.blocks[2].kind,
BlockKind::Html { tag: Some(ref t) } if t == "details"
));
}
#[test]
fn glued_details_with_no_blank_line_anywhere_stays_an_unfolded_html_leaf() {
let src = "<details>\n<summary>A</summary>\n<details>\n<summary>Nested</summary>\n</details>\n</details>\n";
let doc = Doc::parse(src);
assert_eq!(doc.blocks.len(), 1);
assert!(matches!(
doc.blocks[0].kind,
BlockKind::Html { tag: Some(ref t) } if t == "details"
));
}
#[test]
fn details_nested_inside_a_list_item_folds_at_that_level_too() {
let src = "- item\n\n <details>\n <summary>S</summary>\n\n body\n\n </details>\n";
let doc = Doc::parse(src);
let item = &doc.blocks[0].children[0];
let details = item
.children
.iter()
.find(|b| matches!(b.kind, BlockKind::Details { .. }))
.expect("expected a folded Details block inside the list item");
let BlockKind::Details { summary, .. } = &details.kind else {
unreachable!()
};
assert_eq!(summary, "S");
}
#[test]
fn details_nested_inside_a_quote_folds_at_that_level_too() {
let src = "> <details>\n> <summary>S</summary>\n>\n> body\n>\n> </details>\n";
let doc = Doc::parse(src);
let quote = &doc.blocks[0];
assert!(matches!(quote.kind, BlockKind::Quote { .. }));
let details = quote
.children
.iter()
.find(|b| matches!(b.kind, BlockKind::Details { .. }))
.expect("expected a folded Details block inside the quote");
let BlockKind::Details { summary, .. } = &details.kind else {
unreachable!()
};
assert_eq!(summary, "S");
}
#[test]
fn a_tag_name_prefix_is_not_confused_with_details_itself() {
let doc = Doc::parse("<detailsx>\nhi\n</detailsx>\n");
let BlockKind::Html { tag } = &doc.blocks[0].kind else {
panic!("expected an html block")
};
assert_eq!(
tag.as_deref(),
Some("detailsx"),
"not truncated down to \"details\""
);
}
#[test]
fn thematic_break_is_reported_with_its_own_source_range() {
let src = "a\n\n---\n\nb\n";
let doc = Doc::parse(src);
assert!(matches!(doc.blocks[1].kind, BlockKind::ThematicBreak));
assert_eq!(&src[doc.blocks[1].src.clone()], "---\n");
}
#[test]
fn nested_list_inside_a_list_item_is_a_child_block_not_flattened() {
let doc = Doc::parse("- a\n - b\n");
let outer_item = &doc.blocks[0].children[0];
assert_eq!(outer_item.children.len(), 2);
assert!(matches!(
outer_item.children[0].kind,
BlockKind::Paragraph { .. }
));
assert!(matches!(
outer_item.children[1].kind,
BlockKind::List { .. }
));
}
#[test]
fn code_block_nested_inside_a_list_item_is_a_real_child_block() {
let doc = Doc::parse("- item\n\n ```rust\n code\n ```\n");
let item = &doc.blocks[0].children[0];
let code = item
.children
.iter()
.find(|b| matches!(b.kind, BlockKind::CodeBlock { .. }))
.expect("expected a nested code block");
assert!(matches!(
code.kind,
BlockKind::CodeBlock { fenced: true, .. }
));
}
#[test]
fn cjk_ranges_land_on_char_boundaries_and_slice_correctly() {
let src = "# 見出し\n\n```rust\nfn 関数() {}\n```\n";
let doc = Doc::parse(src);
assert_eq!(&src[doc.blocks[0].src.clone()], "# 見出し\n");
let BlockKind::Heading { inline, .. } = &doc.blocks[0].kind else {
panic!("expected a heading")
};
assert_eq!(inline_plain_text(&doc, inline), "見出し");
let BlockKind::CodeBlock { body_spans, .. } = &doc.blocks[1].kind else {
panic!("expected a code block")
};
for r in body_spans {
assert!(src.is_char_boundary(r.start) && src.is_char_boundary(r.end));
}
assert_eq!(code_body_text(body_spans, src), "fn 関数() {}");
}
#[test]
fn empty_document_has_no_blocks() {
assert_eq!(Doc::parse("").blocks, Vec::new());
}
fn check_invariants(
blocks: &[Block],
src: &str,
events_len: usize,
parent: Option<&Range<usize>>,
path: &str,
out: &mut Vec<String>,
) {
let mut prev_end: Option<usize> = None;
for (i, b) in blocks.iter().enumerate() {
let here = format!("{path}[{i}]");
if b.src.start > b.src.end {
out.push(format!("{here}: src.start > src.end ({:?})", b.src));
}
if b.src.end > src.len() {
out.push(format!(
"{here}: src.end past the input's length ({:?})",
b.src
));
} else if !src.is_char_boundary(b.src.start) || !src.is_char_boundary(b.src.end) {
out.push(format!("{here}: src {:?} is not on a char boundary", b.src));
}
if let Some(pe) = prev_end {
if b.src.start < pe {
out.push(format!(
"{here}: overlaps the previous sibling (starts at {} before it ends at {pe})",
b.src.start
));
}
}
if let Some(p) = parent {
if b.src.start < p.start || b.src.end > p.end {
out.push(format!("{here}: src {:?} escapes its parent {p:?}", b.src));
}
}
if let BlockKind::CodeBlock { body_spans, .. } = &b.kind {
let mut prev_span_end: Option<usize> = None;
for (si, span) in body_spans.iter().enumerate() {
let here = format!("{here}.body_spans[{si}]");
if span.start > span.end {
out.push(format!("{here}: start > end ({span:?})"));
} else if span.end > src.len() {
out.push(format!("{here}: end past the input's length ({span:?})"));
} else if !src.is_char_boundary(span.start) || !src.is_char_boundary(span.end) {
out.push(format!("{here}: {span:?} is not on a char boundary"));
} else if span.start < b.src.start || span.end > b.src.end {
out.push(format!(
"{here}: {span:?} escapes its own block's src {:?}",
b.src
));
}
if let Some(pe) = prev_span_end {
if span.start < pe {
out.push(format!(
"{here}: overlaps the previous span (starts at {} before it ends at {pe})",
span.start
));
}
}
prev_span_end = Some(span.end);
}
}
if let BlockKind::ListItem { task: Some(t) } = &b.kind {
if !src.is_char_boundary(t.state_at) {
out.push(format!(
"{here}: task.state_at {} is not on a char boundary",
t.state_at
));
} else if !src[t.state_at..].starts_with(t.state) {
out.push(format!(
"{here}: task.state_at {} does not point at task.state {:?}",
t.state_at, t.state
));
} else if src.as_bytes().get(t.state_at.wrapping_sub(1)) != Some(&b'[')
|| src.as_bytes().get(t.state_at + 1) != Some(&b']')
{
out.push(format!(
"{here}: task.state_at {} is not bracketed by '[' and ']'",
t.state_at
));
}
}
let inline = match &b.kind {
BlockKind::Heading { inline, .. } => Some(inline),
BlockKind::Paragraph { inline } => Some(inline),
_ => None,
};
if let Some(inline) = inline {
if inline.start > inline.end {
out.push(format!("{here}: inline.start > inline.end ({inline:?})"));
} else if inline.end > events_len {
out.push(format!(
"{here}: inline.end {} past events_len {events_len}",
inline.end
));
}
}
check_invariants(&b.children, src, events_len, Some(&b.src), &here, out);
prev_end = Some(b.src.end);
}
}
fn all_corpus_texts() -> Vec<(String, String)> {
let mut v: Vec<(String, String)> = Vec::new();
for (name, src) in super::super::task_corpus::cases() {
v.push((format!("task_corpus: {name}"), src.to_string()));
}
for (name, src) in super::super::code_corpus::cases() {
v.push((format!("code_corpus: {name}"), src.to_string()));
}
for case in super::super::code_span_corpus::cases() {
v.push((format!("code_span_corpus: {}", case.name), case.src));
}
for (name, src) in super::super::preprocess_corpus::cases() {
v.push((format!("preprocess_corpus: {name}"), src.to_string()));
}
let mut with_preprocessing: Vec<(String, String)> = Vec::new();
for (name, raw) in &v {
with_preprocessing.push((format!("{name} [preprocessed]"), preprocess_default(raw)));
}
v.extend(with_preprocessing);
v
}
fn preprocess_default(src: &str) -> String {
let (_front_matter, body) = super::super::strip_front_matter(src);
let origin = super::super::identity_origin(&body);
let (s, origin) = super::super::process_footnotes_traced(&body, &origin);
let (pre_src, _origin) = super::super::process_inline_html_traced(&s, &origin);
pre_src
}
#[test]
fn model_invariants_hold_across_the_full_parity_corpus() {
let mut violations = Vec::new();
for (name, src) in all_corpus_texts() {
let doc = Doc::parse(&src);
check_invariants(
&doc.blocks,
&src,
doc.events.len(),
None,
&name,
&mut violations,
);
}
assert!(
violations.is_empty(),
"{} invariant violation(s):\n{}",
violations.len(),
violations.join("\n")
);
}
#[test]
fn model_invariants_hold_across_the_sample_md_files() {
let mut violations = Vec::new();
let mut checked = 0usize;
for name in [
"images.md",
"links.md",
"links.ja.md",
"markdown.md",
"markdown.ja.md",
"README.md",
"tutorial.md",
"tutorial.ja.md",
] {
let p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("samples")
.join(name);
let Ok(raw) = std::fs::read_to_string(&p) else {
continue;
};
checked += 1;
let pre = preprocess_default(&raw);
let doc = Doc::parse(&pre);
check_invariants(
&doc.blocks,
&pre,
doc.events.len(),
None,
name,
&mut violations,
);
}
if checked == 0 {
eprintln!("model_invariants_hold_across_the_sample_md_files: no samples/*.md found (published-crate build?) — skipping");
return;
}
assert!(
violations.is_empty(),
"{} invariant violation(s) across {checked} sample file(s):\n{}",
violations.len(),
violations.join("\n")
);
}
fn code_block_bodies(
blocks: &[Block],
inside_quote: bool,
out: &mut Vec<(bool, Vec<Range<usize>>)>,
) {
for b in blocks {
match &b.kind {
BlockKind::CodeBlock { body_spans, .. } => {
out.push((inside_quote, body_spans.clone()))
}
BlockKind::Quote { .. } => code_block_bodies(&b.children, true, out),
_ => code_block_bodies(&b.children, inside_quote, out),
}
}
}
fn joined_code_block_texts(src: &str) -> Vec<(bool, String)> {
let mut out = Vec::new();
let mut quote_depth = 0usize;
let mut in_quote_at_open = false;
let mut body: Option<String> = None;
for (ev, _) in Parser::new_ext(src, parse_options()).into_offset_iter() {
match ev {
Event::Start(Tag::BlockQuote(_)) => quote_depth += 1,
Event::End(TagEnd::BlockQuote(_)) => quote_depth = quote_depth.saturating_sub(1),
Event::Start(Tag::CodeBlock(_)) => {
in_quote_at_open = quote_depth > 0;
body = Some(String::new());
}
Event::End(TagEnd::CodeBlock) => {
if let Some(b) = body.take() {
let trimmed = b.strip_suffix('\n').unwrap_or(&b).to_string();
out.push((in_quote_at_open, trimmed));
}
}
Event::Text(t) => {
if let Some(b) = body.as_mut() {
b.push_str(&t);
}
}
_ => {}
}
}
out
}
#[test]
fn code_blocks_match_parser_code_blocks_outside_block_quotes_across_the_corpus() {
let mut total_checked = 0usize;
for (name, src) in all_corpus_texts() {
let mut expected = Vec::new();
super::super::parser_code_blocks(&src, &mut expected);
let actual: Vec<String> = joined_code_block_texts(&src)
.into_iter()
.filter(|(in_quote, _)| !in_quote)
.map(|(_, text)| text)
.collect();
assert_eq!(actual, expected, "case {name:?}");
total_checked += expected.len();
}
assert!(
total_checked > 20,
"the corpus's non-quote-nested code block count looks suspiciously small \
({total_checked}) — a corpus-gathering call probably broke"
);
}
#[test]
fn model_body_ranges_are_internally_consistent_with_parser_code_blocks_quote_filtering() {
for (name, src) in all_corpus_texts() {
let doc = Doc::parse(&src);
let mut model = Vec::new();
code_block_bodies(&doc.blocks, false, &mut model);
let joined = joined_code_block_texts(&src);
assert_eq!(
model.len(),
joined.len(),
"case {name:?}: model found {} code blocks, the independent walk found {}",
model.len(),
joined.len()
);
for (i, ((model_in_quote, _), (joined_in_quote, _))) in
model.iter().zip(joined.iter()).enumerate()
{
assert_eq!(
model_in_quote, joined_in_quote,
"case {name:?} block #{i}: model and the independent walk disagree about quote nesting"
);
}
}
}
#[test]
fn code_body_text_matches_parser_code_blocks_for_every_non_quote_code_block_in_the_corpus() {
let mut total_checked = 0usize;
for (name, src) in all_corpus_texts() {
let mut expected = Vec::new();
super::super::parser_code_blocks(&src, &mut expected);
let doc = Doc::parse(&src);
let mut model = Vec::new();
code_block_bodies(&doc.blocks, false, &mut model);
let actual: Vec<String> = model
.into_iter()
.filter(|(in_quote, _)| !in_quote)
.map(|(_, spans)| code_body_text(&spans, &src))
.collect();
assert_eq!(actual, expected, "case {name:?}");
total_checked += expected.len();
}
assert!(
total_checked > 20,
"the corpus's non-quote-nested code block count looks suspiciously small \
({total_checked}) — a corpus-gathering call probably broke"
);
}
#[test]
fn parsing_the_same_text_twice_gives_the_same_tree() {
for (_, src) in all_corpus_texts() {
assert_eq!(Doc::parse(&src), Doc::parse(&src));
}
}
fn violations_of(blocks: &[Block], src: &str, events_len: usize) -> Vec<String> {
let mut out = Vec::new();
check_invariants(blocks, src, events_len, None, "root", &mut out);
out
}
#[test]
#[allow(clippy::reversed_empty_ranges)] fn check_invariants_catches_src_start_after_end() {
let bad = leaf(BlockKind::ThematicBreak, 5..2);
let out = violations_of(std::slice::from_ref(&bad), "0123456789", 0);
assert!(
out.iter().any(|m| m.contains("src.start > src.end")),
"expected a start>end violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_src_end_past_input_length() {
let src = "abc";
let bad = leaf(BlockKind::ThematicBreak, 0..99);
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter().any(|m| m.contains("past the input's length")),
"expected an end-past-length violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_a_range_that_splits_a_multibyte_char() {
let src = "あ"; let bad = leaf(BlockKind::ThematicBreak, 0..1); let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter().any(|m| m.contains("is not on a char boundary")),
"expected a char-boundary violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_a_child_escaping_its_parent() {
let src = "0123456789";
let child = leaf(BlockKind::ThematicBreak, 6..8); let parent = Block {
kind: BlockKind::List {
ordered: false,
start: None,
},
src: 0..5,
children: vec![child],
};
let out = violations_of(std::slice::from_ref(&parent), src, 0);
assert!(
out.iter().any(|m| m.contains("escapes its parent")),
"expected a parent-escape violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_overlapping_siblings() {
let src = "0123456789";
let a = leaf(BlockKind::ThematicBreak, 0..5);
let b = leaf(BlockKind::ThematicBreak, 3..8); let out = violations_of(&[a, b], src, 0);
assert!(
out.iter()
.any(|m| m.contains("overlaps the previous sibling")),
"expected an overlap violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_siblings_out_of_source_order() {
let src = "0123456789";
let a = leaf(BlockKind::ThematicBreak, 5..9);
let b = leaf(BlockKind::ThematicBreak, 0..2);
let out = violations_of(&[a, b], src, 0);
assert!(
!out.is_empty(),
"expected a violation for an out-of-source-order sibling pair, got none"
);
}
#[test]
#[allow(clippy::reversed_empty_ranges, clippy::single_range_in_vec_init)] fn check_invariants_catches_a_code_block_span_start_after_end() {
let src = "0123456789";
let bad = leaf(
BlockKind::CodeBlock {
lang: None,
fenced: true,
body_spans: vec![5..2],
},
0..10,
);
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("body_spans") && m.contains("start > end")),
"expected a body_spans start>end violation, got {out:?}"
);
}
#[test]
#[allow(clippy::single_range_in_vec_init)] fn check_invariants_catches_a_code_block_span_past_input_length() {
let src = "0123456789";
let bad = leaf(
BlockKind::CodeBlock {
lang: None,
fenced: true,
body_spans: vec![0..99],
},
0..10,
);
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("body_spans") && m.contains("past the input's length")),
"expected a body_spans length violation, got {out:?}"
);
}
#[test]
#[allow(clippy::single_range_in_vec_init)] fn check_invariants_catches_a_code_block_span_splitting_a_multibyte_char() {
let src = "0あ23456789"; let bad = leaf(
BlockKind::CodeBlock {
lang: None,
fenced: true,
body_spans: vec![0..2], },
0..src.len(),
);
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("body_spans") && m.contains("is not on a char boundary")),
"expected a body_spans char-boundary violation, got {out:?}"
);
}
#[test]
#[allow(clippy::single_range_in_vec_init)] fn check_invariants_catches_a_code_block_span_escaping_its_own_block() {
let src = "0123456789";
let bad = leaf(
BlockKind::CodeBlock {
lang: None,
fenced: true,
body_spans: vec![6..8], },
0..5,
);
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("body_spans") && m.contains("escapes its own block's src")),
"expected a body_spans own-block-escape violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_overlapping_code_block_spans() {
let src = "0123456789";
let bad = leaf(
BlockKind::CodeBlock {
lang: None,
fenced: true,
body_spans: vec![0..5, 3..8], },
0..10,
);
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("body_spans") && m.contains("overlaps the previous span")),
"expected an overlapping-spans violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_a_task_state_at_splitting_a_multibyte_char() {
let src = "[あ]"; let bad = Block {
kind: BlockKind::ListItem {
task: Some(Task {
state: 'x',
state_at: 2, }),
},
src: 0..src.len(),
children: Vec::new(),
};
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("task.state_at") && m.contains("is not on a char boundary")),
"expected a task char-boundary violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_a_task_state_that_does_not_match_the_byte_at_state_at() {
let src = "[y]";
let bad = Block {
kind: BlockKind::ListItem {
task: Some(Task {
state: 'x', state_at: 1,
}),
},
src: 0..src.len(),
children: Vec::new(),
};
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter()
.any(|m| m.contains("does not point at task.state")),
"expected a task state-mismatch violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_a_task_state_at_not_bracketed() {
let src = "(x)"; let bad = Block {
kind: BlockKind::ListItem {
task: Some(Task {
state: 'x',
state_at: 1,
}),
},
src: 0..src.len(),
children: Vec::new(),
};
let out = violations_of(std::slice::from_ref(&bad), src, 0);
assert!(
out.iter().any(|m| m.contains("is not bracketed by")),
"expected a task bracket violation, got {out:?}"
);
}
#[test]
#[allow(clippy::reversed_empty_ranges)] fn check_invariants_catches_a_headings_inline_start_after_end() {
let src = "# hi\n";
let bad = leaf(
BlockKind::Heading {
level: 1,
inline: 5..2,
id: None,
classes: Vec::new(),
attrs: Vec::new(),
},
0..src.len(),
);
let out = violations_of(std::slice::from_ref(&bad), src, 10);
assert!(
out.iter().any(|m| m.contains("inline.start > inline.end")),
"expected an inline start>end violation, got {out:?}"
);
}
#[test]
fn check_invariants_catches_a_paragraphs_inline_end_past_events_len() {
let src = "hi\n";
let bad = leaf(BlockKind::Paragraph { inline: 0..99 }, 0..src.len());
let out = violations_of(std::slice::from_ref(&bad), src, 3); assert!(
out.iter()
.any(|m| m.contains("inline.end") && m.contains("past events_len")),
"expected an inline-past-events_len violation, got {out:?}"
);
}
#[test]
#[allow(clippy::reversed_empty_ranges)] fn check_invariants_reports_every_violation_at_once_not_just_the_first() {
let src = "0123456789";
let a = leaf(BlockKind::ThematicBreak, 5..2); let b = leaf(BlockKind::ThematicBreak, 0..99); let out = violations_of(&[a, b], src, 0);
assert!(
out.iter().any(|m| m.contains("src.start > src.end")),
"{out:?}"
);
assert!(
out.iter().any(|m| m.contains("past the input's length")),
"{out:?}"
);
}
fn events_claimed_by_a_leaf(blocks: &[Block], claimed: &mut [bool]) {
for b in blocks {
let inline = match &b.kind {
BlockKind::Heading { inline, .. } => Some(inline),
BlockKind::Paragraph { inline } => Some(inline),
_ => None,
};
if let Some(inline) = inline {
for i in inline.clone() {
if let Some(slot) = claimed.get_mut(i) {
*slot = true;
}
}
}
events_claimed_by_a_leaf(&b.children, claimed);
}
}
#[test]
fn completeness_proxy_detects_an_inline_event_dropped_from_every_leaf() {
let src = "# Title\n\nhello world\n\n---\n";
let doc = Doc::parse(src);
assert_eq!(
doc.blocks.len(),
3,
"heading, paragraph, thematic break: {:?}",
doc.blocks
);
let mut claimed = vec![false; doc.events.len()];
events_claimed_by_a_leaf(&doc.blocks, &mut claimed);
assert!(
claimed.iter().any(|&c| c),
"sanity: the real model should claim at least one event"
);
let mut corrupted = doc.blocks.clone();
let BlockKind::Paragraph { inline } = &mut corrupted[1].kind else {
panic!("expected the paragraph at index 1: {:?}", corrupted[1].kind)
};
inline.end = inline.start;
let mut claimed_after = vec![false; doc.events.len()];
events_claimed_by_a_leaf(&corrupted, &mut claimed_after);
assert!(
claimed
.iter()
.zip(claimed_after.iter())
.any(|(&before, &after)| before && !after),
"corrupting the paragraph's own inline range should un-claim at least one event the \
real model claims"
);
}
fn count_task_list_markers_in_events(doc: &Doc<'_>) -> usize {
doc.events
.iter()
.filter(|(ev, _)| matches!(ev, Event::TaskListMarker(_)))
.count()
}
fn count_tasks_recorded_in_tree(blocks: &[Block]) -> usize {
let mut n = 0;
for b in blocks {
if let BlockKind::ListItem { task: Some(_) } = &b.kind {
n += 1;
}
n += count_tasks_recorded_in_tree(&b.children);
}
n
}
#[test]
fn completeness_proxy_detects_a_task_list_marker_dropped_from_the_tree() {
let src = "- [ ] a\n- [x] b\n";
let doc = Doc::parse(src);
assert_eq!(
count_task_list_markers_in_events(&doc),
2,
"sanity: two markers in the raw event stream"
);
assert_eq!(
count_tasks_recorded_in_tree(&doc.blocks),
2,
"sanity: the real model records both"
);
let mut corrupted = doc.blocks.clone();
let BlockKind::ListItem { task } = &mut corrupted[0].children[0].kind else {
panic!("expected the first list item")
};
*task = None;
assert_eq!(
count_tasks_recorded_in_tree(&corrupted),
1,
"the corrupted copy under-counts relative to the raw stream's own marker count"
);
assert_ne!(
count_task_list_markers_in_events(&doc),
count_tasks_recorded_in_tree(&corrupted),
"a TaskListMarker present in the raw stream but absent from every ListItem.task is \
exactly the completeness gap this proxy exists to catch"
);
}
fn count_code_block_starts_in_raw_events(src: &str) -> usize {
Parser::new_ext(src, parse_options())
.into_offset_iter()
.filter(|(ev, _)| matches!(ev, Event::Start(Tag::CodeBlock(_))))
.count()
}
fn count_code_blocks_recorded_in_tree(blocks: &[Block]) -> usize {
let mut n = 0;
for b in blocks {
if matches!(b.kind, BlockKind::CodeBlock { .. }) {
n += 1;
}
n += count_code_blocks_recorded_in_tree(&b.children);
}
n
}
#[test]
fn completeness_proxy_detects_a_code_block_dropped_from_the_tree() {
let src = "```rust\nfn a() {}\n```\n\npara\n\n```\nplain\n```\n";
let doc = Doc::parse(src);
let raw = count_code_block_starts_in_raw_events(src);
assert_eq!(
raw, 2,
"sanity: two Start(CodeBlock) events in the raw stream"
);
assert_eq!(
count_code_blocks_recorded_in_tree(&doc.blocks),
2,
"sanity: the real model records both"
);
let mut corrupted = doc.blocks.clone();
corrupted[0].kind = BlockKind::ThematicBreak;
assert_eq!(
count_code_blocks_recorded_in_tree(&corrupted),
1,
"the corrupted copy under-counts relative to the raw stream's own CodeBlock count"
);
assert_ne!(
raw,
count_code_blocks_recorded_in_tree(&corrupted),
"a Start(CodeBlock) present in the raw stream but represented by no \
BlockKind::CodeBlock anywhere in the tree is exactly the completeness gap this proxy \
exists to catch"
);
}
fn walker_for(src: &str) -> Walker<'_> {
Walker {
iter: Parser::new_ext(src, parse_options())
.into_offset_iter()
.peekable(),
events: Vec::new(),
}
}
#[test]
fn parse_blocks_raw_silently_ignores_a_top_level_task_list_marker() {
let src = "- [ ] a\n";
let mut w = walker_for(src);
let _ = w.next(); let _ = w.next(); let out = parse_blocks_raw(&mut w, src);
assert_eq!(
out.len(),
1,
"the item's own text still becomes a block: {out:?}"
);
let BlockKind::Paragraph { inline } = &out[0].kind else {
panic!("expected a synthetic paragraph: {:?}", out[0].kind)
};
assert_eq!(
inline_plain_text(
&Doc {
events: w.events.clone(),
blocks: Vec::new()
},
inline
),
"a"
);
}
#[test]
fn collect_stray_inline_run_handles_an_exhausted_event_stream() {
let mut w = walker_for("");
let range = collect_stray_inline_run((Event::Text("x".into()), 3..4), &mut w);
assert_eq!(
range,
3..4,
"an exhausted stream ends the run at just its own first event"
);
}
#[test]
fn collect_code_body_spans_defensively_skips_an_unexpected_start_event() {
let src = "**bold** x\n";
let mut w = walker_for(src);
let spans = collect_code_body_spans(&mut w);
assert!(
spans.is_empty(),
"no Event::Text was ever seen at code-block level: {spans:?}"
);
}
#[test]
fn collect_code_body_spans_defensively_skips_an_event_that_is_neither_text_nor_end() {
let src = "a\nb\n"; let mut w = walker_for(src);
let _ = w.next(); let spans = collect_code_body_spans(&mut w);
assert_eq!(
spans.len(),
2,
"both Text(\"a\") and Text(\"b\") still get collected: {spans:?}"
);
}
#[test]
fn collect_table_rows_defensively_skips_an_unexpected_start_event() {
let src = "**bold** x\n";
let mut w = walker_for(src);
let rows = collect_table_rows(&mut w);
assert!(
rows.is_empty(),
"no TableHead/TableRow was ever seen: {rows:?}"
);
}
#[test]
fn collect_table_rows_defensively_skips_an_event_that_is_neither_a_row_kind_nor_end() {
let src = "a\nb\n";
let mut w = walker_for(src);
let _ = w.next(); let rows = collect_table_rows(&mut w);
assert!(
rows.is_empty(),
"no TableHead/TableRow/End(Table) event ever appears in this stream: {rows:?}"
);
}
#[test]
fn collect_row_cells_defensively_skips_an_unexpected_start_event() {
let src = "**bold** x\n";
let mut w = walker_for(src);
let cells = collect_row_cells(&mut w, TagEnd::TableRow);
assert!(cells.is_empty(), "no TableCell was ever seen: {cells:?}");
}
#[test]
fn collect_row_cells_defensively_skips_an_event_that_is_neither_a_cell_nor_its_own_end() {
let src = "a\nb\n";
let mut w = walker_for(src);
let _ = w.next(); let cells = collect_row_cells(&mut w, TagEnd::TableRow);
assert!(cells.is_empty(), "no TableCell was ever seen: {cells:?}");
}
#[test]
fn collect_html_tag_defensively_skips_an_unexpected_start_event() {
let src = "**bold** x\n";
let mut w = walker_for(src);
let range = 0..src.len();
let tag = collect_html_tag(&mut w, &range, src);
assert_eq!(tag, None);
}
#[test]
fn html_tag_name_reports_none_when_no_name_characters_follow_the_opening_bracket() {
assert_eq!(
html_tag_name("< foo>"),
None,
"space right after '<' leaves no name chars"
);
assert_eq!(html_tag_name("<"), None, "nothing at all after '<'");
assert_eq!(
html_tag_name("</"),
None,
"nothing after the closing-tag slash either"
);
assert_eq!(html_tag_name("<div>"), Some("div".to_string()));
assert_eq!(html_tag_name("</div>"), Some("div".to_string()));
assert_eq!(html_tag_name(""), None, "doesn't even start with '<'");
assert_eq!(
html_tag_name("<!doctype>"),
None,
"excluded by the '!' guard, not this branch"
);
}
#[test]
fn unclosed_glued_details_with_no_close_anywhere_in_the_document_stays_unfolded() {
let src = "<details>\n<summary>S</summary>\nno close anywhere\n";
let doc = Doc::parse(src);
assert_eq!(doc.blocks.len(), 1);
assert!(matches!(
doc.blocks[0].kind,
BlockKind::Html { tag: Some(ref t) } if t == "details"
));
}
#[test]
fn glued_details_with_unrelated_content_after_the_close_stays_unfolded() {
let src = "<details>\n<summary>A</summary>\n</details>\nextra content here\n";
let doc = Doc::parse(src);
assert_eq!(doc.blocks.len(), 1);
assert!(matches!(
doc.blocks[0].kind,
BlockKind::Html { tag: Some(ref t) } if t == "details"
));
assert_eq!(
&src[doc.blocks[0].src.clone()],
src,
"the whole glued leaf is kept intact"
);
}
#[test]
fn unclosed_details_with_no_trailing_newline_stays_unfolded() {
let src = "<details>";
let doc = Doc::parse(src);
assert_eq!(doc.blocks.len(), 1);
assert!(matches!(
doc.blocks[0].kind,
BlockKind::Html { tag: Some(ref t) } if t == "details"
));
assert_eq!(doc.blocks[0].src, 0..src.len());
}
fn collect_model_details_open(blocks: &[Block], out: &mut Vec<bool>) {
for b in blocks {
if let BlockKind::Details { open_attr, .. } = &b.kind {
out.push(*open_attr);
}
collect_model_details_open(&b.children, out);
}
}
#[test]
fn model_details_ordinal_matches_collect_details_open_for_well_formed_non_quote_nesting() {
let cases = [
"<details>\n<summary>S</summary>\n\nbody\n\n</details>\n",
"<details>\n<summary>A</summary>\n\na\n\n</details>\n\n\
<details open>\n<summary>B</summary>\n\nb\n\n</details>\n",
"- item\n\n <details>\n <summary>S</summary>\n\n body\n\n </details>\n",
"<details>\n<summary>S</summary>\n\nbody one\n\nbody two\n",
"<details>\n<summary>Outer</summary>\n\n\
<details open>\n<summary>Inner</summary>\n\ninner body\n\n</details>\n\n\
outer body after inner\n\n</details>\n",
"- item\n\n <details>\n <summary>Outer</summary>\n\n body\n\n </details>\n\n\
- item2\n\n <details open>\n <summary>Second</summary>\n\n body2\n\n </details>\n",
"<details open>\n<summary>A</summary>\n\na\n\n</details>\n\n\
<details>\n<summary>B</summary>\n\nb\n\n</details>\n\n\
<details open>\n<summary>C</summary>\n\nc\n\n</details>\n",
];
for src in cases {
let doc = Doc::parse(src);
let mut model_open: Vec<bool> = Vec::new();
collect_model_details_open(&doc.blocks, &mut model_open);
let split_open = super::super::collect_details_open(src);
assert_eq!(model_open, split_open, "case {src:?}");
}
}
#[test]
fn model_details_ordinal_diverges_from_collect_details_open_for_a_quote_nested_details() {
let src = "> <details>\n> <summary>S</summary>\n>\n> body\n>\n> </details>\n";
let doc = Doc::parse(src);
let mut model_open: Vec<bool> = Vec::new();
collect_model_details_open(&doc.blocks, &mut model_open);
assert_eq!(
model_open,
vec![false],
"the model itself still folds a quote-nested details"
);
assert_eq!(
super::super::collect_details_open(src),
Vec::<bool>::new(),
"collect_details_open misses it — the same '>'-prefix mechanism its own doc comment \
documents for alerts specifically also applies to a plain quote"
);
}
fn extreme_input_corpus() -> Vec<(String, String)> {
let mut v: Vec<(String, String)> = Vec::new();
let mut push = |name: &str, src: String| v.push((name.to_string(), src));
push("unclosed fence", "```rust\nfn a() {}\n".to_string());
push("unclosed fence, no lang", "```\nplain\n".to_string());
push(
"unclosed fence nested in a list item",
"- item\n\n ```rust\n fn a() {}\n".to_string(),
);
push(
"unclosed details, well-formed open/summary",
"<details>\n<summary>S</summary>\n\nbody\n".to_string(),
);
push(
"unclosed details, no summary tag",
"<details>\nbody only\n".to_string(),
);
push(
"unclosed html comment",
"<!-- never closes\nmore text\nstill in the comment\n".to_string(),
);
push(
"unclosed emphasis (asterisk)",
"*never closes\nmore text\n".to_string(),
);
push(
"unclosed emphasis (underscore)",
"_never closes\nmore text\n".to_string(),
);
push("unclosed strong", "**never closes\nmore text\n".to_string());
push(
"unclosed link, no matching bracket",
"[never closes (no matching bracket\n".to_string(),
);
push(
"unclosed link, no matching paren",
"[text](never closes\n".to_string(),
);
push(
"unclosed inline code",
"`never closes\nmore text\n".to_string(),
);
push(
"unclosed strikethrough",
"~~never closes\nmore text\n".to_string(),
);
push(
"table header with no body rows",
"| a | b |\n|---|---|\n".to_string(),
);
push(
"table header row with no delimiter row at all",
"| a | b |\nnot a delimiter row\n".to_string(),
);
push("deeply nested bullet list (10 levels)", {
let mut s = String::new();
for i in 0..10 {
s.push_str(&" ".repeat(i));
s.push_str("- level\n");
}
s
});
push("deeply nested block quote (10 levels)", {
let mut s = "> ".repeat(10);
s.push_str("deep quote\n");
s
});
push(
"list inside quote inside list",
"- outer\n\n > quoted\n >\n > - inner list\n > - inner inner\n".to_string(),
);
push("alternating quote/list nesting (8 levels)", {
let mut s = String::new();
for i in 0..8 {
if i % 2 == 0 {
s.push_str(&"> ".repeat(i / 2 + 1));
} else {
s.push_str(&" ".repeat(i));
s.push_str("- ");
}
}
s.push_str("bottom\n");
s
});
push("huge single line, no newline (~1MB)", "x".repeat(1_000_000));
push("huge single line, unclosed emphasis (~1MB)", {
let mut s = String::from("*");
s.push_str(&"a".repeat(1_000_000));
s
});
push("many blank lines (5000)", "\n".repeat(5000));
push("many sibling paragraphs (2000)", {
let mut s = String::new();
for i in 0..2000 {
s.push_str(&format!("para {i}\n\n"));
}
s
});
push("many sibling list items (3000)", {
let mut s = String::new();
for i in 0..3000 {
s.push_str(&format!("- item {i}\n"));
}
s
});
push(
"cjk heavy",
"# 見出しタイトル\n\n本文は日本語です。**強調**も*斜体*も入ります。\n\n\
- 項目一\n- 項目二\n\n> 引用も日本語\n"
.to_string(),
);
push(
"emoji throughout",
"# 🎉 Title 🚀\n\nHello 👋 world 🌍! `code 💻` and **bold 🔥**.\n\n- [ ] 🧪 test\n"
.to_string(),
);
push(
"combining marks stacked",
"e\u{0301}\u{0301}\u{0301}\u{0301}\u{0301} five combining acutes on one base\n"
.to_string(),
);
push(
"zero width characters",
"zero\u{200B}width\u{200C}space\u{200D}joiner\u{FEFF}mixed\n".to_string(),
);
push(
"rtl text mixed with ltr",
"\u{202B}שלום עולם\u{202C} mixed with English text\n".to_string(),
);
push(
"rtl heading",
"# \u{0645}\u{0631}\u{062D}\u{0628}\u{0627} Arabic heading\n".to_string(),
);
push(
"cjk task list and code fence",
"- [ ] 日本語のタスク\n- [x] 完了したタスク\n\n```rust\nfn 関数() -> 文字列 {}\n```\n"
.to_string(),
);
push(
"crlf throughout, mixed constructs",
"# Title\r\n\r\nBody line one\r\nBody line two\r\n\r\n\
- item one\r\n- item two\r\n\r\n```rust\r\nfn a() {}\r\n```\r\n\r\n\
> quoted\r\n> line two\r\n"
.to_string(),
);
push(
"no trailing newline, plain text",
"just text, no newline at all".to_string(),
);
push(
"no trailing newline, heading",
"# Title, no newline".to_string(),
);
push(
"no trailing newline, fenced code",
"```\ncode\n```".to_string(),
);
push("no trailing newline, list item", "- one\n- two".to_string());
push(
"bom prefixed document",
"\u{FEFF}# Title\n\nBody\n".to_string(),
);
push("nul byte embedded", "before\u{0}after\n".to_string());
push(
"assorted c0 control chars",
"col1\u{1}col2\u{2}col3\u{7}bell\u{8}backspace\n".to_string(),
);
push(
"tab-indented code block",
"para\n\n\tline one\n\tline two\n".to_string(),
);
push(
"tab after a list marker",
"-\ttab then item text\n".to_string(),
);
push(
"fullwidth space, not real indentation",
" - looks indented but is U+3000, not ascii space\n".to_string(),
);
push(
"mixed tab and space indentation",
"- item\n \t - mixed indent nested item\n".to_string(),
);
push("empty string", String::new());
push("single space", " ".to_string());
push(
"whitespace only (spaces and tabs)",
" \t \t\t ".to_string(),
);
push("single newline only", "\n".to_string());
push("many newlines only", "\n\n\n\n\n\n\n\n".to_string());
for esc in ["\\*", "\\_", "\\[", "\\]", "\\(", "\\)", "\\$", "\\\\"] {
push(
&format!("escape {esc:?} mid-text"),
format!("text {esc} more text\n"),
);
push(
&format!("escape {esc:?} at start of line"),
format!("{esc} more text\n"),
);
push(
&format!("escape {esc:?} at end of input, no newline"),
format!("text {esc}"),
);
}
push(
"backslash immediately before cjk",
"before \\あ after\n".to_string(),
);
push(
"backslash immediately before emoji",
"before \\🎉 after\n".to_string(),
);
push(
"backslash immediately before combining mark",
"before \\\u{0301} after\n".to_string(),
);
push(
"input ends with a lone trailing backslash",
"text ends with backslash\\".to_string(),
);
push(
"currency dollar, not math (ENABLE_MATH is off anyway)",
"price is \\$5 not math\n".to_string(),
);
push(
"raw dollar-delimited text, unprocessed",
"inline $x + y$ and $$z$$ display\n".to_string(),
);
push(
"raw footnote-shaped text, unprocessed",
"See [^1] here.\n\n[^1]: definition\n".to_string(),
);
v
}
#[test]
fn doc_parse_never_panics_and_invariants_hold_across_extreme_inputs() {
let mut failures: Vec<String> = Vec::new();
let mut checked = 0usize;
for (name, src) in extreme_input_corpus() {
checked += 1;
let result = super::super::catch_silent(|| {
let doc = Doc::parse(&src);
let mut out = Vec::new();
check_invariants(&doc.blocks, &src, doc.events.len(), None, &name, &mut out);
out
});
match result {
None => failures.push(format!("{name:?}: PANICKED")),
Some(violations) if !violations.is_empty() => {
failures.push(format!(
"{name:?}: {} invariant violation(s): {violations:?}",
violations.len()
));
}
Some(_) => {}
}
}
assert!(
checked > 60,
"the extreme-input corpus looks suspiciously small ({checked}) — did a `push` call \
break?"
);
assert!(
failures.is_empty(),
"{} case(s) failed out of {checked}:\n{}",
failures.len(),
failures.join("\n")
);
}
#[test]
fn doc_parse_never_panics_on_deeply_nested_alternating_containers() {
let mut src = String::new();
for i in 0..15 {
if i % 2 == 0 {
src.push_str("> ");
} else {
src.push_str("- ");
}
}
src.push_str("- [ ] deep task\n");
for i in 0..15 {
let prefix = if i % 2 == 0 { "> " } else { " " };
src.push_str(prefix);
}
src.push_str("```rust\nfn deep() {}\n");
let doc = super::super::catch_silent(|| Doc::parse(&src));
let doc = doc.expect("Doc::parse must not panic on deep, unclosed, alternating nesting");
let mut out = Vec::new();
check_invariants(
&doc.blocks,
&src,
doc.events.len(),
None,
"deeply_nested_alternating",
&mut out,
);
assert!(out.is_empty(), "{out:?}");
}
#[test]
fn code_block_body_spans_reconstruct_byte_exact_content_across_extreme_containers() {
let cases: &[(&str, &str, &str)] = &[
(
"fenced code nested inside a block quote",
"> ```rust\n> fn a() {}\n> ```\n",
"fn a() {}",
),
(
"fenced code nested inside a GFM alert",
"> [!WARNING]\n> ```\n> code in alert\n> ```\n",
"code in alert",
),
(
"fenced code nested inside a details body",
"<details>\n<summary>S</summary>\n\n```rust\nfn a() {}\n```\n\n</details>\n",
"fn a() {}",
),
(
"fenced code nested inside a list item, CRLF throughout",
"- item\r\n\r\n ```rust\r\n fn a() {}\r\n ```\r\n",
"fn a() {}",
),
(
"indented code, CRLF throughout",
"para\r\n\r\n line one\r\n line two\r\n",
"line one\nline two",
),
(
"indented code nested inside a block quote",
"> para\n>\n> indented code\n> more code\n",
"indented code\nmore code",
),
(
"cjk code content",
"```rust\nfn 関数() -> 文字列 {}\n```\n",
"fn 関数() -> 文字列 {}",
),
(
"a literal tab as fenced code content, not indentation",
"```\n\tindented with tab inside fence\n```\n",
"\tindented with tab inside fence",
),
(
"empty fenced code nested inside a list item",
"- item\n\n ```\n ```\n",
"",
),
];
for (name, src, expected) in cases {
let doc = Doc::parse(src);
let mut spans_by_block: Vec<Vec<Range<usize>>> = Vec::new();
code_block_bodies_ignoring_quote_flag(&doc.blocks, &mut spans_by_block);
assert_eq!(
spans_by_block.len(),
1,
"case {name:?}: expected exactly one code block"
);
let actual = code_body_text(&spans_by_block[0], src);
assert_eq!(actual, *expected, "case {name:?}");
let mut violations = Vec::new();
check_invariants(
&doc.blocks,
src,
doc.events.len(),
None,
name,
&mut violations,
);
assert!(violations.is_empty(), "case {name:?}: {violations:?}");
}
}
fn code_block_bodies_ignoring_quote_flag(blocks: &[Block], out: &mut Vec<Vec<Range<usize>>>) {
for b in blocks {
if let BlockKind::CodeBlock { body_spans, .. } = &b.kind {
out.push(body_spans.clone());
}
code_block_bodies_ignoring_quote_flag(&b.children, out);
}
}
#[test]
fn task_state_at_contract_holds_across_extreme_containers() {
let cases = [
(
"task list nested inside a block quote",
"> - [ ] a\n> - [x] b\n> - [X] c\n",
),
(
"task list nested inside a GFM alert",
"> [!NOTE]\n> - [ ] a\n> - [x] b\n> - [X] c\n",
),
(
"task list nested inside a details body",
"<details>\n<summary>S</summary>\n\n- [ ] a\n- [x] b\n- [X] c\n\n</details>\n",
),
(
"task list, CRLF throughout",
"- [ ] a\r\n- [x] b\r\n- [X] c\r\n",
),
(
"task list, cjk label text",
"- [ ] 日本語のタスク\n- [x] 完了したタスク\n",
),
(
"task list nested two list levels deep",
"- outer\n - [ ] nested a\n - [x] nested b\n",
),
];
for (name, src) in cases {
let doc = Doc::parse(src);
let mut tasks: Vec<Task> = Vec::new();
collect_tasks(&doc.blocks, &mut tasks);
assert!(
!tasks.is_empty(),
"case {name:?}: expected at least one task item"
);
for t in &tasks {
assert!(
src.is_char_boundary(t.state_at),
"case {name:?}: state_at {} not on a char boundary",
t.state_at
);
assert!(
src[t.state_at..].starts_with(t.state),
"case {name:?}: byte at state_at does not match state {:?}",
t.state
);
assert_eq!(
src.as_bytes().get(t.state_at - 1),
Some(&b'['),
"case {name:?}: byte before state_at is not '['"
);
assert_eq!(
src.as_bytes().get(t.state_at + 1),
Some(&b']'),
"case {name:?}: byte after state_at is not ']'"
);
}
let mut violations = Vec::new();
check_invariants(
&doc.blocks,
src,
doc.events.len(),
None,
name,
&mut violations,
);
assert!(violations.is_empty(), "case {name:?}: {violations:?}");
}
}
fn collect_tasks(blocks: &[Block], out: &mut Vec<Task>) {
for b in blocks {
if let BlockKind::ListItem { task: Some(t) } = &b.kind {
out.push(*t);
}
collect_tasks(&b.children, out);
}
}
}