mod inline;
mod structure;
use std::ops::Range;
use pulldown_cmark::{
Alignment as MarkdownAlignment, CodeBlockKind, CowStr, Event as CmarkEvent,
HeadingLevel as MarkdownHeadingLevel, Options, Parser, Tag, TagEnd,
};
pub use inline::{tokenize_inline, InlineToken};
pub use structure::{
BorderMode, BoxSpec, ColumnSpec, ColumnWidth, ColumnsSpec, IndentSpec, Padding, WidthMode,
};
const MAX_LAYOUT_DEPTH: usize = 16;
#[derive(Clone, Debug)]
pub struct Document<'a> {
nodes: Vec<Node<'a>>,
}
impl<'a> Document<'a> {
pub fn nodes(&self) -> &[Node<'a>] {
&self.nodes
}
}
pub fn parse(source: &str) -> Result<Document<'_>, ParseError> {
let nodes = structure::parse(source, MAX_LAYOUT_DEPTH)
.map_err(|message| ParseError { message })?
.into_iter()
.map(parse_node)
.collect();
Ok(Document { nodes })
}
fn parse_node(node: structure::Node<'_>) -> Node<'_> {
match node {
structure::Node::Markup(source) => Node::Markdown(Markdown::parse(source)),
structure::Node::Box { spec, children } => Node::Box {
spec,
children: children.into_iter().map(parse_node).collect(),
},
structure::Node::Center { children } => Node::Center {
children: children.into_iter().map(parse_node).collect(),
},
structure::Node::Right { children } => Node::Right {
children: children.into_iter().map(parse_node).collect(),
},
structure::Node::Indent { spec, children } => Node::Indent {
spec,
children: children.into_iter().map(parse_node).collect(),
},
structure::Node::Columns { spec, children } => Node::Columns {
spec,
children: children.into_iter().map(parse_node).collect(),
},
structure::Node::Column { spec, children } => Node::Column {
spec,
children: children.into_iter().map(parse_node).collect(),
},
}
}
#[derive(Clone, Debug)]
pub enum Node<'a> {
Markdown(Markdown<'a>),
Box {
spec: BoxSpec,
children: Vec<Node<'a>>,
},
Center {
children: Vec<Node<'a>>,
},
Right {
children: Vec<Node<'a>>,
},
Indent {
spec: IndentSpec,
children: Vec<Node<'a>>,
},
Columns {
spec: ColumnsSpec,
children: Vec<Node<'a>>,
},
Column {
spec: ColumnSpec,
children: Vec<Node<'a>>,
},
}
#[derive(Clone, Debug)]
pub struct Markdown<'a> {
source: &'a str,
events: Vec<SpannedEvent<'a>>,
}
impl<'a> Markdown<'a> {
fn parse(source: &'a str) -> Self {
let options =
Options::ENABLE_TABLES | Options::ENABLE_STRIKETHROUGH | Options::ENABLE_TASKLISTS;
let mut parsed: Vec<SpannedEvent<'a>> = Vec::new();
for (event, span) in Parser::new_ext(source, options).into_offset_iter() {
let event = Event::from_cmark(event);
if let Event::Text(text) = &event {
if let Some(SpannedEvent {
event: Event::Text(previous),
span: previous_span,
}) = parsed.last_mut()
{
if previous_span.end == span.start {
let mut combined = previous.to_string();
combined.push_str(text);
*previous = CowStr::from(combined);
previous_span.end = span.end;
continue;
}
}
}
parsed.push(SpannedEvent { event, span });
}
let mut events = Vec::new();
let mut code_block = false;
for item in parsed {
match &item.event {
Event::Start(Container::CodeBlock(_)) => {
code_block = true;
events.push(item);
}
Event::End(ContainerEnd::CodeBlock) => {
code_block = false;
events.push(item);
}
_ if code_block => events.push(item),
_ => events.extend(split_inline_tags(source, item)),
}
}
Self { source, events }
}
pub fn source(&self) -> &'a str {
self.source
}
pub fn events(&self) -> &[SpannedEvent<'a>] {
&self.events
}
}
#[derive(Clone, Debug)]
pub struct SpannedEvent<'a> {
pub event: Event<'a>,
pub span: Range<usize>,
}
#[derive(Clone, Debug, PartialEq)]
pub enum Event<'a> {
Start(Container<'a>),
End(ContainerEnd),
Text(CowStr<'a>),
Hashtag(CowStr<'a>),
WikiLink(CowStr<'a>),
InlineTag(InlineTag<'a>),
Code(CowStr<'a>),
Html(CowStr<'a>),
InlineHtml(CowStr<'a>),
FootnoteReference(CowStr<'a>),
SoftBreak,
HardBreak,
Rule,
TaskListMarker(bool),
InlineMath(CowStr<'a>),
DisplayMath(CowStr<'a>),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct InlineTag<'a> {
pub raw: CowStr<'a>,
pub name: String,
pub value: Option<String>,
pub closing: bool,
}
fn split_inline_tags<'a>(source: &'a str, item: SpannedEvent<'a>) -> Vec<SpannedEvent<'a>> {
let Event::Text(text) = &item.event else {
return vec![item];
};
let Some(original) = source.get(item.span.clone()) else {
return vec![item];
};
if original != text.as_ref() {
return vec![item];
}
if item.span.start > 0
&& source.as_bytes().get(item.span.start - 1) == Some(&b'\\')
&& original.starts_with(['#', '['])
{
return vec![item];
}
let mut offset = item.span.start;
tokenize_inline(original)
.into_iter()
.map(|token| match token {
InlineToken::Text(text) => {
let start = offset;
offset += text.len();
SpannedEvent {
event: Event::Text(CowStr::from(text)),
span: start..offset,
}
}
InlineToken::Hashtag(tag) => {
let start = offset;
offset += tag.len() + 1;
SpannedEvent {
event: Event::Hashtag(CowStr::from(tag)),
span: start..offset,
}
}
InlineToken::WikiLink(target) => {
let start = offset;
offset += target.len() + 4;
SpannedEvent {
event: Event::WikiLink(CowStr::from(target)),
span: start..offset,
}
}
InlineToken::Tag {
raw,
name,
value,
closing,
} => {
let start = offset;
offset += raw.len();
SpannedEvent {
event: Event::InlineTag(InlineTag {
raw: CowStr::from(raw),
name,
value,
closing,
}),
span: start..offset,
}
}
})
.collect()
}
impl<'a> Event<'a> {
fn from_cmark(event: CmarkEvent<'a>) -> Self {
match event {
CmarkEvent::Start(tag) => Self::Start(Container::from_cmark(tag)),
CmarkEvent::End(tag) => Self::End(ContainerEnd::from_cmark(tag)),
CmarkEvent::Text(value) => Self::Text(value),
CmarkEvent::Code(value) => Self::Code(value),
CmarkEvent::Html(value) => Self::Html(value),
CmarkEvent::InlineHtml(value) => Self::InlineHtml(value),
CmarkEvent::FootnoteReference(value) => Self::FootnoteReference(value),
CmarkEvent::SoftBreak => Self::SoftBreak,
CmarkEvent::HardBreak => Self::HardBreak,
CmarkEvent::Rule => Self::Rule,
CmarkEvent::TaskListMarker(checked) => Self::TaskListMarker(checked),
CmarkEvent::InlineMath(value) => Self::InlineMath(value),
CmarkEvent::DisplayMath(value) => Self::DisplayMath(value),
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum Container<'a> {
Paragraph,
Heading(HeadingLevel),
BlockQuote,
CodeBlock(Option<CowStr<'a>>),
HtmlBlock,
List(Option<u64>),
Item,
FootnoteDefinition(CowStr<'a>),
DefinitionList,
DefinitionListTitle,
DefinitionListDefinition,
Table(Vec<Alignment>),
TableHead,
TableRow,
TableCell,
Emphasis,
Strong,
Strikethrough,
Superscript,
Subscript,
Link {
target: CowStr<'a>,
title: CowStr<'a>,
},
Image {
target: CowStr<'a>,
title: CowStr<'a>,
},
MetadataBlock,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum HeadingLevel {
H1,
H2,
H3,
H4,
H5,
H6,
}
impl From<MarkdownHeadingLevel> for HeadingLevel {
fn from(value: MarkdownHeadingLevel) -> Self {
match value {
MarkdownHeadingLevel::H1 => Self::H1,
MarkdownHeadingLevel::H2 => Self::H2,
MarkdownHeadingLevel::H3 => Self::H3,
MarkdownHeadingLevel::H4 => Self::H4,
MarkdownHeadingLevel::H5 => Self::H5,
MarkdownHeadingLevel::H6 => Self::H6,
}
}
}
impl<'a> Container<'a> {
fn from_cmark(tag: Tag<'a>) -> Self {
match tag {
Tag::Paragraph => Self::Paragraph,
Tag::Heading { level, .. } => Self::Heading(level.into()),
Tag::BlockQuote(_) => Self::BlockQuote,
Tag::CodeBlock(kind) => Self::CodeBlock(match kind {
CodeBlockKind::Indented => None,
CodeBlockKind::Fenced(info) => Some(info),
}),
Tag::HtmlBlock => Self::HtmlBlock,
Tag::List(first) => Self::List(first),
Tag::Item => Self::Item,
Tag::FootnoteDefinition(name) => Self::FootnoteDefinition(name),
Tag::DefinitionList => Self::DefinitionList,
Tag::DefinitionListTitle => Self::DefinitionListTitle,
Tag::DefinitionListDefinition => Self::DefinitionListDefinition,
Tag::Table(alignments) => {
Self::Table(alignments.into_iter().map(Alignment::from).collect())
}
Tag::TableHead => Self::TableHead,
Tag::TableRow => Self::TableRow,
Tag::TableCell => Self::TableCell,
Tag::Emphasis => Self::Emphasis,
Tag::Strong => Self::Strong,
Tag::Strikethrough => Self::Strikethrough,
Tag::Superscript => Self::Superscript,
Tag::Subscript => Self::Subscript,
Tag::Link {
dest_url, title, ..
} => Self::Link {
target: dest_url,
title,
},
Tag::Image {
dest_url, title, ..
} => Self::Image {
target: dest_url,
title,
},
Tag::MetadataBlock(_) => Self::MetadataBlock,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ContainerEnd {
Paragraph,
Heading,
BlockQuote,
CodeBlock,
HtmlBlock,
List(bool),
Item,
FootnoteDefinition,
DefinitionList,
DefinitionListTitle,
DefinitionListDefinition,
Table,
TableHead,
TableRow,
TableCell,
Emphasis,
Strong,
Strikethrough,
Superscript,
Subscript,
Link,
Image,
MetadataBlock,
}
impl ContainerEnd {
fn from_cmark(tag: TagEnd) -> Self {
match tag {
TagEnd::Paragraph => Self::Paragraph,
TagEnd::Heading(_) => Self::Heading,
TagEnd::BlockQuote(_) => Self::BlockQuote,
TagEnd::CodeBlock => Self::CodeBlock,
TagEnd::HtmlBlock => Self::HtmlBlock,
TagEnd::List(ordered) => Self::List(ordered),
TagEnd::Item => Self::Item,
TagEnd::FootnoteDefinition => Self::FootnoteDefinition,
TagEnd::DefinitionList => Self::DefinitionList,
TagEnd::DefinitionListTitle => Self::DefinitionListTitle,
TagEnd::DefinitionListDefinition => Self::DefinitionListDefinition,
TagEnd::Table => Self::Table,
TagEnd::TableHead => Self::TableHead,
TagEnd::TableRow => Self::TableRow,
TagEnd::TableCell => Self::TableCell,
TagEnd::Emphasis => Self::Emphasis,
TagEnd::Strong => Self::Strong,
TagEnd::Strikethrough => Self::Strikethrough,
TagEnd::Superscript => Self::Superscript,
TagEnd::Subscript => Self::Subscript,
TagEnd::Link => Self::Link,
TagEnd::Image => Self::Image,
TagEnd::MetadataBlock(_) => Self::MetadataBlock,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Alignment {
None,
Left,
Center,
Right,
}
impl From<MarkdownAlignment> for Alignment {
fn from(value: MarkdownAlignment) -> Self {
match value {
MarkdownAlignment::None => Self::None,
MarkdownAlignment::Left => Self::Left,
MarkdownAlignment::Center => Self::Center,
MarkdownAlignment::Right => Self::Right,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ParseError {
message: String,
}
impl std::fmt::Display for ParseError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl std::error::Error for ParseError {}
pub fn is_structural_tag_name(name: &str) -> bool {
structure::is_structural_tag_name(name)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_markdown_and_structural_nodes() {
let document = parse("# Title\n\n[box title=Info width=full]\n**body**\n[/box]").unwrap();
assert_eq!(document.nodes().len(), 2);
assert!(matches!(document.nodes()[0], Node::Markdown(_)));
assert!(matches!(
document.nodes()[1],
Node::Box {
spec: BoxSpec {
width: WidthMode::Full,
..
},
..
}
));
}
#[test]
fn parses_right_aligned_blocks_as_structure() {
let document = parse("[right]aligned[/right]").unwrap();
assert!(matches!(document.nodes(), [Node::Right { .. }]));
assert!(parse("[right gap=2]x[/right]").is_err());
}
#[test]
fn parses_structural_shorthand_aliases() {
let document = parse(concat!(
"[c]center[/c]",
"[r]right[/r]",
"[cols gap=1 px=2]",
"[col width=1fr p=1]left[/col]",
"[col width=1fr]right[/col]",
"[/cols]"
))
.unwrap();
assert!(matches!(document.nodes()[0], Node::Center { .. }));
assert!(matches!(document.nodes()[1], Node::Right { .. }));
assert!(matches!(document.nodes()[2], Node::Columns { .. }));
assert!(is_structural_tag_name("c"));
assert!(is_structural_tag_name("r"));
assert!(is_structural_tag_name("cols"));
assert!(is_structural_tag_name("col"));
assert!(parse("[c gap=1]x[/c]").is_err());
assert!(parse("[cols][col]a[/col x][/cols]").is_err());
}
#[test]
fn keeps_source_spans_for_navigation() {
let document = parse("alpha\n\n[link](target)").unwrap();
let Node::Markdown(markdown) = &document.nodes()[0] else {
panic!("markdown node");
};
assert!(markdown.events().iter().any(|event| {
matches!(&event.event, Event::Text(text) if text.as_ref() == "link")
&& &markdown.source()[event.span.clone()] == "link"
}));
}
#[test]
fn promotes_bbcode_to_ast_but_preserves_escaped_tags() {
let document = parse("[red]hot[/red] \\[blue\\]").unwrap();
let Node::Markdown(markdown) = &document.nodes()[0] else {
panic!("markdown node");
};
assert!(markdown.events().iter().any(|event| {
matches!(
&event.event,
Event::InlineTag(InlineTag {
name,
closing: false,
..
}) if name == "red"
)
}));
assert!(!markdown.events().iter().any(|event| {
matches!(
&event.event,
Event::InlineTag(InlineTag { name, .. }) if name == "blue"
)
}));
}
#[test]
fn promotes_hashtags_and_wikilinks_to_semantic_events() {
let source = "# Heading\n\nSee #开发/日志 and [[项目计划]], not word#part. `#code [[raw]]`";
let document = parse(source).unwrap();
let Node::Markdown(markdown) = &document.nodes()[0] else {
panic!("markdown node");
};
assert!(markdown.events().iter().any(|item| {
matches!(&item.event, Event::Hashtag(tag) if tag.as_ref() == "开发/日志")
&& &source[item.span.clone()] == "#开发/日志"
}));
assert!(markdown.events().iter().any(|item| {
matches!(&item.event, Event::WikiLink(target) if target.as_ref() == "项目计划")
&& &source[item.span.clone()] == "[[项目计划]]"
}));
assert!(!markdown.events().iter().any(|item| {
matches!(&item.event, Event::Hashtag(tag) if tag.as_ref() == "Heading" || tag.as_ref() == "part" || tag.as_ref() == "code")
}));
assert!(!markdown.events().iter().any(|item| {
matches!(&item.event, Event::WikiLink(target) if target.as_ref() == "raw")
}));
}
#[test]
fn escaped_hashtags_and_wikilinks_stay_literal() {
let document = parse(r"\#literal \[\[literal\]\]").unwrap();
let Node::Markdown(markdown) = &document.nodes()[0] else {
panic!("markdown node");
};
assert!(!markdown
.events()
.iter()
.any(|item| { matches!(item.event, Event::Hashtag(_) | Event::WikiLink(_)) }));
}
}