use std::collections::HashMap;
#[cfg(feature = "python")]
use pyo3::prelude::*;
#[cfg(feature = "python")]
use pyo3::types::{PyDict, PyList};
use rushdown::as_kind_data;
use rushdown::ast::{self, KindData, NodeRef, TextQualifier};
use rushdown_emoji::Emoji;
use crate::extensions::{BlockComponent, InlineComponent, Slot, SpanAttributes};
#[cfg_attr(feature = "python", pyclass(from_py_object))]
#[derive(Clone, Debug)]
pub struct AstNode {
pub kind: String,
pub children: Vec<AstNode>,
pub text: Option<String>,
pub attributes: HashMap<String, String>,
pub metadata: Option<HashMap<String, String>>,
pub props: Option<ast::Meta>,
}
impl AstNode {
pub fn new(
kind: String,
children: Vec<AstNode>,
text: Option<String>,
attributes: HashMap<String, String>,
metadata: Option<HashMap<String, String>>,
) -> Self {
AstNode {
kind,
children,
text,
attributes,
metadata,
props: None,
}
}
pub fn walk(&self) -> Vec<AstNode> {
let mut result = Vec::new();
self.walk_recursive(&mut result);
result
}
fn walk_recursive(&self, result: &mut Vec<AstNode>) {
result.push(self.clone());
for child in &self.children {
child.walk_recursive(result);
}
}
#[cfg(any(test, feature = "python"))]
pub(crate) fn repr_string(&self) -> String {
if let Some(ref t) = self.text {
format!("AstNode(kind={:?}, text={:?})", self.kind, t)
} else {
format!(
"AstNode(kind={:?}, children={})",
self.kind,
self.children.len()
)
}
}
}
#[cfg(feature = "python")]
#[pymethods]
impl AstNode {
#[new]
#[pyo3(signature = (kind, children=None, text=None, attributes=None, metadata=None))]
fn py_new(
kind: String,
children: Option<Vec<AstNode>>,
text: Option<String>,
attributes: Option<HashMap<String, String>>,
metadata: Option<HashMap<String, String>>,
) -> Self {
AstNode::new(
kind,
children.unwrap_or_default(),
text,
attributes.unwrap_or_default(),
metadata,
)
}
#[getter]
fn get_kind(&self) -> String {
self.kind.clone()
}
#[setter]
fn set_kind(&mut self, value: String) {
self.kind = value;
}
#[getter]
fn get_children(&self) -> Vec<AstNode> {
self.children.clone()
}
#[setter]
fn set_children(&mut self, value: Vec<AstNode>) {
self.children = value;
}
#[getter]
fn get_text(&self) -> Option<String> {
self.text.clone()
}
#[setter]
fn set_text(&mut self, value: Option<String>) {
self.text = value;
}
#[getter]
fn get_attributes(&self) -> HashMap<String, String> {
self.attributes.clone()
}
#[setter]
fn set_attributes(&mut self, value: HashMap<String, String>) {
self.attributes = value;
}
#[getter]
fn get_metadata(&self) -> Option<HashMap<String, String>> {
self.metadata.clone()
}
#[setter]
fn set_metadata(&mut self, value: Option<HashMap<String, String>>) {
self.metadata = value;
}
#[getter]
fn get_props(&self, py: Python<'_>) -> PyResult<Option<Py<PyDict>>> {
match &self.props {
None => Ok(None),
Some(ast::Meta::Mapping(map)) => {
let dict = PyDict::new(py);
for (k, v) in map.iter() {
dict.set_item(k, meta_to_py(py, v)?)?;
}
Ok(Some(dict.unbind()))
}
Some(other) => {
let dict = PyDict::new(py);
dict.set_item("value", meta_to_py(py, other)?)?;
Ok(Some(dict.unbind()))
}
}
}
#[pyo3(name = "walk")]
fn py_walk(&self) -> Vec<AstNode> {
self.walk()
}
fn __repr__(&self) -> String {
self.repr_string()
}
}
#[cfg_attr(feature = "python", pyclass(skip_from_py_object))]
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Heading {
pub level: u8,
pub id: String,
pub text: String,
pub children: Vec<Heading>,
}
#[cfg(feature = "python")]
#[pymethods]
impl Heading {
#[getter]
fn level(&self) -> u8 {
self.level
}
#[getter]
fn id(&self) -> String {
self.id.clone()
}
#[getter]
fn text(&self) -> String {
self.text.clone()
}
#[getter]
fn children(&self) -> Vec<Heading> {
self.children.clone()
}
fn __repr__(&self) -> String {
format!(
"Heading(level={}, id={:?}, text={:?}, children={})",
self.level,
self.id,
self.text,
self.children.len()
)
}
}
pub(crate) fn collect_headings(
arena: &ast::Arena,
document_ref: NodeRef,
source: &str,
) -> Vec<Heading> {
let mut refs: Vec<NodeRef> = Vec::new();
crate::extensions::collect_headings(arena, document_ref, &mut refs);
refs.into_iter()
.map(|heading_ref| {
let node = &arena[heading_ref];
let level = match node.kind_data() {
KindData::Heading(h) => h.level(),
_ => 0,
};
let id = node
.attributes()
.get("id")
.map(|v| v.str(source).to_string())
.unwrap_or_default();
Heading {
level,
id,
text: crate::extensions::heading_text(arena, heading_ref, source),
children: Vec::new(),
}
})
.collect()
}
pub(crate) fn build_toc(flat: &[Heading]) -> Vec<Heading> {
let mut roots: Vec<Heading> = Vec::new();
let mut stack: Vec<usize> = Vec::new();
for heading in flat {
let node = Heading {
level: heading.level,
id: heading.id.clone(),
text: heading.text.clone(),
children: Vec::new(),
};
while !stack.is_empty() {
let parent = heading_at(&roots, &stack);
if parent.level < node.level {
break;
}
stack.pop();
}
if stack.is_empty() {
roots.push(node);
stack.push(roots.len() - 1);
} else {
let parent = heading_at_mut(&mut roots, &stack);
parent.children.push(node);
let index = parent.children.len() - 1;
stack.push(index);
}
}
roots
}
fn heading_at<'a>(roots: &'a [Heading], path: &[usize]) -> &'a Heading {
let mut node = &roots[path[0]];
for &index in &path[1..] {
node = &node.children[index];
}
node
}
fn heading_at_mut<'a>(roots: &'a mut [Heading], path: &[usize]) -> &'a mut Heading {
let mut node = &mut roots[path[0]];
for &index in &path[1..] {
node = &mut node.children[index];
}
node
}
#[cfg_attr(feature = "python", pyclass)]
pub struct ParseResult {
pub root: AstNode,
pub headings: Vec<Heading>,
pub toc: Vec<Heading>,
pub summary: Option<String>,
pub frontmatter: Option<ast::Meta>,
}
#[cfg(feature = "python")]
#[pymethods]
impl ParseResult {
#[getter]
fn root(&self) -> AstNode {
self.root.clone()
}
#[getter]
fn headings(&self) -> Vec<Heading> {
self.headings.clone()
}
#[getter]
fn toc(&self) -> Vec<Heading> {
self.toc.clone()
}
#[getter]
fn summary(&self) -> Option<String> {
self.summary.clone()
}
#[getter]
fn frontmatter(&self, py: Python<'_>) -> PyResult<Option<Py<PyDict>>> {
match &self.frontmatter {
None => Ok(None),
Some(ast::Meta::Mapping(map)) => {
let dict = PyDict::new(py);
for (k, v) in map.iter() {
dict.set_item(k, meta_to_py(py, v)?)?;
}
Ok(Some(dict.unbind()))
}
Some(other) => {
let dict = PyDict::new(py);
dict.set_item("value", meta_to_py(py, other)?)?;
Ok(Some(dict.unbind()))
}
}
}
fn __repr__(&self) -> String {
format!(
"ParseResult(root={:?}, frontmatter={})",
self.root.kind,
if self.frontmatter.is_some() {
"{...}"
} else {
"None"
}
)
}
}
#[cfg(feature = "python")]
pub(crate) fn meta_to_py(py: Python<'_>, meta: &ast::Meta) -> PyResult<Py<PyAny>> {
let value: Py<PyAny> = match meta {
ast::Meta::Null => py.None(),
ast::Meta::Bool(b) => b.into_pyobject(py)?.to_owned().into_any().unbind(),
ast::Meta::Int(i) => i.into_pyobject(py)?.into_any().unbind(),
ast::Meta::Float(f) => f.into_pyobject(py)?.into_any().unbind(),
ast::Meta::String(s) => s.into_pyobject(py)?.into_any().unbind(),
ast::Meta::Sequence(items) => {
let list = PyList::empty(py);
for item in items {
list.append(meta_to_py(py, item)?)?;
}
list.into_any().unbind()
}
ast::Meta::Mapping(map) => {
let dict = PyDict::new(py);
for (k, v) in map.iter() {
dict.set_item(k, meta_to_py(py, v)?)?;
}
dict.into_any().unbind()
}
};
Ok(value)
}
pub(crate) fn document_meta(arena: &ast::Arena, document_ref: NodeRef) -> Option<ast::Meta> {
let node = &arena[document_ref];
if !matches!(node.kind_data(), KindData::Document(_)) {
return None;
}
let meta = as_kind_data!(arena, document_ref, Document).metadata();
if meta.is_empty() {
return None;
}
Some(ast::Meta::Mapping(meta.clone()))
}
fn kind_name(node: &ast::Node) -> &'static str {
match node.kind_data() {
KindData::Document(_) => "document",
KindData::Paragraph(_) => "paragraph",
KindData::Heading(_) => "heading",
KindData::Blockquote(_) => "blockquote",
KindData::List(_) => "list",
KindData::ListItem(_) => "list_item",
KindData::CodeBlock(_) => "code_block",
KindData::HtmlBlock(_) => "html_block",
KindData::ThematicBreak(_) => "thematic_break",
KindData::Text(_) => "text",
KindData::Emphasis(_) => "emphasis",
KindData::Strong(_) => "strong",
KindData::Link(_) => "link",
KindData::Image(_) => "image",
KindData::CodeSpan(_) => "code_span",
KindData::RawHtml(_) => "raw_html",
KindData::LinkReferenceDefinition(_) => "link_reference_definition",
KindData::Table(_) => "table",
KindData::TableHeader(_) => "table_header",
KindData::TableBody(_) => "table_body",
KindData::TableRow(_) => "table_row",
KindData::TableCell(_) => "table_cell",
KindData::Strikethrough(_) => "strikethrough",
KindData::Extension(ext) => {
if (ext.as_ref() as &dyn std::any::Any)
.downcast_ref::<Emoji>()
.is_some()
{
"emoji"
} else if (ext.as_ref() as &dyn std::any::Any)
.downcast_ref::<BlockComponent>()
.is_some()
{
"block_component"
} else if (ext.as_ref() as &dyn std::any::Any)
.downcast_ref::<InlineComponent>()
.is_some()
{
"inline_component"
} else if (ext.as_ref() as &dyn std::any::Any)
.downcast_ref::<Slot>()
.is_some()
{
"slot"
} else if (ext.as_ref() as &dyn std::any::Any)
.downcast_ref::<SpanAttributes>()
.is_some()
{
"span_attributes"
} else {
"unknown"
}
}
_ => "unknown",
}
}
const RAW_HTML_PLACEHOLDER: &str = "<!-- raw HTML omitted -->";
fn node_text(node: &ast::Node, source: &str) -> Option<String> {
match node.kind_data() {
KindData::Text(t) => Some(t.str(source).to_string()),
KindData::CodeSpan(c) => Some(c.str(source).to_string()),
KindData::RawHtml(_) => Some(RAW_HTML_PLACEHOLDER.to_string()),
KindData::HtmlBlock(_) => Some(format!("{RAW_HTML_PLACEHOLDER}\n")),
KindData::Extension(ext) => (ext.as_ref() as &dyn std::any::Any)
.downcast_ref::<Emoji>()
.map(|emoji| emoji.as_str().to_string()),
_ => None,
}
}
fn is_softbreak(node: &ast::Node) -> bool {
if let KindData::Text(t) = node.kind_data() {
t.has_qualifiers(TextQualifier::SOFT_LINE_BREAK)
} else {
false
}
}
fn is_hardbreak(node: &ast::Node) -> bool {
if let KindData::Text(t) = node.kind_data() {
t.has_qualifiers(TextQualifier::HARD_LINE_BREAK)
} else {
false
}
}
fn node_attributes(node: &ast::Node, source: &str) -> HashMap<String, String> {
let mut attrs = HashMap::new();
match node.kind_data() {
KindData::Link(link) => {
attrs.insert(
"href".to_string(),
link.destination().str(source).to_string(),
);
if let Some(title) = link.title().filter(|t| !t.str(source).is_empty()) {
attrs.insert("title".to_string(), title.str(source).to_string());
}
}
KindData::Image(img) => {
attrs.insert("src".to_string(), img.destination().str(source).to_string());
if let Some(title) = img.title().filter(|t| !t.str(source).is_empty()) {
attrs.insert("title".to_string(), title.str(source).to_string());
}
}
KindData::Heading(h) => {
attrs.insert("level".to_string(), h.level().to_string());
}
KindData::Extension(ext) => {
if let Some(emoji) = (ext.as_ref() as &dyn std::any::Any).downcast_ref::<Emoji>() {
if let Some(sc) = emoji.shortcode() {
attrs.insert("shortcode".to_string(), sc.to_string());
}
} else if let Some(bc) =
(ext.as_ref() as &dyn std::any::Any).downcast_ref::<BlockComponent>()
{
attrs.insert("name".to_string(), bc.name.clone());
} else if let Some(ic) =
(ext.as_ref() as &dyn std::any::Any).downcast_ref::<InlineComponent>()
{
attrs.insert("name".to_string(), ic.name.clone());
} else if let Some(slot) = (ext.as_ref() as &dyn std::any::Any).downcast_ref::<Slot>() {
attrs.insert("name".to_string(), slot.name.clone());
}
}
_ => {}
}
for (key, value) in node.attributes().iter() {
attrs.insert(key.to_string(), value.str(source).to_string());
}
attrs
}
fn node_props(node: &ast::Node) -> Option<ast::Meta> {
let KindData::Extension(ext) = node.kind_data() else {
return None;
};
let bc = (ext.as_ref() as &dyn std::any::Any).downcast_ref::<BlockComponent>()?;
let ast::Meta::Mapping(map) = bc.props.as_ref()? else {
return None;
};
let inline: std::collections::HashSet<&str> =
node.attributes().iter().map(|(k, _)| k.as_str()).collect();
let mut filtered = rushdown::util::StringMap::default();
for (key, value) in map.iter() {
if inline.contains(key.as_str()) {
continue;
}
filtered.insert(key.clone(), value.clone());
}
if filtered.is_empty() {
None
} else {
Some(ast::Meta::Mapping(filtered))
}
}
pub(crate) fn arena_to_ast_node(arena: &ast::Arena, node_ref: NodeRef, source: &str) -> AstNode {
let node = &arena[node_ref];
let kind = kind_name(node).to_string();
let text = node_text(node, source);
let attributes = node_attributes(node, source);
let props = node_props(node);
let metadata = if let KindData::Document(_) = node.kind_data() {
let meta = as_kind_data!(arena, node_ref, Document).metadata();
if meta.is_empty() {
None
} else {
let mut map = HashMap::new();
for (k, v) in meta.iter() {
map.insert(k.to_string(), v.as_str().unwrap_or("").to_string());
}
Some(map)
}
} else {
None
};
let mut children = Vec::new();
let mut child = node.first_child();
while let Some(child_ref) = child {
children.push(arena_to_ast_node(arena, child_ref, source));
let child_node = &arena[child_ref];
if is_softbreak(child_node) {
children.push(AstNode {
kind: "softbreak".to_string(),
children: Vec::new(),
text: None,
attributes: HashMap::new(),
metadata: None,
props: None,
});
} else if is_hardbreak(child_node) {
children.push(AstNode {
kind: "hardbreak".to_string(),
children: Vec::new(),
text: None,
attributes: HashMap::new(),
metadata: None,
props: None,
});
}
child = arena[child_ref].next_sibling();
}
AstNode {
kind,
children,
text,
attributes,
metadata,
props,
}
}
pub(crate) fn extract_summary_blocks(
arena: &ast::Arena,
document_ref: NodeRef,
source: &str,
) -> Option<Vec<AstNode>> {
let mut before = Vec::new();
let mut child = arena[document_ref].first_child();
while let Some(child_ref) = child {
if crate::extensions::is_more_marker(arena, child_ref, source) {
return Some(before);
}
before.push(arena_to_ast_node(arena, child_ref, source));
child = arena[child_ref].next_sibling();
}
None
}
#[cfg(test)]
mod toc_tests {
use super::*;
fn flat(level: u8, id: &str) -> Heading {
Heading {
level,
id: id.to_string(),
text: id.to_string(),
children: Vec::new(),
}
}
fn ids(headings: &[Heading]) -> Vec<&str> {
headings.iter().map(|h| h.id.as_str()).collect()
}
#[test]
fn empty_input_yields_empty_tree() {
assert!(build_toc(&[]).is_empty());
}
#[test]
fn omep_example_nests_as_specified() {
let flat_list = vec![
flat(1, "title"),
flat(2, "setup"),
flat(2, "usage"),
flat(3, "cli"),
flat(3, "library"),
flat(2, "faq"),
];
let toc = build_toc(&flat_list);
assert_eq!(ids(&toc), ["title"]);
assert_eq!(ids(&toc[0].children), ["setup", "usage", "faq"]);
assert_eq!(ids(&toc[0].children[1].children), ["cli", "library"]);
assert!(toc[0].children[0].children.is_empty());
}
#[test]
fn level_skips_are_tolerated() {
let toc = build_toc(&[flat(1, "a"), flat(3, "b")]);
assert_eq!(ids(&toc), ["a"]);
assert_eq!(ids(&toc[0].children), ["b"]);
assert_eq!(toc[0].children[0].level, 3);
}
#[test]
fn siblings_at_top_level_are_multiple_roots() {
let toc = build_toc(&[flat(1, "a"), flat(1, "b"), flat(2, "b1")]);
assert_eq!(ids(&toc), ["a", "b"]);
assert!(toc[0].children.is_empty());
assert_eq!(ids(&toc[1].children), ["b1"]);
}
#[test]
fn shallower_heading_pops_open_ancestors() {
let toc = build_toc(&[flat(2, "a"), flat(4, "a1"), flat(3, "a2"), flat(2, "b")]);
assert_eq!(ids(&toc), ["a", "b"]);
assert_eq!(ids(&toc[0].children), ["a1", "a2"]);
}
#[test]
fn document_starting_deep_still_has_roots() {
let toc = build_toc(&[flat(3, "a"), flat(2, "b")]);
assert_eq!(ids(&toc), ["a", "b"]);
}
#[test]
fn flat_entries_are_left_untouched() {
let flat_list = vec![flat(1, "a"), flat(2, "b")];
let _ = build_toc(&flat_list);
assert!(flat_list.iter().all(|h| h.children.is_empty()));
}
}
#[cfg(all(test, feature = "python"))]
mod tests {
use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString};
use super::*;
use rushdown::ast::Meta;
use rushdown::util::StringMap;
#[test]
fn meta_to_py_scalars_preserve_native_types() {
Python::attach(|py| {
assert!(meta_to_py(py, &Meta::Null).unwrap().is_none(py));
let b = meta_to_py(py, &Meta::Bool(true)).unwrap();
let b = b.bind(py);
assert!(b.is_instance_of::<PyBool>());
assert!(b.extract::<bool>().unwrap());
let i = meta_to_py(py, &Meta::Int(5)).unwrap();
let i = i.bind(py);
assert!(i.is_instance_of::<PyInt>());
assert_eq!(i.extract::<i64>().unwrap(), 5);
let f = meta_to_py(py, &Meta::Float(1.5)).unwrap();
let f = f.bind(py);
assert!(f.is_instance_of::<PyFloat>());
assert_eq!(f.extract::<f64>().unwrap(), 1.5);
let s = meta_to_py(py, &Meta::String("hi".to_string())).unwrap();
let s = s.bind(py);
assert!(s.is_instance_of::<PyString>());
assert_eq!(s.extract::<String>().unwrap(), "hi");
});
}
#[test]
fn meta_to_py_sequence_becomes_list() {
Python::attach(|py| {
let seq = Meta::Sequence(vec![Meta::Int(1), Meta::String("two".to_string())]);
let obj = meta_to_py(py, &seq).unwrap();
let list = obj.bind(py);
assert!(list.is_instance_of::<PyList>());
let list = list.cast::<PyList>().unwrap();
assert_eq!(list.len(), 2);
assert_eq!(list.get_item(0).unwrap().extract::<i64>().unwrap(), 1);
assert_eq!(
list.get_item(1).unwrap().extract::<String>().unwrap(),
"two"
);
});
}
#[test]
fn meta_to_py_mapping_becomes_nested_dict() {
Python::attach(|py| {
let mut inner = StringMap::default();
inner.insert("name", Meta::String("Ada".to_string()));
let mut outer = StringMap::default();
outer.insert("author", Meta::Mapping(inner));
outer.insert("draft", Meta::Bool(false));
let obj = meta_to_py(py, &Meta::Mapping(outer)).unwrap();
let dict = obj.bind(py);
assert!(dict.is_instance_of::<PyDict>());
let dict = dict.cast::<PyDict>().unwrap();
let author = dict.get_item("author").unwrap().unwrap();
let author = author.cast::<PyDict>().unwrap();
assert_eq!(
author
.get_item("name")
.unwrap()
.unwrap()
.extract::<String>()
.unwrap(),
"Ada"
);
assert!(
!dict
.get_item("draft")
.unwrap()
.unwrap()
.extract::<bool>()
.unwrap()
);
});
}
}