pub mod code;
mod footnotes;
mod frontmatter;
mod math;
mod media;
mod tables;
use comrak::options::{Extension, Parse};
pub use frontmatter::markdown_with_toml_frontmatter;
use comrak::markdown_to_html as comrak_markdown_to_html;
use comrak::{Arena, Options};
use comrak::{escape_commonmark_link_destination, format_commonmark, parse_document};
use kuchiki::NodeRef;
use kuchiki::traits::TendrilSink;
use super::{dom, patterns, serialize};
use crate::MarkdownOptions;
#[derive(Clone, Copy, Default)]
pub struct RenderContext {
in_pre: bool,
list_depth: usize,
}
impl RenderContext {
fn render_children(self, node: &NodeRef) -> String {
let mut output = String::new();
for child in node.children() {
output.push_str(&self.render_node(&child));
}
output
}
fn render_node(self, node: &NodeRef) -> String {
match node.as_text() {
Some(text) => {
let text = text.borrow();
if self.in_pre { text.to_string() } else { patterns::normalize_spaces(&text) }
}
None => match code::is_highlighter_chrome(node) {
true => String::new(),
false => match dom::node_name(node).as_str() {
"h1" => block(format!("# {}", self.inline_children(node))),
"h2" => block(format!("## {}", self.inline_children(node))),
"h3" => block(format!("### {}", self.inline_children(node))),
"h4" => block(format!("#### {}", self.inline_children(node))),
"h5" => block(format!("##### {}", self.inline_children(node))),
"h6" => block(format!("###### {}", self.inline_children(node))),
"p" => block(self.inline_children(node)),
"br" => " \n".to_string(),
"strong" | "b" => format!("**{}**", self.inline_children(node)),
"em" | "i" => format!("*{}*", self.inline_children(node)),
"mark" => wrap_inline("==", self.inline_children(node)),
"del" | "s" | "strike" => wrap_inline("~~", self.inline_children(node)),
"sup"
if dom::class_id_string(node)
.to_ascii_lowercase()
.split_whitespace()
.any(|token| token == "reference") =>
{
self.inline_children(node)
}
"sup" | "sub" | "svg" => {
serialize::serialize_node(node).unwrap_or_else(|_| self.render_children(node))
}
"code" if !self.in_pre => format!("`{}`", self.inline_children(node).replace('`', "\\`")),
"pre" => code::render_code_block(node, self),
"math" | "mjx-container" => {
math::render_math(node, self).unwrap_or_else(|| self.render_children(node))
}
"script" => math::render_math(node, self).unwrap_or_default(),
"span" | "img" if math::render_math(node, self).is_some() => {
math::render_math(node, self).unwrap_or_default()
}
"iframe" | "video" | "audio" | "object" | "embed" => media::render_embed(node).unwrap_or_default(),
"a" => {
let label = self.inline_children(node);
if is_heading_permalink(node, &label) {
String::new()
} else if label.is_empty() {
preserved_empty_inline(node)
} else if let Some(href) = dom::attr(node, "href") {
format!("[{}]({})", label, escape_commonmark_link_destination(&href))
} else {
label
}
}
"img" => space_before_ambiguous_image(node, media::render_image(node)),
"picture" => space_before_ambiguous_image(node, media::render_picture(node)),
"source" => String::new(),
"figure" => media::render_figure(node, self).unwrap_or_else(|| block(self.render_children(node))),
"blockquote" => match media::render_embed(node) {
Some(embed) => block(embed),
None => block(
normalize_markdown(&self.render_children(node))
.lines()
.map(|line| if line.trim().is_empty() { ">".to_string() } else { format!("> {line}") })
.collect::<Vec<_>>()
.join("\n"),
),
},
"ul" => self.render_list(node, false),
"ol" => self.render_list(node, true),
"li" => block(self.inline_children(node)),
"dl" => self.render_definition_list(node),
"table" => code::render_code_table(node, self).unwrap_or_else(|| tables::render_table(node, self)),
"div" => code::render_code_container(node, self).unwrap_or_else(|| self.render_children(node)),
"section" | "article" | "main" | "body" => self.render_children(node),
"figcaption" => block(self.inline_children(node)),
"hr" => "\n\n---\n\n".to_string(),
_ => self.render_children(node),
},
},
}
}
fn render_definition_list(self, node: &NodeRef) -> String {
let mut output = String::new();
for child in node.children().filter(|child| child.as_element().is_some()) {
match dom::node_name(&child).as_str() {
"dt" => output.push_str(&block(format!("**{}**", self.inline_children(&child)))),
"dd" => output.push_str(&block(self.inline_children(&child))),
_ => output.push_str(&self.render_node(&child)),
}
}
output
}
fn render_list(self, node: &NodeRef, ordered: bool) -> String {
let mut output = String::new();
let indent = " ".repeat(self.list_depth);
let mut index = 1;
for child in node.children().filter(|child| dom::node_name(child) == "li") {
let marker = if ordered {
let marker = format!("{index}.");
index += 1;
marker
} else {
"-".to_string()
};
let (label, nested) = RenderContext { list_depth: self.list_depth + 1, ..self }.render_list_item(&child);
output.push_str(&format!("{indent}{marker} {label}\n"));
for line in nested.lines().filter(|line| !line.trim().is_empty()) {
output.push_str(line);
output.push('\n');
}
}
output.push('\n');
output
}
fn render_list_item(self, node: &NodeRef) -> (String, String) {
let mut label = String::new();
let mut nested = String::new();
for child in node.children() {
match dom::node_name(&child).as_str() {
"ul" => nested.push_str(&self.render_list(&child, false)),
"ol" => nested.push_str(&self.render_list(&child, true)),
_ => label.push_str(&self.render_node(&child)),
}
}
(patterns::normalize_spaces(normalize_markdown(&label).trim()), nested)
}
fn inline_children(self, node: &NodeRef) -> String {
patterns::normalize_spaces(self.render_children(node).trim())
}
}
pub fn html_to_markdown(html: &str) -> String {
let document = kuchiki::parse_html().one(format!("<html><body>{html}</body></html>"));
let body = dom::select_nodes(&document, "body")
.into_iter()
.next()
.unwrap_or(document);
let footnotes = footnotes::FootnoteContext::extract(&body);
let mut output = RenderContext::default().render_children(&body);
output.push_str(&footnotes.render_defs());
output = normalize_markdown(&output);
fmt_with_comrak(&output)
}
pub fn markdown_to_html(markdown: &str, opts: &MarkdownOptions) -> String {
comrak_markdown_to_html(markdown, &comrak_opts(opts))
}
pub fn normalize_markdown(value: &str) -> String {
let mut output = String::new();
let mut blank_count = 0;
let mut in_fenced_code = false;
for line in value.lines() {
let line = line.trim_end();
if line.trim_start().starts_with("```") || line.trim_start().starts_with("~~~") {
blank_count = 0;
in_fenced_code = !in_fenced_code;
output.push_str(line.trim_start());
output.push('\n');
continue;
}
if in_fenced_code {
output.push_str(line);
output.push('\n');
continue;
}
if line.trim().is_empty() {
blank_count += 1;
if blank_count <= 1 {
output.push('\n');
}
} else {
blank_count = 0;
output.push_str(line);
output.push('\n');
}
}
output.trim().to_string()
}
pub fn render_children(node: &NodeRef, ctx: RenderContext) -> String {
ctx.render_children(node)
}
fn inline_children(node: &NodeRef, ctx: RenderContext) -> String {
ctx.inline_children(node)
}
fn block(value: String) -> String {
let value = value.trim();
if value.is_empty() { String::new() } else { format!("\n\n{value}\n\n") }
}
fn wrap_inline(marker: &str, value: String) -> String {
if value.is_empty() { String::new() } else { format!("{marker}{value}{marker}") }
}
fn preserved_empty_inline(node: &NodeRef) -> String {
let prev_text = node
.previous_sibling()
.and_then(|sibling| sibling.as_text().map(|text| text.borrow().to_string()))
.unwrap_or_default();
let next_text = node
.next_sibling()
.and_then(|sibling| sibling.as_text().map(|text| text.borrow().to_string()))
.unwrap_or_default();
if prev_text.ends_with(char::is_whitespace) && next_text.starts_with(char::is_whitespace) {
" ".to_string()
} else {
String::new()
}
}
fn space_before_ambiguous_image(node: &NodeRef, image: String) -> String {
if image.starts_with("![") && previous_inline_text(node).trim_end().ends_with('!') {
format!(" {image}")
} else {
image
}
}
fn previous_inline_text(node: &NodeRef) -> String {
let mut output = String::new();
let mut sibling = node.previous_sibling();
while let Some(current) = sibling {
output.insert_str(0, &dom::inner_text(¤t));
sibling = current.previous_sibling();
}
output
}
fn is_heading_permalink(node: &NodeRef, label: &str) -> bool {
let label = label.trim();
if !(label.is_empty() || matches!(label, "#" | "¶" | "§" | "Permalink")) {
return false;
}
let attrs = dom::class_id_string(node).to_ascii_lowercase();
attrs.contains("anchor")
|| attrs.contains("permalink")
|| dom::attr(node, "href").is_some_and(|href| href.starts_with('#'))
}
fn fmt_with_comrak(markdown: &str) -> String {
let arena = Arena::new();
let opts = Options {
extension: Extension {
footnotes: true,
math_dollars: true,
strikethrough: true,
table: true,
..Default::default()
},
parse: Parse { leave_footnote_definitions: true, ..Default::default() },
..Default::default()
};
let root = parse_document(&arena, markdown, &opts);
let mut output = String::new();
match format_commonmark(root, &opts, &mut output) {
Err(_) => markdown.to_string(),
Ok(_) => output.trim().to_string(),
}
}
fn comrak_opts(opts: &MarkdownOptions) -> Options<'static> {
let mut comrak = Options::default();
if opts.gfm {
comrak.extension.autolink = true;
comrak.extension.strikethrough = true;
comrak.extension.table = true;
comrak.extension.tagfilter = true;
comrak.extension.tasklist = true;
comrak.parse.tasklist_in_table = true;
}
if opts.footnotes {
comrak.extension.footnotes = true;
comrak.extension.inline_footnotes = true;
}
if opts.math {
comrak.extension.math_code = true;
comrak.extension.math_dollars = true;
}
comrak.render.r#unsafe = opts.allow_raw_html;
comrak
}
#[cfg(test)]
mod tests {
use super::{html_to_markdown, markdown_to_html};
use crate::{MarkdownOptions, ReadabilityOptions, extract};
#[test]
fn converts_representative_article_html() {
let markdown = html_to_markdown(
r#"<div><h1>Title</h1><p>Hello <strong>bold</strong> <a href="https://example.com">link</a>.</p><ul><li>One</li><li>Two</li></ul><pre><code>let x = 1;</code></pre></div>"#,
);
assert!(markdown.contains("# Title"));
assert!(markdown.contains("Hello **bold** [link](https://example.com)."));
assert!(markdown.contains("- One"));
assert!(markdown.contains(" let x = 1;"));
}
#[test]
fn preserves_nested_list_indentation() {
let markdown = html_to_markdown(
r#"<ul><li>Parent<ul><li>Child<ul><li>Grandchild</li></ul></li></ul></li><li>Sibling</li></ul>"#,
);
assert!(markdown.contains("- Parent\n - Child\n - Grandchild"), "{markdown}");
assert!(markdown.contains("- Sibling"), "{markdown}");
}
#[test]
fn converts_simple_tables_to_pipe_tables() {
let markdown = html_to_markdown(
r#"<table><thead><tr><th>Name</th><th>Value</th></tr></thead><tbody><tr><td>A</td><td>x|y</td></tr><tr><td>B</td><td><a href="https://example.com">z</a></td></tr></tbody></table>"#,
);
assert!(markdown.contains("| Name | Value |"));
assert!(markdown.contains("| --- | --- |"));
assert!(markdown.contains(r"| A | x\|y |"));
assert!(markdown.contains("| B | [z](https://example.com) |"));
}
#[test]
fn spaces_definition_lists() {
let markdown = html_to_markdown(
r#"<dl><dt><a href="struct.Value.html">Value</a></dt><dd>A borrowed value.</dd><dt><a href="trait.Serialize.html">Serialize</a></dt><dd>Serializes data.</dd></dl>"#,
);
assert!(
markdown.contains("**[Value](struct.Value.html)**\n\nA borrowed value.\n\n**[Serialize](trait.Serialize.html)**\n\nSerializes data."),
"{markdown}"
);
}
#[test]
fn renders_gfm_markdown_to_html_by_default() {
let html = markdown_to_html(
"| Done | Item |\n| --- | --- |\n| yes | ~~ship it~~ |\n\n- [x] publish\n",
&MarkdownOptions::default(),
);
assert!(html.contains("<table>"), "{html}");
assert!(html.contains("type=\"checkbox\""), "{html}");
assert!(html.contains("<del>ship it</del>"), "{html}");
}
#[test]
fn omits_raw_html_when_markdown_html_is_not_allowed() {
let html = markdown_to_html("<script>alert(1)</script>\n", &MarkdownOptions::default());
assert!(!html.contains("<script>"), "{html}");
assert!(html.contains("raw HTML omitted"), "{html}");
}
#[test]
fn allows_raw_html_when_configured() {
let html = markdown_to_html(
"<span data-x=\"1\">ok</span>\n",
&MarkdownOptions { allow_raw_html: true, ..MarkdownOptions::default() },
);
assert!(html.contains("<span data-x=\"1\">ok</span>"), "{html}");
}
#[test]
fn unwraps_layout_tables() {
let markdown = html_to_markdown(
r#"<table role="presentation"><tr><td><p>Left</p></td><td><p>Right <strong>side</strong></p></td></tr></table>"#,
);
assert!(!markdown.contains("| --- |"));
assert!(markdown.contains("Left"));
assert!(markdown.contains("Right **side**"));
}
#[test]
fn preserves_spanning_tables_as_html() {
let markdown =
html_to_markdown(r#"<table><tr><th colspan="2">Group</th></tr><tr><td>A</td><td>B</td></tr></table>"#);
assert!(markdown.contains("<table>"));
assert!(markdown.contains("colspan=\"2\""));
assert!(markdown.contains("<td>A</td>"));
}
#[test]
fn converts_numeric_footnotes() {
let markdown = html_to_markdown(
r##"<p>See<sup><a id="fnref:1" href="#fn:1">1</a></sup>.</p><ol><li id="fn:1">A note <a href="#fnref:1">↩</a></li></ol>"##,
);
assert!(markdown.contains("See[^1]."), "{markdown}");
assert!(markdown.contains("[^1]:\n A note"), "{markdown}");
assert!(!markdown.contains("↩"), "{markdown}");
assert!(!markdown.contains("[1](#fn:1)"), "{markdown}");
}
#[test]
fn converts_mediawiki_citations() {
let markdown = html_to_markdown(
r##"<p>Claim<sup class="reference"><a href="#cite_note-smith-2">[2]</a></sup>.</p><ol class="references"><li id="cite_note-smith-2"><span class="mw-cite-backlink"><a href="#cite_ref-smith-2">^</a></span>Smith source</li></ol>"##,
);
assert!(markdown.contains("Claim[^2]."), "{markdown}");
assert!(markdown.contains("[^2]:\n Smith source"), "{markdown}");
assert!(!markdown.contains("cite_note"), "{markdown}");
}
#[test]
fn converts_absolute_mediawiki_citation_fragments() {
let markdown = html_to_markdown(
r##"<p>Claim<sup class="reference"><a href="https://en.wikipedia.org/wiki/Mozilla#cite_note-smith-2">[2]</a></sup>.</p><ol class="references"><li id="cite_note-smith-2"><span class="mw-cite-backlink"><a href="#cite_ref-smith-2">^</a></span>Smith source</li></ol>"##,
);
assert!(markdown.contains("Claim[^2]."), "{markdown}");
assert!(markdown.contains("[^2]:\n Smith source"), "{markdown}");
assert!(!markdown.contains("<sup"), "{markdown}");
assert!(!markdown.contains("\\^"), "{markdown}");
}
#[test]
fn converts_google_docs_footnotes() {
let markdown = html_to_markdown(
r##"<p>Word<a id="ftnt_ref1" href="#ftnt1">[1]</a></p><div id="ftnt1"><p><a href="#ftnt_ref1">[1]</a> Google Docs note.</p></div>"##,
);
assert!(markdown.contains("Word[^1]"), "{markdown}");
assert!(markdown.contains("[^1]:\n Google Docs note."), "{markdown}");
assert!(!markdown.contains("ftnt_ref1"), "{markdown}");
}
#[test]
fn converts_mathml_to_latex() {
let markdown = html_to_markdown(
r#"<p>When <math><mi>a</mi><mo>≠</mo><mn>0</mn></math>, solve <math display="block"><mi>x</mi><mo>=</mo><mfrac><mrow><mo>−</mo><mi>b</mi></mrow><mrow><mn>2</mn><mi>a</mi></mrow></mfrac></math></p>"#,
);
assert!(markdown.contains("When $a \\ne 0$"), "{markdown}");
assert!(markdown.contains("$$\nx = \\frac{- b}{2 a}\n$$"), "{markdown}");
}
#[test]
fn extracts_katex_annotations_and_latex_attrs() {
let markdown = html_to_markdown(
r#"<p><span class="katex"><span class="katex-mathml"><math><semantics><mrow></mrow><annotation encoding="application/x-tex">E=mc^2</annotation></semantics></math></span></span> and <span data-latex="\alpha+\beta"></span></p>"#,
);
assert!(markdown.contains("$E=mc^2$"), "{markdown}");
assert!(markdown.contains("$\\alpha+\\beta$"), "{markdown}");
}
#[test]
fn unwraps_equation_tables_without_flattening_math() {
let markdown = html_to_markdown(
r#"<table><tr><td><math display="block"><mi>y</mi><mo>=</mo><msup><mi>x</mi><mn>2</mn></msup></math></td></tr></table>"#,
);
assert!(markdown.contains("$$\ny = x^{2}\n$$"), "{markdown}");
assert!(!markdown.contains("| --- |"), "{markdown}");
assert!(!markdown.contains("<table>"), "{markdown}");
}
#[test]
fn preserves_inline_highlight_and_strikethrough_semantics() {
let markdown = html_to_markdown(
r#"<p>Use <mark>exact match</mark>, not <del>approximate</del>, <s>old</s>, or <strike>stale</strike> terms.</p>"#,
);
assert!(
markdown.contains("Use ==exact match==, not ~~approximate~~, ~~old~~, or ~~stale~~ terms."),
"{markdown}"
);
}
#[test]
fn preserves_non_footnote_superscript_and_subscript_semantics() {
let markdown = html_to_markdown(r#"<p>Area is 10<sup>2</sup> m<sup>2</sup>; water is H<sub>2</sub>O.</p>"#);
assert!(
markdown.contains("Area is 10<sup>2</sup> m<sup>2</sup>; water is H<sub>2</sub>O."),
"{markdown}"
);
}
#[test]
fn unwraps_unlinked_mediawiki_reference_markers() {
let markdown = html_to_markdown(r#"<p>Book<sup class="reference nowrap"><span>: 227</span></sup>.</p>"#);
assert!(markdown.contains("Book: 227."), "{markdown}");
assert!(!markdown.contains("<sup"), "{markdown}");
}
#[test]
fn preserves_inline_svg_semantics() {
let markdown = html_to_markdown(
r#"<p>Status <svg viewBox="0 0 10 10" role="img" aria-label="circle"><circle cx="5" cy="5" r="4"></circle></svg> active.</p>"#,
);
assert!(markdown.contains("<svg "), "{markdown}");
assert!(markdown.contains(r#"viewBox="0 0 10 10""#), "{markdown}");
assert!(markdown.contains(r#"role="img""#), "{markdown}");
assert!(markdown.contains(r#"aria-label="circle""#), "{markdown}");
assert!(
markdown.contains(r#"<circle cx="5" cy="5" r="4"></circle>"#),
"{markdown}"
);
assert!(markdown.contains("active."), "{markdown}");
}
#[test]
fn removes_heading_permalink_anchors() {
let markdown =
html_to_markdown(r##"<h2><a class="heading-anchor" href="#overview">#</a> Overview</h2><p>Body.</p>"##);
assert!(markdown.contains("## Overview"), "{markdown}");
assert!(!markdown.contains("[#](#overview)"), "{markdown}");
}
#[test]
fn strips_wbr_without_inserting_spaces() {
let markdown = html_to_markdown(r#"<p>https://example.com/very<wbr>long<wbr>path</p>"#);
assert!(markdown.contains("https://example.com/verylongpath"), "{markdown}");
assert!(!markdown.contains("very long path"), "{markdown}");
}
#[test]
fn removes_empty_links_but_preserves_images() {
let markdown =
html_to_markdown(r#"<p>Before <a href="https://example.com"></a> after <img src="photo.jpg" alt=""></p>"#);
assert!(!markdown.contains("[](https://example.com)"), "{markdown}");
assert!(markdown.contains("Before after "), "{markdown}");
}
#[test]
fn separates_exclamation_marks_from_image_markdown() {
let markdown = html_to_markdown(r#"<p>Yey!!<img src="photo.jpg" alt="A photo"></p>"#);
assert!(markdown.contains(r"Yey\!\! "), "{markdown}");
assert!(!markdown.contains(r"Yey\!\!"), "{markdown}");
}
#[test]
fn article_markdown_omits_duplicate_title_heading() {
let html = r#"
<html>
<head><title>Maker's Schedule, Manager's Schedule</title></head>
<body>
<article>
<h1>Maker's Schedule, Manager's Schedule</h1>
<p>One reason programmers dislike meetings so much is that
they're on a different type of schedule from other people.</p>
<p>Meetings cost them more.</p>
</article>
</body>
</html>
"#;
let article = extract(
html,
Some("https://www.paulgraham.com/makersschedule.html"),
&ReadabilityOptions::default(),
)
.unwrap()
.unwrap();
assert_eq!(article.title.as_deref(), Some("Maker's Schedule, Manager's Schedule"));
assert!(
!article.markdown.contains("# Maker's Schedule, Manager's Schedule"),
"{}",
article.markdown
);
assert!(
article.markdown.contains("One reason programmers dislike meetings"),
"{}",
article.markdown
);
}
#[test]
fn fixture_preserves_inline_semantics_in_article_markdown() {
let fixture = lectito_fixtures::load_fixture("inline-semantic-elements").unwrap();
let article = extract(
&fixture.source,
Some("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/mark"),
&ReadabilityOptions::default(),
)
.unwrap()
.unwrap();
assert!(
article.markdown.contains("==highlight matching terms=="),
"{}",
article.markdown
);
assert!(article.markdown.contains("~~removed wording~~"), "{}", article.markdown);
assert!(article.markdown.contains("~~obsolete labels~~"), "{}", article.markdown);
assert!(article.markdown.contains("H<sub>2</sub>O"), "{}", article.markdown);
assert!(
article.markdown.contains("10<sup>2</sup> m<sup>2</sup>"),
"{}",
article.markdown
);
assert!(article.markdown.contains("<svg "), "{}", article.markdown);
assert!(
article
.markdown
.contains("[Video](https://www.youtube.com/watch?v=LtOGa5M8AuU)"),
"{}",
article.markdown
);
}
#[test]
fn expanded_markdown_fixtures_cover_tables_media_math_and_footnotes() {
let cases = [
(
"elements--data-table",
"https://en.wikipedia.org/wiki/Hermitian_matrix",
&[
"| Entry | Value | Conjugate |",
"![{\\\\displaystyle A\\_",
"https://wikimedia.org/api/rest_v1/media/math/render/svg/example",
][..],
),
(
"elements--complex-tables",
"https://www.rfc-editor.org/rfc/rfc7540",
&["<table>", "colspan=\"3\"", "Frame Type Registry"][..],
),
(
"elements--srcset-normalization",
"https://web.dev/articles/responsive-images",
&[
"",
][..],
),
(
"elements--embedded-videos",
"https://developer.mozilla.org/en-US/docs/Learn_web_development/Core/Structuring_content/HTML_images",
&[
"[Video](https://www.youtube.com/watch?v=dQw4w9WgXcQ)",
"Embedded lesson video",
][..],
),
(
"math--katex",
"https://www.intmath.com/cg5/katex-mathjax-comparison.php",
&["$E=mc^2$", "$$\n\\int_0^1 x\\,dx\n$$"][..],
),
(
"math--mathjax-svg",
"https://mathjax.github.io/MathJax-demos-web/input/tex-mml2chtml.html",
&["$$\n\\frac{a + b}{c}\n$$"][..],
),
(
"footnotes--numeric-anchor-id",
"https://math.meta.stackexchange.com/questions/5020/mathjax-basic-tutorial-and-quick-reference",
&[
"references to later notes.[^1]",
"[^1]:\n Use dollar signs around short TeX fragments.",
][..],
),
(
"footnotes--google-docs-ftnt",
"https://stackprinter.appspot.com/export?question=5020&service=math.meta.stackexchange&language=en&hideAnswers=false&width=640",
&[
"ids for footnotes.[^1]",
"[^1]:\n This note came from a Google Docs-style footnote block.",
][..],
),
];
for (fixture_name, url, expected_snippets) in cases {
let fixture = lectito_fixtures::load_fixture(fixture_name).unwrap();
let article = extract(
&fixture.source,
Some(url),
&ReadabilityOptions { char_threshold: 0, ..Default::default() },
)
.unwrap()
.unwrap();
for snippet in expected_snippets {
assert!(
article.markdown.contains(snippet),
"{fixture_name} markdown should contain {snippet:?}:\n{}",
article.markdown
);
}
}
}
#[test]
fn prefers_largest_srcset_candidate_and_preserves_image_title() {
let markdown = html_to_markdown(
r#"<p><img src="small.jpg" srcset="https://cdn.example.com/image,w_400.jpg 400w, https://cdn.example.com/image,w_1600.jpg 1600w" alt="Hero" title="Launch view"></p>"#,
);
assert!(
markdown.contains(r#""#),
"{markdown}"
);
assert!(!markdown.contains("small.jpg"), "{markdown}");
}
#[test]
fn normalizes_picture_lazy_and_placeholder_images() {
let markdown = html_to_markdown(
r#"<picture><source data-srcset="wide.webp 1200w, wide@2x.webp 2400w"><img src="data:image/gif;base64,R0lGODlhAQABAAAAACw=" data-src="fallback.jpg" alt="Wide"></picture>"#,
);
assert!(markdown.contains(""), "{markdown}");
assert!(!markdown.contains("data:image"), "{markdown}");
}
#[test]
fn converts_image_figures_with_captions_but_leaves_content_wrappers() {
let image_figure = html_to_markdown(
r#"<figure><img src="photo.jpg" alt="Photo"><figcaption>Photo caption <em>here</em>.</figcaption></figure>"#,
);
assert!(
image_figure.contains("\n\nPhoto caption *here*."),
"{image_figure}"
);
let wrapper = html_to_markdown(
r#"<figure><h2>Section</h2><p>Intro text.</p><img src="photo.jpg" alt="Photo"><figcaption>Caption</figcaption></figure>"#,
);
assert!(wrapper.contains("## Section"), "{wrapper}");
assert!(wrapper.contains("Intro text."), "{wrapper}");
assert!(wrapper.contains(""), "{wrapper}");
}
#[test]
fn converts_figures_with_wrapped_images() {
let markdown = html_to_markdown(
r#"<figure><p><a href="photo.html"><picture><source srcset="photo-large.jpg 1200w"><img src="photo-small.jpg" alt="Photo"></picture></a></p><figcaption>Wrapped caption.</figcaption></figure>"#,
);
assert!(
markdown.contains("\n\nWrapped caption."),
"{markdown}"
);
assert!(!markdown.contains("photo-small.jpg"), "{markdown}");
assert!(!markdown.contains("photo.html"), "{markdown}");
}
#[test]
fn converts_youtube_and_twitter_embeds() {
let markdown = html_to_markdown(
r#"<iframe src="https://www.youtube.com/embed/dQw4w9WgXcQ"></iframe><blockquote class="twitter-tweet"><p>Tweet</p><a href="https://twitter.com/example/status/12345">May 1</a></blockquote>"#,
);
assert!(
markdown.contains("[Video](https://www.youtube.com/watch?v=dQw4w9WgXcQ)"),
"{markdown}"
);
assert!(
markdown.contains("[Video](https://twitter.com/example/status/12345)"),
"{markdown}"
);
assert!(!markdown.contains("> Tweet"), "{markdown}");
}
}