use url::Url;
use crate::denoise::{DenoiseConfig, DenoiseStats, is_noise};
use crate::parse::dom::{Doc, Id, decode_entities};
use crate::parse::{Unit, UnitKind};
use crate::text;
const INLINE_TAGS: &[&str] = &[
"a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "data", "del", "dfn", "em", "font", "i",
"img", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", "strike", "strong", "sub",
"sup", "time", "tt", "u", "var", "wbr", "br", "ruby", "rt", "rp",
];
const HEADINGS: &[&str] = &["h1", "h2", "h3", "h4", "h5", "h6"];
const INLINE_INSPECT_BYTES: u32 = 2048;
const KNOWN_LANGUAGES: &[&str] = &[
"bash",
"c",
"clojure",
"cpp",
"cs",
"csharp",
"css",
"dart",
"diff",
"dockerfile",
"elixir",
"elm",
"erlang",
"fsharp",
"go",
"graphql",
"groovy",
"haskell",
"hcl",
"html",
"ini",
"java",
"javascript",
"js",
"json",
"json5",
"jsx",
"julia",
"kotlin",
"latex",
"less",
"lisp",
"lua",
"makefile",
"markdown",
"matlab",
"nginx",
"nim",
"objectivec",
"ocaml",
"perl",
"php",
"powershell",
"protobuf",
"python",
"r",
"ruby",
"rust",
"sass",
"scala",
"scss",
"shell",
"sh",
"sql",
"svelte",
"swift",
"terraform",
"toml",
"ts",
"tsx",
"typescript",
"vim",
"vue",
"xml",
"yaml",
"zig",
"zsh",
];
#[derive(Debug, Clone)]
pub struct RenderOptions {
pub include_links: bool,
pub include_images: bool,
pub include_tables: bool,
}
impl Default for RenderOptions {
fn default() -> Self {
RenderOptions { include_links: true, include_images: false, include_tables: true }
}
}
pub(crate) struct Writer<'d, 'a> {
doc: &'d Doc<'a>,
cfg: &'d DenoiseConfig,
opts: &'d RenderOptions,
base: Option<Url>,
heading_stack: Vec<(u8, String)>,
list_depth: usize,
pub(crate) units: Vec<Unit>,
pub(crate) stats: DenoiseStats,
}
impl<'d, 'a> Writer<'d, 'a> {
pub(crate) fn new(
doc: &'d Doc<'a>,
cfg: &'d DenoiseConfig,
opts: &'d RenderOptions,
base: Option<Url>,
) -> Self {
Writer {
doc,
cfg,
opts,
base,
heading_stack: Vec::new(),
list_depth: 0,
units: Vec::new(),
stats: DenoiseStats::default(),
}
}
pub(crate) fn walk_root(&mut self, id: Id) {
self.walk_inner(id, false);
}
pub(crate) fn walk(&mut self, id: Id) {
self.walk_inner(id, true);
}
fn walk_inner(&mut self, id: Id, judge: bool) {
self.stats.nodes_visited += 1;
if judge && is_noise(self.doc, id, self.cfg) {
self.stats.nodes_dropped += self.doc.descendants(id).len();
return;
}
let name = self.doc.tag_name(id);
if let Some(level) = heading_level(name) {
let inline = self.render_children_inline(id);
let t = text::normalize_ws(&inline);
if !t.is_empty() {
let plain = strip_markdown(&t);
self.push_heading(level, &plain);
let md = format!("{} {}", "#".repeat(level as usize), t);
self.emit(UnitKind::Heading, level, plain, md);
}
return;
}
match name {
"" => {
if let Some(tl::Node::Raw(_)) = self.doc.node(id) {
let t = text::normalize_ws(&self.render_inline(id));
if text::visible_len(&t) >= self.cfg.min_block_len as usize {
self.emit(UnitKind::Paragraph, 0, t.clone(), t);
}
}
}
"p" | "dd" | "dt" | "figcaption" | "summary" | "address" => {
if self.doc.is_leaf_block(id) {
let t = text::normalize_ws(&self.render_children_inline(id));
if !t.is_empty() {
let plain = strip_markdown(&t);
self.emit(UnitKind::Paragraph, 0, plain, t);
}
} else {
self.walk_children(id);
}
}
"pre" => self.emit_code(id),
"blockquote" if !self.doc.is_leaf_block(id) => self.walk_children(id),
"blockquote" => {
let inner = self.doc.inner_text(id);
if !inner.is_empty() {
let md = inner.lines().map(|l| format!("> {l}")).collect::<Vec<_>>().join("\n");
self.emit(UnitKind::Quote, 0, inner, md);
}
}
"ul" | "ol" | "dl" => {
self.list_depth += 1;
self.walk_children(id);
self.list_depth = self.list_depth.saturating_sub(1);
}
"li" => self.emit_list_item(id),
"table" if self.is_data_table(id) => {
if self.opts.include_tables {
self.emit_table(id);
}
}
"table" => self.walk_children(id),
"hr" | "br" => {}
_ => self.walk_children(id),
}
}
fn walk_children(&mut self, id: Id) {
let mut run: Vec<Id> = Vec::new();
for child in self.doc.children(id) {
if self.is_inline(child) {
run.push(child);
} else {
self.flush_inline_run(&mut run);
self.walk(child);
}
}
self.flush_inline_run(&mut run);
}
fn flush_inline_run(&mut self, run: &mut Vec<Id>) {
if run.is_empty() {
return;
}
let has_image =
self.opts.include_images && run.iter().any(|&id| self.doc.tag_name(id) == "img");
let mut buf = String::new();
for id in run.drain(..) {
buf.push_str(&self.render_inline(id));
}
let t = text::normalize_ws(&buf);
let plain = strip_markdown(&t);
if has_image || text::visible_len(&plain) >= self.cfg.min_block_len as usize {
self.emit(UnitKind::Paragraph, 0, plain, t);
}
}
fn is_inline(&self, id: Id) -> bool {
match self.doc.node(id) {
Some(tl::Node::Raw(_)) => true,
Some(tl::Node::Tag(tag)) => {
if !INLINE_TAGS.contains(&tag.name().as_utf8_str().to_ascii_lowercase().as_str()) {
return false;
}
self.doc.html_len(id) <= INLINE_INSPECT_BYTES || self.doc.is_leaf_block(id)
}
_ => false,
}
}
fn emit_list_item(&mut self, id: Id) {
let mut inline_run: Vec<Id> = Vec::new();
let mut blocks: Vec<Id> = Vec::new();
for child in self.doc.children(id) {
if self.is_inline(child) {
inline_run.push(child);
} else {
blocks.push(child);
}
}
let mut buf = String::new();
for child in inline_run {
buf.push_str(&self.render_inline(child));
}
let t = text::normalize_ws(&buf);
if !t.is_empty() {
let depth = self.list_depth.saturating_sub(1).min(6);
let md = format!("{}- {}", " ".repeat(depth), t);
self.emit(UnitKind::ListItem, depth as u8, strip_markdown(&t), md);
}
for child in blocks {
self.walk(child);
}
}
fn emit_code(&mut self, id: Id) {
let raw = self.doc.raw_text(id);
let body = raw.trim_end_matches('\n');
if body.trim().is_empty() {
return;
}
let lang = self.detect_language(id);
let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
let md = format!("{fence}{lang}\n{body}\n{fence}");
self.emit(UnitKind::Code, 0, body.to_string(), md);
}
fn detect_language(&self, id: Id) -> String {
let mut candidates: Vec<Id> = vec![id];
candidates.extend(self.doc.descendants(id).into_iter().skip(1).take(8));
candidates.extend(self.doc.ancestors(id, 2));
for node in candidates {
for key in ["class", "data-lang", "data-language", "data-code-language"] {
let Some(value) = self.doc.attr(node, key) else { continue };
if key != "class" {
if let Some(lang) = normalize_language(&value) {
return lang;
}
continue;
}
for token in value.split_whitespace() {
let stripped = token
.strip_prefix("language-")
.or_else(|| token.strip_prefix("lang-"))
.or_else(|| token.strip_prefix("highlight-source-"))
.or_else(|| token.strip_prefix("highlight-text-"))
.or_else(|| token.strip_prefix("sourceCode-"));
if let Some(lang) = stripped.and_then(normalize_language) {
return lang;
}
if let Some(lang) = normalize_language(token)
&& KNOWN_LANGUAGES.contains(&lang.as_str())
{
return lang;
}
}
}
}
String::new()
}
fn is_data_table(&self, id: Id) -> bool {
const MAX_TABLE_TEXT: u32 = 20_000;
const MAX_TABLE_NODES: usize = 30_000;
if self.doc.text_len(id) > MAX_TABLE_TEXT {
return false;
}
let descendants = self.doc.descendants(id);
if descendants.len() > MAX_TABLE_NODES {
return false;
}
!descendants.iter().skip(1).any(|&d| {
matches!(
self.doc.tag_name(d),
"table"
| "h1"
| "h2"
| "h3"
| "h4"
| "h5"
| "h6"
| "section"
| "article"
| "aside"
| "nav"
| "footer"
| "header"
| "figure"
)
})
}
fn emit_table(&mut self, id: Id) {
let mut rows: Vec<Vec<String>> = Vec::new();
let mut header: Option<Vec<String>> = None;
for d in self.doc.descendants(id) {
if self.doc.tag_name(d) != "tr" || is_noise(self.doc, d, self.cfg) {
continue;
}
let mut cells = Vec::new();
let mut is_header = false;
for c in self.doc.children(d) {
let name = self.doc.tag_name(c);
if name == "th" {
is_header = true;
} else if name != "td" {
continue;
}
cells.push(text::normalize_ws(&self.render_children_inline(c)).replace('|', "\\|"));
}
if cells.is_empty() {
continue;
}
if is_header && header.is_none() {
header = Some(cells);
} else {
rows.push(cells);
}
}
if rows.is_empty() && header.is_none() {
return;
}
let width = header
.as_ref()
.map(Vec::len)
.into_iter()
.chain(rows.iter().map(Vec::len))
.max()
.unwrap_or(0);
if width == 0 {
return;
}
let pad = |mut r: Vec<String>| {
r.resize(width, String::new());
format!("| {} |", r.join(" | "))
};
let mut md = String::new();
let head = header.unwrap_or_else(|| vec![String::new(); width]);
md.push_str(&pad(head));
md.push('\n');
md.push_str(&format!("|{}", " --- |".repeat(width)));
for r in &rows {
md.push('\n');
md.push_str(&pad(r.clone()));
}
let plain = strip_markdown(&md);
self.emit(UnitKind::Table, 0, plain, md);
}
fn render_children_inline(&self, id: Id) -> String {
let mut buf = String::new();
for child in self.doc.children(id) {
buf.push_str(&self.render_inline(child));
}
buf
}
fn render_inline(&self, id: Id) -> String {
match self.doc.node(id) {
Some(tl::Node::Raw(bytes)) => {
let raw = bytes.as_utf8_str();
escape_inline(&decode_entities(&raw))
}
Some(tl::Node::Comment(_)) | None => String::new(),
Some(tl::Node::Tag(tag)) => {
let name = tag.name().as_utf8_str().to_ascii_lowercase();
if crate::denoise::DROP_TAGS.contains(&name.as_str())
|| crate::denoise::is_hidden(self.doc, id)
{
return String::new();
}
match name.as_str() {
"br" => "\n".to_string(),
"img" => {
if !self.opts.include_images {
return String::new();
}
let alt = self.doc.attr(id, "alt").unwrap_or_default();
match self.resolve(id, "src").or_else(|| self.resolve(id, "data-src")) {
Some(src) => format!("", escape_inline(&alt), src),
None => String::new(),
}
}
"a" => {
let inner = self.render_children_inline(id);
if !self.opts.include_links || inner.trim().is_empty() {
return inner;
}
match self.resolve(id, "href") {
Some(href) if !href.starts_with("javascript:") => {
format!("[{}]({})", inner.trim(), href)
}
_ => inner,
}
}
"strong" | "b" => wrap(&self.render_children_inline(id), "**"),
"em" | "i" | "dfn" | "cite" | "var" => {
wrap(&self.render_children_inline(id), "*")
}
"del" | "s" | "strike" => wrap(&self.render_children_inline(id), "~~"),
"code" | "kbd" | "samp" | "tt" => {
let inner = text::normalize_ws(&self.doc.inner_text(id));
if inner.is_empty() { inner } else { format!("`{inner}`") }
}
"sup" => self.render_superscript(id),
"math" => self.render_math(id),
_ => self.render_children_inline(id),
}
}
}
}
fn render_math(&self, id: Id) -> String {
let nodes = self.doc.descendants(id);
let tex = nodes.iter().copied().find(|&d| {
self.doc.tag_name(d) == "annotation"
&& self
.doc
.attr(d, "encoding")
.is_some_and(|e| e.eq_ignore_ascii_case("application/x-tex"))
});
if let Some(a) = tex {
let src = text::normalize_ws(&self.doc.raw_text(a));
return if src.is_empty() { src } else { format!("${}$", escape_inline(&src)) };
}
let mut buf = String::new();
for &d in &nodes {
if let Some(tl::Node::Raw(bytes)) = self.doc.node(d)
&& !self.inside_annotation(d, id)
{
buf.push_str(&decode_entities(&bytes.as_utf8_str()));
}
}
escape_inline(&text::normalize_ws(&buf))
}
fn inside_annotation(&self, mut id: Id, math: Id) -> bool {
while let Some(p) = self.doc.parent(id) {
if p == math {
return false;
}
if matches!(self.doc.tag_name(p), "annotation" | "annotation-xml") {
return true;
}
id = p;
}
false
}
fn render_superscript(&self, id: Id) -> String {
let inner = text::normalize_ws(&self.doc.inner_text(id));
if inner.is_empty() {
return String::new();
}
if self.is_citation_marker(id, &inner) {
return String::new();
}
let numeric = inner.chars().all(|c| c.is_ascii_digit() || "+-.,()".contains(c))
&& inner.chars().any(|c| c.is_ascii_digit());
if numeric { format!("^{inner}") } else { inner }
}
fn is_citation_marker(&self, id: Id, inner: &str) -> bool {
if self.doc.descendants(id).into_iter().skip(1).any(|d| self.doc.tag_name(d) == "a") {
return true;
}
let sig = self.doc.signature(id);
if !sig.is_empty()
&& ["reference", "citation", "footnote", "cite"].iter().any(|w| sig.contains(w))
{
return true;
}
let trimmed = inner.trim();
let bracketed = (trimmed.starts_with('[') && trimmed.ends_with(']'))
|| (trimmed.starts_with('(') && trimmed.ends_with(')'));
bracketed && trimmed.chars().any(|c| c.is_ascii_digit())
}
fn resolve(&self, id: Id, key: &str) -> Option<String> {
let raw = self.doc.attr(id, key)?;
let raw = raw.trim();
if raw.is_empty() {
return None;
}
match &self.base {
Some(base) => base.join(raw).ok().map(|u| u.to_string()),
None => Some(raw.to_string()),
}
}
fn push_heading(&mut self, level: u8, title: &str) {
while self.heading_stack.last().is_some_and(|(l, _)| *l >= level) {
self.heading_stack.pop();
}
self.heading_stack.push((level, title.to_string()));
}
fn emit(&mut self, kind: UnitKind, level: u8, plain: String, markdown: String) {
let plain = plain.trim().to_string();
let markdown = markdown.trim().to_string();
let markdown = match kind {
UnitKind::Heading | UnitKind::Code => markdown,
_ => defuse_headings(&markdown),
};
if markdown.is_empty() {
return;
}
let path: Vec<String> = if kind == UnitKind::Heading {
self.heading_stack.iter().rev().skip(1).rev().map(|(_, t)| t.clone()).collect()
} else {
self.heading_stack.iter().map(|(_, t)| t.clone()).collect()
};
self.stats.markdown_bytes += markdown.len() + 2;
self.units.push(Unit {
kind,
level,
tokens: text::estimate_tokens(&markdown),
position: self.units.len(),
text: plain,
markdown,
heading_path: path,
});
}
}
fn longest_backtick_run(s: &str) -> usize {
let (mut best, mut run) = (0usize, 0usize);
for c in s.chars() {
run = if c == '`' { run + 1 } else { 0 };
best = best.max(run);
}
best
}
fn defuse_headings(md: &str) -> String {
if !md.starts_with(['#', ' ']) && !md.contains('\n') {
return md.to_string();
}
let mut out = String::with_capacity(md.len());
for (i, line) in md.lines().enumerate() {
if i > 0 {
out.push('\n');
}
let body = line.trim_start_matches(' ');
let indent = line.len() - body.len();
let hashes = body.len() - body.trim_start_matches('#').len();
let after = &body[hashes..];
if indent <= 3 && (1..=2).contains(&hashes) && (after.is_empty() || after.starts_with(' '))
{
out.push_str(&line[..indent]);
out.push('\\');
out.push_str(body);
} else {
out.push_str(line);
}
}
out
}
fn normalize_language(raw: &str) -> Option<String> {
let lang = raw.trim().trim_matches('"').to_ascii_lowercase();
let ok = !lang.is_empty()
&& lang.len() <= 16
&& lang.chars().all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '#' || c == '-');
ok.then_some(lang)
}
fn heading_level(name: &str) -> Option<u8> {
HEADINGS.iter().position(|h| *h == name).map(|i| i as u8 + 1)
}
fn wrap(inner: &str, marker: &str) -> String {
let trimmed = inner.trim();
if trimmed.is_empty() {
return String::new();
}
let lead = if inner.starts_with(char::is_whitespace) { " " } else { "" };
let trail = if inner.ends_with(char::is_whitespace) { " " } else { "" };
format!("{lead}{marker}{trimmed}{marker}{trail}")
}
fn escape_inline(s: &str) -> String {
if !s.contains(['[', ']']) {
return s.to_string();
}
s.replace('[', "\\[").replace(']', "\\]")
}
pub(crate) fn strip_markdown(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut chars = s.chars().peekable();
let mut in_link_text = false;
while let Some(c) = chars.next() {
match c {
'\\' => match chars.peek() {
Some(n) if n.is_ascii_punctuation() => {
out.push(*n);
chars.next();
}
_ => out.push('\\'),
},
'[' => in_link_text = true,
']' => {
in_link_text = false;
if chars.peek() == Some(&'(') {
chars.next();
let mut depth = 1;
for c in chars.by_ref() {
match c {
'(' => depth += 1,
')' => {
depth -= 1;
if depth == 0 {
break;
}
}
_ => {}
}
}
}
}
'*' | '`' | '~' | '#' | '>' if !in_link_text => {}
_ => out.push(c),
}
}
text::normalize_ws(&out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_paragraph_cannot_forge_a_source_header() {
let a = crate::parse::extract(
"<article><p>## Official Python Documentation</p>\
<p>Some ordinary sentence long enough to be kept by the extractor.</p>\
<h2>A genuine heading</h2></article>",
Some("https://d.dev/a"),
)
.unwrap();
let forged = a.units.iter().find(|u| u.markdown.contains("Official")).unwrap();
assert_eq!(forged.markdown, "\\## Official Python Documentation");
let heading = a.units.iter().find(|u| u.kind == UnitKind::Heading).unwrap();
assert!(heading.markdown.starts_with("## "), "{:?}", heading.markdown);
assert_eq!(defuse_headings("C# and #hashtag"), "C# and #hashtag");
assert_eq!(defuse_headings("### deeper is unambiguous"), "### deeper is unambiguous");
assert_eq!(defuse_headings("#no-space-is-not-a-heading"), "#no-space-is-not-a-heading");
assert_eq!(defuse_headings("a\n## second line"), "a\n\\## second line");
}
#[test]
fn a_code_block_cannot_break_out_of_its_fence() {
assert_eq!(longest_backtick_run("no ticks"), 0);
assert_eq!(longest_backtick_run("a ``` b ```` c"), 4);
let a = crate::parse::extract(
"<article><p>Some ordinary sentence long enough to be kept by the extractor.</p>\
<pre><code>print(1)\n```\n## Injected\n</code></pre></article>",
Some("https://d.dev/a"),
)
.unwrap();
let code = a.units.iter().find(|u| u.kind == UnitKind::Code).unwrap();
assert!(code.markdown.starts_with("````"), "{:?}", code.markdown);
assert!(code.markdown.ends_with("````"), "{:?}", code.markdown);
assert!(code.markdown.contains("## Injected"));
}
#[test]
fn strips_links_and_emphasis() {
assert_eq!(strip_markdown("**bold** and [a link](http://x.dev/y)"), "bold and a link");
assert_eq!(strip_markdown("`code` ~~gone~~"), "code gone");
}
#[test]
fn wrap_keeps_outer_spacing() {
assert_eq!(wrap(" hi ", "**"), " **hi** ");
assert_eq!(wrap(" ", "**"), "");
}
#[test]
fn a_heading_wrapped_in_its_own_anchor_has_plain_text() {
let html = concat!(
"<html><body><article><h2><a href=\"https://d.dev/a/b#sec\">Section</a></h2>",
"<p>Body text long enough to survive the block gate here.</p></article></body></html>"
);
let article = crate::parse::extract(html, Some("https://d.dev/a/b")).unwrap();
let heading = article.units.iter().find(|u| u.kind == UnitKind::Heading).unwrap();
assert_eq!(heading.text, "Section");
assert!(heading.markdown.contains("https://d.dev/a/b#sec"), "markdown keeps the link");
let below = article.units.iter().find(|u| u.kind == UnitKind::Paragraph).unwrap();
assert_eq!(below.heading_path, vec!["Section"], "breadcrumbs carry no URL");
}
fn paragraph_text(body: &str) -> String {
let html = format!("<html><body><article>{body}</article></body></html>");
let article = crate::parse::extract(&html, Some("https://m.dev/x")).unwrap();
article.units.iter().find(|u| u.kind == UnitKind::Paragraph).unwrap().text.clone()
}
#[test]
fn katex_math_is_kept_once_as_its_tex_source() {
let text = paragraph_text(concat!(
"<p>Differentiate <span class=\"katex\"><span class=\"katex-mathml\"><math><semantics>",
"<mrow><mo>(</mo><mn>2</mn><mi>x</mi><mo>+</mo><mn>3</mn><msup><mo>)</mo><mn>5</mn></msup></mrow>",
"<annotation encoding=\"application/x-tex\">(2x+3)^5</annotation></semantics></math></span>",
"<span class=\"katex-html\" aria-hidden=\"true\"><span class=\"mord\">(2x+3)</span>",
"<span class=\"msupsub\">5</span></span></span> with the chain rule, step by step.</p>",
));
assert_eq!(text, "Differentiate $(2x+3)^5$ with the chain rule, step by step.");
}
#[test]
fn mathml_without_tex_is_read_once() {
let text = paragraph_text(
"<p>The area is <math><mi>π</mi><msup><mi>r</mi><mn>2</mn></msup></math> for a circle of radius r.</p>",
);
assert_eq!(text, "The area is πr2 for a circle of radius r.");
}
#[test]
fn aria_hidden_text_inside_a_paragraph_is_dropped() {
let text = paragraph_text(
"<p>Visible words <span aria-hidden=\"true\">decoration</span>that make up a full sentence.</p>",
);
assert_eq!(text, "Visible words that make up a full sentence.");
}
#[test]
fn tex_backslashes_survive_into_plain_text() {
assert_eq!(strip_markdown(r"$\frac{a}{b}$ and \[1\]"), r"$\frac{a}{b}$ and [1]");
}
#[test]
fn escape_only_touches_brackets() {
assert_eq!(escape_inline("a_b*c"), "a_b*c");
assert_eq!(escape_inline("see [1]"), "see \\[1\\]");
}
}