1use url::Url;
9
10use crate::denoise::{DenoiseConfig, DenoiseStats, is_noise};
11use crate::parse::dom::{Doc, Id, decode_entities};
12use crate::parse::{Unit, UnitKind};
13use crate::text;
14
15const INLINE_TAGS: &[&str] = &[
16 "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "data", "del", "dfn", "em", "font", "i",
17 "img", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", "strike", "strong", "sub",
18 "sup", "time", "tt", "u", "var", "wbr", "br", "ruby", "rt", "rp",
19];
20
21const HEADINGS: &[&str] = &["h1", "h2", "h3", "h4", "h5", "h6"];
22
23const INLINE_INSPECT_BYTES: u32 = 2048;
32
33const KNOWN_LANGUAGES: &[&str] = &[
39 "bash",
40 "c",
41 "clojure",
42 "cpp",
43 "cs",
44 "csharp",
45 "css",
46 "dart",
47 "diff",
48 "dockerfile",
49 "elixir",
50 "elm",
51 "erlang",
52 "fsharp",
53 "go",
54 "graphql",
55 "groovy",
56 "haskell",
57 "hcl",
58 "html",
59 "ini",
60 "java",
61 "javascript",
62 "js",
63 "json",
64 "json5",
65 "jsx",
66 "julia",
67 "kotlin",
68 "latex",
69 "less",
70 "lisp",
71 "lua",
72 "makefile",
73 "markdown",
74 "matlab",
75 "nginx",
76 "nim",
77 "objectivec",
78 "ocaml",
79 "perl",
80 "php",
81 "powershell",
82 "protobuf",
83 "python",
84 "r",
85 "ruby",
86 "rust",
87 "sass",
88 "scala",
89 "scss",
90 "shell",
91 "sh",
92 "sql",
93 "svelte",
94 "swift",
95 "terraform",
96 "toml",
97 "ts",
98 "tsx",
99 "typescript",
100 "vim",
101 "vue",
102 "xml",
103 "yaml",
104 "zig",
105 "zsh",
106];
107
108#[derive(Debug, Clone)]
110pub struct RenderOptions {
111 pub include_links: bool,
113 pub include_images: bool,
115 pub include_tables: bool,
117}
118
119impl Default for RenderOptions {
120 fn default() -> Self {
121 RenderOptions { include_links: true, include_images: false, include_tables: true }
122 }
123}
124
125pub(crate) struct Writer<'d, 'a> {
126 doc: &'d Doc<'a>,
127 cfg: &'d DenoiseConfig,
128 opts: &'d RenderOptions,
129 base: Option<Url>,
130 heading_stack: Vec<(u8, String)>,
131 list_depth: usize,
132 pub(crate) units: Vec<Unit>,
133 pub(crate) stats: DenoiseStats,
134}
135
136impl<'d, 'a> Writer<'d, 'a> {
137 pub(crate) fn new(
138 doc: &'d Doc<'a>,
139 cfg: &'d DenoiseConfig,
140 opts: &'d RenderOptions,
141 base: Option<Url>,
142 ) -> Self {
143 Writer {
144 doc,
145 cfg,
146 opts,
147 base,
148 heading_stack: Vec::new(),
149 list_depth: 0,
150 units: Vec::new(),
151 stats: DenoiseStats::default(),
152 }
153 }
154
155 pub(crate) fn walk_root(&mut self, id: Id) {
164 self.walk_inner(id, false);
165 }
166
167 pub(crate) fn walk(&mut self, id: Id) {
169 self.walk_inner(id, true);
170 }
171
172 fn walk_inner(&mut self, id: Id, judge: bool) {
173 self.stats.nodes_visited += 1;
174 if judge && is_noise(self.doc, id, self.cfg) {
175 self.stats.nodes_dropped += self.doc.descendants(id).len();
176 return;
177 }
178 let name = self.doc.tag_name(id);
179
180 if let Some(level) = heading_level(name) {
181 let inline = self.render_children_inline(id);
182 let t = text::normalize_ws(&inline);
183 if !t.is_empty() {
184 let plain = strip_markdown(&t);
190 self.push_heading(level, &plain);
191 let md = format!("{} {}", "#".repeat(level as usize), t);
192 self.emit(UnitKind::Heading, level, plain, md);
193 }
194 return;
195 }
196
197 match name {
198 "" => {
199 if let Some(tl::Node::Raw(_)) = self.doc.node(id) {
201 let t = text::normalize_ws(&self.render_inline(id));
202 if text::visible_len(&t) >= self.cfg.min_block_len as usize {
203 self.emit(UnitKind::Paragraph, 0, t.clone(), t);
204 }
205 }
206 }
207 "p" | "dd" | "dt" | "figcaption" | "summary" | "address" => {
214 if self.doc.is_leaf_block(id) {
215 let t = text::normalize_ws(&self.render_children_inline(id));
216 if !t.is_empty() {
217 let plain = strip_markdown(&t);
218 self.emit(UnitKind::Paragraph, 0, plain, t);
219 }
220 } else {
221 self.walk_children(id);
222 }
223 }
224 "pre" => self.emit_code(id),
225 "blockquote" if !self.doc.is_leaf_block(id) => self.walk_children(id),
226 "blockquote" => {
227 let inner = self.doc.inner_text(id);
228 if !inner.is_empty() {
229 let md = inner.lines().map(|l| format!("> {l}")).collect::<Vec<_>>().join("\n");
230 self.emit(UnitKind::Quote, 0, inner, md);
231 }
232 }
233 "ul" | "ol" | "dl" => {
234 self.list_depth += 1;
235 self.walk_children(id);
236 self.list_depth = self.list_depth.saturating_sub(1);
237 }
238 "li" => self.emit_list_item(id),
239 "table" if self.is_data_table(id) => {
245 if self.opts.include_tables {
246 self.emit_table(id);
247 }
248 }
249 "table" => self.walk_children(id),
250 "hr" | "br" => {}
251 _ => self.walk_children(id),
252 }
253 }
254
255 fn walk_children(&mut self, id: Id) {
256 let mut run: Vec<Id> = Vec::new();
259 for child in self.doc.children(id) {
260 if self.is_inline(child) {
261 run.push(child);
262 } else {
263 self.flush_inline_run(&mut run);
264 self.walk(child);
265 }
266 }
267 self.flush_inline_run(&mut run);
268 }
269
270 fn flush_inline_run(&mut self, run: &mut Vec<Id>) {
271 if run.is_empty() {
272 return;
273 }
274 let has_image =
278 self.opts.include_images && run.iter().any(|&id| self.doc.tag_name(id) == "img");
279 let mut buf = String::new();
280 for id in run.drain(..) {
281 buf.push_str(&self.render_inline(id));
282 }
283 let t = text::normalize_ws(&buf);
284 let plain = strip_markdown(&t);
285 if has_image || text::visible_len(&plain) >= self.cfg.min_block_len as usize {
286 self.emit(UnitKind::Paragraph, 0, plain, t);
287 }
288 }
289
290 fn is_inline(&self, id: Id) -> bool {
291 match self.doc.node(id) {
292 Some(tl::Node::Raw(_)) => true,
293 Some(tl::Node::Tag(tag)) => {
294 if !INLINE_TAGS.contains(&tag.name().as_utf8_str().to_ascii_lowercase().as_str()) {
295 return false;
296 }
297 self.doc.html_len(id) <= INLINE_INSPECT_BYTES || self.doc.is_leaf_block(id)
298 }
299 _ => false,
300 }
301 }
302
303 fn emit_list_item(&mut self, id: Id) {
304 let mut inline_run: Vec<Id> = Vec::new();
310 let mut blocks: Vec<Id> = Vec::new();
311 for child in self.doc.children(id) {
312 if self.is_inline(child) {
313 inline_run.push(child);
314 } else {
315 blocks.push(child);
316 }
317 }
318
319 let mut buf = String::new();
320 for child in inline_run {
321 buf.push_str(&self.render_inline(child));
322 }
323 let t = text::normalize_ws(&buf);
324 if !t.is_empty() {
325 let depth = self.list_depth.saturating_sub(1).min(6);
326 let md = format!("{}- {}", " ".repeat(depth), t);
327 self.emit(UnitKind::ListItem, depth as u8, strip_markdown(&t), md);
328 }
329 for child in blocks {
330 self.walk(child);
331 }
332 }
333
334 fn emit_code(&mut self, id: Id) {
335 let raw = self.doc.raw_text(id);
336 let body = raw.trim_end_matches('\n');
337 if body.trim().is_empty() {
338 return;
339 }
340 let lang = self.detect_language(id);
341 let md = format!("```{lang}\n{body}\n```");
342 self.emit(UnitKind::Code, 0, body.to_string(), md);
343 }
344
345 fn detect_language(&self, id: Id) -> String {
354 let mut candidates: Vec<Id> = vec![id];
355 candidates.extend(self.doc.descendants(id).into_iter().skip(1).take(8));
356 candidates.extend(self.doc.ancestors(id, 2));
357
358 for node in candidates {
359 for key in ["class", "data-lang", "data-language", "data-code-language"] {
360 let Some(value) = self.doc.attr(node, key) else { continue };
361 if key != "class" {
362 if let Some(lang) = normalize_language(&value) {
363 return lang;
364 }
365 continue;
366 }
367 for token in value.split_whitespace() {
368 let stripped = token
369 .strip_prefix("language-")
370 .or_else(|| token.strip_prefix("lang-"))
371 .or_else(|| token.strip_prefix("highlight-source-"))
372 .or_else(|| token.strip_prefix("highlight-text-"))
373 .or_else(|| token.strip_prefix("sourceCode-"));
374 if let Some(lang) = stripped.and_then(normalize_language) {
375 return lang;
376 }
377 if let Some(lang) = normalize_language(token)
381 && KNOWN_LANGUAGES.contains(&lang.as_str())
382 {
383 return lang;
384 }
385 }
386 }
387 }
388 String::new()
389 }
390
391 fn is_data_table(&self, id: Id) -> bool {
405 const MAX_TABLE_TEXT: u32 = 20_000;
406 const MAX_TABLE_NODES: usize = 30_000;
407
408 if self.doc.text_len(id) > MAX_TABLE_TEXT {
409 return false;
410 }
411 let descendants = self.doc.descendants(id);
412 if descendants.len() > MAX_TABLE_NODES {
413 return false;
414 }
415 !descendants.iter().skip(1).any(|&d| {
416 matches!(
417 self.doc.tag_name(d),
418 "table"
419 | "h1"
420 | "h2"
421 | "h3"
422 | "h4"
423 | "h5"
424 | "h6"
425 | "section"
426 | "article"
427 | "aside"
428 | "nav"
429 | "footer"
430 | "header"
431 | "figure"
432 )
433 })
434 }
435
436 fn emit_table(&mut self, id: Id) {
437 let mut rows: Vec<Vec<String>> = Vec::new();
438 let mut header: Option<Vec<String>> = None;
439 for d in self.doc.descendants(id) {
440 if self.doc.tag_name(d) != "tr" || is_noise(self.doc, d, self.cfg) {
441 continue;
442 }
443 let mut cells = Vec::new();
444 let mut is_header = false;
445 for c in self.doc.children(d) {
446 let name = self.doc.tag_name(c);
447 if name == "th" {
448 is_header = true;
449 } else if name != "td" {
450 continue;
451 }
452 cells.push(text::normalize_ws(&self.render_children_inline(c)).replace('|', "\\|"));
453 }
454 if cells.is_empty() {
455 continue;
456 }
457 if is_header && header.is_none() {
458 header = Some(cells);
459 } else {
460 rows.push(cells);
461 }
462 }
463 if rows.is_empty() && header.is_none() {
464 return;
465 }
466 let width = header
467 .as_ref()
468 .map(Vec::len)
469 .into_iter()
470 .chain(rows.iter().map(Vec::len))
471 .max()
472 .unwrap_or(0);
473 if width == 0 {
474 return;
475 }
476 let pad = |mut r: Vec<String>| {
477 r.resize(width, String::new());
478 format!("| {} |", r.join(" | "))
479 };
480 let mut md = String::new();
481 let head = header.unwrap_or_else(|| vec![String::new(); width]);
482 md.push_str(&pad(head));
483 md.push('\n');
484 md.push_str(&format!("|{}", " --- |".repeat(width)));
485 for r in &rows {
486 md.push('\n');
487 md.push_str(&pad(r.clone()));
488 }
489 let plain = strip_markdown(&md);
495 self.emit(UnitKind::Table, 0, plain, md);
496 }
497
498 fn render_children_inline(&self, id: Id) -> String {
499 let mut buf = String::new();
500 for child in self.doc.children(id) {
501 buf.push_str(&self.render_inline(child));
502 }
503 buf
504 }
505
506 fn render_inline(&self, id: Id) -> String {
508 match self.doc.node(id) {
509 Some(tl::Node::Raw(bytes)) => {
510 let raw = bytes.as_utf8_str();
511 escape_inline(&decode_entities(&raw))
512 }
513 Some(tl::Node::Comment(_)) | None => String::new(),
514 Some(tl::Node::Tag(tag)) => {
515 let name = tag.name().as_utf8_str().to_ascii_lowercase();
516 if crate::denoise::DROP_TAGS.contains(&name.as_str()) {
517 return String::new();
518 }
519 match name.as_str() {
520 "br" => "\n".to_string(),
521 "img" => {
522 if !self.opts.include_images {
523 return String::new();
524 }
525 let alt = self.doc.attr(id, "alt").unwrap_or_default();
526 match self.resolve(id, "src").or_else(|| self.resolve(id, "data-src")) {
527 Some(src) => format!("", escape_inline(&alt), src),
528 None => String::new(),
529 }
530 }
531 "a" => {
532 let inner = self.render_children_inline(id);
533 if !self.opts.include_links || inner.trim().is_empty() {
534 return inner;
535 }
536 match self.resolve(id, "href") {
537 Some(href) if !href.starts_with("javascript:") => {
538 format!("[{}]({})", inner.trim(), href)
539 }
540 _ => inner,
541 }
542 }
543 "strong" | "b" => wrap(&self.render_children_inline(id), "**"),
544 "em" | "i" | "dfn" | "cite" | "var" => {
545 wrap(&self.render_children_inline(id), "*")
546 }
547 "del" | "s" | "strike" => wrap(&self.render_children_inline(id), "~~"),
548 "code" | "kbd" | "samp" | "tt" => {
549 let inner = text::normalize_ws(&self.doc.inner_text(id));
550 if inner.is_empty() { inner } else { format!("`{inner}`") }
551 }
552 "sup" => self.render_superscript(id),
553 _ => self.render_children_inline(id),
554 }
555 }
556 }
557 }
558
559 fn render_superscript(&self, id: Id) -> String {
565 let inner = text::normalize_ws(&self.doc.inner_text(id));
566 if inner.is_empty() {
567 return String::new();
568 }
569 if self.is_citation_marker(id, &inner) {
570 return String::new();
571 }
572 let numeric = inner.chars().all(|c| c.is_ascii_digit() || "+-.,()".contains(c))
575 && inner.chars().any(|c| c.is_ascii_digit());
576 if numeric { format!("^{inner}") } else { inner }
577 }
578
579 fn is_citation_marker(&self, id: Id, inner: &str) -> bool {
581 if self.doc.descendants(id).into_iter().skip(1).any(|d| self.doc.tag_name(d) == "a") {
583 return true;
584 }
585 let sig = self.doc.signature(id);
586 if !sig.is_empty()
587 && ["reference", "citation", "footnote", "cite"].iter().any(|w| sig.contains(w))
588 {
589 return true;
590 }
591 let trimmed = inner.trim();
593 let bracketed = (trimmed.starts_with('[') && trimmed.ends_with(']'))
594 || (trimmed.starts_with('(') && trimmed.ends_with(')'));
595 bracketed && trimmed.chars().any(|c| c.is_ascii_digit())
596 }
597
598 fn resolve(&self, id: Id, key: &str) -> Option<String> {
600 let raw = self.doc.attr(id, key)?;
601 let raw = raw.trim();
602 if raw.is_empty() {
603 return None;
604 }
605 match &self.base {
606 Some(base) => base.join(raw).ok().map(|u| u.to_string()),
607 None => Some(raw.to_string()),
608 }
609 }
610
611 fn push_heading(&mut self, level: u8, title: &str) {
612 while self.heading_stack.last().is_some_and(|(l, _)| *l >= level) {
613 self.heading_stack.pop();
614 }
615 self.heading_stack.push((level, title.to_string()));
616 }
617
618 fn emit(&mut self, kind: UnitKind, level: u8, plain: String, markdown: String) {
619 let plain = plain.trim().to_string();
620 let markdown = markdown.trim().to_string();
621 if markdown.is_empty() {
622 return;
623 }
624 let path: Vec<String> = if kind == UnitKind::Heading {
625 self.heading_stack.iter().rev().skip(1).rev().map(|(_, t)| t.clone()).collect()
626 } else {
627 self.heading_stack.iter().map(|(_, t)| t.clone()).collect()
628 };
629 self.stats.markdown_bytes += markdown.len() + 2;
630 self.units.push(Unit {
631 kind,
632 level,
633 tokens: text::estimate_tokens(&markdown),
634 position: self.units.len(),
635 text: plain,
636 markdown,
637 heading_path: path,
638 });
639 }
640}
641
642fn normalize_language(raw: &str) -> Option<String> {
644 let lang = raw.trim().trim_matches('"').to_ascii_lowercase();
645 let ok = !lang.is_empty()
646 && lang.len() <= 16
647 && lang.chars().all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '#' || c == '-');
648 ok.then_some(lang)
649}
650
651fn heading_level(name: &str) -> Option<u8> {
652 HEADINGS.iter().position(|h| *h == name).map(|i| i as u8 + 1)
653}
654
655fn wrap(inner: &str, marker: &str) -> String {
656 let trimmed = inner.trim();
657 if trimmed.is_empty() {
658 return String::new();
659 }
660 let lead = if inner.starts_with(char::is_whitespace) { " " } else { "" };
662 let trail = if inner.ends_with(char::is_whitespace) { " " } else { "" };
663 format!("{lead}{marker}{trimmed}{marker}{trail}")
664}
665
666fn escape_inline(s: &str) -> String {
671 if !s.contains(['[', ']']) {
672 return s.to_string();
673 }
674 s.replace('[', "\\[").replace(']', "\\]")
675}
676
677pub(crate) fn strip_markdown(s: &str) -> String {
679 let mut out = String::with_capacity(s.len());
680 let mut chars = s.chars().peekable();
681 let mut in_link_text = false;
682 while let Some(c) = chars.next() {
683 match c {
684 '\\' => {
685 if let Some(n) = chars.next() {
686 out.push(n);
687 }
688 }
689 '[' => in_link_text = true,
690 ']' => {
691 in_link_text = false;
692 if chars.peek() == Some(&'(') {
694 chars.next();
695 let mut depth = 1;
696 for c in chars.by_ref() {
697 match c {
698 '(' => depth += 1,
699 ')' => {
700 depth -= 1;
701 if depth == 0 {
702 break;
703 }
704 }
705 _ => {}
706 }
707 }
708 }
709 }
710 '*' | '`' | '~' | '#' | '>' if !in_link_text => {}
711 _ => out.push(c),
712 }
713 }
714 text::normalize_ws(&out)
715}
716
717#[cfg(test)]
718mod tests {
719 use super::*;
720
721 #[test]
722 fn strips_links_and_emphasis() {
723 assert_eq!(strip_markdown("**bold** and [a link](http://x.dev/y)"), "bold and a link");
724 assert_eq!(strip_markdown("`code` ~~gone~~"), "code gone");
725 }
726
727 #[test]
728 fn wrap_keeps_outer_spacing() {
729 assert_eq!(wrap(" hi ", "**"), " **hi** ");
730 assert_eq!(wrap(" ", "**"), "");
731 }
732
733 #[test]
734 fn a_heading_wrapped_in_its_own_anchor_has_plain_text() {
735 let html = concat!(
737 "<html><body><article><h2><a href=\"https://d.dev/a/b#sec\">Section</a></h2>",
738 "<p>Body text long enough to survive the block gate here.</p></article></body></html>"
739 );
740 let article = crate::parse::extract(html, Some("https://d.dev/a/b")).unwrap();
741 let heading = article.units.iter().find(|u| u.kind == UnitKind::Heading).unwrap();
742 assert_eq!(heading.text, "Section");
743 assert!(heading.markdown.contains("https://d.dev/a/b#sec"), "markdown keeps the link");
744 let below = article.units.iter().find(|u| u.kind == UnitKind::Paragraph).unwrap();
745 assert_eq!(below.heading_path, vec!["Section"], "breadcrumbs carry no URL");
746 }
747
748 #[test]
749 fn escape_only_touches_brackets() {
750 assert_eq!(escape_inline("a_b*c"), "a_b*c");
751 assert_eq!(escape_inline("see [1]"), "see \\[1\\]");
752 }
753}