1use url::Url;
9
10use crate::denoise::{DenoiseConfig, DenoiseStats, is_noise};
11use crate::parse::dom::{Doc, Id, decode_entities};
12use crate::parse::{Unit, UnitKind};
13use crate::text;
14
15const INLINE_TAGS: &[&str] = &[
16 "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "data", "del", "dfn", "em", "font", "i",
17 "img", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", "strike", "strong", "sub",
18 "sup", "time", "tt", "u", "var", "wbr", "br", "ruby", "rt", "rp",
19];
20
21const HEADINGS: &[&str] = &["h1", "h2", "h3", "h4", "h5", "h6"];
22
23const INLINE_INSPECT_BYTES: u32 = 2048;
32
33const KNOWN_LANGUAGES: &[&str] = &[
39 "bash",
40 "c",
41 "clojure",
42 "cpp",
43 "cs",
44 "csharp",
45 "css",
46 "dart",
47 "diff",
48 "dockerfile",
49 "elixir",
50 "elm",
51 "erlang",
52 "fsharp",
53 "go",
54 "graphql",
55 "groovy",
56 "haskell",
57 "hcl",
58 "html",
59 "ini",
60 "java",
61 "javascript",
62 "js",
63 "json",
64 "json5",
65 "jsx",
66 "julia",
67 "kotlin",
68 "latex",
69 "less",
70 "lisp",
71 "lua",
72 "makefile",
73 "markdown",
74 "matlab",
75 "nginx",
76 "nim",
77 "objectivec",
78 "ocaml",
79 "perl",
80 "php",
81 "powershell",
82 "protobuf",
83 "python",
84 "r",
85 "ruby",
86 "rust",
87 "sass",
88 "scala",
89 "scss",
90 "shell",
91 "sh",
92 "sql",
93 "svelte",
94 "swift",
95 "terraform",
96 "toml",
97 "ts",
98 "tsx",
99 "typescript",
100 "vim",
101 "vue",
102 "xml",
103 "yaml",
104 "zig",
105 "zsh",
106];
107
108#[derive(Debug, Clone)]
110pub struct RenderOptions {
111 pub include_links: bool,
113 pub include_images: bool,
115 pub include_tables: bool,
117}
118
119impl Default for RenderOptions {
120 fn default() -> Self {
121 RenderOptions { include_links: true, include_images: false, include_tables: true }
122 }
123}
124
125pub(crate) struct Writer<'d, 'a> {
126 doc: &'d Doc<'a>,
127 cfg: &'d DenoiseConfig,
128 opts: &'d RenderOptions,
129 base: Option<Url>,
130 heading_stack: Vec<(u8, String)>,
131 list_depth: usize,
132 pub(crate) units: Vec<Unit>,
133 pub(crate) stats: DenoiseStats,
134}
135
136impl<'d, 'a> Writer<'d, 'a> {
137 pub(crate) fn new(
138 doc: &'d Doc<'a>,
139 cfg: &'d DenoiseConfig,
140 opts: &'d RenderOptions,
141 base: Option<Url>,
142 ) -> Self {
143 Writer {
144 doc,
145 cfg,
146 opts,
147 base,
148 heading_stack: Vec::new(),
149 list_depth: 0,
150 units: Vec::new(),
151 stats: DenoiseStats::default(),
152 }
153 }
154
155 pub(crate) fn walk_root(&mut self, id: Id) {
164 self.walk_inner(id, false);
165 }
166
167 pub(crate) fn walk(&mut self, id: Id) {
169 self.walk_inner(id, true);
170 }
171
172 fn walk_inner(&mut self, id: Id, judge: bool) {
173 self.stats.nodes_visited += 1;
174 if judge && is_noise(self.doc, id, self.cfg) {
175 self.stats.nodes_dropped += self.doc.descendants(id).len();
176 return;
177 }
178 let name = self.doc.tag_name(id);
179
180 if let Some(level) = heading_level(name) {
181 let inline = self.render_children_inline(id);
182 let t = text::normalize_ws(&inline);
183 if !t.is_empty() {
184 self.push_heading(level, &t);
185 let md = format!("{} {}", "#".repeat(level as usize), t);
186 self.emit(UnitKind::Heading, level, t, md);
187 }
188 return;
189 }
190
191 match name {
192 "" => {
193 if let Some(tl::Node::Raw(_)) = self.doc.node(id) {
195 let t = text::normalize_ws(&self.render_inline(id));
196 if text::visible_len(&t) >= self.cfg.min_block_len as usize {
197 self.emit(UnitKind::Paragraph, 0, t.clone(), t);
198 }
199 }
200 }
201 "p" | "dd" | "dt" | "figcaption" | "summary" | "address" => {
208 if self.doc.is_leaf_block(id) {
209 let t = text::normalize_ws(&self.render_children_inline(id));
210 if !t.is_empty() {
211 let plain = strip_markdown(&t);
212 self.emit(UnitKind::Paragraph, 0, plain, t);
213 }
214 } else {
215 self.walk_children(id);
216 }
217 }
218 "pre" => self.emit_code(id),
219 "blockquote" if !self.doc.is_leaf_block(id) => self.walk_children(id),
220 "blockquote" => {
221 let inner = self.doc.inner_text(id);
222 if !inner.is_empty() {
223 let md = inner.lines().map(|l| format!("> {l}")).collect::<Vec<_>>().join("\n");
224 self.emit(UnitKind::Quote, 0, inner, md);
225 }
226 }
227 "ul" | "ol" | "dl" => {
228 self.list_depth += 1;
229 self.walk_children(id);
230 self.list_depth = self.list_depth.saturating_sub(1);
231 }
232 "li" => self.emit_list_item(id),
233 "table" if self.is_data_table(id) => {
239 if self.opts.include_tables {
240 self.emit_table(id);
241 }
242 }
243 "table" => self.walk_children(id),
244 "hr" | "br" => {}
245 _ => self.walk_children(id),
246 }
247 }
248
249 fn walk_children(&mut self, id: Id) {
250 let mut run: Vec<Id> = Vec::new();
253 for child in self.doc.children(id) {
254 if self.is_inline(child) {
255 run.push(child);
256 } else {
257 self.flush_inline_run(&mut run);
258 self.walk(child);
259 }
260 }
261 self.flush_inline_run(&mut run);
262 }
263
264 fn flush_inline_run(&mut self, run: &mut Vec<Id>) {
265 if run.is_empty() {
266 return;
267 }
268 let has_image =
272 self.opts.include_images && run.iter().any(|&id| self.doc.tag_name(id) == "img");
273 let mut buf = String::new();
274 for id in run.drain(..) {
275 buf.push_str(&self.render_inline(id));
276 }
277 let t = text::normalize_ws(&buf);
278 let plain = strip_markdown(&t);
279 if has_image || text::visible_len(&plain) >= self.cfg.min_block_len as usize {
280 self.emit(UnitKind::Paragraph, 0, plain, t);
281 }
282 }
283
284 fn is_inline(&self, id: Id) -> bool {
285 match self.doc.node(id) {
286 Some(tl::Node::Raw(_)) => true,
287 Some(tl::Node::Tag(tag)) => {
288 if !INLINE_TAGS.contains(&tag.name().as_utf8_str().to_ascii_lowercase().as_str()) {
289 return false;
290 }
291 self.doc.html_len(id) <= INLINE_INSPECT_BYTES || self.doc.is_leaf_block(id)
292 }
293 _ => false,
294 }
295 }
296
297 fn emit_list_item(&mut self, id: Id) {
298 let mut inline_run: Vec<Id> = Vec::new();
304 let mut blocks: Vec<Id> = Vec::new();
305 for child in self.doc.children(id) {
306 if self.is_inline(child) {
307 inline_run.push(child);
308 } else {
309 blocks.push(child);
310 }
311 }
312
313 let mut buf = String::new();
314 for child in inline_run {
315 buf.push_str(&self.render_inline(child));
316 }
317 let t = text::normalize_ws(&buf);
318 if !t.is_empty() {
319 let depth = self.list_depth.saturating_sub(1).min(6);
320 let md = format!("{}- {}", " ".repeat(depth), t);
321 self.emit(UnitKind::ListItem, depth as u8, strip_markdown(&t), md);
322 }
323 for child in blocks {
324 self.walk(child);
325 }
326 }
327
328 fn emit_code(&mut self, id: Id) {
329 let raw = self.doc.raw_text(id);
330 let body = raw.trim_end_matches('\n');
331 if body.trim().is_empty() {
332 return;
333 }
334 let lang = self.detect_language(id);
335 let md = format!("```{lang}\n{body}\n```");
336 self.emit(UnitKind::Code, 0, body.to_string(), md);
337 }
338
339 fn detect_language(&self, id: Id) -> String {
348 let mut candidates: Vec<Id> = vec![id];
349 candidates.extend(self.doc.descendants(id).into_iter().skip(1).take(8));
350 candidates.extend(self.doc.ancestors(id, 2));
351
352 for node in candidates {
353 for key in ["class", "data-lang", "data-language", "data-code-language"] {
354 let Some(value) = self.doc.attr(node, key) else { continue };
355 if key != "class" {
356 if let Some(lang) = normalize_language(&value) {
357 return lang;
358 }
359 continue;
360 }
361 for token in value.split_whitespace() {
362 let stripped = token
363 .strip_prefix("language-")
364 .or_else(|| token.strip_prefix("lang-"))
365 .or_else(|| token.strip_prefix("highlight-source-"))
366 .or_else(|| token.strip_prefix("highlight-text-"))
367 .or_else(|| token.strip_prefix("sourceCode-"));
368 if let Some(lang) = stripped.and_then(normalize_language) {
369 return lang;
370 }
371 if let Some(lang) = normalize_language(token)
375 && KNOWN_LANGUAGES.contains(&lang.as_str())
376 {
377 return lang;
378 }
379 }
380 }
381 }
382 String::new()
383 }
384
385 fn is_data_table(&self, id: Id) -> bool {
399 const MAX_TABLE_TEXT: u32 = 20_000;
400 const MAX_TABLE_NODES: usize = 30_000;
401
402 if self.doc.text_len(id) > MAX_TABLE_TEXT {
403 return false;
404 }
405 let descendants = self.doc.descendants(id);
406 if descendants.len() > MAX_TABLE_NODES {
407 return false;
408 }
409 !descendants.iter().skip(1).any(|&d| {
410 matches!(
411 self.doc.tag_name(d),
412 "table"
413 | "h1"
414 | "h2"
415 | "h3"
416 | "h4"
417 | "h5"
418 | "h6"
419 | "section"
420 | "article"
421 | "aside"
422 | "nav"
423 | "footer"
424 | "header"
425 | "figure"
426 )
427 })
428 }
429
430 fn emit_table(&mut self, id: Id) {
431 let mut rows: Vec<Vec<String>> = Vec::new();
432 let mut header: Option<Vec<String>> = None;
433 for d in self.doc.descendants(id) {
434 if self.doc.tag_name(d) != "tr" || is_noise(self.doc, d, self.cfg) {
435 continue;
436 }
437 let mut cells = Vec::new();
438 let mut is_header = false;
439 for c in self.doc.children(d) {
440 let name = self.doc.tag_name(c);
441 if name == "th" {
442 is_header = true;
443 } else if name != "td" {
444 continue;
445 }
446 cells.push(text::normalize_ws(&self.render_children_inline(c)).replace('|', "\\|"));
447 }
448 if cells.is_empty() {
449 continue;
450 }
451 if is_header && header.is_none() {
452 header = Some(cells);
453 } else {
454 rows.push(cells);
455 }
456 }
457 if rows.is_empty() && header.is_none() {
458 return;
459 }
460 let width = header
461 .as_ref()
462 .map(Vec::len)
463 .into_iter()
464 .chain(rows.iter().map(Vec::len))
465 .max()
466 .unwrap_or(0);
467 if width == 0 {
468 return;
469 }
470 let pad = |mut r: Vec<String>| {
471 r.resize(width, String::new());
472 format!("| {} |", r.join(" | "))
473 };
474 let mut md = String::new();
475 let head = header.unwrap_or_else(|| vec![String::new(); width]);
476 md.push_str(&pad(head));
477 md.push('\n');
478 md.push_str(&format!("|{}", " --- |".repeat(width)));
479 for r in &rows {
480 md.push('\n');
481 md.push_str(&pad(r.clone()));
482 }
483 let plain = strip_markdown(&md);
489 self.emit(UnitKind::Table, 0, plain, md);
490 }
491
492 fn render_children_inline(&self, id: Id) -> String {
493 let mut buf = String::new();
494 for child in self.doc.children(id) {
495 buf.push_str(&self.render_inline(child));
496 }
497 buf
498 }
499
500 fn render_inline(&self, id: Id) -> String {
502 match self.doc.node(id) {
503 Some(tl::Node::Raw(bytes)) => {
504 let raw = bytes.as_utf8_str();
505 escape_inline(&decode_entities(&raw))
506 }
507 Some(tl::Node::Comment(_)) | None => String::new(),
508 Some(tl::Node::Tag(tag)) => {
509 let name = tag.name().as_utf8_str().to_ascii_lowercase();
510 if crate::denoise::DROP_TAGS.contains(&name.as_str()) {
511 return String::new();
512 }
513 match name.as_str() {
514 "br" => "\n".to_string(),
515 "img" => {
516 if !self.opts.include_images {
517 return String::new();
518 }
519 let alt = self.doc.attr(id, "alt").unwrap_or_default();
520 match self.resolve(id, "src").or_else(|| self.resolve(id, "data-src")) {
521 Some(src) => format!("", escape_inline(&alt), src),
522 None => String::new(),
523 }
524 }
525 "a" => {
526 let inner = self.render_children_inline(id);
527 if !self.opts.include_links || inner.trim().is_empty() {
528 return inner;
529 }
530 match self.resolve(id, "href") {
531 Some(href) if !href.starts_with("javascript:") => {
532 format!("[{}]({})", inner.trim(), href)
533 }
534 _ => inner,
535 }
536 }
537 "strong" | "b" => wrap(&self.render_children_inline(id), "**"),
538 "em" | "i" | "dfn" | "cite" | "var" => {
539 wrap(&self.render_children_inline(id), "*")
540 }
541 "del" | "s" | "strike" => wrap(&self.render_children_inline(id), "~~"),
542 "code" | "kbd" | "samp" | "tt" => {
543 let inner = text::normalize_ws(&self.doc.inner_text(id));
544 if inner.is_empty() { inner } else { format!("`{inner}`") }
545 }
546 "sup" => self.render_superscript(id),
547 _ => self.render_children_inline(id),
548 }
549 }
550 }
551 }
552
553 fn render_superscript(&self, id: Id) -> String {
559 let inner = text::normalize_ws(&self.doc.inner_text(id));
560 if inner.is_empty() {
561 return String::new();
562 }
563 if self.is_citation_marker(id, &inner) {
564 return String::new();
565 }
566 let numeric = inner.chars().all(|c| c.is_ascii_digit() || "+-.,()".contains(c))
569 && inner.chars().any(|c| c.is_ascii_digit());
570 if numeric { format!("^{inner}") } else { inner }
571 }
572
573 fn is_citation_marker(&self, id: Id, inner: &str) -> bool {
575 if self.doc.descendants(id).into_iter().skip(1).any(|d| self.doc.tag_name(d) == "a") {
577 return true;
578 }
579 let sig = self.doc.signature(id);
580 if !sig.is_empty()
581 && ["reference", "citation", "footnote", "cite"].iter().any(|w| sig.contains(w))
582 {
583 return true;
584 }
585 let trimmed = inner.trim();
587 let bracketed = (trimmed.starts_with('[') && trimmed.ends_with(']'))
588 || (trimmed.starts_with('(') && trimmed.ends_with(')'));
589 bracketed && trimmed.chars().any(|c| c.is_ascii_digit())
590 }
591
592 fn resolve(&self, id: Id, key: &str) -> Option<String> {
594 let raw = self.doc.attr(id, key)?;
595 let raw = raw.trim();
596 if raw.is_empty() {
597 return None;
598 }
599 match &self.base {
600 Some(base) => base.join(raw).ok().map(|u| u.to_string()),
601 None => Some(raw.to_string()),
602 }
603 }
604
605 fn push_heading(&mut self, level: u8, title: &str) {
606 while self.heading_stack.last().is_some_and(|(l, _)| *l >= level) {
607 self.heading_stack.pop();
608 }
609 self.heading_stack.push((level, title.to_string()));
610 }
611
612 fn emit(&mut self, kind: UnitKind, level: u8, plain: String, markdown: String) {
613 let plain = plain.trim().to_string();
614 let markdown = markdown.trim().to_string();
615 if markdown.is_empty() {
616 return;
617 }
618 let path: Vec<String> = if kind == UnitKind::Heading {
619 self.heading_stack.iter().rev().skip(1).rev().map(|(_, t)| t.clone()).collect()
620 } else {
621 self.heading_stack.iter().map(|(_, t)| t.clone()).collect()
622 };
623 self.stats.markdown_bytes += markdown.len() + 2;
624 self.units.push(Unit {
625 kind,
626 level,
627 tokens: text::estimate_tokens(&markdown),
628 position: self.units.len(),
629 text: plain,
630 markdown,
631 heading_path: path,
632 });
633 }
634}
635
636fn normalize_language(raw: &str) -> Option<String> {
638 let lang = raw.trim().trim_matches('"').to_ascii_lowercase();
639 let ok = !lang.is_empty()
640 && lang.len() <= 16
641 && lang.chars().all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '#' || c == '-');
642 ok.then_some(lang)
643}
644
645fn heading_level(name: &str) -> Option<u8> {
646 HEADINGS.iter().position(|h| *h == name).map(|i| i as u8 + 1)
647}
648
649fn wrap(inner: &str, marker: &str) -> String {
650 let trimmed = inner.trim();
651 if trimmed.is_empty() {
652 return String::new();
653 }
654 let lead = if inner.starts_with(char::is_whitespace) { " " } else { "" };
656 let trail = if inner.ends_with(char::is_whitespace) { " " } else { "" };
657 format!("{lead}{marker}{trimmed}{marker}{trail}")
658}
659
660fn escape_inline(s: &str) -> String {
665 if !s.contains(['[', ']']) {
666 return s.to_string();
667 }
668 s.replace('[', "\\[").replace(']', "\\]")
669}
670
671pub(crate) fn strip_markdown(s: &str) -> String {
673 let mut out = String::with_capacity(s.len());
674 let mut chars = s.chars().peekable();
675 let mut in_link_text = false;
676 while let Some(c) = chars.next() {
677 match c {
678 '\\' => {
679 if let Some(n) = chars.next() {
680 out.push(n);
681 }
682 }
683 '[' => in_link_text = true,
684 ']' => {
685 in_link_text = false;
686 if chars.peek() == Some(&'(') {
688 chars.next();
689 let mut depth = 1;
690 for c in chars.by_ref() {
691 match c {
692 '(' => depth += 1,
693 ')' => {
694 depth -= 1;
695 if depth == 0 {
696 break;
697 }
698 }
699 _ => {}
700 }
701 }
702 }
703 }
704 '*' | '`' | '~' | '#' | '>' if !in_link_text => {}
705 _ => out.push(c),
706 }
707 }
708 text::normalize_ws(&out)
709}
710
711#[cfg(test)]
712mod tests {
713 use super::*;
714
715 #[test]
716 fn strips_links_and_emphasis() {
717 assert_eq!(strip_markdown("**bold** and [a link](http://x.dev/y)"), "bold and a link");
718 assert_eq!(strip_markdown("`code` ~~gone~~"), "code gone");
719 }
720
721 #[test]
722 fn wrap_keeps_outer_spacing() {
723 assert_eq!(wrap(" hi ", "**"), " **hi** ");
724 assert_eq!(wrap(" ", "**"), "");
725 }
726
727 #[test]
728 fn escape_only_touches_brackets() {
729 assert_eq!(escape_inline("a_b*c"), "a_b*c");
730 assert_eq!(escape_inline("see [1]"), "see \\[1\\]");
731 }
732}