1use url::Url;
9
10use crate::denoise::{DenoiseConfig, DenoiseStats, is_noise};
11use crate::parse::dom::{Doc, Id, decode_entities};
12use crate::parse::{Unit, UnitKind};
13use crate::text;
14
15const INLINE_TAGS: &[&str] = &[
16 "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "data", "del", "dfn", "em", "font", "i",
17 "img", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", "strike", "strong", "sub",
18 "sup", "time", "tt", "u", "var", "wbr", "br", "ruby", "rt", "rp",
19];
20
21const HEADINGS: &[&str] = &["h1", "h2", "h3", "h4", "h5", "h6"];
22
23const INLINE_INSPECT_BYTES: u32 = 2048;
32
33const KNOWN_LANGUAGES: &[&str] = &[
39 "bash",
40 "c",
41 "clojure",
42 "cpp",
43 "cs",
44 "csharp",
45 "css",
46 "dart",
47 "diff",
48 "dockerfile",
49 "elixir",
50 "elm",
51 "erlang",
52 "fsharp",
53 "go",
54 "graphql",
55 "groovy",
56 "haskell",
57 "hcl",
58 "html",
59 "ini",
60 "java",
61 "javascript",
62 "js",
63 "json",
64 "json5",
65 "jsx",
66 "julia",
67 "kotlin",
68 "latex",
69 "less",
70 "lisp",
71 "lua",
72 "makefile",
73 "markdown",
74 "matlab",
75 "nginx",
76 "nim",
77 "objectivec",
78 "ocaml",
79 "perl",
80 "php",
81 "powershell",
82 "protobuf",
83 "python",
84 "r",
85 "ruby",
86 "rust",
87 "sass",
88 "scala",
89 "scss",
90 "shell",
91 "sh",
92 "sql",
93 "svelte",
94 "swift",
95 "terraform",
96 "toml",
97 "ts",
98 "tsx",
99 "typescript",
100 "vim",
101 "vue",
102 "xml",
103 "yaml",
104 "zig",
105 "zsh",
106];
107
108#[derive(Debug, Clone)]
110pub struct RenderOptions {
111 pub include_links: bool,
113 pub include_images: bool,
115 pub include_tables: bool,
117}
118
119impl Default for RenderOptions {
120 fn default() -> Self {
121 RenderOptions { include_links: true, include_images: false, include_tables: true }
122 }
123}
124
125pub(crate) struct Writer<'d, 'a> {
126 doc: &'d Doc<'a>,
127 cfg: &'d DenoiseConfig,
128 opts: &'d RenderOptions,
129 base: Option<Url>,
130 heading_stack: Vec<(u8, String)>,
131 list_depth: usize,
132 pub(crate) units: Vec<Unit>,
133 pub(crate) stats: DenoiseStats,
134}
135
136impl<'d, 'a> Writer<'d, 'a> {
137 pub(crate) fn new(
138 doc: &'d Doc<'a>,
139 cfg: &'d DenoiseConfig,
140 opts: &'d RenderOptions,
141 base: Option<Url>,
142 ) -> Self {
143 Writer {
144 doc,
145 cfg,
146 opts,
147 base,
148 heading_stack: Vec::new(),
149 list_depth: 0,
150 units: Vec::new(),
151 stats: DenoiseStats::default(),
152 }
153 }
154
155 pub(crate) fn walk_root(&mut self, id: Id) {
164 self.walk_inner(id, false);
165 }
166
167 pub(crate) fn walk(&mut self, id: Id) {
169 self.walk_inner(id, true);
170 }
171
172 fn walk_inner(&mut self, id: Id, judge: bool) {
173 self.stats.nodes_visited += 1;
174 if judge && is_noise(self.doc, id, self.cfg) {
175 self.stats.nodes_dropped += self.doc.descendants(id).len();
176 return;
177 }
178 let name = self.doc.tag_name(id);
179
180 if let Some(level) = heading_level(name) {
181 let inline = self.render_children_inline(id);
182 let t = text::normalize_ws(&inline);
183 if !t.is_empty() {
184 let plain = strip_markdown(&t);
190 self.push_heading(level, &plain);
191 let md = format!("{} {}", "#".repeat(level as usize), t);
192 self.emit(UnitKind::Heading, level, plain, md);
193 }
194 return;
195 }
196
197 match name {
198 "" => {
199 if let Some(tl::Node::Raw(_)) = self.doc.node(id) {
201 let t = text::normalize_ws(&self.render_inline(id));
202 if text::visible_len(&t) >= self.cfg.min_block_len as usize {
203 self.emit(UnitKind::Paragraph, 0, t.clone(), t);
204 }
205 }
206 }
207 "p" | "dd" | "dt" | "figcaption" | "summary" | "address" => {
214 if self.doc.is_leaf_block(id) {
215 let t = text::normalize_ws(&self.render_children_inline(id));
216 if !t.is_empty() {
217 let plain = strip_markdown(&t);
218 self.emit(UnitKind::Paragraph, 0, plain, t);
219 }
220 } else {
221 self.walk_children(id);
222 }
223 }
224 "pre" => self.emit_code(id),
225 "blockquote" if !self.doc.is_leaf_block(id) => self.walk_children(id),
226 "blockquote" => {
227 let inner = self.doc.inner_text(id);
228 if !inner.is_empty() {
229 let md = inner.lines().map(|l| format!("> {l}")).collect::<Vec<_>>().join("\n");
230 self.emit(UnitKind::Quote, 0, inner, md);
231 }
232 }
233 "ul" | "ol" | "dl" => {
234 self.list_depth += 1;
235 self.walk_children(id);
236 self.list_depth = self.list_depth.saturating_sub(1);
237 }
238 "li" => self.emit_list_item(id),
239 "table" if self.is_data_table(id) => {
245 if self.opts.include_tables {
246 self.emit_table(id);
247 }
248 }
249 "table" => self.walk_children(id),
250 "hr" | "br" => {}
251 _ => self.walk_children(id),
252 }
253 }
254
255 fn walk_children(&mut self, id: Id) {
256 let mut run: Vec<Id> = Vec::new();
259 for child in self.doc.children(id) {
260 if self.is_inline(child) {
261 run.push(child);
262 } else {
263 self.flush_inline_run(&mut run);
264 self.walk(child);
265 }
266 }
267 self.flush_inline_run(&mut run);
268 }
269
270 fn flush_inline_run(&mut self, run: &mut Vec<Id>) {
271 if run.is_empty() {
272 return;
273 }
274 let has_image =
278 self.opts.include_images && run.iter().any(|&id| self.doc.tag_name(id) == "img");
279 let mut buf = String::new();
280 for id in run.drain(..) {
281 buf.push_str(&self.render_inline(id));
282 }
283 let t = text::normalize_ws(&buf);
284 let plain = strip_markdown(&t);
285 if has_image || text::visible_len(&plain) >= self.cfg.min_block_len as usize {
286 self.emit(UnitKind::Paragraph, 0, plain, t);
287 }
288 }
289
290 fn is_inline(&self, id: Id) -> bool {
291 match self.doc.node(id) {
292 Some(tl::Node::Raw(_)) => true,
293 Some(tl::Node::Tag(tag)) => {
294 if !INLINE_TAGS.contains(&tag.name().as_utf8_str().to_ascii_lowercase().as_str()) {
295 return false;
296 }
297 self.doc.html_len(id) <= INLINE_INSPECT_BYTES || self.doc.is_leaf_block(id)
298 }
299 _ => false,
300 }
301 }
302
303 fn emit_list_item(&mut self, id: Id) {
304 let mut inline_run: Vec<Id> = Vec::new();
310 let mut blocks: Vec<Id> = Vec::new();
311 for child in self.doc.children(id) {
312 if self.is_inline(child) {
313 inline_run.push(child);
314 } else {
315 blocks.push(child);
316 }
317 }
318
319 let mut buf = String::new();
320 for child in inline_run {
321 buf.push_str(&self.render_inline(child));
322 }
323 let t = text::normalize_ws(&buf);
324 if !t.is_empty() {
325 let depth = self.list_depth.saturating_sub(1).min(6);
326 let md = format!("{}- {}", " ".repeat(depth), t);
327 self.emit(UnitKind::ListItem, depth as u8, strip_markdown(&t), md);
328 }
329 for child in blocks {
330 self.walk(child);
331 }
332 }
333
334 fn emit_code(&mut self, id: Id) {
335 let raw = self.doc.raw_text(id);
336 let body = raw.trim_end_matches('\n');
337 if body.trim().is_empty() {
338 return;
339 }
340 let lang = self.detect_language(id);
341 let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
345 let md = format!("{fence}{lang}\n{body}\n{fence}");
346 self.emit(UnitKind::Code, 0, body.to_string(), md);
347 }
348
349 fn detect_language(&self, id: Id) -> String {
358 let mut candidates: Vec<Id> = vec![id];
359 candidates.extend(self.doc.descendants(id).into_iter().skip(1).take(8));
360 candidates.extend(self.doc.ancestors(id, 2));
361
362 for node in candidates {
363 for key in ["class", "data-lang", "data-language", "data-code-language"] {
364 let Some(value) = self.doc.attr(node, key) else { continue };
365 if key != "class" {
366 if let Some(lang) = normalize_language(&value) {
367 return lang;
368 }
369 continue;
370 }
371 for token in value.split_whitespace() {
372 let stripped = token
373 .strip_prefix("language-")
374 .or_else(|| token.strip_prefix("lang-"))
375 .or_else(|| token.strip_prefix("highlight-source-"))
376 .or_else(|| token.strip_prefix("highlight-text-"))
377 .or_else(|| token.strip_prefix("sourceCode-"));
378 if let Some(lang) = stripped.and_then(normalize_language) {
379 return lang;
380 }
381 if let Some(lang) = normalize_language(token)
385 && KNOWN_LANGUAGES.contains(&lang.as_str())
386 {
387 return lang;
388 }
389 }
390 }
391 }
392 String::new()
393 }
394
395 fn is_data_table(&self, id: Id) -> bool {
409 const MAX_TABLE_TEXT: u32 = 20_000;
410 const MAX_TABLE_NODES: usize = 30_000;
411
412 if self.doc.text_len(id) > MAX_TABLE_TEXT {
413 return false;
414 }
415 let descendants = self.doc.descendants(id);
416 if descendants.len() > MAX_TABLE_NODES {
417 return false;
418 }
419 !descendants.iter().skip(1).any(|&d| {
420 matches!(
421 self.doc.tag_name(d),
422 "table"
423 | "h1"
424 | "h2"
425 | "h3"
426 | "h4"
427 | "h5"
428 | "h6"
429 | "section"
430 | "article"
431 | "aside"
432 | "nav"
433 | "footer"
434 | "header"
435 | "figure"
436 )
437 })
438 }
439
440 fn emit_table(&mut self, id: Id) {
441 let mut rows: Vec<Vec<String>> = Vec::new();
442 let mut header: Option<Vec<String>> = None;
443 for d in self.doc.descendants(id) {
444 if self.doc.tag_name(d) != "tr" || is_noise(self.doc, d, self.cfg) {
445 continue;
446 }
447 let mut cells = Vec::new();
448 let mut is_header = false;
449 for c in self.doc.children(d) {
450 let name = self.doc.tag_name(c);
451 if name == "th" {
452 is_header = true;
453 } else if name != "td" {
454 continue;
455 }
456 cells.push(text::normalize_ws(&self.render_children_inline(c)).replace('|', "\\|"));
457 }
458 if cells.is_empty() {
459 continue;
460 }
461 if is_header && header.is_none() {
462 header = Some(cells);
463 } else {
464 rows.push(cells);
465 }
466 }
467 if rows.is_empty() && header.is_none() {
468 return;
469 }
470 let width = header
471 .as_ref()
472 .map(Vec::len)
473 .into_iter()
474 .chain(rows.iter().map(Vec::len))
475 .max()
476 .unwrap_or(0);
477 if width == 0 {
478 return;
479 }
480 let pad = |mut r: Vec<String>| {
481 r.resize(width, String::new());
482 format!("| {} |", r.join(" | "))
483 };
484 let mut md = String::new();
485 let head = header.unwrap_or_else(|| vec![String::new(); width]);
486 md.push_str(&pad(head));
487 md.push('\n');
488 md.push_str(&format!("|{}", " --- |".repeat(width)));
489 for r in &rows {
490 md.push('\n');
491 md.push_str(&pad(r.clone()));
492 }
493 let plain = strip_markdown(&md);
499 self.emit(UnitKind::Table, 0, plain, md);
500 }
501
502 fn render_children_inline(&self, id: Id) -> String {
503 let mut buf = String::new();
504 for child in self.doc.children(id) {
505 buf.push_str(&self.render_inline(child));
506 }
507 buf
508 }
509
510 fn render_inline(&self, id: Id) -> String {
512 match self.doc.node(id) {
513 Some(tl::Node::Raw(bytes)) => {
514 let raw = bytes.as_utf8_str();
515 escape_inline(&decode_entities(&raw))
516 }
517 Some(tl::Node::Comment(_)) | None => String::new(),
518 Some(tl::Node::Tag(tag)) => {
519 let name = tag.name().as_utf8_str().to_ascii_lowercase();
520 if crate::denoise::DROP_TAGS.contains(&name.as_str())
521 || crate::denoise::is_hidden(self.doc, id)
522 {
523 return String::new();
524 }
525 match name.as_str() {
526 "br" => "\n".to_string(),
527 "img" => {
528 if !self.opts.include_images {
529 return String::new();
530 }
531 let alt = self.doc.attr(id, "alt").unwrap_or_default();
532 match self.resolve(id, "src").or_else(|| self.resolve(id, "data-src")) {
533 Some(src) => format!("", escape_inline(&alt), src),
534 None => String::new(),
535 }
536 }
537 "a" => {
538 let inner = self.render_children_inline(id);
539 if !self.opts.include_links || inner.trim().is_empty() {
540 return inner;
541 }
542 match self.resolve(id, "href") {
543 Some(href) if !href.starts_with("javascript:") => {
544 format!("[{}]({})", inner.trim(), href)
545 }
546 _ => inner,
547 }
548 }
549 "strong" | "b" => wrap(&self.render_children_inline(id), "**"),
550 "em" | "i" | "dfn" | "cite" | "var" => {
551 wrap(&self.render_children_inline(id), "*")
552 }
553 "del" | "s" | "strike" => wrap(&self.render_children_inline(id), "~~"),
554 "code" | "kbd" | "samp" | "tt" => {
555 let inner = text::normalize_ws(&self.doc.inner_text(id));
556 if inner.is_empty() { inner } else { format!("`{inner}`") }
557 }
558 "sup" => self.render_superscript(id),
559 "math" => self.render_math(id),
560 _ => self.render_children_inline(id),
561 }
562 }
563 }
564 }
565
566 fn render_math(&self, id: Id) -> String {
571 let nodes = self.doc.descendants(id);
572 let tex = nodes.iter().copied().find(|&d| {
573 self.doc.tag_name(d) == "annotation"
574 && self
575 .doc
576 .attr(d, "encoding")
577 .is_some_and(|e| e.eq_ignore_ascii_case("application/x-tex"))
578 });
579 if let Some(a) = tex {
580 let src = text::normalize_ws(&self.doc.raw_text(a));
581 return if src.is_empty() { src } else { format!("${}$", escape_inline(&src)) };
582 }
583 let mut buf = String::new();
584 for &d in &nodes {
585 if let Some(tl::Node::Raw(bytes)) = self.doc.node(d)
586 && !self.inside_annotation(d, id)
587 {
588 buf.push_str(&decode_entities(&bytes.as_utf8_str()));
589 }
590 }
591 escape_inline(&text::normalize_ws(&buf))
592 }
593
594 fn inside_annotation(&self, mut id: Id, math: Id) -> bool {
595 while let Some(p) = self.doc.parent(id) {
596 if p == math {
597 return false;
598 }
599 if matches!(self.doc.tag_name(p), "annotation" | "annotation-xml") {
600 return true;
601 }
602 id = p;
603 }
604 false
605 }
606
607 fn render_superscript(&self, id: Id) -> String {
613 let inner = text::normalize_ws(&self.doc.inner_text(id));
614 if inner.is_empty() {
615 return String::new();
616 }
617 if self.is_citation_marker(id, &inner) {
618 return String::new();
619 }
620 let numeric = inner.chars().all(|c| c.is_ascii_digit() || "+-.,()".contains(c))
623 && inner.chars().any(|c| c.is_ascii_digit());
624 if numeric { format!("^{inner}") } else { inner }
625 }
626
627 fn is_citation_marker(&self, id: Id, inner: &str) -> bool {
629 if self.doc.descendants(id).into_iter().skip(1).any(|d| self.doc.tag_name(d) == "a") {
631 return true;
632 }
633 let sig = self.doc.signature(id);
634 if !sig.is_empty()
635 && ["reference", "citation", "footnote", "cite"].iter().any(|w| sig.contains(w))
636 {
637 return true;
638 }
639 let trimmed = inner.trim();
641 let bracketed = (trimmed.starts_with('[') && trimmed.ends_with(']'))
642 || (trimmed.starts_with('(') && trimmed.ends_with(')'));
643 bracketed && trimmed.chars().any(|c| c.is_ascii_digit())
644 }
645
646 fn resolve(&self, id: Id, key: &str) -> Option<String> {
648 let raw = self.doc.attr(id, key)?;
649 let raw = raw.trim();
650 if raw.is_empty() {
651 return None;
652 }
653 match &self.base {
654 Some(base) => base.join(raw).ok().map(|u| u.to_string()),
655 None => Some(raw.to_string()),
656 }
657 }
658
659 fn push_heading(&mut self, level: u8, title: &str) {
660 while self.heading_stack.last().is_some_and(|(l, _)| *l >= level) {
661 self.heading_stack.pop();
662 }
663 self.heading_stack.push((level, title.to_string()));
664 }
665
666 fn emit(&mut self, kind: UnitKind, level: u8, plain: String, markdown: String) {
667 let plain = plain.trim().to_string();
668 let markdown = markdown.trim().to_string();
669 let markdown = match kind {
670 UnitKind::Heading | UnitKind::Code => markdown,
673 _ => defuse_headings(&markdown),
674 };
675 if markdown.is_empty() {
676 return;
677 }
678 let path: Vec<String> = if kind == UnitKind::Heading {
679 self.heading_stack.iter().rev().skip(1).rev().map(|(_, t)| t.clone()).collect()
680 } else {
681 self.heading_stack.iter().map(|(_, t)| t.clone()).collect()
682 };
683 self.stats.markdown_bytes += markdown.len() + 2;
684 self.units.push(Unit {
685 kind,
686 level,
687 tokens: text::estimate_tokens(&markdown),
688 position: self.units.len(),
689 text: plain,
690 markdown,
691 heading_path: path,
692 });
693 }
694}
695
696fn longest_backtick_run(s: &str) -> usize {
698 let (mut best, mut run) = (0usize, 0usize);
699 for c in s.chars() {
700 run = if c == '`' { run + 1 } else { 0 };
701 best = best.max(run);
702 }
703 best
704}
705
706fn defuse_headings(md: &str) -> String {
718 if !md.starts_with(['#', ' ']) && !md.contains('\n') {
719 return md.to_string();
720 }
721 let mut out = String::with_capacity(md.len());
722 for (i, line) in md.lines().enumerate() {
723 if i > 0 {
724 out.push('\n');
725 }
726 let body = line.trim_start_matches(' ');
727 let indent = line.len() - body.len();
728 let hashes = body.len() - body.trim_start_matches('#').len();
729 let after = &body[hashes..];
731 if indent <= 3 && (1..=2).contains(&hashes) && (after.is_empty() || after.starts_with(' '))
732 {
733 out.push_str(&line[..indent]);
734 out.push('\\');
735 out.push_str(body);
736 } else {
737 out.push_str(line);
738 }
739 }
740 out
741}
742
743fn normalize_language(raw: &str) -> Option<String> {
745 let lang = raw.trim().trim_matches('"').to_ascii_lowercase();
746 let ok = !lang.is_empty()
747 && lang.len() <= 16
748 && lang.chars().all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '#' || c == '-');
749 ok.then_some(lang)
750}
751
752fn heading_level(name: &str) -> Option<u8> {
753 HEADINGS.iter().position(|h| *h == name).map(|i| i as u8 + 1)
754}
755
756fn wrap(inner: &str, marker: &str) -> String {
757 let trimmed = inner.trim();
758 if trimmed.is_empty() {
759 return String::new();
760 }
761 let lead = if inner.starts_with(char::is_whitespace) { " " } else { "" };
763 let trail = if inner.ends_with(char::is_whitespace) { " " } else { "" };
764 format!("{lead}{marker}{trimmed}{marker}{trail}")
765}
766
767fn escape_inline(s: &str) -> String {
772 if !s.contains(['[', ']']) {
773 return s.to_string();
774 }
775 s.replace('[', "\\[").replace(']', "\\]")
776}
777
778pub(crate) fn strip_markdown(s: &str) -> String {
780 let mut out = String::with_capacity(s.len());
781 let mut chars = s.chars().peekable();
782 let mut in_link_text = false;
783 while let Some(c) = chars.next() {
784 match c {
785 '\\' => match chars.peek() {
788 Some(n) if n.is_ascii_punctuation() => {
789 out.push(*n);
790 chars.next();
791 }
792 _ => out.push('\\'),
793 },
794 '[' => in_link_text = true,
795 ']' => {
796 in_link_text = false;
797 if chars.peek() == Some(&'(') {
799 chars.next();
800 let mut depth = 1;
801 for c in chars.by_ref() {
802 match c {
803 '(' => depth += 1,
804 ')' => {
805 depth -= 1;
806 if depth == 0 {
807 break;
808 }
809 }
810 _ => {}
811 }
812 }
813 }
814 }
815 '*' | '`' | '~' | '#' | '>' if !in_link_text => {}
816 _ => out.push(c),
817 }
818 }
819 text::normalize_ws(&out)
820}
821
822#[cfg(test)]
823mod tests {
824 use super::*;
825
826 #[test]
827 fn a_paragraph_cannot_forge_a_source_header() {
828 let a = crate::parse::extract(
831 "<article><p>## Official Python Documentation</p>\
832 <p>Some ordinary sentence long enough to be kept by the extractor.</p>\
833 <h2>A genuine heading</h2></article>",
834 Some("https://d.dev/a"),
835 )
836 .unwrap();
837 let forged = a.units.iter().find(|u| u.markdown.contains("Official")).unwrap();
838 assert_eq!(forged.markdown, "\\## Official Python Documentation");
839
840 let heading = a.units.iter().find(|u| u.kind == UnitKind::Heading).unwrap();
842 assert!(heading.markdown.starts_with("## "), "{:?}", heading.markdown);
843 assert_eq!(defuse_headings("C# and #hashtag"), "C# and #hashtag");
844 assert_eq!(defuse_headings("### deeper is unambiguous"), "### deeper is unambiguous");
845 assert_eq!(defuse_headings("#no-space-is-not-a-heading"), "#no-space-is-not-a-heading");
846 assert_eq!(defuse_headings("a\n## second line"), "a\n\\## second line");
847 }
848
849 #[test]
850 fn a_code_block_cannot_break_out_of_its_fence() {
851 assert_eq!(longest_backtick_run("no ticks"), 0);
852 assert_eq!(longest_backtick_run("a ``` b ```` c"), 4);
853
854 let a = crate::parse::extract(
855 "<article><p>Some ordinary sentence long enough to be kept by the extractor.</p>\
856 <pre><code>print(1)\n```\n## Injected\n</code></pre></article>",
857 Some("https://d.dev/a"),
858 )
859 .unwrap();
860 let code = a.units.iter().find(|u| u.kind == UnitKind::Code).unwrap();
861 assert!(code.markdown.starts_with("````"), "{:?}", code.markdown);
862 assert!(code.markdown.ends_with("````"), "{:?}", code.markdown);
863 assert!(code.markdown.contains("## Injected"));
865 }
866
867 #[test]
868 fn strips_links_and_emphasis() {
869 assert_eq!(strip_markdown("**bold** and [a link](http://x.dev/y)"), "bold and a link");
870 assert_eq!(strip_markdown("`code` ~~gone~~"), "code gone");
871 }
872
873 #[test]
874 fn wrap_keeps_outer_spacing() {
875 assert_eq!(wrap(" hi ", "**"), " **hi** ");
876 assert_eq!(wrap(" ", "**"), "");
877 }
878
879 #[test]
880 fn a_heading_wrapped_in_its_own_anchor_has_plain_text() {
881 let html = concat!(
883 "<html><body><article><h2><a href=\"https://d.dev/a/b#sec\">Section</a></h2>",
884 "<p>Body text long enough to survive the block gate here.</p></article></body></html>"
885 );
886 let article = crate::parse::extract(html, Some("https://d.dev/a/b")).unwrap();
887 let heading = article.units.iter().find(|u| u.kind == UnitKind::Heading).unwrap();
888 assert_eq!(heading.text, "Section");
889 assert!(heading.markdown.contains("https://d.dev/a/b#sec"), "markdown keeps the link");
890 let below = article.units.iter().find(|u| u.kind == UnitKind::Paragraph).unwrap();
891 assert_eq!(below.heading_path, vec!["Section"], "breadcrumbs carry no URL");
892 }
893
894 fn paragraph_text(body: &str) -> String {
896 let html = format!("<html><body><article>{body}</article></body></html>");
897 let article = crate::parse::extract(&html, Some("https://m.dev/x")).unwrap();
898 article.units.iter().find(|u| u.kind == UnitKind::Paragraph).unwrap().text.clone()
899 }
900
901 #[test]
902 fn katex_math_is_kept_once_as_its_tex_source() {
903 let text = paragraph_text(concat!(
906 "<p>Differentiate <span class=\"katex\"><span class=\"katex-mathml\"><math><semantics>",
907 "<mrow><mo>(</mo><mn>2</mn><mi>x</mi><mo>+</mo><mn>3</mn><msup><mo>)</mo><mn>5</mn></msup></mrow>",
908 "<annotation encoding=\"application/x-tex\">(2x+3)^5</annotation></semantics></math></span>",
909 "<span class=\"katex-html\" aria-hidden=\"true\"><span class=\"mord\">(2x+3)</span>",
910 "<span class=\"msupsub\">5</span></span></span> with the chain rule, step by step.</p>",
911 ));
912 assert_eq!(text, "Differentiate $(2x+3)^5$ with the chain rule, step by step.");
913 }
914
915 #[test]
916 fn mathml_without_tex_is_read_once() {
917 let text = paragraph_text(
918 "<p>The area is <math><mi>π</mi><msup><mi>r</mi><mn>2</mn></msup></math> for a circle of radius r.</p>",
919 );
920 assert_eq!(text, "The area is πr2 for a circle of radius r.");
921 }
922
923 #[test]
924 fn aria_hidden_text_inside_a_paragraph_is_dropped() {
925 let text = paragraph_text(
926 "<p>Visible words <span aria-hidden=\"true\">decoration</span>that make up a full sentence.</p>",
927 );
928 assert_eq!(text, "Visible words that make up a full sentence.");
929 }
930
931 #[test]
932 fn tex_backslashes_survive_into_plain_text() {
933 assert_eq!(strip_markdown(r"$\frac{a}{b}$ and \[1\]"), r"$\frac{a}{b}$ and [1]");
935 }
936
937 #[test]
938 fn escape_only_touches_brackets() {
939 assert_eq!(escape_inline("a_b*c"), "a_b*c");
940 assert_eq!(escape_inline("see [1]"), "see \\[1\\]");
941 }
942}