Skip to main content

rustai_core/parse/
markdown.rs

1//! Markdown writer.
2//!
3//! The output is deliberately plain — ATX headings, `-` bullets, fenced code,
4//! pipe tables — because it is fed to a small local model, not rendered. Every
5//! block becomes a [`Unit`], which is the granularity the context slimmer
6//! later ranks and selects.
7
8use url::Url;
9
10use crate::denoise::{DenoiseConfig, DenoiseStats, is_noise};
11use crate::parse::dom::{Doc, Id, decode_entities};
12use crate::parse::{Unit, UnitKind};
13use crate::text;
14
15const INLINE_TAGS: &[&str] = &[
16    "a", "abbr", "b", "bdi", "bdo", "big", "cite", "code", "data", "del", "dfn", "em", "font", "i",
17    "img", "ins", "kbd", "mark", "q", "s", "samp", "small", "span", "strike", "strong", "sub",
18    "sup", "time", "tt", "u", "var", "wbr", "br", "ruby", "rt", "rp",
19];
20
21const HEADINGS: &[&str] = &["h1", "h2", "h3", "h4", "h5", "h6"];
22
23/// Above this serialised size, an "inline" element is checked for block-level
24/// content before being believed.
25///
26/// Inline elements are small by nature. A `<span>` carrying 50 KB of markup is
27/// not emphasis — it is an unclosed tag that swallowed the rest of the page,
28/// and flattening it would turn the whole document into one paragraph. The
29/// size gate keeps the check off the hot path: `html_len` is O(1) and the vast
30/// majority of inline nodes never pay for the descendant walk.
31const INLINE_INSPECT_BYTES: u32 = 2048;
32
33/// Language names accepted from a bare `class="rust"`-style attribute.
34///
35/// A whitelist rather than a pattern, because that attribute is shared with
36/// styling hooks: `hljs`, `notranslate` and `prettyprint` all look exactly like
37/// a language name to anything less strict.
38const KNOWN_LANGUAGES: &[&str] = &[
39    "bash",
40    "c",
41    "clojure",
42    "cpp",
43    "cs",
44    "csharp",
45    "css",
46    "dart",
47    "diff",
48    "dockerfile",
49    "elixir",
50    "elm",
51    "erlang",
52    "fsharp",
53    "go",
54    "graphql",
55    "groovy",
56    "haskell",
57    "hcl",
58    "html",
59    "ini",
60    "java",
61    "javascript",
62    "js",
63    "json",
64    "json5",
65    "jsx",
66    "julia",
67    "kotlin",
68    "latex",
69    "less",
70    "lisp",
71    "lua",
72    "makefile",
73    "markdown",
74    "matlab",
75    "nginx",
76    "nim",
77    "objectivec",
78    "ocaml",
79    "perl",
80    "php",
81    "powershell",
82    "protobuf",
83    "python",
84    "r",
85    "ruby",
86    "rust",
87    "sass",
88    "scala",
89    "scss",
90    "shell",
91    "sh",
92    "sql",
93    "svelte",
94    "swift",
95    "terraform",
96    "toml",
97    "ts",
98    "tsx",
99    "typescript",
100    "vim",
101    "vue",
102    "xml",
103    "yaml",
104    "zig",
105    "zsh",
106];
107
108/// Options that change what ends up in the Markdown.
109#[derive(Debug, Clone)]
110pub struct RenderOptions {
111    /// Keep `[text](href)` links instead of flattening to text.
112    pub include_links: bool,
113    /// Keep `![alt](src)` images.
114    pub include_images: bool,
115    /// Render `<table>` as a pipe table.
116    pub include_tables: bool,
117}
118
119impl Default for RenderOptions {
120    fn default() -> Self {
121        RenderOptions { include_links: true, include_images: false, include_tables: true }
122    }
123}
124
125pub(crate) struct Writer<'d, 'a> {
126    doc: &'d Doc<'a>,
127    cfg: &'d DenoiseConfig,
128    opts: &'d RenderOptions,
129    base: Option<Url>,
130    heading_stack: Vec<(u8, String)>,
131    list_depth: usize,
132    pub(crate) units: Vec<Unit>,
133    pub(crate) stats: DenoiseStats,
134}
135
136impl<'d, 'a> Writer<'d, 'a> {
137    pub(crate) fn new(
138        doc: &'d Doc<'a>,
139        cfg: &'d DenoiseConfig,
140        opts: &'d RenderOptions,
141        base: Option<Url>,
142    ) -> Self {
143        Writer {
144            doc,
145            cfg,
146            opts,
147            base,
148            heading_stack: Vec::new(),
149            list_depth: 0,
150            units: Vec::new(),
151            stats: DenoiseStats::default(),
152        }
153    }
154
155    /// Walk the chosen content root, without re-judging the root itself.
156    ///
157    /// The root arrives already decided: the content scorer picked it, or it
158    /// is `<body>` as a fallback. Re-asking the boilerplate heuristics about
159    /// it reopens a settled question with the wrong test — an article
160    /// container that also holds the navigation it is about to drop reads as
161    /// a link farm by raw density, and rejecting it discards the whole
162    /// document to save the part that was leaving anyway.
163    pub(crate) fn walk_root(&mut self, id: Id) {
164        self.walk_inner(id, false);
165    }
166
167    /// Walk a subtree, emitting units.
168    pub(crate) fn walk(&mut self, id: Id) {
169        self.walk_inner(id, true);
170    }
171
172    fn walk_inner(&mut self, id: Id, judge: bool) {
173        self.stats.nodes_visited += 1;
174        if judge && is_noise(self.doc, id, self.cfg) {
175            self.stats.nodes_dropped += self.doc.descendants(id).len();
176            return;
177        }
178        let name = self.doc.tag_name(id);
179
180        if let Some(level) = heading_level(name) {
181            let inline = self.render_children_inline(id);
182            let t = text::normalize_ws(&inline);
183            if !t.is_empty() {
184                // The plain form has to lose the link syntax, the same as a
185                // paragraph's does. Modern documentation wraps every heading
186                // in its own anchor, so leaving it in puts the whole URL into
187                // the text that BM25 scores and an embedding model reads --
188                // and into every breadcrumb built from the heading path.
189                let plain = strip_markdown(&t);
190                self.push_heading(level, &plain);
191                let md = format!("{} {}", "#".repeat(level as usize), t);
192                self.emit(UnitKind::Heading, level, plain, md);
193            }
194            return;
195        }
196
197        match name {
198            "" => {
199                // Bare text directly under a container.
200                if let Some(tl::Node::Raw(_)) = self.doc.node(id) {
201                    let t = text::normalize_ws(&self.render_inline(id));
202                    if text::visible_len(&t) >= self.cfg.min_block_len as usize {
203                        self.emit(UnitKind::Paragraph, 0, t.clone(), t);
204                    }
205                }
206            }
207            // A paragraph is only a paragraph if it holds no block-level
208            // element. HTML forbids that nesting, but lenient parsers -- this
209            // one included -- do not insert the implied `</p>` that a browser
210            // would, so an unclosed `<p>` can end up owning the rest of the
211            // document. Recursing costs one check and turns that failure into
212            // ordinary output.
213            "p" | "dd" | "dt" | "figcaption" | "summary" | "address" => {
214                if self.doc.is_leaf_block(id) {
215                    let t = text::normalize_ws(&self.render_children_inline(id));
216                    if !t.is_empty() {
217                        let plain = strip_markdown(&t);
218                        self.emit(UnitKind::Paragraph, 0, plain, t);
219                    }
220                } else {
221                    self.walk_children(id);
222                }
223            }
224            "pre" => self.emit_code(id),
225            "blockquote" if !self.doc.is_leaf_block(id) => self.walk_children(id),
226            "blockquote" => {
227                let inner = self.doc.inner_text(id);
228                if !inner.is_empty() {
229                    let md = inner.lines().map(|l| format!("> {l}")).collect::<Vec<_>>().join("\n");
230                    self.emit(UnitKind::Quote, 0, inner, md);
231                }
232            }
233            "ul" | "ol" | "dl" => {
234                self.list_depth += 1;
235                self.walk_children(id);
236                self.list_depth = self.list_depth.saturating_sub(1);
237            }
238            "li" => self.emit_list_item(id),
239            // A table holding headings, sections or other tables is laying out
240            // a page, not tabulating data. Rendering one as a pipe table turns
241            // a whole document into a single unreadable row — and lenient HTML
242            // parsers, this one included, will happily put an entire page
243            // inside a `<table>` whose close tag they failed to match.
244            "table" if self.is_data_table(id) => {
245                if self.opts.include_tables {
246                    self.emit_table(id);
247                }
248            }
249            "table" => self.walk_children(id),
250            "hr" | "br" => {}
251            _ => self.walk_children(id),
252        }
253    }
254
255    fn walk_children(&mut self, id: Id) {
256        // Group runs of inline children into implicit paragraphs so that text
257        // sitting loose inside a `<div>` is not lost.
258        let mut run: Vec<Id> = Vec::new();
259        for child in self.doc.children(id) {
260            if self.is_inline(child) {
261                run.push(child);
262            } else {
263                self.flush_inline_run(&mut run);
264                self.walk(child);
265            }
266        }
267        self.flush_inline_run(&mut run);
268    }
269
270    fn flush_inline_run(&mut self, run: &mut Vec<Id>) {
271        if run.is_empty() {
272            return;
273        }
274        // An image is a block in its own right, so a run that carries one is
275        // kept whatever its alt text weighs; the length gate exists to drop
276        // stray words, not figures the caller explicitly asked for.
277        let has_image =
278            self.opts.include_images && run.iter().any(|&id| self.doc.tag_name(id) == "img");
279        let mut buf = String::new();
280        for id in run.drain(..) {
281            buf.push_str(&self.render_inline(id));
282        }
283        let t = text::normalize_ws(&buf);
284        let plain = strip_markdown(&t);
285        if has_image || text::visible_len(&plain) >= self.cfg.min_block_len as usize {
286            self.emit(UnitKind::Paragraph, 0, plain, t);
287        }
288    }
289
290    fn is_inline(&self, id: Id) -> bool {
291        match self.doc.node(id) {
292            Some(tl::Node::Raw(_)) => true,
293            Some(tl::Node::Tag(tag)) => {
294                if !INLINE_TAGS.contains(&tag.name().as_utf8_str().to_ascii_lowercase().as_str()) {
295                    return false;
296                }
297                self.doc.html_len(id) <= INLINE_INSPECT_BYTES || self.doc.is_leaf_block(id)
298            }
299            _ => false,
300        }
301    }
302
303    fn emit_list_item(&mut self, id: Id) {
304        // Only the inline run becomes the bullet. Block children — nested
305        // lists, but also the paragraphs and tables that `<li>` is allowed to
306        // contain — are walked so they keep their own structure. Flattening
307        // them with `inner_text` is how a single mis-nested `<li>` ends up
308        // holding an entire page.
309        let mut inline_run: Vec<Id> = Vec::new();
310        let mut blocks: Vec<Id> = Vec::new();
311        for child in self.doc.children(id) {
312            if self.is_inline(child) {
313                inline_run.push(child);
314            } else {
315                blocks.push(child);
316            }
317        }
318
319        let mut buf = String::new();
320        for child in inline_run {
321            buf.push_str(&self.render_inline(child));
322        }
323        let t = text::normalize_ws(&buf);
324        if !t.is_empty() {
325            let depth = self.list_depth.saturating_sub(1).min(6);
326            let md = format!("{}- {}", "  ".repeat(depth), t);
327            self.emit(UnitKind::ListItem, depth as u8, strip_markdown(&t), md);
328        }
329        for child in blocks {
330            self.walk(child);
331        }
332    }
333
334    fn emit_code(&mut self, id: Id) {
335        let raw = self.doc.raw_text(id);
336        let body = raw.trim_end_matches('\n');
337        if body.trim().is_empty() {
338            return;
339        }
340        let lang = self.detect_language(id);
341        let md = format!("```{lang}\n{body}\n```");
342        self.emit(UnitKind::Code, 0, body.to_string(), md);
343    }
344
345    /// Work out what language a code block is in.
346    ///
347    /// There is no single convention. Highlighters write `language-rust`,
348    /// `lang-rust` or a bare `rust`; GitHub writes `highlight-source-rust` on a
349    /// wrapper *above* the `<pre>`; others use `data-lang`. A fenced block
350    /// without its language is markedly less useful to a model reading it, so
351    /// this checks all of them — ancestors included — rather than the first
352    /// class attribute it happens to find.
353    fn detect_language(&self, id: Id) -> String {
354        let mut candidates: Vec<Id> = vec![id];
355        candidates.extend(self.doc.descendants(id).into_iter().skip(1).take(8));
356        candidates.extend(self.doc.ancestors(id, 2));
357
358        for node in candidates {
359            for key in ["class", "data-lang", "data-language", "data-code-language"] {
360                let Some(value) = self.doc.attr(node, key) else { continue };
361                if key != "class" {
362                    if let Some(lang) = normalize_language(&value) {
363                        return lang;
364                    }
365                    continue;
366                }
367                for token in value.split_whitespace() {
368                    let stripped = token
369                        .strip_prefix("language-")
370                        .or_else(|| token.strip_prefix("lang-"))
371                        .or_else(|| token.strip_prefix("highlight-source-"))
372                        .or_else(|| token.strip_prefix("highlight-text-"))
373                        .or_else(|| token.strip_prefix("sourceCode-"));
374                    if let Some(lang) = stripped.and_then(normalize_language) {
375                        return lang;
376                    }
377                    // A bare class name, but only if it names a language we
378                    // recognise — `hljs`, `prettyprint` and `notranslate` all
379                    // sit in the same attribute.
380                    if let Some(lang) = normalize_language(token)
381                        && KNOWN_LANGUAGES.contains(&lang.as_str())
382                    {
383                        return lang;
384                    }
385                }
386            }
387        }
388        String::new()
389    }
390
391    /// Is this a table of data, or a table used for layout?
392    ///
393    /// Structural content inside cells is the giveaway. The caps are a second
394    /// line of defence against a `<table>` whose close tag the parser never
395    /// found, which swallows the rest of the document.
396    ///
397    /// Only the text cap speaks to context budget, and it is the tighter of
398    /// the two by far. The node cap has to stay well clear of what a real
399    /// table costs: a 223-row table of national GDP figures, six columns of
400    /// linked and footnoted cells, comes to 4,522 nodes around 6.4 KB of
401    /// text. Rejecting that emits nothing at all -- walking a data table as
402    /// ordinary blocks yields cells too short to survive as paragraphs -- so a
403    /// cap set near real tables does not degrade the output, it deletes it.
404    fn is_data_table(&self, id: Id) -> bool {
405        const MAX_TABLE_TEXT: u32 = 20_000;
406        const MAX_TABLE_NODES: usize = 30_000;
407
408        if self.doc.text_len(id) > MAX_TABLE_TEXT {
409            return false;
410        }
411        let descendants = self.doc.descendants(id);
412        if descendants.len() > MAX_TABLE_NODES {
413            return false;
414        }
415        !descendants.iter().skip(1).any(|&d| {
416            matches!(
417                self.doc.tag_name(d),
418                "table"
419                    | "h1"
420                    | "h2"
421                    | "h3"
422                    | "h4"
423                    | "h5"
424                    | "h6"
425                    | "section"
426                    | "article"
427                    | "aside"
428                    | "nav"
429                    | "footer"
430                    | "header"
431                    | "figure"
432            )
433        })
434    }
435
436    fn emit_table(&mut self, id: Id) {
437        let mut rows: Vec<Vec<String>> = Vec::new();
438        let mut header: Option<Vec<String>> = None;
439        for d in self.doc.descendants(id) {
440            if self.doc.tag_name(d) != "tr" || is_noise(self.doc, d, self.cfg) {
441                continue;
442            }
443            let mut cells = Vec::new();
444            let mut is_header = false;
445            for c in self.doc.children(d) {
446                let name = self.doc.tag_name(c);
447                if name == "th" {
448                    is_header = true;
449                } else if name != "td" {
450                    continue;
451                }
452                cells.push(text::normalize_ws(&self.render_children_inline(c)).replace('|', "\\|"));
453            }
454            if cells.is_empty() {
455                continue;
456            }
457            if is_header && header.is_none() {
458                header = Some(cells);
459            } else {
460                rows.push(cells);
461            }
462        }
463        if rows.is_empty() && header.is_none() {
464            return;
465        }
466        let width = header
467            .as_ref()
468            .map(Vec::len)
469            .into_iter()
470            .chain(rows.iter().map(Vec::len))
471            .max()
472            .unwrap_or(0);
473        if width == 0 {
474            return;
475        }
476        let pad = |mut r: Vec<String>| {
477            r.resize(width, String::new());
478            format!("| {} |", r.join(" | "))
479        };
480        let mut md = String::new();
481        let head = header.unwrap_or_else(|| vec![String::new(); width]);
482        md.push_str(&pad(head));
483        md.push('\n');
484        md.push_str(&format!("|{}", " --- |".repeat(width)));
485        for r in &rows {
486            md.push('\n');
487            md.push_str(&pad(r.clone()));
488        }
489        // Derived from what was actually written, not from the source subtree.
490        // A mis-nested table can hold far more text than it renders, and a unit
491        // whose `text` disagrees with its `markdown` corrupts every downstream
492        // measurement — token budgets, density, and the "did we find an
493        // article?" test alike.
494        let plain = strip_markdown(&md);
495        self.emit(UnitKind::Table, 0, plain, md);
496    }
497
498    fn render_children_inline(&self, id: Id) -> String {
499        let mut buf = String::new();
500        for child in self.doc.children(id) {
501            buf.push_str(&self.render_inline(child));
502        }
503        buf
504    }
505
506    /// Render a subtree as inline Markdown.
507    fn render_inline(&self, id: Id) -> String {
508        match self.doc.node(id) {
509            Some(tl::Node::Raw(bytes)) => {
510                let raw = bytes.as_utf8_str();
511                escape_inline(&decode_entities(&raw))
512            }
513            Some(tl::Node::Comment(_)) | None => String::new(),
514            Some(tl::Node::Tag(tag)) => {
515                let name = tag.name().as_utf8_str().to_ascii_lowercase();
516                if crate::denoise::DROP_TAGS.contains(&name.as_str()) {
517                    return String::new();
518                }
519                match name.as_str() {
520                    "br" => "\n".to_string(),
521                    "img" => {
522                        if !self.opts.include_images {
523                            return String::new();
524                        }
525                        let alt = self.doc.attr(id, "alt").unwrap_or_default();
526                        match self.resolve(id, "src").or_else(|| self.resolve(id, "data-src")) {
527                            Some(src) => format!("![{}]({})", escape_inline(&alt), src),
528                            None => String::new(),
529                        }
530                    }
531                    "a" => {
532                        let inner = self.render_children_inline(id);
533                        if !self.opts.include_links || inner.trim().is_empty() {
534                            return inner;
535                        }
536                        match self.resolve(id, "href") {
537                            Some(href) if !href.starts_with("javascript:") => {
538                                format!("[{}]({})", inner.trim(), href)
539                            }
540                            _ => inner,
541                        }
542                    }
543                    "strong" | "b" => wrap(&self.render_children_inline(id), "**"),
544                    "em" | "i" | "dfn" | "cite" | "var" => {
545                        wrap(&self.render_children_inline(id), "*")
546                    }
547                    "del" | "s" | "strike" => wrap(&self.render_children_inline(id), "~~"),
548                    "code" | "kbd" | "samp" | "tt" => {
549                        let inner = text::normalize_ws(&self.doc.inner_text(id));
550                        if inner.is_empty() { inner } else { format!("`{inner}`") }
551                    }
552                    "sup" => self.render_superscript(id),
553                    _ => self.render_children_inline(id),
554                }
555            }
556        }
557    }
558
559    /// `<sup>` is two unrelated things wearing one tag.
560    ///
561    /// A citation marker is noise an LLM should never see. An exponent is data:
562    /// dropping it silently turns 10^23 into 10, which is the kind of error
563    /// that survives every downstream check. Telling them apart is the point.
564    fn render_superscript(&self, id: Id) -> String {
565        let inner = text::normalize_ws(&self.doc.inner_text(id));
566        if inner.is_empty() {
567            return String::new();
568        }
569        if self.is_citation_marker(id, &inner) {
570            return String::new();
571        }
572        // `^` only where it disambiguates: `10^23` and `mol^-1` need it,
573        // `1st` reads worse as `1^st`.
574        let numeric = inner.chars().all(|c| c.is_ascii_digit() || "+-.,()".contains(c))
575            && inner.chars().any(|c| c.is_ascii_digit());
576        if numeric { format!("^{inner}") } else { inner }
577    }
578
579    /// Is this `<sup>` a footnote or reference marker?
580    fn is_citation_marker(&self, id: Id, inner: &str) -> bool {
581        // Wikipedia and most CMSes wrap the marker in a link to the note.
582        if self.doc.descendants(id).into_iter().skip(1).any(|d| self.doc.tag_name(d) == "a") {
583            return true;
584        }
585        let sig = self.doc.signature(id);
586        if !sig.is_empty()
587            && ["reference", "citation", "footnote", "cite"].iter().any(|w| sig.contains(w))
588        {
589            return true;
590        }
591        // A bracketed number standing alone: `[1]`, `(12)`.
592        let trimmed = inner.trim();
593        let bracketed = (trimmed.starts_with('[') && trimmed.ends_with(']'))
594            || (trimmed.starts_with('(') && trimmed.ends_with(')'));
595        bracketed && trimmed.chars().any(|c| c.is_ascii_digit())
596    }
597
598    /// Absolutise an attribute against the document's base URL.
599    fn resolve(&self, id: Id, key: &str) -> Option<String> {
600        let raw = self.doc.attr(id, key)?;
601        let raw = raw.trim();
602        if raw.is_empty() {
603            return None;
604        }
605        match &self.base {
606            Some(base) => base.join(raw).ok().map(|u| u.to_string()),
607            None => Some(raw.to_string()),
608        }
609    }
610
611    fn push_heading(&mut self, level: u8, title: &str) {
612        while self.heading_stack.last().is_some_and(|(l, _)| *l >= level) {
613            self.heading_stack.pop();
614        }
615        self.heading_stack.push((level, title.to_string()));
616    }
617
618    fn emit(&mut self, kind: UnitKind, level: u8, plain: String, markdown: String) {
619        let plain = plain.trim().to_string();
620        let markdown = markdown.trim().to_string();
621        if markdown.is_empty() {
622            return;
623        }
624        let path: Vec<String> = if kind == UnitKind::Heading {
625            self.heading_stack.iter().rev().skip(1).rev().map(|(_, t)| t.clone()).collect()
626        } else {
627            self.heading_stack.iter().map(|(_, t)| t.clone()).collect()
628        };
629        self.stats.markdown_bytes += markdown.len() + 2;
630        self.units.push(Unit {
631            kind,
632            level,
633            tokens: text::estimate_tokens(&markdown),
634            position: self.units.len(),
635            text: plain,
636            markdown,
637            heading_path: path,
638        });
639    }
640}
641
642/// Lowercase and sanity-check a language token.
643fn normalize_language(raw: &str) -> Option<String> {
644    let lang = raw.trim().trim_matches('"').to_ascii_lowercase();
645    let ok = !lang.is_empty()
646        && lang.len() <= 16
647        && lang.chars().all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '#' || c == '-');
648    ok.then_some(lang)
649}
650
651fn heading_level(name: &str) -> Option<u8> {
652    HEADINGS.iter().position(|h| *h == name).map(|i| i as u8 + 1)
653}
654
655fn wrap(inner: &str, marker: &str) -> String {
656    let trimmed = inner.trim();
657    if trimmed.is_empty() {
658        return String::new();
659    }
660    // Preserve the surrounding spacing the original markup implied.
661    let lead = if inner.starts_with(char::is_whitespace) { " " } else { "" };
662    let trail = if inner.ends_with(char::is_whitespace) { " " } else { "" };
663    format!("{lead}{marker}{trimmed}{marker}{trail}")
664}
665
666/// Escape only what would otherwise change Markdown block structure.
667///
668/// Aggressive escaping makes the text harder for a small model to read, so we
669/// leave `*` and `_` alone inside words and only defuse link syntax.
670fn escape_inline(s: &str) -> String {
671    if !s.contains(['[', ']']) {
672        return s.to_string();
673    }
674    s.replace('[', "\\[").replace(']', "\\]")
675}
676
677/// Strip the Markdown we just added, to recover plain text for ranking.
678pub(crate) fn strip_markdown(s: &str) -> String {
679    let mut out = String::with_capacity(s.len());
680    let mut chars = s.chars().peekable();
681    let mut in_link_text = false;
682    while let Some(c) = chars.next() {
683        match c {
684            '\\' => {
685                if let Some(n) = chars.next() {
686                    out.push(n);
687                }
688            }
689            '[' => in_link_text = true,
690            ']' => {
691                in_link_text = false;
692                // Drop the following `(...)` target.
693                if chars.peek() == Some(&'(') {
694                    chars.next();
695                    let mut depth = 1;
696                    for c in chars.by_ref() {
697                        match c {
698                            '(' => depth += 1,
699                            ')' => {
700                                depth -= 1;
701                                if depth == 0 {
702                                    break;
703                                }
704                            }
705                            _ => {}
706                        }
707                    }
708                }
709            }
710            '*' | '`' | '~' | '#' | '>' if !in_link_text => {}
711            _ => out.push(c),
712        }
713    }
714    text::normalize_ws(&out)
715}
716
717#[cfg(test)]
718mod tests {
719    use super::*;
720
721    #[test]
722    fn strips_links_and_emphasis() {
723        assert_eq!(strip_markdown("**bold** and [a link](http://x.dev/y)"), "bold and a link");
724        assert_eq!(strip_markdown("`code` ~~gone~~"), "code gone");
725    }
726
727    #[test]
728    fn wrap_keeps_outer_spacing() {
729        assert_eq!(wrap(" hi ", "**"), " **hi** ");
730        assert_eq!(wrap("   ", "**"), "");
731    }
732
733    #[test]
734    fn a_heading_wrapped_in_its_own_anchor_has_plain_text() {
735        // MDN, docs.rs and every MkDocs site render headings this way.
736        let html = concat!(
737            "<html><body><article><h2><a href=\"https://d.dev/a/b#sec\">Section</a></h2>",
738            "<p>Body text long enough to survive the block gate here.</p></article></body></html>"
739        );
740        let article = crate::parse::extract(html, Some("https://d.dev/a/b")).unwrap();
741        let heading = article.units.iter().find(|u| u.kind == UnitKind::Heading).unwrap();
742        assert_eq!(heading.text, "Section");
743        assert!(heading.markdown.contains("https://d.dev/a/b#sec"), "markdown keeps the link");
744        let below = article.units.iter().find(|u| u.kind == UnitKind::Paragraph).unwrap();
745        assert_eq!(below.heading_path, vec!["Section"], "breadcrumbs carry no URL");
746    }
747
748    #[test]
749    fn escape_only_touches_brackets() {
750        assert_eq!(escape_inline("a_b*c"), "a_b*c");
751        assert_eq!(escape_inline("see [1]"), "see \\[1\\]");
752    }
753}