Skip to main content

rustai_core/denoise/
mod.rs

1//! Semantic denoising: find the article, drop everything else.
2//!
3//! Three signals, applied in this order, do nearly all the work:
4//!
5//! 1. **Structural priors** — `<nav>`, `<footer>`, `<aside>`, `role="banner"`
6//!    and a class/id vocabulary that has been stable across the web for a
7//!    decade.
8//! 2. **Link density** — anchor text over total text. Navigation and "related
9//!    posts" rails sit near 1.0; prose sits near 0.
10//! 3. **Text-to-HTML ratio** — visible characters over serialised markup
11//!    bytes. Ad slots and JS widgets are almost pure markup.
12//!
13//! Only then do we run a Readability-style content score to pick the container
14//! that actually holds the article, because scoring alone happily selects a
15//! comment thread on a thin page.
16
17use std::sync::LazyLock;
18
19use regex::Regex;
20
21use crate::parse::dom::{Doc, Id};
22
23/// Tags removed outright, whatever they contain.
24pub const DROP_TAGS: &[&str] = &[
25    "script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed",
26    "form", "input", "select", "textarea", "button", "label", "fieldset", "dialog", "audio",
27    "video", "map", "area", "link", "meta", "base",
28];
29
30/// Tags that are boilerplate by definition in the HTML5 outline.
31pub const CHROME_TAGS: &[&str] = &["nav", "footer", "aside", "menu"];
32
33/// Elements exempt from the two prose-shaped heuristics.
34///
35/// Link density and text-to-markup ratio both ask "does this container read
36/// like prose?", and a data table answers no however good it is: cells hold a
37/// word or a number, and a header row that cites its sources is *mostly*
38/// links. Judging a table by those thresholds drops the GDP of every nation
39/// for looking insufficiently like a paragraph. What a table is still gets
40/// decided -- by the vocabulary above, which catches an ad wherever it sits,
41/// and by the writer's own structural test for layout tables.
42///
43/// Code blocks fail the same tests for the same reason, and a highlighted one
44/// fails them badly: `<span class="line"><span>` around every token, with the
45/// theme inlined as `style`.
46const TABLE_TAGS: &[&str] = &[
47    "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption", "colgroup", "col",
48    // Code blocks answer those two questions the same way a table does. A
49    // syntax highlighter emits a `<span>` per token and a theme's worth of
50    // inline style, so the markup dwarfs the text; the text itself is source,
51    // which does not read like prose and never will.
52    "pre", "code", "samp", "kbd",
53];
54
55/// Class/id tokens that are never content, and that no positive marker
56/// outranks.
57///
58/// Split out from [`CONCLUSIVE`] because "positive wins ties" is the wrong
59/// rule here: `class="sponsored-content"` matches both vocabularies, and it is
60/// an advertisement. Commercial markers are unambiguous in a way that
61/// structural ones ("comments", "related") are not.
62static ABSOLUTE: LazyLock<Regex> = LazyLock::new(|| {
63    Regex::new(
64        r"(?i)(^|[-_\s])(ads?|adbox|adslot|adunit|advert|adverts|advertising|advertisement|advertisements|sponsors?|sponsored|sponsorship|promos?|promoted|promotions?|banners?|adsense|adsbygoogle|doubleclick|googlead|googleads|taboola|outbrain|revcontent|zergnet|mgid|criteo|popups?|interstitials?|paywall|cookies?|consent|gdpr)([-_\s]|$)",
65    )
66    .expect("static absolute regex")
67});
68
69/// Class/id tokens that justify dropping a subtree **at any size**.
70///
71/// The vocabulary below is high-precision: nothing here is ever the article. A
72/// reference list or a comment thread can run to thousands of characters, so
73/// the size guard that protects [`NEGATIVE`] from false positives would let
74/// them straight through — and a Wikipedia citation list will happily eat a
75/// whole context budget.
76static CONCLUSIVE: LazyLock<Regex> = LazyLock::new(|| {
77    Regex::new(
78        r"(?i)(^|[-_\s])(ads?|adbox|advert|advertisement|sponsors?|sponsored|promos?|promotions?|banners?|popups?|modals?|overlays?|interstitials?|cookies?|consent|gdpr|newsletters?|subscribe|signup|paywall|shares?|sharing|socials?|comments?|disqus|livefyre|related|recommend|recommended|recirc|outbrain|taboola|trending|breadcrumbs?|pagination|pagers?|navbars?|navigation|navbox|navboxes|masthead|footers?|toolbars?|skip|sr-only|screen-reader|visually-hidden|noprint|references?|reference-list|reflist|refbegin|refend|citations?|footnotes?|catlinks|authority-control|mw-editsection|mw-references|printfooter|sitesub|jump-link|mw-jump-link)([-_\s]|$)",
79    )
80    .expect("static conclusive regex")
81});
82
83static NEGATIVE: LazyLock<Regex> = LazyLock::new(|| {
84    Regex::new(
85        r"(?i)(^|[-_\s])(ad|ads|adbox|advert|advertisement|sponsor|sponsored|promo|promotion|banner|popup|modal|overlay|interstitial|cookie|consent|gdpr|newsletter|subscribe|signup|paywall|share|sharing|social|follow|comment|comments|disqus|livefyre|reply|sidebar|side-bar|widget|related|recommend|recirc|outbrain|taboola|trending|popular|breadcrumb|pagination|pager|paging|nav|navbar|navigation|menu|masthead|footer|header|topbar|toolbar|utility|skip|hidden|invisible|screen-reader|sr-only|visually-hidden|meta|byline|tags|tag-list|author-box|bio|cta|newsl|toc|table-of-contents|infobox|navbox|metadata|mw-editsection|reference|citation|footnote)([-_\s]|$)",
86    )
87    .expect("static negative regex")
88});
89
90/// `header` and `footer` qualified by a word that scopes them to a component
91/// rather than to the page.
92///
93/// The chrome vocabulary matches tokens, not whole class names, so `header`
94/// fires inside `sticky-header-multi` exactly as it does inside `site-header`.
95/// The first is a data table describing its own sticky column headings — the
96/// markup Wikipedia puts on every sortable table — and dropping it costs the
97/// reader the table. Compounds listed here are stripped from a signature
98/// before the vocabulary sees it; unqualified `header`, and page-scoped
99/// compounds like `site-header`, are left alone.
100static COMPONENT_SCOPED: LazyLock<Regex> = LazyLock::new(|| {
101    Regex::new(
102        r"(?i)(^|[-_\s])(sticky|table|col|column|row|grid|data|sort|sortable|cell|group|thead|tfoot)[-_](headers?|footers?)([-_\s]|$)",
103    )
104    .expect("static component-scoped regex")
105});
106
107static POSITIVE: LazyLock<Regex> = LazyLock::new(|| {
108    Regex::new(
109        r"(?i)(^|[-_\s])(article|articlebody|post|postbody|post-content|entry|entry-content|content|contents|main|maincontent|story|storybody|body|bodytext|text|prose|markdown|readme|blog|page-content|rich-text|document|paragraph|section-content|mw-parser-output|mw-content-text)([-_\s]|$)",
110    )
111    .expect("static positive regex")
112});
113
114/// Tuning knobs for the denoiser.
115#[derive(Debug, Clone)]
116pub struct DenoiseConfig {
117    /// A block shorter than this (in grapheme clusters) is dropped unless it is
118    /// a heading, list item or table cell.
119    pub min_block_len: u32,
120    /// Blocks above this anchor-text fraction are treated as navigation.
121    pub max_link_density: f32,
122    /// Blocks below this text-to-markup ratio are treated as widgets.
123    pub min_text_ratio: f32,
124    /// Drop `<nav>`, `<footer>`, `<aside>` and friends outright.
125    pub drop_chrome: bool,
126    /// Also drop nodes whose class/id matches the boilerplate vocabulary.
127    pub drop_by_class: bool,
128    /// Keep `<table>` content as Markdown tables rather than dropping them.
129    pub keep_tables: bool,
130}
131
132impl Default for DenoiseConfig {
133    /// Tuned against `benches/quality.py`, which scores extraction on 23 real
134    /// pages two ways: five-word shingle overlap with the article container
135    /// (recall and precision), and a count of site furniture that leaked in.
136    /// Both are needed. Judged on overlap alone the right answer is to switch
137    /// every threshold off -- it scores best -- but that quadruples the
138    /// furniture, because the article container the score compares against
139    /// never contained the navigation in the first place.
140    ///
141    /// From the defaults these replace, F1 goes 59.1% to 63.1% with furniture
142    /// unchanged at 4 occurrences over 23 pages.
143    fn default() -> Self {
144        DenoiseConfig {
145            // Holds the line at 25. This is the one threshold that guards
146            // against furniture rather than against widgets: "Jump to
147            // content" is 17 characters and "Skip to main" is 12, so
148            // lowering it to 15 takes leaked furniture from 4 to 16.
149            min_block_len: 25,
150            // 0.5 was costing recall for nothing measurable. Between 0.5 and
151            // 0.7 the link share of the output moves 6.7% to 7.1% while F1
152            // gains a point; 0.6 takes most of that for half the loosening.
153            max_link_density: 0.6,
154            // 0.06 was the single most expensive default: it drops legitimate
155            // markup-heavy content -- highlighted code, annotated tables --
156            // and buys almost no precision. Anything at or below 0.02 scores
157            // the same, so this keeps a floor rather than removing the test.
158            min_text_ratio: 0.02,
159            drop_chrome: true,
160            drop_by_class: true,
161            keep_tables: true,
162        }
163    }
164}
165
166/// What the denoiser threw away, for observability and tuning.
167///
168/// Counts are over DOM *nodes*, and a rejected node counts its whole subtree —
169/// dropping one `<nav>` removes everything under it, and a figure that said
170/// "1 node dropped" would be useless for judging whether the heuristics fired.
171#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
172pub struct DenoiseStats {
173    /// Nodes in the document.
174    pub nodes_visited: usize,
175    /// Nodes removed, counting whole rejected subtrees and everything outside
176    /// the chosen content root.
177    pub nodes_dropped: usize,
178    /// Bytes of HTML handed in.
179    pub html_bytes: usize,
180    /// Bytes of Markdown handed back.
181    pub markdown_bytes: usize,
182}
183
184impl DenoiseStats {
185    /// How much smaller the output is than the input, in `[0, 1]`.
186    pub fn compression(&self) -> f32 {
187        if self.html_bytes == 0 {
188            return 0.0;
189        }
190        1.0 - (self.markdown_bytes as f32 / self.html_bytes as f32).min(1.0)
191    }
192}
193
194/// Class/id weighting, mirroring Readability's `getClassWeight`.
195/// A node's class/id signature with component-scoped `header`/`footer`
196/// compounds removed, ready to match against the vocabularies.
197///
198/// See [`COMPONENT_SCOPED`] for why they are removed rather than matched.
199fn signature_for_match(doc: &Doc<'_>, id: Id) -> String {
200    let raw = doc.signature(id);
201    match COMPONENT_SCOPED.replace_all(&raw, " ") {
202        std::borrow::Cow::Borrowed(_) => raw,
203        std::borrow::Cow::Owned(cleaned) => cleaned,
204    }
205}
206
207pub(crate) fn class_weight(doc: &Doc<'_>, id: Id) -> f32 {
208    let sig = signature_for_match(doc, id);
209    if sig.is_empty() {
210        return 0.0;
211    }
212    let mut w = 0.0;
213    if NEGATIVE.is_match(&sig) || CONCLUSIVE.is_match(&sig) || ABSOLUTE.is_match(&sig) {
214        w -= 25.0;
215    }
216    if POSITIVE.is_match(&sig) {
217        w += 25.0;
218    }
219    w
220}
221
222/// Is this node's text mostly inside a code block?
223///
224/// A syntax highlighter turns ten lines of source into a `<span>` per token
225/// with a theme inlined as `style`, which reads as pure scaffolding to the
226/// text-to-markup ratio. The code is the content; the scaffolding is what
227/// displaying code costs.
228fn holds_mostly_code(doc: &Doc<'_>, id: Id) -> bool {
229    let text = doc.text_len(id);
230    if text == 0 {
231        return false;
232    }
233    let mut in_code = 0u32;
234    let mut stack = vec![id];
235    while let Some(node) = stack.pop() {
236        if matches!(doc.tag_name(node), "pre" | "code") {
237            in_code = in_code.saturating_add(doc.text_len(node));
238            continue; // 중첩된 code 를 두 번 세지 않는다
239        }
240        stack.extend(doc.children(node));
241    }
242    in_code * 2 >= text
243}
244
245/// Did the page itself mark this node invisible? Inline rendering asks too:
246/// KaTeX puts an `aria-hidden` copy of every formula inside the paragraph.
247pub(crate) fn is_hidden(doc: &Doc<'_>, id: Id) -> bool {
248    doc.attr(id, "hidden").is_some()
249        || doc.attr(id, "aria-hidden").as_deref() == Some("true")
250        || doc.attr(id, "style").is_some_and(|s| {
251            let s = s.replace(' ', "").to_ascii_lowercase();
252            s.contains("display:none") || s.contains("visibility:hidden")
253        })
254}
255
256/// Is this node boilerplate that should never reach the Markdown writer?
257pub(crate) fn is_noise(doc: &Doc<'_>, id: Id, cfg: &DenoiseConfig) -> bool {
258    let name = doc.tag_name(id);
259    if name.is_empty() {
260        return false; // raw text; judged through its parent
261    }
262    if DROP_TAGS.contains(&name) {
263        return true;
264    }
265    if cfg.drop_chrome && CHROME_TAGS.contains(&name) {
266        return true;
267    }
268    if !cfg.keep_tables && name == "table" {
269        return true;
270    }
271    if is_hidden(doc, id) {
272        return true;
273    }
274    // Past this point every rule is a heuristic about *where* content usually
275    // is not. None of them may overrule the arithmetic fact that this node
276    // holds most of the page.
277    if doc.is_dominant(id) {
278        return false;
279    }
280
281    if matches!(
282        doc.attr(id, "role").as_deref(),
283        Some(
284            "navigation"
285                | "banner"
286                | "complementary"
287                | "search"
288                | "contentinfo"
289                | "alert"
290                | "dialog"
291        )
292    ) {
293        return true;
294    }
295
296    let text = doc.text_len(id);
297
298    if cfg.drop_by_class {
299        let sig = signature_for_match(doc, id);
300        // Commercial markers outrank everything, including a positive one.
301        if ABSOLUTE.is_match(&sig) {
302            return true;
303        }
304        // High-precision vocabulary: never the article, whatever its size.
305        if CONCLUSIVE.is_match(&sig) && !POSITIVE.is_match(&sig) {
306            return true;
307        }
308        // The rest is suggestive, not conclusive: `<div id="main-content">` and
309        // `<div class="content-ad">` both match something, so require the node
310        // to also look thin before dropping it on the vocabulary alone.
311        if class_weight(doc, id) < 0.0 && text < 400 {
312            return true;
313        }
314    }
315
316    if TABLE_TAGS.contains(&name) {
317        return false;
318    }
319
320    // Link farms. Headings and list items legitimately run link-heavy, so only
321    // apply this to containers that carry real text.
322    if text >= cfg.min_block_len
323        && doc.link_density(id) > cfg.max_link_density
324        && !matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6")
325    {
326        return true;
327    }
328
329    // Markup-heavy, text-poor: a widget, a player shell, an ad slot -- unless
330    // the markup is a highlighter's and the text is source code. Exempting
331    // `<pre>` itself is not enough: a wrapper `<div>` around it is judged on
332    // its own, fails, and takes the code with it. Checked only on the way to
333    // rejecting, so the walk stays off the common path.
334    if doc.html_len(id) > 512
335        && doc.text_ratio(id) < cfg.min_text_ratio
336        && !holds_mostly_code(doc, id)
337    {
338        return true;
339    }
340
341    false
342}
343
344/// Pick the element that most likely holds the article body.
345///
346/// Readability's scoring, with two changes: the ratio and link-density signals
347/// above gate which paragraphs are allowed to contribute at all, and the climb
348/// out of the winning candidate stops at the first ancestor that does not
349/// improve, which keeps multi-section articles whole without swallowing the
350/// page furniture around them.
351pub(crate) fn find_content_root(doc: &Doc<'_>, cfg: &DenoiseConfig) -> Option<Id> {
352    let n = doc.dom.nodes().len();
353    let mut score = vec![0.0f32; n];
354    let mut scored = vec![false; n];
355
356    for &id in &doc.preorder {
357        let name = doc.tag_name(id);
358        if !matches!(
359            name,
360            "p" | "pre" | "td" | "blockquote" | "article" | "section" | "div" | "li" | "dd"
361        ) {
362            continue;
363        }
364        let text = doc.text_len(id);
365        if text < cfg.min_block_len {
366            continue;
367        }
368        if doc.link_density(id) > cfg.max_link_density {
369            continue;
370        }
371        if !doc.is_leaf_block(id) {
372            continue;
373        }
374
375        let inner = doc.inner_text(id);
376        let commas = inner.chars().filter(|c| matches!(c, ',' | ',' | '、' | ';')).count() as f32;
377        // Sentence-ish punctuation is the cheapest proxy for "this is prose,
378        // not a list of link labels".
379        let stops =
380            inner.chars().filter(|c| matches!(c, '.' | '。' | '?' | '!' | '?' | '!')).count()
381                as f32;
382        let base = 1.0 + commas + stops * 0.5 + (text as f32 / 100.0).min(3.0);
383
384        for (level, anc) in doc.ancestors(id, 3).into_iter().enumerate() {
385            let divisor = match level {
386                0 => 1.0,
387                1 => 2.0,
388                _ => 3.0,
389            };
390            score[anc] += base / divisor;
391            scored[anc] = true;
392        }
393    }
394
395    let mut best: Option<(Id, f32)> = None;
396    for id in 0..n {
397        if !scored[id] {
398            continue;
399        }
400        let name = doc.tag_name(id);
401        if name == "body" || name == "html" || name.is_empty() {
402            continue;
403        }
404        let final_score = (score[id] + class_weight(doc, id)) * (1.0 - doc.link_density(id));
405        if final_score <= 0.0 {
406            continue;
407        }
408        if best.is_none_or(|(_, b)| final_score > b) {
409            best = Some((id, final_score));
410        }
411    }
412
413    let (mut root, mut root_score) = best?;
414    // Climb while the parent genuinely scores better — that means the sibling
415    // sections around `root` are content too.
416    while let Some(p) = doc.parent(root) {
417        let name = doc.tag_name(p);
418        if name == "body" || name == "html" || name.is_empty() {
419            break;
420        }
421        let parent_score = (score[p] + class_weight(doc, p)) * (1.0 - doc.link_density(p));
422        if parent_score > root_score {
423            root = p;
424            root_score = parent_score;
425        } else {
426            break;
427        }
428    }
429    Some(root)
430}
431
432#[cfg(test)]
433mod tests {
434    use super::*;
435    use crate::parse::dom::Doc;
436
437    const PAGE: &str = r##"<html><body>
438      <header class="site-header"><a href="/">Home</a><a href="/about">About</a></header>
439      <nav class="main-nav"><a href="/x">X</a><a href="/y">Y</a><a href="/z">Z</a></nav>
440      <div class="ad-slot"><div><div><span></span></div></div></div>
441      <div id="content">
442        <article class="post-content">
443          <h1>Rust makes crawling cheap</h1>
444          <p>The first paragraph explains, at some length, why memory footprint matters, and it contains commas.</p>
445          <p>The second paragraph continues the argument with more prose, more commas, and several sentences. It ends here.</p>
446        </article>
447      </div>
448      <aside class="related-posts"><a href="/1">One</a><a href="/2">Two</a></aside>
449      <footer class="site-footer">© 2026</footer>
450    </body></html>"##;
451
452    fn id_of(doc: &Doc<'_>, sig: &str) -> Id {
453        doc.preorder
454            .iter()
455            .copied()
456            .find(|&i| doc.signature(i).contains(sig))
457            .unwrap_or_else(|| panic!("no node with signature {sig}"))
458    }
459
460    #[test]
461    fn content_root_is_the_article() {
462        let doc = Doc::parse(PAGE).unwrap();
463        let root = find_content_root(&doc, &DenoiseConfig::default()).expect("a root");
464        assert_eq!(doc.tag_name(root), "article", "picked {:?}", doc.signature(root));
465    }
466
467    #[test]
468    fn chrome_and_ads_are_noise() {
469        let doc = Doc::parse(PAGE).unwrap();
470        let cfg = DenoiseConfig::default();
471        for sig in ["main-nav", "ad-slot", "related-posts", "site-footer"] {
472            assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} should be noise");
473        }
474    }
475
476    #[test]
477    fn article_body_is_not_noise() {
478        let doc = Doc::parse(PAGE).unwrap();
479        let cfg = DenoiseConfig::default();
480        assert!(!is_noise(&doc, id_of(&doc, "post-content"), &cfg));
481    }
482
483    #[test]
484    fn large_reference_and_comment_blocks_are_dropped() {
485        // Wikipedia's citation list is `<ol class="mw-references references">` and
486        // runs to thousands of characters — far past the size guard that keeps
487        // the suggestive vocabulary honest.
488        let long = "Author, A. (2020). Some paper title here. Journal of Things. ".repeat(40);
489        let html = format!(
490            r#"<html><body><div id="content">
491                 <p>{long}</p>
492                 <ol class="mw-references references"><li>{long}</li></ol>
493                 <div class="comments"><p>{long}</p></div>
494                 <section class="related-articles"><p>{long}</p></section>
495               </div></body></html>"#
496        );
497        let doc = Doc::parse(&html).unwrap();
498        let cfg = DenoiseConfig::default();
499        for sig in ["mw-references", "comments", "related-articles"] {
500            assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} survived at full size");
501        }
502    }
503
504    #[test]
505    fn plural_class_names_are_matched() {
506        // The article has to be present and larger, or the dominance rule
507        // correctly refuses to call the only content on the page boilerplate.
508        let prose = "Real article prose that runs well past any length filter here. ".repeat(30);
509        for sig in ["references", "citations", "footnotes", "comments", "ads", "related"] {
510            let html = format!(
511                r#"<html><body><article class="post"><p>{prose}</p></article>
512                   <div class="{sig}"><p>some boilerplate text</p></div></body></html>"#
513            );
514            let doc = Doc::parse(&html).unwrap();
515            assert!(
516                is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
517                "class {sig:?} was not recognised"
518            );
519        }
520    }
521
522    #[test]
523    fn commercial_markers_outrank_a_positive_class() {
524        // `sponsored-content` matches both vocabularies. It is an advert.
525        let prose = "Real article prose that runs past any length filter here. ".repeat(30);
526        for sig in ["sponsored-content", "ad-content", "promoted-story", "adsbygoogle"] {
527            let html = format!(
528                r#"<html><body><article class="post"><p>{prose}</p></article>
529                   <div class="{sig}"><p>{prose}</p></div></body></html>"#
530            );
531            let doc = Doc::parse(&html).unwrap();
532            assert!(
533                is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
534                "{sig:?} survived"
535            );
536        }
537    }
538
539    #[test]
540    fn ad_tech_attributes_are_recognised() {
541        let prose = "Real article prose that runs past any length filter here. ".repeat(30);
542        let html = format!(
543            r#"<html><body><article class="post"><p>{prose}</p></article>
544               <div data-ad-unit="/1234/banner"><p>banner copy</p></div>
545               <div data-ad-client="ca-pub-1"><p>more banner copy</p></div>
546               </body></html>"#
547        );
548        let doc = Doc::parse(&html).unwrap();
549        let cfg = DenoiseConfig::default();
550        let ads: Vec<Id> = doc
551            .preorder
552            .iter()
553            .copied()
554            .filter(|&i| {
555                doc.attr(i, "data-ad-unit").is_some() || doc.attr(i, "data-ad-client").is_some()
556            })
557            .collect();
558        assert_eq!(ads.len(), 2);
559        for ad in ads {
560            assert!(is_noise(&doc, ad, &cfg), "ad-tech attribute not recognised");
561        }
562    }
563
564    #[test]
565    fn a_positive_class_overrides_the_conclusive_list() {
566        // `article-share-content` should not be dropped just because it says
567        // "share": an explicit content marker wins.
568        let html = r#"<html><body><div class="share entry-content"><p>Real prose that is long enough to matter here.</p></div></body></html>"#;
569        let doc = Doc::parse(html).unwrap();
570        assert!(!is_noise(&doc, id_of(&doc, "entry-content"), &DenoiseConfig::default()));
571    }
572
573    #[test]
574    fn a_node_holding_the_whole_page_is_never_boilerplate() {
575        // The shape a lenient parser produces when it cannot close a tag: the
576        // entire article ends up inside a navigation element. Dropping it on
577        // the class name would discard the page.
578        let prose = "Real article prose that carries the paragraph well past any length filter. "
579            .repeat(30);
580        let html = format!(
581            r#"<html><body>
582                 <table class="sidebar nomobile" role="navigation"><tr><td>
583                   <h1>The article</h1><p>{prose}</p>
584                 </td></tr></table>
585               </body></html>"#
586        );
587        let doc = Doc::parse(&html).unwrap();
588        let sidebar = id_of(&doc, "sidebar");
589        assert!(doc.is_dominant(sidebar));
590        assert!(!is_noise(&doc, sidebar, &DenoiseConfig::default()), "the page was discarded");
591    }
592
593    #[test]
594    fn a_genuine_sidebar_is_still_dropped() {
595        let prose =
596            "Real article prose that carries the paragraph past any length filter. ".repeat(30);
597        let html = format!(
598            r#"<html><body>
599                 <article class="post"><h1>T</h1><p>{prose}</p></article>
600                 <aside class="sidebar" role="navigation"><a href="/1">One</a><a href="/2">Two</a></aside>
601               </body></html>"#
602        );
603        let doc = Doc::parse(&html).unwrap();
604        let aside = id_of(&doc, "sidebar");
605        assert!(!doc.is_dominant(aside));
606        assert!(is_noise(&doc, aside, &DenoiseConfig::default()));
607    }
608
609    #[test]
610    fn class_weight_signs_are_right() {
611        let doc = Doc::parse(PAGE).unwrap();
612        assert!(class_weight(&doc, id_of(&doc, "post-content")) > 0.0);
613        assert!(class_weight(&doc, id_of(&doc, "ad-slot")) < 0.0);
614    }
615}
616
617#[cfg(test)]
618mod code_wrapper_tests {
619    use super::*;
620    use crate::parse::dom::Doc;
621
622    /// A highlighter's output is mostly markup by weight, and the `<div>` that
623    /// wraps it is judged on its own: it fails the ratio test, and the code
624    /// goes with it. Exempting `<pre>` alone does not save the wrapper.
625    #[test]
626    fn a_wrapper_around_code_survives_the_ratio_test() {
627        // Shaped like the real thing: a `<span>` per token, each carrying a
628        // slice of the theme, wrapped in a `<pre>` carrying the rest of it.
629        const TOKEN: &str =
630            "<span class=\"tok\" style=\"color:#E06C75;font-weight:400;font-style:normal\">";
631        let mut code = String::new();
632        for i in 0..40 {
633            code.push_str(
634                "<span class=\"line\" style=\"display:block;min-height:1lh;white-space:pre\">",
635            );
636            for tok in ["l", "x", "=", "1", ";", " "] {
637                code.push_str(TOKEN);
638                code.push_str(tok);
639                code.push_str("</span>");
640            }
641            code.push_str("</span>\n");
642            let _ = i;
643        }
644        let html = format!(
645            "<html><body><article><p>{}</p>\
646             <div class=\"code-block-wrapper\"><pre class=\"astro-code\" \
647             style=\"--shiki-light:#abb2bf;--shiki-dark:#383A42;--shiki-light-bg:#282c34;\
648             --shiki-dark-bg:#FAFAFA;overflow-x:auto;white-space:pre-wrap\">\
649             <code>{code}</code></pre></div></article></body></html>",
650            "Prose with commas, at some length. ".repeat(6)
651        );
652        let doc = Doc::parse(&html).unwrap();
653        let cfg = DenoiseConfig::default();
654        let wrapper = doc
655            .preorder
656            .iter()
657            .copied()
658            .find(|&i| doc.signature(i).contains("code-block-wrapper"))
659            .expect("wrapper");
660        assert!(doc.text_ratio(wrapper) < cfg.min_text_ratio, "fixture is not markup-heavy");
661        assert!(!is_noise(&doc, wrapper, &cfg), "code wrapper was dropped");
662    }
663
664    /// The exemption is for code, not for every markup-heavy container.
665    #[test]
666    fn an_ordinary_widget_shell_is_still_dropped() {
667        let filler = "<div><span></span></div>".repeat(80);
668        let html = format!(
669            "<html><body><article><p>{}</p>\
670             <div class=\"player\">{filler}<i>x</i></div></article></body></html>",
671            "Prose with commas, at some length. ".repeat(20)
672        );
673        let doc = Doc::parse(&html).unwrap();
674        let cfg = DenoiseConfig::default();
675        let shell = doc
676            .preorder
677            .iter()
678            .copied()
679            .find(|&i| doc.signature(i).contains("player"))
680            .expect("shell");
681        assert!(is_noise(&doc, shell, &cfg), "widget shell survived");
682    }
683}