Skip to main content

rustai_core/denoise/
mod.rs

1//! Semantic denoising: find the article, drop everything else.
2//!
3//! Three signals, applied in this order, do nearly all the work:
4//!
5//! 1. **Structural priors** — `<nav>`, `<footer>`, `<aside>`, `role="banner"`
6//!    and a class/id vocabulary that has been stable across the web for a
7//!    decade.
8//! 2. **Link density** — anchor text over total text. Navigation and "related
9//!    posts" rails sit near 1.0; prose sits near 0.
10//! 3. **Text-to-HTML ratio** — visible characters over serialised markup
11//!    bytes. Ad slots and JS widgets are almost pure markup.
12//!
13//! Only then do we run a Readability-style content score to pick the container
14//! that actually holds the article, because scoring alone happily selects a
15//! comment thread on a thin page.
16
17use std::sync::LazyLock;
18
19use regex::Regex;
20
21use crate::parse::dom::{Doc, Id};
22
23/// Tags removed outright, whatever they contain.
24pub const DROP_TAGS: &[&str] = &[
25    "script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed",
26    "form", "input", "select", "textarea", "button", "label", "fieldset", "dialog", "audio",
27    "video", "map", "area", "link", "meta", "base",
28];
29
30/// Tags that are boilerplate by definition in the HTML5 outline.
31pub const CHROME_TAGS: &[&str] = &["nav", "footer", "aside", "menu"];
32
33/// Elements exempt from the two prose-shaped heuristics.
34///
35/// Link density and text-to-markup ratio both ask "does this container read
36/// like prose?", and a data table answers no however good it is: cells hold a
37/// word or a number, and a header row that cites its sources is *mostly*
38/// links. Judging a table by those thresholds drops the GDP of every nation
39/// for looking insufficiently like a paragraph. What a table is still gets
40/// decided -- by the vocabulary above, which catches an ad wherever it sits,
41/// and by the writer's own structural test for layout tables.
42///
43/// Code blocks fail the same tests for the same reason, and a highlighted one
44/// fails them badly: `<span class="line"><span>` around every token, with the
45/// theme inlined as `style`.
46const TABLE_TAGS: &[&str] = &[
47    "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption", "colgroup", "col",
48    // Code blocks answer those two questions the same way a table does. A
49    // syntax highlighter emits a `<span>` per token and a theme's worth of
50    // inline style, so the markup dwarfs the text; the text itself is source,
51    // which does not read like prose and never will.
52    "pre", "code", "samp", "kbd",
53];
54
55/// Class/id tokens that are never content, and that no positive marker
56/// outranks.
57///
58/// Split out from [`CONCLUSIVE`] because "positive wins ties" is the wrong
59/// rule here: `class="sponsored-content"` matches both vocabularies, and it is
60/// an advertisement. Commercial markers are unambiguous in a way that
61/// structural ones ("comments", "related") are not.
62static ABSOLUTE: LazyLock<Regex> = LazyLock::new(|| {
63    Regex::new(
64        r"(?i)(^|[-_\s])(ads?|adbox|adslot|adunit|advert|adverts|advertising|advertisement|advertisements|sponsors?|sponsored|sponsorship|promos?|promoted|promotions?|banners?|adsense|adsbygoogle|doubleclick|googlead|googleads|taboola|outbrain|revcontent|zergnet|mgid|criteo|popups?|interstitials?|paywall|cookies?|consent|gdpr)([-_\s]|$)",
65    )
66    .expect("static absolute regex")
67});
68
69/// Class/id tokens that justify dropping a subtree **at any size**.
70///
71/// The vocabulary below is high-precision: nothing here is ever the article. A
72/// reference list or a comment thread can run to thousands of characters, so
73/// the size guard that protects [`NEGATIVE`] from false positives would let
74/// them straight through — and a Wikipedia citation list will happily eat a
75/// whole context budget.
76static CONCLUSIVE: LazyLock<Regex> = LazyLock::new(|| {
77    Regex::new(
78        r"(?i)(^|[-_\s])(ads?|adbox|advert|advertisement|sponsors?|sponsored|promos?|promotions?|banners?|popups?|modals?|overlays?|interstitials?|cookies?|consent|gdpr|newsletters?|subscribe|signup|paywall|shares?|sharing|socials?|comments?|disqus|livefyre|related|recommend|recommended|recirc|outbrain|taboola|trending|breadcrumbs?|pagination|pagers?|navbars?|navigation|navbox|navboxes|masthead|footers?|toolbars?|skip|sr-only|screen-reader|visually-hidden|noprint|references?|reference-list|reflist|refbegin|refend|citations?|footnotes?|catlinks|authority-control|mw-editsection|mw-references|printfooter|sitesub|jump-link|mw-jump-link)([-_\s]|$)",
79    )
80    .expect("static conclusive regex")
81});
82
83static NEGATIVE: LazyLock<Regex> = LazyLock::new(|| {
84    Regex::new(
85        r"(?i)(^|[-_\s])(ad|ads|adbox|advert|advertisement|sponsor|sponsored|promo|promotion|banner|popup|modal|overlay|interstitial|cookie|consent|gdpr|newsletter|subscribe|signup|paywall|share|sharing|social|follow|comment|comments|disqus|livefyre|reply|sidebar|side-bar|widget|related|recommend|recirc|outbrain|taboola|trending|popular|breadcrumb|pagination|pager|paging|nav|navbar|navigation|menu|masthead|footer|header|topbar|toolbar|utility|skip|hidden|invisible|screen-reader|sr-only|visually-hidden|meta|byline|tags|tag-list|author-box|bio|cta|newsl|toc|table-of-contents|infobox|navbox|metadata|mw-editsection|reference|citation|footnote)([-_\s]|$)",
86    )
87    .expect("static negative regex")
88});
89
90/// `header` and `footer` qualified by a word that scopes them to a component
91/// rather than to the page.
92///
93/// The chrome vocabulary matches tokens, not whole class names, so `header`
94/// fires inside `sticky-header-multi` exactly as it does inside `site-header`.
95/// The first is a data table describing its own sticky column headings — the
96/// markup Wikipedia puts on every sortable table — and dropping it costs the
97/// reader the table. Compounds listed here are stripped from a signature
98/// before the vocabulary sees it; unqualified `header`, and page-scoped
99/// compounds like `site-header`, are left alone.
100static COMPONENT_SCOPED: LazyLock<Regex> = LazyLock::new(|| {
101    Regex::new(
102        r"(?i)(^|[-_\s])(sticky|table|col|column|row|grid|data|sort|sortable|cell|group|thead|tfoot)[-_](headers?|footers?)([-_\s]|$)",
103    )
104    .expect("static component-scoped regex")
105});
106
107static POSITIVE: LazyLock<Regex> = LazyLock::new(|| {
108    Regex::new(
109        r"(?i)(^|[-_\s])(article|articlebody|post|postbody|post-content|entry|entry-content|content|contents|main|maincontent|story|storybody|body|bodytext|text|prose|markdown|readme|blog|page-content|rich-text|document|paragraph|section-content|mw-parser-output|mw-content-text)([-_\s]|$)",
110    )
111    .expect("static positive regex")
112});
113
114/// Tuning knobs for the denoiser.
115#[derive(Debug, Clone)]
116pub struct DenoiseConfig {
117    /// A block shorter than this (in grapheme clusters) is dropped unless it is
118    /// a heading, list item or table cell.
119    pub min_block_len: u32,
120    /// Blocks above this anchor-text fraction are treated as navigation.
121    pub max_link_density: f32,
122    /// Blocks below this text-to-markup ratio are treated as widgets.
123    pub min_text_ratio: f32,
124    /// Drop `<nav>`, `<footer>`, `<aside>` and friends outright.
125    pub drop_chrome: bool,
126    /// Also drop nodes whose class/id matches the boilerplate vocabulary.
127    pub drop_by_class: bool,
128    /// Keep `<table>` content as Markdown tables rather than dropping them.
129    pub keep_tables: bool,
130}
131
132impl Default for DenoiseConfig {
133    /// Tuned against `benches/quality.py`, which scores extraction on 23 real
134    /// pages two ways: five-word shingle overlap with the article container
135    /// (recall and precision), and a count of site furniture that leaked in.
136    /// Both are needed. Judged on overlap alone the right answer is to switch
137    /// every threshold off -- it scores best -- but that quadruples the
138    /// furniture, because the article container the score compares against
139    /// never contained the navigation in the first place.
140    ///
141    /// From the defaults these replace, F1 goes 59.1% to 63.1% with furniture
142    /// unchanged at 4 occurrences over 23 pages.
143    fn default() -> Self {
144        DenoiseConfig {
145            // Holds the line at 25. This is the one threshold that guards
146            // against furniture rather than against widgets: "Jump to
147            // content" is 17 characters and "Skip to main" is 12, so
148            // lowering it to 15 takes leaked furniture from 4 to 16.
149            min_block_len: 25,
150            // 0.5 was costing recall for nothing measurable. Between 0.5 and
151            // 0.7 the link share of the output moves 6.7% to 7.1% while F1
152            // gains a point; 0.6 takes most of that for half the loosening.
153            max_link_density: 0.6,
154            // 0.06 was the single most expensive default: it drops legitimate
155            // markup-heavy content -- highlighted code, annotated tables --
156            // and buys almost no precision. Anything at or below 0.02 scores
157            // the same, so this keeps a floor rather than removing the test.
158            min_text_ratio: 0.02,
159            drop_chrome: true,
160            drop_by_class: true,
161            keep_tables: true,
162        }
163    }
164}
165
166/// What the denoiser threw away, for observability and tuning.
167///
168/// Counts are over DOM *nodes*, and a rejected node counts its whole subtree —
169/// dropping one `<nav>` removes everything under it, and a figure that said
170/// "1 node dropped" would be useless for judging whether the heuristics fired.
171#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
172pub struct DenoiseStats {
173    /// Nodes in the document.
174    pub nodes_visited: usize,
175    /// Nodes removed, counting whole rejected subtrees and everything outside
176    /// the chosen content root.
177    pub nodes_dropped: usize,
178    /// Bytes of HTML handed in.
179    pub html_bytes: usize,
180    /// Bytes of Markdown handed back.
181    pub markdown_bytes: usize,
182}
183
184impl DenoiseStats {
185    /// How much smaller the output is than the input, in `[0, 1]`.
186    pub fn compression(&self) -> f32 {
187        if self.html_bytes == 0 {
188            return 0.0;
189        }
190        1.0 - (self.markdown_bytes as f32 / self.html_bytes as f32).min(1.0)
191    }
192}
193
194/// Class/id weighting, mirroring Readability's `getClassWeight`.
195/// A node's class/id signature with component-scoped `header`/`footer`
196/// compounds removed, ready to match against the vocabularies.
197///
198/// See [`COMPONENT_SCOPED`] for why they are removed rather than matched.
199fn signature_for_match(doc: &Doc<'_>, id: Id) -> String {
200    let raw = doc.signature(id);
201    match COMPONENT_SCOPED.replace_all(&raw, " ") {
202        std::borrow::Cow::Borrowed(_) => raw,
203        std::borrow::Cow::Owned(cleaned) => cleaned,
204    }
205}
206
207pub(crate) fn class_weight(doc: &Doc<'_>, id: Id) -> f32 {
208    let sig = signature_for_match(doc, id);
209    if sig.is_empty() {
210        return 0.0;
211    }
212    let mut w = 0.0;
213    if NEGATIVE.is_match(&sig) || CONCLUSIVE.is_match(&sig) || ABSOLUTE.is_match(&sig) {
214        w -= 25.0;
215    }
216    if POSITIVE.is_match(&sig) {
217        w += 25.0;
218    }
219    w
220}
221
222/// Is this node's text mostly inside a code block?
223///
224/// A syntax highlighter turns ten lines of source into a `<span>` per token
225/// with a theme inlined as `style`, which reads as pure scaffolding to the
226/// text-to-markup ratio. The code is the content; the scaffolding is what
227/// displaying code costs.
228fn holds_mostly_code(doc: &Doc<'_>, id: Id) -> bool {
229    let text = doc.text_len(id);
230    if text == 0 {
231        return false;
232    }
233    let mut in_code = 0u32;
234    let mut stack = vec![id];
235    while let Some(node) = stack.pop() {
236        if matches!(doc.tag_name(node), "pre" | "code") {
237            in_code = in_code.saturating_add(doc.text_len(node));
238            continue; // 중첩된 code 를 두 번 세지 않는다
239        }
240        stack.extend(doc.children(node));
241    }
242    in_code * 2 >= text
243}
244
245/// Is this node boilerplate that should never reach the Markdown writer?
246pub(crate) fn is_noise(doc: &Doc<'_>, id: Id, cfg: &DenoiseConfig) -> bool {
247    let name = doc.tag_name(id);
248    if name.is_empty() {
249        return false; // raw text; judged through its parent
250    }
251    if DROP_TAGS.contains(&name) {
252        return true;
253    }
254    if cfg.drop_chrome && CHROME_TAGS.contains(&name) {
255        return true;
256    }
257    if !cfg.keep_tables && name == "table" {
258        return true;
259    }
260    if doc.attr(id, "hidden").is_some()
261        || doc.attr(id, "aria-hidden").as_deref() == Some("true")
262        || doc.attr(id, "style").is_some_and(|s| {
263            let s = s.replace(' ', "").to_ascii_lowercase();
264            s.contains("display:none") || s.contains("visibility:hidden")
265        })
266    {
267        return true;
268    }
269    // Past this point every rule is a heuristic about *where* content usually
270    // is not. None of them may overrule the arithmetic fact that this node
271    // holds most of the page.
272    if doc.is_dominant(id) {
273        return false;
274    }
275
276    if matches!(
277        doc.attr(id, "role").as_deref(),
278        Some(
279            "navigation"
280                | "banner"
281                | "complementary"
282                | "search"
283                | "contentinfo"
284                | "alert"
285                | "dialog"
286        )
287    ) {
288        return true;
289    }
290
291    let text = doc.text_len(id);
292
293    if cfg.drop_by_class {
294        let sig = signature_for_match(doc, id);
295        // Commercial markers outrank everything, including a positive one.
296        if ABSOLUTE.is_match(&sig) {
297            return true;
298        }
299        // High-precision vocabulary: never the article, whatever its size.
300        if CONCLUSIVE.is_match(&sig) && !POSITIVE.is_match(&sig) {
301            return true;
302        }
303        // The rest is suggestive, not conclusive: `<div id="main-content">` and
304        // `<div class="content-ad">` both match something, so require the node
305        // to also look thin before dropping it on the vocabulary alone.
306        if class_weight(doc, id) < 0.0 && text < 400 {
307            return true;
308        }
309    }
310
311    if TABLE_TAGS.contains(&name) {
312        return false;
313    }
314
315    // Link farms. Headings and list items legitimately run link-heavy, so only
316    // apply this to containers that carry real text.
317    if text >= cfg.min_block_len
318        && doc.link_density(id) > cfg.max_link_density
319        && !matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6")
320    {
321        return true;
322    }
323
324    // Markup-heavy, text-poor: a widget, a player shell, an ad slot -- unless
325    // the markup is a highlighter's and the text is source code. Exempting
326    // `<pre>` itself is not enough: a wrapper `<div>` around it is judged on
327    // its own, fails, and takes the code with it. Checked only on the way to
328    // rejecting, so the walk stays off the common path.
329    if doc.html_len(id) > 512
330        && doc.text_ratio(id) < cfg.min_text_ratio
331        && !holds_mostly_code(doc, id)
332    {
333        return true;
334    }
335
336    false
337}
338
339/// Pick the element that most likely holds the article body.
340///
341/// Readability's scoring, with two changes: the ratio and link-density signals
342/// above gate which paragraphs are allowed to contribute at all, and the climb
343/// out of the winning candidate stops at the first ancestor that does not
344/// improve, which keeps multi-section articles whole without swallowing the
345/// page furniture around them.
346pub(crate) fn find_content_root(doc: &Doc<'_>, cfg: &DenoiseConfig) -> Option<Id> {
347    let n = doc.dom.nodes().len();
348    let mut score = vec![0.0f32; n];
349    let mut scored = vec![false; n];
350
351    for &id in &doc.preorder {
352        let name = doc.tag_name(id);
353        if !matches!(
354            name,
355            "p" | "pre" | "td" | "blockquote" | "article" | "section" | "div" | "li" | "dd"
356        ) {
357            continue;
358        }
359        let text = doc.text_len(id);
360        if text < cfg.min_block_len {
361            continue;
362        }
363        if doc.link_density(id) > cfg.max_link_density {
364            continue;
365        }
366        if !doc.is_leaf_block(id) {
367            continue;
368        }
369
370        let inner = doc.inner_text(id);
371        let commas = inner.chars().filter(|c| matches!(c, ',' | ',' | '、' | ';')).count() as f32;
372        // Sentence-ish punctuation is the cheapest proxy for "this is prose,
373        // not a list of link labels".
374        let stops =
375            inner.chars().filter(|c| matches!(c, '.' | '。' | '?' | '!' | '?' | '!')).count()
376                as f32;
377        let base = 1.0 + commas + stops * 0.5 + (text as f32 / 100.0).min(3.0);
378
379        for (level, anc) in doc.ancestors(id, 3).into_iter().enumerate() {
380            let divisor = match level {
381                0 => 1.0,
382                1 => 2.0,
383                _ => 3.0,
384            };
385            score[anc] += base / divisor;
386            scored[anc] = true;
387        }
388    }
389
390    let mut best: Option<(Id, f32)> = None;
391    for id in 0..n {
392        if !scored[id] {
393            continue;
394        }
395        let name = doc.tag_name(id);
396        if name == "body" || name == "html" || name.is_empty() {
397            continue;
398        }
399        let final_score = (score[id] + class_weight(doc, id)) * (1.0 - doc.link_density(id));
400        if final_score <= 0.0 {
401            continue;
402        }
403        if best.is_none_or(|(_, b)| final_score > b) {
404            best = Some((id, final_score));
405        }
406    }
407
408    let (mut root, mut root_score) = best?;
409    // Climb while the parent genuinely scores better — that means the sibling
410    // sections around `root` are content too.
411    while let Some(p) = doc.parent(root) {
412        let name = doc.tag_name(p);
413        if name == "body" || name == "html" || name.is_empty() {
414            break;
415        }
416        let parent_score = (score[p] + class_weight(doc, p)) * (1.0 - doc.link_density(p));
417        if parent_score > root_score {
418            root = p;
419            root_score = parent_score;
420        } else {
421            break;
422        }
423    }
424    Some(root)
425}
426
427#[cfg(test)]
428mod tests {
429    use super::*;
430    use crate::parse::dom::Doc;
431
432    const PAGE: &str = r##"<html><body>
433      <header class="site-header"><a href="/">Home</a><a href="/about">About</a></header>
434      <nav class="main-nav"><a href="/x">X</a><a href="/y">Y</a><a href="/z">Z</a></nav>
435      <div class="ad-slot"><div><div><span></span></div></div></div>
436      <div id="content">
437        <article class="post-content">
438          <h1>Rust makes crawling cheap</h1>
439          <p>The first paragraph explains, at some length, why memory footprint matters, and it contains commas.</p>
440          <p>The second paragraph continues the argument with more prose, more commas, and several sentences. It ends here.</p>
441        </article>
442      </div>
443      <aside class="related-posts"><a href="/1">One</a><a href="/2">Two</a></aside>
444      <footer class="site-footer">© 2026</footer>
445    </body></html>"##;
446
447    fn id_of(doc: &Doc<'_>, sig: &str) -> Id {
448        doc.preorder
449            .iter()
450            .copied()
451            .find(|&i| doc.signature(i).contains(sig))
452            .unwrap_or_else(|| panic!("no node with signature {sig}"))
453    }
454
455    #[test]
456    fn content_root_is_the_article() {
457        let doc = Doc::parse(PAGE).unwrap();
458        let root = find_content_root(&doc, &DenoiseConfig::default()).expect("a root");
459        assert_eq!(doc.tag_name(root), "article", "picked {:?}", doc.signature(root));
460    }
461
462    #[test]
463    fn chrome_and_ads_are_noise() {
464        let doc = Doc::parse(PAGE).unwrap();
465        let cfg = DenoiseConfig::default();
466        for sig in ["main-nav", "ad-slot", "related-posts", "site-footer"] {
467            assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} should be noise");
468        }
469    }
470
471    #[test]
472    fn article_body_is_not_noise() {
473        let doc = Doc::parse(PAGE).unwrap();
474        let cfg = DenoiseConfig::default();
475        assert!(!is_noise(&doc, id_of(&doc, "post-content"), &cfg));
476    }
477
478    #[test]
479    fn large_reference_and_comment_blocks_are_dropped() {
480        // Wikipedia's citation list is `<ol class="mw-references references">` and
481        // runs to thousands of characters — far past the size guard that keeps
482        // the suggestive vocabulary honest.
483        let long = "Author, A. (2020). Some paper title here. Journal of Things. ".repeat(40);
484        let html = format!(
485            r#"<html><body><div id="content">
486                 <p>{long}</p>
487                 <ol class="mw-references references"><li>{long}</li></ol>
488                 <div class="comments"><p>{long}</p></div>
489                 <section class="related-articles"><p>{long}</p></section>
490               </div></body></html>"#
491        );
492        let doc = Doc::parse(&html).unwrap();
493        let cfg = DenoiseConfig::default();
494        for sig in ["mw-references", "comments", "related-articles"] {
495            assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} survived at full size");
496        }
497    }
498
499    #[test]
500    fn plural_class_names_are_matched() {
501        // The article has to be present and larger, or the dominance rule
502        // correctly refuses to call the only content on the page boilerplate.
503        let prose = "Real article prose that runs well past any length filter here. ".repeat(30);
504        for sig in ["references", "citations", "footnotes", "comments", "ads", "related"] {
505            let html = format!(
506                r#"<html><body><article class="post"><p>{prose}</p></article>
507                   <div class="{sig}"><p>some boilerplate text</p></div></body></html>"#
508            );
509            let doc = Doc::parse(&html).unwrap();
510            assert!(
511                is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
512                "class {sig:?} was not recognised"
513            );
514        }
515    }
516
517    #[test]
518    fn commercial_markers_outrank_a_positive_class() {
519        // `sponsored-content` matches both vocabularies. It is an advert.
520        let prose = "Real article prose that runs past any length filter here. ".repeat(30);
521        for sig in ["sponsored-content", "ad-content", "promoted-story", "adsbygoogle"] {
522            let html = format!(
523                r#"<html><body><article class="post"><p>{prose}</p></article>
524                   <div class="{sig}"><p>{prose}</p></div></body></html>"#
525            );
526            let doc = Doc::parse(&html).unwrap();
527            assert!(
528                is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
529                "{sig:?} survived"
530            );
531        }
532    }
533
534    #[test]
535    fn ad_tech_attributes_are_recognised() {
536        let prose = "Real article prose that runs past any length filter here. ".repeat(30);
537        let html = format!(
538            r#"<html><body><article class="post"><p>{prose}</p></article>
539               <div data-ad-unit="/1234/banner"><p>banner copy</p></div>
540               <div data-ad-client="ca-pub-1"><p>more banner copy</p></div>
541               </body></html>"#
542        );
543        let doc = Doc::parse(&html).unwrap();
544        let cfg = DenoiseConfig::default();
545        let ads: Vec<Id> = doc
546            .preorder
547            .iter()
548            .copied()
549            .filter(|&i| {
550                doc.attr(i, "data-ad-unit").is_some() || doc.attr(i, "data-ad-client").is_some()
551            })
552            .collect();
553        assert_eq!(ads.len(), 2);
554        for ad in ads {
555            assert!(is_noise(&doc, ad, &cfg), "ad-tech attribute not recognised");
556        }
557    }
558
559    #[test]
560    fn a_positive_class_overrides_the_conclusive_list() {
561        // `article-share-content` should not be dropped just because it says
562        // "share": an explicit content marker wins.
563        let html = r#"<html><body><div class="share entry-content"><p>Real prose that is long enough to matter here.</p></div></body></html>"#;
564        let doc = Doc::parse(html).unwrap();
565        assert!(!is_noise(&doc, id_of(&doc, "entry-content"), &DenoiseConfig::default()));
566    }
567
568    #[test]
569    fn a_node_holding_the_whole_page_is_never_boilerplate() {
570        // The shape a lenient parser produces when it cannot close a tag: the
571        // entire article ends up inside a navigation element. Dropping it on
572        // the class name would discard the page.
573        let prose = "Real article prose that carries the paragraph well past any length filter. "
574            .repeat(30);
575        let html = format!(
576            r#"<html><body>
577                 <table class="sidebar nomobile" role="navigation"><tr><td>
578                   <h1>The article</h1><p>{prose}</p>
579                 </td></tr></table>
580               </body></html>"#
581        );
582        let doc = Doc::parse(&html).unwrap();
583        let sidebar = id_of(&doc, "sidebar");
584        assert!(doc.is_dominant(sidebar));
585        assert!(!is_noise(&doc, sidebar, &DenoiseConfig::default()), "the page was discarded");
586    }
587
588    #[test]
589    fn a_genuine_sidebar_is_still_dropped() {
590        let prose =
591            "Real article prose that carries the paragraph past any length filter. ".repeat(30);
592        let html = format!(
593            r#"<html><body>
594                 <article class="post"><h1>T</h1><p>{prose}</p></article>
595                 <aside class="sidebar" role="navigation"><a href="/1">One</a><a href="/2">Two</a></aside>
596               </body></html>"#
597        );
598        let doc = Doc::parse(&html).unwrap();
599        let aside = id_of(&doc, "sidebar");
600        assert!(!doc.is_dominant(aside));
601        assert!(is_noise(&doc, aside, &DenoiseConfig::default()));
602    }
603
604    #[test]
605    fn class_weight_signs_are_right() {
606        let doc = Doc::parse(PAGE).unwrap();
607        assert!(class_weight(&doc, id_of(&doc, "post-content")) > 0.0);
608        assert!(class_weight(&doc, id_of(&doc, "ad-slot")) < 0.0);
609    }
610}
611
612#[cfg(test)]
613mod code_wrapper_tests {
614    use super::*;
615    use crate::parse::dom::Doc;
616
617    /// A highlighter's output is mostly markup by weight, and the `<div>` that
618    /// wraps it is judged on its own: it fails the ratio test, and the code
619    /// goes with it. Exempting `<pre>` alone does not save the wrapper.
620    #[test]
621    fn a_wrapper_around_code_survives_the_ratio_test() {
622        // Shaped like the real thing: a `<span>` per token, each carrying a
623        // slice of the theme, wrapped in a `<pre>` carrying the rest of it.
624        const TOKEN: &str =
625            "<span class=\"tok\" style=\"color:#E06C75;font-weight:400;font-style:normal\">";
626        let mut code = String::new();
627        for i in 0..40 {
628            code.push_str(
629                "<span class=\"line\" style=\"display:block;min-height:1lh;white-space:pre\">",
630            );
631            for tok in ["l", "x", "=", "1", ";", " "] {
632                code.push_str(TOKEN);
633                code.push_str(tok);
634                code.push_str("</span>");
635            }
636            code.push_str("</span>\n");
637            let _ = i;
638        }
639        let html = format!(
640            "<html><body><article><p>{}</p>\
641             <div class=\"code-block-wrapper\"><pre class=\"astro-code\" \
642             style=\"--shiki-light:#abb2bf;--shiki-dark:#383A42;--shiki-light-bg:#282c34;\
643             --shiki-dark-bg:#FAFAFA;overflow-x:auto;white-space:pre-wrap\">\
644             <code>{code}</code></pre></div></article></body></html>",
645            "Prose with commas, at some length. ".repeat(6)
646        );
647        let doc = Doc::parse(&html).unwrap();
648        let cfg = DenoiseConfig::default();
649        let wrapper = doc
650            .preorder
651            .iter()
652            .copied()
653            .find(|&i| doc.signature(i).contains("code-block-wrapper"))
654            .expect("wrapper");
655        assert!(doc.text_ratio(wrapper) < cfg.min_text_ratio, "fixture is not markup-heavy");
656        assert!(!is_noise(&doc, wrapper, &cfg), "code wrapper was dropped");
657    }
658
659    /// The exemption is for code, not for every markup-heavy container.
660    #[test]
661    fn an_ordinary_widget_shell_is_still_dropped() {
662        let filler = "<div><span></span></div>".repeat(80);
663        let html = format!(
664            "<html><body><article><p>{}</p>\
665             <div class=\"player\">{filler}<i>x</i></div></article></body></html>",
666            "Prose with commas, at some length. ".repeat(20)
667        );
668        let doc = Doc::parse(&html).unwrap();
669        let cfg = DenoiseConfig::default();
670        let shell = doc
671            .preorder
672            .iter()
673            .copied()
674            .find(|&i| doc.signature(i).contains("player"))
675            .expect("shell");
676        assert!(is_noise(&doc, shell, &cfg), "widget shell survived");
677    }
678}