1use std::sync::LazyLock;
18
19use regex::Regex;
20
21use crate::parse::dom::{Doc, Id};
22
23pub const DROP_TAGS: &[&str] = &[
25 "script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed",
26 "form", "input", "select", "textarea", "button", "label", "fieldset", "dialog", "audio",
27 "video", "map", "area", "link", "meta", "base",
28];
29
30pub const CHROME_TAGS: &[&str] = &["nav", "footer", "aside", "menu"];
32
33const TABLE_TAGS: &[&str] = &[
47 "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption", "colgroup", "col",
48 "pre", "code", "samp", "kbd",
53];
54
55static ABSOLUTE: LazyLock<Regex> = LazyLock::new(|| {
63 Regex::new(
64 r"(?i)(^|[-_\s])(ads?|adbox|adslot|adunit|advert|adverts|advertising|advertisement|advertisements|sponsors?|sponsored|sponsorship|promos?|promoted|promotions?|banners?|adsense|adsbygoogle|doubleclick|googlead|googleads|taboola|outbrain|revcontent|zergnet|mgid|criteo|popups?|interstitials?|paywall|cookies?|consent|gdpr)([-_\s]|$)",
65 )
66 .expect("static absolute regex")
67});
68
69static CONCLUSIVE: LazyLock<Regex> = LazyLock::new(|| {
77 Regex::new(
78 r"(?i)(^|[-_\s])(ads?|adbox|advert|advertisement|sponsors?|sponsored|promos?|promotions?|banners?|popups?|modals?|overlays?|interstitials?|cookies?|consent|gdpr|newsletters?|subscribe|signup|paywall|shares?|sharing|socials?|comments?|disqus|livefyre|related|recommend|recommended|recirc|outbrain|taboola|trending|breadcrumbs?|pagination|pagers?|navbars?|navigation|navbox|navboxes|masthead|footers?|toolbars?|skip|sr-only|screen-reader|visually-hidden|noprint|references?|reference-list|reflist|refbegin|refend|citations?|footnotes?|catlinks|authority-control|mw-editsection|mw-references|printfooter|sitesub|jump-link|mw-jump-link)([-_\s]|$)",
79 )
80 .expect("static conclusive regex")
81});
82
83static NEGATIVE: LazyLock<Regex> = LazyLock::new(|| {
84 Regex::new(
85 r"(?i)(^|[-_\s])(ad|ads|adbox|advert|advertisement|sponsor|sponsored|promo|promotion|banner|popup|modal|overlay|interstitial|cookie|consent|gdpr|newsletter|subscribe|signup|paywall|share|sharing|social|follow|comment|comments|disqus|livefyre|reply|sidebar|side-bar|widget|related|recommend|recirc|outbrain|taboola|trending|popular|breadcrumb|pagination|pager|paging|nav|navbar|navigation|menu|masthead|footer|header|topbar|toolbar|utility|skip|hidden|invisible|screen-reader|sr-only|visually-hidden|meta|byline|tags|tag-list|author-box|bio|cta|newsl|toc|table-of-contents|infobox|navbox|metadata|mw-editsection|reference|citation|footnote)([-_\s]|$)",
86 )
87 .expect("static negative regex")
88});
89
90static COMPONENT_SCOPED: LazyLock<Regex> = LazyLock::new(|| {
101 Regex::new(
102 r"(?i)(^|[-_\s])(sticky|table|col|column|row|grid|data|sort|sortable|cell|group|thead|tfoot)[-_](headers?|footers?)([-_\s]|$)",
103 )
104 .expect("static component-scoped regex")
105});
106
107static POSITIVE: LazyLock<Regex> = LazyLock::new(|| {
108 Regex::new(
109 r"(?i)(^|[-_\s])(article|articlebody|post|postbody|post-content|entry|entry-content|content|contents|main|maincontent|story|storybody|body|bodytext|text|prose|markdown|readme|blog|page-content|rich-text|document|paragraph|section-content|mw-parser-output|mw-content-text)([-_\s]|$)",
110 )
111 .expect("static positive regex")
112});
113
114#[derive(Debug, Clone)]
116pub struct DenoiseConfig {
117 pub min_block_len: u32,
120 pub max_link_density: f32,
122 pub min_text_ratio: f32,
124 pub drop_chrome: bool,
126 pub drop_by_class: bool,
128 pub keep_tables: bool,
130}
131
132impl Default for DenoiseConfig {
133 fn default() -> Self {
144 DenoiseConfig {
145 min_block_len: 25,
150 max_link_density: 0.6,
154 min_text_ratio: 0.02,
159 drop_chrome: true,
160 drop_by_class: true,
161 keep_tables: true,
162 }
163 }
164}
165
166#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
172pub struct DenoiseStats {
173 pub nodes_visited: usize,
175 pub nodes_dropped: usize,
178 pub html_bytes: usize,
180 pub markdown_bytes: usize,
182}
183
184impl DenoiseStats {
185 pub fn compression(&self) -> f32 {
187 if self.html_bytes == 0 {
188 return 0.0;
189 }
190 1.0 - (self.markdown_bytes as f32 / self.html_bytes as f32).min(1.0)
191 }
192}
193
194fn signature_for_match(doc: &Doc<'_>, id: Id) -> String {
200 let raw = doc.signature(id);
201 match COMPONENT_SCOPED.replace_all(&raw, " ") {
202 std::borrow::Cow::Borrowed(_) => raw,
203 std::borrow::Cow::Owned(cleaned) => cleaned,
204 }
205}
206
207pub(crate) fn class_weight(doc: &Doc<'_>, id: Id) -> f32 {
208 let sig = signature_for_match(doc, id);
209 if sig.is_empty() {
210 return 0.0;
211 }
212 let mut w = 0.0;
213 if NEGATIVE.is_match(&sig) || CONCLUSIVE.is_match(&sig) || ABSOLUTE.is_match(&sig) {
214 w -= 25.0;
215 }
216 if POSITIVE.is_match(&sig) {
217 w += 25.0;
218 }
219 w
220}
221
222fn holds_mostly_code(doc: &Doc<'_>, id: Id) -> bool {
229 let text = doc.text_len(id);
230 if text == 0 {
231 return false;
232 }
233 let mut in_code = 0u32;
234 let mut stack = vec![id];
235 while let Some(node) = stack.pop() {
236 if matches!(doc.tag_name(node), "pre" | "code") {
237 in_code = in_code.saturating_add(doc.text_len(node));
238 continue; }
240 stack.extend(doc.children(node));
241 }
242 in_code * 2 >= text
243}
244
245pub(crate) fn is_noise(doc: &Doc<'_>, id: Id, cfg: &DenoiseConfig) -> bool {
247 let name = doc.tag_name(id);
248 if name.is_empty() {
249 return false; }
251 if DROP_TAGS.contains(&name) {
252 return true;
253 }
254 if cfg.drop_chrome && CHROME_TAGS.contains(&name) {
255 return true;
256 }
257 if !cfg.keep_tables && name == "table" {
258 return true;
259 }
260 if doc.attr(id, "hidden").is_some()
261 || doc.attr(id, "aria-hidden").as_deref() == Some("true")
262 || doc.attr(id, "style").is_some_and(|s| {
263 let s = s.replace(' ', "").to_ascii_lowercase();
264 s.contains("display:none") || s.contains("visibility:hidden")
265 })
266 {
267 return true;
268 }
269 if doc.is_dominant(id) {
273 return false;
274 }
275
276 if matches!(
277 doc.attr(id, "role").as_deref(),
278 Some(
279 "navigation"
280 | "banner"
281 | "complementary"
282 | "search"
283 | "contentinfo"
284 | "alert"
285 | "dialog"
286 )
287 ) {
288 return true;
289 }
290
291 let text = doc.text_len(id);
292
293 if cfg.drop_by_class {
294 let sig = signature_for_match(doc, id);
295 if ABSOLUTE.is_match(&sig) {
297 return true;
298 }
299 if CONCLUSIVE.is_match(&sig) && !POSITIVE.is_match(&sig) {
301 return true;
302 }
303 if class_weight(doc, id) < 0.0 && text < 400 {
307 return true;
308 }
309 }
310
311 if TABLE_TAGS.contains(&name) {
312 return false;
313 }
314
315 if text >= cfg.min_block_len
318 && doc.link_density(id) > cfg.max_link_density
319 && !matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6")
320 {
321 return true;
322 }
323
324 if doc.html_len(id) > 512
330 && doc.text_ratio(id) < cfg.min_text_ratio
331 && !holds_mostly_code(doc, id)
332 {
333 return true;
334 }
335
336 false
337}
338
339pub(crate) fn find_content_root(doc: &Doc<'_>, cfg: &DenoiseConfig) -> Option<Id> {
347 let n = doc.dom.nodes().len();
348 let mut score = vec![0.0f32; n];
349 let mut scored = vec![false; n];
350
351 for &id in &doc.preorder {
352 let name = doc.tag_name(id);
353 if !matches!(
354 name,
355 "p" | "pre" | "td" | "blockquote" | "article" | "section" | "div" | "li" | "dd"
356 ) {
357 continue;
358 }
359 let text = doc.text_len(id);
360 if text < cfg.min_block_len {
361 continue;
362 }
363 if doc.link_density(id) > cfg.max_link_density {
364 continue;
365 }
366 if !doc.is_leaf_block(id) {
367 continue;
368 }
369
370 let inner = doc.inner_text(id);
371 let commas = inner.chars().filter(|c| matches!(c, ',' | ',' | '、' | ';')).count() as f32;
372 let stops =
375 inner.chars().filter(|c| matches!(c, '.' | '。' | '?' | '!' | '?' | '!')).count()
376 as f32;
377 let base = 1.0 + commas + stops * 0.5 + (text as f32 / 100.0).min(3.0);
378
379 for (level, anc) in doc.ancestors(id, 3).into_iter().enumerate() {
380 let divisor = match level {
381 0 => 1.0,
382 1 => 2.0,
383 _ => 3.0,
384 };
385 score[anc] += base / divisor;
386 scored[anc] = true;
387 }
388 }
389
390 let mut best: Option<(Id, f32)> = None;
391 for id in 0..n {
392 if !scored[id] {
393 continue;
394 }
395 let name = doc.tag_name(id);
396 if name == "body" || name == "html" || name.is_empty() {
397 continue;
398 }
399 let final_score = (score[id] + class_weight(doc, id)) * (1.0 - doc.link_density(id));
400 if final_score <= 0.0 {
401 continue;
402 }
403 if best.is_none_or(|(_, b)| final_score > b) {
404 best = Some((id, final_score));
405 }
406 }
407
408 let (mut root, mut root_score) = best?;
409 while let Some(p) = doc.parent(root) {
412 let name = doc.tag_name(p);
413 if name == "body" || name == "html" || name.is_empty() {
414 break;
415 }
416 let parent_score = (score[p] + class_weight(doc, p)) * (1.0 - doc.link_density(p));
417 if parent_score > root_score {
418 root = p;
419 root_score = parent_score;
420 } else {
421 break;
422 }
423 }
424 Some(root)
425}
426
427#[cfg(test)]
428mod tests {
429 use super::*;
430 use crate::parse::dom::Doc;
431
432 const PAGE: &str = r##"<html><body>
433 <header class="site-header"><a href="/">Home</a><a href="/about">About</a></header>
434 <nav class="main-nav"><a href="/x">X</a><a href="/y">Y</a><a href="/z">Z</a></nav>
435 <div class="ad-slot"><div><div><span></span></div></div></div>
436 <div id="content">
437 <article class="post-content">
438 <h1>Rust makes crawling cheap</h1>
439 <p>The first paragraph explains, at some length, why memory footprint matters, and it contains commas.</p>
440 <p>The second paragraph continues the argument with more prose, more commas, and several sentences. It ends here.</p>
441 </article>
442 </div>
443 <aside class="related-posts"><a href="/1">One</a><a href="/2">Two</a></aside>
444 <footer class="site-footer">© 2026</footer>
445 </body></html>"##;
446
447 fn id_of(doc: &Doc<'_>, sig: &str) -> Id {
448 doc.preorder
449 .iter()
450 .copied()
451 .find(|&i| doc.signature(i).contains(sig))
452 .unwrap_or_else(|| panic!("no node with signature {sig}"))
453 }
454
455 #[test]
456 fn content_root_is_the_article() {
457 let doc = Doc::parse(PAGE).unwrap();
458 let root = find_content_root(&doc, &DenoiseConfig::default()).expect("a root");
459 assert_eq!(doc.tag_name(root), "article", "picked {:?}", doc.signature(root));
460 }
461
462 #[test]
463 fn chrome_and_ads_are_noise() {
464 let doc = Doc::parse(PAGE).unwrap();
465 let cfg = DenoiseConfig::default();
466 for sig in ["main-nav", "ad-slot", "related-posts", "site-footer"] {
467 assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} should be noise");
468 }
469 }
470
471 #[test]
472 fn article_body_is_not_noise() {
473 let doc = Doc::parse(PAGE).unwrap();
474 let cfg = DenoiseConfig::default();
475 assert!(!is_noise(&doc, id_of(&doc, "post-content"), &cfg));
476 }
477
478 #[test]
479 fn large_reference_and_comment_blocks_are_dropped() {
480 let long = "Author, A. (2020). Some paper title here. Journal of Things. ".repeat(40);
484 let html = format!(
485 r#"<html><body><div id="content">
486 <p>{long}</p>
487 <ol class="mw-references references"><li>{long}</li></ol>
488 <div class="comments"><p>{long}</p></div>
489 <section class="related-articles"><p>{long}</p></section>
490 </div></body></html>"#
491 );
492 let doc = Doc::parse(&html).unwrap();
493 let cfg = DenoiseConfig::default();
494 for sig in ["mw-references", "comments", "related-articles"] {
495 assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} survived at full size");
496 }
497 }
498
499 #[test]
500 fn plural_class_names_are_matched() {
501 let prose = "Real article prose that runs well past any length filter here. ".repeat(30);
504 for sig in ["references", "citations", "footnotes", "comments", "ads", "related"] {
505 let html = format!(
506 r#"<html><body><article class="post"><p>{prose}</p></article>
507 <div class="{sig}"><p>some boilerplate text</p></div></body></html>"#
508 );
509 let doc = Doc::parse(&html).unwrap();
510 assert!(
511 is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
512 "class {sig:?} was not recognised"
513 );
514 }
515 }
516
517 #[test]
518 fn commercial_markers_outrank_a_positive_class() {
519 let prose = "Real article prose that runs past any length filter here. ".repeat(30);
521 for sig in ["sponsored-content", "ad-content", "promoted-story", "adsbygoogle"] {
522 let html = format!(
523 r#"<html><body><article class="post"><p>{prose}</p></article>
524 <div class="{sig}"><p>{prose}</p></div></body></html>"#
525 );
526 let doc = Doc::parse(&html).unwrap();
527 assert!(
528 is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
529 "{sig:?} survived"
530 );
531 }
532 }
533
534 #[test]
535 fn ad_tech_attributes_are_recognised() {
536 let prose = "Real article prose that runs past any length filter here. ".repeat(30);
537 let html = format!(
538 r#"<html><body><article class="post"><p>{prose}</p></article>
539 <div data-ad-unit="/1234/banner"><p>banner copy</p></div>
540 <div data-ad-client="ca-pub-1"><p>more banner copy</p></div>
541 </body></html>"#
542 );
543 let doc = Doc::parse(&html).unwrap();
544 let cfg = DenoiseConfig::default();
545 let ads: Vec<Id> = doc
546 .preorder
547 .iter()
548 .copied()
549 .filter(|&i| {
550 doc.attr(i, "data-ad-unit").is_some() || doc.attr(i, "data-ad-client").is_some()
551 })
552 .collect();
553 assert_eq!(ads.len(), 2);
554 for ad in ads {
555 assert!(is_noise(&doc, ad, &cfg), "ad-tech attribute not recognised");
556 }
557 }
558
559 #[test]
560 fn a_positive_class_overrides_the_conclusive_list() {
561 let html = r#"<html><body><div class="share entry-content"><p>Real prose that is long enough to matter here.</p></div></body></html>"#;
564 let doc = Doc::parse(html).unwrap();
565 assert!(!is_noise(&doc, id_of(&doc, "entry-content"), &DenoiseConfig::default()));
566 }
567
568 #[test]
569 fn a_node_holding_the_whole_page_is_never_boilerplate() {
570 let prose = "Real article prose that carries the paragraph well past any length filter. "
574 .repeat(30);
575 let html = format!(
576 r#"<html><body>
577 <table class="sidebar nomobile" role="navigation"><tr><td>
578 <h1>The article</h1><p>{prose}</p>
579 </td></tr></table>
580 </body></html>"#
581 );
582 let doc = Doc::parse(&html).unwrap();
583 let sidebar = id_of(&doc, "sidebar");
584 assert!(doc.is_dominant(sidebar));
585 assert!(!is_noise(&doc, sidebar, &DenoiseConfig::default()), "the page was discarded");
586 }
587
588 #[test]
589 fn a_genuine_sidebar_is_still_dropped() {
590 let prose =
591 "Real article prose that carries the paragraph past any length filter. ".repeat(30);
592 let html = format!(
593 r#"<html><body>
594 <article class="post"><h1>T</h1><p>{prose}</p></article>
595 <aside class="sidebar" role="navigation"><a href="/1">One</a><a href="/2">Two</a></aside>
596 </body></html>"#
597 );
598 let doc = Doc::parse(&html).unwrap();
599 let aside = id_of(&doc, "sidebar");
600 assert!(!doc.is_dominant(aside));
601 assert!(is_noise(&doc, aside, &DenoiseConfig::default()));
602 }
603
604 #[test]
605 fn class_weight_signs_are_right() {
606 let doc = Doc::parse(PAGE).unwrap();
607 assert!(class_weight(&doc, id_of(&doc, "post-content")) > 0.0);
608 assert!(class_weight(&doc, id_of(&doc, "ad-slot")) < 0.0);
609 }
610}
611
612#[cfg(test)]
613mod code_wrapper_tests {
614 use super::*;
615 use crate::parse::dom::Doc;
616
617 #[test]
621 fn a_wrapper_around_code_survives_the_ratio_test() {
622 const TOKEN: &str =
625 "<span class=\"tok\" style=\"color:#E06C75;font-weight:400;font-style:normal\">";
626 let mut code = String::new();
627 for i in 0..40 {
628 code.push_str(
629 "<span class=\"line\" style=\"display:block;min-height:1lh;white-space:pre\">",
630 );
631 for tok in ["l", "x", "=", "1", ";", " "] {
632 code.push_str(TOKEN);
633 code.push_str(tok);
634 code.push_str("</span>");
635 }
636 code.push_str("</span>\n");
637 let _ = i;
638 }
639 let html = format!(
640 "<html><body><article><p>{}</p>\
641 <div class=\"code-block-wrapper\"><pre class=\"astro-code\" \
642 style=\"--shiki-light:#abb2bf;--shiki-dark:#383A42;--shiki-light-bg:#282c34;\
643 --shiki-dark-bg:#FAFAFA;overflow-x:auto;white-space:pre-wrap\">\
644 <code>{code}</code></pre></div></article></body></html>",
645 "Prose with commas, at some length. ".repeat(6)
646 );
647 let doc = Doc::parse(&html).unwrap();
648 let cfg = DenoiseConfig::default();
649 let wrapper = doc
650 .preorder
651 .iter()
652 .copied()
653 .find(|&i| doc.signature(i).contains("code-block-wrapper"))
654 .expect("wrapper");
655 assert!(doc.text_ratio(wrapper) < cfg.min_text_ratio, "fixture is not markup-heavy");
656 assert!(!is_noise(&doc, wrapper, &cfg), "code wrapper was dropped");
657 }
658
659 #[test]
661 fn an_ordinary_widget_shell_is_still_dropped() {
662 let filler = "<div><span></span></div>".repeat(80);
663 let html = format!(
664 "<html><body><article><p>{}</p>\
665 <div class=\"player\">{filler}<i>x</i></div></article></body></html>",
666 "Prose with commas, at some length. ".repeat(20)
667 );
668 let doc = Doc::parse(&html).unwrap();
669 let cfg = DenoiseConfig::default();
670 let shell = doc
671 .preorder
672 .iter()
673 .copied()
674 .find(|&i| doc.signature(i).contains("player"))
675 .expect("shell");
676 assert!(is_noise(&doc, shell, &cfg), "widget shell survived");
677 }
678}