1use std::sync::LazyLock;
18
19use regex::Regex;
20
21use crate::parse::dom::{Doc, Id};
22
23pub const DROP_TAGS: &[&str] = &[
25 "script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed",
26 "form", "input", "select", "textarea", "button", "label", "fieldset", "dialog", "audio",
27 "video", "map", "area", "link", "meta", "base",
28];
29
30pub const CHROME_TAGS: &[&str] = &["nav", "footer", "aside", "menu"];
32
33const TABLE_TAGS: &[&str] = &[
47 "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption", "colgroup", "col",
48 "pre", "code", "samp", "kbd",
53];
54
55static ABSOLUTE: LazyLock<Regex> = LazyLock::new(|| {
63 Regex::new(
64 r"(?i)(^|[-_\s])(ads?|adbox|adslot|adunit|advert|adverts|advertising|advertisement|advertisements|sponsors?|sponsored|sponsorship|promos?|promoted|promotions?|banners?|adsense|adsbygoogle|doubleclick|googlead|googleads|taboola|outbrain|revcontent|zergnet|mgid|criteo|popups?|interstitials?|paywall|cookies?|consent|gdpr)([-_\s]|$)",
65 )
66 .expect("static absolute regex")
67});
68
69static CONCLUSIVE: LazyLock<Regex> = LazyLock::new(|| {
77 Regex::new(
78 r"(?i)(^|[-_\s])(ads?|adbox|advert|advertisement|sponsors?|sponsored|promos?|promotions?|banners?|popups?|modals?|overlays?|interstitials?|cookies?|consent|gdpr|newsletters?|subscribe|signup|paywall|shares?|sharing|socials?|comments?|disqus|livefyre|related|recommend|recommended|recirc|outbrain|taboola|trending|breadcrumbs?|pagination|pagers?|navbars?|navigation|navbox|navboxes|masthead|footers?|toolbars?|skip|sr-only|screen-reader|visually-hidden|noprint|references?|reference-list|reflist|refbegin|refend|citations?|footnotes?|catlinks|authority-control|mw-editsection|mw-references|printfooter|sitesub|jump-link|mw-jump-link)([-_\s]|$)",
79 )
80 .expect("static conclusive regex")
81});
82
83static NEGATIVE: LazyLock<Regex> = LazyLock::new(|| {
84 Regex::new(
85 r"(?i)(^|[-_\s])(ad|ads|adbox|advert|advertisement|sponsor|sponsored|promo|promotion|banner|popup|modal|overlay|interstitial|cookie|consent|gdpr|newsletter|subscribe|signup|paywall|share|sharing|social|follow|comment|comments|disqus|livefyre|reply|sidebar|side-bar|widget|related|recommend|recirc|outbrain|taboola|trending|popular|breadcrumb|pagination|pager|paging|nav|navbar|navigation|menu|masthead|footer|header|topbar|toolbar|utility|skip|hidden|invisible|screen-reader|sr-only|visually-hidden|meta|byline|tags|tag-list|author-box|bio|cta|newsl|toc|table-of-contents|infobox|navbox|metadata|mw-editsection|reference|citation|footnote)([-_\s]|$)",
86 )
87 .expect("static negative regex")
88});
89
90static COMPONENT_SCOPED: LazyLock<Regex> = LazyLock::new(|| {
101 Regex::new(
102 r"(?i)(^|[-_\s])(sticky|table|col|column|row|grid|data|sort|sortable|cell|group|thead|tfoot)[-_](headers?|footers?)([-_\s]|$)",
103 )
104 .expect("static component-scoped regex")
105});
106
107static POSITIVE: LazyLock<Regex> = LazyLock::new(|| {
108 Regex::new(
109 r"(?i)(^|[-_\s])(article|articlebody|post|postbody|post-content|entry|entry-content|content|contents|main|maincontent|story|storybody|body|bodytext|text|prose|markdown|readme|blog|page-content|rich-text|document|paragraph|section-content|mw-parser-output|mw-content-text)([-_\s]|$)",
110 )
111 .expect("static positive regex")
112});
113
114#[derive(Debug, Clone)]
116pub struct DenoiseConfig {
117 pub min_block_len: u32,
120 pub max_link_density: f32,
122 pub min_text_ratio: f32,
124 pub drop_chrome: bool,
126 pub drop_by_class: bool,
128 pub keep_tables: bool,
130}
131
132impl Default for DenoiseConfig {
133 fn default() -> Self {
144 DenoiseConfig {
145 min_block_len: 25,
150 max_link_density: 0.6,
154 min_text_ratio: 0.02,
159 drop_chrome: true,
160 drop_by_class: true,
161 keep_tables: true,
162 }
163 }
164}
165
166#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
172pub struct DenoiseStats {
173 pub nodes_visited: usize,
175 pub nodes_dropped: usize,
178 pub html_bytes: usize,
180 pub markdown_bytes: usize,
182}
183
184impl DenoiseStats {
185 pub fn compression(&self) -> f32 {
187 if self.html_bytes == 0 {
188 return 0.0;
189 }
190 1.0 - (self.markdown_bytes as f32 / self.html_bytes as f32).min(1.0)
191 }
192}
193
194fn signature_for_match(doc: &Doc<'_>, id: Id) -> String {
200 let raw = doc.signature(id);
201 match COMPONENT_SCOPED.replace_all(&raw, " ") {
202 std::borrow::Cow::Borrowed(_) => raw,
203 std::borrow::Cow::Owned(cleaned) => cleaned,
204 }
205}
206
207pub(crate) fn class_weight(doc: &Doc<'_>, id: Id) -> f32 {
208 let sig = signature_for_match(doc, id);
209 if sig.is_empty() {
210 return 0.0;
211 }
212 let mut w = 0.0;
213 if NEGATIVE.is_match(&sig) || CONCLUSIVE.is_match(&sig) || ABSOLUTE.is_match(&sig) {
214 w -= 25.0;
215 }
216 if POSITIVE.is_match(&sig) {
217 w += 25.0;
218 }
219 w
220}
221
222fn holds_mostly_code(doc: &Doc<'_>, id: Id) -> bool {
229 let text = doc.text_len(id);
230 if text == 0 {
231 return false;
232 }
233 let mut in_code = 0u32;
234 let mut stack = vec![id];
235 while let Some(node) = stack.pop() {
236 if matches!(doc.tag_name(node), "pre" | "code") {
237 in_code = in_code.saturating_add(doc.text_len(node));
238 continue; }
240 stack.extend(doc.children(node));
241 }
242 in_code * 2 >= text
243}
244
245pub(crate) fn is_hidden(doc: &Doc<'_>, id: Id) -> bool {
248 doc.attr(id, "hidden").is_some()
249 || doc.attr(id, "aria-hidden").as_deref() == Some("true")
250 || doc.attr(id, "style").is_some_and(|s| {
251 let s = s.replace(' ', "").to_ascii_lowercase();
252 s.contains("display:none") || s.contains("visibility:hidden")
253 })
254}
255
256pub(crate) fn is_noise(doc: &Doc<'_>, id: Id, cfg: &DenoiseConfig) -> bool {
258 let name = doc.tag_name(id);
259 if name.is_empty() {
260 return false; }
262 if DROP_TAGS.contains(&name) {
263 return true;
264 }
265 if cfg.drop_chrome && CHROME_TAGS.contains(&name) {
266 return true;
267 }
268 if !cfg.keep_tables && name == "table" {
269 return true;
270 }
271 if is_hidden(doc, id) {
272 return true;
273 }
274 if doc.is_dominant(id) {
278 return false;
279 }
280
281 if matches!(
282 doc.attr(id, "role").as_deref(),
283 Some(
284 "navigation"
285 | "banner"
286 | "complementary"
287 | "search"
288 | "contentinfo"
289 | "alert"
290 | "dialog"
291 )
292 ) {
293 return true;
294 }
295
296 let text = doc.text_len(id);
297
298 if cfg.drop_by_class {
299 let sig = signature_for_match(doc, id);
300 if ABSOLUTE.is_match(&sig) {
302 return true;
303 }
304 if CONCLUSIVE.is_match(&sig) && !POSITIVE.is_match(&sig) {
306 return true;
307 }
308 if class_weight(doc, id) < 0.0 && text < 400 {
312 return true;
313 }
314 }
315
316 if TABLE_TAGS.contains(&name) {
317 return false;
318 }
319
320 if text >= cfg.min_block_len
323 && doc.link_density(id) > cfg.max_link_density
324 && !matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6")
325 {
326 return true;
327 }
328
329 if doc.html_len(id) > 512
335 && doc.text_ratio(id) < cfg.min_text_ratio
336 && !holds_mostly_code(doc, id)
337 {
338 return true;
339 }
340
341 false
342}
343
344pub(crate) fn find_content_root(doc: &Doc<'_>, cfg: &DenoiseConfig) -> Option<Id> {
352 let n = doc.dom.nodes().len();
353 let mut score = vec![0.0f32; n];
354 let mut scored = vec![false; n];
355
356 for &id in &doc.preorder {
357 let name = doc.tag_name(id);
358 if !matches!(
359 name,
360 "p" | "pre" | "td" | "blockquote" | "article" | "section" | "div" | "li" | "dd"
361 ) {
362 continue;
363 }
364 let text = doc.text_len(id);
365 if text < cfg.min_block_len {
366 continue;
367 }
368 if doc.link_density(id) > cfg.max_link_density {
369 continue;
370 }
371 if !doc.is_leaf_block(id) {
372 continue;
373 }
374
375 let inner = doc.inner_text(id);
376 let commas = inner.chars().filter(|c| matches!(c, ',' | ',' | '、' | ';')).count() as f32;
377 let stops =
380 inner.chars().filter(|c| matches!(c, '.' | '。' | '?' | '!' | '?' | '!')).count()
381 as f32;
382 let base = 1.0 + commas + stops * 0.5 + (text as f32 / 100.0).min(3.0);
383
384 for (level, anc) in doc.ancestors(id, 3).into_iter().enumerate() {
385 let divisor = match level {
386 0 => 1.0,
387 1 => 2.0,
388 _ => 3.0,
389 };
390 score[anc] += base / divisor;
391 scored[anc] = true;
392 }
393 }
394
395 let mut best: Option<(Id, f32)> = None;
396 for id in 0..n {
397 if !scored[id] {
398 continue;
399 }
400 let name = doc.tag_name(id);
401 if name == "body" || name == "html" || name.is_empty() {
402 continue;
403 }
404 let final_score = (score[id] + class_weight(doc, id)) * (1.0 - doc.link_density(id));
405 if final_score <= 0.0 {
406 continue;
407 }
408 if best.is_none_or(|(_, b)| final_score > b) {
409 best = Some((id, final_score));
410 }
411 }
412
413 let (mut root, mut root_score) = best?;
414 while let Some(p) = doc.parent(root) {
417 let name = doc.tag_name(p);
418 if name == "body" || name == "html" || name.is_empty() {
419 break;
420 }
421 let parent_score = (score[p] + class_weight(doc, p)) * (1.0 - doc.link_density(p));
422 if parent_score > root_score {
423 root = p;
424 root_score = parent_score;
425 } else {
426 break;
427 }
428 }
429 Some(root)
430}
431
432#[cfg(test)]
433mod tests {
434 use super::*;
435 use crate::parse::dom::Doc;
436
437 const PAGE: &str = r##"<html><body>
438 <header class="site-header"><a href="/">Home</a><a href="/about">About</a></header>
439 <nav class="main-nav"><a href="/x">X</a><a href="/y">Y</a><a href="/z">Z</a></nav>
440 <div class="ad-slot"><div><div><span></span></div></div></div>
441 <div id="content">
442 <article class="post-content">
443 <h1>Rust makes crawling cheap</h1>
444 <p>The first paragraph explains, at some length, why memory footprint matters, and it contains commas.</p>
445 <p>The second paragraph continues the argument with more prose, more commas, and several sentences. It ends here.</p>
446 </article>
447 </div>
448 <aside class="related-posts"><a href="/1">One</a><a href="/2">Two</a></aside>
449 <footer class="site-footer">© 2026</footer>
450 </body></html>"##;
451
452 fn id_of(doc: &Doc<'_>, sig: &str) -> Id {
453 doc.preorder
454 .iter()
455 .copied()
456 .find(|&i| doc.signature(i).contains(sig))
457 .unwrap_or_else(|| panic!("no node with signature {sig}"))
458 }
459
460 #[test]
461 fn content_root_is_the_article() {
462 let doc = Doc::parse(PAGE).unwrap();
463 let root = find_content_root(&doc, &DenoiseConfig::default()).expect("a root");
464 assert_eq!(doc.tag_name(root), "article", "picked {:?}", doc.signature(root));
465 }
466
467 #[test]
468 fn chrome_and_ads_are_noise() {
469 let doc = Doc::parse(PAGE).unwrap();
470 let cfg = DenoiseConfig::default();
471 for sig in ["main-nav", "ad-slot", "related-posts", "site-footer"] {
472 assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} should be noise");
473 }
474 }
475
476 #[test]
477 fn article_body_is_not_noise() {
478 let doc = Doc::parse(PAGE).unwrap();
479 let cfg = DenoiseConfig::default();
480 assert!(!is_noise(&doc, id_of(&doc, "post-content"), &cfg));
481 }
482
483 #[test]
484 fn large_reference_and_comment_blocks_are_dropped() {
485 let long = "Author, A. (2020). Some paper title here. Journal of Things. ".repeat(40);
489 let html = format!(
490 r#"<html><body><div id="content">
491 <p>{long}</p>
492 <ol class="mw-references references"><li>{long}</li></ol>
493 <div class="comments"><p>{long}</p></div>
494 <section class="related-articles"><p>{long}</p></section>
495 </div></body></html>"#
496 );
497 let doc = Doc::parse(&html).unwrap();
498 let cfg = DenoiseConfig::default();
499 for sig in ["mw-references", "comments", "related-articles"] {
500 assert!(is_noise(&doc, id_of(&doc, sig), &cfg), "{sig} survived at full size");
501 }
502 }
503
504 #[test]
505 fn plural_class_names_are_matched() {
506 let prose = "Real article prose that runs well past any length filter here. ".repeat(30);
509 for sig in ["references", "citations", "footnotes", "comments", "ads", "related"] {
510 let html = format!(
511 r#"<html><body><article class="post"><p>{prose}</p></article>
512 <div class="{sig}"><p>some boilerplate text</p></div></body></html>"#
513 );
514 let doc = Doc::parse(&html).unwrap();
515 assert!(
516 is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
517 "class {sig:?} was not recognised"
518 );
519 }
520 }
521
522 #[test]
523 fn commercial_markers_outrank_a_positive_class() {
524 let prose = "Real article prose that runs past any length filter here. ".repeat(30);
526 for sig in ["sponsored-content", "ad-content", "promoted-story", "adsbygoogle"] {
527 let html = format!(
528 r#"<html><body><article class="post"><p>{prose}</p></article>
529 <div class="{sig}"><p>{prose}</p></div></body></html>"#
530 );
531 let doc = Doc::parse(&html).unwrap();
532 assert!(
533 is_noise(&doc, id_of(&doc, sig), &DenoiseConfig::default()),
534 "{sig:?} survived"
535 );
536 }
537 }
538
539 #[test]
540 fn ad_tech_attributes_are_recognised() {
541 let prose = "Real article prose that runs past any length filter here. ".repeat(30);
542 let html = format!(
543 r#"<html><body><article class="post"><p>{prose}</p></article>
544 <div data-ad-unit="/1234/banner"><p>banner copy</p></div>
545 <div data-ad-client="ca-pub-1"><p>more banner copy</p></div>
546 </body></html>"#
547 );
548 let doc = Doc::parse(&html).unwrap();
549 let cfg = DenoiseConfig::default();
550 let ads: Vec<Id> = doc
551 .preorder
552 .iter()
553 .copied()
554 .filter(|&i| {
555 doc.attr(i, "data-ad-unit").is_some() || doc.attr(i, "data-ad-client").is_some()
556 })
557 .collect();
558 assert_eq!(ads.len(), 2);
559 for ad in ads {
560 assert!(is_noise(&doc, ad, &cfg), "ad-tech attribute not recognised");
561 }
562 }
563
564 #[test]
565 fn a_positive_class_overrides_the_conclusive_list() {
566 let html = r#"<html><body><div class="share entry-content"><p>Real prose that is long enough to matter here.</p></div></body></html>"#;
569 let doc = Doc::parse(html).unwrap();
570 assert!(!is_noise(&doc, id_of(&doc, "entry-content"), &DenoiseConfig::default()));
571 }
572
573 #[test]
574 fn a_node_holding_the_whole_page_is_never_boilerplate() {
575 let prose = "Real article prose that carries the paragraph well past any length filter. "
579 .repeat(30);
580 let html = format!(
581 r#"<html><body>
582 <table class="sidebar nomobile" role="navigation"><tr><td>
583 <h1>The article</h1><p>{prose}</p>
584 </td></tr></table>
585 </body></html>"#
586 );
587 let doc = Doc::parse(&html).unwrap();
588 let sidebar = id_of(&doc, "sidebar");
589 assert!(doc.is_dominant(sidebar));
590 assert!(!is_noise(&doc, sidebar, &DenoiseConfig::default()), "the page was discarded");
591 }
592
593 #[test]
594 fn a_genuine_sidebar_is_still_dropped() {
595 let prose =
596 "Real article prose that carries the paragraph past any length filter. ".repeat(30);
597 let html = format!(
598 r#"<html><body>
599 <article class="post"><h1>T</h1><p>{prose}</p></article>
600 <aside class="sidebar" role="navigation"><a href="/1">One</a><a href="/2">Two</a></aside>
601 </body></html>"#
602 );
603 let doc = Doc::parse(&html).unwrap();
604 let aside = id_of(&doc, "sidebar");
605 assert!(!doc.is_dominant(aside));
606 assert!(is_noise(&doc, aside, &DenoiseConfig::default()));
607 }
608
609 #[test]
610 fn class_weight_signs_are_right() {
611 let doc = Doc::parse(PAGE).unwrap();
612 assert!(class_weight(&doc, id_of(&doc, "post-content")) > 0.0);
613 assert!(class_weight(&doc, id_of(&doc, "ad-slot")) < 0.0);
614 }
615}
616
617#[cfg(test)]
618mod code_wrapper_tests {
619 use super::*;
620 use crate::parse::dom::Doc;
621
622 #[test]
626 fn a_wrapper_around_code_survives_the_ratio_test() {
627 const TOKEN: &str =
630 "<span class=\"tok\" style=\"color:#E06C75;font-weight:400;font-style:normal\">";
631 let mut code = String::new();
632 for i in 0..40 {
633 code.push_str(
634 "<span class=\"line\" style=\"display:block;min-height:1lh;white-space:pre\">",
635 );
636 for tok in ["l", "x", "=", "1", ";", " "] {
637 code.push_str(TOKEN);
638 code.push_str(tok);
639 code.push_str("</span>");
640 }
641 code.push_str("</span>\n");
642 let _ = i;
643 }
644 let html = format!(
645 "<html><body><article><p>{}</p>\
646 <div class=\"code-block-wrapper\"><pre class=\"astro-code\" \
647 style=\"--shiki-light:#abb2bf;--shiki-dark:#383A42;--shiki-light-bg:#282c34;\
648 --shiki-dark-bg:#FAFAFA;overflow-x:auto;white-space:pre-wrap\">\
649 <code>{code}</code></pre></div></article></body></html>",
650 "Prose with commas, at some length. ".repeat(6)
651 );
652 let doc = Doc::parse(&html).unwrap();
653 let cfg = DenoiseConfig::default();
654 let wrapper = doc
655 .preorder
656 .iter()
657 .copied()
658 .find(|&i| doc.signature(i).contains("code-block-wrapper"))
659 .expect("wrapper");
660 assert!(doc.text_ratio(wrapper) < cfg.min_text_ratio, "fixture is not markup-heavy");
661 assert!(!is_noise(&doc, wrapper, &cfg), "code wrapper was dropped");
662 }
663
664 #[test]
666 fn an_ordinary_widget_shell_is_still_dropped() {
667 let filler = "<div><span></span></div>".repeat(80);
668 let html = format!(
669 "<html><body><article><p>{}</p>\
670 <div class=\"player\">{filler}<i>x</i></div></article></body></html>",
671 "Prose with commas, at some length. ".repeat(20)
672 );
673 let doc = Doc::parse(&html).unwrap();
674 let cfg = DenoiseConfig::default();
675 let shell = doc
676 .preorder
677 .iter()
678 .copied()
679 .find(|&i| doc.signature(i).contains("player"))
680 .expect("shell");
681 assert!(is_noise(&doc, shell, &cfg), "widget shell survived");
682 }
683}