1pub(crate) mod dom;
10mod index;
11mod markdown;
12mod meta;
13
14use rayon::prelude::*;
15use url::Url;
16
17const COMPARE_GAP_PERCENT: u64 = 120;
27
28use crate::denoise::{DenoiseConfig, DenoiseStats};
29
30use crate::error::Result;
31use crate::text;
32
33pub use index::Link;
34pub use markdown::RenderOptions;
35pub use meta::Meta;
36
37#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
39#[serde(rename_all = "snake_case")]
40pub enum ArticleKind {
41 Article,
43 Index,
45}
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
49pub enum IndexMode {
50 #[default]
52 Auto,
53 Never,
55 Always,
57}
58
59#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)]
61#[serde(rename_all = "snake_case")]
62pub enum UnitKind {
63 Heading,
65 Paragraph,
67 ListItem,
69 Code,
71 Quote,
73 Table,
75}
76
77impl UnitKind {
78 pub(crate) fn prior(self) -> f32 {
83 match self {
84 UnitKind::Heading => 1.25,
85 UnitKind::Paragraph => 1.0,
86 UnitKind::Code => 0.95,
87 UnitKind::Table => 0.9,
88 UnitKind::Quote => 0.85,
89 UnitKind::ListItem => 0.8,
90 }
91 }
92}
93
94#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
99pub struct Unit {
100 pub kind: UnitKind,
102 pub level: u8,
104 pub text: String,
106 pub markdown: String,
108 pub heading_path: Vec<String>,
111 pub position: usize,
113 pub tokens: usize,
115}
116
117#[derive(Debug, Clone, serde::Serialize)]
119pub struct Article {
120 pub url: Option<String>,
122 pub kind: ArticleKind,
124 pub meta: Meta,
126 pub markdown: String,
128 pub text: String,
130 pub units: Vec<Unit>,
132 pub links: Vec<Link>,
135 pub stats: DenoiseStats,
137}
138
139impl Article {
140 pub fn tokens(&self) -> usize {
142 self.units.iter().map(|u| u.tokens).sum()
143 }
144
145 pub fn title(&self) -> Option<&str> {
147 self.meta
148 .title
149 .as_deref()
150 .or_else(|| {
151 self.units.iter().find(|u| u.kind == UnitKind::Heading).map(|u| u.text.as_str())
152 })
153 .filter(|t| !t.is_empty())
154 }
155}
156
157#[derive(Debug, Clone)]
159pub struct ExtractOptions {
160 pub denoise: DenoiseConfig,
162 pub index_mode: IndexMode,
164 pub render: RenderOptions,
166 pub min_content_chars: usize,
170}
171
172impl ExtractOptions {
173 pub fn new() -> Self {
175 ExtractOptions {
176 denoise: DenoiseConfig::default(),
177 index_mode: IndexMode::default(),
178 render: RenderOptions::default(),
179 min_content_chars: 200,
180 }
181 }
182}
183
184pub fn extract(html: &str, url: Option<&str>) -> Result<Article> {
189 extract_with(html, url, &ExtractOptions::new())
190}
191
192pub fn extract_with(html: &str, url: Option<&str>, opts: &ExtractOptions) -> Result<Article> {
194 let unraw = dom::neutralise_raw_text(html);
201 let repaired = dom::neutralise_bogus_markup(&unraw);
202 let html = repaired.as_ref();
203 let doc = dom::Doc::parse(html)?;
204 let meta = meta::extract(&doc);
205
206 let base = url
207 .and_then(|u| Url::parse(u).ok())
208 .or_else(|| meta.canonical.as_deref().and_then(|c| Url::parse(c).ok()));
209
210 let body = {
215 let doc_root = doc.preorder.first().copied();
216 let body_tag = doc.preorder.iter().copied().find(|&id| doc.tag_name(id) == "body");
217 match (body_tag, doc_root) {
218 (Some(b), Some(r)) if doc.text_len(r) > doc.text_len(b).saturating_mul(2) => Some(r),
219 (Some(b), _) => Some(b),
220 (None, r) => r,
221 }
222 };
223
224 let root = crate::denoise::find_content_root(&doc, &opts.denoise).or(body);
225 let Some(root) = root else {
226 return Ok(Article {
227 url: url.map(str::to_string),
228 kind: ArticleKind::Article,
229 meta,
230 markdown: String::new(),
231 text: String::new(),
232 units: Vec::new(),
233 links: Vec::new(),
234 stats: DenoiseStats { html_bytes: html.len(), ..Default::default() },
235 });
236 };
237
238 let mut writer = markdown::Writer::new(&doc, &opts.denoise, &opts.render, base.clone());
239 writer.walk_root(root);
240
241 let extracted = |w: &markdown::Writer<'_, '_>| -> usize {
255 w.units.iter().map(|u| text::visible_len(&u.text)).sum()
256 };
257 let current_len = extracted(&writer);
258
259 let worth_comparing = body.is_some_and(|b| {
263 b != root && doc.text_len(b) as u64 * 100 > doc.text_len(root) as u64 * COMPARE_GAP_PERCENT
264 });
265
266 if (current_len < opts.min_content_chars || worth_comparing)
267 && let Some(body) = body
268 && body != root
269 {
270 let mut fallback = markdown::Writer::new(&doc, &opts.denoise, &opts.render, base.clone());
271 fallback.walk_root(body);
272 let fallback_len = extracted(&fallback);
273 if fallback_len > current_len {
274 writer = fallback;
275 }
276 }
277
278 let mut kind = ArticleKind::Article;
282 let mut links = Vec::new();
283 let article_chars = extracted(&writer);
284 if opts.index_mode != IndexMode::Never && index::could_be_index(&doc, article_chars) {
285 let harvested = index::harvest(&doc, base.as_ref(), &opts.denoise);
286 if index::looks_like_index(&harvested, article_chars) {
287 kind = ArticleKind::Index;
288 writer.units = index::units_from_links(&harvested);
289 links = harvested;
290 } else if opts.index_mode == IndexMode::Always {
291 links = harvested;
292 }
293 }
294
295 let mut units = writer.units;
296 for (i, u) in units.iter_mut().enumerate() {
297 u.position = i;
298 }
299 let markdown = units.iter().map(|u| u.markdown.as_str()).collect::<Vec<_>>().join("\n\n");
300 let text = units.iter().map(|u| u.text.as_str()).collect::<Vec<_>>().join("\n");
301
302 let mut stats = writer.stats;
303 let total_nodes = doc.preorder.len();
306 let outside_root = total_nodes.saturating_sub(doc.descendants(root).len());
307 stats.nodes_visited = total_nodes;
308 stats.nodes_dropped = (stats.nodes_dropped + outside_root).min(total_nodes);
309 stats.html_bytes = html.len();
310 stats.markdown_bytes = markdown.len();
311
312 Ok(Article { url: url.map(str::to_string), kind, meta, markdown, text, units, links, stats })
313}
314
315pub fn extract_many<'a, I>(docs: I, opts: &ExtractOptions) -> Vec<Result<Article>>
321where
322 I: IntoIterator<Item = (&'a str, Option<&'a str>)>,
323 I::IntoIter: Send,
324{
325 let docs: Vec<_> = docs.into_iter().collect();
326 docs.into_par_iter().map(|(html, url)| extract_with(html, url, opts)).collect()
327}
328
329#[cfg(test)]
330mod tests {
331 use super::*;
332
333 const PAGE: &str = r##"<html lang="en"><head><title>Zero-cost crawling</title></head><body>
334 <nav><a href="/a">A</a><a href="/b">B</a><a href="/c">C</a><a href="/d">D</a></nav>
335 <main><article class="post">
336 <h1>Zero-cost crawling</h1>
337 <p>Python crawlers pay for a DOM they never asked for, and it shows in RSS, in latency, and in bills.</p>
338 <h2>Why Rust</h2>
339 <p>An arena parser, a work-stealing pool, and no garbage collector at all: the memory ceiling is a design choice.</p>
340 <ul><li>Low memory</li><li>Fast parsing</li></ul>
341 <pre><code class="language-rust">let x = 1;</code></pre>
342 <table><tr><th>Tool</th><th>RSS</th></tr><tr><td>rustai</td><td>28 MB</td></tr></table>
343 </article></main>
344 <footer class="footer">© 2026 Example, all rights reserved, contact us</footer>
345 </body></html>"##;
346
347 #[test]
348 fn extracts_structure_and_drops_chrome() {
349 let art = extract(PAGE, Some("https://example.com/post")).unwrap();
350 assert_eq!(art.title(), Some("Zero-cost crawling"));
351 assert!(art.markdown.contains("# Zero-cost crawling"));
352 assert!(art.markdown.contains("## Why Rust"));
353 assert!(art.markdown.contains("- Low memory"));
354 assert!(art.markdown.contains("```rust\nlet x = 1;\n```"));
355 assert!(art.markdown.contains("| Tool | RSS |"));
356 assert!(!art.text.contains("all rights reserved"), "footer survived:\n{}", art.markdown);
357 assert!(!art.markdown.contains("](/a)"), "nav survived");
358 }
359
360 #[test]
361 fn units_carry_their_heading_path() {
362 let art = extract(PAGE, None).unwrap();
363 let para = art
364 .units
365 .iter()
366 .find(|u| u.text.starts_with("An arena parser"))
367 .expect("second paragraph");
368 assert_eq!(para.heading_path, ["Zero-cost crawling", "Why Rust"]);
369 assert!(para.tokens > 0);
370 }
371
372 #[test]
373 fn relative_links_are_absolutised() {
374 let html = r#"<article><p>Read the <a href="/docs/guide">guide</a> for the full walkthrough of the pipeline.</p></article>"#;
375 let art = extract(html, Some("https://example.com/blog/post")).unwrap();
376 assert!(art.markdown.contains("(https://example.com/docs/guide)"), "{}", art.markdown);
377 }
378
379 #[test]
380 fn compression_is_reported() {
381 let art = extract(PAGE, None).unwrap();
382 assert!(art.stats.compression() > 0.3, "only {}", art.stats.compression());
383 assert!(art.tokens() > 0);
384 }
385
386 #[test]
387 fn dropped_nodes_include_everything_outside_the_content_root() {
388 let art = extract(PAGE, None).unwrap();
389 assert!(art.stats.nodes_dropped > 0, "{:?}", art.stats);
391 assert!(art.stats.nodes_dropped < art.stats.nodes_visited);
392 }
393
394 #[test]
395 fn an_unclosed_paragraph_does_not_swallow_the_document() {
396 let html = "<article><p>Intro sentence that runs on <h2>A real heading</h2> <p>A second paragraph with enough words in it to be kept.</p> <h2>Another heading</h2> <p>A third paragraph, also long enough to survive the filter.</p> </article>";
400 let art = extract(html, None).unwrap();
401 assert!(
402 art.units.len() >= 4,
403 "collapsed into {} units:\n{}",
404 art.units.len(),
405 art.markdown
406 );
407 assert!(art.markdown.contains("## A real heading"));
408 assert!(art.markdown.contains("## Another heading"));
409 let biggest = art.units.iter().map(|u| u.tokens).max().unwrap();
410 assert!(biggest < 60, "one unit swallowed the rest: {biggest} tokens");
411 }
412
413 #[test]
414 fn an_inline_element_holding_blocks_does_not_flatten_the_page() {
415 let filler = "Real prose that carries the paragraph past the length filter. ".repeat(20);
424 let html = format!(
425 "<article><div class=\"notice\"><span>Notice text.\
426 <h2>First section</h2><p>{filler}</p>\
427 <h2>Second section</h2><p>{filler}</p></span></div></article>"
428 );
429 let art = extract(&html, None).unwrap();
430 assert!(art.markdown.contains("## First section"), "headings lost:\n{}", art.markdown);
431 assert!(art.markdown.contains("## Second section"));
432 assert!(
434 art.units.len() >= 4,
435 "collapsed into {} units:\n{}",
436 art.units.len(),
437 art.markdown
438 );
439 }
440
441 #[test]
442 fn a_list_item_holding_blocks_keeps_their_structure() {
443 let prose = "Real prose inside a list item, long enough to survive the filter. ";
444 let html = format!(
445 "<article><ul><li>Bullet text\
446 <h2>A heading inside the item</h2><p>{prose}</p>\
447 <ul><li>Nested bullet</li></ul></li></ul></article>"
448 );
449 let art = extract(&html, None).unwrap();
450 assert!(art.markdown.contains("- Bullet text"));
451 assert!(art.markdown.contains("## A heading inside the item"), "{}", art.markdown);
452 assert!(art.markdown.contains("- Nested bullet"));
453 let biggest = art.units.iter().map(|u| u.tokens).max().unwrap();
454 assert!(biggest < 40, "the item swallowed its blocks: {biggest} tokens");
455 }
456
457 #[test]
458 fn small_inline_elements_stay_inline() {
459 let html = "<article><p>A sentence with <strong>bold</strong> and <em>italic</em> and <code>code</code> in it, long enough to keep.</p></article>";
460 let art = extract(html, None).unwrap();
461 assert_eq!(art.units.len(), 1, "inline runs were split:\n{}", art.markdown);
462 assert!(art.markdown.contains("**bold**"));
463 assert!(art.markdown.contains("*italic*"));
464 assert!(art.markdown.contains("`code`"));
465 }
466
467 #[test]
468 fn a_layout_table_is_walked_not_tabulated() {
469 let html = "<table><tr><td> <h1>Page title</h1> <p>A paragraph of real prose that is long enough to keep around.</p> <h2>Section</h2> <p>Another paragraph of real prose, also long enough to keep.</p> </td></tr></table>";
473 let art = extract(html, None).unwrap();
474 assert!(!art.units.iter().any(|u| u.kind == UnitKind::Table), "{}", art.markdown);
475 assert!(art.markdown.contains("# Page title"));
476 assert!(art.markdown.contains("## Section"));
477 }
478
479 #[test]
480 fn a_real_data_table_is_still_tabulated() {
481 let html = "<article><p>Some prose introducing the numbers below it.</p> <table><tr><th>Engine</th><th>RSS</th></tr> <tr><td>arena</td><td>4.8 MiB</td></tr></table></article>";
482 let art = extract(html, None).unwrap();
483 assert!(art.markdown.contains("| Engine | RSS |"), "{}", art.markdown);
484 assert!(art.units.iter().any(|u| u.kind == UnitKind::Table));
485 }
486
487 #[test]
488 fn empty_input_is_not_an_error() {
489 let art = extract("", None).unwrap();
490 assert!(art.units.is_empty());
491 }
492
493 #[test]
494 fn many_documents_extract_in_parallel() {
495 let inputs: Vec<(&str, Option<&str>)> = (0..8).map(|_| (PAGE, None)).collect();
496 let out = extract_many(inputs, &ExtractOptions::new());
497 assert_eq!(out.len(), 8);
498 assert!(out.iter().all(|r| r.as_ref().is_ok_and(|a| !a.units.is_empty())));
499 }
500}
501
502#[cfg(test)]
503mod superscript_tests {
504 use super::*;
505
506 fn md(fragment: &str) -> String {
507 let html = format!("<article><p>{} {fragment}</p></article>", "Prose. ".repeat(12));
508 extract(&html, None).unwrap().markdown
509 }
510
511 #[test]
512 fn exponents_and_units_survive() {
513 assert!(md("6.02 × 10<sup>23</sup> mol<sup>-1</sup>").contains("10^23 mol^-1"));
514 assert!(md("area is 5 m<sup>2</sup>").contains("m^2"));
515 }
516
517 #[test]
518 fn ordinals_read_naturally() {
519 assert!(md("the 1<sup>st</sup> release").contains("1st release"));
520 }
521
522 #[test]
523 fn citation_markers_are_dropped() {
524 for marker in [
525 r#"<sup class="reference">[1]</sup>"#,
526 r##"<sup><a href="#cite_note-1">[12]</a></sup>"##,
527 "<sup>[3]</sup>",
528 ] {
529 let out = md(&format!("A claim{marker} follows."));
530 assert!(out.contains("A claim follows.") || out.contains("A claim follows."), "{out}");
531 assert!(!out.contains('['), "marker survived: {out}");
532 }
533 }
534
535 #[test]
536 fn attribute_entities_are_decoded() {
537 let html = r#"<html><head><meta property="og:title" content="Steve Irwin's family & friends"></head>
538 <body><article><p>Body text long enough to keep around here.</p></article></body></html>"#;
539 let art = extract(html, None).unwrap();
540 assert_eq!(art.meta.title.as_deref(), Some("Steve Irwin's family & friends"));
541 }
542}
543
544#[cfg(test)]
545mod table_regressions {
546 use super::*;
547
548 fn page(table: &str) -> String {
552 format!(
553 "<html><body><article><h1>Figures</h1><p>{}</p>{table}</article></body></html>",
554 "A sentence of ordinary prose, with commas, to anchor the page. ".repeat(6)
555 )
556 }
557
558 fn table_rows(html: &str) -> usize {
559 let art = extract(html, Some("https://example.com/x")).expect("extract");
560 art.markdown.lines().filter(|l| l.starts_with('|')).count()
561 }
562
563 const ROWS: &str = "<tr><th>Country</th><th>GDP</th></tr>\
564 <tr><td>Korea</td><td>1,870,000</td></tr>\
565 <tr><td>Japan</td><td>4,230,000</td></tr>";
566
567 #[test]
571 fn component_scoped_header_is_not_page_chrome() {
572 for cls in [
573 "wikitable sortable sticky-header-multi static-row-numbers",
574 "sticky-header-multi",
575 "static-row-header",
576 "table-footer",
577 "column-header",
578 ] {
579 let html = page(&format!("<table class=\"{cls}\">{ROWS}</table>"));
580 assert!(table_rows(&html) >= 4, "class={cls} lost the table");
581 }
582 }
583
584 #[test]
586 fn page_scoped_header_is_still_chrome() {
587 for cls in ["site-header", "page-header", "global-header", "header", "site-footer"] {
588 let html = format!(
589 "<html><body><article><h1>T</h1><p>{}</p>\
590 <div class=\"{cls}\"><a href=\"/a\">Menu one</a><a href=\"/b\">Menu two</a></div>\
591 </article></body></html>",
592 "Ordinary prose, with commas, at length. ".repeat(6)
593 );
594 let art = extract(&html, Some("https://example.com/x")).expect("extract");
595 assert!(!art.text.contains("Menu one"), "class={cls} survived");
596 }
597 }
598
599 #[test]
602 fn giant_attribute_does_not_bury_a_table() {
603 let blob = "y".repeat(3000);
604 let table = format!(
605 "<table><tr><th>Country</th><th>GDP</th></tr>\
606 <tr><td data-mw='{{\"parts\":\"{blob}\"}}'>Korea</td><td>1,870,000</td></tr></table>"
607 );
608 assert!(table_rows(&page(&table)) >= 3, "attribute payload sank the table");
609 }
610
611 #[test]
614 fn link_heavy_cells_survive() {
615 let table = "<table>\
616 <tr><th><a href=\"/imf\">IMF</a></th><th><a href=\"/wb\">World Bank</a></th></tr>\
617 <tr><td><a href=\"/kr\">Korea</a></td><td>1,870,000</td></tr></table>";
618 assert!(table_rows(&page(table)) >= 3, "link density dropped the table");
619 }
620
621 #[test]
625 fn large_data_table_is_still_tabulated() {
626 let mut t = String::from("<table><tr><th>Country</th><th>GDP</th></tr>");
627 for i in 0..400 {
628 t.push_str(&format!(
629 "<tr><td><a href=\"/c{i}\" title=\"country {i}\">Country {i}</a></td>\
630 <td>{i},000</td></tr>"
631 ));
632 }
633 t.push_str("</table>");
634 assert!(table_rows(&page(&t)) > 300, "large table was not tabulated");
635 }
636}
637
638#[cfg(test)]
639mod root_fallback {
640 use super::*;
641
642 fn sectioned() -> String {
649 let mut s = String::from("<html><body><div class=\"mw-parser-output\">");
650 for i in 0..12 {
651 s.push_str(&format!(
652 "<section><h2>Section {i}</h2><p>This section argues its point at \
653 length, with commas, clauses and several sentences. It runs on \
654 for a while so that it reads as prose rather than a label. \
655 Marker{i} appears here, and the paragraph continues past it with \
656 further discussion, more commas, and a closing sentence.</p>\
657 <p>A second paragraph follows, also of a reasonable length, so \
658 that the section carries real weight when the scorer looks at \
659 it. It too has commas and sentences.</p></section>"
660 ));
661 }
662 s.push_str("<ol class=\"references\">");
663 for i in 0..40 {
664 s.push_str(&format!(
665 "<li>Author {i}, <a href=\"/r{i}\">a cited work</a>, somewhere.</li>"
666 ));
667 }
668 s.push_str("</ol></div></body></html>");
669 s
670 }
671
672 fn as_article(html: &str) -> Article {
673 let opts = ExtractOptions { index_mode: IndexMode::Never, ..ExtractOptions::new() };
674 extract_with(html, Some("https://example.com/x"), &opts).expect("extract")
675 }
676
677 #[test]
678 fn partial_root_loses_to_the_fuller_container() {
679 let art = as_article(§ioned());
680 for i in 0..6 {
681 assert!(
682 art.text.contains(&format!("Marker{i}")),
683 "section {i} missing; extraction stopped early:\n{}",
684 &art.markdown[..art.markdown.len().min(400)]
685 );
686 }
687 }
688
689 #[test]
692 fn the_fuller_container_still_gets_denoised() {
693 let art = as_article(§ioned());
694 assert!(!art.text.contains("a cited work"), "reference list survived");
695 }
696}