Skip to main content

rustai_core/parse/
mod.rs

1//! HTML → denoised Markdown units.
2//!
3//! [`extract`] is the whole parsing story: parse once with `tl`, index the
4//! arena, pick the content root, and write Markdown while dropping boilerplate.
5//! It never touches the network and never allocates a second copy of the
6//! document, so it is cheap enough to run over a whole result set in parallel
7//! via [`extract_many`].
8
9pub(crate) mod dom;
10mod index;
11mod markdown;
12mod meta;
13
14use rayon::prelude::*;
15use url::Url;
16
17/// How much more text the container must hold than the chosen root before
18/// the second walk is worth doing, as a percentage.
19///
20/// When root scoring worked, the root already covers nearly all of the
21/// container and there is nothing to win — so this gate is what keeps the
22/// comparison off the hot path. Measured over the same 34 pages, 120 keeps
23/// every bit of the 20% gain that comparing unconditionally gives, and costs
24/// nothing: 1,808 docs/s against 1,820 before, where comparing at 102 dropped
25/// throughput to 1,250.
26const COMPARE_GAP_PERCENT: u64 = 120;
27
28use crate::denoise::{DenoiseConfig, DenoiseStats};
29
30use crate::error::Result;
31use crate::text;
32
33pub use index::Link;
34pub use markdown::RenderOptions;
35pub use meta::Meta;
36
37/// Whether a page turned out to be an article or a listing.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
39#[serde(rename_all = "snake_case")]
40pub enum ArticleKind {
41    /// Prose: the usual case.
42    Article,
43    /// A front page, feed or archive, where the link list is the content.
44    Index,
45}
46
47/// When to fall back to index-page extraction.
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
49pub enum IndexMode {
50    /// Try it only when article extraction came up empty. The default.
51    #[default]
52    Auto,
53    /// Never; a page with no prose extracts to nothing.
54    Never,
55    /// Always harvest links, even from a page that did yield an article.
56    Always,
57}
58
59/// What kind of block a [`Unit`] came from.
60#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)]
61#[serde(rename_all = "snake_case")]
62pub enum UnitKind {
63    /// `<h1>` … `<h6>`.
64    Heading,
65    /// A paragraph or other prose block.
66    Paragraph,
67    /// One `<li>`.
68    ListItem,
69    /// A fenced code block.
70    Code,
71    /// A `<blockquote>`.
72    Quote,
73    /// A Markdown pipe table.
74    Table,
75}
76
77impl UnitKind {
78    /// Weight applied when ranking, before relevance.
79    ///
80    /// Headings are short but carry the document's structure, so they punch
81    /// above their token count; boilerplate-prone list items punch below.
82    pub(crate) fn prior(self) -> f32 {
83        match self {
84            UnitKind::Heading => 1.25,
85            UnitKind::Paragraph => 1.0,
86            UnitKind::Code => 0.95,
87            UnitKind::Table => 0.9,
88            UnitKind::Quote => 0.85,
89            UnitKind::ListItem => 0.8,
90        }
91    }
92}
93
94/// One addressable block of an article.
95///
96/// This is the unit the context slimmer ranks, selects and emits, so it carries
97/// both its rendered Markdown and the plain text used for scoring.
98#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
99pub struct Unit {
100    /// Block type.
101    pub kind: UnitKind,
102    /// Heading level (1–6), or list nesting depth.
103    pub level: u8,
104    /// Plain text, with Markdown syntax removed.
105    pub text: String,
106    /// Rendered Markdown for this block.
107    pub markdown: String,
108    /// Enclosing headings, outermost first — the breadcrumb a model needs to
109    /// make sense of a block lifted out of its document.
110    pub heading_path: Vec<String>,
111    /// Index in document order.
112    pub position: usize,
113    /// Estimated LLM tokens for [`Unit::markdown`].
114    pub tokens: usize,
115}
116
117/// A cleaned document.
118#[derive(Debug, Clone, serde::Serialize)]
119pub struct Article {
120    /// Source URL, if one was supplied.
121    pub url: Option<String>,
122    /// Whether this page read as prose or as a listing.
123    pub kind: ArticleKind,
124    /// Metadata read from `<head>`.
125    pub meta: Meta,
126    /// The full cleaned document as Markdown.
127    pub markdown: String,
128    /// The full cleaned document as plain text.
129    pub text: String,
130    /// The Markdown split into rankable blocks.
131    pub units: Vec<Unit>,
132    /// Links harvested from a listing page. Empty for ordinary articles unless
133    /// [`IndexMode::Always`] was requested.
134    pub links: Vec<Link>,
135    /// What the denoiser dropped.
136    pub stats: DenoiseStats,
137}
138
139impl Article {
140    /// Estimated LLM tokens for the whole article.
141    pub fn tokens(&self) -> usize {
142        self.units.iter().map(|u| u.tokens).sum()
143    }
144
145    /// Title, falling back to the first heading.
146    pub fn title(&self) -> Option<&str> {
147        self.meta
148            .title
149            .as_deref()
150            .or_else(|| {
151                self.units.iter().find(|u| u.kind == UnitKind::Heading).map(|u| u.text.as_str())
152            })
153            .filter(|t| !t.is_empty())
154    }
155}
156
157/// Knobs for [`extract`].
158#[derive(Debug, Clone)]
159pub struct ExtractOptions {
160    /// Denoiser tuning.
161    pub denoise: DenoiseConfig,
162    /// When to fall back to harvesting links instead of prose.
163    pub index_mode: IndexMode,
164    /// Markdown rendering tuning.
165    pub render: RenderOptions,
166    /// If the chosen content root yields fewer than this many characters, redo
167    /// the extraction from `<body>`. Guards against a scoring miss on pages
168    /// with unusual markup.
169    pub min_content_chars: usize,
170}
171
172impl ExtractOptions {
173    /// Defaults tuned for article-shaped pages.
174    pub fn new() -> Self {
175        ExtractOptions {
176            denoise: DenoiseConfig::default(),
177            index_mode: IndexMode::default(),
178            render: RenderOptions::default(),
179            min_content_chars: 200,
180        }
181    }
182}
183
184/// Turn a raw HTML document into denoised Markdown units.
185///
186/// `url` is used to absolutise relative links and is echoed back on the
187/// [`Article`]; pass `None` for standalone fragments.
188pub fn extract(html: &str, url: Option<&str>) -> Result<Article> {
189    extract_with(html, url, &ExtractOptions::new())
190}
191
192/// [`extract`] with explicit options.
193pub fn extract_with(html: &str, url: Option<&str>, opts: &ExtractOptions) -> Result<Article> {
194    // Two things `tl` gets wrong badly enough to lose the document, both of
195    // which have to be repaired before parsing. Script and style bodies are
196    // not treated as raw text, so a `<` in a comparison opens a phantom
197    // element; and a `<` followed by punctuation opens one too, where HTML
198    // says it is a bogus comment or not markup at all. Each pass borrows when
199    // it finds nothing, so a clean document is never copied.
200    let unraw = dom::neutralise_raw_text(html);
201    let repaired = dom::neutralise_bogus_markup(&unraw);
202    let html = repaired.as_ref();
203    let doc = dom::Doc::parse(html)?;
204    let meta = meta::extract(&doc);
205
206    let base = url
207        .and_then(|u| Url::parse(u).ok())
208        .or_else(|| meta.canonical.as_deref().and_then(|c| Url::parse(c).ok()));
209
210    // The container to fall back to when root scoring does badly. Normally
211    // `<body>`, but a page whose markup strands its text outside `<body>` --
212    // an unclosed tag earlier in the document -- needs the parse root instead,
213    // or the fallback has nothing to offer.
214    let body = {
215        let doc_root = doc.preorder.first().copied();
216        let body_tag = doc.preorder.iter().copied().find(|&id| doc.tag_name(id) == "body");
217        match (body_tag, doc_root) {
218            (Some(b), Some(r)) if doc.text_len(r) > doc.text_len(b).saturating_mul(2) => Some(r),
219            (Some(b), _) => Some(b),
220            (None, r) => r,
221        }
222    };
223
224    let root = crate::denoise::find_content_root(&doc, &opts.denoise).or(body);
225    let Some(root) = root else {
226        return Ok(Article {
227            url: url.map(str::to_string),
228            kind: ArticleKind::Article,
229            meta,
230            markdown: String::new(),
231            text: String::new(),
232            units: Vec::new(),
233            links: Vec::new(),
234            stats: DenoiseStats { html_bytes: html.len(), ..Default::default() },
235        });
236    };
237
238    let mut writer = markdown::Writer::new(&doc, &opts.denoise, &opts.render, base.clone());
239    writer.walk_root(root);
240
241    // Content-root scoring can land in the wrong subtree — on a sidebar, on a
242    // reference list, or inside a maintenance banner that a lenient parser
243    // nested the whole article into. Rather than guess from thresholds whether
244    // that happened, extract the container too and keep whichever yielded more.
245    //
246    // The old test asked whether the root was *starved* — under five percent
247    // of the document's text. That catches a root that collapsed to nothing
248    // and misses every root that merely stopped early, which is the common
249    // failure: an encyclopaedia article split into sibling `<section>`s gives
250    // a root holding its lead and nothing else, comfortably over the
251    // threshold and comfortably wrong. Both walks run the same denoiser, so
252    // the larger result is not the noisier one — measured over 34 pages this
253    // gained 20% more text with nothing regressing.
254    let extracted = |w: &markdown::Writer<'_, '_>| -> usize {
255        w.units.iter().map(|u| text::visible_len(&u.text)).sum()
256    };
257    let current_len = extracted(&writer);
258
259    // Nothing to gain when the root already covers the container's text, and
260    // this is the common case — skipping it keeps the second walk off the hot
261    // path for pages where root scoring did its job.
262    let worth_comparing = body.is_some_and(|b| {
263        b != root && doc.text_len(b) as u64 * 100 > doc.text_len(root) as u64 * COMPARE_GAP_PERCENT
264    });
265
266    if (current_len < opts.min_content_chars || worth_comparing)
267        && let Some(body) = body
268        && body != root
269    {
270        let mut fallback = markdown::Writer::new(&doc, &opts.denoise, &opts.render, base.clone());
271        fallback.walk_root(body);
272        let fallback_len = extracted(&fallback);
273        if fallback_len > current_len {
274            writer = fallback;
275        }
276    }
277
278    // A page with no prose may still be a listing, where the links are the
279    // content rather than the boilerplate. Harvesting only runs once article
280    // extraction has already failed, so ordinary pages never take this path.
281    let mut kind = ArticleKind::Article;
282    let mut links = Vec::new();
283    let article_chars = extracted(&writer);
284    if opts.index_mode != IndexMode::Never && index::could_be_index(&doc, article_chars) {
285        let harvested = index::harvest(&doc, base.as_ref(), &opts.denoise);
286        if index::looks_like_index(&harvested, article_chars) {
287            kind = ArticleKind::Index;
288            writer.units = index::units_from_links(&harvested);
289            links = harvested;
290        } else if opts.index_mode == IndexMode::Always {
291            links = harvested;
292        }
293    }
294
295    let mut units = writer.units;
296    for (i, u) in units.iter_mut().enumerate() {
297        u.position = i;
298    }
299    let markdown = units.iter().map(|u| u.markdown.as_str()).collect::<Vec<_>>().join("\n\n");
300    let text = units.iter().map(|u| u.text.as_str()).collect::<Vec<_>>().join("\n");
301
302    let mut stats = writer.stats;
303    // Everything outside the chosen content root was removed just as surely as
304    // a node the noise rules rejected, and it is usually the larger share.
305    let total_nodes = doc.preorder.len();
306    let outside_root = total_nodes.saturating_sub(doc.descendants(root).len());
307    stats.nodes_visited = total_nodes;
308    stats.nodes_dropped = (stats.nodes_dropped + outside_root).min(total_nodes);
309    stats.html_bytes = html.len();
310    stats.markdown_bytes = markdown.len();
311
312    Ok(Article { url: url.map(str::to_string), kind, meta, markdown, text, units, links, stats })
313}
314
315/// Extract many documents in parallel across the rayon pool.
316///
317/// Parsing is CPU-bound and per-document independent, which is exactly the
318/// shape rayon is for: a 20-result search collapses to roughly one document's
319/// worth of wall time on a multi-core machine.
320pub fn extract_many<'a, I>(docs: I, opts: &ExtractOptions) -> Vec<Result<Article>>
321where
322    I: IntoIterator<Item = (&'a str, Option<&'a str>)>,
323    I::IntoIter: Send,
324{
325    let docs: Vec<_> = docs.into_iter().collect();
326    docs.into_par_iter().map(|(html, url)| extract_with(html, url, opts)).collect()
327}
328
329#[cfg(test)]
330mod tests {
331    use super::*;
332
333    const PAGE: &str = r##"<html lang="en"><head><title>Zero-cost crawling</title></head><body>
334      <nav><a href="/a">A</a><a href="/b">B</a><a href="/c">C</a><a href="/d">D</a></nav>
335      <main><article class="post">
336        <h1>Zero-cost crawling</h1>
337        <p>Python crawlers pay for a DOM they never asked for, and it shows in RSS, in latency, and in bills.</p>
338        <h2>Why Rust</h2>
339        <p>An arena parser, a work-stealing pool, and no garbage collector at all: the memory ceiling is a design choice.</p>
340        <ul><li>Low memory</li><li>Fast parsing</li></ul>
341        <pre><code class="language-rust">let x = 1;</code></pre>
342        <table><tr><th>Tool</th><th>RSS</th></tr><tr><td>rustai</td><td>28 MB</td></tr></table>
343      </article></main>
344      <footer class="footer">© 2026 Example, all rights reserved, contact us</footer>
345    </body></html>"##;
346
347    #[test]
348    fn extracts_structure_and_drops_chrome() {
349        let art = extract(PAGE, Some("https://example.com/post")).unwrap();
350        assert_eq!(art.title(), Some("Zero-cost crawling"));
351        assert!(art.markdown.contains("# Zero-cost crawling"));
352        assert!(art.markdown.contains("## Why Rust"));
353        assert!(art.markdown.contains("- Low memory"));
354        assert!(art.markdown.contains("```rust\nlet x = 1;\n```"));
355        assert!(art.markdown.contains("| Tool | RSS |"));
356        assert!(!art.text.contains("all rights reserved"), "footer survived:\n{}", art.markdown);
357        assert!(!art.markdown.contains("](/a)"), "nav survived");
358    }
359
360    #[test]
361    fn units_carry_their_heading_path() {
362        let art = extract(PAGE, None).unwrap();
363        let para = art
364            .units
365            .iter()
366            .find(|u| u.text.starts_with("An arena parser"))
367            .expect("second paragraph");
368        assert_eq!(para.heading_path, ["Zero-cost crawling", "Why Rust"]);
369        assert!(para.tokens > 0);
370    }
371
372    #[test]
373    fn relative_links_are_absolutised() {
374        let html = r#"<article><p>Read the <a href="/docs/guide">guide</a> for the full walkthrough of the pipeline.</p></article>"#;
375        let art = extract(html, Some("https://example.com/blog/post")).unwrap();
376        assert!(art.markdown.contains("(https://example.com/docs/guide)"), "{}", art.markdown);
377    }
378
379    #[test]
380    fn compression_is_reported() {
381        let art = extract(PAGE, None).unwrap();
382        assert!(art.stats.compression() > 0.3, "only {}", art.stats.compression());
383        assert!(art.tokens() > 0);
384    }
385
386    #[test]
387    fn dropped_nodes_include_everything_outside_the_content_root() {
388        let art = extract(PAGE, None).unwrap();
389        // The nav and footer live outside the article and must still be counted.
390        assert!(art.stats.nodes_dropped > 0, "{:?}", art.stats);
391        assert!(art.stats.nodes_dropped < art.stats.nodes_visited);
392    }
393
394    #[test]
395    fn an_unclosed_paragraph_does_not_swallow_the_document() {
396        // HTML implies `</p>` before a block-level start tag. Lenient parsers
397        // do not insert it, so without a guard the whole page becomes one
398        // paragraph. This is the shape real pages hit it with.
399        let html = "<article><p>Intro sentence that runs on                    <h2>A real heading</h2>                    <p>A second paragraph with enough words in it to be kept.</p>                    <h2>Another heading</h2>                    <p>A third paragraph, also long enough to survive the filter.</p>                    </article>";
400        let art = extract(html, None).unwrap();
401        assert!(
402            art.units.len() >= 4,
403            "collapsed into {} units:\n{}",
404            art.units.len(),
405            art.markdown
406        );
407        assert!(art.markdown.contains("## A real heading"));
408        assert!(art.markdown.contains("## Another heading"));
409        let biggest = art.units.iter().map(|u| u.tokens).max().unwrap();
410        assert!(biggest < 60, "one unit swallowed the rest: {biggest} tokens");
411    }
412
413    #[test]
414    fn an_inline_element_holding_blocks_does_not_flatten_the_page() {
415        // The shape an unclosed `<span>` produces: an inline tag that has taken
416        // ownership of the rest of the document. Flattening it would emit the
417        // whole page as a single paragraph with no headings at all.
418        //
419        // The filler is sized past `INLINE_INSPECT_BYTES` on purpose. Below
420        // that the guard deliberately does not run: a sub-2 KB span that
421        // flattens is a cosmetic wrinkle, while the descendant walk on every
422        // small `<a>` and `<strong>` would be a real cost on every document.
423        let filler = "Real prose that carries the paragraph past the length filter. ".repeat(20);
424        let html = format!(
425            "<article><div class=\"notice\"><span>Notice text.\
426             <h2>First section</h2><p>{filler}</p>\
427             <h2>Second section</h2><p>{filler}</p></span></div></article>"
428        );
429        let art = extract(&html, None).unwrap();
430        assert!(art.markdown.contains("## First section"), "headings lost:\n{}", art.markdown);
431        assert!(art.markdown.contains("## Second section"));
432        // Two headings and two paragraphs, not one merged blob.
433        assert!(
434            art.units.len() >= 4,
435            "collapsed into {} units:\n{}",
436            art.units.len(),
437            art.markdown
438        );
439    }
440
441    #[test]
442    fn a_list_item_holding_blocks_keeps_their_structure() {
443        let prose = "Real prose inside a list item, long enough to survive the filter. ";
444        let html = format!(
445            "<article><ul><li>Bullet text\
446             <h2>A heading inside the item</h2><p>{prose}</p>\
447             <ul><li>Nested bullet</li></ul></li></ul></article>"
448        );
449        let art = extract(&html, None).unwrap();
450        assert!(art.markdown.contains("- Bullet text"));
451        assert!(art.markdown.contains("## A heading inside the item"), "{}", art.markdown);
452        assert!(art.markdown.contains("- Nested bullet"));
453        let biggest = art.units.iter().map(|u| u.tokens).max().unwrap();
454        assert!(biggest < 40, "the item swallowed its blocks: {biggest} tokens");
455    }
456
457    #[test]
458    fn small_inline_elements_stay_inline() {
459        let html = "<article><p>A sentence with <strong>bold</strong> and                     <em>italic</em> and <code>code</code> in it, long enough to keep.</p></article>";
460        let art = extract(html, None).unwrap();
461        assert_eq!(art.units.len(), 1, "inline runs were split:\n{}", art.markdown);
462        assert!(art.markdown.contains("**bold**"));
463        assert!(art.markdown.contains("*italic*"));
464        assert!(art.markdown.contains("`code`"));
465    }
466
467    #[test]
468    fn a_layout_table_is_walked_not_tabulated() {
469        // Navboxes, page shells and unclosed tables all look like this: block
470        // content inside cells. Rendering it as a pipe table would produce one
471        // giant unreadable row.
472        let html = "<table><tr><td>                    <h1>Page title</h1>                    <p>A paragraph of real prose that is long enough to keep around.</p>                    <h2>Section</h2>                    <p>Another paragraph of real prose, also long enough to keep.</p>                    </td></tr></table>";
473        let art = extract(html, None).unwrap();
474        assert!(!art.units.iter().any(|u| u.kind == UnitKind::Table), "{}", art.markdown);
475        assert!(art.markdown.contains("# Page title"));
476        assert!(art.markdown.contains("## Section"));
477    }
478
479    #[test]
480    fn a_real_data_table_is_still_tabulated() {
481        let html = "<article><p>Some prose introducing the numbers below it.</p>                    <table><tr><th>Engine</th><th>RSS</th></tr>                    <tr><td>arena</td><td>4.8 MiB</td></tr></table></article>";
482        let art = extract(html, None).unwrap();
483        assert!(art.markdown.contains("| Engine | RSS |"), "{}", art.markdown);
484        assert!(art.units.iter().any(|u| u.kind == UnitKind::Table));
485    }
486
487    #[test]
488    fn empty_input_is_not_an_error() {
489        let art = extract("", None).unwrap();
490        assert!(art.units.is_empty());
491    }
492
493    #[test]
494    fn many_documents_extract_in_parallel() {
495        let inputs: Vec<(&str, Option<&str>)> = (0..8).map(|_| (PAGE, None)).collect();
496        let out = extract_many(inputs, &ExtractOptions::new());
497        assert_eq!(out.len(), 8);
498        assert!(out.iter().all(|r| r.as_ref().is_ok_and(|a| !a.units.is_empty())));
499    }
500}
501
502#[cfg(test)]
503mod superscript_tests {
504    use super::*;
505
506    fn md(fragment: &str) -> String {
507        let html = format!("<article><p>{} {fragment}</p></article>", "Prose. ".repeat(12));
508        extract(&html, None).unwrap().markdown
509    }
510
511    #[test]
512    fn exponents_and_units_survive() {
513        assert!(md("6.02 × 10<sup>23</sup> mol<sup>-1</sup>").contains("10^23 mol^-1"));
514        assert!(md("area is 5 m<sup>2</sup>").contains("m^2"));
515    }
516
517    #[test]
518    fn ordinals_read_naturally() {
519        assert!(md("the 1<sup>st</sup> release").contains("1st release"));
520    }
521
522    #[test]
523    fn citation_markers_are_dropped() {
524        for marker in [
525            r#"<sup class="reference">[1]</sup>"#,
526            r##"<sup><a href="#cite_note-1">[12]</a></sup>"##,
527            "<sup>[3]</sup>",
528        ] {
529            let out = md(&format!("A claim{marker} follows."));
530            assert!(out.contains("A claim follows.") || out.contains("A claim  follows."), "{out}");
531            assert!(!out.contains('['), "marker survived: {out}");
532        }
533    }
534
535    #[test]
536    fn attribute_entities_are_decoded() {
537        let html = r#"<html><head><meta property="og:title" content="Steve Irwin&#x27;s family &amp; friends"></head>
538            <body><article><p>Body text long enough to keep around here.</p></article></body></html>"#;
539        let art = extract(html, None).unwrap();
540        assert_eq!(art.meta.title.as_deref(), Some("Steve Irwin's family & friends"));
541    }
542}
543
544#[cfg(test)]
545mod table_regressions {
546    use super::*;
547
548    /// Wrap a table in enough prose that it is not the whole document, so the
549    /// dominance rule cannot rescue it and the heuristics under test are the
550    /// ones that decide.
551    fn page(table: &str) -> String {
552        format!(
553            "<html><body><article><h1>Figures</h1><p>{}</p>{table}</article></body></html>",
554            "A sentence of ordinary prose, with commas, to anchor the page. ".repeat(6)
555        )
556    }
557
558    fn table_rows(html: &str) -> usize {
559        let art = extract(html, Some("https://example.com/x")).expect("extract");
560        art.markdown.lines().filter(|l| l.starts_with('|')).count()
561    }
562
563    const ROWS: &str = "<tr><th>Country</th><th>GDP</th></tr>\
564                        <tr><td>Korea</td><td>1,870,000</td></tr>\
565                        <tr><td>Japan</td><td>4,230,000</td></tr>";
566
567    /// `header` is a chrome token, but `sticky-header-multi` is a data table
568    /// describing its own headings — the class Wikipedia puts on every
569    /// sortable table.
570    #[test]
571    fn component_scoped_header_is_not_page_chrome() {
572        for cls in [
573            "wikitable sortable sticky-header-multi static-row-numbers",
574            "sticky-header-multi",
575            "static-row-header",
576            "table-footer",
577            "column-header",
578        ] {
579            let html = page(&format!("<table class=\"{cls}\">{ROWS}</table>"));
580            assert!(table_rows(&html) >= 4, "class={cls} lost the table");
581        }
582    }
583
584    /// The page-scoped compounds must still read as boilerplate.
585    #[test]
586    fn page_scoped_header_is_still_chrome() {
587        for cls in ["site-header", "page-header", "global-header", "header", "site-footer"] {
588            let html = format!(
589                "<html><body><article><h1>T</h1><p>{}</p>\
590                 <div class=\"{cls}\"><a href=\"/a\">Menu one</a><a href=\"/b\">Menu two</a></div>\
591                 </article></body></html>",
592                "Ordinary prose, with commas, at length. ".repeat(6)
593            );
594            let art = extract(&html, Some("https://example.com/x")).expect("extract");
595            assert!(!art.text.contains("Menu one"), "class={cls} survived");
596        }
597    }
598
599    /// One enormous attribute value must not read as "almost pure markup".
600    /// Wikipedia's parser hangs serialised JSON off every element.
601    #[test]
602    fn giant_attribute_does_not_bury_a_table() {
603        let blob = "y".repeat(3000);
604        let table = format!(
605            "<table><tr><th>Country</th><th>GDP</th></tr>\
606             <tr><td data-mw='{{\"parts\":\"{blob}\"}}'>Korea</td><td>1,870,000</td></tr></table>"
607        );
608        assert!(table_rows(&page(&table)) >= 3, "attribute payload sank the table");
609    }
610
611    /// A header row that cites each column's source is mostly links, and a
612    /// data cell holds one word. Neither makes a table a link farm.
613    #[test]
614    fn link_heavy_cells_survive() {
615        let table = "<table>\
616            <tr><th><a href=\"/imf\">IMF</a></th><th><a href=\"/wb\">World Bank</a></th></tr>\
617            <tr><td><a href=\"/kr\">Korea</a></td><td>1,870,000</td></tr></table>";
618        assert!(table_rows(&page(table)) >= 3, "link density dropped the table");
619    }
620
621    /// A real data table runs to thousands of nodes. Rejecting it does not
622    /// degrade the output, it deletes it — walked as blocks, cells are too
623    /// short to survive as paragraphs.
624    #[test]
625    fn large_data_table_is_still_tabulated() {
626        let mut t = String::from("<table><tr><th>Country</th><th>GDP</th></tr>");
627        for i in 0..400 {
628            t.push_str(&format!(
629                "<tr><td><a href=\"/c{i}\" title=\"country {i}\">Country {i}</a></td>\
630                 <td>{i},000</td></tr>"
631            ));
632        }
633        t.push_str("</table>");
634        assert!(table_rows(&page(&t)) > 300, "large table was not tabulated");
635    }
636}
637
638#[cfg(test)]
639mod root_fallback {
640    use super::*;
641
642    /// A document split into sibling sections under one wrapper, with a
643    /// reference list long enough to outscore any single section.
644    ///
645    /// The article stays the larger half deliberately: a reference list that
646    /// held most of a page's text would be protected by the dominance rule,
647    /// and rightly — that shape is a parse accident, not an encyclopaedia.
648    fn sectioned() -> String {
649        let mut s = String::from("<html><body><div class=\"mw-parser-output\">");
650        for i in 0..12 {
651            s.push_str(&format!(
652                "<section><h2>Section {i}</h2><p>This section argues its point at \
653                 length, with commas, clauses and several sentences. It runs on \
654                 for a while so that it reads as prose rather than a label. \
655                 Marker{i} appears here, and the paragraph continues past it with \
656                 further discussion, more commas, and a closing sentence.</p>\
657                 <p>A second paragraph follows, also of a reasonable length, so \
658                 that the section carries real weight when the scorer looks at \
659                 it. It too has commas and sentences.</p></section>"
660            ));
661        }
662        s.push_str("<ol class=\"references\">");
663        for i in 0..40 {
664            s.push_str(&format!(
665                "<li>Author {i}, <a href=\"/r{i}\">a cited work</a>, somewhere.</li>"
666            ));
667        }
668        s.push_str("</ol></div></body></html>");
669        s
670    }
671
672    fn as_article(html: &str) -> Article {
673        let opts = ExtractOptions { index_mode: IndexMode::Never, ..ExtractOptions::new() };
674        extract_with(html, Some("https://example.com/x"), &opts).expect("extract")
675    }
676
677    #[test]
678    fn partial_root_loses_to_the_fuller_container() {
679        let art = as_article(&sectioned());
680        for i in 0..6 {
681            assert!(
682                art.text.contains(&format!("Marker{i}")),
683                "section {i} missing; extraction stopped early:\n{}",
684                &art.markdown[..art.markdown.len().min(400)]
685            );
686        }
687    }
688
689    /// The comparison must not become an excuse to sweep in boilerplate: the
690    /// reference list is dropped either way.
691    #[test]
692    fn the_fuller_container_still_gets_denoised() {
693        let art = as_article(&sectioned());
694        assert!(!art.text.contains("a cited work"), "reference list survived");
695    }
696}