Skip to main content

rustai_core/parse/
index.rs

1//! Index-page extraction: when the link list *is* the content.
2//!
3//! A front page, a feed, a blog archive and a forum listing all defeat article
4//! extraction for the same reason — the link density that marks navigation as
5//! boilerplate is, on these pages, the point. Running the article heuristics
6//! over one correctly yields nothing.
7//!
8//! What such a page is actually worth is an inventory: which stories exist and
9//! where they live, so the pipeline can go read them. That is what this module
10//! produces, and it is why an index page doubles as a collection source — see
11//! [`crate::search::Provider::Index`].
12//!
13//! Detection only runs after article extraction has already come up empty, so
14//! ordinary pages never take this path.
15
16use url::Url;
17
18use crate::denoise::{DenoiseConfig, is_noise};
19use crate::parse::dom::{BLOCK_TAGS, Doc, Id};
20use crate::search::canonical_key;
21use crate::text;
22
23/// Minimum links before a page is worth calling an index.
24const MIN_INDEX_LINKS: usize = 5;
25
26/// A link must be at least this share of its block to be an entry title.
27///
28/// An entry headline dominates its container; a link inside a sentence is a
29/// few percent of it. That gap is what separates an inventory from prose, and
30/// it holds regardless of whether article extraction happened to succeed.
31const TITULAR_PERCENT: u32 = 35;
32
33/// Share of the page's prose that titular links must account for.
34const INDEX_COVERAGE_PERCENT: usize = 25;
35
36/// Absolute floor for anchor text. Below this it is a control, not a title.
37const MIN_HEADLINE_CHARS: usize = 8;
38
39/// A single-word anchor has to be this long to count, since one word is far
40/// more likely to be a control ("Subscribe", "Comments") than a headline.
41const MIN_SINGLE_WORD_CHARS: usize = 12;
42
43/// Ceiling on anchor text. No headline runs this long; an anchor that does has
44/// swallowed its siblings because the parser never closed it, and its text is
45/// not to be trusted.
46const MAX_HEADLINE_CHARS: usize = 200;
47
48/// Descriptive text kept alongside a link.
49const MAX_SNIPPET_CHARS: usize = 200;
50
51/// How far up to look for a boilerplate container. Deep enough to clear any
52/// realistic nesting, bounded so a pathological tree cannot stall the walk.
53const MAX_ANCESTOR_WALK: usize = 32;
54
55/// Anchor text that is navigation whatever the page is.
56const CONTROL_WORDS: &[&str] = &[
57    "home",
58    "about",
59    "contact",
60    "login",
61    "log in",
62    "sign in",
63    "sign up",
64    "register",
65    "search",
66    "menu",
67    "more",
68    "read more",
69    "next",
70    "previous",
71    "prev",
72    "back",
73    "top",
74    "share",
75    "subscribe",
76    "newsletter",
77    "privacy",
78    "terms",
79    "cookies",
80    "settings",
81    "help",
82    "support",
83    "faq",
84    "rss",
85    "skip to content",
86    "advertisement",
87    "댓글",
88    "로그인",
89    "회원가입",
90    "더보기",
91    "구독",
92];
93
94/// One harvested link.
95#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
96pub struct Link {
97    /// Anchor text.
98    pub text: String,
99    /// Absolute URL.
100    pub url: String,
101    /// Nearby descriptive text, if the page offers any.
102    pub snippet: String,
103    /// Enclosing headings, outermost first.
104    pub heading_path: Vec<String>,
105}
106
107/// Collect the outbound links a listing page exists to offer.
108///
109/// Links inside boilerplate, duplicate targets and navigation controls are all
110/// dropped, so what comes back is the page's actual inventory rather than every
111/// `<a>` in the document.
112/// Nodes sitting inside a table that has header cells.
113///
114/// A `<td>` holding one linked country name is a value, not an entry in a
115/// listing — but the same shape is how Hacker News lays out its front page,
116/// so "inside a table" cannot be the test. Header cells are the discriminator
117/// that survives both: a table of data labels its columns, a table used for
118/// layout has nothing to label.
119fn inside_header_table(doc: &Doc<'_>) -> Vec<bool> {
120    let mut mark = vec![false; doc.dom.nodes().len()];
121    for &id in &doc.preorder {
122        if doc.tag_name(id) != "table" || mark[id] {
123            continue;
124        }
125        let descendants = doc.descendants(id);
126        if descendants.iter().any(|&d| doc.tag_name(d) == "th") {
127            for d in descendants {
128                if let Some(slot) = mark.get_mut(d) {
129                    *slot = true;
130                }
131            }
132        }
133    }
134    mark
135}
136
137pub(crate) fn harvest(doc: &Doc<'_>, base: Option<&Url>, cfg: &DenoiseConfig) -> Vec<Link> {
138    // The article denoiser drops containers for being link-dense and
139    // markup-heavy. On a listing page those are not defects — they are the
140    // format. Structural and vocabulary rules still apply, so links inside a
141    // `<nav>`, a footer or a cookie banner are still rejected.
142    let cfg = &DenoiseConfig { max_link_density: 1.0, min_text_ratio: 0.0, ..cfg.clone() };
143
144    let in_data_table = inside_header_table(doc);
145    let mut links: Vec<Link> = Vec::new();
146    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
147    // Heading ids are kept so a link can exclude the heading it sits inside:
148    // a headline is not its own section.
149    let mut headings: Vec<(u8, String, Id)> = Vec::new();
150    let self_key = base.and_then(|b| canonical_key(b.as_str()));
151
152    for &id in &doc.preorder {
153        let name = doc.tag_name(id);
154
155        if let Some(level) = heading_level(name) {
156            let title = text::normalize_ws(&doc.inner_text(id));
157            if !title.is_empty() {
158                while headings.last().is_some_and(|(l, _, _)| *l >= level) {
159                    headings.pop();
160                }
161                headings.push((level, title, id));
162            }
163            continue;
164        }
165        if name != "a" {
166            continue;
167        }
168        if in_data_table[id] {
169            continue;
170        }
171
172        let Some(text_raw) = Some(text::normalize_ws(&doc.inner_text(id))) else { continue };
173        if !is_headline(&text_raw) {
174            continue;
175        }
176        let Some(url) = resolve(doc, id, base) else { continue };
177        let Some(key) = canonical_key(&url) else { continue };
178        // A link back to the page itself carries no new information.
179        if self_key.as_ref() == Some(&key) || !seen.insert(key) {
180            continue;
181        }
182        if in_boilerplate(doc, id, cfg) {
183            continue;
184        }
185        if !is_titular(doc, id) {
186            continue;
187        }
188
189        links.push(Link {
190            snippet: snippet_for(doc, id, &text_raw),
191            text: text_raw,
192            url,
193            heading_path: headings
194                .iter()
195                .filter(|(_, _, hid)| !is_descendant_of(doc, id, *hid))
196                .map(|(_, t, _)| t.clone())
197                .collect(),
198        });
199    }
200    links
201}
202
203/// Does this page look like a listing rather than an article?
204///
205/// The test is coverage, not absence: how much of the prose the article path
206/// found is attached to a link. On a listing every headline and standfirst
207/// belongs to an entry, so coverage approaches one; on an article with a
208/// "related reading" rail at the end it is a rounding error. Asking instead
209/// whether article extraction *failed* would miss every listing page whose
210/// cards carry a summary — which is most of them.
211pub(crate) fn looks_like_index(links: &[Link], article_chars: usize) -> bool {
212    if links.len() < MIN_INDEX_LINKS {
213        return false;
214    }
215    if article_chars == 0 {
216        return true;
217    }
218    // Titles only. Snippets are lifted from the same prose the article path
219    // measured, so counting them would compare the page against itself.
220    let inventory: usize = links.iter().map(|l| text::visible_len(&l.text)).sum();
221    inventory * 100 > article_chars * INDEX_COVERAGE_PERCENT
222}
223
224/// Anchor text long enough, and specific enough, to be a headline.
225fn is_headline(text: &str) -> bool {
226    let trimmed = text.trim();
227    let chars = text::visible_len(trimmed);
228    if !(MIN_HEADLINE_CHARS..=MAX_HEADLINE_CHARS).contains(&chars) {
229        return false;
230    }
231    // Aggregator titles are short — "GPT-6 Astra" is eleven characters — so the
232    // length bar drops as soon as there is more than one word to look at.
233    if chars < MIN_SINGLE_WORD_CHARS && trimmed.split_whitespace().count() < 2 {
234        return false;
235    }
236    if !trimmed.chars().any(char::is_alphabetic) {
237        return false;
238    }
239    let lower = trimmed.to_lowercase();
240    if CONTROL_WORDS.iter().any(|w| lower == *w) {
241        return false;
242    }
243    !is_entry_metadata(trimmed)
244}
245
246/// Timestamps, comment counts, scores and bare domains.
247///
248/// Aggregators surround every headline with these, and they clear a length
249/// threshold easily — "1501 comments" is longer than plenty of real titles.
250fn is_entry_metadata(text: &str) -> bool {
251    let lower = text.trim().to_lowercase();
252
253    // A bare domain or filename: one token, has a dot, no spaces.
254    if !lower.contains(' ') && lower.contains('.') {
255        return true;
256    }
257
258    // The pattern has to describe the *whole* string. "13 hours ago" is
259    // metadata; "10 years of Rust in production" is a headline that happens to
260    // start the same way, and only length tells them apart.
261    let words: Vec<&str> = lower.split_whitespace().collect();
262    if words.len() > 3 {
263        return false;
264    }
265    let first = words.first().copied().unwrap_or_default();
266    let leads_with_number = first
267        .trim_matches(|c: char| !c.is_alphanumeric())
268        .chars()
269        .all(|c| c.is_ascii_digit() || c == ',' || c == '.')
270        && first.chars().any(|c| c.is_ascii_digit());
271    if !leads_with_number {
272        return false;
273    }
274
275    // "1501 comments", "342 points", "13 hours ago", "3분 전"
276    const UNITS: &[&str] = &[
277        "comment", "comments", "point", "points", "vote", "votes", "reply", "replies", "view",
278        "views", "second", "seconds", "minute", "minutes", "hour", "hours", "day", "days", "week",
279        "weeks", "month", "months", "year", "years", "ago", "분", "시간", "일", "개",
280    ];
281    words[1..].iter().any(|w| UNITS.contains(&w.trim_matches(|c: char| !c.is_alphanumeric())))
282}
283
284/// Is this link an entry title rather than a link inside a sentence?
285fn is_titular(doc: &Doc<'_>, id: Id) -> bool {
286    // A link that *wraps* a heading is titular by any reading of the word.
287    // Modern listings make the whole card the anchor -- `<a><h2>Title</h2>
288    // <span>Read More</span></a>` -- which is the same relationship as the
289    // classic `<h2><a>Title</a></h2>` with the nesting inverted. Checking only
290    // upwards misses it, and misses it silently: the page harvests to nothing.
291    if doc
292        .descendants(id)
293        .iter()
294        .skip(1)
295        .any(|&d| matches!(doc.tag_name(d), "h1" | "h2" | "h3" | "h4" | "h5" | "h6"))
296    {
297        return true;
298    }
299    let mut cur = id;
300    while let Some(parent) = doc.parent(cur) {
301        let name = doc.tag_name(parent);
302        if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
303            return true;
304        }
305        if BLOCK_TAGS.contains(&name) {
306            let block = doc.text_len(parent);
307            return block == 0 || doc.text_len(id) * 100 >= block * TITULAR_PERCENT;
308        }
309        cur = parent;
310    }
311    true
312}
313
314/// Is `id` inside `ancestor`?
315fn is_descendant_of(doc: &Doc<'_>, mut id: Id, ancestor: Id) -> bool {
316    while let Some(p) = doc.parent(id) {
317        if p == ancestor {
318            return true;
319        }
320        id = p;
321    }
322    false
323}
324
325/// Is this link inside something the denoiser would drop?
326///
327/// The walk goes all the way up, not a few levels: a documentation sidebar
328/// nests its `<nav>` half a dozen lists deep, and stopping early harvests the
329/// page's own table of contents as if it were an inventory.
330fn in_boilerplate(doc: &Doc<'_>, id: Id, cfg: &DenoiseConfig) -> bool {
331    doc.ancestors(id, MAX_ANCESTOR_WALK).into_iter().any(|a| is_noise(doc, a, cfg))
332}
333
334/// Descriptive text sitting next to the link, if the page provides any.
335///
336/// Listing pages wrap each entry in a card; the card's text minus the headline
337/// is the standfirst. It is what makes these links rankable against a query.
338fn snippet_for(doc: &Doc<'_>, link: Id, anchor_text: &str) -> String {
339    // The nearest ancestor that adds text beyond the headline is the card. An
340    // ancestor holding many times the headline is a whole page section, and
341    // borrowing its text would attach every other story's standfirst to this one.
342    let anchor_len = doc.text_len(link);
343    let card = doc.ancestors(link, 4).into_iter().find(|&a| {
344        let len = doc.text_len(a);
345        len > anchor_len + 20 && len < anchor_len.saturating_mul(10).max(anchor_len + 600)
346    });
347    let Some(card) = card else { return String::new() };
348
349    let full = doc.inner_text(card);
350    let rest = full.replace(anchor_text, " ");
351    let rest = text::normalize_ws(&rest);
352    if text::visible_len(&rest) < 20 {
353        return String::new();
354    }
355    rest.chars().take(MAX_SNIPPET_CHARS).collect()
356}
357
358fn resolve(doc: &Doc<'_>, id: Id, base: Option<&Url>) -> Option<String> {
359    let href = doc.attr(id, "href")?;
360    let href = href.trim();
361    if href.is_empty() || href.starts_with('#') || href.starts_with("javascript:") {
362        return None;
363    }
364    let absolute = match base {
365        Some(base) => base.join(href).ok()?,
366        None => Url::parse(href).ok()?,
367    };
368    matches!(absolute.scheme(), "http" | "https").then(|| absolute.to_string())
369}
370
371fn heading_level(name: &str) -> Option<u8> {
372    ["h1", "h2", "h3", "h4", "h5", "h6"].iter().position(|h| *h == name).map(|i| i as u8 + 1)
373}
374
375/// Render harvested links as rankable Markdown units.
376///
377/// One unit per link, with the section heading emitted whenever it changes.
378/// Keeping the snippet in the unit text is what lets the slimmer rank a listing
379/// against a query rather than just truncating it.
380pub(crate) fn units_from_links(links: &[Link]) -> Vec<crate::parse::Unit> {
381    use crate::parse::{Unit, UnitKind};
382
383    let mut units = Vec::with_capacity(links.len() + 4);
384    let mut last_path: Vec<String> = Vec::new();
385
386    for link in links {
387        if link.heading_path != last_path
388            && let Some(section) = link.heading_path.last()
389        {
390            let markdown = format!("## {section}");
391            units.push(Unit {
392                kind: UnitKind::Heading,
393                level: 2,
394                tokens: text::estimate_tokens(&markdown),
395                position: units.len(),
396                text: section.clone(),
397                markdown,
398                heading_path: link
399                    .heading_path
400                    .iter()
401                    .take(link.heading_path.len().saturating_sub(1))
402                    .cloned()
403                    .collect(),
404            });
405            last_path = link.heading_path.clone();
406        }
407
408        let markdown = if link.snippet.is_empty() {
409            format!("- [{}]({})", link.text, link.url)
410        } else {
411            format!("- [{}]({}) — {}", link.text, link.url, link.snippet)
412        };
413        let plain = if link.snippet.is_empty() {
414            link.text.clone()
415        } else {
416            format!("{} — {}", link.text, link.snippet)
417        };
418        units.push(Unit {
419            kind: UnitKind::ListItem,
420            level: 0,
421            tokens: text::estimate_tokens(&markdown),
422            position: units.len(),
423            text: plain,
424            markdown,
425            heading_path: link.heading_path.clone(),
426        });
427    }
428    units
429}
430
431/// Could this page possibly be an index? A cheap upper bound.
432///
433/// The real harvest resolves URLs, decodes text and gathers snippets — worth
434/// paying for on a listing, pure waste on the article pages that are the
435/// common case. This pass touches only precomputed lengths and the tag name,
436/// and it over-estimates, so a page it rejects could not have qualified.
437pub(crate) fn could_be_index(doc: &Doc<'_>, article_chars: usize) -> bool {
438    let in_data_table = inside_header_table(doc);
439    let mut count = 0usize;
440    let mut chars = 0usize;
441    for &id in &doc.preorder {
442        if doc.tag_name(id) != "a" || in_data_table[id] {
443            continue;
444        }
445        let len = doc.text_len(id) as usize;
446        if !(MIN_HEADLINE_CHARS..=MAX_HEADLINE_CHARS).contains(&len) {
447            continue;
448        }
449        if !is_titular(doc, id) {
450            continue;
451        }
452        count += 1;
453        chars += len;
454    }
455    count >= MIN_INDEX_LINKS
456        && (article_chars == 0 || chars * 100 > article_chars * INDEX_COVERAGE_PERCENT)
457}
458
459#[cfg(test)]
460mod tests {
461    use super::*;
462
463    const LISTING: &str = r##"<html><body>
464      <header><nav><a href="/">Home</a><a href="/about">About us</a></nav></header>
465      <main>
466        <h2>Top stories</h2>
467        <div class="card"><h3><a href="/story/rust-memory">Rust cuts crawler memory by an order of magnitude</a></h3>
468          <p>A new pipeline holds under five megabytes while parsing six hundred documents a second.</p></div>
469        <div class="card"><h3><a href="/story/http3">What actually changed in HTTP/3</a></h3>
470          <p>The transport moved to UDP, and head-of-line blocking went with it.</p></div>
471        <div class="card"><h3><a href="/story/bm25">BM25 is still the baseline to beat</a></h3>
472          <p>Twenty years on, the ranking function remains hard to improve upon.</p></div>
473        <h2>Opinion</h2>
474        <div class="card"><h3><a href="/opinion/agents">Agents are not a product category</a></h3>
475          <p>A short argument about naming things.</p></div>
476        <div class="card"><h3><a href="/opinion/slm">The case for small local models</a></h3>
477          <p>Latency, privacy and cost all point the same way.</p></div>
478        <a href="/page/2">Next</a>
479        <a href="/story/rust-memory?utm_source=footer">Rust cuts crawler memory by an order of magnitude</a>
480      </main>
481      <footer><a href="/privacy">Privacy policy</a><a href="/terms">Terms of service</a></footer>
482    </body></html>"##;
483
484    fn harvested() -> Vec<Link> {
485        let doc = Doc::parse(LISTING).unwrap();
486        let base = Url::parse("https://wire.example/").unwrap();
487        harvest(&doc, Some(&base), &DenoiseConfig::default())
488    }
489
490    #[test]
491    fn collects_the_stories_and_nothing_else() {
492        let links = harvested();
493        let urls: Vec<&str> = links.iter().map(|l| l.url.as_str()).collect();
494        assert_eq!(links.len(), 5, "{urls:#?}");
495        assert!(urls.contains(&"https://wire.example/story/bm25"));
496        // Navigation, pagination and footer controls are not inventory.
497        assert!(!urls.iter().any(|u| u.contains("/privacy")));
498        assert!(!urls.iter().any(|u| u.contains("/about")));
499        assert!(!urls.iter().any(|u| u.contains("/page/2")));
500    }
501
502    #[test]
503    fn duplicate_targets_collapse_across_decorations() {
504        let links = harvested();
505        let dupes = links.iter().filter(|l| l.url.contains("rust-memory")).count();
506        assert_eq!(dupes, 1, "the utm-tagged repeat survived");
507    }
508
509    #[test]
510    fn section_headings_become_the_breadcrumb() {
511        let links = harvested();
512        let opinion: Vec<&Link> = links.iter().filter(|l| l.url.contains("/opinion/")).collect();
513        assert_eq!(opinion.len(), 2);
514        assert!(opinion.iter().all(|l| l.heading_path == ["Opinion"]), "{opinion:#?}");
515        let top = links.iter().find(|l| l.url.contains("/story/bm25")).unwrap();
516        assert_eq!(top.heading_path, ["Top stories"]);
517    }
518
519    #[test]
520    fn standfirsts_are_kept_as_snippets() {
521        let links = harvested();
522        let http3 = links.iter().find(|l| l.url.contains("http3")).unwrap();
523        assert!(http3.snippet.contains("head-of-line blocking"), "{:?}", http3.snippet);
524    }
525
526    #[test]
527    fn detection_measures_how_much_prose_belongs_to_links() {
528        let links = harvested();
529        // Nothing but links: unambiguous.
530        assert!(looks_like_index(&links, 0));
531        // A listing whose cards carry standfirsts: the headlines still account
532        // for a large share of the prose, so it reads as an index even though
533        // article extraction succeeded.
534        let titles: usize = links.iter().map(|l| text::visible_len(&l.text)).sum();
535        assert!(looks_like_index(&links, titles * 2));
536        // An article that merely ends with a reading list is still an article.
537        assert!(!looks_like_index(&links, titles * 20));
538        // And an inventory too small to be one.
539        assert!(!looks_like_index(&links[..2], 0));
540    }
541
542    #[test]
543    fn links_inside_sentences_are_not_inventory() {
544        let html = r#"<html><body><article>
545            <p>A paragraph of real prose that runs on for a while and happens to mention
546               <a href="/guide">the full guide to everything</a> somewhere in the middle of it,
547               plus <a href="/other">another long link title here</a> for good measure.</p>
548            <p>A second paragraph, similarly long, that also links to
549               <a href="/third">a third destination with a title</a> inside a sentence.</p>
550        </article></body></html>"#;
551        let doc = Doc::parse(html).unwrap();
552        let base = Url::parse("https://example.com/post").unwrap();
553        let links = harvest(&doc, Some(&base), &DenoiseConfig::default());
554        assert!(links.is_empty(), "prose links were harvested: {links:#?}");
555    }
556
557    #[test]
558    fn short_and_control_anchors_are_rejected() {
559        assert!(!is_headline("Next"));
560        assert!(!is_headline("Read more"));
561        assert!(!is_headline("2026"));
562        assert!(!is_headline("   "));
563        assert!(is_headline("Rust cuts crawler memory"));
564        assert!(is_headline("GPT-6 Astra"), "short aggregator titles must survive");
565        assert!(!is_headline("Subscribe"));
566        assert!(!is_headline("upvote"));
567        // An anchor that swallowed the rest of the menu is not a headline.
568        assert!(!is_headline(&"Some navigation label ".repeat(30)));
569        assert!(is_headline("작은 로컬 모델을 위한 근거"));
570    }
571
572    #[test]
573    fn aggregator_metadata_is_rejected() {
574        // All of these clear the length threshold and are not stories.
575        for junk in [
576            "1501 comments",
577            "342 points",
578            "13 hours ago",
579            "2 days ago",
580            "neil.fraser.name",
581            "docs.rs/tokio",
582        ] {
583            assert!(!is_headline(junk), "{junk:?} was taken for a headline");
584        }
585        // Headlines that merely begin with a number must survive.
586        for real in [
587            "3 things we learned about small models",
588            "1.76.0 is out, and it changes the defaults",
589            "10 years of Rust in production",
590        ] {
591            assert!(is_headline(real), "{real:?} was rejected");
592        }
593    }
594}
595
596#[cfg(test)]
597mod titular_tests {
598    use super::*;
599
600    /// The card-is-the-link pattern: `<a><h2>Title</h2><span>Read More</span></a>`.
601    /// It is the same relationship as `<h2><a>Title</a></h2>` with the nesting
602    /// inverted, and checking only upwards used to miss it -- silently, because
603    /// the page then harvested to nothing at all.
604    #[test]
605    fn an_anchor_wrapping_a_heading_is_titular() {
606        let html = "<html><body><div class=\"cards\">\
607            <div><a href=\"/a\"><h2>The first post about something</h2>\
608            <span>Read More</span></a></div>\
609            <div><a href=\"/b\"><h2>The second post about something</h2>\
610            <span>Read More</span></a></div></div></body></html>";
611        let doc = Doc::parse(html).unwrap();
612        let anchors: Vec<_> =
613            doc.preorder.iter().copied().filter(|&i| doc.tag_name(i) == "a").collect();
614        assert_eq!(anchors.len(), 2);
615        for a in anchors {
616            assert!(is_titular(&doc, a), "anchor wrapping a heading rejected");
617        }
618    }
619}