1use url::Url;
17
18use crate::denoise::{DenoiseConfig, is_noise};
19use crate::parse::dom::{BLOCK_TAGS, Doc, Id};
20use crate::search::canonical_key;
21use crate::text;
22
23const MIN_INDEX_LINKS: usize = 5;
25
26const TITULAR_PERCENT: u32 = 35;
32
33const INDEX_COVERAGE_PERCENT: usize = 25;
35
36const MIN_HEADLINE_CHARS: usize = 8;
38
39const MIN_SINGLE_WORD_CHARS: usize = 12;
42
43const MAX_HEADLINE_CHARS: usize = 200;
47
48const MAX_SNIPPET_CHARS: usize = 200;
50
51const MAX_ANCESTOR_WALK: usize = 32;
54
55const CONTROL_WORDS: &[&str] = &[
57 "home",
58 "about",
59 "contact",
60 "login",
61 "log in",
62 "sign in",
63 "sign up",
64 "register",
65 "search",
66 "menu",
67 "more",
68 "read more",
69 "next",
70 "previous",
71 "prev",
72 "back",
73 "top",
74 "share",
75 "subscribe",
76 "newsletter",
77 "privacy",
78 "terms",
79 "cookies",
80 "settings",
81 "help",
82 "support",
83 "faq",
84 "rss",
85 "skip to content",
86 "advertisement",
87 "댓글",
88 "로그인",
89 "회원가입",
90 "더보기",
91 "구독",
92];
93
94#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
96pub struct Link {
97 pub text: String,
99 pub url: String,
101 pub snippet: String,
103 pub heading_path: Vec<String>,
105}
106
107fn inside_header_table(doc: &Doc<'_>) -> Vec<bool> {
120 let mut mark = vec![false; doc.dom.nodes().len()];
121 for &id in &doc.preorder {
122 if doc.tag_name(id) != "table" || mark[id] {
123 continue;
124 }
125 let descendants = doc.descendants(id);
126 if descendants.iter().any(|&d| doc.tag_name(d) == "th") {
127 for d in descendants {
128 if let Some(slot) = mark.get_mut(d) {
129 *slot = true;
130 }
131 }
132 }
133 }
134 mark
135}
136
137pub(crate) fn harvest(doc: &Doc<'_>, base: Option<&Url>, cfg: &DenoiseConfig) -> Vec<Link> {
138 let cfg = &DenoiseConfig { max_link_density: 1.0, min_text_ratio: 0.0, ..cfg.clone() };
143
144 let in_data_table = inside_header_table(doc);
145 let mut links: Vec<Link> = Vec::new();
146 let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
147 let mut headings: Vec<(u8, String, Id)> = Vec::new();
150 let self_key = base.and_then(|b| canonical_key(b.as_str()));
151
152 for &id in &doc.preorder {
153 let name = doc.tag_name(id);
154
155 if let Some(level) = heading_level(name) {
156 let title = text::normalize_ws(&doc.inner_text(id));
157 if !title.is_empty() {
158 while headings.last().is_some_and(|(l, _, _)| *l >= level) {
159 headings.pop();
160 }
161 headings.push((level, title, id));
162 }
163 continue;
164 }
165 if name != "a" {
166 continue;
167 }
168 if in_data_table[id] {
169 continue;
170 }
171
172 let Some(text_raw) = Some(text::normalize_ws(&doc.inner_text(id))) else { continue };
173 if !is_headline(&text_raw) {
174 continue;
175 }
176 let Some(url) = resolve(doc, id, base) else { continue };
177 let Some(key) = canonical_key(&url) else { continue };
178 if self_key.as_ref() == Some(&key) || !seen.insert(key) {
180 continue;
181 }
182 if in_boilerplate(doc, id, cfg) {
183 continue;
184 }
185 if !is_titular(doc, id) {
186 continue;
187 }
188
189 links.push(Link {
190 snippet: snippet_for(doc, id, &text_raw),
191 text: text_raw,
192 url,
193 heading_path: headings
194 .iter()
195 .filter(|(_, _, hid)| !is_descendant_of(doc, id, *hid))
196 .map(|(_, t, _)| t.clone())
197 .collect(),
198 });
199 }
200 links
201}
202
203pub(crate) fn looks_like_index(links: &[Link], article_chars: usize) -> bool {
212 if links.len() < MIN_INDEX_LINKS {
213 return false;
214 }
215 if article_chars == 0 {
216 return true;
217 }
218 let inventory: usize = links.iter().map(|l| text::visible_len(&l.text)).sum();
221 inventory * 100 > article_chars * INDEX_COVERAGE_PERCENT
222}
223
224fn is_headline(text: &str) -> bool {
226 let trimmed = text.trim();
227 let chars = text::visible_len(trimmed);
228 if !(MIN_HEADLINE_CHARS..=MAX_HEADLINE_CHARS).contains(&chars) {
229 return false;
230 }
231 if chars < MIN_SINGLE_WORD_CHARS && trimmed.split_whitespace().count() < 2 {
234 return false;
235 }
236 if !trimmed.chars().any(char::is_alphabetic) {
237 return false;
238 }
239 let lower = trimmed.to_lowercase();
240 if CONTROL_WORDS.iter().any(|w| lower == *w) {
241 return false;
242 }
243 !is_entry_metadata(trimmed)
244}
245
246fn is_entry_metadata(text: &str) -> bool {
251 let lower = text.trim().to_lowercase();
252
253 if !lower.contains(' ') && lower.contains('.') {
255 return true;
256 }
257
258 let words: Vec<&str> = lower.split_whitespace().collect();
262 if words.len() > 3 {
263 return false;
264 }
265 let first = words.first().copied().unwrap_or_default();
266 let leads_with_number = first
267 .trim_matches(|c: char| !c.is_alphanumeric())
268 .chars()
269 .all(|c| c.is_ascii_digit() || c == ',' || c == '.')
270 && first.chars().any(|c| c.is_ascii_digit());
271 if !leads_with_number {
272 return false;
273 }
274
275 const UNITS: &[&str] = &[
277 "comment", "comments", "point", "points", "vote", "votes", "reply", "replies", "view",
278 "views", "second", "seconds", "minute", "minutes", "hour", "hours", "day", "days", "week",
279 "weeks", "month", "months", "year", "years", "ago", "분", "시간", "일", "개",
280 ];
281 words[1..].iter().any(|w| UNITS.contains(&w.trim_matches(|c: char| !c.is_alphanumeric())))
282}
283
284fn is_titular(doc: &Doc<'_>, id: Id) -> bool {
286 if doc
292 .descendants(id)
293 .iter()
294 .skip(1)
295 .any(|&d| matches!(doc.tag_name(d), "h1" | "h2" | "h3" | "h4" | "h5" | "h6"))
296 {
297 return true;
298 }
299 let mut cur = id;
300 while let Some(parent) = doc.parent(cur) {
301 let name = doc.tag_name(parent);
302 if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
303 return true;
304 }
305 if BLOCK_TAGS.contains(&name) {
306 let block = doc.text_len(parent);
307 return block == 0 || doc.text_len(id) * 100 >= block * TITULAR_PERCENT;
308 }
309 cur = parent;
310 }
311 true
312}
313
314fn is_descendant_of(doc: &Doc<'_>, mut id: Id, ancestor: Id) -> bool {
316 while let Some(p) = doc.parent(id) {
317 if p == ancestor {
318 return true;
319 }
320 id = p;
321 }
322 false
323}
324
325fn in_boilerplate(doc: &Doc<'_>, id: Id, cfg: &DenoiseConfig) -> bool {
331 doc.ancestors(id, MAX_ANCESTOR_WALK).into_iter().any(|a| is_noise(doc, a, cfg))
332}
333
334fn snippet_for(doc: &Doc<'_>, link: Id, anchor_text: &str) -> String {
339 let anchor_len = doc.text_len(link);
343 let card = doc.ancestors(link, 4).into_iter().find(|&a| {
344 let len = doc.text_len(a);
345 len > anchor_len + 20 && len < anchor_len.saturating_mul(10).max(anchor_len + 600)
346 });
347 let Some(card) = card else { return String::new() };
348
349 let full = doc.inner_text(card);
350 let rest = full.replace(anchor_text, " ");
351 let rest = text::normalize_ws(&rest);
352 if text::visible_len(&rest) < 20 {
353 return String::new();
354 }
355 rest.chars().take(MAX_SNIPPET_CHARS).collect()
356}
357
358fn resolve(doc: &Doc<'_>, id: Id, base: Option<&Url>) -> Option<String> {
359 let href = doc.attr(id, "href")?;
360 let href = href.trim();
361 if href.is_empty() || href.starts_with('#') || href.starts_with("javascript:") {
362 return None;
363 }
364 let absolute = match base {
365 Some(base) => base.join(href).ok()?,
366 None => Url::parse(href).ok()?,
367 };
368 matches!(absolute.scheme(), "http" | "https").then(|| absolute.to_string())
369}
370
371fn heading_level(name: &str) -> Option<u8> {
372 ["h1", "h2", "h3", "h4", "h5", "h6"].iter().position(|h| *h == name).map(|i| i as u8 + 1)
373}
374
375pub(crate) fn units_from_links(links: &[Link]) -> Vec<crate::parse::Unit> {
381 use crate::parse::{Unit, UnitKind};
382
383 let mut units = Vec::with_capacity(links.len() + 4);
384 let mut last_path: Vec<String> = Vec::new();
385
386 for link in links {
387 if link.heading_path != last_path
388 && let Some(section) = link.heading_path.last()
389 {
390 let markdown = format!("## {section}");
391 units.push(Unit {
392 kind: UnitKind::Heading,
393 level: 2,
394 tokens: text::estimate_tokens(&markdown),
395 position: units.len(),
396 text: section.clone(),
397 markdown,
398 heading_path: link
399 .heading_path
400 .iter()
401 .take(link.heading_path.len().saturating_sub(1))
402 .cloned()
403 .collect(),
404 });
405 last_path = link.heading_path.clone();
406 }
407
408 let markdown = if link.snippet.is_empty() {
409 format!("- [{}]({})", link.text, link.url)
410 } else {
411 format!("- [{}]({}) — {}", link.text, link.url, link.snippet)
412 };
413 let plain = if link.snippet.is_empty() {
414 link.text.clone()
415 } else {
416 format!("{} — {}", link.text, link.snippet)
417 };
418 units.push(Unit {
419 kind: UnitKind::ListItem,
420 level: 0,
421 tokens: text::estimate_tokens(&markdown),
422 position: units.len(),
423 text: plain,
424 markdown,
425 heading_path: link.heading_path.clone(),
426 });
427 }
428 units
429}
430
431pub(crate) fn could_be_index(doc: &Doc<'_>, article_chars: usize) -> bool {
438 let in_data_table = inside_header_table(doc);
439 let mut count = 0usize;
440 let mut chars = 0usize;
441 for &id in &doc.preorder {
442 if doc.tag_name(id) != "a" || in_data_table[id] {
443 continue;
444 }
445 let len = doc.text_len(id) as usize;
446 if !(MIN_HEADLINE_CHARS..=MAX_HEADLINE_CHARS).contains(&len) {
447 continue;
448 }
449 if !is_titular(doc, id) {
450 continue;
451 }
452 count += 1;
453 chars += len;
454 }
455 count >= MIN_INDEX_LINKS
456 && (article_chars == 0 || chars * 100 > article_chars * INDEX_COVERAGE_PERCENT)
457}
458
459#[cfg(test)]
460mod tests {
461 use super::*;
462
463 const LISTING: &str = r##"<html><body>
464 <header><nav><a href="/">Home</a><a href="/about">About us</a></nav></header>
465 <main>
466 <h2>Top stories</h2>
467 <div class="card"><h3><a href="/story/rust-memory">Rust cuts crawler memory by an order of magnitude</a></h3>
468 <p>A new pipeline holds under five megabytes while parsing six hundred documents a second.</p></div>
469 <div class="card"><h3><a href="/story/http3">What actually changed in HTTP/3</a></h3>
470 <p>The transport moved to UDP, and head-of-line blocking went with it.</p></div>
471 <div class="card"><h3><a href="/story/bm25">BM25 is still the baseline to beat</a></h3>
472 <p>Twenty years on, the ranking function remains hard to improve upon.</p></div>
473 <h2>Opinion</h2>
474 <div class="card"><h3><a href="/opinion/agents">Agents are not a product category</a></h3>
475 <p>A short argument about naming things.</p></div>
476 <div class="card"><h3><a href="/opinion/slm">The case for small local models</a></h3>
477 <p>Latency, privacy and cost all point the same way.</p></div>
478 <a href="/page/2">Next</a>
479 <a href="/story/rust-memory?utm_source=footer">Rust cuts crawler memory by an order of magnitude</a>
480 </main>
481 <footer><a href="/privacy">Privacy policy</a><a href="/terms">Terms of service</a></footer>
482 </body></html>"##;
483
484 fn harvested() -> Vec<Link> {
485 let doc = Doc::parse(LISTING).unwrap();
486 let base = Url::parse("https://wire.example/").unwrap();
487 harvest(&doc, Some(&base), &DenoiseConfig::default())
488 }
489
490 #[test]
491 fn collects_the_stories_and_nothing_else() {
492 let links = harvested();
493 let urls: Vec<&str> = links.iter().map(|l| l.url.as_str()).collect();
494 assert_eq!(links.len(), 5, "{urls:#?}");
495 assert!(urls.contains(&"https://wire.example/story/bm25"));
496 assert!(!urls.iter().any(|u| u.contains("/privacy")));
498 assert!(!urls.iter().any(|u| u.contains("/about")));
499 assert!(!urls.iter().any(|u| u.contains("/page/2")));
500 }
501
502 #[test]
503 fn duplicate_targets_collapse_across_decorations() {
504 let links = harvested();
505 let dupes = links.iter().filter(|l| l.url.contains("rust-memory")).count();
506 assert_eq!(dupes, 1, "the utm-tagged repeat survived");
507 }
508
509 #[test]
510 fn section_headings_become_the_breadcrumb() {
511 let links = harvested();
512 let opinion: Vec<&Link> = links.iter().filter(|l| l.url.contains("/opinion/")).collect();
513 assert_eq!(opinion.len(), 2);
514 assert!(opinion.iter().all(|l| l.heading_path == ["Opinion"]), "{opinion:#?}");
515 let top = links.iter().find(|l| l.url.contains("/story/bm25")).unwrap();
516 assert_eq!(top.heading_path, ["Top stories"]);
517 }
518
519 #[test]
520 fn standfirsts_are_kept_as_snippets() {
521 let links = harvested();
522 let http3 = links.iter().find(|l| l.url.contains("http3")).unwrap();
523 assert!(http3.snippet.contains("head-of-line blocking"), "{:?}", http3.snippet);
524 }
525
526 #[test]
527 fn detection_measures_how_much_prose_belongs_to_links() {
528 let links = harvested();
529 assert!(looks_like_index(&links, 0));
531 let titles: usize = links.iter().map(|l| text::visible_len(&l.text)).sum();
535 assert!(looks_like_index(&links, titles * 2));
536 assert!(!looks_like_index(&links, titles * 20));
538 assert!(!looks_like_index(&links[..2], 0));
540 }
541
542 #[test]
543 fn links_inside_sentences_are_not_inventory() {
544 let html = r#"<html><body><article>
545 <p>A paragraph of real prose that runs on for a while and happens to mention
546 <a href="/guide">the full guide to everything</a> somewhere in the middle of it,
547 plus <a href="/other">another long link title here</a> for good measure.</p>
548 <p>A second paragraph, similarly long, that also links to
549 <a href="/third">a third destination with a title</a> inside a sentence.</p>
550 </article></body></html>"#;
551 let doc = Doc::parse(html).unwrap();
552 let base = Url::parse("https://example.com/post").unwrap();
553 let links = harvest(&doc, Some(&base), &DenoiseConfig::default());
554 assert!(links.is_empty(), "prose links were harvested: {links:#?}");
555 }
556
557 #[test]
558 fn short_and_control_anchors_are_rejected() {
559 assert!(!is_headline("Next"));
560 assert!(!is_headline("Read more"));
561 assert!(!is_headline("2026"));
562 assert!(!is_headline(" "));
563 assert!(is_headline("Rust cuts crawler memory"));
564 assert!(is_headline("GPT-6 Astra"), "short aggregator titles must survive");
565 assert!(!is_headline("Subscribe"));
566 assert!(!is_headline("upvote"));
567 assert!(!is_headline(&"Some navigation label ".repeat(30)));
569 assert!(is_headline("작은 로컬 모델을 위한 근거"));
570 }
571
572 #[test]
573 fn aggregator_metadata_is_rejected() {
574 for junk in [
576 "1501 comments",
577 "342 points",
578 "13 hours ago",
579 "2 days ago",
580 "neil.fraser.name",
581 "docs.rs/tokio",
582 ] {
583 assert!(!is_headline(junk), "{junk:?} was taken for a headline");
584 }
585 for real in [
587 "3 things we learned about small models",
588 "1.76.0 is out, and it changes the defaults",
589 "10 years of Rust in production",
590 ] {
591 assert!(is_headline(real), "{real:?} was rejected");
592 }
593 }
594}
595
596#[cfg(test)]
597mod titular_tests {
598 use super::*;
599
600 #[test]
605 fn an_anchor_wrapping_a_heading_is_titular() {
606 let html = "<html><body><div class=\"cards\">\
607 <div><a href=\"/a\"><h2>The first post about something</h2>\
608 <span>Read More</span></a></div>\
609 <div><a href=\"/b\"><h2>The second post about something</h2>\
610 <span>Read More</span></a></div></div></body></html>";
611 let doc = Doc::parse(html).unwrap();
612 let anchors: Vec<_> =
613 doc.preorder.iter().copied().filter(|&i| doc.tag_name(i) == "a").collect();
614 assert_eq!(anchors.len(), 2);
615 for a in anchors {
616 assert!(is_titular(&doc, a), "anchor wrapping a heading rejected");
617 }
618 }
619}