webfetch/
extract.rs

1use std::collections::HashMap;
2
3use ego_tree::{NodeId, NodeRef};
4use once_cell::sync::Lazy;
5use scraper::node::Node;
6use scraper::{ElementRef, Html, Selector};
7
8use crate::types::Metadata;
9
10/// Selectors are compiled once per process rather than on every call — this
11/// runs on the hot conversion path, several times per page.
12fn selector(sel: &str) -> Selector {
13    Selector::parse(sel).expect("static selector")
14}
15
16static CONTENT_ROOTS: Lazy<[Selector; 3]> = Lazy::new(|| {
17    [
18        selector("article"),
19        selector("main"),
20        selector("[role=main]"),
21    ]
22});
23static DIV: Lazy<Selector> = Lazy::new(|| selector("div"));
24static BODY: Lazy<Selector> = Lazy::new(|| selector("body"));
25static TITLE: Lazy<[Selector; 2]> = Lazy::new(|| [selector("title"), selector("h1")]);
26static HTML_EL: Lazy<Selector> = Lazy::new(|| selector("html"));
27
28static META_DESCRIPTION: Lazy<[Selector; 2]> = Lazy::new(|| {
29    [
30        selector("meta[name=description]"),
31        selector("meta[property='og:description']"),
32    ]
33});
34static META_AUTHOR: Lazy<[Selector; 2]> = Lazy::new(|| {
35    [
36        selector("meta[name=author]"),
37        selector("meta[property='article:author']"),
38    ]
39});
40static META_PUBLISHED: Lazy<[Selector; 2]> = Lazy::new(|| {
41    [
42        selector("meta[property='article:published_time']"),
43        selector("meta[name='date']"),
44    ]
45});
46static META_SITE_NAME: Lazy<[Selector; 1]> =
47    Lazy::new(|| [selector("meta[property='og:site_name']")]);
48
49/// Sum the trimmed length of every descendant text node, for every node in the
50/// tree, in a single bottom-up pass.
51///
52/// The previous "largest `<div>`" heuristic called `el.text()` (a full subtree
53/// walk) once per `<div>`; on nested DOMs the same text was re-summed at every
54/// ancestor, making it ~O(n²). Computing each node's subtree text length once
55/// and reading it back from the map keeps the identical "largest text-bearing
56/// container" semantics in O(n).
57fn subtree_text_lengths(root: NodeRef<Node>, out: &mut HashMap<NodeId, usize>) -> usize {
58    let mut total = match root.value() {
59        Node::Text(t) => t.trim().len(),
60        _ => 0,
61    };
62    for child in root.children() {
63        total += subtree_text_lengths(child, out);
64    }
65    out.insert(root.id(), total);
66    total
67}
68
69/// Pick the element most likely to contain the primary article content.
70///
71/// Heuristic, in priority order: `<article>`, `<main>`, `[role=main]`,
72/// then the largest `<div>` by text length, falling back to `<body>`.
73pub fn content_root(doc: &Html) -> Option<ElementRef<'_>> {
74    for selector in CONTENT_ROOTS.iter() {
75        if let Some(el) = doc.select(selector).next() {
76            return Some(el);
77        }
78    }
79
80    // Fall back to the largest text-bearing <div>, using one bottom-up pass to
81    // compute every node's subtree text length up front.
82    let mut lengths: HashMap<NodeId, usize> = HashMap::new();
83    subtree_text_lengths(doc.tree.root(), &mut lengths);
84
85    let mut best: Option<(usize, ElementRef)> = None;
86    for el in doc.select(&DIV) {
87        let len = lengths.get(&el.id()).copied().unwrap_or(0);
88        if best.as_ref().is_none_or(|(b, _)| len > *b) {
89            best = Some((len, el));
90        }
91    }
92    if let Some((len, el)) = best {
93        if len > 0 {
94            return Some(el);
95        }
96    }
97
98    doc.select(&BODY).next()
99}
100
101/// Extract the page title from `<title>` or the first `<h1>`.
102pub fn extract_title(doc: &Html) -> String {
103    for selector in TITLE.iter() {
104        if let Some(el) = doc.select(selector).next() {
105            let t = el.text().collect::<String>().trim().to_string();
106            if !t.is_empty() {
107                return t;
108            }
109        }
110    }
111    String::new()
112}
113
114/// Read the `content` attribute of the first matching `<meta>` selector.
115fn meta(doc: &Html, selectors: &[Selector]) -> Option<String> {
116    for selector in selectors {
117        if let Some(el) = doc.select(selector).next() {
118            if let Some(c) = el.value().attr("content") {
119                let c = c.trim();
120                if !c.is_empty() {
121                    return Some(c.to_string());
122                }
123            }
124        }
125    }
126    None
127}
128
129/// Extract citation-oriented metadata: description, author, publish date,
130/// language, and site name (from standard `<meta>`/OpenGraph tags).
131pub fn extract_metadata(doc: &Html) -> Metadata {
132    let lang = doc
133        .select(&HTML_EL)
134        .next()
135        .and_then(|el| el.value().attr("lang"))
136        .map(|s| s.trim().to_string())
137        .filter(|s| !s.is_empty());
138
139    Metadata {
140        description: meta(doc, &*META_DESCRIPTION),
141        author: meta(doc, &*META_AUTHOR),
142        published: meta(doc, &*META_PUBLISHED),
143        site_name: meta(doc, &*META_SITE_NAME),
144        lang,
145        charset: None,
146    }
147}
webfetch/extract.rs

webfetch/
extract.rs
