webtools-fetch 0.1.10

Token-efficient web content fetcher with reference-style URL preservation
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

use scraper::{ElementRef, Html, Selector};

use crate::types::Metadata;

/// Pick the element most likely to contain the primary article content.
///
/// Heuristic, in priority order: `<article>`, `<main>`, `[role=main]`,
/// then the largest `<div>` by text length, falling back to `<body>`.
pub fn content_root(doc: &Html) -> Option<ElementRef<'_>> {
    for sel in ["article", "main", "[role=main]"] {
        if let Ok(selector) = Selector::parse(sel) {
            if let Some(el) = doc.select(&selector).next() {
                return Some(el);
            }
        }
    }

    // Fall back to the largest text-bearing <div>.
    if let Ok(div_sel) = Selector::parse("div") {
        let mut best: Option<(usize, ElementRef)> = None;
        for el in doc.select(&div_sel) {
            let len = el.text().map(|t| t.trim().len()).sum::<usize>();
            if best.as_ref().is_none_or(|(b, _)| len > *b) {
                best = Some((len, el));
            }
        }
        if let Some((len, el)) = best {
            if len > 0 {
                return Some(el);
            }
        }
    }

    Selector::parse("body")
        .ok()
        .and_then(|sel| doc.select(&sel).next())
}

/// Extract the page title from `<title>` or the first `<h1>`.
pub fn extract_title(doc: &Html) -> String {
    for sel in ["title", "h1"] {
        if let Ok(selector) = Selector::parse(sel) {
            if let Some(el) = doc.select(&selector).next() {
                let t = el.text().collect::<String>().trim().to_string();
                if !t.is_empty() {
                    return t;
                }
            }
        }
    }
    String::new()
}

/// Read the `content` attribute of the first matching `<meta>` selector.
fn meta(doc: &Html, selectors: &[&str]) -> Option<String> {
    for sel in selectors {
        if let Ok(selector) = Selector::parse(sel) {
            if let Some(el) = doc.select(&selector).next() {
                if let Some(c) = el.value().attr("content") {
                    let c = c.trim();
                    if !c.is_empty() {
                        return Some(c.to_string());
                    }
                }
            }
        }
    }
    None
}

/// Extract citation-oriented metadata: description, author, publish date,
/// language, and site name (from standard `<meta>`/OpenGraph tags).
pub fn extract_metadata(doc: &Html) -> Metadata {
    let lang = Selector::parse("html")
        .ok()
        .and_then(|sel| doc.select(&sel).next())
        .and_then(|el| el.value().attr("lang"))
        .map(|s| s.trim().to_string())
        .filter(|s| !s.is_empty());

    Metadata {
        description: meta(
            doc,
            &["meta[name=description]", "meta[property='og:description']"],
        ),
        author: meta(
            doc,
            &["meta[name=author]", "meta[property='article:author']"],
        ),
        published: meta(
            doc,
            &[
                "meta[property='article:published_time']",
                "meta[name='date']",
            ],
        ),
        site_name: meta(doc, &["meta[property='og:site_name']"]),
        lang,
    }
}