Skip to main content

rustai_core/parse/
meta.rs

1//! Document metadata from `<head>`, Open Graph, Twitter cards and JSON-LD.
2
3use crate::parse::dom::{Doc, Id};
4use crate::text;
5
6/// Everything we can learn about a page without reading its body.
7#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
8pub struct Meta {
9    /// Best available title.
10    pub title: Option<String>,
11    /// Meta description or `og:description`.
12    pub description: Option<String>,
13    /// Author, if the page declares one.
14    pub byline: Option<String>,
15    /// Publication timestamp as the page states it (usually ISO 8601).
16    pub published: Option<String>,
17    /// BCP-47 language tag from `<html lang>` or `og:locale`.
18    pub language: Option<String>,
19    /// `<link rel="canonical">`.
20    pub canonical: Option<String>,
21    /// `og:site_name`.
22    pub site_name: Option<String>,
23}
24
25/// Read metadata out of a parsed document.
26pub(crate) fn extract(doc: &Doc<'_>) -> Meta {
27    let mut meta = Meta::default();
28
29    for &id in &doc.preorder {
30        match doc.tag_name(id) {
31            "html" => {
32                meta.language = meta.language.take().or_else(|| clean(doc.attr(id, "lang")));
33            }
34            "title" if meta.title.is_none() => {
35                meta.title = clean(Some(doc.raw_text(id)));
36            }
37            "link" => {
38                if doc.attr(id, "rel").as_deref().map(str::trim) == Some("canonical") {
39                    meta.canonical = meta.canonical.take().or_else(|| clean(doc.attr(id, "href")));
40                }
41            }
42            "meta" => apply_meta_tag(doc, id, &mut meta),
43            "script" if doc.attr(id, "type").as_deref() == Some("application/ld+json") => {
44                apply_json_ld(&doc.raw_text(id), &mut meta);
45            }
46            _ => {}
47        }
48    }
49
50    // A `<title>` is usually "Headline — Site Name"; prefer og:title, which
51    // normally is not decorated, and otherwise trim the site suffix.
52    if let (Some(title), Some(site)) = (meta.title.clone(), meta.site_name.clone())
53        && let Some(stripped) = strip_site_suffix(&title, &site)
54    {
55        meta.title = Some(stripped);
56    }
57    meta
58}
59
60fn apply_meta_tag(doc: &Doc<'_>, id: Id, meta: &mut Meta) {
61    let key = doc
62        .attr(id, "property")
63        .or_else(|| doc.attr(id, "name"))
64        .or_else(|| doc.attr(id, "itemprop"))
65        .map(|k| k.trim().to_ascii_lowercase());
66    let Some(key) = key else { return };
67    let Some(value) = clean(doc.attr(id, "content")) else { return };
68
69    match key.as_str() {
70        "og:title" | "twitter:title" => meta.title = Some(value),
71        "og:description" | "twitter:description" | "description" => {
72            meta.description.get_or_insert(value);
73        }
74        "og:site_name" => meta.site_name = Some(value),
75        "og:locale" => {
76            meta.language.get_or_insert(value.replace('_', "-"));
77        }
78        "og:url" => {
79            meta.canonical.get_or_insert(value);
80        }
81        "author" | "article:author" | "dc.creator" | "citation_author" | "byl" => {
82            meta.byline.get_or_insert(value);
83        }
84        "article:published_time"
85        | "datepublished"
86        | "dc.date"
87        | "date"
88        | "citation_publication_date"
89        | "pubdate" => {
90            meta.published.get_or_insert(value);
91        }
92        _ => {}
93    }
94}
95
96/// Pull the handful of fields we care about out of schema.org JSON-LD.
97fn apply_json_ld(raw: &str, meta: &mut Meta) {
98    let Ok(value) = serde_json::from_str::<serde_json::Value>(raw.trim()) else { return };
99    let mut queue = vec![value];
100    while let Some(node) = queue.pop() {
101        match node {
102            serde_json::Value::Array(items) => queue.extend(items),
103            serde_json::Value::Object(map) => {
104                if let Some(graph) = map.get("@graph") {
105                    queue.push(graph.clone());
106                }
107                if let Some(v) = map.get("headline").and_then(|v| v.as_str()) {
108                    meta.title.get_or_insert_with(|| v.to_string());
109                }
110                if let Some(v) = map.get("datePublished").and_then(|v| v.as_str()) {
111                    meta.published.get_or_insert_with(|| v.to_string());
112                }
113                if let Some(v) = map.get("description").and_then(|v| v.as_str()) {
114                    meta.description.get_or_insert_with(|| v.to_string());
115                }
116                match map.get("author") {
117                    Some(serde_json::Value::String(s)) => {
118                        meta.byline.get_or_insert_with(|| s.clone());
119                    }
120                    Some(serde_json::Value::Object(a)) => {
121                        if let Some(n) = a.get("name").and_then(|v| v.as_str()) {
122                            meta.byline.get_or_insert_with(|| n.to_string());
123                        }
124                    }
125                    Some(serde_json::Value::Array(a)) => {
126                        let names: Vec<String> = a
127                            .iter()
128                            .filter_map(|v| {
129                                v.as_str()
130                                    .map(str::to_string)
131                                    .or_else(|| v.get("name")?.as_str().map(str::to_string))
132                            })
133                            .collect();
134                        if !names.is_empty() {
135                            meta.byline.get_or_insert_with(|| names.join(", "));
136                        }
137                    }
138                    _ => {}
139                }
140            }
141            _ => {}
142        }
143    }
144}
145
146/// `"Headline - Example News"` with site `"Example News"` becomes `"Headline"`.
147fn strip_site_suffix(title: &str, site: &str) -> Option<String> {
148    let site = site.trim();
149    if site.is_empty() || title.len() <= site.len() {
150        return None;
151    }
152    for sep in [" - ", " | ", " — ", " – ", " :: ", " · "] {
153        // Two grapheme clusters, not two words: a Korean or Chinese headline is
154        // routinely shorter than any sensible Latin-script minimum.
155        if let Some(head) = title.strip_suffix(&format!("{sep}{site}"))
156            && text::visible_len(head.trim()) >= 2
157        {
158            return Some(head.trim().to_string());
159        }
160    }
161    None
162}
163
164fn clean(v: Option<String>) -> Option<String> {
165    let v = text::normalize_ws(&v?);
166    if v.is_empty() { None } else { Some(v) }
167}
168
169#[cfg(test)]
170mod tests {
171    use super::*;
172
173    #[test]
174    fn reads_head_and_open_graph() {
175        let html = r#"<html lang="ko"><head>
176            <title>제목 - 예시 뉴스</title>
177            <meta property="og:site_name" content="예시 뉴스">
178            <meta name="description" content="설명입니다">
179            <meta property="article:published_time" content="2026-01-02T03:04:05Z">
180            <link rel="canonical" href="https://example.com/a">
181        </head><body><p>본문</p></body></html>"#;
182        let doc = Doc::parse(html).unwrap();
183        let meta = extract(&doc);
184        assert_eq!(meta.title.as_deref(), Some("제목"));
185        assert_eq!(meta.language.as_deref(), Some("ko"));
186        assert_eq!(meta.description.as_deref(), Some("설명입니다"));
187        assert_eq!(meta.published.as_deref(), Some("2026-01-02T03:04:05Z"));
188        assert_eq!(meta.canonical.as_deref(), Some("https://example.com/a"));
189    }
190
191    #[test]
192    fn reads_json_ld_author() {
193        let html = r#"<html><head><script type="application/ld+json">
194            {"@type":"Article","headline":"H","author":{"name":"Ada Lovelace"}}
195        </script></head><body></body></html>"#;
196        let doc = Doc::parse(html).unwrap();
197        let meta = extract(&doc);
198        assert_eq!(meta.byline.as_deref(), Some("Ada Lovelace"));
199    }
200}