rustai_core/parse/
meta.rs1use crate::parse::dom::{Doc, Id};
4use crate::text;
5
6#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
8pub struct Meta {
9 pub title: Option<String>,
11 pub description: Option<String>,
13 pub byline: Option<String>,
15 pub published: Option<String>,
17 pub language: Option<String>,
19 pub canonical: Option<String>,
21 pub site_name: Option<String>,
23}
24
25pub(crate) fn extract(doc: &Doc<'_>) -> Meta {
27 let mut meta = Meta::default();
28
29 for &id in &doc.preorder {
30 match doc.tag_name(id) {
31 "html" => {
32 meta.language = meta.language.take().or_else(|| clean(doc.attr(id, "lang")));
33 }
34 "title" if meta.title.is_none() => {
35 meta.title = clean(Some(doc.raw_text(id)));
36 }
37 "link" => {
38 if doc.attr(id, "rel").as_deref().map(str::trim) == Some("canonical") {
39 meta.canonical = meta.canonical.take().or_else(|| clean(doc.attr(id, "href")));
40 }
41 }
42 "meta" => apply_meta_tag(doc, id, &mut meta),
43 "script" if doc.attr(id, "type").as_deref() == Some("application/ld+json") => {
44 apply_json_ld(&doc.raw_text(id), &mut meta);
45 }
46 _ => {}
47 }
48 }
49
50 if let (Some(title), Some(site)) = (meta.title.clone(), meta.site_name.clone())
53 && let Some(stripped) = strip_site_suffix(&title, &site)
54 {
55 meta.title = Some(stripped);
56 }
57 meta
58}
59
60fn apply_meta_tag(doc: &Doc<'_>, id: Id, meta: &mut Meta) {
61 let key = doc
62 .attr(id, "property")
63 .or_else(|| doc.attr(id, "name"))
64 .or_else(|| doc.attr(id, "itemprop"))
65 .map(|k| k.trim().to_ascii_lowercase());
66 let Some(key) = key else { return };
67 let Some(value) = clean(doc.attr(id, "content")) else { return };
68
69 match key.as_str() {
70 "og:title" | "twitter:title" => meta.title = Some(value),
71 "og:description" | "twitter:description" | "description" => {
72 meta.description.get_or_insert(value);
73 }
74 "og:site_name" => meta.site_name = Some(value),
75 "og:locale" => {
76 meta.language.get_or_insert(value.replace('_', "-"));
77 }
78 "og:url" => {
79 meta.canonical.get_or_insert(value);
80 }
81 "author" | "article:author" | "dc.creator" | "citation_author" | "byl" => {
82 meta.byline.get_or_insert(value);
83 }
84 "article:published_time"
85 | "datepublished"
86 | "dc.date"
87 | "date"
88 | "citation_publication_date"
89 | "pubdate" => {
90 meta.published.get_or_insert(value);
91 }
92 _ => {}
93 }
94}
95
96fn apply_json_ld(raw: &str, meta: &mut Meta) {
98 let Ok(value) = serde_json::from_str::<serde_json::Value>(raw.trim()) else { return };
99 let mut queue = vec![value];
100 while let Some(node) = queue.pop() {
101 match node {
102 serde_json::Value::Array(items) => queue.extend(items),
103 serde_json::Value::Object(map) => {
104 if let Some(graph) = map.get("@graph") {
105 queue.push(graph.clone());
106 }
107 if let Some(v) = map.get("headline").and_then(|v| v.as_str()) {
108 meta.title.get_or_insert_with(|| v.to_string());
109 }
110 if let Some(v) = map.get("datePublished").and_then(|v| v.as_str()) {
111 meta.published.get_or_insert_with(|| v.to_string());
112 }
113 if let Some(v) = map.get("description").and_then(|v| v.as_str()) {
114 meta.description.get_or_insert_with(|| v.to_string());
115 }
116 match map.get("author") {
117 Some(serde_json::Value::String(s)) => {
118 meta.byline.get_or_insert_with(|| s.clone());
119 }
120 Some(serde_json::Value::Object(a)) => {
121 if let Some(n) = a.get("name").and_then(|v| v.as_str()) {
122 meta.byline.get_or_insert_with(|| n.to_string());
123 }
124 }
125 Some(serde_json::Value::Array(a)) => {
126 let names: Vec<String> = a
127 .iter()
128 .filter_map(|v| {
129 v.as_str()
130 .map(str::to_string)
131 .or_else(|| v.get("name")?.as_str().map(str::to_string))
132 })
133 .collect();
134 if !names.is_empty() {
135 meta.byline.get_or_insert_with(|| names.join(", "));
136 }
137 }
138 _ => {}
139 }
140 }
141 _ => {}
142 }
143 }
144}
145
146fn strip_site_suffix(title: &str, site: &str) -> Option<String> {
148 let site = site.trim();
149 if site.is_empty() || title.len() <= site.len() {
150 return None;
151 }
152 for sep in [" - ", " | ", " — ", " – ", " :: ", " · "] {
153 if let Some(head) = title.strip_suffix(&format!("{sep}{site}"))
156 && text::visible_len(head.trim()) >= 2
157 {
158 return Some(head.trim().to_string());
159 }
160 }
161 None
162}
163
164fn clean(v: Option<String>) -> Option<String> {
165 let v = text::normalize_ws(&v?);
166 if v.is_empty() { None } else { Some(v) }
167}
168
169#[cfg(test)]
170mod tests {
171 use super::*;
172
173 #[test]
174 fn reads_head_and_open_graph() {
175 let html = r#"<html lang="ko"><head>
176 <title>제목 - 예시 뉴스</title>
177 <meta property="og:site_name" content="예시 뉴스">
178 <meta name="description" content="설명입니다">
179 <meta property="article:published_time" content="2026-01-02T03:04:05Z">
180 <link rel="canonical" href="https://example.com/a">
181 </head><body><p>본문</p></body></html>"#;
182 let doc = Doc::parse(html).unwrap();
183 let meta = extract(&doc);
184 assert_eq!(meta.title.as_deref(), Some("제목"));
185 assert_eq!(meta.language.as_deref(), Some("ko"));
186 assert_eq!(meta.description.as_deref(), Some("설명입니다"));
187 assert_eq!(meta.published.as_deref(), Some("2026-01-02T03:04:05Z"));
188 assert_eq!(meta.canonical.as_deref(), Some("https://example.com/a"));
189 }
190
191 #[test]
192 fn reads_json_ld_author() {
193 let html = r#"<html><head><script type="application/ld+json">
194 {"@type":"Article","headline":"H","author":{"name":"Ada Lovelace"}}
195 </script></head><body></body></html>"#;
196 let doc = Doc::parse(html).unwrap();
197 let meta = extract(&doc);
198 assert_eq!(meta.byline.as_deref(), Some("Ada Lovelace"));
199 }
200}