crawlkit_parser/
metadata.rs

1//! 页面元数据提取模块
2//!
3//! 从 HTML 文档中提取标题、描述、关键词、Open Graph、Twitter Card、
4//! 结构化数据等各类元数据。改编自 halldyll-parser。
5
6use scraper::{Html, ElementRef};
7use std::collections::HashMap;
8use url::Url;
9
10use crate::selector::{
11    SELECTORS, try_parse_selector, meta_name_selector, meta_property_selector, link_rel_selector,
12};
13use crate::types::{
14    PageMetadata, OpenGraph, TwitterCard, RobotsMeta, AlternateLink,
15    StructuredData, ParserResult,
16};
17
18// ============================================================================
19// 主入口
20// ============================================================================
21
22/// 从 HTML 文档中提取完整页面元数据
23pub fn extract_metadata(document: &Html, base_url: Option<&Url>) -> ParserResult<PageMetadata> {
24    let title = extract_title(document);
25    let charset = extract_charset(document);
26    let language = extract_language(document);
27    let base = extract_base_url(document);
28    let description = extract_meta_content(document, "description");
29    let author = extract_meta_content(document, "author");
30    let generator = extract_meta_content(document, "generator");
31    let viewport = extract_meta_content(document, "viewport");
32    let theme_color = extract_meta_content(document, "theme-color");
33    let published_date = extract_meta_content(document, "date")
34        .or_else(|| extract_meta_content(document, "article:published_time"));
35    let modified_date = extract_meta_content(document, "article:modified_time");
36    let keywords = extract_keywords(document);
37    let canonical = extract_canonical(document, base_url);
38    let favicon = extract_favicon(document, base_url);
39    let apple_touch = extract_apple_touch_icon(document, base_url);
40    let robots = extract_robots(document);
41    let opengraph = extract_opengraph(document);
42    let twitter = extract_twitter_card(document);
43    let alternates = extract_alternates(document, base_url);
44    let structured = extract_structured_data(document);
45    let schema_type = structured.first()
46        .and_then(|s| s.schema_type.clone());
47    let custom = extract_custom_meta(document);
48
49    Ok(PageMetadata {
50        title,
51        description,
52        keywords,
53        author,
54        generator,
55        canonical,
56        base_url: base,
57        language,
58        charset,
59        viewport,
60        robots,
61        opengraph,
62        twitter,
63        alternates,
64        favicon,
65        apple_touch_icon: apple_touch,
66        theme_color,
67        published_date,
68        modified_date,
69        schema_type,
70        custom,
71    })
72}
73
74// ============================================================================
75// 基本元数据
76// ============================================================================
77
78/// 提取页面标题：依次尝试 og:title、twitter:title、`<title>`、h1
79pub fn extract_title(document: &Html) -> Option<String> {
80    if let Some(sel) = try_parse_selector(&meta_property_selector("og:title")) {
81        if let Some(el) = document.select(&sel).next() {
82            if let Some(content) = el.value().attr("content") {
83                let text = content.trim().to_string();
84                if !text.is_empty() {
85                    return Some(text);
86                }
87            }
88        }
89    }
90    if let Some(sel) = try_parse_selector(&meta_name_selector("twitter:title")) {
91        if let Some(el) = document.select(&sel).next() {
92            if let Some(content) = el.value().attr("content") {
93                let text = content.trim().to_string();
94                if !text.is_empty() {
95                    return Some(text);
96                }
97            }
98        }
99    }
100    let title_sel = &SELECTORS.title;
101    if let Some(el) = document.select(title_sel).next() {
102        let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
103        if !text.is_empty() {
104            return Some(text);
105        }
106    }
107    if let Some(sel) = try_parse_selector("h1") {
108        if let Some(el) = document.select(&sel).next() {
109            let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
110            if !text.is_empty() {
111                return Some(text);
112            }
113        }
114    }
115    None
116}
117
118/// 提取字符编码声明
119pub fn extract_charset(document: &Html) -> Option<String> {
120    if let Some(sel) = try_parse_selector("meta[charset]") {
121        if let Some(el) = document.select(&sel).next() {
122            if let Some(charset) = el.value().attr("charset") {
123                let val = charset.trim().to_string();
124                if !val.is_empty() {
125                    return Some(val);
126                }
127            }
128        }
129    }
130    if let Some(sel) = try_parse_selector(r#"meta[http-equiv="Content-Type"]"#) {
131        if let Some(el) = document.select(&sel).next() {
132            if let Some(content) = el.value().attr("content") {
133                if let Some(pos) = content.to_lowercase().find("charset=") {
134                    let charset = content[pos + 8..].trim().to_string();
135                    if !charset.is_empty() {
136                        return Some(charset);
137                    }
138                }
139            }
140        }
141    }
142    None
143}
144
145/// 提取页面语言：优先 `<html lang>`，其次 `http-equiv="content-language"`
146pub fn extract_language(document: &Html) -> Option<String> {
147    let html_sel = &SELECTORS.html;
148    if let Some(el) = document.select(html_sel).next() {
149        if let Some(lang) = el.value().attr("lang") {
150            let val = lang.trim().to_string();
151            if !val.is_empty() {
152                return Some(val);
153            }
154        }
155    }
156    if let Some(sel) = try_parse_selector(r#"meta[http-equiv="content-language"]"#) {
157        if let Some(el) = document.select(&sel).next() {
158            if let Some(content) = el.value().attr("content") {
159                let val = content.trim().to_string();
160                if !val.is_empty() {
161                    return Some(val);
162                }
163            }
164        }
165    }
166    None
167}
168
169/// 提取 `<base>` 标签的 href 值
170pub fn extract_base_url(document: &Html) -> Option<String> {
171    let base_sel = &SELECTORS.base;
172    if let Some(el) = document.select(base_sel).next() {
173        if let Some(href) = el.value().attr("href") {
174            let val = href.trim().to_string();
175            if !val.is_empty() {
176                return Some(val);
177            }
178        }
179    }
180    None
181}
182
183/// 通用 meta 内容提取，同时尝试 `name` 和 `property` 属性
184pub fn extract_meta_content(document: &Html, name: &str) -> Option<String> {
185    if let Some(sel) = try_parse_selector(&meta_name_selector(name)) {
186        if let Some(el) = document.select(&sel).next() {
187            if let Some(content) = el.value().attr("content") {
188                let val = content.trim().to_string();
189                if !val.is_empty() {
190                    return Some(val);
191                }
192            }
193        }
194    }
195    if let Some(sel) = try_parse_selector(&meta_property_selector(name)) {
196        if let Some(el) = document.select(&sel).next() {
197            if let Some(content) = el.value().attr("content") {
198                let val = content.trim().to_string();
199                if !val.is_empty() {
200                    return Some(val);
201                }
202            }
203        }
204    }
205    None
206}
207
208/// 提取关键词：从 `<meta name="keywords">` 解析，逗号分隔
209pub fn extract_keywords(document: &Html) -> Vec<String> {
210    if let Some(sel) = try_parse_selector(&meta_name_selector("keywords")) {
211        if let Some(el) = document.select(&sel).next() {
212            if let Some(content) = el.value().attr("content") {
213                return content
214                    .split(',')
215                    .map(|s| s.trim().to_string())
216                    .filter(|s| !s.is_empty())
217                    .collect();
218            }
219        }
220    }
221    Vec::new()
222}
223
224// ============================================================================
225// 链接资源
226// ============================================================================
227
228/// 提取规范链接 `<link rel="canonical">`
229pub fn extract_canonical(document: &Html, base_url: Option<&Url>) -> Option<String> {
230    if let Some(sel) = try_parse_selector(&link_rel_selector("canonical")) {
231        if let Some(el) = document.select(&sel).next() {
232            if let Some(href) = el.value().attr("href") {
233                return resolve_url(base_url, href);
234            }
235        }
236    }
237    None
238}
239
240/// 提取 favicon：优先选择尺寸最大的图标
241pub fn extract_favicon(document: &Html, base_url: Option<&Url>) -> Option<String> {
242    if let Some(sel) = try_parse_selector("link[rel~='icon'], link[rel='shortcut icon'], link[rel='apple-touch-icon-precomposed']") {
243        let mut candidates: Vec<(String, String)> = Vec::new();
244        for el in document.select(&sel) {
245            if let Some(href) = el.value().attr("href") {
246                let sizes = el.value().attr("sizes").unwrap_or("").to_string();
247                candidates.push((href.to_string(), sizes));
248            }
249        }
250        if candidates.is_empty() {
251            return None;
252        }
253        let best = find_largest_icon(&candidates);
254        resolve_url(base_url, &best)
255    } else {
256        None
257    }
258}
259
260/// 提取苹果触控图标
261pub fn extract_apple_touch_icon(document: &Html, base_url: Option<&Url>) -> Option<String> {
262    if let Some(sel) = try_parse_selector("link[rel='apple-touch-icon'], link[rel='apple-touch-icon-precomposed']") {
263        if let Some(el) = document.select(&sel).next() {
264            if let Some(href) = el.value().attr("href") {
265                return resolve_url(base_url, href);
266            }
267        }
268    }
269    None
270}
271
272// ============================================================================
273// 爬虫规则（robots）
274// ============================================================================
275
276/// 提取 robots meta 标签，解析指令
277pub fn extract_robots(document: &Html) -> RobotsMeta {
278    let content = if let Some(sel) = try_parse_selector("meta[name='robots'], meta[name='ROBOTS']") {
279        document.select(&sel).next()
280            .and_then(|el| el.value().attr("content").map(|c| c.to_string()))
281    } else {
282        None
283    };
284
285    let directives = match content {
286        Some(ref c) => c.split(',').map(|s| s.trim().to_lowercase()).collect::<Vec<_>>(),
287        None => return RobotsMeta::allowed(),
288    };
289
290    let mut meta = RobotsMeta::allowed();
291    meta.raw = content;
292
293    for directive in &directives {
294        match directive.as_str() {
295            "noindex" => meta.index = false,
296            "nofollow" => meta.follow = false,
297            "noarchive" => meta.archive = false,
298            "nocache" => meta.cache = false,
299            "nosnippet" => meta.snippet = false,
300            "all" | "index" | "follow" => {}
301            v if v.starts_with("max-snippet:") => {
302                let val = v.trim_start_matches("max-snippet:").trim();
303                meta.max_snippet = val.parse().unwrap_or(-1);
304            }
305            v if v.starts_with("max-image-preview:") => {
306                let val = v.trim_start_matches("max-image-preview:").trim();
307                meta.max_image_preview = Some(val.to_string());
308            }
309            v if v.starts_with("max-video-preview:") => {
310                let val = v.trim_start_matches("max-video-preview:").trim();
311                meta.max_video_preview = val.parse().unwrap_or(-1);
312            }
313            _ => {}
314        }
315    }
316
317    meta
318}
319
320// ============================================================================
321// Open Graph
322// ============================================================================
323
324/// 提取 Open Graph 元数据
325pub fn extract_opengraph(document: &Html) -> OpenGraph {
326    let mut og = OpenGraph::default();
327    if let Some(title) = extract_og_property(document, "og:title") {
328        og.title = Some(title);
329    }
330    if let Some(og_type) = extract_og_property(document, "og:type") {
331        og.og_type = Some(og_type);
332    }
333    if let Some(url) = extract_og_property(document, "og:url") {
334        og.url = Some(url);
335    }
336    if let Some(image) = extract_og_property(document, "og:image") {
337        og.image = Some(image);
338    }
339    if let Some(desc) = extract_og_property(document, "og:description") {
340        og.description = Some(desc);
341    }
342    if let Some(site) = extract_og_property(document, "og:site_name") {
343        og.site_name = Some(site);
344    }
345    if let Some(locale) = extract_og_property(document, "og:locale") {
346        og.locale = Some(locale);
347    }
348    if let Some(video) = extract_og_property(document, "og:video") {
349        og.video = Some(video);
350    }
351    if let Some(audio) = extract_og_property(document, "og:audio") {
352        og.audio = Some(audio);
353    }
354    og.extra = extract_all_og_properties(document);
355    og
356}
357
358/// 提取单个 Open Graph 属性值
359pub fn extract_og_property(document: &Html, property: &str) -> Option<String> {
360    if let Some(sel) = try_parse_selector(&meta_property_selector(property)) {
361        if let Some(el) = document.select(&sel).next() {
362            if let Some(content) = el.value().attr("content") {
363                let val = content.trim().to_string();
364                if !val.is_empty() {
365                    return Some(val);
366                }
367            }
368        }
369    }
370    None
371}
372
373/// 提取所有 Open Graph 属性到 HashMap（不含标准字段）
374pub fn extract_all_og_properties(document: &Html) -> HashMap<String, String> {
375    let mut extras = HashMap::new();
376    let meta_sel = &SELECTORS.meta;
377    let known = [
378        "og:title", "og:type", "og:url", "og:image", "og:description",
379        "og:site_name", "og:locale", "og:video", "og:audio",
380    ];
381    for el in document.select(meta_sel) {
382        if let Some(property) = el.value().attr("property") {
383            let prop_lower = property.to_lowercase();
384            if prop_lower.starts_with("og:") && !known.contains(&prop_lower.as_str()) {
385                if let Some(content) = el.value().attr("content") {
386                    extras.insert(property.to_string(), content.to_string());
387                }
388            }
389        }
390    }
391    extras
392}
393
394// ============================================================================
395// Twitter Card
396// ============================================================================
397
398/// 提取 Twitter Card 元数据
399pub fn extract_twitter_card(document: &Html) -> TwitterCard {
400    let mut card = TwitterCard::default();
401    if let Some(val) = extract_twitter_property(document, "twitter:card") {
402        card.card = Some(val);
403    }
404    if let Some(val) = extract_twitter_property(document, "twitter:site") {
405        card.site = Some(val);
406    }
407    if let Some(val) = extract_twitter_property(document, "twitter:creator") {
408        card.creator = Some(val);
409    }
410    if let Some(val) = extract_twitter_property(document, "twitter:title") {
411        card.title = Some(val);
412    }
413    if let Some(val) = extract_twitter_property(document, "twitter:description") {
414        card.description = Some(val);
415    }
416    if let Some(val) = extract_twitter_property(document, "twitter:image") {
417        card.image = Some(val);
418    }
419    card.extra = extract_all_twitter_properties(document);
420    card
421}
422
423/// 提取单个 Twitter Card 属性值
424pub fn extract_twitter_property(document: &Html, name: &str) -> Option<String> {
425    if let Some(sel) = try_parse_selector(&meta_name_selector(name)) {
426        if let Some(el) = document.select(&sel).next() {
427            if let Some(content) = el.value().attr("content") {
428                let val = content.trim().to_string();
429                if !val.is_empty() {
430                    return Some(val);
431                }
432            }
433        }
434    }
435    None
436}
437
438/// 提取所有 Twitter Card 属性到 HashMap（不含标准字段）
439pub fn extract_all_twitter_properties(document: &Html) -> HashMap<String, String> {
440    let mut extras = HashMap::new();
441    let meta_sel = &SELECTORS.meta;
442    let known = [
443        "twitter:card", "twitter:site", "twitter:creator",
444        "twitter:title", "twitter:description", "twitter:image",
445    ];
446    for el in document.select(meta_sel) {
447        if let Some(name) = el.value().attr("name") {
448            let name_lower = name.to_lowercase();
449            if name_lower.starts_with("twitter:") && !known.contains(&name_lower.as_str()) {
450                if let Some(content) = el.value().attr("content") {
451                    extras.insert(name.to_string(), content.to_string());
452                }
453            }
454        }
455    }
456    extras
457}
458
459// ============================================================================
460// 备用链接
461// ============================================================================
462
463/// 提取 `<link rel="alternate">` 标签（如多语言版本、RSS 等）
464pub fn extract_alternates(document: &Html, base_url: Option<&Url>) -> Vec<AlternateLink> {
465    let mut alternates = Vec::new();
466    if let Some(sel) = try_parse_selector(&link_rel_selector("alternate")) {
467        for el in document.select(&sel) {
468            let hreflang = el.value().attr("hreflang")
469                .unwrap_or("")
470                .trim()
471                .to_string();
472            let href = el.value().attr("href")
473                .unwrap_or("")
474                .trim()
475                .to_string();
476            if href.is_empty() {
477                continue;
478            }
479            let resolved = resolve_url(base_url, &href).unwrap_or(href);
480            alternates.push(AlternateLink { hreflang, href: resolved });
481        }
482    }
483    alternates
484}
485
486// ============================================================================
487// 结构化数据
488// ============================================================================
489
490/// 提取全部结构化数据（JSON-LD + Microdata）
491pub fn extract_structured_data(document: &Html) -> Vec<StructuredData> {
492    let mut results = Vec::new();
493    results.extend(extract_json_ld(document));
494    results.extend(extract_microdata(document));
495    results
496}
497
498/// 提取 JSON-LD 结构化数据
499pub fn extract_json_ld(document: &Html) -> Vec<StructuredData> {
500    let mut results = Vec::new();
501    let json_ld_sel = &SELECTORS.json_ld;
502    for el in document.select(json_ld_sel) {
503        let raw = el.text().collect::<Vec<_>>().join("").trim().to_string();
504        if raw.is_empty() {
505            continue;
506        }
507        let mut sd = StructuredData::json_ld(&raw);
508        if let Ok(val) = serde_json::from_str::<serde_json::Value>(&raw) {
509            if let Some(obj) = val.as_object() {
510                for (k, v) in obj {
511                    sd.properties.insert(k.clone(), v.clone());
512                }
513            }
514            if let Some(typ) = val.get("@type").and_then(|t| t.as_str()) {
515                sd.schema_type = Some(typ.to_string());
516            }
517            if let Some(typ) = val.get("type").and_then(|t| t.as_str()) {
518                sd.schema_type = Some(typ.to_string());
519            }
520        }
521        results.push(sd);
522    }
523    results
524}
525
526/// 提取 Microdata 结构化数据
527pub fn extract_microdata(document: &Html) -> Vec<StructuredData> {
528    let mut results = Vec::new();
529    if let Some(sel) = try_parse_selector("*[itemscope]") {
530        for el in document.select(&sel) {
531            let itemtype = el.value().attr("itemtype")
532                .and_then(|t| t.rsplit('/').next())
533                .or_else(|| el.value().attr("itemtype"))
534                .map(|t| t.to_string());
535
536            let mut sd = match itemtype {
537                Some(ref t) => StructuredData::microdata(t),
538                None => continue,
539            };
540
541            let mut props = HashMap::new();
542            extract_microdata_properties(&el, &mut props);
543            sd.properties = props;
544            results.push(sd);
545        }
546    }
547    results
548}
549
550/// 递归提取一个 Microdata 元素的所有属性
551fn extract_microdata_properties(element: &ElementRef, props: &mut HashMap<String, serde_json::Value>) {
552    if let Some(sel) = try_parse_selector("*[itemprop]") {
553        for child in element.select(&sel) {
554            if child.value().attr("itemscope").is_some() {
555                continue;
556            }
557            if let Some(prop_name) = child.value().attr("itemprop") {
558                let value = get_microdata_value(&child);
559                props.insert(prop_name.to_string(), serde_json::Value::String(value));
560            }
561        }
562    }
563}
564
565/// 获取 Microdata 元素的值（按优先级：content → src → href → text）
566fn get_microdata_value(element: &ElementRef) -> String {
567    if let Some(content) = element.value().attr("content") {
568        return content.trim().to_string();
569    }
570    if let Some(src) = element.value().attr("src") {
571        return src.trim().to_string();
572    }
573    if let Some(href) = element.value().attr("href") {
574        return href.trim().to_string();
575    }
576    element.text().collect::<Vec<_>>().join("").trim().to_string()
577}
578
579// ============================================================================
580// 自定义元数据
581// ============================================================================
582
583/// 提取所有未归类到其他方法的自定义 meta 标签
584pub fn extract_custom_meta(document: &Html) -> HashMap<String, String> {
585    let mut custom = HashMap::new();
586    let meta_sel = &SELECTORS.meta;
587    let skip_names = [
588        "description", "keywords", "author", "generator", "viewport",
589        "theme-color", "robots", "googlebot", "date",
590    ];
591    let skip_prefixes = ["og:", "twitter:", "article:", "al:"];
592    for el in document.select(meta_sel) {
593        let name = el.value().attr("name")
594            .or_else(|| el.value().attr("property"))
595            .map(|n| n.to_string());
596        let content = el.value().attr("content")
597            .map(|c| c.to_string());
598        match (name, content) {
599            (Some(n), Some(c)) if !n.is_empty() && !c.is_empty() => {
600                let n_lower = n.to_lowercase();
601                if skip_names.contains(&n_lower.as_str()) {
602                    continue;
603                }
604                if skip_prefixes.iter().any(|p| n_lower.starts_with(p)) {
605                    continue;
606                }
607                custom.insert(n, c);
608            }
609            _ => {}
610        }
611    }
612    custom
613}
614
615// ============================================================================
616// 内部辅助函数
617// ============================================================================
618
619/// 将相对 URL 解析为绝对 URL，base_url 为 None 时直接返回原始值
620fn resolve_url(base_url: Option<&Url>, url_str: &str) -> Option<String> {
621    let trimmed = url_str.trim();
622    if trimmed.is_empty() || trimmed.starts_with("data:") || trimmed.starts_with("javascript:") {
623        return None;
624    }
625    match base_url {
626        Some(base) => base.join(trimmed).ok().map(|u| u.to_string()),
627        None => Some(trimmed.to_string()),
628    }
629}
630
631/// 从候选图标列表中找出尺寸最大的一个
632fn find_largest_icon(candidates: &[(String, String)]) -> String {
633    let mut best: Option<&str> = None;
634    let mut best_size: i32 = -1;
635
636    for (href, sizes) in candidates {
637        if let Some(dim) = parse_icon_size(sizes) {
638            if dim > best_size {
639                best_size = dim;
640                best = Some(href);
641            }
642        } else if best.is_none() {
643            best = Some(href);
644        }
645    }
646
647    best.unwrap_or_else(|| candidates[0].0.as_str()).to_string()
648}
649
650/// 解析 sizes 属性（如 "32x32"）返回最大边长
651fn parse_icon_size(sizes: &str) -> Option<i32> {
652    let sizes = sizes.trim();
653    if sizes.is_empty() || sizes.eq_ignore_ascii_case("any") {
654        return None;
655    }
656    sizes.split_whitespace()
657        .filter_map(|s| {
658            let parts: Vec<&str> = s.split('x').collect();
659            if parts.len() == 2 {
660                let w = parts[0].parse::<i32>().ok()?;
661                let h = parts[1].parse::<i32>().ok()?;
662                Some(w.max(h))
663            } else {
664                None
665            }
666        })
667        .max()
668}
669
670// ============================================================================
671// 测试
672// ============================================================================
673
674#[cfg(test)]
675mod tests {
676    use super::*;
677
678    /// 解析 HTML 字符串为 `Html` 文档
679    fn parse_html(html: &str) -> Html {
680        Html::parse_document(html)
681    }
682
683    /// 创建测试用 base URL
684    fn test_base_url() -> Url {
685        Url::parse("https://example.com").unwrap()
686    }
687
688    // -----------------------------------------------------------------------
689    // 标题提取
690    // -----------------------------------------------------------------------
691
692    #[test]
693    fn 提取标题_og_title优先() {
694        let html = parse_html(r#"
695            <html><head>
696                <meta property="og:title" content="OG Title">
697                <title>Page Title</title>
698            </head></html>
699        "#);
700        assert_eq!(extract_title(&html), Some("OG Title".to_string()));
701    }
702
703    #[test]
704    fn 提取标题_twitter_title其次() {
705        let html = parse_html(r#"
706            <html><head>
707                <meta name="twitter:title" content="Twitter Title">
708                <title>Page Title</title>
709            </head></html>
710        "#);
711        assert_eq!(extract_title(&html), Some("Twitter Title".to_string()));
712    }
713
714    #[test]
715    fn 提取标题_title标签兜底() {
716        let html = parse_html("<html><head><title>Page Title</title></head></html>");
717        assert_eq!(extract_title(&html), Some("Page Title".to_string()));
718    }
719
720    #[test]
721    fn 提取标题_h1最后兜底() {
722        let html = parse_html("<html><body><h1>H1 Title</h1></body></html>");
723        assert_eq!(extract_title(&html), Some("H1 Title".to_string()));
724    }
725
726    #[test]
727    fn 提取标题_无标题返回None() {
728        let html = parse_html("<html><head></head><body><p>no title</p></body></html>");
729        assert_eq!(extract_title(&html), None);
730    }
731
732    // -----------------------------------------------------------------------
733    // 字符编码
734    // -----------------------------------------------------------------------
735
736    #[test]
737    fn 提取charset_meta标签() {
738        let html = parse_html(r#"<html><head><meta charset="utf-8"></head></html>"#);
739        assert_eq!(extract_charset(&html), Some("utf-8".to_string()));
740    }
741
742    #[test]
743    fn 提取charset_http_equiv() {
744        let html = parse_html(r#"<html><head><meta http-equiv="Content-Type" content="text/html; charset=gb2312"></head></html>"#);
745        assert_eq!(extract_charset(&html), Some("gb2312".to_string()));
746    }
747
748    #[test]
749    fn 提取charset_无声明返回None() {
750        let html = parse_html("<html><head></head></html>");
751        assert_eq!(extract_charset(&html), None);
752    }
753
754    // -----------------------------------------------------------------------
755    // 语言
756    // -----------------------------------------------------------------------
757
758    #[test]
759    fn 提取语言_html_lang属性() {
760        let html = parse_html(r#"<html lang="en"><head></head></html>"#);
761        assert_eq!(extract_language(&html), Some("en".to_string()));
762    }
763
764    #[test]
765    fn 提取语言_http_equiv() {
766        let html = parse_html(r#"<html><head><meta http-equiv="content-language" content="zh-CN"></head></html>"#);
767        assert_eq!(extract_language(&html), Some("zh-CN".to_string()));
768    }
769
770    // -----------------------------------------------------------------------
771    // base URL
772    // -----------------------------------------------------------------------
773
774    #[test]
775    fn 提取base_url() {
776        let html = parse_html(r#"<html><head><base href="https://example.com/base/"></head></html>"#);
777        assert_eq!(extract_base_url(&html), Some("https://example.com/base/".to_string()));
778    }
779
780    // -----------------------------------------------------------------------
781    // 关键词
782    // -----------------------------------------------------------------------
783
784    #[test]
785    fn 提取关键词_逗号分隔() {
786        let html = parse_html(r#"<html><head><meta name="keywords" content="rust, web, scraping"></head></html>"#);
787        assert_eq!(extract_keywords(&html), vec!["rust", "web", "scraping"]);
788    }
789
790    #[test]
791    fn 提取关键词_无关键词返回空() {
792        let html = parse_html("<html><head></head></html>");
793        assert!(extract_keywords(&html).is_empty());
794    }
795
796    // -----------------------------------------------------------------------
797    // 规范链接
798    // -----------------------------------------------------------------------
799
800    #[test]
801    fn 提取canonical_绝对路径() {
802        let html = parse_html(r#"<html><head><link rel="canonical" href="https://example.com/page"></head></html>"#);
803        assert_eq!(
804            extract_canonical(&html, None),
805            Some("https://example.com/page".to_string())
806        );
807    }
808
809    #[test]
810    fn 提取canonical_相对路径() {
811        let html = parse_html(r#"<html><head><link rel="canonical" href="/blog/post"></head></html>"#);
812        let base = test_base_url();
813        assert_eq!(
814            extract_canonical(&html, Some(&base)),
815            Some("https://example.com/blog/post".to_string())
816        );
817    }
818
819    // -----------------------------------------------------------------------
820    // Favicon
821    // -----------------------------------------------------------------------
822
823    #[test]
824    fn 提取favicon() {
825        let html = parse_html(r#"<html><head><link rel="icon" href="/favicon.ico"></head></html>"#);
826        let base = test_base_url();
827        assert_eq!(
828            extract_favicon(&html, Some(&base)),
829            Some("https://example.com/favicon.ico".to_string())
830        );
831    }
832
833    #[test]
834    fn 提取favicon_选最大尺寸() {
835        let html = parse_html(r#"
836            <html><head>
837                <link rel="icon" href="/small.png" sizes="16x16">
838                <link rel="icon" href="/large.png" sizes="64x64">
839            </head></html>
840        "#);
841        let base = test_base_url();
842        assert_eq!(
843            extract_favicon(&html, Some(&base)),
844            Some("https://example.com/large.png".to_string())
845        );
846    }
847
848    // -----------------------------------------------------------------------
849    // Apple Touch Icon
850    // -----------------------------------------------------------------------
851
852    #[test]
853    fn 提取apple_touch_icon() {
854        let html = parse_html(r#"<html><head><link rel="apple-touch-icon" href="/apple-icon.png"></head></html>"#);
855        let base = test_base_url();
856        assert_eq!(
857            extract_apple_touch_icon(&html, Some(&base)),
858            Some("https://example.com/apple-icon.png".to_string())
859        );
860    }
861
862    // -----------------------------------------------------------------------
863    // Robots
864    // -----------------------------------------------------------------------
865
866    #[test]
867    fn 提取robots_允许全部() {
868        let html = parse_html(r#"<html><head><meta name="robots" content="all"></head></html>"#);
869        let robots = extract_robots(&html);
870        assert!(robots.index);
871        assert!(robots.follow);
872    }
873
874    #[test]
875    fn 提取robots_noindex_nofollow() {
876        let html = parse_html(r#"<html><head><meta name="robots" content="noindex, nofollow"></head></html>"#);
877        let robots = extract_robots(&html);
878        assert!(!robots.index);
879        assert!(!robots.follow);
880    }
881
882    #[test]
883    fn 提取robots_无标签默认允许() {
884        let html = parse_html("<html><head></head></html>");
885        let robots = extract_robots(&html);
886        assert!(robots.index);
887        assert!(robots.follow);
888    }
889
890    #[test]
891    fn 提取robots_max_snippet() {
892        let html = parse_html(r#"<html><head><meta name="robots" content="max-snippet:50"></head></html>"#);
893        let robots = extract_robots(&html);
894        assert_eq!(robots.max_snippet, 50);
895    }
896
897    // -----------------------------------------------------------------------
898    // Open Graph
899    // -----------------------------------------------------------------------
900
901    #[test]
902    fn 提取opengraph_基本字段() {
903        let html = parse_html(r#"
904            <html><head>
905                <meta property="og:title" content="OG Title">
906                <meta property="og:description" content="OG Description">
907                <meta property="og:url" content="https://example.com">
908                <meta property="og:type" content="article">
909            </head></html>
910        "#);
911        let og = extract_opengraph(&html);
912        assert_eq!(og.title, Some("OG Title".to_string()));
913        assert_eq!(og.description, Some("OG Description".to_string()));
914        assert_eq!(og.url, Some("https://example.com".to_string()));
915        assert_eq!(og.og_type, Some("article".to_string()));
916        assert!(og.is_present());
917    }
918
919    #[test]
920    fn 提取opengraph_扩展字段() {
921        let html = parse_html(r#"
922            <html><head>
923                <meta property="og:title" content="Title">
924                <meta property="og:custom" content="value">
925            </head></html>
926        "#);
927        let og = extract_opengraph(&html);
928        assert_eq!(og.extra.get("og:custom"), Some(&"value".to_string()));
929    }
930
931    // -----------------------------------------------------------------------
932    // Twitter Card
933    // -----------------------------------------------------------------------
934
935    #[test]
936    fn 提取twitter_card_基本字段() {
937        let html = parse_html(r#"
938            <html><head>
939                <meta name="twitter:card" content="summary_large_image">
940                <meta name="twitter:site" content="@example">
941                <meta name="twitter:title" content="Tweet Title">
942            </head></html>
943        "#);
944        let card = extract_twitter_card(&html);
945        assert_eq!(card.card, Some("summary_large_image".to_string()));
946        assert_eq!(card.site, Some("@example".to_string()));
947        assert_eq!(card.title, Some("Tweet Title".to_string()));
948    }
949
950    // -----------------------------------------------------------------------
951    // 备用链接
952    // -----------------------------------------------------------------------
953
954    #[test]
955    fn 提取alternates() {
956        let html = parse_html(r#"
957            <html><head>
958                <link rel="alternate" hreflang="zh" href="/zh/">
959                <link rel="alternate" hreflang="en" href="/en/">
960            </head></html>
961        "#);
962        let base = test_base_url();
963        let alts = extract_alternates(&html, Some(&base));
964        assert_eq!(alts.len(), 2);
965        assert_eq!(alts[0].hreflang, "zh");
966        assert_eq!(alts[0].href, "https://example.com/zh/");
967        assert_eq!(alts[1].hreflang, "en");
968    }
969
970    // -----------------------------------------------------------------------
971    // JSON-LD 结构化数据
972    // -----------------------------------------------------------------------
973
974    #[test]
975    fn 提取json_ld() {
976        let html = parse_html(r#"
977            <html><head>
978                <script type="application/ld+json">
979                    {"@type": "WebPage", "name": "Test Page", "description": "A test"}
980                </script>
981            </head></html>
982        "#);
983        let data = extract_json_ld(&html);
984        assert_eq!(data.len(), 1);
985        assert_eq!(data[0].schema_type, Some("WebPage".to_string()));
986        assert!(data[0].raw_json.is_some());
987    }
988
989    // -----------------------------------------------------------------------
990    // 综合入口
991    // -----------------------------------------------------------------------
992
993    #[test]
994    fn extract_metadata_完整提取() {
995        let html = parse_html(r#"
996            <html lang="zh-CN">
997            <head>
998                <meta charset="utf-8">
999                <title>完整测试页面</title>
1000                <meta name="description" content="这是一个完整的测试页面">
1001                <meta name="keywords" content="测试, 元数据, 提取">
1002                <meta name="author" content="测试作者">
1003                <meta property="og:title" content="OG 标题">
1004                <meta property="og:type" content="website">
1005                <meta name="twitter:card" content="summary">
1006                <link rel="canonical" href="/canonical">
1007                <link rel="icon" href="/favicon.ico">
1008                <link rel="alternate" hreflang="en" href="/en/">
1009                <base href="https://example.com/blog/">
1010            </head>
1011            <body></body>
1012            </html>
1013        "#);
1014        let base = Url::parse("https://example.com").ok();
1015        let meta = extract_metadata(&html, base.as_ref()).unwrap();
1016        assert_eq!(meta.title, Some("OG 标题".to_string()));
1017        assert_eq!(meta.charset, Some("utf-8".to_string()));
1018        assert_eq!(meta.language, Some("zh-CN".to_string()));
1019        assert_eq!(meta.base_url, Some("https://example.com/blog/".to_string()));
1020        assert_eq!(meta.description, Some("这是一个完整的测试页面".to_string()));
1021        assert_eq!(meta.keywords, vec!["测试", "元数据", "提取"]);
1022        assert_eq!(meta.author, Some("测试作者".to_string()));
1023        assert_eq!(meta.canonical, Some("https://example.com/canonical".to_string()));
1024        assert!(meta.favicon.unwrap().contains("favicon.ico"));
1025        assert_eq!(meta.alternates.len(), 1);
1026    }
1027
1028    // -----------------------------------------------------------------------
1029    // 辅助函数
1030    // -----------------------------------------------------------------------
1031
1032    #[test]
1033    fn resolve_url_相对路径() {
1034        let base = Url::parse("https://example.com/path/").ok();
1035        assert_eq!(
1036            resolve_url(base.as_ref(), "/page.html"),
1037            Some("https://example.com/page.html".to_string())
1038        );
1039    }
1040
1041    #[test]
1042    fn resolve_url_绝对路径() {
1043        let base = Url::parse("https://example.com/path/").ok();
1044        assert_eq!(
1045            resolve_url(base.as_ref(), "https://other.com/page"),
1046            Some("https://other.com/page".to_string())
1047        );
1048    }
1049
1050    #[test]
1051    fn resolve_url_无效返回None() {
1052        let base = Url::parse("https://example.com").ok();
1053        assert_eq!(resolve_url(base.as_ref(), ""), None);
1054        assert_eq!(resolve_url(base.as_ref(), "data:image/png;base64,abc"), None);
1055        assert_eq!(resolve_url(base.as_ref(), "javascript:void(0)"), None);
1056    }
1057
1058    #[test]
1059    fn find_largest_icon_选择最大() {
1060        let candidates = vec![
1061            ("/small.png".to_string(), "16x16".to_string()),
1062            ("/medium.png".to_string(), "32x32".to_string()),
1063            ("/large.png".to_string(), "64x64".to_string()),
1064        ];
1065        assert_eq!(find_largest_icon(&candidates), "/large.png");
1066    }
1067
1068    #[test]
1069    fn find_largest_icon_无尺寸时选第一个() {
1070        let candidates = vec![
1071            ("/icon.png".to_string(), "".to_string()),
1072            ("/other.png".to_string(), "any".to_string()),
1073        ];
1074        assert_eq!(find_largest_icon(&candidates), "/icon.png");
1075    }
1076
1077    #[test]
1078    fn parse_icon_size_标准格式() {
1079        assert_eq!(parse_icon_size("32x32"), Some(32));
1080        assert_eq!(parse_icon_size("64x32"), Some(64));
1081    }
1082
1083    #[test]
1084    fn parse_icon_size_any返回None() {
1085        assert_eq!(parse_icon_size("any"), None);
1086        assert_eq!(parse_icon_size(""), None);
1087    }
1088
1089    #[test]
1090    fn extract_meta_content_by_name() {
1091        let html = parse_html(r#"<html><head><meta name="generator" content="crawlkit"></head></html>"#);
1092        assert_eq!(extract_meta_content(&html, "generator"), Some("crawlkit".to_string()));
1093    }
1094
1095    #[test]
1096    fn extract_meta_content_by_property() {
1097        let html = parse_html(r#"<html><head><meta property="article:section" content="tech"></head></html>"#);
1098        assert_eq!(extract_meta_content(&html, "article:section"), Some("tech".to_string()));
1099    }
1100
1101    #[test]
1102    fn extract_meta_content_无匹配返回None() {
1103        let html = parse_html("<html><head></head></html>");
1104        assert_eq!(extract_meta_content(&html, "nonexistent"), None);
1105    }
1106
1107    #[test]
1108    fn extract_custom_meta_过滤已知字段() {
1109        let html = parse_html(r#"
1110            <html><head>
1111                <meta name="description" content="skip me">
1112                <meta name="og:title" content="skip me too">
1113                <meta name="custom-field" content="keep me">
1114            </head></html>
1115        "#);
1116        let custom = extract_custom_meta(&html);
1117        assert_eq!(custom.len(), 1);
1118        assert_eq!(custom.get("custom-field"), Some(&"keep me".to_string()));
1119    }
1120
1121    #[test]
1122    fn extract_microdata_基本提取() {
1123        let html = parse_html(r#"
1124            <html><body>
1125                <div itemscope itemtype="http://schema.org/Person">
1126                    <span itemprop="name">John Doe</span>
1127                    <span itemprop="email">john@example.com</span>
1128                </div>
1129            </body></html>
1130        "#);
1131        let data = extract_microdata(&html);
1132        assert_eq!(data.len(), 1);
1133        assert_eq!(data[0].schema_type, Some("Person".to_string()));
1134        assert_eq!(
1135            data[0].properties.get("name"),
1136            Some(&serde_json::Value::String("John Doe".to_string()))
1137        );
1138    }
1139
1140    #[test]
1141    fn extract_og_property_存在() {
1142        let html = parse_html(r#"<html><head><meta property="og:image" content="https://example.com/img.jpg"></head></html>"#);
1143        assert_eq!(
1144            extract_og_property(&html, "og:image"),
1145            Some("https://example.com/img.jpg".to_string())
1146        );
1147    }
1148
1149    #[test]
1150    fn extract_og_property_不存在返回None() {
1151        let html = parse_html("<html><head></head></html>");
1152        assert_eq!(extract_og_property(&html, "og:image"), None);
1153    }
1154
1155    #[test]
1156    fn extract_twitter_property_存在() {
1157        let html = parse_html(r#"<html><head><meta name="twitter:image" content="https://example.com/img.jpg"></head></html>"#);
1158        assert_eq!(
1159            extract_twitter_property(&html, "twitter:image"),
1160            Some("https://example.com/img.jpg".to_string())
1161        );
1162    }
1163
1164    #[test]
1165    fn 提取robots_高级指令() {
1166        let html = parse_html(r#"<html><head><meta name="robots" content="noindex, max-image-preview:standard, max-video-preview:30"></head></html>"#);
1167        let robots = extract_robots(&html);
1168        assert!(!robots.index);
1169        assert_eq!(robots.max_image_preview, Some("standard".to_string()));
1170        assert_eq!(robots.max_video_preview, 30);
1171    }
1172
1173    #[test]
1174    fn extract_metadata_无base_url() {
1175        let html = parse_html(r#"
1176            <html><head>
1177                <title>No Base</title>
1178                <link rel="canonical" href="/page">
1179            </head></html>
1180        "#);
1181        let meta = extract_metadata(&html, None).unwrap();
1182        assert_eq!(meta.title, Some("No Base".to_string()));
1183    }
1184
1185    #[test]
1186    fn extract_metadata_错误处理_无效选择器不崩溃() {
1187        let html = parse_html("<html><head><title>Safe</title></head></html>");
1188        let meta = extract_metadata(&html, None).unwrap();
1189        assert_eq!(meta.title, Some("Safe".to_string()));
1190    }
1191}
crawlkit_parser/metadata.rs

crawlkit_parser/
metadata.rs
