crawlkit_parser/
metadata.rs

1//! 页面元数据提取模块
2//!
3//! 从 HTML 文档中提取标题、描述、关键词、Open Graph、Twitter Card、
4//! 结构化数据等各类元数据。改编自 halldyll-parser。
5
6use scraper::{Html, ElementRef};
7use std::collections::HashMap;
8use url::Url;
9
10use crate::selector::{
11    SELECTORS, try_parse_selector, meta_name_selector, meta_property_selector, link_rel_selector,
12};
13use crate::types::{
14    PageMetadata, OpenGraph, TwitterCard, RobotsMeta, AlternateLink,
15    StructuredData, ParserResult,
16};
17
18// ============================================================================
19// 主入口
20// ============================================================================
21
22/// 从 HTML 文档中提取完整页面元数据
23pub fn extract_metadata(document: &Html, base_url: Option<&Url>) -> ParserResult<PageMetadata> {
24    let title = extract_title(document);
25    let charset = extract_charset(document);
26    let language = extract_language(document);
27    let base = extract_base_url(document);
28    let description = extract_meta_content(document, "description");
29    let author = extract_meta_content(document, "author");
30    let generator = extract_meta_content(document, "generator");
31    let viewport = extract_meta_content(document, "viewport");
32    let theme_color = extract_meta_content(document, "theme-color");
33    let published_date = extract_meta_content(document, "date")
34        .or_else(|| extract_meta_content(document, "article:published_time"));
35    let modified_date = extract_meta_content(document, "article:modified_time");
36    let keywords = extract_keywords(document);
37    let canonical = extract_canonical(document, base_url);
38    let favicon = extract_favicon(document, base_url);
39    let apple_touch = extract_apple_touch_icon(document, base_url);
40    let robots = extract_robots(document);
41    let opengraph = extract_opengraph(document);
42    let twitter = extract_twitter_card(document);
43    let alternates = extract_alternates(document, base_url);
44    let structured = extract_structured_data(document);
45    let schema_type = structured.first()
46        .and_then(|s| s.schema_type.clone());
47    let custom = extract_custom_meta(document);
48
49    Ok(PageMetadata {
50        title,
51        description,
52        keywords,
53        author,
54        generator,
55        canonical,
56        base_url: base,
57        language,
58        charset,
59        viewport,
60        robots,
61        opengraph,
62        twitter,
63        alternates,
64        favicon,
65        apple_touch_icon: apple_touch,
66        theme_color,
67        published_date,
68        modified_date,
69        schema_type,
70        custom,
71    })
72}
73
74// ============================================================================
75// 基本元数据
76// ============================================================================
77
78/// 提取页面标题：依次尝试 og:title、twitter:title、`<title>`、h1
79pub fn extract_title(document: &Html) -> Option<String> {
80    if let Some(sel) = try_parse_selector(&meta_property_selector("og:title"))
81        && let Some(el) = document.select(&sel).next()
82            && let Some(content) = el.value().attr("content") {
83                let text = content.trim().to_string();
84                if !text.is_empty() {
85                    return Some(text);
86                }
87            }
88    if let Some(sel) = try_parse_selector(&meta_name_selector("twitter:title"))
89        && let Some(el) = document.select(&sel).next()
90            && let Some(content) = el.value().attr("content") {
91                let text = content.trim().to_string();
92                if !text.is_empty() {
93                    return Some(text);
94                }
95            }
96    let title_sel = &SELECTORS.title;
97    if let Some(el) = document.select(title_sel).next() {
98        let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
99        if !text.is_empty() {
100            return Some(text);
101        }
102    }
103    if let Some(sel) = try_parse_selector("h1")
104        && let Some(el) = document.select(&sel).next() {
105            let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
106            if !text.is_empty() {
107                return Some(text);
108            }
109        }
110    None
111}
112
113/// 提取字符编码声明
114pub fn extract_charset(document: &Html) -> Option<String> {
115    if let Some(sel) = try_parse_selector("meta[charset]")
116        && let Some(el) = document.select(&sel).next()
117            && let Some(charset) = el.value().attr("charset") {
118                let val = charset.trim().to_string();
119                if !val.is_empty() {
120                    return Some(val);
121                }
122            }
123    if let Some(sel) = try_parse_selector(r#"meta[http-equiv="Content-Type"]"#)
124        && let Some(el) = document.select(&sel).next()
125            && let Some(content) = el.value().attr("content")
126                && let Some(pos) = content.to_lowercase().find("charset=") {
127                    let charset = content[pos + 8..].trim().to_string();
128                    if !charset.is_empty() {
129                        return Some(charset);
130                    }
131                }
132    None
133}
134
135/// 提取页面语言：优先 `<html lang>`，其次 `http-equiv="content-language"`
136pub fn extract_language(document: &Html) -> Option<String> {
137    let html_sel = &SELECTORS.html;
138    if let Some(el) = document.select(html_sel).next()
139        && let Some(lang) = el.value().attr("lang") {
140            let val = lang.trim().to_string();
141            if !val.is_empty() {
142                return Some(val);
143            }
144        }
145    if let Some(sel) = try_parse_selector(r#"meta[http-equiv="content-language"]"#)
146        && let Some(el) = document.select(&sel).next()
147            && let Some(content) = el.value().attr("content") {
148                let val = content.trim().to_string();
149                if !val.is_empty() {
150                    return Some(val);
151                }
152            }
153    None
154}
155
156/// 提取 `<base>` 标签的 href 值
157pub fn extract_base_url(document: &Html) -> Option<String> {
158    let base_sel = &SELECTORS.base;
159    if let Some(el) = document.select(base_sel).next()
160        && let Some(href) = el.value().attr("href") {
161            let val = href.trim().to_string();
162            if !val.is_empty() {
163                return Some(val);
164            }
165        }
166    None
167}
168
169/// 通用 meta 内容提取，同时尝试 `name` 和 `property` 属性
170pub fn extract_meta_content(document: &Html, name: &str) -> Option<String> {
171    if let Some(sel) = try_parse_selector(&meta_name_selector(name))
172        && let Some(el) = document.select(&sel).next()
173            && let Some(content) = el.value().attr("content") {
174                let val = content.trim().to_string();
175                if !val.is_empty() {
176                    return Some(val);
177                }
178            }
179    if let Some(sel) = try_parse_selector(&meta_property_selector(name))
180        && let Some(el) = document.select(&sel).next()
181            && let Some(content) = el.value().attr("content") {
182                let val = content.trim().to_string();
183                if !val.is_empty() {
184                    return Some(val);
185                }
186            }
187    None
188}
189
190/// 提取关键词：从 `<meta name="keywords">` 解析，逗号分隔
191pub fn extract_keywords(document: &Html) -> Vec<String> {
192    if let Some(sel) = try_parse_selector(&meta_name_selector("keywords"))
193        && let Some(el) = document.select(&sel).next()
194            && let Some(content) = el.value().attr("content") {
195                return content
196                    .split(',')
197                    .map(|s| s.trim().to_string())
198                    .filter(|s| !s.is_empty())
199                    .collect();
200            }
201    Vec::new()
202}
203
204// ============================================================================
205// 链接资源
206// ============================================================================
207
208/// 提取规范链接 `<link rel="canonical">`
209pub fn extract_canonical(document: &Html, base_url: Option<&Url>) -> Option<String> {
210    if let Some(sel) = try_parse_selector(&link_rel_selector("canonical"))
211        && let Some(el) = document.select(&sel).next()
212            && let Some(href) = el.value().attr("href") {
213                return resolve_url(base_url, href);
214            }
215    None
216}
217
218/// 提取 favicon：优先选择尺寸最大的图标
219pub fn extract_favicon(document: &Html, base_url: Option<&Url>) -> Option<String> {
220    if let Some(sel) = try_parse_selector("link[rel~='icon'], link[rel='shortcut icon'], link[rel='apple-touch-icon-precomposed']") {
221        let mut candidates: Vec<(String, String)> = Vec::new();
222        for el in document.select(&sel) {
223            if let Some(href) = el.value().attr("href") {
224                let sizes = el.value().attr("sizes").unwrap_or("").to_string();
225                candidates.push((href.to_string(), sizes));
226            }
227        }
228        if candidates.is_empty() {
229            return None;
230        }
231        let best = find_largest_icon(&candidates);
232        resolve_url(base_url, &best)
233    } else {
234        None
235    }
236}
237
238/// 提取苹果触控图标
239pub fn extract_apple_touch_icon(document: &Html, base_url: Option<&Url>) -> Option<String> {
240    if let Some(sel) = try_parse_selector("link[rel='apple-touch-icon'], link[rel='apple-touch-icon-precomposed']")
241        && let Some(el) = document.select(&sel).next()
242            && let Some(href) = el.value().attr("href") {
243                return resolve_url(base_url, href);
244            }
245    None
246}
247
248// ============================================================================
249// 爬虫规则（robots）
250// ============================================================================
251
252/// 提取 robots meta 标签，解析指令
253pub fn extract_robots(document: &Html) -> RobotsMeta {
254    let content = if let Some(sel) = try_parse_selector("meta[name='robots'], meta[name='ROBOTS']") {
255        document.select(&sel).next()
256            .and_then(|el| el.value().attr("content").map(std::string::ToString::to_string))
257    } else {
258        None
259    };
260
261    let directives = match content {
262        Some(ref c) => c.split(',').map(|s| s.trim().to_lowercase()).collect::<Vec<_>>(),
263        None => return RobotsMeta::allowed(),
264    };
265
266    let mut meta = RobotsMeta::allowed();
267    meta.raw = content;
268
269    for directive in &directives {
270        match directive.as_str() {
271            "noindex" => meta.index = false,
272            "nofollow" => meta.follow = false,
273            "noarchive" => meta.archive = false,
274            "nocache" => meta.cache = false,
275            "nosnippet" => meta.snippet = false,
276            "all" | "index" | "follow" => {}
277            v if v.starts_with("max-snippet:") => {
278                let val = v.trim_start_matches("max-snippet:").trim();
279                meta.max_snippet = val.parse().unwrap_or(-1);
280            }
281            v if v.starts_with("max-image-preview:") => {
282                let val = v.trim_start_matches("max-image-preview:").trim();
283                meta.max_image_preview = Some(val.to_string());
284            }
285            v if v.starts_with("max-video-preview:") => {
286                let val = v.trim_start_matches("max-video-preview:").trim();
287                meta.max_video_preview = val.parse().unwrap_or(-1);
288            }
289            _ => {}
290        }
291    }
292
293    meta
294}
295
296// ============================================================================
297// Open Graph
298// ============================================================================
299
300/// 提取 Open Graph 元数据
301pub fn extract_opengraph(document: &Html) -> OpenGraph {
302    let mut og = OpenGraph::default();
303    if let Some(title) = extract_og_property(document, "og:title") {
304        og.title = Some(title);
305    }
306    if let Some(og_type) = extract_og_property(document, "og:type") {
307        og.og_type = Some(og_type);
308    }
309    if let Some(url) = extract_og_property(document, "og:url") {
310        og.url = Some(url);
311    }
312    if let Some(image) = extract_og_property(document, "og:image") {
313        og.image = Some(image);
314    }
315    if let Some(desc) = extract_og_property(document, "og:description") {
316        og.description = Some(desc);
317    }
318    if let Some(site) = extract_og_property(document, "og:site_name") {
319        og.site_name = Some(site);
320    }
321    if let Some(locale) = extract_og_property(document, "og:locale") {
322        og.locale = Some(locale);
323    }
324    if let Some(video) = extract_og_property(document, "og:video") {
325        og.video = Some(video);
326    }
327    if let Some(audio) = extract_og_property(document, "og:audio") {
328        og.audio = Some(audio);
329    }
330    og.extra = extract_all_og_properties(document);
331    og
332}
333
334/// 提取单个 Open Graph 属性值
335pub fn extract_og_property(document: &Html, property: &str) -> Option<String> {
336    if let Some(sel) = try_parse_selector(&meta_property_selector(property))
337        && let Some(el) = document.select(&sel).next()
338            && let Some(content) = el.value().attr("content") {
339                let val = content.trim().to_string();
340                if !val.is_empty() {
341                    return Some(val);
342                }
343            }
344    None
345}
346
347/// 提取所有 Open Graph 属性到 HashMap（不含标准字段）
348pub fn extract_all_og_properties(document: &Html) -> HashMap<String, String> {
349    let mut extras = HashMap::new();
350    let meta_sel = &SELECTORS.meta;
351    let known = [
352        "og:title", "og:type", "og:url", "og:image", "og:description",
353        "og:site_name", "og:locale", "og:video", "og:audio",
354    ];
355    for el in document.select(meta_sel) {
356        if let Some(property) = el.value().attr("property") {
357            let prop_lower = property.to_lowercase();
358            if prop_lower.starts_with("og:") && !known.contains(&prop_lower.as_str())
359                && let Some(content) = el.value().attr("content") {
360                    extras.insert(property.to_string(), content.to_string());
361                }
362        }
363    }
364    extras
365}
366
367// ============================================================================
368// Twitter Card
369// ============================================================================
370
371/// 提取 Twitter Card 元数据
372pub fn extract_twitter_card(document: &Html) -> TwitterCard {
373    let mut card = TwitterCard::default();
374    if let Some(val) = extract_twitter_property(document, "twitter:card") {
375        card.card = Some(val);
376    }
377    if let Some(val) = extract_twitter_property(document, "twitter:site") {
378        card.site = Some(val);
379    }
380    if let Some(val) = extract_twitter_property(document, "twitter:creator") {
381        card.creator = Some(val);
382    }
383    if let Some(val) = extract_twitter_property(document, "twitter:title") {
384        card.title = Some(val);
385    }
386    if let Some(val) = extract_twitter_property(document, "twitter:description") {
387        card.description = Some(val);
388    }
389    if let Some(val) = extract_twitter_property(document, "twitter:image") {
390        card.image = Some(val);
391    }
392    card.extra = extract_all_twitter_properties(document);
393    card
394}
395
396/// 提取单个 Twitter Card 属性值
397pub fn extract_twitter_property(document: &Html, name: &str) -> Option<String> {
398    if let Some(sel) = try_parse_selector(&meta_name_selector(name))
399        && let Some(el) = document.select(&sel).next()
400            && let Some(content) = el.value().attr("content") {
401                let val = content.trim().to_string();
402                if !val.is_empty() {
403                    return Some(val);
404                }
405            }
406    None
407}
408
409/// 提取所有 Twitter Card 属性到 HashMap（不含标准字段）
410pub fn extract_all_twitter_properties(document: &Html) -> HashMap<String, String> {
411    let mut extras = HashMap::new();
412    let meta_sel = &SELECTORS.meta;
413    let known = [
414        "twitter:card", "twitter:site", "twitter:creator",
415        "twitter:title", "twitter:description", "twitter:image",
416    ];
417    for el in document.select(meta_sel) {
418        if let Some(name) = el.value().attr("name") {
419            let name_lower = name.to_lowercase();
420            if name_lower.starts_with("twitter:") && !known.contains(&name_lower.as_str())
421                && let Some(content) = el.value().attr("content") {
422                    extras.insert(name.to_string(), content.to_string());
423                }
424        }
425    }
426    extras
427}
428
429// ============================================================================
430// 备用链接
431// ============================================================================
432
433/// 提取 `<link rel="alternate">` 标签（如多语言版本、RSS 等）
434pub fn extract_alternates(document: &Html, base_url: Option<&Url>) -> Vec<AlternateLink> {
435    let mut alternates = Vec::new();
436    if let Some(sel) = try_parse_selector(&link_rel_selector("alternate")) {
437        for el in document.select(&sel) {
438            let hreflang = el.value().attr("hreflang")
439                .unwrap_or("")
440                .trim()
441                .to_string();
442            let href = el.value().attr("href")
443                .unwrap_or("")
444                .trim()
445                .to_string();
446            if href.is_empty() {
447                continue;
448            }
449            let resolved = resolve_url(base_url, &href).unwrap_or(href);
450            alternates.push(AlternateLink { hreflang, href: resolved });
451        }
452    }
453    alternates
454}
455
456// ============================================================================
457// 结构化数据
458// ============================================================================
459
460/// 提取全部结构化数据（JSON-LD + Microdata）
461pub fn extract_structured_data(document: &Html) -> Vec<StructuredData> {
462    let mut results = Vec::new();
463    results.extend(extract_json_ld(document));
464    results.extend(extract_microdata(document));
465    results
466}
467
468/// 提取 JSON-LD 结构化数据
469pub fn extract_json_ld(document: &Html) -> Vec<StructuredData> {
470    let mut results = Vec::new();
471    let json_ld_sel = &SELECTORS.json_ld;
472    for el in document.select(json_ld_sel) {
473        let raw = el.text().collect::<Vec<_>>().join("").trim().to_string();
474        if raw.is_empty() {
475            continue;
476        }
477        let mut sd = StructuredData::json_ld(&raw);
478        if let Ok(val) = serde_json::from_str::<serde_json::Value>(&raw) {
479            if let Some(obj) = val.as_object() {
480                for (k, v) in obj {
481                    sd.properties.insert(k.clone(), v.clone());
482                }
483            }
484            if let Some(typ) = val.get("@type").and_then(|t| t.as_str()) {
485                sd.schema_type = Some(typ.to_string());
486            }
487            if let Some(typ) = val.get("type").and_then(|t| t.as_str()) {
488                sd.schema_type = Some(typ.to_string());
489            }
490        }
491        results.push(sd);
492    }
493    results
494}
495
496/// 提取 Microdata 结构化数据
497pub fn extract_microdata(document: &Html) -> Vec<StructuredData> {
498    let mut results = Vec::new();
499    if let Some(sel) = try_parse_selector("*[itemscope]") {
500        for el in document.select(&sel) {
501            let itemtype = el.value().attr("itemtype")
502                .and_then(|t| t.rsplit('/').next())
503                .or_else(|| el.value().attr("itemtype"))
504                .map(std::string::ToString::to_string);
505
506            let mut sd = match itemtype {
507                Some(ref t) => StructuredData::microdata(t),
508                None => continue,
509            };
510
511            let mut props = HashMap::new();
512            extract_microdata_properties(&el, &mut props);
513            sd.properties = props;
514            results.push(sd);
515        }
516    }
517    results
518}
519
520/// 递归提取一个 Microdata 元素的所有属性
521fn extract_microdata_properties(element: &ElementRef, props: &mut HashMap<String, serde_json::Value>) {
522    if let Some(sel) = try_parse_selector("*[itemprop]") {
523        for child in element.select(&sel) {
524            if child.value().attr("itemscope").is_some() {
525                continue;
526            }
527            if let Some(prop_name) = child.value().attr("itemprop") {
528                let value = get_microdata_value(&child);
529                props.insert(prop_name.to_string(), serde_json::Value::String(value));
530            }
531        }
532    }
533}
534
535/// 获取 Microdata 元素的值（按优先级：content → src → href → text）
536fn get_microdata_value(element: &ElementRef) -> String {
537    if let Some(content) = element.value().attr("content") {
538        return content.trim().to_string();
539    }
540    if let Some(src) = element.value().attr("src") {
541        return src.trim().to_string();
542    }
543    if let Some(href) = element.value().attr("href") {
544        return href.trim().to_string();
545    }
546    element.text().collect::<Vec<_>>().join("").trim().to_string()
547}
548
549// ============================================================================
550// 自定义元数据
551// ============================================================================
552
553/// 提取所有未归类到其他方法的自定义 meta 标签
554pub fn extract_custom_meta(document: &Html) -> HashMap<String, String> {
555    let mut custom = HashMap::new();
556    let meta_sel = &SELECTORS.meta;
557    let skip_names = [
558        "description", "keywords", "author", "generator", "viewport",
559        "theme-color", "robots", "googlebot", "date",
560    ];
561    let skip_prefixes = ["og:", "twitter:", "article:", "al:"];
562    for el in document.select(meta_sel) {
563        let name = el.value().attr("name")
564            .or_else(|| el.value().attr("property"))
565            .map(std::string::ToString::to_string);
566        let content = el.value().attr("content")
567            .map(std::string::ToString::to_string);
568        match (name, content) {
569            (Some(n), Some(c)) if !n.is_empty() && !c.is_empty() => {
570                let n_lower = n.to_lowercase();
571                if skip_names.contains(&n_lower.as_str()) {
572                    continue;
573                }
574                if skip_prefixes.iter().any(|p| n_lower.starts_with(p)) {
575                    continue;
576                }
577                custom.insert(n, c);
578            }
579            _ => {}
580        }
581    }
582    custom
583}
584
585// ============================================================================
586// 内部辅助函数
587// ============================================================================
588
589/// 将相对 URL 解析为绝对 URL，base_url 为 None 时直接返回原始值
590fn resolve_url(base_url: Option<&Url>, url_str: &str) -> Option<String> {
591    let trimmed = url_str.trim();
592    if trimmed.is_empty() || trimmed.starts_with("data:") || trimmed.starts_with("javascript:") {
593        return None;
594    }
595    match base_url {
596        Some(base) => base.join(trimmed).ok().map(|u| u.to_string()),
597        None => Some(trimmed.to_string()),
598    }
599}
600
601/// 从候选图标列表中找出尺寸最大的一个
602fn find_largest_icon(candidates: &[(String, String)]) -> String {
603    let mut best: Option<&str> = None;
604    let mut best_size: i32 = -1;
605
606    for (href, sizes) in candidates {
607        if let Some(dim) = parse_icon_size(sizes) {
608            if dim > best_size {
609                best_size = dim;
610                best = Some(href);
611            }
612        } else if best.is_none() {
613            best = Some(href);
614        }
615    }
616
617    best.unwrap_or_else(|| candidates[0].0.as_str()).to_string()
618}
619
620/// 解析 sizes 属性（如 "32x32"）返回最大边长
621fn parse_icon_size(sizes: &str) -> Option<i32> {
622    let sizes = sizes.trim();
623    if sizes.is_empty() || sizes.eq_ignore_ascii_case("any") {
624        return None;
625    }
626    sizes.split_whitespace()
627        .filter_map(|s| {
628            let parts: Vec<&str> = s.split('x').collect();
629            if parts.len() == 2 {
630                let w = parts[0].parse::<i32>().ok()?;
631                let h = parts[1].parse::<i32>().ok()?;
632                Some(w.max(h))
633            } else {
634                None
635            }
636        })
637        .max()
638}
639
640// ============================================================================
641// 测试
642// ============================================================================
643
644#[cfg(test)]
645mod tests {
646    use super::*;
647
648    /// 解析 HTML 字符串为 `Html` 文档
649    fn parse_html(html: &str) -> Html {
650        Html::parse_document(html)
651    }
652
653    /// 创建测试用 base URL
654    fn test_base_url() -> Url {
655        Url::parse("https://example.com").unwrap()
656    }
657
658    // -----------------------------------------------------------------------
659    // 标题提取
660    // -----------------------------------------------------------------------
661
662    #[test]
663    fn 提取标题_og_title优先() {
664        let html = parse_html(r#"
665            <html><head>
666                <meta property="og:title" content="OG Title">
667                <title>Page Title</title>
668            </head></html>
669        "#);
670        assert_eq!(extract_title(&html), Some("OG Title".to_string()));
671    }
672
673    #[test]
674    fn 提取标题_twitter_title其次() {
675        let html = parse_html(r#"
676            <html><head>
677                <meta name="twitter:title" content="Twitter Title">
678                <title>Page Title</title>
679            </head></html>
680        "#);
681        assert_eq!(extract_title(&html), Some("Twitter Title".to_string()));
682    }
683
684    #[test]
685    fn 提取标题_title标签兜底() {
686        let html = parse_html("<html><head><title>Page Title</title></head></html>");
687        assert_eq!(extract_title(&html), Some("Page Title".to_string()));
688    }
689
690    #[test]
691    fn 提取标题_h1最后兜底() {
692        let html = parse_html("<html><body><h1>H1 Title</h1></body></html>");
693        assert_eq!(extract_title(&html), Some("H1 Title".to_string()));
694    }
695
696    #[test]
697    fn 提取标题_无标题返回None() {
698        let html = parse_html("<html><head></head><body><p>no title</p></body></html>");
699        assert_eq!(extract_title(&html), None);
700    }
701
702    // -----------------------------------------------------------------------
703    // 字符编码
704    // -----------------------------------------------------------------------
705
706    #[test]
707    fn 提取charset_meta标签() {
708        let html = parse_html(r#"<html><head><meta charset="utf-8"></head></html>"#);
709        assert_eq!(extract_charset(&html), Some("utf-8".to_string()));
710    }
711
712    #[test]
713    fn 提取charset_http_equiv() {
714        let html = parse_html(r#"<html><head><meta http-equiv="Content-Type" content="text/html; charset=gb2312"></head></html>"#);
715        assert_eq!(extract_charset(&html), Some("gb2312".to_string()));
716    }
717
718    #[test]
719    fn 提取charset_无声明返回None() {
720        let html = parse_html("<html><head></head></html>");
721        assert_eq!(extract_charset(&html), None);
722    }
723
724    // -----------------------------------------------------------------------
725    // 语言
726    // -----------------------------------------------------------------------
727
728    #[test]
729    fn 提取语言_html_lang属性() {
730        let html = parse_html(r#"<html lang="en"><head></head></html>"#);
731        assert_eq!(extract_language(&html), Some("en".to_string()));
732    }
733
734    #[test]
735    fn 提取语言_http_equiv() {
736        let html = parse_html(r#"<html><head><meta http-equiv="content-language" content="zh-CN"></head></html>"#);
737        assert_eq!(extract_language(&html), Some("zh-CN".to_string()));
738    }
739
740    // -----------------------------------------------------------------------
741    // base URL
742    // -----------------------------------------------------------------------
743
744    #[test]
745    fn 提取base_url() {
746        let html = parse_html(r#"<html><head><base href="https://example.com/base/"></head></html>"#);
747        assert_eq!(extract_base_url(&html), Some("https://example.com/base/".to_string()));
748    }
749
750    // -----------------------------------------------------------------------
751    // 关键词
752    // -----------------------------------------------------------------------
753
754    #[test]
755    fn 提取关键词_逗号分隔() {
756        let html = parse_html(r#"<html><head><meta name="keywords" content="rust, web, scraping"></head></html>"#);
757        assert_eq!(extract_keywords(&html), vec!["rust", "web", "scraping"]);
758    }
759
760    #[test]
761    fn 提取关键词_无关键词返回空() {
762        let html = parse_html("<html><head></head></html>");
763        assert!(extract_keywords(&html).is_empty());
764    }
765
766    // -----------------------------------------------------------------------
767    // 规范链接
768    // -----------------------------------------------------------------------
769
770    #[test]
771    fn 提取canonical_绝对路径() {
772        let html = parse_html(r#"<html><head><link rel="canonical" href="https://example.com/page"></head></html>"#);
773        assert_eq!(
774            extract_canonical(&html, None),
775            Some("https://example.com/page".to_string())
776        );
777    }
778
779    #[test]
780    fn 提取canonical_相对路径() {
781        let html = parse_html(r#"<html><head><link rel="canonical" href="/blog/post"></head></html>"#);
782        let base = test_base_url();
783        assert_eq!(
784            extract_canonical(&html, Some(&base)),
785            Some("https://example.com/blog/post".to_string())
786        );
787    }
788
789    // -----------------------------------------------------------------------
790    // Favicon
791    // -----------------------------------------------------------------------
792
793    #[test]
794    fn 提取favicon() {
795        let html = parse_html(r#"<html><head><link rel="icon" href="/favicon.ico"></head></html>"#);
796        let base = test_base_url();
797        assert_eq!(
798            extract_favicon(&html, Some(&base)),
799            Some("https://example.com/favicon.ico".to_string())
800        );
801    }
802
803    #[test]
804    fn 提取favicon_选最大尺寸() {
805        let html = parse_html(r#"
806            <html><head>
807                <link rel="icon" href="/small.png" sizes="16x16">
808                <link rel="icon" href="/large.png" sizes="64x64">
809            </head></html>
810        "#);
811        let base = test_base_url();
812        assert_eq!(
813            extract_favicon(&html, Some(&base)),
814            Some("https://example.com/large.png".to_string())
815        );
816    }
817
818    // -----------------------------------------------------------------------
819    // Apple Touch Icon
820    // -----------------------------------------------------------------------
821
822    #[test]
823    fn 提取apple_touch_icon() {
824        let html = parse_html(r#"<html><head><link rel="apple-touch-icon" href="/apple-icon.png"></head></html>"#);
825        let base = test_base_url();
826        assert_eq!(
827            extract_apple_touch_icon(&html, Some(&base)),
828            Some("https://example.com/apple-icon.png".to_string())
829        );
830    }
831
832    // -----------------------------------------------------------------------
833    // Robots
834    // -----------------------------------------------------------------------
835
836    #[test]
837    fn 提取robots_允许全部() {
838        let html = parse_html(r#"<html><head><meta name="robots" content="all"></head></html>"#);
839        let robots = extract_robots(&html);
840        assert!(robots.index);
841        assert!(robots.follow);
842    }
843
844    #[test]
845    fn 提取robots_noindex_nofollow() {
846        let html = parse_html(r#"<html><head><meta name="robots" content="noindex, nofollow"></head></html>"#);
847        let robots = extract_robots(&html);
848        assert!(!robots.index);
849        assert!(!robots.follow);
850    }
851
852    #[test]
853    fn 提取robots_无标签默认允许() {
854        let html = parse_html("<html><head></head></html>");
855        let robots = extract_robots(&html);
856        assert!(robots.index);
857        assert!(robots.follow);
858    }
859
860    #[test]
861    fn 提取robots_max_snippet() {
862        let html = parse_html(r#"<html><head><meta name="robots" content="max-snippet:50"></head></html>"#);
863        let robots = extract_robots(&html);
864        assert_eq!(robots.max_snippet, 50);
865    }
866
867    // -----------------------------------------------------------------------
868    // Open Graph
869    // -----------------------------------------------------------------------
870
871    #[test]
872    fn 提取opengraph_基本字段() {
873        let html = parse_html(r#"
874            <html><head>
875                <meta property="og:title" content="OG Title">
876                <meta property="og:description" content="OG Description">
877                <meta property="og:url" content="https://example.com">
878                <meta property="og:type" content="article">
879            </head></html>
880        "#);
881        let og = extract_opengraph(&html);
882        assert_eq!(og.title, Some("OG Title".to_string()));
883        assert_eq!(og.description, Some("OG Description".to_string()));
884        assert_eq!(og.url, Some("https://example.com".to_string()));
885        assert_eq!(og.og_type, Some("article".to_string()));
886        assert!(og.is_present());
887    }
888
889    #[test]
890    fn 提取opengraph_扩展字段() {
891        let html = parse_html(r#"
892            <html><head>
893                <meta property="og:title" content="Title">
894                <meta property="og:custom" content="value">
895            </head></html>
896        "#);
897        let og = extract_opengraph(&html);
898        assert_eq!(og.extra.get("og:custom"), Some(&"value".to_string()));
899    }
900
901    // -----------------------------------------------------------------------
902    // Twitter Card
903    // -----------------------------------------------------------------------
904
905    #[test]
906    fn 提取twitter_card_基本字段() {
907        let html = parse_html(r#"
908            <html><head>
909                <meta name="twitter:card" content="summary_large_image">
910                <meta name="twitter:site" content="@example">
911                <meta name="twitter:title" content="Tweet Title">
912            </head></html>
913        "#);
914        let card = extract_twitter_card(&html);
915        assert_eq!(card.card, Some("summary_large_image".to_string()));
916        assert_eq!(card.site, Some("@example".to_string()));
917        assert_eq!(card.title, Some("Tweet Title".to_string()));
918    }
919
920    // -----------------------------------------------------------------------
921    // 备用链接
922    // -----------------------------------------------------------------------
923
924    #[test]
925    fn 提取alternates() {
926        let html = parse_html(r#"
927            <html><head>
928                <link rel="alternate" hreflang="zh" href="/zh/">
929                <link rel="alternate" hreflang="en" href="/en/">
930            </head></html>
931        "#);
932        let base = test_base_url();
933        let alts = extract_alternates(&html, Some(&base));
934        assert_eq!(alts.len(), 2);
935        assert_eq!(alts[0].hreflang, "zh");
936        assert_eq!(alts[0].href, "https://example.com/zh/");
937        assert_eq!(alts[1].hreflang, "en");
938    }
939
940    // -----------------------------------------------------------------------
941    // JSON-LD 结构化数据
942    // -----------------------------------------------------------------------
943
944    #[test]
945    fn 提取json_ld() {
946        let html = parse_html(r#"
947            <html><head>
948                <script type="application/ld+json">
949                    {"@type": "WebPage", "name": "Test Page", "description": "A test"}
950                </script>
951            </head></html>
952        "#);
953        let data = extract_json_ld(&html);
954        assert_eq!(data.len(), 1);
955        assert_eq!(data[0].schema_type, Some("WebPage".to_string()));
956        assert!(data[0].raw_json.is_some());
957    }
958
959    // -----------------------------------------------------------------------
960    // 综合入口
961    // -----------------------------------------------------------------------
962
963    #[test]
964    fn extract_metadata_完整提取() {
965        let html = parse_html(r#"
966            <html lang="zh-CN">
967            <head>
968                <meta charset="utf-8">
969                <title>完整测试页面</title>
970                <meta name="description" content="这是一个完整的测试页面">
971                <meta name="keywords" content="测试, 元数据, 提取">
972                <meta name="author" content="测试作者">
973                <meta property="og:title" content="OG 标题">
974                <meta property="og:type" content="website">
975                <meta name="twitter:card" content="summary">
976                <link rel="canonical" href="/canonical">
977                <link rel="icon" href="/favicon.ico">
978                <link rel="alternate" hreflang="en" href="/en/">
979                <base href="https://example.com/blog/">
980            </head>
981            <body></body>
982            </html>
983        "#);
984        let base = Url::parse("https://example.com").ok();
985        let meta = extract_metadata(&html, base.as_ref()).unwrap();
986        assert_eq!(meta.title, Some("OG 标题".to_string()));
987        assert_eq!(meta.charset, Some("utf-8".to_string()));
988        assert_eq!(meta.language, Some("zh-CN".to_string()));
989        assert_eq!(meta.base_url, Some("https://example.com/blog/".to_string()));
990        assert_eq!(meta.description, Some("这是一个完整的测试页面".to_string()));
991        assert_eq!(meta.keywords, vec!["测试", "元数据", "提取"]);
992        assert_eq!(meta.author, Some("测试作者".to_string()));
993        assert_eq!(meta.canonical, Some("https://example.com/canonical".to_string()));
994        assert!(meta.favicon.unwrap().contains("favicon.ico"));
995        assert_eq!(meta.alternates.len(), 1);
996    }
997
998    // -----------------------------------------------------------------------
999    // 辅助函数
1000    // -----------------------------------------------------------------------
1001
1002    #[test]
1003    fn resolve_url_相对路径() {
1004        let base = Url::parse("https://example.com/path/").ok();
1005        assert_eq!(
1006            resolve_url(base.as_ref(), "/page.html"),
1007            Some("https://example.com/page.html".to_string())
1008        );
1009    }
1010
1011    #[test]
1012    fn resolve_url_绝对路径() {
1013        let base = Url::parse("https://example.com/path/").ok();
1014        assert_eq!(
1015            resolve_url(base.as_ref(), "https://other.com/page"),
1016            Some("https://other.com/page".to_string())
1017        );
1018    }
1019
1020    #[test]
1021    fn resolve_url_无效返回None() {
1022        let base = Url::parse("https://example.com").ok();
1023        assert_eq!(resolve_url(base.as_ref(), ""), None);
1024        assert_eq!(resolve_url(base.as_ref(), "data:image/png;base64,abc"), None);
1025        assert_eq!(resolve_url(base.as_ref(), "javascript:void(0)"), None);
1026    }
1027
1028    #[test]
1029    fn find_largest_icon_选择最大() {
1030        let candidates = vec![
1031            ("/small.png".to_string(), "16x16".to_string()),
1032            ("/medium.png".to_string(), "32x32".to_string()),
1033            ("/large.png".to_string(), "64x64".to_string()),
1034        ];
1035        assert_eq!(find_largest_icon(&candidates), "/large.png");
1036    }
1037
1038    #[test]
1039    fn find_largest_icon_无尺寸时选第一个() {
1040        let candidates = vec![
1041            ("/icon.png".to_string(), String::new()),
1042            ("/other.png".to_string(), "any".to_string()),
1043        ];
1044        assert_eq!(find_largest_icon(&candidates), "/icon.png");
1045    }
1046
1047    #[test]
1048    fn parse_icon_size_标准格式() {
1049        assert_eq!(parse_icon_size("32x32"), Some(32));
1050        assert_eq!(parse_icon_size("64x32"), Some(64));
1051    }
1052
1053    #[test]
1054    fn parse_icon_size_any返回None() {
1055        assert_eq!(parse_icon_size("any"), None);
1056        assert_eq!(parse_icon_size(""), None);
1057    }
1058
1059    #[test]
1060    fn extract_meta_content_by_name() {
1061        let html = parse_html(r#"<html><head><meta name="generator" content="crawlkit"></head></html>"#);
1062        assert_eq!(extract_meta_content(&html, "generator"), Some("crawlkit".to_string()));
1063    }
1064
1065    #[test]
1066    fn extract_meta_content_by_property() {
1067        let html = parse_html(r#"<html><head><meta property="article:section" content="tech"></head></html>"#);
1068        assert_eq!(extract_meta_content(&html, "article:section"), Some("tech".to_string()));
1069    }
1070
1071    #[test]
1072    fn extract_meta_content_无匹配返回None() {
1073        let html = parse_html("<html><head></head></html>");
1074        assert_eq!(extract_meta_content(&html, "nonexistent"), None);
1075    }
1076
1077    #[test]
1078    fn extract_custom_meta_过滤已知字段() {
1079        let html = parse_html(r#"
1080            <html><head>
1081                <meta name="description" content="skip me">
1082                <meta name="og:title" content="skip me too">
1083                <meta name="custom-field" content="keep me">
1084            </head></html>
1085        "#);
1086        let custom = extract_custom_meta(&html);
1087        assert_eq!(custom.len(), 1);
1088        assert_eq!(custom.get("custom-field"), Some(&"keep me".to_string()));
1089    }
1090
1091    #[test]
1092    fn extract_microdata_基本提取() {
1093        let html = parse_html(r#"
1094            <html><body>
1095                <div itemscope itemtype="http://schema.org/Person">
1096                    <span itemprop="name">John Doe</span>
1097                    <span itemprop="email">john@example.com</span>
1098                </div>
1099            </body></html>
1100        "#);
1101        let data = extract_microdata(&html);
1102        assert_eq!(data.len(), 1);
1103        assert_eq!(data[0].schema_type, Some("Person".to_string()));
1104        assert_eq!(
1105            data[0].properties.get("name"),
1106            Some(&serde_json::Value::String("John Doe".to_string()))
1107        );
1108    }
1109
1110    #[test]
1111    fn extract_og_property_存在() {
1112        let html = parse_html(r#"<html><head><meta property="og:image" content="https://example.com/img.jpg"></head></html>"#);
1113        assert_eq!(
1114            extract_og_property(&html, "og:image"),
1115            Some("https://example.com/img.jpg".to_string())
1116        );
1117    }
1118
1119    #[test]
1120    fn extract_og_property_不存在返回None() {
1121        let html = parse_html("<html><head></head></html>");
1122        assert_eq!(extract_og_property(&html, "og:image"), None);
1123    }
1124
1125    #[test]
1126    fn extract_twitter_property_存在() {
1127        let html = parse_html(r#"<html><head><meta name="twitter:image" content="https://example.com/img.jpg"></head></html>"#);
1128        assert_eq!(
1129            extract_twitter_property(&html, "twitter:image"),
1130            Some("https://example.com/img.jpg".to_string())
1131        );
1132    }
1133
1134    #[test]
1135    fn 提取robots_高级指令() {
1136        let html = parse_html(r#"<html><head><meta name="robots" content="noindex, max-image-preview:standard, max-video-preview:30"></head></html>"#);
1137        let robots = extract_robots(&html);
1138        assert!(!robots.index);
1139        assert_eq!(robots.max_image_preview, Some("standard".to_string()));
1140        assert_eq!(robots.max_video_preview, 30);
1141    }
1142
1143    #[test]
1144    fn extract_metadata_无base_url() {
1145        let html = parse_html(r#"
1146            <html><head>
1147                <title>No Base</title>
1148                <link rel="canonical" href="/page">
1149            </head></html>
1150        "#);
1151        let meta = extract_metadata(&html, None).unwrap();
1152        assert_eq!(meta.title, Some("No Base".to_string()));
1153    }
1154
1155    #[test]
1156    fn extract_metadata_错误处理_无效选择器不崩溃() {
1157        let html = parse_html("<html><head><title>Safe</title></head></html>");
1158        let meta = extract_metadata(&html, None).unwrap();
1159        assert_eq!(meta.title, Some("Safe".to_string()));
1160    }
1161}
crawlkit_parser/metadata.rs

crawlkit_parser/
metadata.rs
