crawlkit_parser/
links.rs

1//! 链接提取与分析模块
2//!
3//! 基于 halldyll-parser 的链接提取逻辑改写。
4//! 提供从 HTML 文档中提取链接、URL 解析、分类、过滤与统计分析功能。
5
6use scraper::{ElementRef, Html};
7use std::collections::HashSet;
8use url::Url;
9
10use crate::selector::SELECTORS;
11use crate::types::{Link, LinkRel, LinkType, ParserConfig, ParserResult};
12
13// ============================================================================
14// 链接统计
15// ============================================================================
16
17/// 链接统计信息
18#[derive(Debug, Clone, Default)]
19pub struct LinkStats {
20    /// 链接总数
21    pub total: usize,
22    /// 内部链接数
23    pub internal: usize,
24    /// 外部链接数
25    pub external: usize,
26    /// nofollow 链接数
27    pub nofollow: usize,
28    /// sponsored 链接数
29    pub sponsored: usize,
30    /// ugc 链接数
31    pub ugc: usize,
32    /// 可跟随链接数
33    pub followable: usize,
34    /// 唯一外部域名数
35    pub unique_external_domains: usize,
36    /// 外部域名列表
37    pub external_domains: Vec<String>,
38}
39
40// ============================================================================
41// 链接提取
42// ============================================================================
43
44/// 从 HTML 文档中提取所有链接。
45///
46/// 使用 `SELECTORS` 中定义的 `a[href]` 选择器匹配链接元素，
47/// 并对每个元素依次调用 `extract_link` 进行解析。
48pub fn extract_links(document: &Html, config: &ParserConfig) -> ParserResult<Vec<Link>> {
49    let selector = &SELECTORS.a;
50    let mut links = Vec::new();
51    let mut seen = HashSet::new();
52
53    for element in document.select(selector) {
54        if let Some(link) = extract_link(&element, config.base_url.as_ref())
55            && seen.insert(link.href.clone()) {
56                links.push(link);
57            }
58    }
59
60    Ok(links)
61}
62
63/// 从单个 `<a>` 元素提取 Link 结构。
64///
65/// 提取 href、文本、title、rel、target、hreflang 等属性，
66/// 并解析 URL、链接类型、nofollow 状态等。
67pub fn extract_link(element: &ElementRef, base_url: Option<&Url>) -> Option<Link> {
68    let href = element.value().attr("href")?;
69    let href = href.trim();
70    if href.is_empty()
71        || href.starts_with('#')
72        || href.starts_with("javascript:")
73        || href.starts_with("mailto:")
74        || href.starts_with("tel:")
75        || href.starts_with("data:")
76    {
77        return None;
78    }
79
80    let text: String = element.text().collect::<Vec<_>>().join(" ").trim().to_string();
81
82    let resolved = resolve_url(href, base_url);
83    let normalized = resolved.as_deref().and_then(normalize_url);
84
85    let rel_attr = element.value().attr("rel").unwrap_or("");
86    let rel = parse_rel_attribute(rel_attr);
87    let is_nofollow = is_nofollow(&rel);
88
89    let link_type = match normalized.as_deref() {
90        Some(url) => determine_link_type(url, base_url),
91        None => LinkType::Unknown,
92    };
93
94    let title = element.value().attr("title").map(ToString::to_string);
95    let target = element.value().attr("target").map(ToString::to_string);
96    let hreflang = element.value().attr("hreflang").map(ToString::to_string);
97
98    Some(Link {
99        href: href.to_string(),
100        url: normalized,
101        text: if text.is_empty() { href.to_string() } else { text },
102        title,
103        rel,
104        link_type,
105        is_nofollow,
106        target,
107        hreflang,
108    })
109}
110
111// ============================================================================
112// URL 处理
113// ============================================================================
114
115/// 将 href 属性值解析为绝对 URL。
116///
117/// 支持相对路径解析（基于 base_url）以及协议相对 URL（`//example.com/path`）。
118pub fn resolve_url(href: &str, base_url: Option<&Url>) -> Option<String> {
119    let href = href.trim();
120    if href.is_empty() {
121        return None;
122    }
123
124    // 协议相对 URL：//example.com/path
125    if href.starts_with("//") {
126        let scheme = base_url.map_or("https", url::Url::scheme);
127        return Some(format!("{scheme}:{href}"));
128    }
129
130    // 已经是绝对 URL
131    if href.starts_with("http://") || href.starts_with("https://") {
132        return Url::parse(href).ok().map(|u| u.to_string());
133    }
134
135    // 相对 URL，需要 base_url
136    let base = base_url?;
137    base.join(href).ok().map(|u| u.to_string())
138}
139
140/// 标准化 URL：移除片段标识符（fragment）、末尾斜杠（可选的）等。
141pub fn normalize_url(url: &str) -> Option<String> {
142    let mut parsed = Url::parse(url).ok()?;
143
144    // 只处理 http/https
145    if parsed.scheme() != "http" && parsed.scheme() != "https" {
146        return Some(url.to_string());
147    }
148
149    // 移除片段
150    parsed.set_fragment(None);
151
152    // 移除默认端口
153    match parsed.port() {
154        Some(80) if parsed.scheme() == "http" => {
155            let _ = parsed.set_port(None);
156        }
157        Some(443) if parsed.scheme() == "https" => {
158            let _ = parsed.set_port(None);
159        }
160        _ => {}
161    }
162
163    Some(parsed.to_string())
164}
165
166// ============================================================================
167// 链接类型判定
168// ============================================================================
169
170/// 判断链接类型（内部 / 外部 / 未知）。
171///
172/// - 内部链接：与 base_url 域名相同（包括 www 前缀差异）
173/// - 外部链接：不同域名，并且与 base_url 属于不同的一级域名
174/// - 子域名链接与主站同属一个一级域名时视为内部链接
175pub fn determine_link_type(url: &str, base_url: Option<&Url>) -> LinkType {
176    let parsed = match Url::parse(url) {
177        Ok(u) => u,
178        Err(_) => return LinkType::Unknown,
179    };
180
181    let base = match base_url {
182        Some(b) => b,
183        None => return LinkType::External,
184    };
185
186    let url_host = match parsed.host_str() {
187        Some(h) => h,
188        None => return LinkType::Internal,
189    };
190
191    let base_host = match base.host_str() {
192        Some(h) => h,
193        None => return LinkType::External,
194    };
195
196    if url_host == base_host {
197        return LinkType::Internal;
198    }
199
200    // 检查是否为子域名关系
201    if let Some(rest) = url_host.strip_suffix(base_host)
202        && rest.ends_with('.')
203    {
204        return LinkType::Internal;
205    }
206
207    LinkType::External
208}
209
210// ============================================================================
211// Rel 属性解析
212// ============================================================================
213
214/// 解析 `rel` 属性值为 `LinkRel` 枚举列表。
215pub fn parse_rel_attribute(rel: &str) -> Vec<LinkRel> {
216    if rel.is_empty() {
217        return Vec::new();
218    }
219
220    rel.split_whitespace()
221        .map(|token| match token.to_lowercase().as_str() {
222            "nofollow" => LinkRel::NoFollow,
223            "ugc" => LinkRel::Ugc,
224            "sponsored" => LinkRel::Sponsored,
225            "external" => LinkRel::External,
226            "noopener" => LinkRel::NoOpener,
227            "noreferrer" => LinkRel::NoReferrer,
228            "follow" => LinkRel::Follow,
229            _ => LinkRel::Other,
230        })
231        .collect()
232}
233
234/// 判断 rel 列表中是否包含 nofollow。
235pub fn is_nofollow(rel: &[LinkRel]) -> bool {
236    rel.iter().any(|r| matches!(r, LinkRel::NoFollow))
237}
238
239/// 判断 rel 列表中是否包含 sponsored。
240pub fn is_sponsored(rel: &[LinkRel]) -> bool {
241    rel.iter().any(|r| matches!(r, LinkRel::Sponsored))
242}
243
244/// 判断 rel 列表中是否包含 ugc。
245pub fn is_ugc(rel: &[LinkRel]) -> bool {
246    rel.iter().any(|r| matches!(r, LinkRel::Ugc))
247}
248
249// ============================================================================
250// 链接过滤
251// ============================================================================
252
253/// 过滤出内部链接。
254pub fn filter_internal_links(links: &[Link]) -> Vec<&Link> {
255    links.iter().filter(|l| l.link_type == LinkType::Internal).collect()
256}
257
258/// 过滤出外部链接。
259pub fn filter_external_links(links: &[Link]) -> Vec<&Link> {
260    links.iter().filter(|l| l.link_type == LinkType::External).collect()
261}
262
263/// 过滤出可跟随的链接（非 nofollow / sponsored / ugc）。
264pub fn filter_followable_links(links: &[Link]) -> Vec<&Link> {
265    links.iter().filter(|l| l.should_follow()).collect()
266}
267
268/// 获取所有外部链接的唯一域名列表。
269///
270/// 从每个外部链接中提取域名，去重后返回。
271pub fn get_external_domains(links: &[Link]) -> Vec<String> {
272    let mut domains = HashSet::new();
273
274    for link in links {
275        if link.link_type != LinkType::External {
276            continue;
277        }
278        if let Some(ref url_str) = link.url
279            && let Ok(parsed) = Url::parse(url_str)
280                && let Some(host) = parsed.host_str() {
281                    domains.insert(host.to_string());
282                }
283    }
284
285    let mut result: Vec<_> = domains.into_iter().collect();
286    result.sort();
287    result
288}
289
290// ============================================================================
291// 链接统计
292// ============================================================================
293
294/// 计算链接统计信息。
295pub fn calculate_link_stats(links: &[Link]) -> LinkStats {
296    let total = links.len();
297    let internal = links.iter().filter(|l| l.link_type == LinkType::Internal).count();
298    let external = links.iter().filter(|l| l.link_type == LinkType::External).count();
299
300    let nofollow = links.iter().filter(|l| l.is_nofollow).count();
301    let sponsored = links.iter().filter(|l| is_sponsored(&l.rel)).count();
302    let ugc = links.iter().filter(|l| is_ugc(&l.rel)).count();
303    let followable = links.iter().filter(|l| l.should_follow()).count();
304
305    let external_domains = get_external_domains(links);
306    let unique_external_domains = external_domains.len();
307
308    LinkStats {
309        total,
310        internal,
311        external,
312        nofollow,
313        sponsored,
314        ugc,
315        followable,
316        unique_external_domains,
317        external_domains,
318    }
319}
320
321// ============================================================================
322// 测试
323// ============================================================================
324
325#[cfg(test)]
326mod tests {
327    use super::*;
328    
329    use scraper::Selector;
330
331    // 辅助：构造测试用的 ParserConfig
332    fn test_config(base_url: Option<&str>) -> ParserConfig {
333        ParserConfig {
334            base_url: base_url.and_then(|u| Url::parse(u).ok()),
335            ..ParserConfig::default()
336        }
337    }
338
339    // 辅助：从 HTML 字符串解析文档并提取链接
340    fn extract_from_html(html: &str, base_url: Option<&str>) -> Vec<Link> {
341        let document = Html::parse_document(html);
342        let config = test_config(base_url);
343        extract_links(&document, &config).unwrap()
344    }
345
346    #[test]
347    fn test_extract_links_basic() {
348        let html = r#"<html><body><a href="https://example.com">Example</a></body></html>"#;
349        let links = extract_from_html(html, None);
350        assert_eq!(links.len(), 1);
351        assert_eq!(links[0].href, "https://example.com");
352        assert_eq!(links[0].text, "Example");
353    }
354
355    #[test]
356    fn test_extract_links_filters_anchors_and_javascript() {
357        let html = r##"<html><body>
358            <a href="#section1">锚点</a>
359            <a href="javascript:void(0)">脚本</a>
360            <a href="mailto:user at host">邮箱</a>
361            <a href="tel:+123456789">电话</a>
362            <a href="data:text/plain,hello">数据</a>
363            <a href="https://example.com/valid">有效</a>
364        </body></html>"##;
365        let links = extract_from_html(html, None);
366        assert_eq!(links.len(), 1);
367        assert_eq!(links[0].href, "https://example.com/valid");
368    }
369
370    #[test]
371    fn test_extract_links_deduplicates() {
372        let html = r#"<html><body>
373            <a href="https://example.com">A</a>
374            <a href="https://example.com">B</a>
375        </body></html>"#;
376        let links = extract_from_html(html, None);
377        assert_eq!(links.len(), 1);
378    }
379
380    #[test]
381    fn test_extract_links_resolves_relative_with_base() {
382        let html = r#"<html><body><a href="/page">相对链接</a></body></html>"#;
383        let links = extract_from_html(html, Some("https://example.com"));
384        assert_eq!(links.len(), 1);
385        assert_eq!(links[0].url.as_deref(), Some("https://example.com/page"));
386        assert_eq!(links[0].link_type, LinkType::Internal);
387    }
388
389    #[test]
390    fn test_extract_links_protocol_relative() {
391        let html = r#"<html><body><a href="//cdn.example.com/file.js">CDN</a></body></html>"#;
392        let links = extract_from_html(html, Some("https://example.com"));
393        assert_eq!(links.len(), 1);
394        assert_eq!(links[0].url.as_deref(), Some("https://cdn.example.com/file.js"));
395    }
396
397    #[test]
398    fn test_extract_link_returns_none_for_element_without_href() {
399        let html = Html::parse_document(r"<html><body><span>不是链接</span></body></html>");
400        let selector = Selector::parse("span").unwrap();
401        let element = html.select(&selector).next().unwrap();
402        let result = extract_link(&element, None);
403        assert!(result.is_none());
404    }
405
406    #[test]
407    fn test_resolve_url_absolute() {
408        let base = Url::parse("https://example.com").ok();
409        let result = resolve_url("https://other.com/path", base.as_ref());
410        assert_eq!(result.as_deref(), Some("https://other.com/path"));
411    }
412
413    #[test]
414    fn test_resolve_url_relative() {
415        let base = Url::parse("https://example.com/base/").ok();
416        let result = resolve_url("../page", base.as_ref());
417        assert_eq!(result.as_deref(), Some("https://example.com/page"));
418    }
419
420    #[test]
421    fn test_resolve_url_protocol_relative() {
422        let base = Url::parse("https://example.com").ok();
423        let result = resolve_url("//other.com/path", base.as_ref());
424        assert_eq!(result.as_deref(), Some("https://other.com/path"));
425    }
426
427    #[test]
428    fn test_resolve_url_empty_or_skip() {
429        assert!(resolve_url("", None).is_none());
430        assert!(resolve_url("  ", None).is_none());
431    }
432
433    #[test]
434    fn test_normalize_url_removes_fragment() {
435        let result = normalize_url("https://example.com/page#section");
436        assert_eq!(result.as_deref(), Some("https://example.com/page"));
437    }
438
439    #[test]
440    fn test_normalize_url_removes_default_port() {
441        let result = normalize_url("https://example.com:443/page");
442        assert_eq!(result.as_deref(), Some("https://example.com/page"));
443    }
444
445    #[test]
446    fn test_normalize_url_preserves_non_standard_port() {
447        let result = normalize_url("https://example.com:8080/page");
448        assert_eq!(result.as_deref(), Some("https://example.com:8080/page"));
449    }
450
451    #[test]
452    fn test_normalize_url_keeps_http_and_https() {
453        let result = normalize_url("ftp://example.com/file");
454        assert_eq!(result.as_deref(), Some("ftp://example.com/file"));
455    }
456
457    #[test]
458    fn test_determine_link_type_internal_exact() {
459        let base = Url::parse("https://example.com").ok();
460        let result = determine_link_type("https://example.com/page", base.as_ref());
461        assert_eq!(result, LinkType::Internal);
462    }
463
464    #[test]
465    fn test_determine_link_type_internal_subdomain() {
466        let base = Url::parse("https://example.com").ok();
467        let result = determine_link_type("https://blog.example.com/page", base.as_ref());
468        assert_eq!(result, LinkType::Internal);
469    }
470
471    #[test]
472    fn test_determine_link_type_external() {
473        let base = Url::parse("https://example.com").ok();
474        let result = determine_link_type("https://other.com/page", base.as_ref());
475        assert_eq!(result, LinkType::External);
476    }
477
478    #[test]
479    fn test_determine_link_type_no_base() {
480        let result = determine_link_type("https://example.com/page", None);
481        assert_eq!(result, LinkType::External);
482    }
483
484    #[test]
485    fn test_parse_rel_attribute_nofollow() {
486        let rel = parse_rel_attribute("nofollow");
487        assert!(is_nofollow(&rel));
488        assert!(!is_sponsored(&rel));
489        assert!(!is_ugc(&rel));
490    }
491
492    #[test]
493    fn test_parse_rel_attribute_multiple() {
494        let rel = parse_rel_attribute("nofollow ugc sponsored");
495        assert!(is_nofollow(&rel));
496        assert!(is_sponsored(&rel));
497        assert!(is_ugc(&rel));
498    }
499
500    #[test]
501    fn test_parse_rel_attribute_empty() {
502        let rel = parse_rel_attribute("");
503        assert!(rel.is_empty());
504    }
505
506    #[test]
507    fn test_parse_rel_attribute_case_insensitive() {
508        let rel = parse_rel_attribute("NoFollow UGC Sponsored");
509        assert!(is_nofollow(&rel));
510        assert!(is_sponsored(&rel));
511        assert!(is_ugc(&rel));
512    }
513
514    #[test]
515    fn test_filter_internal_links() {
516        let mut a = Link::new("https://example.com/internal", "内部");
517        a.link_type = LinkType::Internal;
518        let mut b = Link::new("https://external.com/external", "外部");
519        b.link_type = LinkType::External;
520        let links = vec![a, b];
521        let internal = filter_internal_links(&links);
522        assert_eq!(internal.len(), 1);
523        assert_eq!(internal[0].href, "https://example.com/internal");
524    }
525
526    #[test]
527    fn test_filter_external_links() {
528        let mut a = Link::new("https://example.com/internal", "内部");
529        a.link_type = LinkType::Internal;
530        let mut b = Link::new("https://external.com/external", "外部");
531        b.link_type = LinkType::External;
532        let links = vec![a, b];
533        let external = filter_external_links(&links);
534        assert_eq!(external.len(), 1);
535        assert_eq!(external[0].href, "https://external.com/external");
536    }
537
538    #[test]
539    fn test_filter_followable_links() {
540        let mut follow = Link::new("https://example.com/follow", "可跟随");
541        follow.link_type = LinkType::Internal;
542        let mut nofollow = Link::new("https://example.com/nofollow", "不可跟随");
543        nofollow.link_type = LinkType::Internal;
544        nofollow.is_nofollow = true;
545
546        let links = vec![follow, nofollow];
547        let followable = filter_followable_links(&links);
548        assert_eq!(followable.len(), 1);
549        assert_eq!(followable[0].href, "https://example.com/follow");
550    }
551
552    #[test]
553    fn test_get_external_domains() {
554        let mut a = Link::new("https://example.com/internal", "内部");
555        a.url = Some("https://example.com/internal".to_string());
556        a.link_type = LinkType::Internal;
557        let mut b = Link::new("https://external.com/page1", "外部1");
558        b.url = Some("https://external.com/page1".to_string());
559        b.link_type = LinkType::External;
560        let mut c = Link::new("https://external.com/page2", "外部2");
561        c.url = Some("https://external.com/page2".to_string());
562        c.link_type = LinkType::External;
563        let mut d = Link::new("https://other.org/page", "外部3");
564        d.url = Some("https://other.org/page".to_string());
565        d.link_type = LinkType::External;
566        let links = vec![a, b, c, d];
567        let domains = get_external_domains(&links);
568        assert_eq!(domains, vec!["external.com", "other.org"]);
569    }
570
571    #[test]
572    fn test_calculate_link_stats() {
573        let mut ext = Link::new("https://external.com/page", "外部链接");
574        ext.url = Some("https://external.com/page".to_string());
575        ext.link_type = LinkType::External;
576
577        let mut int = Link::new("https://example.com/page", "内部链接");
578        int.url = Some("https://example.com/page".to_string());
579        int.link_type = LinkType::Internal;
580        int.is_nofollow = true;
581
582        let links = vec![ext, int];
583
584        let stats = calculate_link_stats(&links);
585        assert_eq!(stats.total, 2);
586        assert_eq!(stats.internal, 1);
587        assert_eq!(stats.external, 1);
588        assert_eq!(stats.nofollow, 1);
589        assert_eq!(stats.followable, 1);
590        assert_eq!(stats.unique_external_domains, 1);
591    }
592
593    #[test]
594    fn test_extract_link_with_title_and_target() {
595        let html = r#"<html><body><a href="https://example.com" title="示例" target="_blank" rel="nofollow">链接</a></body></html>"#;
596        let document = Html::parse_document(html);
597        let selector = Selector::parse("a").unwrap();
598        let element = document.select(&selector).next().unwrap();
599        let link = extract_link(&element, None).unwrap();
600
601        assert_eq!(link.title.as_deref(), Some("示例"));
602        assert_eq!(link.target.as_deref(), Some("_blank"));
603        assert!(link.is_nofollow);
604        assert!(link.opens_new_tab());
605    }
606
607    #[test]
608    fn test_should_skip_link_with_empty_href() {
609        let html = r#"<html><body><a href="">空链接</a></body></html>"#;
610        let links = extract_from_html(html, None);
611        assert!(links.is_empty());
612    }
613
614    #[test]
615    fn test_only_whitespace_href_skipped() {
616        let html = r#"<html><body><a href="   ">空白链接</a></body></html>"#;
617        let links = extract_from_html(html, None);
618        assert!(links.is_empty());
619    }
620
621    #[test]
622    fn test_same_domain_different_scheme_is_external() {
623        // 链接为 http，基准为 https，但域名相同 —— 仍设为内部链接
624        let base = Url::parse("https://example.com").ok();
625        let result = determine_link_type("http://example.com/page", base.as_ref());
626        assert_eq!(result, LinkType::Internal);
627    }
628
629    #[test]
630    fn test_extract_links_multiple_selectors() {
631        let html = r#"<html><body>
632            <a href="https://a.com">A</a>
633            <a href="https://b.com">B</a>
634            <a href="https://c.com">C</a>
635        </body></html>"#;
636        let links = extract_from_html(html, None);
637        assert_eq!(links.len(), 3);
638    }
639
640    #[test]
641    fn test_normalize_url_noop() {
642        let result = normalize_url("https://example.com/path/to/page");
643        assert_eq!(result.as_deref(), Some("https://example.com/path/to/page"));
644    }
645
646    #[test]
647    fn test_calculate_link_stats_empty() {
648        let stats = calculate_link_stats(&[]);
649        assert_eq!(stats.total, 0);
650        assert_eq!(stats.internal, 0);
651        assert_eq!(stats.external, 0);
652        assert_eq!(stats.followable, 0);
653    }
654}
crawlkit_parser/links.rs

crawlkit_parser/
links.rs
