crawlkit_parser/
links.rs

1//! 链接提取与分析模块
2//!
3//! 基于 halldyll-parser 的链接提取逻辑改写。
4//! 提供从 HTML 文档中提取链接、URL 解析、分类、过滤与统计分析功能。
5
6use scraper::{ElementRef, Html};
7use std::collections::HashSet;
8use url::Url;
9
10use crate::selector::SELECTORS;
11use crate::types::{Link, LinkRel, LinkType, ParserConfig, ParserResult};
12
13// ============================================================================
14// 链接统计
15// ============================================================================
16
17/// 链接统计信息
18#[derive(Debug, Clone, Default)]
19pub struct LinkStats {
20    /// 链接总数
21    pub total: usize,
22    /// 内部链接数
23    pub internal: usize,
24    /// 外部链接数
25    pub external: usize,
26    /// nofollow 链接数
27    pub nofollow: usize,
28    /// sponsored 链接数
29    pub sponsored: usize,
30    /// ugc 链接数
31    pub ugc: usize,
32    /// 可跟随链接数
33    pub followable: usize,
34    /// 唯一外部域名数
35    pub unique_external_domains: usize,
36    /// 外部域名列表
37    pub external_domains: Vec<String>,
38}
39
40// ============================================================================
41// 链接提取
42// ============================================================================
43
44/// 从 HTML 文档中提取所有链接。
45///
46/// 使用 `SELECTORS` 中定义的 `a[href]` 选择器匹配链接元素，
47/// 并对每个元素依次调用 `extract_link` 进行解析。
48pub fn extract_links(document: &Html, config: &ParserConfig) -> ParserResult<Vec<Link>> {
49    let selector = &SELECTORS.a;
50    let mut links = Vec::new();
51    let mut seen = HashSet::new();
52
53    for element in document.select(selector) {
54        if let Some(link) = extract_link(&element, config.base_url.as_ref()) {
55            if seen.insert(link.href.clone()) {
56                links.push(link);
57            }
58        }
59    }
60
61    Ok(links)
62}
63
64/// 从单个 `<a>` 元素提取 Link 结构。
65///
66/// 提取 href、文本、title、rel、target、hreflang 等属性，
67/// 并解析 URL、链接类型、nofollow 状态等。
68pub fn extract_link(element: &ElementRef, base_url: Option<&Url>) -> Option<Link> {
69    let href = element.value().attr("href")?;
70    let href = href.trim();
71    if href.is_empty()
72        || href.starts_with('#')
73        || href.starts_with("javascript:")
74        || href.starts_with("mailto:")
75        || href.starts_with("tel:")
76        || href.starts_with("data:")
77    {
78        return None;
79    }
80
81    let text: String = element.text().collect::<Vec<_>>().join(" ").trim().to_string();
82
83    let resolved = resolve_url(href, base_url);
84    let normalized = resolved.as_deref().and_then(normalize_url);
85
86    let rel_attr = element.value().attr("rel").unwrap_or("");
87    let rel = parse_rel_attribute(rel_attr);
88    let is_nofollow = is_nofollow(&rel);
89
90    let link_type = match normalized.as_deref() {
91        Some(url) => determine_link_type(url, base_url),
92        None => LinkType::Unknown,
93    };
94
95    let title = element.value().attr("title").map(ToString::to_string);
96    let target = element.value().attr("target").map(ToString::to_string);
97    let hreflang = element.value().attr("hreflang").map(ToString::to_string);
98
99    Some(Link {
100        href: href.to_string(),
101        url: normalized,
102        text: if text.is_empty() { href.to_string() } else { text },
103        title,
104        rel,
105        link_type,
106        is_nofollow,
107        target,
108        hreflang,
109    })
110}
111
112// ============================================================================
113// URL 处理
114// ============================================================================
115
116/// 将 href 属性值解析为绝对 URL。
117///
118/// 支持相对路径解析（基于 base_url）以及协议相对 URL（`//example.com/path`）。
119pub fn resolve_url(href: &str, base_url: Option<&Url>) -> Option<String> {
120    let href = href.trim();
121    if href.is_empty() {
122        return None;
123    }
124
125    // 协议相对 URL：//example.com/path
126    if href.starts_with("//") {
127        let scheme = base_url.map(|u| u.scheme()).unwrap_or("https");
128        return Some(format!("{scheme}:{href}"));
129    }
130
131    // 已经是绝对 URL
132    if href.starts_with("http://") || href.starts_with("https://") {
133        return Url::parse(href).ok().map(|u| u.to_string());
134    }
135
136    // 相对 URL，需要 base_url
137    let base = base_url?;
138    base.join(href).ok().map(|u| u.to_string())
139}
140
141/// 标准化 URL：移除片段标识符（fragment）、末尾斜杠（可选的）等。
142pub fn normalize_url(url: &str) -> Option<String> {
143    let mut parsed = Url::parse(url).ok()?;
144
145    // 只处理 http/https
146    if parsed.scheme() != "http" && parsed.scheme() != "https" {
147        return Some(url.to_string());
148    }
149
150    // 移除片段
151    parsed.set_fragment(None);
152
153    // 移除默认端口
154    match parsed.port() {
155        Some(80) if parsed.scheme() == "http" => {
156            let _ = parsed.set_port(None);
157        }
158        Some(443) if parsed.scheme() == "https" => {
159            let _ = parsed.set_port(None);
160        }
161        _ => {}
162    }
163
164    Some(parsed.to_string())
165}
166
167// ============================================================================
168// 链接类型判定
169// ============================================================================
170
171/// 判断链接类型（内部 / 外部 / 未知）。
172///
173/// - 内部链接：与 base_url 域名相同（包括 www 前缀差异）
174/// - 外部链接：不同域名，并且与 base_url 属于不同的一级域名
175/// - 子域名链接与主站同属一个一级域名时视为内部链接
176pub fn determine_link_type(url: &str, base_url: Option<&Url>) -> LinkType {
177    let parsed = match Url::parse(url) {
178        Ok(u) => u,
179        Err(_) => return LinkType::Unknown,
180    };
181
182    let base = match base_url {
183        Some(b) => b,
184        None => return LinkType::External,
185    };
186
187    let url_host = match parsed.host_str() {
188        Some(h) => h,
189        None => return LinkType::Internal,
190    };
191
192    let base_host = match base.host_str() {
193        Some(h) => h,
194        None => return LinkType::External,
195    };
196
197    if url_host == base_host {
198        return LinkType::Internal;
199    }
200
201    // 检查是否为子域名关系
202    if let Some(rest) = url_host.strip_suffix(base_host)
203        && rest.ends_with('.')
204    {
205        return LinkType::Internal;
206    }
207
208    LinkType::External
209}
210
211// ============================================================================
212// Rel 属性解析
213// ============================================================================
214
215/// 解析 `rel` 属性值为 `LinkRel` 枚举列表。
216pub fn parse_rel_attribute(rel: &str) -> Vec<LinkRel> {
217    if rel.is_empty() {
218        return Vec::new();
219    }
220
221    rel.split_whitespace()
222        .map(|token| match token.to_lowercase().as_str() {
223            "nofollow" => LinkRel::NoFollow,
224            "ugc" => LinkRel::Ugc,
225            "sponsored" => LinkRel::Sponsored,
226            "external" => LinkRel::External,
227            "noopener" => LinkRel::NoOpener,
228            "noreferrer" => LinkRel::NoReferrer,
229            "follow" => LinkRel::Follow,
230            _ => LinkRel::Other,
231        })
232        .collect()
233}
234
235/// 判断 rel 列表中是否包含 nofollow。
236pub fn is_nofollow(rel: &[LinkRel]) -> bool {
237    rel.iter().any(|r| matches!(r, LinkRel::NoFollow))
238}
239
240/// 判断 rel 列表中是否包含 sponsored。
241pub fn is_sponsored(rel: &[LinkRel]) -> bool {
242    rel.iter().any(|r| matches!(r, LinkRel::Sponsored))
243}
244
245/// 判断 rel 列表中是否包含 ugc。
246pub fn is_ugc(rel: &[LinkRel]) -> bool {
247    rel.iter().any(|r| matches!(r, LinkRel::Ugc))
248}
249
250// ============================================================================
251// 链接过滤
252// ============================================================================
253
254/// 过滤出内部链接。
255pub fn filter_internal_links(links: &[Link]) -> Vec<&Link> {
256    links.iter().filter(|l| l.link_type == LinkType::Internal).collect()
257}
258
259/// 过滤出外部链接。
260pub fn filter_external_links(links: &[Link]) -> Vec<&Link> {
261    links.iter().filter(|l| l.link_type == LinkType::External).collect()
262}
263
264/// 过滤出可跟随的链接（非 nofollow / sponsored / ugc）。
265pub fn filter_followable_links(links: &[Link]) -> Vec<&Link> {
266    links.iter().filter(|l| l.should_follow()).collect()
267}
268
269/// 获取所有外部链接的唯一域名列表。
270///
271/// 从每个外部链接中提取域名，去重后返回。
272pub fn get_external_domains(links: &[Link]) -> Vec<String> {
273    let mut domains = HashSet::new();
274
275    for link in links {
276        if link.link_type != LinkType::External {
277            continue;
278        }
279        if let Some(ref url_str) = link.url {
280            if let Ok(parsed) = Url::parse(url_str) {
281                if let Some(host) = parsed.host_str() {
282                    domains.insert(host.to_string());
283                }
284            }
285        }
286    }
287
288    let mut result: Vec<_> = domains.into_iter().collect();
289    result.sort();
290    result
291}
292
293// ============================================================================
294// 链接统计
295// ============================================================================
296
297/// 计算链接统计信息。
298pub fn calculate_link_stats(links: &[Link]) -> LinkStats {
299    let total = links.len();
300    let internal = links.iter().filter(|l| l.link_type == LinkType::Internal).count();
301    let external = links.iter().filter(|l| l.link_type == LinkType::External).count();
302
303    let nofollow = links.iter().filter(|l| l.is_nofollow).count();
304    let sponsored = links.iter().filter(|l| is_sponsored(&l.rel)).count();
305    let ugc = links.iter().filter(|l| is_ugc(&l.rel)).count();
306    let followable = links.iter().filter(|l| l.should_follow()).count();
307
308    let external_domains = get_external_domains(links);
309    let unique_external_domains = external_domains.len();
310
311    LinkStats {
312        total,
313        internal,
314        external,
315        nofollow,
316        sponsored,
317        ugc,
318        followable,
319        unique_external_domains,
320        external_domains,
321    }
322}
323
324// ============================================================================
325// 测试
326// ============================================================================
327
328#[cfg(test)]
329mod tests {
330    use super::*;
331    use crate::selector::SELECTORS;
332    use scraper::Selector;
333
334    // 辅助：构造测试用的 ParserConfig
335    fn test_config(base_url: Option<&str>) -> ParserConfig {
336        ParserConfig {
337            base_url: base_url.and_then(|u| Url::parse(u).ok()),
338            ..ParserConfig::default()
339        }
340    }
341
342    // 辅助：从 HTML 字符串解析文档并提取链接
343    fn extract_from_html(html: &str, base_url: Option<&str>) -> Vec<Link> {
344        let document = Html::parse_document(html);
345        let config = test_config(base_url);
346        extract_links(&document, &config).unwrap()
347    }
348
349    #[test]
350    fn test_extract_links_basic() {
351        let html = r#"<html><body><a href="https://example.com">Example</a></body></html>"#;
352        let links = extract_from_html(html, None);
353        assert_eq!(links.len(), 1);
354        assert_eq!(links[0].href, "https://example.com");
355        assert_eq!(links[0].text, "Example");
356    }
357
358    #[test]
359    fn test_extract_links_filters_anchors_and_javascript() {
360        let html = r##"<html><body>
361            <a href="#section1">锚点</a>
362            <a href="javascript:void(0)">脚本</a>
363            <a href="mailto:user at host">邮箱</a>
364            <a href="tel:+123456789">电话</a>
365            <a href="data:text/plain,hello">数据</a>
366            <a href="https://example.com/valid">有效</a>
367        </body></html>"##;
368        let links = extract_from_html(html, None);
369        assert_eq!(links.len(), 1);
370        assert_eq!(links[0].href, "https://example.com/valid");
371    }
372
373    #[test]
374    fn test_extract_links_deduplicates() {
375        let html = r#"<html><body>
376            <a href="https://example.com">A</a>
377            <a href="https://example.com">B</a>
378        </body></html>"#;
379        let links = extract_from_html(html, None);
380        assert_eq!(links.len(), 1);
381    }
382
383    #[test]
384    fn test_extract_links_resolves_relative_with_base() {
385        let html = r#"<html><body><a href="/page">相对链接</a></body></html>"#;
386        let links = extract_from_html(html, Some("https://example.com"));
387        assert_eq!(links.len(), 1);
388        assert_eq!(links[0].url.as_deref(), Some("https://example.com/page"));
389        assert_eq!(links[0].link_type, LinkType::Internal);
390    }
391
392    #[test]
393    fn test_extract_links_protocol_relative() {
394        let html = r#"<html><body><a href="//cdn.example.com/file.js">CDN</a></body></html>"#;
395        let links = extract_from_html(html, Some("https://example.com"));
396        assert_eq!(links.len(), 1);
397        assert_eq!(links[0].url.as_deref(), Some("https://cdn.example.com/file.js"));
398    }
399
400    #[test]
401    fn test_extract_link_returns_none_for_element_without_href() {
402        let html = Html::parse_document(r#"<html><body><span>不是链接</span></body></html>"#);
403        let selector = Selector::parse("span").unwrap();
404        let element = html.select(&selector).next().unwrap();
405        let result = extract_link(&element, None);
406        assert!(result.is_none());
407    }
408
409    #[test]
410    fn test_resolve_url_absolute() {
411        let base = Url::parse("https://example.com").ok();
412        let result = resolve_url("https://other.com/path", base.as_ref());
413        assert_eq!(result.as_deref(), Some("https://other.com/path"));
414    }
415
416    #[test]
417    fn test_resolve_url_relative() {
418        let base = Url::parse("https://example.com/base/").ok();
419        let result = resolve_url("../page", base.as_ref());
420        assert_eq!(result.as_deref(), Some("https://example.com/page"));
421    }
422
423    #[test]
424    fn test_resolve_url_protocol_relative() {
425        let base = Url::parse("https://example.com").ok();
426        let result = resolve_url("//other.com/path", base.as_ref());
427        assert_eq!(result.as_deref(), Some("https://other.com/path"));
428    }
429
430    #[test]
431    fn test_resolve_url_empty_or_skip() {
432        assert!(resolve_url("", None).is_none());
433        assert!(resolve_url("  ", None).is_none());
434    }
435
436    #[test]
437    fn test_normalize_url_removes_fragment() {
438        let result = normalize_url("https://example.com/page#section");
439        assert_eq!(result.as_deref(), Some("https://example.com/page"));
440    }
441
442    #[test]
443    fn test_normalize_url_removes_default_port() {
444        let result = normalize_url("https://example.com:443/page");
445        assert_eq!(result.as_deref(), Some("https://example.com/page"));
446    }
447
448    #[test]
449    fn test_normalize_url_preserves_non_standard_port() {
450        let result = normalize_url("https://example.com:8080/page");
451        assert_eq!(result.as_deref(), Some("https://example.com:8080/page"));
452    }
453
454    #[test]
455    fn test_normalize_url_keeps_http_and_https() {
456        let result = normalize_url("ftp://example.com/file");
457        assert_eq!(result.as_deref(), Some("ftp://example.com/file"));
458    }
459
460    #[test]
461    fn test_determine_link_type_internal_exact() {
462        let base = Url::parse("https://example.com").ok();
463        let result = determine_link_type("https://example.com/page", base.as_ref());
464        assert_eq!(result, LinkType::Internal);
465    }
466
467    #[test]
468    fn test_determine_link_type_internal_subdomain() {
469        let base = Url::parse("https://example.com").ok();
470        let result = determine_link_type("https://blog.example.com/page", base.as_ref());
471        assert_eq!(result, LinkType::Internal);
472    }
473
474    #[test]
475    fn test_determine_link_type_external() {
476        let base = Url::parse("https://example.com").ok();
477        let result = determine_link_type("https://other.com/page", base.as_ref());
478        assert_eq!(result, LinkType::External);
479    }
480
481    #[test]
482    fn test_determine_link_type_no_base() {
483        let result = determine_link_type("https://example.com/page", None);
484        assert_eq!(result, LinkType::External);
485    }
486
487    #[test]
488    fn test_parse_rel_attribute_nofollow() {
489        let rel = parse_rel_attribute("nofollow");
490        assert!(is_nofollow(&rel));
491        assert!(!is_sponsored(&rel));
492        assert!(!is_ugc(&rel));
493    }
494
495    #[test]
496    fn test_parse_rel_attribute_multiple() {
497        let rel = parse_rel_attribute("nofollow ugc sponsored");
498        assert!(is_nofollow(&rel));
499        assert!(is_sponsored(&rel));
500        assert!(is_ugc(&rel));
501    }
502
503    #[test]
504    fn test_parse_rel_attribute_empty() {
505        let rel = parse_rel_attribute("");
506        assert!(rel.is_empty());
507    }
508
509    #[test]
510    fn test_parse_rel_attribute_case_insensitive() {
511        let rel = parse_rel_attribute("NoFollow UGC Sponsored");
512        assert!(is_nofollow(&rel));
513        assert!(is_sponsored(&rel));
514        assert!(is_ugc(&rel));
515    }
516
517    #[test]
518    fn test_filter_internal_links() {
519        let mut a = Link::new("https://example.com/internal", "内部");
520        a.link_type = LinkType::Internal;
521        let mut b = Link::new("https://external.com/external", "外部");
522        b.link_type = LinkType::External;
523        let links = vec![a, b];
524        let internal = filter_internal_links(&links);
525        assert_eq!(internal.len(), 1);
526        assert_eq!(internal[0].href, "https://example.com/internal");
527    }
528
529    #[test]
530    fn test_filter_external_links() {
531        let mut a = Link::new("https://example.com/internal", "内部");
532        a.link_type = LinkType::Internal;
533        let mut b = Link::new("https://external.com/external", "外部");
534        b.link_type = LinkType::External;
535        let links = vec![a, b];
536        let external = filter_external_links(&links);
537        assert_eq!(external.len(), 1);
538        assert_eq!(external[0].href, "https://external.com/external");
539    }
540
541    #[test]
542    fn test_filter_followable_links() {
543        let mut follow = Link::new("https://example.com/follow", "可跟随");
544        follow.link_type = LinkType::Internal;
545        let mut nofollow = Link::new("https://example.com/nofollow", "不可跟随");
546        nofollow.link_type = LinkType::Internal;
547        nofollow.is_nofollow = true;
548
549        let links = vec![follow, nofollow];
550        let followable = filter_followable_links(&links);
551        assert_eq!(followable.len(), 1);
552        assert_eq!(followable[0].href, "https://example.com/follow");
553    }
554
555    #[test]
556    fn test_get_external_domains() {
557        let mut a = Link::new("https://example.com/internal", "内部");
558        a.url = Some("https://example.com/internal".to_string());
559        a.link_type = LinkType::Internal;
560        let mut b = Link::new("https://external.com/page1", "外部1");
561        b.url = Some("https://external.com/page1".to_string());
562        b.link_type = LinkType::External;
563        let mut c = Link::new("https://external.com/page2", "外部2");
564        c.url = Some("https://external.com/page2".to_string());
565        c.link_type = LinkType::External;
566        let mut d = Link::new("https://other.org/page", "外部3");
567        d.url = Some("https://other.org/page".to_string());
568        d.link_type = LinkType::External;
569        let links = vec![a, b, c, d];
570        let domains = get_external_domains(&links);
571        assert_eq!(domains, vec!["external.com", "other.org"]);
572    }
573
574    #[test]
575    fn test_calculate_link_stats() {
576        let mut ext = Link::new("https://external.com/page", "外部链接");
577        ext.url = Some("https://external.com/page".to_string());
578        ext.link_type = LinkType::External;
579
580        let mut int = Link::new("https://example.com/page", "内部链接");
581        int.url = Some("https://example.com/page".to_string());
582        int.link_type = LinkType::Internal;
583        int.is_nofollow = true;
584
585        let links = vec![ext, int];
586
587        let stats = calculate_link_stats(&links);
588        assert_eq!(stats.total, 2);
589        assert_eq!(stats.internal, 1);
590        assert_eq!(stats.external, 1);
591        assert_eq!(stats.nofollow, 1);
592        assert_eq!(stats.followable, 1);
593        assert_eq!(stats.unique_external_domains, 1);
594    }
595
596    #[test]
597    fn test_extract_link_with_title_and_target() {
598        let html = r#"<html><body><a href="https://example.com" title="示例" target="_blank" rel="nofollow">链接</a></body></html>"#;
599        let document = Html::parse_document(html);
600        let selector = Selector::parse("a").unwrap();
601        let element = document.select(&selector).next().unwrap();
602        let link = extract_link(&element, None).unwrap();
603
604        assert_eq!(link.title.as_deref(), Some("示例"));
605        assert_eq!(link.target.as_deref(), Some("_blank"));
606        assert!(link.is_nofollow);
607        assert!(link.opens_new_tab());
608    }
609
610    #[test]
611    fn test_should_skip_link_with_empty_href() {
612        let html = r#"<html><body><a href="">空链接</a></body></html>"#;
613        let links = extract_from_html(html, None);
614        assert!(links.is_empty());
615    }
616
617    #[test]
618    fn test_only_whitespace_href_skipped() {
619        let html = r#"<html><body><a href="   ">空白链接</a></body></html>"#;
620        let links = extract_from_html(html, None);
621        assert!(links.is_empty());
622    }
623
624    #[test]
625    fn test_same_domain_different_scheme_is_external() {
626        // 链接为 http，基准为 https，但域名相同 —— 仍设为内部链接
627        let base = Url::parse("https://example.com").ok();
628        let result = determine_link_type("http://example.com/page", base.as_ref());
629        assert_eq!(result, LinkType::Internal);
630    }
631
632    #[test]
633    fn test_extract_links_multiple_selectors() {
634        let html = r#"<html><body>
635            <a href="https://a.com">A</a>
636            <a href="https://b.com">B</a>
637            <a href="https://c.com">C</a>
638        </body></html>"#;
639        let links = extract_from_html(html, None);
640        assert_eq!(links.len(), 3);
641    }
642
643    #[test]
644    fn test_normalize_url_noop() {
645        let result = normalize_url("https://example.com/path/to/page");
646        assert_eq!(result.as_deref(), Some("https://example.com/path/to/page"));
647    }
648
649    #[test]
650    fn test_calculate_link_stats_empty() {
651        let stats = calculate_link_stats(&[]);
652        assert_eq!(stats.total, 0);
653        assert_eq!(stats.internal, 0);
654        assert_eq!(stats.external, 0);
655        assert_eq!(stats.followable, 0);
656    }
657}
crawlkit_parser/links.rs

crawlkit_parser/
links.rs
