crawlkit_parser/
extractor.rs

1//! 基于 DOM 结构聚类 + 多信号打分的文章链接提取器。
2//!
3//! 与纯正则方案（见 `rules` 模块）的区别：
4//! - 正则只能判断"这个 URL 长得像不像文章"，遇到 aei.org / newamerica.org
5//!   这种纯 slug 结构（`/program/area/slug`）就完全失效。
6//! - 本模块从页面结构入手：真正的文章列表在 DOM 里通常是"同一父级路径下
7//!   重复出现的相似节点"（比如同一个 `<ul class="news-list">` 下的十几个
8//!   `<li><a>`）。把链接按其父级路径签名聚类，聚类越大，越可能是文章列表。
9//! - 再结合锚文本长度、URL 路径深度、导航词过滤、以及复用 UrlRule 的正则
10//!   规则，做加权打分，避免"一条正则命中/不命中"的非黑即白判定。
11//!
12//! # 示例
13//!
14//! ```rust
15//! use crawlkit_parser::{LinkExtractor, ExtractorConfig};
16//!
17//! let html = r#"
18//! <html><body>
19//!     <div class="news-list">
20//!         <a href="/2024/01/15/story-one">第一条新闻标题</a>
21//!         <a href="/2024/01/16/story-two">第二条新闻标题</a>
22//!         <a href="/2024/01/17/story-three">第三条新闻标题</a>
23//!         <a href="/2024/01/18/story-four">第四条新闻标题</a>
24//!     </div>
25//! </body></html>
26//! "#;
27//!
28//! let extractor = LinkExtractor::new(ExtractorConfig::default());
29//! let results = extractor.extract(html, "https://example.com/");
30//!
31//! assert_eq!(results.len(), 4);
32//! for link in &results {
33//!     println!("{} (score: {})", link.url, link.score);
34//! }
35//! ```
36//!
37
38use std::collections::HashMap;
39
40use scraper::{ElementRef, Html, Selector};
41use url::Url;
42
43use crate::rules::UrlRule;
44
45/// 提取到的候选文章链接。
46///
47/// 包含 URL、锚文本、综合得分以及所在列表结构的聚类大小。
48///
49/// ```rust
50/// use crawlkit_parser::{LinkExtractor, ExtractorConfig};
51///
52/// let html = r#"<a href="/2024/01/15/story">文章标题</a>"#;
53/// let extractor = LinkExtractor::new(ExtractorConfig::default());
54/// let results = extractor.extract(html, "https://example.com/");
55/// for link in &results {
56///     println!("{} | score={} | cluster={}", link.url, link.score, link.cluster_size);
57/// }
58/// ```
59#[derive(Debug, Clone)]
60pub struct ExtractedLink {
61    /// 完整 URL（已解析为绝对地址）
62    pub url: String,
63    /// 锚文本（已清理多余空白）
64    pub text: String,
65    /// 综合评分（8 个信号的加权和，越高越可能是文章链接）
66    pub score: f32,
67    /// 该链接所在 DOM 路径下同类链接的数量（列表结构强度）
68    pub cluster_size: usize,
69}
70
71/// 提取器配置，按需调整以适配不同站点。
72///
73/// # 默认值
74///
75/// | 字段 | 默认值 | 说明 |
76/// |------|--------|------|
77/// | `score_threshold` | `1.5` | 低于此分的链接会被过滤 |
78/// | `cluster_depth` | `4` | DOM 路径签名向上遍历层数 |
79/// | `cluster_min_size` | `4` | 聚类大小 >= 此值视为强信号 |
80/// | `min_text_len` | `8` | 锚文本短于此值扣分 |
81/// | `use_url_rule` | `true` | 是否开启 UrlRule 辅助信号 |
82/// | `non_article_paths` | 24 条默认路径 | 含 `/people/`、`/about/` 等 |
83///
84/// ```rust
85/// use crawlkit_parser::ExtractorConfig;
86///
87/// // 低阈值模式：捕获更多候选链接
88/// let config = ExtractorConfig {
89///     score_threshold: 0.5,
90///     min_text_len: 4,
91///     ..ExtractorConfig::default()
92/// };
93///
94/// // 严格模式：只保留高置信度链接
95/// let strict = ExtractorConfig {
96///     score_threshold: 3.0,
97///     cluster_min_size: 6,
98///     ..ExtractorConfig::default()
99/// };
100/// ```
101#[derive(Debug, Clone)]
102pub struct ExtractorConfig {
103    /// 最终判定为"文章链接"的分数阈值
104    pub score_threshold: f32,
105    /// 向上查找父节点计算路径签名的层数（越大越精细，但也越容易把
106    /// 结构略有差异的同类项拆散）
107    pub cluster_depth: usize,
108    /// 一个聚类至少要有多少条链接，才被视为"列表结构"强信号
109    pub cluster_min_size: usize,
110    /// 锚文本短于此长度的链接会被判定为可能是导航/按钮而非标题
111    pub min_text_len: usize,
112    /// 是否叠加 UrlRule 的正则规则作为附加信号
113    pub use_url_rule: bool,
114    /// URL 路径关键词惩罚列表（如 /people/ /about/），可追加站点自定义路径
115    pub non_article_paths: Vec<String>,
116}
117
118fn default_non_article_paths() -> Vec<String> {
119    vec![
120        "/people/", "/team/", "/about/", "/contact/",
121        "/careers/", "/staff/", "/leadership/", "/board/",
122        "/privacy", "/join", "/copyright", "/terms", "/subscribe",
123        "/accessibility", "/press-inquiries", "/internships",
124        "/jurisdiction", "/committee-", "/subcommittee-",
125        "/chairman-", "/ranking-member", "/whistleblower",
126        "/sitemap", "/faq", "/donate", "/support-",
127    ]
128    .into_iter()
129    .map(String::from)
130    .collect()
131}
132
133impl Default for ExtractorConfig {
134    fn default() -> Self {
135        Self {
136            score_threshold: 1.5,
137            cluster_depth: 4,
138            cluster_min_size: 4,
139            min_text_len: 8,
140            use_url_rule: true,
141            non_article_paths: default_non_article_paths(),
142        }
143    }
144}
145
146/// 基于 DOM 结构聚类 + 多信号打分的文章链接提取器。
147///
148/// 算法流程：
149/// 1. 提取页面中所有 `<a[href]>` 元素
150/// 2. 对每个链接计算 DOM 路径签名（`dom_path_signature`），按签名聚类
151/// 3. 对每个候选链接计算 8 个信号的加权评分
152/// 4. 同一 URL 保留最高分，过滤低于阈值的链接，按分数降序排列
153///
154/// # 示例
155///
156/// ```rust
157/// use crawlkit_parser::{LinkExtractor, ExtractorConfig};
158///
159/// let html = r#"
160/// <html><body>
161///     <div class="list">
162///         <a href="/2024/01/15/story-one">第一篇新闻标题</a>
163///         <a href="/2024/01/16/story-two">第二篇新闻标题</a>
164///         <a href="/2024/01/17/story-three">第三篇新闻标题</a>
165///         <a href="/2024/01/18/story-four">第四篇新闻标题</a>
166///     </div>
167/// </body></html>
168/// "#;
169/// let extractor = LinkExtractor::new(ExtractorConfig::default());
170/// let results = extractor.extract(html, "https://example.com/");
171/// assert_eq!(results.len(), 4);
172/// ```
173pub struct LinkExtractor {
174    config: ExtractorConfig,
175    url_rule: UrlRule,
176}
177
178struct Candidate {
179    url: String,
180    text: String,
181    path_sig: String,
182    href_raw: String,
183}
184
185impl LinkExtractor {
186    /// 创建一个新的提取器。
187    ///
188    /// 内部使用默认的 `UrlRule`，可通过 `with_url_rule` 覆盖。
189    ///
190    /// ```rust
191    /// use crawlkit_parser::{LinkExtractor, ExtractorConfig};
192    ///
193    /// let extractor = LinkExtractor::new(ExtractorConfig::default());
194    /// ```
195    pub fn new(config: ExtractorConfig) -> Self {
196        Self {
197            config,
198            url_rule: UrlRule::default(),
199        }
200    }
201
202    /// 设置自定义 UrlRule，覆盖默认规则。
203    ///
204    /// ```rust
205    /// use crawlkit_parser::{LinkExtractor, ExtractorConfig, UrlRule};
206    ///
207    /// let rule = UrlRule::default().with_include(r"/press/");
208    /// let extractor = LinkExtractor::new(ExtractorConfig::default())
209    ///     .with_url_rule(rule);
210    /// ```
211    pub fn with_url_rule(mut self, rule: UrlRule) -> Self {
212        self.url_rule = rule;
213        self
214    }
215
216    /// 从 HTML 中提取候选文章链接，按 score 从高到低排列。
217    ///
218    /// # 参数
219    /// - `html`: 页面 HTML 源码
220    /// - `base_url`: 页面 URL，用于相对路径解析和站外链接判定
221    ///
222    /// # 返回值
223    /// 按评分降序排列的文章链接列表，已过滤低于 `score_threshold` 的链接。
224    ///
225    /// ```rust
226    /// use crawlkit_parser::{LinkExtractor, ExtractorConfig};
227    ///
228    /// let html = r#"
229    /// <ul>
230    ///     <li><a href="/news/2025/01/01/story-a">2025年第一篇新闻</a></li>
231    ///     <li><a href="/news/2025/01/02/story-b">2025年第二篇新闻</a></li>
232    ///     <li><a href="/news/2025/01/03/story-c">2025年第三篇新闻</a></li>
233    ///     <li><a href="/news/2025/01/04/story-d">2025年第四篇新闻</a></li>
234    /// </ul>
235    /// "#;
236    /// let extractor = LinkExtractor::new(ExtractorConfig::default());
237    /// let results = extractor.extract(html, "https://example.com/");
238    /// assert!(results.len() >= 4);
239    /// for link in &results {
240    ///     println!("{} score={}", link.url, link.score);
241    /// }
242    /// ```
243    pub fn extract(&self, html: &str, base_url: &str) -> Vec<ExtractedLink> {
244        let document = Html::parse_document(html);
245        let a_selector = Selector::parse("a[href]").expect("valid selector");
246        let base = Url::parse(base_url).ok();
247        let base_host = base.as_ref().and_then(|u| u.host_str()).unwrap_or("");
248
249        let mut candidates: Vec<Candidate> = Vec::new();
250
251        for a in document.select(&a_selector) {
252            let href = match a.value().attr("href") {
253                Some(h) => h.trim(),
254                None => continue,
255            };
256            if href.is_empty() || href.starts_with('#') {
257                continue;
258            }
259            if href.starts_with("javascript:") || href.starts_with("mailto:") || href.starts_with("tel:") {
260                continue;
261            }
262
263            let resolved = resolve_url(&base, href);
264            let text = clean_text(&a.text().collect::<Vec<_>>().join(" "));
265            let path_sig = dom_path_signature(a, self.config.cluster_depth);
266
267            candidates.push(Candidate {
268                url: resolved,
269                text,
270                path_sig,
271                href_raw: href.to_string(),
272            });
273        }
274
275        let mut cluster_counts: HashMap<String, usize> = HashMap::new();
276        for c in &candidates {
277            *cluster_counts.entry(c.path_sig.clone()).or_insert(0) += 1;
278        }
279
280        let mut best_by_url: HashMap<String, ExtractedLink> = HashMap::new();
281
282        for c in &candidates {
283            let cluster_size = *cluster_counts.get(&c.path_sig).unwrap_or(&0);
284            let score = self.score_candidate(c, cluster_size, base_host);
285
286            best_by_url
287                .entry(c.url.clone())
288                .and_modify(|entry| {
289                    if c.text.chars().count() > entry.text.chars().count() {
290                        entry.text = c.text.clone();
291                    }
292                    entry.score = entry.score.max(score);
293                    entry.cluster_size = entry.cluster_size.max(cluster_size);
294                })
295                .or_insert_with(|| ExtractedLink {
296                    url: c.url.clone(),
297                    text: c.text.clone(),
298                    score,
299                    cluster_size,
300                });
301        }
302
303        let mut results: Vec<ExtractedLink> = best_by_url
304            .into_values()
305            .filter(|l| l.score >= self.config.score_threshold)
306            .collect();
307
308        results.sort_by(|a, b| {
309            b.score
310                .partial_cmp(&a.score)
311                .unwrap_or(std::cmp::Ordering::Equal)
312        });
313        results
314    }
315
316    /// 对单个候选链接计算 8 个信号的加权评分。
317    ///
318    /// | # | 信号 | 权重 | 说明 |
319    /// |---|------|------|------|
320    /// | 1 | 锚文本长度 | +1.0 / +0.5 / -0.5 | >= min_text_len 加分；<= 120 额外加分；空文本扣分 |
321    /// | 2 | 聚类大小 | +1.5 / +0.5 | >= cluster_min_size 强加分；>= 2 小幅加分 |
322    /// | 3 | UrlRule 匹配 | +0.5 | 复用正则规则作为辅助信号 |
323    /// | 4 | 路径深度 | +0.3 / -1.0 | 路径段 >= 2 加分，浅路径扣分 |
324    /// | 5 | 导航词 | -1.5 | 匹配 NAV_WORDS 列表扣分 |
325    /// | 6 | 分页/数字 | -2.0 / -1.5 | 含 page 参数或纯数字文本扣分 |
326    /// | 7 | 外站链接 | -2.0 | 不同主机名的链接扣分 |
327    /// | 8 | 非文章路径 | -2.0 | 匹配 non_article_paths 扣分 |
328    fn score_candidate(&self, c: &Candidate, cluster_size: usize, base_host: &str) -> f32 {
329        let mut score = 0.0f32;
330
331        let text_len = c.text.chars().count();
332        if text_len >= self.config.min_text_len {
333            score += 1.0;
334            if text_len <= 120 {
335                score += 0.5;
336            }
337        } else if text_len == 0 {
338            score -= 0.5;
339        }
340
341        if cluster_size >= self.config.cluster_min_size {
342            score += 1.5;
343        } else if cluster_size >= 2 {
344            score += 0.5;
345        }
346
347        if self.config.use_url_rule && self.url_rule.is_article_url(&c.url) {
348            score += 0.5;
349        }
350
351        let path_segments = c
352            .url
353            .split('/')
354            .skip(3)
355            .filter(|s| !s.is_empty())
356            .count();
357        if path_segments >= 2 {
358            score += 0.3;
359        } else {
360            score -= 1.0;
361        }
362
363        let lower_text = c.text.to_lowercase();
364        const NAV_WORDS: [&str; 12] = [
365            "home", "login", "sign in", "sign up", "subscribe", "next", "prev",
366            "previous", "more", "更多", "首页", "登录",
367        ];
368        if NAV_WORDS.iter().any(|w| lower_text == *w) {
369            score -= 1.5;
370        }
371
372        if c.href_raw.contains("?page=") || c.href_raw.contains("&page=") {
373            score -= 2.0;
374        }
375        if !c.text.trim().is_empty() && c.text.trim().chars().all(|c| c.is_ascii_digit()) {
376            score -= 1.5;
377        }
378
379        if let Ok(url) = Url::parse(&c.url)
380            && let Some(host) = url.host_str()
381                && !host.is_empty() && host != base_host {
382                    score -= 2.0;
383                }
384
385        if self.config.non_article_paths.iter().any(|p| c.url.contains(p)) {
386            score -= 2.0;
387        }
388
389        score
390    }
391}
392
393fn resolve_url(base: &Option<Url>, href: &str) -> String {
394    match base {
395        Some(b) => b
396            .join(href).map_or_else(|_| href.to_string(), |u| u.to_string()),
397        None => href.to_string(),
398    }
399}
400
401fn clean_text(s: &str) -> String {
402    s.split_whitespace().collect::<Vec<_>>().join(" ")
403}
404
405fn dom_path_signature(a: ElementRef, depth: usize) -> String {
406    let mut parts = Vec::new();
407    let mut current = a.parent();
408    let mut d = 0;
409
410    while let Some(node) = current {
411        if d >= depth {
412            break;
413        }
414        if let Some(el) = ElementRef::wrap(node) {
415            let tag = el.value().name();
416            if tag == "html" || tag == "body" {
417                break;
418            }
419            let mut classes: Vec<&str> = el.value().classes().collect();
420            classes.sort_unstable();
421            let class_part = if classes.is_empty() {
422                String::new()
423            } else {
424                format!(".{}", classes.join("."))
425            };
426            parts.push(format!("{tag}{class_part}"));
427        }
428        current = node.parent();
429        d += 1;
430    }
431
432    parts.reverse();
433    parts.join(">")
434}
435
436#[cfg(test)]
437mod tests {
438    use super::*;
439
440    #[test]
441    fn test_extracts_repeated_list_items_over_scattered_nav_links() {
442        let html = r#"
443        <html><body>
444            <nav>
445                <a href="/">首页</a>
446                <a href="/login">登录</a>
447            </nav>
448            <div class="news-list">
449                <ul>
450                    <li><a href="/2024/01/15/story-one">这是第一条新闻标题</a></li>
451                    <li><a href="/2024/01/16/story-two">这是第二条新闻标题</a></li>
452                    <li><a href="/2024/01/17/story-three">这是第三条新闻标题</a></li>
453                    <li><a href="/2024/01/18/story-four">这是第四条新闻标题</a></li>
454                    <li><a href="/2024/01/19/story-five">这是第五条新闻标题</a></li>
455                </ul>
456            </div>
457        </body></html>
458        "#;
459
460        let extractor = LinkExtractor::new(ExtractorConfig::default());
461        let results = extractor.extract(html, "https://example.com/");
462        let urls: Vec<&str> = results.iter().map(|l| l.url.as_str()).collect();
463
464        assert!(urls.iter().any(|u| u.contains("story-one")));
465        assert!(urls.iter().any(|u| u.contains("story-five")));
466        assert!(!urls.iter().any(|u| u.ends_with("/login")));
467    }
468
469    #[test]
470    fn test_short_nav_text_is_penalized() {
471        let html = r#"
472        <html><body>
473            <a href="/next">Next</a>
474            <a href="/article/full-title-of-a-real-article">完整的一篇真实文章标题在这里</a>
475        </body></html>
476        "#;
477        let extractor = LinkExtractor::new(ExtractorConfig::default());
478        let results = extractor.extract(html, "https://example.com/");
479        assert!(results[0].url.contains("full-title-of-a-real-article"));
480    }
481
482    #[test]
483    fn test_relative_url_resolution() {
484        let html = r#"<html><body><a href="/news/foo">新闻标题足够长这样才算数</a></body></html>"#;
485        let extractor = LinkExtractor::new(ExtractorConfig::default());
486        let results = extractor.extract(html, "https://example.com/section/");
487        assert_eq!(results[0].url, "https://example.com/news/foo");
488    }
489
490    #[test]
491    fn test_slug_only_site_without_regex_hints() {
492        let html = r#"
493        <html><body>
494            <div class="views-row"><a href="/health-policy/making-a-deposit/">Making a Deposit in Health Policy</a></div>
495            <div class="views-row"><a href="/economics/inflation-outlook/">The Inflation Outlook</a></div>
496            <div class="views-row"><a href="/foreign-policy/china-strategy/">Rethinking China Strategy</a></div>
497            <div class="views-row"><a href="/education/school-choice/">The Case for School Choice</a></div>
498        </body></html>
499        "#;
500        let extractor = LinkExtractor::new(ExtractorConfig {
501            use_url_rule: false,
502            ..ExtractorConfig::default()
503        });
504        let results = extractor.extract(html, "https://www.aei.org/");
505        assert!(results.iter().any(|l| l.url.contains("making-a-deposit")));
506        assert!(results.iter().any(|l| l.url.contains("china-strategy")));
507    }
508
509    #[test]
510    fn test_with_url_rule_boosts_score() {
511        let html = r#"<html><body><a href="/2024/01/15/some-story">一篇独立的新闻文章标题</a></body></html>"#;
512        let with_rule = LinkExtractor::new(ExtractorConfig::default());
513        let without_rule = LinkExtractor::new(ExtractorConfig {
514            use_url_rule: false,
515            ..ExtractorConfig::default()
516        });
517        let r1 = with_rule.extract(html, "https://example.com/");
518        let r2 = without_rule.extract(html, "https://example.com/");
519        assert!(r1[0].score > r2[0].score);
520    }
521}
crawlkit_parser/extractor.rs

crawlkit_parser/
extractor.rs
