use std::collections::HashMap;
use scraper::{ElementRef, Html, Selector};
use url::Url;
use crate::rules::UrlRule;
#[derive(Debug, Clone)]
pub struct ExtractedLink {
pub url: String,
pub text: String,
pub score: f32,
pub cluster_size: usize,
}
#[derive(Debug, Clone)]
pub struct ExtractorConfig {
pub score_threshold: f32,
pub cluster_depth: usize,
pub cluster_min_size: usize,
pub min_text_len: usize,
pub use_url_rule: bool,
pub non_article_paths: Vec<String>,
}
fn default_non_article_paths() -> Vec<String> {
vec![
"/people/", "/team/", "/about/", "/contact/",
"/careers/", "/staff/", "/leadership/", "/board/",
"/privacy", "/join", "/copyright", "/terms", "/subscribe",
"/accessibility", "/press-inquiries", "/internships",
"/jurisdiction", "/committee-", "/subcommittee-",
"/chairman-", "/ranking-member", "/whistleblower",
"/sitemap", "/faq", "/donate", "/support-",
]
.into_iter()
.map(String::from)
.collect()
}
impl Default for ExtractorConfig {
fn default() -> Self {
Self {
score_threshold: 1.5,
cluster_depth: 4,
cluster_min_size: 4,
min_text_len: 8,
use_url_rule: true,
non_article_paths: default_non_article_paths(),
}
}
}
pub struct LinkExtractor {
config: ExtractorConfig,
url_rule: UrlRule,
}
struct Candidate {
url: String,
text: String,
path_sig: String,
href_raw: String,
}
impl LinkExtractor {
pub fn new(config: ExtractorConfig) -> Self {
Self {
config,
url_rule: UrlRule::default(),
}
}
pub fn with_url_rule(mut self, rule: UrlRule) -> Self {
self.url_rule = rule;
self
}
pub fn extract(&self, html: &str, base_url: &str) -> Vec<ExtractedLink> {
let document = Html::parse_document(html);
let a_selector = Selector::parse("a[href]").expect("valid selector");
let base = Url::parse(base_url).ok();
let base_host = base.as_ref().and_then(|u| u.host_str()).unwrap_or("");
let mut candidates: Vec<Candidate> = Vec::new();
for a in document.select(&a_selector) {
let href = match a.value().attr("href") {
Some(h) => h.trim(),
None => continue,
};
if href.is_empty() || href.starts_with('#') {
continue;
}
if href.starts_with("javascript:") || href.starts_with("mailto:") || href.starts_with("tel:") {
continue;
}
let resolved = resolve_url(&base, href);
let text = clean_text(&a.text().collect::<Vec<_>>().join(" "));
let path_sig = dom_path_signature(a, self.config.cluster_depth);
candidates.push(Candidate {
url: resolved,
text,
path_sig,
href_raw: href.to_string(),
});
}
let mut cluster_counts: HashMap<String, usize> = HashMap::new();
for c in &candidates {
*cluster_counts.entry(c.path_sig.clone()).or_insert(0) += 1;
}
let mut best_by_url: HashMap<String, ExtractedLink> = HashMap::new();
for c in &candidates {
let cluster_size = *cluster_counts.get(&c.path_sig).unwrap_or(&0);
let score = self.score_candidate(c, cluster_size, base_host);
best_by_url
.entry(c.url.clone())
.and_modify(|entry| {
if c.text.chars().count() > entry.text.chars().count() {
entry.text = c.text.clone();
}
entry.score = entry.score.max(score);
entry.cluster_size = entry.cluster_size.max(cluster_size);
})
.or_insert_with(|| ExtractedLink {
url: c.url.clone(),
text: c.text.clone(),
score,
cluster_size,
});
}
let mut results: Vec<ExtractedLink> = best_by_url
.into_values()
.filter(|l| l.score >= self.config.score_threshold)
.collect();
results.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
results
}
fn score_candidate(&self, c: &Candidate, cluster_size: usize, base_host: &str) -> f32 {
let mut score = 0.0f32;
let text_len = c.text.chars().count();
if text_len >= self.config.min_text_len {
score += 1.0;
if text_len <= 120 {
score += 0.5;
}
} else if text_len == 0 {
score -= 0.5;
}
if cluster_size >= self.config.cluster_min_size {
score += 1.5;
} else if cluster_size >= 2 {
score += 0.5;
}
if self.config.use_url_rule && self.url_rule.is_article_url(&c.url) {
score += 0.5;
}
let path_segments = c
.url
.split('/')
.skip(3)
.filter(|s| !s.is_empty())
.count();
if path_segments >= 2 {
score += 0.3;
} else {
score -= 1.0;
}
let lower_text = c.text.to_lowercase();
const NAV_WORDS: [&str; 12] = [
"home", "login", "sign in", "sign up", "subscribe", "next", "prev",
"previous", "more", "更多", "首页", "登录",
];
if NAV_WORDS.iter().any(|w| lower_text == *w) {
score -= 1.5;
}
if c.href_raw.contains("?page=") || c.href_raw.contains("&page=") {
score -= 2.0;
}
if !c.text.trim().is_empty() && c.text.trim().chars().all(|c| c.is_ascii_digit()) {
score -= 1.5;
}
if let Ok(url) = Url::parse(&c.url) {
if let Some(host) = url.host_str() {
if !host.is_empty() && host != base_host {
score -= 2.0;
}
}
}
if self.config.non_article_paths.iter().any(|p| c.url.contains(p)) {
score -= 2.0;
}
score
}
}
fn resolve_url(base: &Option<Url>, href: &str) -> String {
match base {
Some(b) => b
.join(href)
.map(|u| u.to_string())
.unwrap_or_else(|_| href.to_string()),
None => href.to_string(),
}
}
fn clean_text(s: &str) -> String {
s.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn dom_path_signature(a: ElementRef, depth: usize) -> String {
let mut parts = Vec::new();
let mut current = a.parent();
let mut d = 0;
while let Some(node) = current {
if d >= depth {
break;
}
if let Some(el) = ElementRef::wrap(node) {
let tag = el.value().name();
if tag == "html" || tag == "body" {
break;
}
let mut classes: Vec<&str> = el.value().classes().collect();
classes.sort_unstable();
let class_part = if classes.is_empty() {
String::new()
} else {
format!(".{}", classes.join("."))
};
parts.push(format!("{tag}{class_part}"));
}
current = node.parent();
d += 1;
}
parts.reverse();
parts.join(">")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extracts_repeated_list_items_over_scattered_nav_links() {
let html = r#"
<html><body>
<nav>
<a href="/">首页</a>
<a href="/login">登录</a>
</nav>
<div class="news-list">
<ul>
<li><a href="/2024/01/15/story-one">这是第一条新闻标题</a></li>
<li><a href="/2024/01/16/story-two">这是第二条新闻标题</a></li>
<li><a href="/2024/01/17/story-three">这是第三条新闻标题</a></li>
<li><a href="/2024/01/18/story-four">这是第四条新闻标题</a></li>
<li><a href="/2024/01/19/story-five">这是第五条新闻标题</a></li>
</ul>
</div>
</body></html>
"#;
let extractor = LinkExtractor::new(ExtractorConfig::default());
let results = extractor.extract(html, "https://example.com/");
let urls: Vec<&str> = results.iter().map(|l| l.url.as_str()).collect();
assert!(urls.iter().any(|u| u.contains("story-one")));
assert!(urls.iter().any(|u| u.contains("story-five")));
assert!(!urls.iter().any(|u| u.ends_with("/login")));
}
#[test]
fn test_short_nav_text_is_penalized() {
let html = r#"
<html><body>
<a href="/next">Next</a>
<a href="/article/full-title-of-a-real-article">完整的一篇真实文章标题在这里</a>
</body></html>
"#;
let extractor = LinkExtractor::new(ExtractorConfig::default());
let results = extractor.extract(html, "https://example.com/");
assert!(results[0].url.contains("full-title-of-a-real-article"));
}
#[test]
fn test_relative_url_resolution() {
let html = r#"<html><body><a href="/news/foo">新闻标题足够长这样才算数</a></body></html>"#;
let extractor = LinkExtractor::new(ExtractorConfig::default());
let results = extractor.extract(html, "https://example.com/section/");
assert_eq!(results[0].url, "https://example.com/news/foo");
}
#[test]
fn test_slug_only_site_without_regex_hints() {
let html = r#"
<html><body>
<div class="views-row"><a href="/health-policy/making-a-deposit/">Making a Deposit in Health Policy</a></div>
<div class="views-row"><a href="/economics/inflation-outlook/">The Inflation Outlook</a></div>
<div class="views-row"><a href="/foreign-policy/china-strategy/">Rethinking China Strategy</a></div>
<div class="views-row"><a href="/education/school-choice/">The Case for School Choice</a></div>
</body></html>
"#;
let extractor = LinkExtractor::new(ExtractorConfig {
use_url_rule: false,
..ExtractorConfig::default()
});
let results = extractor.extract(html, "https://www.aei.org/");
assert!(results.iter().any(|l| l.url.contains("making-a-deposit")));
assert!(results.iter().any(|l| l.url.contains("china-strategy")));
}
#[test]
fn test_with_url_rule_boosts_score() {
let html = r#"<html><body><a href="/2024/01/15/some-story">一篇独立的新闻文章标题</a></body></html>"#;
let with_rule = LinkExtractor::new(ExtractorConfig::default());
let without_rule = LinkExtractor::new(ExtractorConfig {
use_url_rule: false,
..ExtractorConfig::default()
});
let r1 = with_rule.extract(html, "https://example.com/");
let r2 = without_rule.extract(html, "https://example.com/");
assert!(r1[0].score > r2[0].score);
}
}