crawlkit_parser/
html.rs

1//! HTML 解析工具集
2//!
3//! 提供链接提取、文章内容提取、可读性提取等实用功能。
4//! 底层使用 `scraper`（基于 html5ever）和 `dom_smoothie`（Readability 模式）。
5
6use dom_smoothie::{Config, TextMode};
7use scraper::{Html, Selector};
8use skyscraper::{
9    html as xpath_html,
10    xpath::{
11        self, XpathItemTree,
12        grammar::{
13            NonTreeXpathNode, XpathItemTreeNodeData,
14            data_model::{Node, XpathItem},
15        },
16    },
17};
18use url::Url;
19
20use crawlkit_core::error::{CrawlError, Result};
21
22/// 链接选择器类型
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum LinkSelectorType {
25    /// CSS 选择器，例如 `a[href]`
26    Css,
27    /// XPath 表达式，例如 `//a/@href`
28    Xpath,
29}
30
31/// 从 HTML 中提取所有匹配 CSS 选择器的链接（href 属性）
32///
33/// - `html_content`: 完整 HTML 字符串
34/// - `selector`: CSS 选择器，如 `"a[href]"` 或 `"div.news a"`
35///
36/// 返回去重后的链接列表
37pub fn extract_links(html_content: &str, selector: &str) -> Vec<String> {
38    try_extract_links(html_content, selector).unwrap_or_default()
39}
40
41/// 从 HTML 中提取所有匹配 CSS 选择器的链接（href 属性）
42///
43/// 和 `extract_links` 相比，此函数会返回选择器解析错误，适合需要严格错误处理的场景。
44pub fn try_extract_links(html_content: &str, selector: &str) -> Result<Vec<String>> {
45    let document = Html::parse_document(html_content);
46    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
47        selector: selector.to_string(),
48        message: e.to_string(),
49    })?;
50
51    let mut seen = std::collections::HashSet::new();
52    let mut links = Vec::new();
53
54    for element in document.select(&sel) {
55        if let Some(href) = element.value().attr("href")
56            && seen.insert(href.to_string())
57        {
58            links.push(href.to_string());
59        }
60    }
61
62    Ok(links)
63}
64
65/// 按选择器类型提取链接。
66///
67/// XPath 表达式既可以指向属性节点（如 `//a/@href`），也可以指向元素节点（如 `//a`）。
68pub fn extract_links_by_selector(
69    html_content: &str,
70    selector: &str,
71    selector_type: LinkSelectorType,
72) -> Result<Vec<String>> {
73    match selector_type {
74        LinkSelectorType::Css => try_extract_links(html_content, selector),
75        LinkSelectorType::Xpath => extract_links_by_xpath(html_content, selector),
76    }
77}
78
79/// 使用 XPath 表达式提取链接。
80pub fn extract_links_by_xpath(html_content: &str, selector: &str) -> Result<Vec<String>> {
81    let sanitized = sanitize_for_xpath(html_content);
82    let document = xpath_html::parse(&sanitized).map_err(|e| CrawlError::Html(e.to_string()))?;
83    let tree = XpathItemTree::from(&document);
84    let xpath_expr = xpath::parse(selector).map_err(|e| CrawlError::Selector {
85        selector: selector.to_string(),
86        message: e.to_string(),
87    })?;
88    let item_set = xpath_expr
89        .apply(&tree)
90        .map_err(|e| CrawlError::Html(e.to_string()))?;
91
92    let mut seen = std::collections::HashSet::new();
93    let mut links = Vec::new();
94
95    for item in &item_set {
96        let href = match item {
97            XpathItem::Node(Node::NonTreeNode(NonTreeXpathNode::AttributeNode(attribute))) => {
98                attribute.value.clone()
99            }
100            XpathItem::Node(Node::TreeNode(tree_node)) => match tree_node.data {
101                XpathItemTreeNodeData::ElementNode(element) => element
102                    .get_attribute("href")
103                    .map(str::to_string)
104                    .unwrap_or_default(),
105                _ => String::new(),
106            },
107            _ => String::new(),
108        };
109
110        let href = href.trim();
111        if !href.is_empty() && seen.insert(href.to_string()) {
112            links.push(href.to_string());
113        }
114    }
115
116    Ok(links)
117}
118
119/// 将相对 URL 转为绝对 URL
120///
121/// - `base_url`: 基准 URL（当前页面）
122/// - `relative`: 相对路径或完整 URL
123pub fn resolve_url(base_url: &str, relative: &str) -> Option<String> {
124    let relative = relative.trim();
125    if should_skip_link(relative) {
126        return None;
127    }
128    let base = Url::parse(base_url).ok()?;
129    let resolved = base.join(relative).ok()?;
130    match resolved.scheme() {
131        "http" | "https" => Some(resolved.to_string()),
132        _ => None,
133    }
134}
135
136/// 提取并解析为绝对 URL，自动去重和过滤非页面链接。
137pub fn extract_absolute_links(
138    html_content: &str,
139    selector: &str,
140    selector_type: LinkSelectorType,
141    base_url: &str,
142) -> Result<Vec<String>> {
143    let links = extract_links_by_selector(html_content, selector, selector_type)?;
144    let mut seen = std::collections::HashSet::new();
145    let mut absolute_links = Vec::new();
146
147    for link in links {
148        if let Some(url) = resolve_url(base_url, &link)
149            && seen.insert(url.clone())
150        {
151            absolute_links.push(url);
152        }
153    }
154
155    Ok(absolute_links)
156}
157
158fn should_skip_link(link: &str) -> bool {
159    link.is_empty()
160        || link.starts_with('#')
161        || link.starts_with("javascript:")
162        || link.starts_with("mailto:")
163        || link.starts_with("tel:")
164        || link.starts_with("data:")
165}
166
167/// 文章内容提取结果
168#[derive(Debug, Clone, Default)]
169pub struct Article {
170    /// 文章标题
171    pub title: String,
172    /// 文章正文（纯文本）
173    pub content: String,
174    /// 发布日期
175    pub date: Option<String>,
176    /// 作者
177    pub author: Option<String>,
178    /// 描述/摘要
179    pub description: Option<String>,
180}
181
182/// 从 HTML 中提取文章内容
183///
184/// 使用启发式规则提取：按优先级尝试多种常见 DOM 结构。
185///
186/// # 提取策略
187/// 1. 优先查找 `<article>` 标签
188/// 2. 查找 `article-body` / `post-content` / `entry-content` 等常见 class
189/// 3. 查找 `<h1>` 作为标题，最大的 `<div>` 块作为正文
190///
191/// `base_url` 参数预留用于将文章中的相对路径转为绝对路径，当前版本暂未使用。
192pub fn extract_article(html_content: &str, _base_url: &str) -> Article {
193    let document = Html::parse_document(html_content);
194
195    Article {
196        title: extract_title(&document),
197        content: extract_content_heuristic(&document),
198        date: extract_meta_content(&document, "date")
199            .or_else(|| extract_meta_content(&document, "article:published_time")),
200        author: extract_meta_content(&document, "author")
201            .or_else(|| extract_meta_content(&document, "article:author")),
202        description: extract_meta_content(&document, "description")
203            .or_else(|| extract_meta_content(&document, "og:description")),
204    }
205}
206
207/// 提取页面标题：优先 og:title → h1 → <title>
208fn extract_title(document: &Html) -> String {
209    if let Ok(sel) = Selector::parse(r#"meta[property="og:title"]"#)
210        && let Some(el) = document.select(&sel).next()
211        && let Some(content) = el.value().attr("content")
212        && !content.is_empty()
213    {
214        return content.to_string();
215    }
216    if let Ok(sel) = Selector::parse("h1")
217        && let Some(el) = document.select(&sel).next()
218    {
219        let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
220        if !text.is_empty() {
221            return text;
222        }
223    }
224    if let Ok(sel) = Selector::parse("title")
225        && let Some(el) = document.select(&sel).next()
226    {
227        let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
228        if !text.is_empty() {
229            return text;
230        }
231    }
232    String::new()
233}
234
235/// 提取正文：按优先级尝试多种策略
236fn extract_content_heuristic(document: &Html) -> String {
237    // 策略 1：<article> 标签
238    if let Ok(sel) = Selector::parse("article")
239        && let Some(el) = document.select(&sel).next()
240    {
241        let text = element_to_text(&el);
242        if text.len() > 100 {
243            return text;
244        }
245    }
246
247    // 策略 2：常见文章容器 class
248    let content_selectors = &[
249        "article-body",
250        "post-content",
251        "entry-content",
252        "article-content",
253        "news-content",
254        "story-body",
255        ".content-article",
256        "#article-body",
257        ".article-body",
258        ".post-body",
259    ];
260
261    for selector_str in content_selectors {
262        if let Ok(sel) = Selector::parse(selector_str)
263            && let Some(el) = document.select(&sel).next()
264        {
265            let text = element_to_text(&el);
266            if text.len() > 100 {
267                return text;
268            }
269        }
270    }
271
272    // 策略 3：找最大的文本块（启发式兜底）
273    if let Ok(sel) = Selector::parse("div") {
274        let divs: Vec<_> = document.select(&sel).collect();
275        let mut best = String::new();
276        for div in divs {
277            let text = element_to_text(&div);
278            if text.len() > 200 && text.len() < 50_000 && text.len() > best.len() {
279                best = text;
280            }
281        }
282        if !best.is_empty() {
283            return best;
284        }
285    }
286
287    String::new()
288}
289
290/// 从 <meta> 标签提取 content 属性
291fn extract_meta_content(document: &Html, name: &str) -> Option<String> {
292    let sel_str = format!(r#"meta[name="{name}"]"#);
293    if let Ok(sel) = Selector::parse(&sel_str)
294        && let Some(el) = document.select(&sel).next()
295        && let Some(content) = el.value().attr("content")
296    {
297        let content = content.trim().to_string();
298        if !content.is_empty() {
299            return Some(content);
300        }
301    }
302    let sel_str = format!(r#"meta[property="{name}"]"#);
303    if let Ok(sel) = Selector::parse(&sel_str)
304        && let Some(el) = document.select(&sel).next()
305        && let Some(content) = el.value().attr("content")
306    {
307        let content = content.trim().to_string();
308        if !content.is_empty() {
309            return Some(content);
310        }
311    }
312    None
313}
314
315// ──────────────────────────────────────────────
316// 可读性提取（基于 dom_smoothie）
317// ──────────────────────────────────────────────
318
319/// 使用 dom_smoothie 提取文章正文（Readability 模式）
320///
321/// ```rust
322/// let html = r#"<html><body><article><p>正文内容</p></article></body></html>"#;
323/// let content = crawlkit_parser::html::extract_readable_content(html).unwrap();
324/// ```
325pub fn extract_readable_content(raw_html: &str) -> Result<String> {
326    let cfg = Config {
327        text_mode: TextMode::Markdown,
328        ..Default::default()
329    };
330
331    let mut readability = dom_smoothie::Readability::new(raw_html, None, Some(cfg))
332        .map_err(|e| CrawlError::Readability(e.to_string()))?;
333
334    let article = readability
335        .parse()
336        .map_err(|e| CrawlError::Readability(e.to_string()))?;
337
338    Ok(article.content.to_string())
339}
340
341/// 使用 CSS 选择器提取文章正文
342///
343/// ```rust
344/// let html = r#"<html><body><div class="content">正文</div></body></html>"#;
345/// let content = crawlkit_parser::html::extract_content_by_selector(html, "div.content").unwrap();
346/// ```
347pub fn extract_content_by_selector(raw_html: &str, content_selector: &str) -> Result<String> {
348    let document = Html::parse_document(raw_html);
349    let selector = Selector::parse(content_selector).map_err(|e| CrawlError::Selector {
350        selector: content_selector.to_string(),
351        message: e.to_string(),
352    })?;
353
354    let content = document
355        .select(&selector)
356        .next()
357        .map(|el| el.text().collect::<Vec<_>>().join("\n").trim().to_string())
358        .unwrap_or_default();
359
360    Ok(content)
361}
362
363/// 智能提取：优先 Readability，失败则回退到 CSS 选择器
364///
365/// ```rust
366/// let html = r#"<html><body><article><p>正文</p></article></body></html>"#;
367/// let content = crawlkit_parser::html::extract_content(html, "article").unwrap();
368/// ```
369pub fn extract_content(raw_html: &str, content_selector: &str) -> Result<String> {
370    match extract_readable_content(raw_html) {
371        Ok(content) if !content.is_empty() && content.len() > 100 => Ok(content),
372        _ => extract_content_by_selector(raw_html, content_selector),
373    }
374}
375
376/// 提取匹配指定选择器的所有文本内容
377///
378/// ```rust
379/// let html = r#"<html><body><p>段落1</p><p>段落2</p></body></html>"#;
380/// let texts = crawlkit_parser::html::extract_texts(html, "p").unwrap();
381/// ```
382pub fn extract_texts(raw_html: &str, selector: &str) -> Result<Vec<String>> {
383    let document = Html::parse_document(raw_html);
384    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
385        selector: selector.to_string(),
386        message: e.to_string(),
387    })?;
388
389    let texts: Vec<String> = document
390        .select(&sel)
391        .filter_map(|el| {
392            let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
393            if text.is_empty() { None } else { Some(text) }
394        })
395        .collect();
396
397    Ok(texts)
398}
399
400/// 提取匹配选择器的元素的指定属性值
401pub fn extract_attributes(raw_html: &str, selector: &str, attr: &str) -> Result<Vec<String>> {
402    let document = Html::parse_document(raw_html);
403    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
404        selector: selector.to_string(),
405        message: e.to_string(),
406    })?;
407
408    let values: Vec<String> = document
409        .select(&sel)
410        .filter_map(|el| el.value().attr(attr).map(ToString::to_string))
411        .filter(|v| !v.is_empty())
412        .collect();
413
414    Ok(values)
415}
416
417/// 将 HTML 元素转为纯文本（保留段落分隔）
418fn element_to_text(element: &scraper::ElementRef) -> String {
419    let mut result = String::new();
420    for text_piece in element.text() {
421        let t = text_piece.trim();
422        if !t.is_empty() {
423            result.push_str(t);
424            result.push('\n');
425        }
426    }
427    let lines: Vec<&str> = result
428        .lines()
429        .map(str::trim)
430        .filter(|l| !l.is_empty())
431        .collect();
432    lines.join("\n")
433}
434
435#[cfg(test)]
436mod tests {
437    use super::*;
438
439    fn sample_html() -> &'static str {
440        r##"<!DOCTYPE html>
441<html>
442<body>
443  <a class="link" href="https://example.com/article/1">文章1</a>
444  <a class="link" href="/article/2">文章2</a>
445  <a class="link" href="//cdn.example.com/article/3">文章3</a>
446  <a class="link" href="#top">锚点</a>
447  <a class="link" href="javascript:void(0)">脚本</a>
448  <a class="link" href="mailto:test@example.com">邮箱</a>
449</body>
450</html>"##
451    }
452
453    #[test]
454    fn try_extract_links_returns_selector_error() {
455        let result = try_extract_links("<html></html>", "!!!invalid");
456        assert!(result.is_err());
457    }
458
459    #[test]
460    fn extract_absolute_links_by_css_filters_and_resolves() {
461        let links = extract_absolute_links(
462            sample_html(),
463            "a.link",
464            LinkSelectorType::Css,
465            "https://example.com/base/",
466        )
467        .unwrap();
468
469        assert_eq!(links.len(), 3);
470        assert!(links.contains(&"https://example.com/article/1".to_string()));
471        assert!(links.contains(&"https://example.com/article/2".to_string()));
472        assert!(links.contains(&"https://cdn.example.com/article/3".to_string()));
473    }
474
475    #[test]
476    fn extract_absolute_links_by_xpath_reads_href_attributes() {
477        let links = extract_absolute_links(
478            sample_html(),
479            "//a[@class='link']/@href",
480            LinkSelectorType::Xpath,
481            "https://example.com",
482        )
483        .unwrap();
484
485        assert_eq!(links.len(), 3);
486        assert!(links.contains(&"https://example.com/article/1".to_string()));
487        assert!(links.contains(&"https://example.com/article/2".to_string()));
488        assert!(links.contains(&"https://cdn.example.com/article/3".to_string()));
489    }
490
491    #[test]
492    fn resolve_url_skips_non_page_links() {
493        assert!(resolve_url("https://example.com", "javascript:void(0)").is_none());
494        assert!(resolve_url("https://example.com", "mailto:test@example.com").is_none());
495        assert!(resolve_url("https://example.com", "#top").is_none());
496    }
497
498    #[test]
499    fn test_sanitize_for_xpath_meta() {
500        let html = r#"<html><head><meta charset="utf-8"></head><body></body></html>"#;
501        let sanitized = sanitize_for_xpath(html);
502        assert!(sanitized.contains("meta"));
503        assert!(sanitized.contains("charset"));
504    }
505
506    #[test]
507    fn test_sanitize_for_xpath_self_closing_unchanged() {
508        let html = r#"<html><head><meta charset="utf-8" /></head><body></body></html>"#;
509        let sanitized = sanitize_for_xpath(html);
510        assert!(sanitized.contains("meta"));
511        assert!(sanitized.contains("charset"));
512    }
513
514    #[test]
515    fn test_sanitize_for_xpath_multiple_void_elements() {
516        let html = r#"<br><img src="x.png"><input type="text"><link rel="stylesheet">"#;
517        let sanitized = sanitize_for_xpath(html);
518        assert!(sanitized.contains("br"));
519        assert!(sanitized.contains("img"));
520        assert!(sanitized.contains("input"));
521        assert!(sanitized.contains("link"));
522    }
523
524    #[test]
525    fn test_sanitize_for_xpath_non_void_unchanged() {
526        let html = r#"<div><p>Hello</p></div>"#;
527        let sanitized = sanitize_for_xpath(html);
528        assert!(sanitized.contains("<div>"));
529        assert!(sanitized.contains("<p>"));
530    }
531}
532
533/// 将 HTML 中未自闭合的 void 元素转为自闭合格式
534///
535/// skyscraper 的 HTML 解析器是 XML 严格模式，遇到 `<meta>` 会报错。
536/// 此函数用 scraper（html5ever，宽容解析）先解析再序列化，
537/// 得到格式良好的 HTML，使 skyscraper 能正确解析。
538pub fn sanitize_for_xpath(html: &str) -> String {
539    let doc = Html::parse_document(html);
540    doc.html()
541}
crawlkit_parser/html.rs

crawlkit_parser/
html.rs
