crawlkit_parser/
html.rs

1//! HTML 解析工具集
2//!
3//! 提供链接提取、文章内容提取、可读性提取等实用功能。
4//! 底层使用 `scraper`（基于 html5ever）和 `dom_smoothie`（Readability 模式）。
5
6use dom_smoothie::{Config, Readability, TextMode};
7use scraper::{Html, Selector};
8use skyscraper::{
9    html as xpath_html,
10    xpath::{
11        self, XpathItemTree,
12        grammar::{
13            NonTreeXpathNode, XpathItemTreeNodeData,
14            data_model::{Node, XpathItem},
15        },
16    },
17};
18use url::Url;
19
20use crawlkit_core::error::{CrawlError, Result};
21
22/// 链接选择器类型
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum LinkSelectorType {
25    /// CSS 选择器，例如 `a[href]`
26    Css,
27    /// XPath 表达式，例如 `//a/@href`
28    Xpath,
29}
30
31/// 从 HTML 中提取所有匹配 CSS 选择器的链接（href 属性）
32///
33/// - `html_content`: 完整 HTML 字符串
34/// - `selector`: CSS 选择器，如 `"a[href]"` 或 `"div.news a"`
35///
36/// 返回去重后的链接列表
37pub fn extract_links(html_content: &str, selector: &str) -> Vec<String> {
38    try_extract_links(html_content, selector).unwrap_or_default()
39}
40
41/// 从 HTML 中提取所有匹配 CSS 选择器的链接（href 属性）
42///
43/// 和 `extract_links` 相比，此函数会返回选择器解析错误，适合需要严格错误处理的场景。
44pub fn try_extract_links(html_content: &str, selector: &str) -> Result<Vec<String>> {
45    let document = Html::parse_document(html_content);
46    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
47        selector: selector.to_string(),
48        message: e.to_string(),
49    })?;
50
51    let mut seen = std::collections::HashSet::new();
52    let mut links = Vec::new();
53
54    for element in document.select(&sel) {
55        if let Some(href) = element.value().attr("href")
56            && seen.insert(href.to_string())
57        {
58            links.push(href.to_string());
59        }
60    }
61
62    Ok(links)
63}
64
65/// 按选择器类型提取链接。
66///
67/// XPath 表达式既可以指向属性节点（如 `//a/@href`），也可以指向元素节点（如 `//a`）。
68pub fn extract_links_by_selector(
69    html_content: &str,
70    selector: &str,
71    selector_type: LinkSelectorType,
72) -> Result<Vec<String>> {
73    match selector_type {
74        LinkSelectorType::Css => try_extract_links(html_content, selector),
75        LinkSelectorType::Xpath => extract_links_by_xpath(html_content, selector),
76    }
77}
78
79/// 使用 XPath 表达式提取链接。
80pub fn extract_links_by_xpath(html_content: &str, selector: &str) -> Result<Vec<String>> {
81    let sanitized = sanitize_for_xpath(html_content);
82    let document = xpath_html::parse(&sanitized).map_err(|e| CrawlError::Html(e.to_string()))?;
83    let tree = XpathItemTree::from(&document);
84    let xpath_expr = xpath::parse(selector).map_err(|e| CrawlError::Selector {
85        selector: selector.to_string(),
86        message: e.to_string(),
87    })?;
88    let item_set = xpath_expr
89        .apply(&tree)
90        .map_err(|e| CrawlError::Html(e.to_string()))?;
91
92    let mut seen = std::collections::HashSet::new();
93    let mut links = Vec::new();
94
95    for item in &item_set {
96        let href = match item {
97            XpathItem::Node(Node::NonTreeNode(NonTreeXpathNode::AttributeNode(attribute))) => {
98                attribute.value.clone()
99            }
100            XpathItem::Node(Node::TreeNode(tree_node)) => match tree_node.data {
101                XpathItemTreeNodeData::ElementNode(element) => element
102                    .get_attribute("href")
103                    .map(str::to_string)
104                    .unwrap_or_default(),
105                _ => String::new(),
106            },
107            _ => String::new(),
108        };
109
110        let href = href.trim();
111        if !href.is_empty() && seen.insert(href.to_string()) {
112            links.push(href.to_string());
113        }
114    }
115
116    Ok(links)
117}
118
119/// 将相对 URL 转为绝对 URL
120///
121/// - `base_url`: 基准 URL（当前页面）
122/// - `relative`: 相对路径或完整 URL
123pub fn resolve_url(base_url: &str, relative: &str) -> Option<String> {
124    let relative = relative.trim();
125    if should_skip_link(relative) {
126        return None;
127    }
128    let base = Url::parse(base_url).ok()?;
129    let resolved = base.join(relative).ok()?;
130    match resolved.scheme() {
131        "http" | "https" => Some(resolved.to_string()),
132        _ => None,
133    }
134}
135
136/// 提取并解析为绝对 URL，自动去重和过滤非页面链接。
137pub fn extract_absolute_links(
138    html_content: &str,
139    selector: &str,
140    selector_type: LinkSelectorType,
141    base_url: &str,
142) -> Result<Vec<String>> {
143    let links = extract_links_by_selector(html_content, selector, selector_type)?;
144    let mut seen = std::collections::HashSet::new();
145    let mut absolute_links = Vec::new();
146
147    for link in links {
148        if let Some(url) = resolve_url(base_url, &link)
149            && seen.insert(url.clone())
150        {
151            absolute_links.push(url);
152        }
153    }
154
155    Ok(absolute_links)
156}
157
158fn should_skip_link(link: &str) -> bool {
159    link.is_empty()
160        || link.starts_with('#')
161        || link.starts_with("javascript:")
162        || link.starts_with("mailto:")
163        || link.starts_with("tel:")
164        || link.starts_with("data:")
165}
166
167/// 文章内容提取结果
168#[derive(Debug, Clone, Default)]
169pub struct Article {
170    /// 文章标题
171    pub title: String,
172    /// 文章正文（纯文本）
173    pub content: String,
174    /// 发布日期
175    pub date: Option<String>,
176    /// 作者
177    pub author: Option<String>,
178    /// 描述/摘要
179    pub description: Option<String>,
180}
181
182/// 从 HTML 中提取文章内容
183///
184/// 使用 dom_smoothie（Readability 模式）提取结构化文章内容，
185/// 支持标题、正文、作者、发布日期和摘要的自动提取。
186///
187/// `base_url` 参数传递给 Readability，用于解析相对 URL。
188pub fn extract_article(html_content: &str, base_url: &str) -> Article {
189    let cfg = Config {
190        text_mode: TextMode::Markdown,
191        ..Default::default()
192    };
193
194    let url = if base_url.is_empty() { None } else { Some(base_url) };
195
196    let mut readability = match Readability::new(html_content, url, Some(cfg)) {
197        Ok(r) => r,
198        Err(_) => return Article::default(),
199    };
200
201    let dom_article = match readability.parse() {
202        Ok(a) => a,
203        Err(_) => return Article::default(),
204    };
205
206    let content = dom_article.text_content.to_string();
207
208    Article {
209        title: dom_article.title,
210        content,
211        date: dom_article.published_time,
212        author: dom_article.byline,
213        description: dom_article.excerpt,
214    }
215}
216
217// ──────────────────────────────────────────────
218// 可读性提取（基于 dom_smoothie）
219// ──────────────────────────────────────────────
220
221/// 使用 dom_smoothie 提取文章正文（Readability 模式）
222///
223/// ```rust
224/// let html = r#"<html><body><article><p>正文内容</p></article></body></html>"#;
225/// let content = crawlkit_parser::html::extract_readable_content(html).unwrap();
226/// ```
227pub fn extract_readable_content(raw_html: &str) -> Result<String> {
228    let cfg = Config {
229        text_mode: TextMode::Markdown,
230        ..Default::default()
231    };
232
233    let mut readability = dom_smoothie::Readability::new(raw_html, None, Some(cfg))
234        .map_err(|e| CrawlError::Readability(e.to_string()))?;
235
236    let article = readability
237        .parse()
238        .map_err(|e| CrawlError::Readability(e.to_string()))?;
239
240    Ok(article.content.to_string())
241}
242
243/// 使用 CSS 选择器提取文章正文
244///
245/// ```rust
246/// let html = r#"<html><body><div class="content">正文</div></body></html>"#;
247/// let content = crawlkit_parser::html::extract_content_by_selector(html, "div.content").unwrap();
248/// ```
249pub fn extract_content_by_selector(raw_html: &str, content_selector: &str) -> Result<String> {
250    let document = Html::parse_document(raw_html);
251    let selector = Selector::parse(content_selector).map_err(|e| CrawlError::Selector {
252        selector: content_selector.to_string(),
253        message: e.to_string(),
254    })?;
255
256    let content = document
257        .select(&selector)
258        .next()
259        .map(|el| el.text().collect::<Vec<_>>().join("\n").trim().to_string())
260        .unwrap_or_default();
261
262    Ok(content)
263}
264
265/// 智能提取：优先 Readability，失败则回退到 CSS 选择器
266///
267/// ```rust
268/// let html = r#"<html><body><article><p>正文</p></article></body></html>"#;
269/// let content = crawlkit_parser::html::extract_content(html, "article").unwrap();
270/// ```
271pub fn extract_content(raw_html: &str, content_selector: &str) -> Result<String> {
272    match extract_readable_content(raw_html) {
273        Ok(content) if !content.is_empty() && content.len() > 100 => Ok(content),
274        _ => extract_content_by_selector(raw_html, content_selector),
275    }
276}
277
278/// 提取匹配指定选择器的所有文本内容
279///
280/// ```rust
281/// let html = r#"<html><body><p>段落1</p><p>段落2</p></body></html>"#;
282/// let texts = crawlkit_parser::html::extract_texts(html, "p").unwrap();
283/// ```
284pub fn extract_texts(raw_html: &str, selector: &str) -> Result<Vec<String>> {
285    let document = Html::parse_document(raw_html);
286    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
287        selector: selector.to_string(),
288        message: e.to_string(),
289    })?;
290
291    let texts: Vec<String> = document
292        .select(&sel)
293        .filter_map(|el| {
294            let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
295            if text.is_empty() { None } else { Some(text) }
296        })
297        .collect();
298
299    Ok(texts)
300}
301
302/// 提取匹配选择器的元素的指定属性值
303pub fn extract_attributes(raw_html: &str, selector: &str, attr: &str) -> Result<Vec<String>> {
304    let document = Html::parse_document(raw_html);
305    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
306        selector: selector.to_string(),
307        message: e.to_string(),
308    })?;
309
310    let values: Vec<String> = document
311        .select(&sel)
312        .filter_map(|el| el.value().attr(attr).map(ToString::to_string))
313        .filter(|v| !v.is_empty())
314        .collect();
315
316    Ok(values)
317}
318
319/// 将 HTML 中未自闭合的 void 元素转为自闭合格式
320///
321/// skyscraper 的 HTML 解析器是 XML 严格模式，遇到 `<meta>` 会报错。
322/// 此函数用 scraper（html5ever，宽容解析）先解析再序列化，
323/// 得到格式良好的 HTML，使 skyscraper 能正确解析。
324pub fn sanitize_for_xpath(html: &str) -> String {
325    let doc = Html::parse_document(html);
326    doc.html()
327}
328
329#[cfg(test)]
330mod tests {
331    use super::*;
332
333    fn sample_html() -> &'static str {
334        r##"<!DOCTYPE html>
335<html>
336<body>
337  <a class="link" href="https://example.com/article/1">文章1</a>
338  <a class="link" href="/article/2">文章2</a>
339  <a class="link" href="//cdn.example.com/article/3">文章3</a>
340  <a class="link" href="#top">锚点</a>
341  <a class="link" href="javascript:void(0)">脚本</a>
342  <a class="link" href="mailto:test@example.com">邮箱</a>
343</body>
344</html>"##
345    }
346
347    #[test]
348    fn try_extract_links_returns_selector_error() {
349        let result = try_extract_links("<html></html>", "!!!invalid");
350        assert!(result.is_err());
351    }
352
353    #[test]
354    fn extract_absolute_links_by_css_filters_and_resolves() {
355        let links = extract_absolute_links(
356            sample_html(),
357            "a.link",
358            LinkSelectorType::Css,
359            "https://example.com/base/",
360        )
361        .unwrap();
362
363        assert_eq!(links.len(), 3);
364        assert!(links.contains(&"https://example.com/article/1".to_string()));
365        assert!(links.contains(&"https://example.com/article/2".to_string()));
366        assert!(links.contains(&"https://cdn.example.com/article/3".to_string()));
367    }
368
369    #[test]
370    fn extract_absolute_links_by_xpath_reads_href_attributes() {
371        let links = extract_absolute_links(
372            sample_html(),
373            "//a[@class='link']/@href",
374            LinkSelectorType::Xpath,
375            "https://example.com",
376        )
377        .unwrap();
378
379        assert_eq!(links.len(), 3);
380        assert!(links.contains(&"https://example.com/article/1".to_string()));
381        assert!(links.contains(&"https://example.com/article/2".to_string()));
382        assert!(links.contains(&"https://cdn.example.com/article/3".to_string()));
383    }
384
385    #[test]
386    fn resolve_url_skips_non_page_links() {
387        assert!(resolve_url("https://example.com", "javascript:void(0)").is_none());
388        assert!(resolve_url("https://example.com", "mailto:test@example.com").is_none());
389        assert!(resolve_url("https://example.com", "#top").is_none());
390    }
391
392    #[test]
393    fn test_sanitize_for_xpath_meta() {
394        let html = r#"<html><head><meta charset="utf-8"></head><body></body></html>"#;
395        let sanitized = sanitize_for_xpath(html);
396        assert!(sanitized.contains("meta"));
397        assert!(sanitized.contains("charset"));
398    }
399
400    #[test]
401    fn test_sanitize_for_xpath_self_closing_unchanged() {
402        let html = r#"<html><head><meta charset="utf-8" /></head><body></body></html>"#;
403        let sanitized = sanitize_for_xpath(html);
404        assert!(sanitized.contains("meta"));
405        assert!(sanitized.contains("charset"));
406    }
407
408    #[test]
409    fn test_sanitize_for_xpath_multiple_void_elements() {
410        let html = r#"<br><img src="x.png"><input type="text"><link rel="stylesheet">"#;
411        let sanitized = sanitize_for_xpath(html);
412        assert!(sanitized.contains("br"));
413        assert!(sanitized.contains("img"));
414        assert!(sanitized.contains("input"));
415        assert!(sanitized.contains("link"));
416    }
417
418    #[test]
419    fn test_sanitize_for_xpath_non_void_unchanged() {
420        let html = r"<div><p>Hello</p></div>";
421        let sanitized = sanitize_for_xpath(html);
422        assert!(sanitized.contains("<div>"));
423        assert!(sanitized.contains("<p>"));
424    }
425}
crawlkit_parser/html.rs

crawlkit_parser/
html.rs
