crawlkit_parser/
html.rs

1//! HTML 解析工具集
2//!
3//! 提供链接提取、文章内容提取、可读性提取等实用功能。
4//! 底层使用 `scraper`（基于 html5ever）和 `dom_smoothie`（Readability 模式）。
5
6use dom_smoothie::{Config, Readability, TextMode};
7use regex::Regex;
8use scraper::{Html, Selector};
9use skyscraper::{
10    html as xpath_html,
11    xpath::{
12        self, XpathItemTree,
13        grammar::{
14            NonTreeXpathNode, XpathItemTreeNodeData,
15            data_model::{Node, XpathItem},
16        },
17    },
18};
19use url::Url;
20
21use crawlkit_core::error::{CrawlError, Result};
22
23/// 链接选择器类型
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub enum LinkSelectorType {
26    /// CSS 选择器，例如 `a[href]`
27    Css,
28    /// XPath 表达式，例如 `//a/@href`
29    Xpath,
30}
31
32/// 从 HTML 中提取所有匹配 CSS 选择器的链接（href 属性）
33///
34/// - `html_content`: 完整 HTML 字符串
35/// - `selector`: CSS 选择器，如 `"a[href]"` 或 `"div.news a"`
36///
37/// 返回去重后的链接列表
38pub fn extract_links(html_content: &str, selector: &str) -> Vec<String> {
39    try_extract_links(html_content, selector).unwrap_or_default()
40}
41
42/// 从 HTML 中提取所有匹配 CSS 选择器的链接（href 属性）
43///
44/// 和 `extract_links` 相比，此函数会返回选择器解析错误，适合需要严格错误处理的场景。
45pub fn try_extract_links(html_content: &str, selector: &str) -> Result<Vec<String>> {
46    let document = Html::parse_document(html_content);
47    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
48        selector: selector.to_string(),
49        message: e.to_string(),
50    })?;
51
52    let mut seen = std::collections::HashSet::new();
53    let mut links = Vec::new();
54
55    for element in document.select(&sel) {
56        if let Some(href) = element.value().attr("href")
57            && seen.insert(href.to_string())
58        {
59            links.push(href.to_string());
60        }
61    }
62
63    Ok(links)
64}
65
66/// 按选择器类型提取链接。
67///
68/// XPath 表达式既可以指向属性节点（如 `//a/@href`），也可以指向元素节点（如 `//a`）。
69pub fn extract_links_by_selector(
70    html_content: &str,
71    selector: &str,
72    selector_type: LinkSelectorType,
73) -> Result<Vec<String>> {
74    match selector_type {
75        LinkSelectorType::Css => try_extract_links(html_content, selector),
76        LinkSelectorType::Xpath => extract_links_by_xpath(html_content, selector),
77    }
78}
79
80/// 检测内容是否为 XML（RSS/Atom/Sitemap 等）
81fn is_xml_content(content: &str) -> bool {
82    let trimmed = content.trim_start();
83    trimmed.starts_with("<?xml")
84        || trimmed.starts_with("<rss")
85        || trimmed.starts_with("<feed")
86        || trimmed.starts_with("<sitemap")
87        || trimmed.starts_with("<urlset")
88}
89
90/// 从 RSS/Atom XML 内容中提取 `<link>` 文本链接
91///
92/// 使用正则匹配 `<link>url</link>` 模式，适用于 RSS 2.0 和 Atom。
93fn extract_links_from_xml(xml_content: &str) -> Vec<String> {
94    let mut links = Vec::new();
95    let mut seen = std::collections::HashSet::new();
96
97    // RSS 2.0: <link>https://example.com</link>
98    // Atom: <link href="https://example.com"/> 或 <link>https://example.com</link>
99    let re_text_link = Regex::new(r"<link>\s*(https?://[^<]+?)\s*</link>").unwrap();
100    let re_attr_link = Regex::new(r#"<link[^>]+href="(https?://[^"]+)""#).unwrap();
101
102    for cap in re_text_link.captures_iter(xml_content) {
103        if let Some(url) = cap.get(1) {
104            let url = url.as_str().trim().to_string();
105            if !url.is_empty() && seen.insert(url.clone()) {
106                links.push(url);
107            }
108        }
109    }
110
111    for cap in re_attr_link.captures_iter(xml_content) {
112        if let Some(url) = cap.get(1) {
113            let url = url.as_str().trim().to_string();
114            if !url.is_empty() && seen.insert(url.clone()) {
115                links.push(url);
116            }
117        }
118    }
119
120    links
121}
122
123/// 使用 XPath 表达式提取链接。
124///
125/// 对 HTML 和 XML（RSS/Atom）内容均有效：
126/// - HTML：自动清理为规范格式，支持 `href` 属性和文本节点
127/// - XML（RSS/Atom）：使用正则提取 `<link>` 文本，忽略 XPath 选择器
128pub fn extract_links_by_xpath(html_content: &str, selector: &str) -> Result<Vec<String>> {
129    // XML 内容（RSS/Atom）使用正则提取，跳过 XPath 解析
130    if is_xml_content(html_content) {
131        return Ok(extract_links_from_xml(html_content));
132    }
133
134    let sanitized = sanitize_for_xpath(html_content);
135    let document = xpath_html::parse(&sanitized).map_err(|e| CrawlError::Html(e.to_string()))?;
136    let tree = XpathItemTree::from(&document);
137    let xpath_expr = xpath::parse(selector).map_err(|e| CrawlError::Selector {
138        selector: selector.to_string(),
139        message: e.to_string(),
140    })?;
141    let item_set = xpath_expr
142        .apply(&tree)
143        .map_err(|e| CrawlError::Html(e.to_string()))?;
144
145    let mut seen = std::collections::HashSet::new();
146    let mut links = Vec::new();
147
148    for item in &item_set {
149        let href = match item {
150            XpathItem::Node(Node::NonTreeNode(NonTreeXpathNode::AttributeNode(attribute))) => {
151                attribute.value.clone()
152            }
153            XpathItem::Node(Node::TreeNode(tree_node)) => match &tree_node.data {
154                XpathItemTreeNodeData::ElementNode(element) => element
155                    .get_attribute("href")
156                    .map(str::to_string)
157                    .unwrap_or_default(),
158                XpathItemTreeNodeData::TextNode(text) => text.to_string(),
159                _ => String::new(),
160            },
161            _ => String::new(),
162        };
163
164        let href = href.trim();
165        if !href.is_empty() && seen.insert(href.to_string()) {
166            links.push(href.to_string());
167        }
168    }
169
170    Ok(links)
171}
172
173/// 将相对 URL 转为绝对 URL
174///
175/// - `base_url`: 基准 URL（当前页面）
176/// - `relative`: 相对路径或完整 URL
177pub fn resolve_url(base_url: &str, relative: &str) -> Option<String> {
178    let relative = relative.trim();
179    if should_skip_link(relative) {
180        return None;
181    }
182    let base = Url::parse(base_url).ok()?;
183    let resolved = base.join(relative).ok()?;
184    match resolved.scheme() {
185        "http" | "https" => Some(resolved.to_string()),
186        _ => None,
187    }
188}
189
190/// 提取并解析为绝对 URL，自动去重和过滤非页面链接。
191pub fn extract_absolute_links(
192    html_content: &str,
193    selector: &str,
194    selector_type: LinkSelectorType,
195    base_url: &str,
196) -> Result<Vec<String>> {
197    let links = extract_links_by_selector(html_content, selector, selector_type)?;
198    let mut seen = std::collections::HashSet::new();
199    let mut absolute_links = Vec::new();
200
201    for link in links {
202        if let Some(url) = resolve_url(base_url, &link)
203            && seen.insert(url.clone())
204        {
205            absolute_links.push(url);
206        }
207    }
208
209    Ok(absolute_links)
210}
211
212fn should_skip_link(link: &str) -> bool {
213    link.is_empty()
214        || link.starts_with('#')
215        || link.starts_with("javascript:")
216        || link.starts_with("mailto:")
217        || link.starts_with("tel:")
218        || link.starts_with("data:")
219}
220
221/// 文章内容提取结果
222#[derive(Debug, Clone, Default)]
223pub struct Article {
224    /// 文章标题
225    pub title: String,
226    /// 文章正文（纯文本）
227    pub content: String,
228    /// 发布日期
229    pub date: Option<String>,
230    /// 作者
231    pub author: Option<String>,
232    /// 描述/摘要
233    pub description: Option<String>,
234}
235
236/// 从 HTML 中提取文章内容
237///
238/// 使用 dom_smoothie（Readability 模式）提取结构化文章内容，
239/// 支持标题、正文、作者、发布日期和摘要的自动提取。
240///
241/// `base_url` 参数传递给 Readability，用于解析相对 URL。
242pub fn extract_article(html_content: &str, base_url: &str) -> Article {
243    let cfg = Config {
244        text_mode: TextMode::Markdown,
245        ..Default::default()
246    };
247
248    let url = if base_url.is_empty() { None } else { Some(base_url) };
249
250    let mut readability = match Readability::new(html_content, url, Some(cfg)) {
251        Ok(r) => r,
252        Err(_) => return Article::default(),
253    };
254
255    let dom_article = match readability.parse() {
256        Ok(a) => a,
257        Err(_) => return Article::default(),
258    };
259
260    let content = dom_article.text_content.to_string();
261
262    Article {
263        title: dom_article.title,
264        content,
265        date: dom_article.published_time,
266        author: dom_article.byline,
267        description: dom_article.excerpt,
268    }
269}
270
271// ──────────────────────────────────────────────
272// 可读性提取（基于 dom_smoothie）
273// ──────────────────────────────────────────────
274
275/// 使用 dom_smoothie 提取文章正文（Readability 模式）
276///
277/// ```rust
278/// let html = r#"<html><body><article><p>正文内容</p></article></body></html>"#;
279/// let content = crawlkit_parser::html::extract_readable_content(html).unwrap();
280/// ```
281pub fn extract_readable_content(raw_html: &str) -> Result<String> {
282    let cfg = Config {
283        text_mode: TextMode::Markdown,
284        ..Default::default()
285    };
286
287    let mut readability = dom_smoothie::Readability::new(raw_html, None, Some(cfg))
288        .map_err(|e| CrawlError::Readability(e.to_string()))?;
289
290    let article = readability
291        .parse()
292        .map_err(|e| CrawlError::Readability(e.to_string()))?;
293
294    Ok(article.content.to_string())
295}
296
297/// 使用 CSS 选择器提取文章正文
298///
299/// ```rust
300/// let html = r#"<html><body><div class="content">正文</div></body></html>"#;
301/// let content = crawlkit_parser::html::extract_content_by_selector(html, "div.content").unwrap();
302/// ```
303pub fn extract_content_by_selector(raw_html: &str, content_selector: &str) -> Result<String> {
304    let document = Html::parse_document(raw_html);
305    let selector = Selector::parse(content_selector).map_err(|e| CrawlError::Selector {
306        selector: content_selector.to_string(),
307        message: e.to_string(),
308    })?;
309
310    let content = document
311        .select(&selector)
312        .next()
313        .map(|el| el.text().collect::<Vec<_>>().join("\n").trim().to_string())
314        .unwrap_or_default();
315
316    Ok(content)
317}
318
319/// 智能提取：优先 Readability，失败则回退到 CSS 选择器
320///
321/// ```rust
322/// let html = r#"<html><body><article><p>正文</p></article></body></html>"#;
323/// let content = crawlkit_parser::html::extract_content(html, "article").unwrap();
324/// ```
325pub fn extract_content(raw_html: &str, content_selector: &str) -> Result<String> {
326    match extract_readable_content(raw_html) {
327        Ok(content) if !content.is_empty() && content.len() > 100 => Ok(content),
328        _ => extract_content_by_selector(raw_html, content_selector),
329    }
330}
331
332/// 提取匹配指定选择器的所有文本内容
333///
334/// ```rust
335/// let html = r#"<html><body><p>段落1</p><p>段落2</p></body></html>"#;
336/// let texts = crawlkit_parser::html::extract_texts(html, "p").unwrap();
337/// ```
338pub fn extract_texts(raw_html: &str, selector: &str) -> Result<Vec<String>> {
339    let document = Html::parse_document(raw_html);
340    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
341        selector: selector.to_string(),
342        message: e.to_string(),
343    })?;
344
345    let texts: Vec<String> = document
346        .select(&sel)
347        .filter_map(|el| {
348            let text: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
349            if text.is_empty() { None } else { Some(text) }
350        })
351        .collect();
352
353    Ok(texts)
354}
355
356/// 提取匹配选择器的元素的指定属性值
357pub fn extract_attributes(raw_html: &str, selector: &str, attr: &str) -> Result<Vec<String>> {
358    let document = Html::parse_document(raw_html);
359    let sel = Selector::parse(selector).map_err(|e| CrawlError::Selector {
360        selector: selector.to_string(),
361        message: e.to_string(),
362    })?;
363
364    let values: Vec<String> = document
365        .select(&sel)
366        .filter_map(|el| el.value().attr(attr).map(ToString::to_string))
367        .filter(|v| !v.is_empty())
368        .collect();
369
370    Ok(values)
371}
372
373/// 将 HTML 中未自闭合的 void 元素转为自闭合格式
374///
375/// skyscraper 的 HTML 解析器是 XML 严格模式，遇到 `<meta>` 会报错。
376/// 此函数用 scraper（html5ever，宽容解析）先解析再序列化，
377/// 得到格式良好的 HTML，使 skyscraper 能正确解析。
378pub fn sanitize_for_xpath(html: &str) -> String {
379    let doc = Html::parse_document(html);
380    doc.html()
381}
382
383#[cfg(test)]
384mod tests {
385    use super::*;
386
387    fn sample_html() -> &'static str {
388        r##"<!DOCTYPE html>
389<html>
390<body>
391  <a class="link" href="https://example.com/article/1">文章1</a>
392  <a class="link" href="/article/2">文章2</a>
393  <a class="link" href="//cdn.example.com/article/3">文章3</a>
394  <a class="link" href="#top">锚点</a>
395  <a class="link" href="javascript:void(0)">脚本</a>
396  <a class="link" href="mailto:test@example.com">邮箱</a>
397</body>
398</html>"##
399    }
400
401    #[test]
402    fn try_extract_links_returns_selector_error() {
403        let result = try_extract_links("<html></html>", "!!!invalid");
404        assert!(result.is_err());
405    }
406
407    #[test]
408    fn extract_absolute_links_by_css_filters_and_resolves() {
409        let links = extract_absolute_links(
410            sample_html(),
411            "a.link",
412            LinkSelectorType::Css,
413            "https://example.com/base/",
414        )
415        .unwrap();
416
417        assert_eq!(links.len(), 3);
418        assert!(links.contains(&"https://example.com/article/1".to_string()));
419        assert!(links.contains(&"https://example.com/article/2".to_string()));
420        assert!(links.contains(&"https://cdn.example.com/article/3".to_string()));
421    }
422
423    #[test]
424    fn extract_absolute_links_by_xpath_reads_href_attributes() {
425        let links = extract_absolute_links(
426            sample_html(),
427            "//a[@class='link']/@href",
428            LinkSelectorType::Xpath,
429            "https://example.com",
430        )
431        .unwrap();
432
433        assert_eq!(links.len(), 3);
434        assert!(links.contains(&"https://example.com/article/1".to_string()));
435        assert!(links.contains(&"https://example.com/article/2".to_string()));
436        assert!(links.contains(&"https://cdn.example.com/article/3".to_string()));
437    }
438
439    #[test]
440    fn resolve_url_skips_non_page_links() {
441        assert!(resolve_url("https://example.com", "javascript:void(0)").is_none());
442        assert!(resolve_url("https://example.com", "mailto:test@example.com").is_none());
443        assert!(resolve_url("https://example.com", "#top").is_none());
444    }
445
446    #[test]
447    fn test_sanitize_for_xpath_meta() {
448        let html = r#"<html><head><meta charset="utf-8"></head><body></body></html>"#;
449        let sanitized = sanitize_for_xpath(html);
450        assert!(sanitized.contains("meta"));
451        assert!(sanitized.contains("charset"));
452    }
453
454    #[test]
455    fn test_sanitize_for_xpath_self_closing_unchanged() {
456        let html = r#"<html><head><meta charset="utf-8" /></head><body></body></html>"#;
457        let sanitized = sanitize_for_xpath(html);
458        assert!(sanitized.contains("meta"));
459        assert!(sanitized.contains("charset"));
460    }
461
462    #[test]
463    fn test_sanitize_for_xpath_multiple_void_elements() {
464        let html = r#"<br><img src="x.png"><input type="text"><link rel="stylesheet">"#;
465        let sanitized = sanitize_for_xpath(html);
466        assert!(sanitized.contains("br"));
467        assert!(sanitized.contains("img"));
468        assert!(sanitized.contains("input"));
469        assert!(sanitized.contains("link"));
470    }
471
472    #[test]
473    fn test_sanitize_for_xpath_non_void_unchanged() {
474        let html = r"<div><p>Hello</p></div>";
475        let sanitized = sanitize_for_xpath(html);
476        assert!(sanitized.contains("<div>"));
477        assert!(sanitized.contains("<p>"));
478    }
479
480    #[test]
481    fn test_rss_xml_xpath_text_node() {
482        let rss = r#"<?xml version="1.0" encoding="utf-8"?>
483<rss version="2.0">
484  <channel>
485    <item>
486      <link>https://example.com/article1</link>
487    </item>
488    <item>
489      <link>https://example.com/article2</link>
490    </item>
491  </channel>
492</rss>"#;
493        let links = extract_links_by_xpath(rss, "//item/link").unwrap();
494        assert_eq!(links.len(), 2);
495        assert!(links.contains(&"https://example.com/article1".to_string()));
496        assert!(links.contains(&"https://example.com/article2".to_string()));
497    }
498
499    #[test]
500    fn test_atom_xml_xpath_link_attr() {
501        let atom = r#"<?xml version="1.0" encoding="utf-8"?>
502<feed xmlns="http://www.w3.org/2005/Atom">
503  <entry>
504    <link href="https://example.com/atom1"/>
505  </entry>
506  <entry>
507    <link href="https://example.com/atom2"/>
508  </entry>
509</feed>"#;
510        let links = extract_links_by_xpath(atom, "//entry/link/@href").unwrap();
511        assert_eq!(links.len(), 2);
512        assert!(links.contains(&"https://example.com/atom1".to_string()));
513        assert!(links.contains(&"https://example.com/atom2".to_string()));
514    }
515}
crawlkit_parser/html.rs

crawlkit_parser/
html.rs
