millipede_html/
extract.rs

1use std::sync::Arc;
2
3use millipede_core::{
4    errors::CrawlError,
5    link_extraction::{ExtractedLink, LinkExtractor},
6};
7use scraper::Selector;
8use url::Url;
9
10use crate::SynchronizedHtml;
11
12/// Extracts raw link targets from an already-parsed HTML document.
13pub struct HtmlLinkExtractor {
14    html: Arc<SynchronizedHtml>,
15    document_url: Url,
16}
17
18impl HtmlLinkExtractor {
19    /// Creates an extractor for `html`, resolving document bases against `document_url`.
20    pub fn new(html: Arc<scraper::Html>, document_url: Url) -> Self {
21        Self::from_synchronized(
22            Arc::new(SynchronizedHtml::from_html(Arc::unwrap_or_clone(html))),
23            document_url,
24        )
25    }
26
27    pub(crate) fn from_synchronized(html: Arc<SynchronizedHtml>, document_url: Url) -> Self {
28        Self { html, document_url }
29    }
30}
31
32#[async_trait::async_trait]
33impl LinkExtractor for HtmlLinkExtractor {
34    async fn extract(&self, selector: Option<&str>) -> Result<Vec<ExtractedLink>, CrawlError> {
35        let selector_text = selector.unwrap_or("a[href]");
36        let selector = Selector::parse(selector_text).map_err(|error| {
37            CrawlError::non_retryable(anyhow::anyhow!(
38                "invalid link selector {selector_text:?}: {error}"
39            ))
40        })?;
41        let base_selector = Selector::parse("base[href]")
42            .expect("the built-in base[href] selector must always parse");
43
44        Ok(self.html.with_html(|html| {
45            let effective_base = html
46                .select(&base_selector)
47                .next()
48                .and_then(|element| element.value().attr("href"))
49                .and_then(|href| self.document_url.join(href).ok())
50                .unwrap_or_else(|| self.document_url.clone());
51
52            html.select(&selector)
53                .filter_map(|element| element.value().attr("href"))
54                .map(|href| ExtractedLink {
55                    url: href.to_owned(),
56                    base: Some(effective_base.clone()),
57                })
58                .collect()
59        }))
60    }
61}
62
63#[cfg(test)]
64mod tests {
65    use super::*;
66
67    fn extractor(document: &str) -> HtmlLinkExtractor {
68        HtmlLinkExtractor::new(
69            #[allow(clippy::arc_with_non_send_sync)]
70            Arc::new(scraper::Html::parse_document(document)),
71            Url::parse("https://example.test/landing/page").expect("document URL must parse"),
72        )
73    }
74
75    #[tokio::test]
76    async fn extracts_raw_hrefs_against_the_first_valid_base() {
77        let links = extractor(
78            r#"<base href="/catalog/"><base href="/ignored/">
79               <a href="item">default</a><a class="product" href="../other">other</a>"#,
80        )
81        .extract(None)
82        .await
83        .expect("default selector must extract");
84
85        assert_eq!(links.len(), 2);
86        assert_eq!(links[0].url, "item");
87        assert_eq!(
88            links[0].base.as_ref().map(Url::as_str),
89            Some("https://example.test/catalog/")
90        );
91    }
92
93    #[tokio::test]
94    async fn missing_base_uses_document_url_for_every_link() {
95        let links = extractor(r#"<a href="child">child</a>"#)
96            .extract(None)
97            .await
98            .expect("default selector must extract");
99
100        assert_eq!(links.len(), 1);
101        assert_eq!(
102            links[0].base.as_ref().map(Url::as_str),
103            Some("https://example.test/landing/page")
104        );
105    }
106
107    #[tokio::test]
108    async fn invalid_first_base_falls_back_to_document_url_and_ignores_later_bases() {
109        let links = extractor(
110            r#"<base href="http://["><base href="/later-valid/">
111               <a href="child">child</a>"#,
112        )
113        .extract(None)
114        .await
115        .expect("default selector must extract");
116
117        assert_eq!(links.len(), 1);
118        assert_eq!(
119            links[0].base.as_ref().map(Url::as_str),
120            Some("https://example.test/landing/page")
121        );
122    }
123
124    #[tokio::test]
125    async fn custom_selector_still_skips_elements_without_href() {
126        let links =
127            extractor(r#"<a class="product">missing</a><a class="product" href="kept">kept</a>"#)
128                .extract(Some("a.product"))
129                .await
130                .expect("custom selector must extract");
131        assert_eq!(links.len(), 1);
132        assert_eq!(links[0].url, "kept");
133    }
134
135    #[tokio::test]
136    async fn invalid_selector_reports_source_text() {
137        let error = extractor("")
138            .extract(Some("a["))
139            .await
140            .expect_err("invalid selector must fail");
141        assert!(error.to_string().contains("a["));
142    }
143}
millipede_html/extract.rs

millipede_html/
extract.rs
