use scraper::{ElementRef, Html, Selector};
use serde_json::Value;
use thiserror::Error;
use url::Url;
#[derive(Debug, Error)]
pub enum ExtractionUtilsError {
#[error("Invalid CSS selector: {0}")]
InvalidSelector(String),
#[error("Element not found for selector: {0}")]
ElementNotFound(String),
#[error("URL parsing failed: {0}")]
UrlParseError(#[from] url::ParseError),
}
pub struct ExtractionUtils;
impl ExtractionUtils {
pub fn extract_element_value<F>(
element: ElementRef,
attr: &str,
base: Option<&Url>,
text_fallback: F,
) -> Option<String>
where
F: FnOnce() -> Option<String>,
{
let val = element.value().attr(attr).map(|s| s.to_string());
match (val, base) {
(Some(v), Some(b)) if attr == "href" || attr == "src" => {
let v = v.strip_prefix('/').unwrap_or(&v);
let mut new_url = b.clone();
new_url.set_path(&format!("{}/{}", b.path().trim_end_matches('/'), v));
Some(new_url.to_string())
}
(Some(v), _) => Some(v),
(None, _) => text_fallback(),
}
}
pub fn extract_array_values<'a, 'b, F>(
elements: impl Iterator<Item = ElementRef<'a>>,
attr: &'b str,
base: Option<&'b Url>,
mut text_fallback: F,
) -> Vec<Value>
where
F: FnMut(ElementRef<'a>) -> Option<String>,
{
elements
.filter_map(|element| {
let value =
Self::extract_element_value(element, attr, base, || text_fallback(element));
value.filter(|v| !v.is_empty()).map(Value::String)
})
.collect()
}
pub fn extract_single_value(
html: &str,
selector_str: &str,
attr: Option<&str>,
base: Option<&Url>,
) -> Result<Value, ExtractionUtilsError> {
let document = Html::parse_document(html);
let selector = Selector::parse(selector_str)
.map_err(|e| ExtractionUtilsError::InvalidSelector(e.to_string()))?;
if let Some(element) = document.select(&selector).next() {
let value = attr.map_or_else(
|| {
Some(
element
.text()
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string(),
)
},
|attr_name| {
Self::extract_element_value(element, attr_name, base, || {
Some(
element
.text()
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string(),
)
})
},
);
Ok(value.map(Value::String).unwrap_or(Value::Null))
} else {
Err(ExtractionUtilsError::ElementNotFound(
selector_str.to_string(),
))
}
}
}
pub trait ExtractableRule {
fn selector(&self) -> &str;
fn is_array(&self) -> bool;
fn attr(&self) -> Option<&str>;
fn description(&self) -> &str;
}
#[cfg(test)]
mod tests {
use super::*;
use scraper::Html;
#[test]
fn test_extract_element_value_with_href() {
let html = r#"<a href="relative/path">Link</a>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("a").unwrap())
.next()
.unwrap();
let base_url = Url::parse("https://example.com/base").unwrap();
let result =
ExtractionUtils::extract_element_value(element, "href", Some(&base_url), || None);
assert_eq!(
result,
Some("https://example.com/base/relative/path".to_string())
);
}
#[test]
fn test_extract_element_value_with_text() {
let html = r#"<p>Hello World</p>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("p").unwrap())
.next()
.unwrap();
let result = ExtractionUtils::extract_element_value(
element,
"href", None,
|| {
Some(
element
.text()
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string(),
)
},
);
assert_eq!(result, Some("Hello World".to_string()));
}
#[test]
fn test_extract_array_values() {
let html = r#"
<div>
<a href="/link1">Link 1</a>
<a href="/link2">Link 2</a>
<a href="/link3">Link 3</a>
</div>
"#;
let document = Html::parse_document(html);
let selector = Selector::parse("a").unwrap();
let elements = document.select(&selector);
let base_url = Url::parse("https://example.com").unwrap();
let results =
ExtractionUtils::extract_array_values(elements, "href", Some(&base_url), |_| None);
assert_eq!(results.len(), 3);
assert_eq!(
results[0],
Value::String("https://example.com/link1".to_string())
);
assert_eq!(
results[1],
Value::String("https://example.com/link2".to_string())
);
assert_eq!(
results[2],
Value::String("https://example.com/link3".to_string())
);
}
#[test]
fn test_extract_single_value() {
let html = r#"<meta name="description" content="Test Description">"#;
let result = ExtractionUtils::extract_single_value(
html,
"meta[name='description']",
Some("content"),
None,
);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
Value::String("Test Description".to_string())
);
}
#[test]
fn test_extract_single_value_not_found() {
let html = r#"<div>No meta tag</div>"#;
let result = ExtractionUtils::extract_single_value(
html,
"meta[name='description']",
Some("content"),
None,
);
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
ExtractionUtilsError::ElementNotFound(_)
));
}
#[test]
fn test_extract_element_value_with_src_attribute() {
let html = r#"<img src="image.png">"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("img").expect("valid selector"))
.next()
.expect("img element exists");
let base_url = Url::parse("https://example.com/page").expect("valid url");
let result =
ExtractionUtils::extract_element_value(element, "src", Some(&base_url), || None);
assert_eq!(
result,
Some("https://example.com/page/image.png".to_string())
);
}
#[test]
fn test_extract_element_value_non_href_src_attr_returned_as_is() {
let html = r#"<div class="container main">Content</div>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("div").expect("valid selector"))
.next()
.expect("div element exists");
let base_url = Url::parse("https://example.com/base").expect("valid url");
let result =
ExtractionUtils::extract_element_value(element, "class", Some(&base_url), || None);
assert_eq!(result, Some("container main".to_string()));
}
#[test]
fn test_extract_element_value_non_href_src_attr_without_base() {
let html = r#"<span data-id="12345">Text</span>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("span").expect("valid selector"))
.next()
.expect("span element exists");
let result = ExtractionUtils::extract_element_value(element, "data-id", None, || None);
assert_eq!(result, Some("12345".to_string()));
}
#[test]
fn test_extract_element_value_no_attr_fallback_returns_none() {
let html = r#"<a>Link without href</a>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("a").expect("valid selector"))
.next()
.expect("a element exists");
let result = ExtractionUtils::extract_element_value(element, "href", None, || None);
assert!(
result.is_none(),
"should return None when attr is absent and fallback returns None"
);
}
#[test]
fn test_extract_element_value_href_without_base_returns_raw() {
let html = r#"<a href="/path/page">Link</a>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("a").expect("valid selector"))
.next()
.expect("a element exists");
let result = ExtractionUtils::extract_element_value(element, "href", None, || None);
assert_eq!(result, Some("/path/page".to_string()));
}
#[test]
fn test_extract_element_value_href_strips_leading_slash_when_joining() {
let html = r#"<a href="/absolute/path">Link</a>"#;
let document = Html::parse_document(html);
let element = document
.select(&Selector::parse("a").expect("valid selector"))
.next()
.expect("a element exists");
let base_url = Url::parse("https://example.com").expect("valid url");
let result =
ExtractionUtils::extract_element_value(element, "href", Some(&base_url), || None);
assert_eq!(
result,
Some("https://example.com/absolute/path".to_string())
);
}
#[test]
fn test_extract_array_values_uses_text_fallback_when_attr_missing() {
let html = r#"
<ul>
<li>Item 1</li>
<li>Item 2</li>
</ul>
"#;
let document = Html::parse_document(html);
let selector = Selector::parse("li").expect("valid selector");
let elements = document.select(&selector);
let results = ExtractionUtils::extract_array_values(elements, "href", None, |el| {
Some(el.text().collect::<Vec<_>>().join(" ").trim().to_string())
});
assert_eq!(results.len(), 2);
assert_eq!(results[0], Value::String("Item 1".to_string()));
assert_eq!(results[1], Value::String("Item 2".to_string()));
}
#[test]
fn test_extract_array_values_filters_empty_strings() {
let html = r#"
<div>
<a href="">Empty</a>
<a href="/link1">Link 1</a>
<a href="">Also Empty</a>
<a href="/link2">Link 2</a>
</div>
"#;
let document = Html::parse_document(html);
let selector = Selector::parse("a").expect("valid selector");
let elements = document.select(&selector);
let results = ExtractionUtils::extract_array_values(elements, "href", None, |_| None);
assert_eq!(results.len(), 2, "empty values should be filtered out");
}
#[test]
fn test_extract_array_values_returns_empty_when_no_elements() {
let html = r#"<div>No links here</div>"#;
let document = Html::parse_document(html);
let selector = Selector::parse("a").expect("valid selector");
let elements = document.select(&selector);
let results = ExtractionUtils::extract_array_values(elements, "href", None, |_| None);
assert!(
results.is_empty(),
"should return empty vec when no elements match"
);
}
#[test]
fn test_extract_array_values_mixed_attr_and_fallback() {
let html = r#"
<div>
<a href="/link1">Link 1</a>
<a>Link 2 without href</a>
<a href="/link3">Link 3</a>
</div>
"#;
let document = Html::parse_document(html);
let selector = Selector::parse("a").expect("valid selector");
let elements = document.select(&selector);
let base_url = Url::parse("https://example.com").expect("valid url");
let results =
ExtractionUtils::extract_array_values(elements, "href", Some(&base_url), |el| {
Some(el.text().collect::<Vec<_>>().join(" ").trim().to_string())
});
assert_eq!(results.len(), 3);
assert_eq!(
results[0],
Value::String("https://example.com/link1".to_string())
);
assert_eq!(results[1], Value::String("Link 2 without href".to_string()));
assert_eq!(
results[2],
Value::String("https://example.com/link3".to_string())
);
}
#[test]
fn test_extract_single_value_without_attr_extracts_text() {
let html = r#"<h1>Title Text</h1>"#;
let result = ExtractionUtils::extract_single_value(html, "h1", None, None);
assert!(result.is_ok());
assert_eq!(result.expect("ok"), Value::String("Title Text".to_string()));
}
#[test]
fn test_extract_single_value_without_attr_empty_text_returns_empty_string() {
let html = r#"<h1></h1>"#;
let result = ExtractionUtils::extract_single_value(html, "h1", None, None);
assert!(result.is_ok());
assert_eq!(result.expect("ok"), Value::String("".to_string()));
}
#[test]
fn test_extract_single_value_with_base_url_joins_href() {
let html = r#"<a href="page.html">Link</a>"#;
let base_url = Url::parse("https://example.com/base").expect("valid url");
let result =
ExtractionUtils::extract_single_value(html, "a", Some("href"), Some(&base_url));
assert!(result.is_ok());
assert_eq!(
result.expect("ok"),
Value::String("https://example.com/base/page.html".to_string())
);
}
#[test]
fn test_extract_single_value_invalid_selector_returns_error() {
let html = r#"<div>content</div>"#;
let result = ExtractionUtils::extract_single_value(html, "!!!invalid", None, None);
assert!(result.is_err());
match result.expect_err("should be error") {
ExtractionUtilsError::InvalidSelector(msg) => {
assert!(!msg.is_empty(), "error message should not be empty");
}
other => panic!("expected InvalidSelector, got {:?}", other),
}
}
#[test]
fn test_extract_single_value_attr_present_but_empty_returns_null() {
let html = r#"<meta name="description" content="">"#;
let result = ExtractionUtils::extract_single_value(
html,
"meta[name='description']",
Some("content"),
None,
);
assert!(result.is_ok());
assert_eq!(result.expect("ok"), Value::String("".to_string()));
}
#[test]
fn test_extract_single_value_attr_missing_falls_back_to_text() {
let html = r#"<a>Click here</a>"#;
let result = ExtractionUtils::extract_single_value(html, "a", Some("href"), None);
assert!(result.is_ok());
assert_eq!(result.expect("ok"), Value::String("Click here".to_string()));
}
#[test]
fn test_extraction_utils_error_invalid_selector_display() {
let err = ExtractionUtilsError::InvalidSelector("bad syntax".to_string());
assert!(err.to_string().contains("Invalid CSS selector"));
assert!(err.to_string().contains("bad syntax"));
}
#[test]
fn test_extraction_utils_error_element_not_found_display() {
let err = ExtractionUtilsError::ElementNotFound("div.missing".to_string());
assert!(err.to_string().contains("Element not found for selector"));
assert!(err.to_string().contains("div.missing"));
}
#[test]
fn test_extraction_utils_error_url_parse_error_display() {
let parse_err = "http://[invalid"
.parse::<url::Url>()
.expect_err("should fail");
let err = ExtractionUtilsError::UrlParseError(parse_err);
assert!(err.to_string().contains("URL parsing failed"));
}
#[test]
fn test_extraction_utils_error_url_parse_from_conversion() {
let parse_err = "not a url".parse::<url::Url>().expect_err("should fail");
let err: ExtractionUtilsError = parse_err.into();
match err {
ExtractionUtilsError::UrlParseError(_) => { }
other => panic!("expected UrlParseError, got {:?}", other),
}
}
struct TestRule {
sel: String,
array: bool,
attribute: Option<String>,
desc: String,
}
impl ExtractableRule for TestRule {
fn selector(&self) -> &str {
&self.sel
}
fn is_array(&self) -> bool {
self.array
}
fn attr(&self) -> Option<&str> {
self.attribute.as_deref()
}
fn description(&self) -> &str {
&self.desc
}
}
#[test]
fn test_extractable_rule_returns_correct_values() {
let rule = TestRule {
sel: "a.link".to_string(),
array: true,
attribute: Some("href".to_string()),
desc: "Extract all links".to_string(),
};
assert_eq!(rule.selector(), "a.link");
assert!(rule.is_array());
assert_eq!(rule.attr(), Some("href"));
assert_eq!(rule.description(), "Extract all links");
}
#[test]
fn test_extractable_rule_with_no_attr() {
let rule = TestRule {
sel: "p".to_string(),
array: false,
attribute: None,
desc: "Extract paragraph text".to_string(),
};
assert_eq!(rule.selector(), "p");
assert!(!rule.is_array());
assert!(rule.attr().is_none());
assert_eq!(rule.description(), "Extract paragraph text");
}
#[test]
fn test_extract_single_value_non_href_attr_with_base_url() {
let html = r#"<meta name="author" content="Kirky">"#;
let base_url = Url::parse("https://example.com/base").expect("valid url");
let result = ExtractionUtils::extract_single_value(
html,
"meta[name='author']",
Some("content"),
Some(&base_url),
);
assert!(result.is_ok());
assert_eq!(result.expect("ok"), Value::String("Kirky".to_string()));
}
#[test]
fn test_extract_single_value_href_without_base_returns_raw() {
let html = r#"<a href="/path/page">Link</a>"#;
let result = ExtractionUtils::extract_single_value(html, "a", Some("href"), None);
assert!(result.is_ok());
assert_eq!(result.expect("ok"), Value::String("/path/page".to_string()));
}
}