use scraper::{ElementRef, Html, Selector};
pub struct Doc {
pub title: String,
pub url: String,
pub lang: String,
pub desc: String,
pub blocks: Vec<Block>,
}
#[derive(Clone, Copy, PartialEq, Debug)]
pub enum Filter {
Headings,
Code,
Tables,
Links,
Img,
Text,
Meta,
}
impl Filter {
pub fn parse(s: &str) -> Result<Vec<Filter>, String> {
s.split(',')
.map(|f| match f.trim() {
"headings" => Ok(Filter::Headings),
"code" => Ok(Filter::Code),
"tables" => Ok(Filter::Tables),
"links" => Ok(Filter::Links),
"img" => Ok(Filter::Img),
"text" => Ok(Filter::Text),
"meta" => Ok(Filter::Meta),
other => Err(format!("unknown filter: {other}")),
})
.collect()
}
}
pub enum Block {
Heading(u8, String),
Para(String),
Item(u8, String),
Quote(String),
Code(String),
Link(String, String),
Img(String, String),
Table {
headers: Vec<String>,
rows: Vec<Vec<String>>,
},
}
impl Block {
fn matches(&self, filters: &[Filter]) -> bool {
if filters.is_empty() {
return !matches!(self, Block::Link(..) | Block::Img(..));
}
let f = match self {
Block::Heading(..) => Filter::Headings,
Block::Code(_) => Filter::Code,
Block::Table { .. } => Filter::Tables,
Block::Link(..) => Filter::Links,
Block::Img(..) => Filter::Img,
Block::Para(_) | Block::Item(..) | Block::Quote(_) => Filter::Text,
};
filters.contains(&f)
}
}
const SKIP: &[&str] = &[
"script", "style", "nav", "footer", "aside", "noscript", "iframe", "form", "svg", "button",
"select", "template",
];
pub fn run(html: &str, url: &str, select: Option<&str>, filters: &[Filter]) -> Doc {
let dom = Html::parse_document(html);
let mut doc = Doc {
title: meta_title(&dom),
url: url.to_string(),
lang: attr(&dom, "html", "lang"),
desc: meta(&dom, "description"),
blocks: Vec::new(),
};
let root = pick_root(&dom, select);
if let Some(el) = root {
walk(el, &mut doc.blocks, 0);
}
dedup(&mut doc.blocks);
doc.blocks.retain(|b| b.matches(filters));
doc
}
pub fn out_links(html: &str) -> Vec<String> {
let dom = Html::parse_document(html);
let sel = Selector::parse("a[href]").unwrap();
let mut seen = std::collections::HashSet::new();
dom.select(&sel)
.filter_map(|a| a.value().attr("href"))
.filter(|h| h.starts_with("http"))
.filter(|h| seen.insert(h.to_string()))
.map(String::from)
.collect()
}
fn pick_root<'a>(dom: &'a Html, select: Option<&str>) -> Option<ElementRef<'a>> {
if let Some(s) = select {
let sel = Selector::parse(s).ok()?;
return dom.select(&sel).next();
}
for s in &["main", "[role=main]"] {
if let Ok(sel) = Selector::parse(s) {
if let Some(el) = dom.select(&sel).next() {
return Some(el);
}
}
}
if let Ok(sel) = Selector::parse("article") {
let best = dom.select(&sel).max_by_key(|el| el.text().count());
if best.is_some() {
return best;
}
}
Selector::parse("body")
.ok()
.and_then(|s| dom.select(&s).next())
}
fn walk(el: ElementRef, out: &mut Vec<Block>, list_depth: u8) {
let tag = el.value().name();
if SKIP.contains(&tag) || is_hidden(&el) {
return;
}
match tag {
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
let lv = tag.as_bytes()[1] - b'0';
push_text(out, text_of(&el), |t| Block::Heading(lv, t));
}
"p" => {
push_text(out, text_of(&el), Block::Para);
collect_media(&el, out);
}
"li" => {
push_text(out, own_text(&el), |t| Block::Item(list_depth, t));
for c in el.child_elements() {
if matches!(c.value().name(), "ul" | "ol") {
walk(c, out, list_depth + 1);
}
}
}
"blockquote" => push_text(out, text_of(&el), Block::Quote),
"pre" => push_text(out, raw_text(&el), Block::Code),
"a" => {
let href = el.value().attr("href").unwrap_or("");
let t = text_of(&el);
if href.starts_with("http") && !t.is_empty() {
out.push(Block::Link(href.to_string(), t));
}
for c in el.child_elements() {
walk(c, out, list_depth);
}
}
"img" | "source" => {
if let Some(src) = img_src(&el) {
let alt = el.value().attr("alt").unwrap_or("").to_string();
out.push(Block::Img(src, alt));
}
}
"picture" | "figure" => {
for c in el.child_elements() {
walk(c, out, list_depth);
}
}
"table" => {
if let Some(tbl) = extract_table(&el) {
out.push(tbl);
}
}
"ul" | "ol" => {
for c in el.child_elements() {
walk(c, out, list_depth);
}
}
_ => {
for c in el.child_elements() {
walk(c, out, list_depth);
}
if el.child_elements().next().is_none() && matches!(tag, "div" | "span" | "td") {
push_text(out, own_text(&el), Block::Para);
}
}
}
}
fn collect_media(el: &ElementRef, out: &mut Vec<Block>) {
for c in el.child_elements() {
match c.value().name() {
"a" => {
let href = c.value().attr("href").unwrap_or("");
let t = text_of(&c);
if href.starts_with("http") && !t.is_empty() {
out.push(Block::Link(href.to_string(), t));
}
}
"img" => {
if let Some(src) = img_src(&c) {
let alt = c.value().attr("alt").unwrap_or("").to_string();
out.push(Block::Img(src, alt));
}
}
_ => collect_media(&c, out),
}
}
}
fn extract_table(el: &ElementRef) -> Option<Block> {
let tr_sel = Selector::parse("tr").unwrap();
let th_sel = Selector::parse("th").unwrap();
let td_sel = Selector::parse("td").unwrap();
let mut headers: Vec<String> = Vec::new();
let mut rows: Vec<Vec<String>> = Vec::new();
for tr in el.select(&tr_sel) {
let ths: Vec<String> = tr.select(&th_sel).map(|c| text_of(&c)).collect();
if !ths.is_empty() && headers.is_empty() {
headers = ths;
continue;
}
let tds: Vec<String> = tr.select(&td_sel).map(|c| text_of(&c)).collect();
if tds.iter().any(|c| !c.is_empty()) {
rows.push(tds);
}
}
if headers.is_empty() && rows.len() > 1 {
headers = rows.remove(0);
}
if rows.is_empty() {
return None;
}
Some(Block::Table { headers, rows })
}
fn img_src(el: &ElementRef) -> Option<String> {
let v = el.value();
for attr in &["src", "data-src", "data-lazy-src", "data-original"] {
if let Some(s) = v.attr(attr) {
if !s.is_empty() && !s.starts_with("data:") {
return Some(s.to_string());
}
}
}
if let Some(ss) = v.attr("srcset") {
if let Some(first) = ss.split(',').next() {
let url = first.split_whitespace().next().unwrap_or("");
if !url.is_empty() && !url.starts_with("data:") {
return Some(url.to_string());
}
}
}
None
}
fn is_hidden(el: &ElementRef) -> bool {
let v = el.value();
v.attr("hidden").is_some()
|| v.attr("aria-hidden") == Some("true")
|| v.attr("style").is_some_and(|s| s.contains("display:none"))
}
fn push_text(out: &mut Vec<Block>, t: String, f: impl Fn(String) -> Block) {
if t.chars().count() > 1 {
out.push(f(t));
}
}
fn text_of(el: &ElementRef) -> String {
squeeze(&el.text().collect::<String>())
}
fn own_text(el: &ElementRef) -> String {
let mut s = String::new();
for node in el.children() {
if let Some(t) = node.value().as_text() {
s.push_str(t);
} else if let Some(c) = ElementRef::wrap(node) {
if !matches!(c.value().name(), "ul" | "ol") && !SKIP.contains(&c.value().name()) {
s.push_str(&c.text().collect::<String>());
}
}
}
squeeze(&s)
}
fn raw_text(el: &ElementRef) -> String {
el.text().collect::<String>().trim().to_string()
}
fn squeeze(s: &str) -> String {
s.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn dedup(blocks: &mut Vec<Block>) {
let mut seen = std::collections::HashSet::new();
blocks.retain(|b| {
let key = match b {
Block::Para(t) | Block::Heading(_, t) | Block::Quote(t) => t.clone(),
Block::Link(h, _) | Block::Img(h, _) => h.clone(),
_ => return true,
};
seen.insert(key)
});
}
fn meta_title(dom: &Html) -> String {
let sel = Selector::parse("title").unwrap();
dom.select(&sel)
.next()
.map(|e| squeeze(&e.text().collect::<String>()))
.unwrap_or_default()
}
fn meta(dom: &Html, name: &str) -> String {
let sel = Selector::parse(&format!(r#"meta[name="{name}"]"#)).unwrap();
dom.select(&sel)
.next()
.and_then(|e| e.value().attr("content"))
.map(squeeze)
.unwrap_or_default()
}
fn attr(dom: &Html, tag: &str, name: &str) -> String {
let sel = Selector::parse(tag).unwrap();
dom.select(&sel)
.next()
.and_then(|e| e.value().attr(name))
.unwrap_or_default()
.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn run_default(html: &str) -> Doc {
run(html, "http://t/", None, &[])
}
fn headings(doc: &Doc) -> Vec<(u8, String)> {
doc.blocks
.iter()
.filter_map(|b| match b {
Block::Heading(l, t) => Some((*l, t.clone())),
_ => None,
})
.collect()
}
fn paras(doc: &Doc) -> Vec<String> {
doc.blocks
.iter()
.filter_map(|b| match b {
Block::Para(t) => Some(t.clone()),
_ => None,
})
.collect()
}
#[test]
fn filter_parse_single_and_multi() {
let f = Filter::parse("code, tables").unwrap();
assert_eq!(f.len(), 2);
assert!(f.contains(&Filter::Code));
assert!(f.contains(&Filter::Tables));
}
#[test]
fn filter_parse_trims_whitespace() {
let f = Filter::parse(" links , img ").unwrap();
assert!(f.contains(&Filter::Links));
assert!(f.contains(&Filter::Img));
}
#[test]
fn filter_parse_all_variants() {
for (s, want) in [
("headings", Filter::Headings),
("code", Filter::Code),
("tables", Filter::Tables),
("links", Filter::Links),
("img", Filter::Img),
("text", Filter::Text),
("meta", Filter::Meta),
] {
assert_eq!(Filter::parse(s).unwrap(), vec![want]);
}
}
#[test]
fn filter_parse_rejects_unknown() {
let e = Filter::parse("headings,bogus").unwrap_err();
assert!(e.contains("bogus"), "error was: {e}");
}
#[test]
fn extracts_title_lang_desc() {
let html = r#"<!doctype html><html lang="ja"><head>
<title> Hello World </title>
<meta name="description" content="a page">
</head><body><p>hi there</p></body></html>"#;
let doc = run_default(html);
assert_eq!(doc.title, "Hello World"); assert_eq!(doc.lang, "ja");
assert_eq!(doc.desc, "a page");
assert_eq!(doc.url, "http://t/");
}
#[test]
fn missing_metadata_is_empty() {
let doc = run_default("<html><body><p>content here</p></body></html>");
assert_eq!(doc.title, "");
assert_eq!(doc.lang, "");
assert_eq!(doc.desc, "");
}
#[test]
fn headings_h1_to_h6() {
let html = "<body><h1>a title</h1><h3>sub head</h3></body>";
let doc = run_default(html);
assert_eq!(
headings(&doc),
vec![(1, "a title".into()), (3, "sub head".into())]
);
}
#[test]
fn single_char_blocks_are_dropped() {
let doc = run_default("<body><p>x</p><p>hello</p></body>");
assert_eq!(paras(&doc), vec!["hello".to_string()]);
}
#[test]
fn nested_list_depth() {
let html = "<body><ul><li>outer<ul><li>inner</li></ul></li></ul></body>";
let doc = run_default(html);
let items: Vec<(u8, String)> = doc
.blocks
.iter()
.filter_map(|b| match b {
Block::Item(d, t) => Some((*d, t.clone())),
_ => None,
})
.collect();
assert_eq!(items, vec![(0, "outer".into()), (1, "inner".into())]);
}
#[test]
fn blockquote_and_pre() {
let html = "<body><blockquote>a quote</blockquote><pre> code line </pre></body>";
let doc = run_default(html);
assert!(doc
.blocks
.iter()
.any(|b| matches!(b, Block::Quote(t) if t == "a quote")));
assert!(doc
.blocks
.iter()
.any(|b| matches!(b, Block::Code(t) if t == "code line")));
}
#[test]
fn skip_tags_removed() {
let html =
"<body><script>var x=1</script><p>real content</p><footer>foot text</footer></body>";
let doc = run_default(html);
let ps = paras(&doc);
assert!(ps.contains(&"real content".to_string()));
assert!(!ps.iter().any(|p| p.contains("var x")));
assert!(!ps.iter().any(|p| p.contains("foot")));
}
#[test]
fn hidden_elements_removed() {
let html = r#"<body>
<p hidden>hidden one</p>
<p aria-hidden="true">hidden two</p>
<p style="display:none">hidden three</p>
<p>visible content</p>
</body>"#;
let ps = paras(&run_default(html));
assert_eq!(ps, vec!["visible content".to_string()]);
}
#[test]
fn dedup_repeated_paragraphs() {
let html = "<body><p>same line</p><p>same line</p><p>other line</p></body>";
assert_eq!(
paras(&run_default(html)),
vec!["same line".to_string(), "other line".to_string()]
);
}
#[test]
fn links_and_imgs_off_by_default() {
let html = r#"<body><p>text</p><a href="https://x.com">x</a><img src="https://y.com/a.png" alt="pic"></body>"#;
let doc = run_default(html);
assert!(!doc
.blocks
.iter()
.any(|b| matches!(b, Block::Link(..) | Block::Img(..))));
}
#[test]
fn links_extracted_with_filter() {
let html =
r#"<body><a href="https://x.com/p">label</a><a href="/relative">skip</a></body>"#;
let doc = run(html, "http://t/", None, &[Filter::Links]);
let links: Vec<(String, String)> = doc
.blocks
.iter()
.filter_map(|b| match b {
Block::Link(h, t) => Some((h.clone(), t.clone())),
_ => None,
})
.collect();
assert_eq!(links, vec![("https://x.com/p".into(), "label".into())]);
}
#[test]
fn img_filter_and_alt() {
let html = r#"<body><img src="https://y.com/a.png" alt="alt text"></body>"#;
let doc = run(html, "http://t/", None, &[Filter::Img]);
assert!(doc.blocks.iter().any(
|b| matches!(b, Block::Img(s, a) if s == "https://y.com/a.png" && a == "alt text")
));
}
#[test]
fn table_with_headers() {
let html = "<body><table><tr><th>name</th><th>price</th></tr>\
<tr><td>a</td><td>100</td></tr><tr><td>b</td><td>200</td></tr></table></body>";
let doc = run(html, "http://t/", None, &[Filter::Tables]);
let tbl = doc.blocks.iter().find_map(|b| match b {
Block::Table { headers, rows } => Some((headers.clone(), rows.clone())),
_ => None,
});
let (headers, rows) = tbl.expect("table block");
assert_eq!(headers, vec!["name", "price"]);
assert_eq!(rows, vec![vec!["a", "100"], vec!["b", "200"]]);
}
#[test]
fn table_without_th_uses_first_row_as_header() {
let html = "<body><table><tr><td>h1</td><td>h2</td></tr>\
<tr><td>a</td><td>b</td></tr></table></body>";
let doc = run(html, "http://t/", None, &[Filter::Tables]);
let (headers, rows) = doc
.blocks
.iter()
.find_map(|b| match b {
Block::Table { headers, rows } => Some((headers.clone(), rows.clone())),
_ => None,
})
.expect("table");
assert_eq!(headers, vec!["h1", "h2"]);
assert_eq!(rows, vec![vec!["a", "b"]]);
}
#[test]
fn select_scopes_extraction() {
let html = r#"<body><p>outside</p><div class="post"><p>inside</p></div></body>"#;
let doc = run(html, "http://t/", Some("div.post"), &[]);
assert_eq!(paras(&doc), vec!["inside".to_string()]);
}
#[test]
fn main_preferred_over_body() {
let html = "<body><p>chrome</p><main><p>article body</p></main></body>";
let doc = run_default(html);
let ps = paras(&doc);
assert!(ps.contains(&"article body".to_string()));
assert!(!ps.contains(&"chrome".to_string()));
}
#[test]
fn out_links_http_only_and_deduped() {
let html = r#"<a href="https://a.com">1</a><a href="/rel">2</a>
<a href="https://a.com">dup</a><a href="http://b.com">3</a>"#;
assert_eq!(
out_links(html),
vec!["https://a.com".to_string(), "http://b.com".to_string()]
);
}
#[test]
fn squeeze_collapses_whitespace() {
assert_eq!(squeeze(" a\n\t b c "), "a b c");
assert_eq!(squeeze(""), "");
}
#[test]
fn img_src_prefers_src_and_rejects_data_uri() {
let dom = Html::parse_fragment(r#"<img src="https://x/a.png">"#);
let sel = Selector::parse("img").unwrap();
let el = dom.select(&sel).next().unwrap();
assert_eq!(img_src(&el).as_deref(), Some("https://x/a.png"));
let dom = Html::parse_fragment(r#"<img src="data:image/png;base64,AAA">"#);
let el = dom.select(&sel).next().unwrap();
assert_eq!(img_src(&el), None);
}
#[test]
fn img_src_falls_back_to_srcset_first_url() {
let dom = Html::parse_fragment(r#"<img srcset="https://x/a.png 1x, https://x/b.png 2x">"#);
let sel = Selector::parse("img").unwrap();
let el = dom.select(&sel).next().unwrap();
assert_eq!(img_src(&el).as_deref(), Some("https://x/a.png"));
}
}