//! HTML to Markdown conversion handler.
//!
//! Uses Mozilla-style readability extraction to extract clean article content
//! before converting to markdown. Falls back to raw html2md for non-article pages.
//!
//! # Pipeline
//!
//! 1. **SPA data extraction** (Next.js/Nuxt): Extract from `__NEXT_DATA__` / `__NUXT__` JSON
//! 2. **Pre-strip hidden sections**: Remove `<details>`, `<noscript>`, `<dialog>` (browser-hidden)
//! 3. **Pre-strip embedded code**: Remove `<script>`, `<style>`, `<template>` noise
//! 4. **Pre-strip noise sections**: Remove advisories, cookie banners, vulnerability text
//! 5. **Pre-strip comments**: Remove comment threads, Disqus, discussion sections
//! 6. **Readability extraction** (default): Extract main article content, strip boilerplate
//! 7. **Fallback to raw html2md**: If extraction fails, use cleaned HTML with basic filtering
//!
//! The readability step significantly improves output quality by removing navigation,
//! footers, ads, and other noise before markdown conversion.
use anyhow::Result;
use scraper::Selector;
use std::sync::LazyLock;
use super::quality;
use super::readability;
use super::spa_extract;
use super::{ContentHandler, ConversionResult};
static HIDDEN_SECTION_SELECTOR: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("details, noscript, dialog").expect("static hidden section selector")
});
static EMBEDDED_NON_SCRIPT_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("style, template").expect("static embedded selector"));
static SCRIPT_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("script").expect("static script selector"));
static BODY_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("body").expect("static body selector"));
static ATTR_SELECTOR: LazyLock<Selector> =
LazyLock::new(|| Selector::parse("[class], [id]").expect("static attr selector"));
static HEADING_WITH_ID_SELECTOR: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("h1[id], h2[id], h3[id], h4[id], h5[id], h6[id]")
.expect("static heading selector")
});
static COMMENT_CONTAINER_SELECTOR: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("div[class], div[id], section[class], section[id]")
.expect("static comment container selector")
});
/// Converts HTML responses to clean markdown.
pub struct HtmlHandler;
/// HTML-specific extraction controls threaded from higher-level commands.
#[derive(Debug, Clone, Copy)]
pub struct HtmlConversionOptions {
/// Allow local SPA hydration-data extraction (`__NEXT_DATA__`, JSON-LD, etc.).
pub allow_spa_extraction: bool,
/// Allow remote thin-content recovery via `r.jina.ai`.
pub allow_jina_fallback: bool,
}
impl Default for HtmlConversionOptions {
fn default() -> Self {
Self {
allow_spa_extraction: true,
allow_jina_fallback: true,
}
}
}
impl ContentHandler for HtmlHandler {
fn supported_types(&self) -> &[&str] {
&["text/html", "application/xhtml+xml"]
}
fn to_markdown(&self, bytes: &[u8], content_type: &str) -> Result<ConversionResult> {
self.to_markdown_with_url(bytes, content_type, None)
}
}
impl HtmlHandler {
/// Convert HTML to markdown, using the real page URL for readability heuristics.
///
/// Providing `url` improves extraction quality: the readability crate uses it
/// for relative link resolution and site-specific scoring.
///
/// # Errors
///
/// Infallible in practice — returns `Err` only if the internal `Ok(...)` somehow
/// fails, which cannot happen with the current implementation.
pub fn to_markdown_with_url(
&self,
bytes: &[u8],
content_type: &str,
url: Option<&str>,
) -> Result<ConversionResult> {
self.to_markdown_with_url_and_options(
bytes,
content_type,
url,
HtmlConversionOptions::default(),
)
}
/// Convert HTML to markdown with explicit extraction controls.
pub fn to_markdown_with_url_and_options(
&self,
bytes: &[u8],
content_type: &str,
url: Option<&str>,
options: HtmlConversionOptions,
) -> Result<ConversionResult> {
let start = std::time::Instant::now();
let html = String::from_utf8_lossy(bytes);
let markdown = html_to_markdown_with_url_and_options(&html, url, options);
let quality = quality::score_extraction(bytes, &markdown);
Ok(ConversionResult {
markdown,
page_count: None,
content_type: content_type.to_string(),
elapsed_ms: start.elapsed().as_secs_f64() * 1000.0,
quality: Some(quality),
})
}
}
/// Convert HTML to markdown with URL-aware readability extraction.
///
/// # Pipeline
///
/// 1. **SPA extraction**: Try `__NEXT_DATA__` / `__NUXT_DATA__` for React/Vue SPAs
/// 2. **Pre-strip**: Hidden elements → embedded code → noise sections → comment sections
/// 3. **Readability extraction**: Extract article content from cleaned HTML
/// 4. **Fallback**: If extraction fails, use cleaned HTML with basic filtering
/// 5. **Thin-content warning**: Emit a tracing warning when output is disproportionately
/// small vs. the HTML body size, indicating JS-rendered content may be missing.
///
/// Passing the real `url` significantly improves readability quality for sites with
/// complex DOM structures (`LessWrong`, `Ghost CMS`, etc.).
#[must_use]
pub fn html_to_markdown_with_url(html: &str, url: Option<&str>) -> String {
html_to_markdown_with_url_and_options(html, url, HtmlConversionOptions::default())
}
/// Convert HTML to markdown with URL-aware readability extraction and explicit controls.
#[must_use]
pub fn html_to_markdown_with_url_and_options(
html: &str,
url: Option<&str>,
options: HtmlConversionOptions,
) -> String {
html_to_markdown_with_url_and_fetcher(html, url, options, fetch_jina_reader)
}
fn html_to_markdown_with_url_and_fetcher<F>(
html: &str,
url: Option<&str>,
options: HtmlConversionOptions,
jina_fetcher: F,
) -> String
where
F: Fn(&str) -> Option<String>,
{
html_to_markdown_with_url_and_sources(
html,
url,
options,
spa_extract::extract_spa_data,
jina_fetcher,
)
}
fn html_to_markdown_with_url_and_sources<S, F>(
html: &str,
url: Option<&str>,
options: HtmlConversionOptions,
spa_extractor: S,
jina_fetcher: F,
) -> String
where
S: Fn(&str) -> Option<String>,
F: Fn(&str) -> Option<String>,
{
const MIN_READABILITY_LEN: usize = 50;
// Try SPA data extraction first (Next.js, Nuxt, etc.)
let spa_candidate = if options.allow_spa_extraction {
spa_extractor(html)
} else {
None
};
if let Some(spa_content) = spa_candidate.as_ref() {
if !is_thin_content(html.len(), spa_content.len()) {
return spa_content.clone();
}
tracing::debug!(
"Thin SPA extraction detected ({} chars from {} bytes HTML) — continuing to fallback paths",
spa_content.len(),
html.len()
);
}
// Pre-strip noise before readability to prevent non-article content
// from dominating the scoring. Order: hidden elements → embedded code → noise sections → comments.
let cleaned_html = strip_hidden_sections(html);
let cleaned_html = strip_embedded_data_sections(&cleaned_html);
let cleaned_html = strip_noise_sections(&cleaned_html);
let cleaned_html = strip_comment_sections(&cleaned_html);
// Try readability extraction with real URL (or fallback placeholder)
let effective_url = url.unwrap_or("https://example.com");
let readability_md =
readability::extract_article(&cleaned_html, effective_url).map(|article| {
let md = html2md::parse_html(&article.content_html);
let lines: Vec<&str> = md
.lines()
.map(str::trim)
.filter(|l| !l.is_empty())
.collect();
let md_result = lines.join("\n");
// html2md sometimes truncates list-heavy content (<ol>/<li>).
// If the article's plain text is significantly longer than the
// markdown output, fall back to the plain text which preserves
// all content from the DOM.
if article.text_content.len() > md_result.len() + 100 {
article.text_content
} else {
md_result
}
});
// Direct html2md on cleaned HTML (preserves tables, lists, etc.)
let direct_md = html_to_markdown(&cleaned_html);
// Pick the better result. Readability produces clean article content but
// is much shorter than direct conversion (which includes navigation, CSS,
// JSON-LD, footers, etc.). Prefer readability whenever it extracted
// non-trivial content (>= 50 chars), since readability strips boilerplate
// that direct_md preserves. Only fall back to direct when readability
// produced nearly nothing.
let markdown = match readability_md {
Some(ref r_md) if r_md.len() >= MIN_READABILITY_LEN => r_md.clone(),
Some(ref r_md) if r_md.len() > direct_md.len() => r_md.clone(),
_ => direct_md,
};
// Post-process: strip markdown noise from html2md output
let markdown = clean_markdown_noise(&markdown);
let markdown = match spa_candidate {
Some(spa_content) if spa_content.len() > markdown.len() => spa_content,
_ => markdown,
};
// Auto-recover when output is suspiciously thin relative to the HTML input.
// A ratio below 2% usually means JS-rendered content was not captured.
// Re-try SPA extraction on the *original* (uncleaned) HTML — the first
// extraction pass may have missed embedded JSON that only appears in
// deeply nested script tags or variable assignments.
if is_thin_content(html.len(), markdown.len()) {
tracing::debug!(
"Thin content detected ({} chars from {} bytes HTML) — attempting recovery",
markdown.len(),
html.len()
);
// Last-resort fallback: Jina reader can render JS-heavy pages.
// Only attempt when we have a URL and local extraction failed.
if options.allow_jina_fallback
&& let Some(page_url) = url
&& let Some(jina_md) = jina_fetcher(page_url)
{
tracing::info!(
"Thin content recovered via Jina reader ({} chars)",
jina_md.len()
);
return jina_md;
}
// Jina fallback either wasn't attempted (no URL) or failed.
// Emit actionable guidance as a warning.
tracing::warn!(
"Output is suspiciously thin ({} chars from {} bytes of HTML). \
The page likely uses JavaScript rendering. Try:\n \
1. nab spa <url> (extract embedded SPA data)\n \
2. nab fetch <url> (uses default browser cookies automatically)\n \
3. nab fetch --cookies brave <url> (override the browser profile if needed)",
markdown.len(),
html.len()
);
}
markdown
}
/// Check if markdown output is suspiciously thin relative to HTML input size.
///
/// Returns `true` when the markdown is disproportionately small compared to
/// the raw HTML. This typically indicates JS-rendered content not captured.
///
/// Thresholds: HTML >= 5 KB, markdown < 800 chars, ratio < 2%.
#[must_use]
fn is_thin_content(html_len: usize, markdown_len: usize) -> bool {
const MIN_HTML_LEN: usize = 5_000;
const MIN_MARKDOWN_LEN: usize = 800;
const THIN_RATIO_PERCENT: usize = 2;
if html_len < MIN_HTML_LEN || markdown_len >= MIN_MARKDOWN_LEN {
return false;
}
let ratio_percent = (markdown_len * 100) / html_len.max(1);
ratio_percent < THIN_RATIO_PERCENT
}
/// Fetch content from Jina reader as a last-resort fallback for JS-rendered pages.
///
/// Jina reader (`r.jina.ai`) renders JavaScript and returns clean markdown.
/// This is used when local extraction produces suspiciously thin content,
/// indicating the page relies on client-side rendering.
///
/// This helper performs blocking network I/O. Async fetch paths should call it
/// only from blocking conversion contexts.
///
/// Returns `Some(markdown)` if Jina returns substantial content (> 200 chars),
/// `None` on any failure (network error, empty response, timeout).
fn fetch_jina_reader(url: &str) -> Option<String> {
const MIN_JINA_CONTENT_LEN: usize = 200;
const JINA_TIMEOUT_SECS: u64 = 15;
let jina_url = format!("https://r.jina.ai/{url}");
tracing::debug!("Fetching Jina reader fallback: {}", jina_url);
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(JINA_TIMEOUT_SECS))
.build()
.ok()?;
let response = client
.get(&jina_url)
.header("Accept", "text/markdown")
.send()
.ok()?;
if !response.status().is_success() {
tracing::debug!("Jina reader returned HTTP {}", response.status().as_u16());
return None;
}
let body = response.text().ok()?;
let trimmed = body.trim();
if trimmed.len() >= MIN_JINA_CONTENT_LEN {
Some(trimmed.to_string())
} else {
tracing::debug!(
"Jina reader returned thin content ({} chars), discarding",
trimmed.len()
);
None
}
}
/// Detect suspiciously thin markdown output relative to HTML input size.
///
/// Returns a warning message when the markdown is disproportionately small
/// compared to the raw HTML. This typically indicates JavaScript-rendered
/// content that was not captured by the static HTML parser.
///
/// # Returns
///
/// `Some(warning)` when the ratio is below the threshold, `None` otherwise.
#[must_use]
pub fn detect_thin_content(html_len: usize, markdown_len: usize) -> Option<String> {
if is_thin_content(html_len, markdown_len) {
let ratio_percent = (markdown_len * 100) / html_len.max(1);
Some(format!(
"Output is suspiciously thin ({markdown_len} chars from {html_len} bytes of HTML, \
{ratio_percent}% ratio). The page likely uses JavaScript rendering — \
the article body may be missing. Try:\n \
1. nab spa <url> (extract embedded SPA data)\n \
2. nab fetch <url> (uses default browser cookies automatically)\n \
3. nab fetch --cookies brave <url> (override the browser profile if needed)"
))
} else {
None
}
}
/// Convert HTML to markdown with readability extraction (URL-unaware).
///
/// Calls [`html_to_markdown_with_url`] with no URL. Prefer the URL-aware
/// variant when the fetch URL is available.
#[must_use]
pub fn html_to_markdown_with_readability(html: &str) -> String {
html_to_markdown_with_url_and_options(html, None, HtmlConversionOptions::default())
}
/// Remove hidden/collapsed sections from HTML before readability processing.
///
/// Browsers collapse `<details>` elements by default and hide `<noscript>` content
/// when JavaScript is enabled. However, readability and html2md see the full DOM
/// text, so these elements can dominate scoring and crowd out the actual page content.
///
/// This is critical for pages like `deps.rs` where dozens of `<details>` elements
/// containing security advisory text overwhelm the dependency status table.
///
/// # Targeted elements
///
/// - `<details>` — collapsed by default unless `open` attribute is present
/// - `<noscript>` — hidden when JS is available (which static fetch implies)
/// - `<dialog>` — hidden unless `open` attribute is present
///
/// Elements with the `open` attribute are preserved since the page author
/// explicitly intended them to be visible.
#[must_use]
pub fn strip_hidden_sections(html: &str) -> String {
let document = scraper::Html::parse_document(html);
// Collect node IDs of hidden elements to remove
// Keep <details open>, <dialog open> — those are intentionally visible
let hidden_ids: std::collections::HashSet<ego_tree::NodeId> = document
.select(&HIDDEN_SECTION_SELECTOR)
.filter(|el| {
let name = el.value().name();
// noscript is always hidden in a JS-capable context
if name == "noscript" {
return true;
}
// details/dialog: remove only if NOT open
el.value().attr("open").is_none()
})
.map(|el| el.id())
.collect();
rebuild_document_excluding(&document, html, &hidden_ids)
}
/// Remove embedded code and templating payloads before fallback extraction.
///
/// JS-heavy pages often ship large inline bootstrap blobs in `<script>` tags or
/// templating payloads in `<template>` blocks. When readability cannot recover the
/// real article body, `html2md` may otherwise emit those blobs as markdown, which
/// prevents the thin-content heuristic from reaching the existing Jina fallback.
///
/// Preserve Schema.org JSON-LD scripts: they are structured content rather than
/// bootstrap noise, and some pages rely on them as the only meaningful fallback.
#[must_use]
pub fn strip_embedded_data_sections(html: &str) -> String {
let document = scraper::Html::parse_document(html);
let mut embedded_ids = std::collections::HashSet::new();
embedded_ids.extend(
document
.select(&EMBEDDED_NON_SCRIPT_SELECTOR)
.map(|el| el.id()),
);
embedded_ids.extend(
document
.select(&SCRIPT_SELECTOR)
.filter(|script| !is_jsonld_script(script))
.map(|script| script.id()),
);
rebuild_document_excluding(&document, html, &embedded_ids)
}
fn is_jsonld_script(script: &scraper::ElementRef<'_>) -> bool {
script
.value()
.attr("type")
.and_then(|script_type| script_type.split(';').next())
.is_some_and(|script_type| {
script_type
.trim()
.eq_ignore_ascii_case("application/ld+json")
})
}
/// Recursively serialize a subtree, skipping nodes whose IDs are in `exclude`.
///
/// This handles nested hidden elements at any depth (not just direct children
/// of `<body>`), which is essential for pages where `<details>` elements appear
/// inside `<div>` wrappers or table cells.
fn serialize_children_excluding(
document: &scraper::Html,
parent_id: ego_tree::NodeId,
exclude: &std::collections::HashSet<ego_tree::NodeId>,
) -> String {
let Some(node) = document.tree.get(parent_id) else {
return String::new();
};
let mut out = String::new();
for child in node.children() {
if exclude.contains(&child.id()) {
continue;
}
match child.value() {
scraper::Node::Element(el) => {
// Open tag
out.push('<');
out.push_str(el.name());
for (k, v) in el.attrs() {
out.push(' ');
out.push_str(k);
out.push_str("=\"");
out.push_str(&v.replace('"', """));
out.push('"');
}
out.push('>');
// Recurse into children (may also contain excluded nodes)
out.push_str(&serialize_children_excluding(document, child.id(), exclude));
// Close tag (skip void elements)
if !is_void_element(el.name()) {
out.push_str("</");
out.push_str(el.name());
out.push('>');
}
}
scraper::Node::Text(text) => {
out.push_str(text);
}
_ => {}
}
}
out
}
/// Rebuild a document while excluding a set of node IDs from both `<head>` and `<body>`.
fn rebuild_document_excluding(
document: &scraper::Html,
original_html: &str,
exclude: &std::collections::HashSet<ego_tree::NodeId>,
) -> String {
if exclude.is_empty() {
return original_html.to_string();
}
let head = scraper::Selector::parse("head")
.ok()
.and_then(|sel| document.select(&sel).next());
let body = scraper::Selector::parse("body")
.ok()
.and_then(|sel| document.select(&sel).next());
let Some(body) = body else {
return original_html.to_string();
};
let head_html = head
.map(|head| serialize_children_excluding(document, head.id(), exclude))
.unwrap_or_default();
let body_html = serialize_children_excluding(document, body.id(), exclude);
if head_html.is_empty() {
format!("<html><body>{body_html}</body></html>")
} else {
format!("<html><head>{head_html}</head><body>{body_html}</body></html>")
}
}
/// Returns `true` for HTML void elements that must not have a closing tag.
fn is_void_element(name: &str) -> bool {
matches!(
name,
"area"
| "base"
| "br"
| "col"
| "embed"
| "hr"
| "img"
| "input"
| "link"
| "meta"
| "param"
| "source"
| "track"
| "wbr"
)
}
/// Remove noise sections (advisories, cookie banners, etc.) from HTML.
///
/// Identifies elements whose class or id attributes match known noise patterns
/// and strips them using deep recursive exclusion. Unlike `strip_comment_sections`
/// (which only filters direct children of `<body>`), this handles noise elements
/// nested at any depth — critical for pages like `deps.rs` where advisory `<div>`
/// elements sit inside wrapper containers.
///
/// # Targeted patterns
///
/// - `#vulnerabilities`, `.advisories`, `.security-advisories` — security advisory sections
/// - `.cookie-banner`, `.consent-banner`, `.gdpr-banner` — cookie/consent UI
/// - `.newsletter-signup`, `.subscribe-form` — signup noise
/// - Headings (`<h1>`–`<h6>`) with noise IDs and all sibling elements that follow
#[must_use]
pub fn strip_noise_sections(html: &str) -> String {
let document = scraper::Html::parse_document(html);
// Collect node IDs of noise elements
let mut noise_ids: std::collections::HashSet<ego_tree::NodeId> = document
.select(&ATTR_SELECTOR)
.filter(|el| {
let class = el.value().attr("class").unwrap_or("");
let id = el.value().attr("id").unwrap_or("");
let combined = format!("{class} {id}").to_lowercase();
is_noise_section(&combined)
})
.map(|el| el.id())
.collect();
// Special case: headings with noise IDs (e.g., <h3 id="vulnerabilities">)
// Strip all subsequent siblings of such headings, since the advisory content
// follows the heading as sibling <div> elements (deps.rs pattern).
for heading in document.select(&HEADING_WITH_ID_SELECTOR) {
let id = heading.value().attr("id").unwrap_or("");
if is_noise_section(&id.to_lowercase()) {
// Mark the heading itself
noise_ids.insert(heading.id());
// Mark all subsequent siblings
let mut sibling = document
.tree
.get(heading.id())
.and_then(|n| n.next_sibling());
while let Some(sib) = sibling {
noise_ids.insert(sib.id());
sibling = sib.next_sibling();
}
}
}
rebuild_document_excluding(&document, html, &noise_ids)
}
/// Remove comment section DOM nodes from HTML before readability processing.
///
/// Sites like `LessWrong` have dense comment sections that confuse the readability
/// crate's scoring heuristics. We identify comment containers by common class/id
/// patterns and blank out their text content, preserving the DOM structure so
/// the readability crate can still score remaining nodes correctly.
///
/// This is a fast string-scan approach — not full DOM manipulation — so it
/// operates on the raw HTML string and handles nested structures by targeting
/// outermost comment containers.
///
/// # Panics
///
/// Never panics in practice — the `"div"` CSS selector used as a last-resort
/// fallback is always valid and cannot fail to parse.
#[must_use]
pub fn strip_comment_sections(html: &str) -> String {
let document = scraper::Html::parse_document(html);
// We can't mutate the scraper DOM, so we serialize element HTML of everything
// EXCEPT comment containers, then reconstruct a valid document.
// Build a set of comment container hashes to skip
let comment_container_ids: std::collections::HashSet<u64> = document
.select(&COMMENT_CONTAINER_SELECTOR)
.filter(|el| is_comment_container(el))
.map(|el| element_hash(el))
.collect();
if comment_container_ids.is_empty() {
return html.to_string();
}
// Rebuild HTML without comment containers by serializing the body's
// direct children that aren't comment containers, then wrapping.
// For simplicity and correctness, we blank the comment node HTML.
let body_html = document.select(&BODY_SELECTOR).next().map_or_else(
|| html.to_string(),
|body| {
// Filter children: collect HTML of non-comment children
body.children()
.filter_map(|child| {
let el_ref = scraper::ElementRef::wrap(child)?;
if comment_container_ids.contains(&element_hash(el_ref)) {
None
} else {
Some(el_ref.html())
}
})
.collect::<String>()
},
);
// Preserve head and wrap body
let head_html = scraper::Selector::parse("head")
.ok()
.and_then(|sel| document.select(&sel).next())
.map(|h| h.html())
.unwrap_or_default();
format!("<html>{head_html}<body>{body_html}</body></html>")
}
/// Returns `true` if this element looks like a comment section container.
fn is_comment_container(element: &scraper::ElementRef<'_>) -> bool {
let class = element.value().attr("class").unwrap_or("");
let id = element.value().attr("id").unwrap_or("");
let combined = format!("{class} {id}").to_lowercase();
is_noise_section(&combined)
}
/// Returns `true` if a class/id string indicates a noise section that should
/// be stripped before readability processing.
///
/// Covers: comment sections, vulnerability advisories, cookie banners,
/// newsletter signup forms, and similar non-article content that tends to
/// dominate readability scoring due to high text density.
fn is_noise_section(combined: &str) -> bool {
const NOISE_MARKERS: &[&str] = &[
// Comment sections
"comment-section",
"comments-section",
"comments-container",
"comment-list",
"comment-thread",
"disqus",
"discussion-section",
"replies-section",
// Security advisories / vulnerability sections (deps.rs, GitHub, etc.)
"vulnerabilities",
"advisories",
"security-advisories",
"advisory-list",
// Cookie/consent banners
"cookie-banner",
"cookie-consent",
"consent-banner",
"gdpr-banner",
// Newsletter/signup noise
"newsletter-signup",
"subscribe-form",
// Footer noise
"site-footer",
"global-footer",
];
NOISE_MARKERS.iter().any(|marker| combined.contains(marker))
}
/// Stable hash for a DOM element based on its outer HTML.
///
/// Used to identify specific elements across two passes over the parsed DOM.
fn element_hash(element: scraper::ElementRef<'_>) -> u64 {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
// Hash the element's name + attributes as a structural fingerprint
element.value().name().hash(&mut hasher);
for (k, v) in element.value().attrs() {
k.hash(&mut hasher);
v.hash(&mut hasher);
}
hasher.finish()
}
/// Strip common markdown noise produced by html2md.
///
/// Removes artefacts that are technically valid markdown but carry no useful
/// information for an LLM or human reader:
///
/// - **Base64 data URIs**: Inline `` badges/icons (often huge)
/// - **Empty links**: `[](url)` from `<a><img/></a>` patterns (icon-only links)
/// - **Bare image noise in tables**: `[](url)` cells that are just crate icons
///
/// Applied as a final post-processing pass on the markdown string.
#[must_use]
fn clean_markdown_noise(md: &str) -> String {
md.lines()
.map(|line| {
let mut cleaned = line.to_string();
// Remove inline base64 data URIs:  or 
while let Some(start) = cleaned.find(" = after_bang.find("](data:") {
// Find the closing )
let data_start = start + 2 + paren + 2; // past ](
if let Some(close) = cleaned[data_start..].find(')') {
let end = data_start + close + 1;
cleaned = format!("{}{}", &cleaned[..start], &cleaned[end..]);
continue;
}
}
break;
}
// Remove empty link artefacts: [](url) — no link text, just icon wrappers
while let Some(start) = cleaned.find("[](") {
if let Some(close) = cleaned[start + 3..].find(')') {
let end = start + 3 + close + 1;
let replacement = &cleaned[end..];
cleaned = format!("{}{}", cleaned[..start].trim_end(), replacement);
continue;
}
break;
}
cleaned
})
.filter(|l| {
let trimmed = l.trim();
// Drop lines that are entirely base64 data or became empty after cleaning
!trimmed.is_empty()
&& !trimmed.starts_with("![](data:")
&& !trimmed.starts_with("```\n[![")
})
.collect::<Vec<_>>()
.join("\n")
}
/// Convert HTML to markdown with boilerplate filtering (fallback).
///
/// Uses `html2md` for the heavy lifting, then post-processes to remove
/// common web boilerplate (cookie notices, navigation, privacy footers)
/// and collapse excessive whitespace.
pub fn html_to_markdown(html: &str) -> String {
let md = html2md::parse_html(html);
let lines: Vec<&str> = md
.lines()
.map(str::trim)
.filter(|l| !l.is_empty())
.filter(|l| !is_boilerplate(l))
.collect();
lines.join("\n")
}
/// Returns `true` if a line looks like web boilerplate.
pub fn is_boilerplate(line: &str) -> bool {
// Preserve markdown links -- never filter lines containing link syntax
if line.contains("](") {
return false;
}
let lower = line.to_lowercase();
lower.contains("skip to content")
|| lower.contains("cookie")
|| lower.contains("privacy policy")
|| lower.contains("terms of service")
|| lower.starts_with("©")
|| lower.starts_with("copyright")
|| (lower.len() < 3 && !lower.chars().any(char::is_alphanumeric))
}
#[cfg(test)]
mod tests {
use super::{
HtmlConversionOptions, html_to_markdown_with_url, html_to_markdown_with_url_and_fetcher,
html_to_markdown_with_url_and_sources, is_thin_content, strip_embedded_data_sections,
strip_hidden_sections, strip_noise_sections,
};
#[test]
fn is_thin_content_returns_false_for_small_html_below_threshold() {
// GIVEN: HTML smaller than the 5 KB minimum threshold
// WHEN: checking thin content
let result = is_thin_content(1_000, 10);
// THEN: not considered thin (threshold not reached)
assert!(!result);
}
#[test]
fn is_thin_content_returns_false_for_adequate_markdown() {
// GIVEN: large HTML but markdown of 900 chars (>= 800 minimum)
// WHEN: checking thin content
let result = is_thin_content(10_000, 900);
// THEN: not considered thin (markdown exceeds the partial-content floor)
assert!(!result);
}
#[test]
fn is_thin_content_returns_true_for_thin_spa_page() {
// GIVEN: 50 KB HTML with only 50 chars of markdown output
// WHEN: checking thin content
let result = is_thin_content(50_000, 50);
// THEN: flagged as thin (50/50000 = 0.1%, well below 2% threshold)
assert!(result);
}
#[test]
fn is_thin_content_returns_true_for_partial_extraction_below_new_floor() {
// GIVEN: a JS-heavy page where only a short excerpt was extracted
// WHEN: checking thin content with 600 chars from 50 KB HTML
let result = is_thin_content(50_000, 600);
// THEN: still flagged as thin so fallback recovery can run
assert!(result);
}
#[test]
fn is_thin_content_boundary_at_799_chars_is_thin() {
// GIVEN: 50 KB HTML with 799 chars of markdown (one below the new 800-char boundary)
// WHEN: checking thin content
let result = is_thin_content(50_000, 799);
// THEN: flagged as thin (799 < 800 minimum, ratio < 2%)
assert!(result);
}
#[test]
fn is_thin_content_boundary_at_800_chars_is_not_thin() {
// GIVEN: 50 KB HTML with exactly 800 chars of markdown (at the boundary)
// WHEN: checking thin content
let result = is_thin_content(50_000, 800);
// THEN: NOT flagged as thin (800 >= 800 minimum satisfies the condition)
assert!(!result);
}
#[test]
fn strip_embedded_data_sections_removes_bootstrap_but_preserves_jsonld() {
let html = r#"<html>
<head>
<style>.hidden { display: none; }</style>
<script>window.__BOOT__ = {"title":"Hidden"};</script>
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"Article","description":"Structured fallback"}
</script>
</head>
<body>
<article><p>Visible content</p></article>
<template><div>Template payload</div></template>
<script>window.__NEXT_DATA__ = {"props":{"pageProps":{"body":"noise"}}};</script>
</body>
</html>"#;
let result = strip_embedded_data_sections(html);
assert!(result.contains("Visible content"));
assert!(!result.contains("window.__BOOT__"));
assert!(!result.contains("Template payload"));
assert!(!result.contains("__NEXT_DATA__"));
assert!(!result.contains("display: none"));
assert!(result.contains("application/ld+json"));
assert!(result.contains("Structured fallback"));
}
#[test]
fn html_to_markdown_with_url_drops_inline_script_bootstrap_noise() {
let bootstrap = "const boot='very noisy js bootstrap';".repeat(120);
let html = format!(
"<html><body><div id='app'>Loading…</div><script>{bootstrap}</script></body></html>"
);
let markdown = html_to_markdown_with_url(&html, None);
assert!(!markdown.contains("very noisy js bootstrap"));
assert!(!markdown.contains("const boot="));
}
#[test]
fn html_to_markdown_uses_jina_fallback_when_enabled() {
let bootstrap = "const boot='very noisy js bootstrap';".repeat(220);
let html = format!(
"<html><body><div id='app'>Loading…</div><script>{bootstrap}</script></body></html>"
);
assert!(html.len() > 5_000);
let calls = std::cell::Cell::new(0);
let markdown = html_to_markdown_with_url_and_fetcher(
&html,
Some("https://example.com/article"),
HtmlConversionOptions::default(),
|url| {
calls.set(calls.get() + 1);
Some(format!("Recovered remotely from {url}"))
},
);
assert_eq!(calls.get(), 1);
assert_eq!(
markdown,
"Recovered remotely from https://example.com/article"
);
}
#[test]
fn thin_spa_extraction_uses_jina_fallback_when_enabled() {
let code_sample = "from vllm import LLM\\n".repeat(18);
let bootstrap = "const boot='very noisy js bootstrap';".repeat(1_000);
let html = format!(
"<html><body><div id='app'>Loading…</div><script>{bootstrap}</script></body></html>"
);
let calls = std::cell::Cell::new(0);
let markdown = html_to_markdown_with_url_and_sources(
&html,
Some("https://example.com/article"),
HtmlConversionOptions::default(),
|_html| Some(code_sample.clone()),
|url| {
calls.set(calls.get() + 1);
Some(format!("Recovered remotely from {url}"))
},
);
assert_eq!(calls.get(), 1);
assert_eq!(
markdown,
"Recovered remotely from https://example.com/article"
);
}
#[test]
fn html_to_markdown_skips_jina_fallback_when_disabled() {
let bootstrap = "const boot='very noisy js bootstrap';".repeat(220);
let html = format!(
"<html><body><div id='app'>Loading…</div><script>{bootstrap}</script></body></html>"
);
assert!(html.len() > 5_000);
let calls = std::cell::Cell::new(0);
let markdown = html_to_markdown_with_url_and_fetcher(
&html,
Some("https://example.com/article"),
HtmlConversionOptions {
allow_spa_extraction: true,
allow_jina_fallback: false,
},
|_url| {
calls.set(calls.get() + 1);
Some("Recovered remotely".to_string())
},
);
assert_eq!(calls.get(), 0);
assert!(markdown.contains("Loading"));
assert!(!markdown.contains("Recovered remotely"));
}
#[test]
fn thin_spa_extraction_is_retained_when_jina_fallback_disabled() {
let code_sample = "from vllm import LLM\\n".repeat(18);
let bootstrap = "const boot='very noisy js bootstrap';".repeat(1_000);
let html = format!(
"<html><body><div id='app'>Loading…</div><script>{bootstrap}</script></body></html>"
);
let calls = std::cell::Cell::new(0);
let markdown = html_to_markdown_with_url_and_sources(
&html,
Some("https://example.com/article"),
HtmlConversionOptions {
allow_spa_extraction: true,
allow_jina_fallback: false,
},
|_html| Some(code_sample.clone()),
|_url| {
calls.set(calls.get() + 1);
Some("Recovered remotely".to_string())
},
);
assert_eq!(calls.get(), 0);
assert!(markdown.contains("from vllm import LLM"));
assert!(!markdown.contains("Recovered remotely"));
}
#[test]
fn html_to_markdown_can_disable_spa_extraction() {
let article_body = "A".repeat(320);
let html = format!(
r#"<html><body>
<p>Loading…</p>
<script id="__NEXT_DATA__" type="application/json">
{{"props":{{"pageProps":{{"body":"{article_body}"}}}}}}
</script>
</body></html>"#
);
let markdown = html_to_markdown_with_url_and_fetcher(
&html,
Some("https://example.com/article"),
HtmlConversionOptions {
allow_spa_extraction: false,
allow_jina_fallback: false,
},
|_url| Some("Recovered remotely".to_string()),
);
assert!(!markdown.contains(&article_body));
assert!(markdown.contains("Loading"));
}
#[test]
fn strip_hidden_sections_removes_closed_details() {
let html = r"<html><body>
<h1>Status</h1>
<p>All good</p>
<details><summary>Advisory</summary><p>CVE-2024-1234</p></details>
</body></html>";
let result = strip_hidden_sections(html);
assert!(!result.contains("CVE-2024-1234"));
assert!(result.contains("All good"));
assert!(result.contains("Status"));
}
#[test]
fn strip_hidden_sections_preserves_open_details() {
let html = r"<html><body>
<details open><summary>Visible</summary><p>Important info</p></details>
<details><summary>Hidden</summary><p>Secret</p></details>
</body></html>";
let result = strip_hidden_sections(html);
assert!(result.contains("Important info"));
assert!(!result.contains("Secret"));
}
#[test]
fn strip_hidden_sections_removes_noscript() {
let html = r"<html><body>
<p>Main content</p>
<noscript><p>Enable JavaScript</p></noscript>
</body></html>";
let result = strip_hidden_sections(html);
assert!(result.contains("Main content"));
assert!(!result.contains("Enable JavaScript"));
}
#[test]
fn strip_hidden_sections_removes_closed_dialog() {
let html = r"<html><body>
<p>Page</p>
<dialog><p>Modal content</p></dialog>
<dialog open><p>Visible modal</p></dialog>
</body></html>";
let result = strip_hidden_sections(html);
assert!(result.contains("Page"));
assert!(!result.contains("Modal content"));
assert!(result.contains("Visible modal"));
}
#[test]
fn strip_hidden_sections_handles_nested_details_in_divs() {
let html = r#"<html><body>
<div class="content"><p>Real content</p></div>
<div class="advisories">
<details><summary>CVE-1</summary><p>Bad thing 1</p></details>
<details><summary>CVE-2</summary><p>Bad thing 2</p></details>
</div>
</body></html>"#;
let result = strip_hidden_sections(html);
assert!(result.contains("Real content"));
assert!(!result.contains("Bad thing 1"));
assert!(!result.contains("Bad thing 2"));
}
#[test]
fn strip_hidden_sections_noop_when_no_hidden_elements() {
let html = r"<html><body><p>Just text</p></body></html>";
let result = strip_hidden_sections(html);
assert!(result.contains("Just text"));
}
#[test]
fn strip_noise_sections_removes_vulnerabilities_heading_and_siblings() {
// Simulates deps.rs structure: h3#vulnerabilities followed by advisory boxes
let html = r#"<html><body>
<table><tr><td>reqwest</td><td>up to date</td></tr></table>
<h3 id="vulnerabilities">Security Vulnerabilities</h3>
<div class="box"><p>CVE-2022-24713 regex advisory</p></div>
<div class="box"><p>chrono segfault advisory</p></div>
</body></html>"#;
let result = strip_noise_sections(html);
assert!(result.contains("reqwest"));
assert!(result.contains("up to date"));
assert!(!result.contains("CVE-2022-24713"));
assert!(!result.contains("chrono segfault"));
assert!(!result.contains("Security Vulnerabilities"));
}
#[test]
fn strip_noise_sections_removes_advisory_by_class() {
let html = r#"<html><body>
<p>Main content</p>
<div class="advisories"><p>RUSTSEC-2020-0159</p></div>
</body></html>"#;
let result = strip_noise_sections(html);
assert!(result.contains("Main content"));
assert!(!result.contains("RUSTSEC-2020-0159"));
}
#[test]
fn strip_noise_sections_removes_cookie_banner() {
let html = r#"<html><body>
<article><p>Article text</p></article>
<div class="cookie-banner"><p>Accept cookies</p></div>
</body></html>"#;
let result = strip_noise_sections(html);
assert!(result.contains("Article text"));
assert!(!result.contains("Accept cookies"));
}
#[test]
fn strip_noise_sections_preserves_all_when_no_noise() {
let html = r"<html><body><p>Clean page</p></body></html>";
let result = strip_noise_sections(html);
assert!(result.contains("Clean page"));
}
#[test]
fn ghost_blog_ordered_list_not_truncated() {
// Ghost CMS article with <ol><li> content.
// html2md truncates ordered lists; the pipeline must detect this
// and fall back to plain text to preserve the full article.
use super::html_to_markdown_with_url;
let html = r#"
<html>
<head><title>Porting Software</title></head>
<body>
<header><nav>Site Nav</nav></header>
<main class="site-main">
<article class="gh-article post tag-ai">
<header class="gh-article-header">
<h1 class="gh-article-title">porting software has been trivial</h1>
</header>
<div class="gh-content gh-canvas">
<p>This one is short and sweet. if you want to port a codebase from one language to another here's the approach:</p>
<ol>
<li>Run a ralph loop which compresses all tests into specs which looks similar to study every file in tests using separate subagents and document in specs and link the implementation as citations in the specification</li>
<li>Then do a separate Ralph loop for all product functionality ensuring there are citations to the specification. Study every file in src using separate subagents per file and link the implementation as citations in the specification</li>
<li>Once you have that within the same repo run a Ralph loop to create a TODO file and then execute a classic ralph doing just one thing and the most important thing per loop. Remind the agent that it can study the specifications and follow the citations to reference source code.</li>
<li>For best outcomes you wanna configure your target language to have strict compilation</li>
</ol>
<p>The key theory here is usage of citations in the specifications which tease the file_read tool to study the original implementation during stage 3. Reducing stage 1 and stage 2 to specs is the precursor which transforms a code base into high level PRDs without coupling the implementation from the source language.</p>
</div>
</article>
</main>
<section class="newsletter-signup"><p>Subscribe</p></section>
<footer>Copyright</footer>
</body>
</html>
"#;
let md = html_to_markdown_with_url(html, Some("https://ghuntley.com/porting/"));
// Must contain the conclusion paragraph (the part html2md truncates)
assert!(
md.contains("high level PRDs without coupling"),
"Missing conclusion paragraph in markdown output ({} chars): {}",
md.len(),
md
);
// Must contain list items
assert!(
md.contains("Ralph loop"),
"Missing list content in markdown output: {}",
md
);
}
#[test]
fn ghost_blog_real_html_not_truncated() {
// Test with actual Ghost CMS HTML structure that triggers html2md truncation.
// The key difference from the simplified test above: the <em> tags inside
// the <li> elements and the inline <ol> (no whitespace between tags).
use super::html_to_markdown_with_url;
let html = r#"<html>
<head><title>porting software has been trivial for a while now. here's how you do it.</title></head>
<body>
<header><nav>Site Nav</nav></header>
<main class="site-main">
<article class="gh-article post tag-ai">
<header class="gh-article-header"><h1 class="gh-article-title">porting software has been trivial</h1></header>
<div class="gh-content gh-canvas">
<p>This one is short and sweet. if you want to port a codebase from one language to another here's the approach:</p><ol><li>Run a ralph loop which compresses all tests into /specs/<em>.md which looks similar to "study every file in tests/</em>* using separate subagents and document in /specs/*.md and link the implementation as citations in the specification"</li><li>Then do a separate Ralph loop for all product functionality - ensuring there's citations to the specification. "study every file in src/* using seperate subagents per file and link the implementation as citations in the specification"</li><li>Once you have that - within the same repo run a Ralph loop to create a TODO file and then execute a classic ralph - doing just one thing and the most important thing per loop. Remind the agent that it can study the specifications and follow the citations to reference source code.</li><li>For best outcomes you wanna configure your target language to have strict compilation </li></ol><p>The key theory here is usage of citations in the specifications which tease the file_read tool to study the original implementation during stage 3. Reducing stage 1 and stage 2 to specs is the precursor which transforms a code base into high level PRDs without coupling the implementation from the source language.</p>
</div>
</article>
</main>
<section class="newsletter-signup"><h3>Subscribe</h3><p>Join subscribers</p></section>
<footer><p>Copyright 2026</p></footer>
</body></html>"#;
let md = html_to_markdown_with_url(html, Some("https://ghuntley.com/porting/"));
// Must contain the conclusion paragraph
assert!(
md.contains("high level PRDs without coupling"),
"Missing conclusion paragraph in markdown output ({} chars): {}",
md.len(),
md
);
// Must contain list item content
assert!(
md.contains("Ralph loop") || md.contains("ralph loop"),
"Missing list content in markdown output: {}",
md
);
// Should be substantial
assert!(
md.len() > 500,
"Markdown too short: {} chars. Content: {}",
md.len(),
md
);
}
}