use std::borrow::Cow;
use std::sync::LazyLock;
use regex::Regex;
use scraper::{Html, Selector};
use crate::domain::compile_bounded_regex;
use crate::error::{AppError, AppResult, ErrorDetail};
static SEL_DROP_SCRIPT_STYLE: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("script, style, noscript")
.expect("hardcoded scraper selector 'script, style, noscript' is valid by construction")
});
pub(super) static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
compile_bounded_regex(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
.expect("hardcoded on* handler strip regex is valid by construction")
});
pub(super) static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
compile_bounded_regex(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
.expect("hardcoded javascript: URL strip regex is valid by construction")
});
static RE_DROP_TAG_PROBE: LazyLock<Regex> = LazyLock::new(|| {
compile_bounded_regex(r"(?i)<(?:script|style|noscript)")
.expect("hardcoded drop-tag probe regex is valid by construction")
});
pub fn sanitize_html_fragment(html: &str) -> String {
let mut drop_html: Vec<String> = if RE_DROP_TAG_PROBE.is_match(html) {
let fragment = Html::parse_fragment(html);
fragment
.select(&SEL_DROP_SCRIPT_STYLE)
.map(|el| el.html())
.collect()
} else {
Vec::new()
};
drop_html.sort_by_key(|s| std::cmp::Reverse(s.len()));
let out: Cow<'_, str> = if drop_html.is_empty() {
Cow::Borrowed(html)
} else {
let mut buf = String::with_capacity(html.len());
buf.push_str(html);
for tag_html in drop_html {
if !tag_html.is_empty() {
buf = buf.replace(&tag_html, "");
}
}
Cow::Owned(buf)
};
let out = scrub_pattern(&RE_ON_HANDLERS, out);
let out = scrub_pattern(&RE_JS_URLS, out);
out.into_owned()
}
pub(super) fn scrub_pattern<'a>(re: &Regex, input: Cow<'a, str>) -> Cow<'a, str> {
if re.is_match(&input) {
Cow::Owned(re.replace_all(&input, "").into_owned())
} else {
input
}
}
pub(super) fn html_to_markdown(html: &str) -> AppResult<String> {
let cleaned = sanitize_html_fragment(html);
let md = htmd::convert(&cleaned)
.map_err(|e| AppError::of_with_source(ErrorDetail::HtmlToMarkdown, e))?;
Ok(scrub_rustdoc_chrome(&md))
}
pub fn scrub_rustdoc_chrome(md: &str) -> String {
let stripped = strip_literal(strip_literal(Cow::Borrowed(md), "§"), "Copy item path");
let mut out = String::with_capacity(stripped.len());
let mut prev_space = false;
for c in stripped.chars() {
if c == ' ' {
if prev_space {
continue;
}
prev_space = true;
out.push(' ');
} else {
prev_space = false;
out.push(c);
}
}
out
}
fn strip_literal<'a>(input: Cow<'a, str>, needle: &str) -> Cow<'a, str> {
if input.contains(needle) {
Cow::Owned(input.replace(needle, ""))
} else {
input
}
}