docsrs_cli/docs_rs/html/sanitize.rs
1//! Security scrub (`on*` handlers, `javascript:` URLs, script/style) + HTML→Markdown.
2
3use std::borrow::Cow;
4use std::sync::LazyLock;
5
6use regex::Regex;
7use scraper::{Html, Selector};
8
9use crate::domain::compile_bounded_regex;
10use crate::error::{AppError, AppResult, ErrorDetail};
11
12static SEL_DROP_SCRIPT_STYLE: LazyLock<Selector> = LazyLock::new(|| {
13 Selector::parse("script, style, noscript")
14 .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction")
15});
16
17/// Compiled once: strip `on*` event handlers from HTML fragments (hot path for every HTML→MD).
18pub(super) static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
19 compile_bounded_regex(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
20 .expect("hardcoded on* handler strip regex is valid by construction")
21});
22
23/// Compiled once: strip `javascript:` URLs from href/src attributes.
24pub(super) static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
25 compile_bounded_regex(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
26 .expect("hardcoded javascript: URL strip regex is valid by construction")
27});
28
29/// Compiled once: literal prescan for the tags [`SEL_DROP_SCRIPT_STYLE`] removes.
30///
31/// Deliberately looser than the selector (no end-of-name anchor) so it can only
32/// over-report, never miss a real start tag.
33static RE_DROP_TAG_PROBE: LazyLock<Regex> = LazyLock::new(|| {
34 compile_bounded_regex(r"(?i)<(?:script|style|noscript)")
35 .expect("hardcoded drop-tag probe regex is valid by construction")
36});
37
38/// Remove script/style nodes and on* handlers before Markdown conversion.
39///
40/// Extraction of doc content uses CSS selectors elsewhere in this module.
41/// Regexes here are **security scrub only** (`on*` / `javascript:`), compiled
42/// once via process-static [`LazyLock`] — not used to scrape fields from HTML.
43///
44/// Precondition: `html` is already body-capped (`HARD_MAX_BODY_BYTES`) and resident in RAM.
45///
46/// When there are no script/style/noscript nodes, skips the multi-replace loop
47/// and only runs the process-static hygiene regexes (SCRAPE-S-006).
48pub fn sanitize_html_fragment(html: &str) -> String {
49 // Cheap literal prescan before the DOM build: html5ever never invents a
50 // script/style/noscript element, so no start-tag substring means no drop
51 // target and the whole `parse_fragment` pass is dead work. A false positive
52 // (the literal inside text or an attribute) only costs the parse we already
53 // paid for unconditionally, so the fast path is fail-safe (SCRAPE-S-006).
54 let mut drop_html: Vec<String> = if RE_DROP_TAG_PROBE.is_match(html) {
55 let fragment = Html::parse_fragment(html);
56 // Collect serialized drop-targets once, longest first so nested matches do
57 // not leave residuals, then strip from a single working buffer (SCRAPE-Q-003).
58 fragment
59 .select(&SEL_DROP_SCRIPT_STYLE)
60 .map(|el| el.html())
61 .collect()
62 } else {
63 Vec::new()
64 };
65 drop_html.sort_by_key(|s| std::cmp::Reverse(s.len()));
66 // SCRAPE-S-006: no script/style → scrub hygiene only (no N× replace allocs).
67 let out: Cow<'_, str> = if drop_html.is_empty() {
68 Cow::Borrowed(html)
69 } else {
70 let mut buf = String::with_capacity(html.len());
71 buf.push_str(html);
72 for tag_html in drop_html {
73 if !tag_html.is_empty() {
74 buf = buf.replace(&tag_html, "");
75 }
76 }
77 Cow::Owned(buf)
78 };
79 // Strip event handlers: onload=, onclick=, etc.
80 let out = scrub_pattern(&RE_ON_HANDLERS, out);
81 // Drop javascript: URLs in href/src
82 let out = scrub_pattern(&RE_JS_URLS, out);
83 out.into_owned()
84}
85
86/// Delete every `re` match from `input`, keeping the borrow when nothing matches.
87///
88/// `Regex::replace_all(..).into_owned()` copies the whole body even on a clean
89/// document, so a script-free 10 MiB page paid one full copy per hygiene regex.
90/// Probing with `is_match` first keeps the untouched case allocation-free; the
91/// extra scan only runs when a rewrite was going to allocate anyway.
92pub(super) fn scrub_pattern<'a>(re: &Regex, input: Cow<'a, str>) -> Cow<'a, str> {
93 if re.is_match(&input) {
94 Cow::Owned(re.replace_all(&input, "").into_owned())
95 } else {
96 input
97 }
98}
99
100pub(super) fn html_to_markdown(html: &str) -> AppResult<String> {
101 let cleaned = sanitize_html_fragment(html);
102 // Cause stays in `Error::source` — never embed `{e}` in Display (ADR 0002 / ERR-O-001).
103 let md = htmd::convert(&cleaned)
104 .map_err(|e| AppError::of_with_source(ErrorDetail::HtmlToMarkdown, e))?;
105 Ok(scrub_rustdoc_chrome(&md))
106}
107
108/// Strip rustdoc UI chrome that pollutes LLM context (`§`, "Copy item path", …).
109pub fn scrub_rustdoc_chrome(md: &str) -> String {
110 // `str::replace` allocates even when the needle is absent. Probe first and
111 // keep the borrow, preserving the original strip order (`§` before the
112 // label) so a `Copy§ item path` artefact still collapses exactly as before.
113 let stripped = strip_literal(strip_literal(Cow::Borrowed(md), "§"), "Copy item path");
114 // Single-pass collapse of accidental double spaces (not full reflow).
115 let mut out = String::with_capacity(stripped.len());
116 let mut prev_space = false;
117 for c in stripped.chars() {
118 if c == ' ' {
119 if prev_space {
120 continue;
121 }
122 prev_space = true;
123 out.push(' ');
124 } else {
125 prev_space = false;
126 out.push(c);
127 }
128 }
129 out
130}
131
132/// Delete every occurrence of `needle`, keeping the borrow when it is absent.
133fn strip_literal<'a>(input: Cow<'a, str>, needle: &str) -> Cow<'a, str> {
134 if input.contains(needle) {
135 Cow::Owned(input.replace(needle, ""))
136 } else {
137 input
138 }
139}