Skip to main content

docsrs_cli/docs_rs/
html.rs

1//! HTML sanitize, version scrape, and markdown extraction (CPU-bound pure paths).
2//!
3//! Runs under `spawn_blocking` via fetch/search callers — no network here.
4//!
5//! Field extraction uses `scraper` CSS selectors + typed path segments — never
6//! regex over the document for content fields. Process-static regexes below are
7//! **security scrub only** (`on*` handlers / `javascript:` URLs).
8//!
9//! Fixed CSS selectors are compiled once via [`LazyLock`] (SCRAPE-R-004) so the
10//! one-shot CPU path does not re-parse the same selector strings on every page.
11
12use std::sync::LazyLock;
13
14use regex::Regex;
15use scraper::{Html, Selector};
16use url::Url;
17
18use crate::config::{HOST_DOC_RUST_LANG_ORG, is_stdlib_crate};
19use crate::domain::compile_bounded_regex;
20use crate::error::{AppError, AppResult, ErrorKind};
21
22/// Process-static CSS selectors (valid by construction — panic on init is a bug).
23static SEL_A_HREF: LazyLock<Selector> = LazyLock::new(|| {
24    Selector::parse("a[href]").expect("hardcoded scraper selector 'a[href]' is valid by construction")
25});
26static SEL_TITLE: LazyLock<Selector> = LazyLock::new(|| {
27    Selector::parse("title").expect("hardcoded scraper selector 'title' is valid by construction")
28});
29static SEL_DROP_SCRIPT_STYLE: LazyLock<Selector> = LazyLock::new(|| {
30    Selector::parse("script, style, noscript")
31        .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction")
32});
33static SEL_RUSTDOC_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
34    Selector::parse(".rustdoc .docblock")
35        .expect("hardcoded scraper selector '.rustdoc .docblock' is valid by construction")
36});
37static SEL_MAIN_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
38    Selector::parse("#main-content .docblock")
39        .expect("hardcoded scraper selector '#main-content .docblock' is valid by construction")
40});
41static SEL_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
42    Selector::parse(".docblock")
43        .expect("hardcoded scraper selector '.docblock' is valid by construction")
44});
45static SEL_RUSTDOC_MAIN_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
46    Selector::parse(".rustdoc-main .item-decl")
47        .expect("hardcoded scraper selector '.rustdoc-main .item-decl' is valid by construction")
48});
49static SEL_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
50    Selector::parse(".item-decl")
51        .expect("hardcoded scraper selector '.item-decl' is valid by construction")
52});
53static SEL_MAIN_CONTENT: LazyLock<Selector> = LazyLock::new(|| {
54    Selector::parse("#main-content")
55        .expect("hardcoded scraper selector '#main-content' is valid by construction")
56});
57/// Any element with an `id` attribute — used for method anchors without dynamic CSS (SCRAPE-S-002).
58static SEL_WITH_ID: LazyLock<Selector> = LazyLock::new(|| {
59    Selector::parse("[id]").expect("hardcoded scraper selector '[id]' is valid by construction")
60});
61
62#[cfg_attr(not(test), allow(dead_code))]
63pub(super) fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
64    extract_resolved_version_for_crate(final_url, requested, None)
65}
66
67/// Resolve version/channel from the final URL, aware of docs.rs vs stdlib layout.
68///
69/// - docs.rs: `/{pkg}/{version}/…` — second path segment is the version
70/// - doc.rust-lang.org: `/{channel}/{crate}/…` — first path segment is channel/version
71pub(super) fn extract_resolved_version_for_crate(
72    final_url: &Url,
73    requested: &str,
74    crate_name: Option<&str>,
75) -> Option<String> {
76    let host = final_url.host_str().unwrap_or("");
77    let segs: Vec<&str> = final_url
78        .path_segments()?
79        .filter(|s| !s.is_empty())
80        .collect();
81    if segs.is_empty() {
82        return None;
83    }
84
85    let is_stdlib_host = host.eq_ignore_ascii_case(HOST_DOC_RUST_LANG_ORG)
86        || crate_name.is_some_and(is_stdlib_crate)
87        || segs
88            .get(1)
89            .is_some_and(|s| matches!(*s, "std" | "core" | "alloc"));
90
91    if is_stdlib_host {
92        // /{channel}/{crate}/… — never emit crate name as version
93        let channel = segs.first()?;
94        if matches!(*channel, "std" | "core" | "alloc") {
95            return None;
96        }
97        return Some((*channel).to_string());
98    }
99
100    // docs.rs: /{pkg}/{version}/…
101    let ver = segs.get(1)?;
102    if *ver == "latest" {
103        return None;
104    }
105    if requested == "latest" || *ver != requested {
106        return Some((*ver).to_string());
107    }
108    Some((*ver).to_string())
109}
110
111/// Try to scrape a concrete SemVer from docs.rs HTML when the URL still says `/latest/`.
112///
113/// Only accepts versions that belong to `crate_name` (never a dependency hit).
114/// Uses `scraper` + path-segment scan — **no** regex field extraction (SCRAPE-Q-002).
115/// Production fetch paths use [`scrape_docs_rs_version_from_document`] after a single
116/// parse; this wrapper remains for unit tests and offline callers.
117#[cfg_attr(not(test), allow(dead_code))]
118pub(super) fn scrape_docs_rs_version_from_html(html: &str, crate_name: &str) -> Option<String> {
119    let document = Html::parse_document(html);
120    scrape_docs_rs_version_from_document(&document, crate_name)
121}
122
123/// Version scrape against an already-parsed document (SCRAPE-S-003: one DOM per body).
124pub(super) fn scrape_docs_rs_version_from_document(
125    document: &Html,
126    crate_name: &str,
127) -> Option<String> {
128    let crate_name = crate_name.trim();
129    if crate_name.is_empty() || is_stdlib_crate(crate_name) {
130        return None;
131    }
132    for a in document.select(&SEL_A_HREF) {
133        if let Some(href) = a.value().attr("href")
134            && let Some(ver) = version_from_docs_rs_href(href, crate_name)
135        {
136            return Some(ver);
137        }
138    }
139    // Title text fallback (rustdoc skins sometimes embed "crate version").
140    if let Some(title) = document.select(&SEL_TITLE).next() {
141        let text: String = title.text().collect();
142        if let Some(ver) = version_token_near_crate_in_text(&text, crate_name) {
143            return Some(ver);
144        }
145    }
146    None
147}
148
149/// Extract `/{crate}/{semver}/` from an href (absolute or path-absolute/relative).
150fn version_from_docs_rs_href(href: &str, crate_name: &str) -> Option<String> {
151    let href = href.trim();
152    if href.is_empty() {
153        return None;
154    }
155    // Prefer Url path_segments when the href is absolute; otherwise scan path pieces.
156    let path = if href.starts_with("http://") || href.starts_with("https://") {
157        Url::parse(href).ok()?.path().to_string()
158    } else {
159        // Strip query/fragment for path-only hrefs.
160        let path = href.split('#').next().unwrap_or(href);
161        path.split('?').next().unwrap_or(path).to_string()
162    };
163    let segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
164    for i in 0..segs.len().saturating_sub(1) {
165        if segs[i].eq_ignore_ascii_case(crate_name)
166            && let Some(ver) = segs.get(i + 1)
167            && looks_like_semver(ver)
168        {
169            return Some((*ver).to_string());
170        }
171    }
172    None
173}
174
175/// True when `s` looks like SemVer core (`X.Y.Z` optional pre-release), no `v` prefix.
176fn looks_like_semver(s: &str) -> bool {
177    if s.is_empty() || s.starts_with('v') || s.starts_with('V') || s.contains('+') {
178        return false;
179    }
180    let (core, pre) = match s.split_once('-') {
181        Some((c, p)) => (c, Some(p)),
182        None => (s, None),
183    };
184    let mut parts = core.split('.');
185    let (Some(a), Some(b), Some(c), None) = (parts.next(), parts.next(), parts.next(), parts.next())
186    else {
187        return false;
188    };
189    if !a.bytes().all(|b| b.is_ascii_digit())
190        || !b.bytes().all(|b| b.is_ascii_digit())
191        || !c.bytes().all(|b| b.is_ascii_digit())
192        || a.is_empty()
193        || b.is_empty()
194        || c.is_empty()
195    {
196        return false;
197    }
198    if let Some(pre) = pre {
199        if pre.is_empty() {
200            return false;
201        }
202        return pre
203            .bytes()
204            .all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'-');
205    }
206    true
207}
208
209/// Scan title-like free text for `crate_name` then a nearby SemVer token.
210fn version_token_near_crate_in_text(text: &str, crate_name: &str) -> Option<String> {
211    let lower = text.to_ascii_lowercase();
212    let crate_l = crate_name.to_ascii_lowercase();
213    let idx = lower.find(&crate_l)?;
214    let after = &text[idx + crate_name.len()..];
215    for token in after.split(|c: char| !(c.is_ascii_alphanumeric() || c == '.' || c == '-')) {
216        if looks_like_semver(token) {
217            return Some(token.to_string());
218        }
219    }
220    None
221}
222
223/// Compiled once: strip `on*` event handlers from HTML fragments (hot path for every HTML→MD).
224static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
225    compile_bounded_regex(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
226        .expect("hardcoded on* handler strip regex is valid by construction")
227});
228
229/// Compiled once: strip `javascript:` URLs from href/src attributes.
230static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
231    compile_bounded_regex(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
232        .expect("hardcoded javascript: URL strip regex is valid by construction")
233});
234
235/// Remove script/style nodes and on* handlers before Markdown conversion.
236///
237/// Extraction of doc content uses CSS selectors elsewhere in this module.
238/// Regexes here are **security scrub only** (`on*` / `javascript:`), compiled
239/// once via process-static [`LazyLock`] — not used to scrape fields from HTML.
240///
241/// Precondition: `html` is already body-capped (`HARD_MAX_BODY_BYTES`) and resident in RAM.
242///
243/// When there are no script/style/noscript nodes, skips the multi-replace loop
244/// and only runs the process-static hygiene regexes (SCRAPE-S-006).
245pub fn sanitize_html_fragment(html: &str) -> String {
246    let fragment = Html::parse_fragment(html);
247    // Collect serialized drop-targets once, longest first so nested matches do not
248    // leave residuals, then strip from a single working buffer (SCRAPE-Q-003).
249    let mut drop_html: Vec<String> = fragment
250        .select(&SEL_DROP_SCRIPT_STYLE)
251        .map(|el| el.html())
252        .collect();
253    drop_html.sort_by_key(|s| std::cmp::Reverse(s.len()));
254    // SCRAPE-S-006: no script/style → scrub hygiene only (no N× replace allocs).
255    let mut out = if drop_html.is_empty() {
256        html.to_string()
257    } else {
258        let mut buf = String::with_capacity(html.len());
259        buf.push_str(html);
260        for tag_html in drop_html {
261            if !tag_html.is_empty() {
262                buf = buf.replace(&tag_html, "");
263            }
264        }
265        buf
266    };
267    // Strip event handlers: onload=, onclick=, etc.
268    out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
269    // Drop javascript: URLs in href/src
270    out = RE_JS_URLS.replace_all(&out, "").into_owned();
271    out
272}
273
274pub(super) fn html_to_markdown(html: &str) -> AppResult<String> {
275    let cleaned = sanitize_html_fragment(html);
276    // Cause stays in `Error::source` — never embed `{e}` in Display (ADR 0002 / ERR-O-001).
277    let md = htmd::convert(&cleaned).map_err(|e| {
278        AppError::with_source(ErrorKind::Parse, "html to markdown conversion failed", e)
279    })?;
280    Ok(scrub_rustdoc_chrome(&md))
281}
282
283/// Strip rustdoc UI chrome that pollutes LLM context (`§`, "Copy item path", …).
284pub fn scrub_rustdoc_chrome(md: &str) -> String {
285    let stripped = md.replace('§', "").replace("Copy item path", "");
286    // Single-pass collapse of accidental double spaces (not full reflow).
287    let mut out = String::with_capacity(stripped.len());
288    let mut prev_space = false;
289    for c in stripped.chars() {
290        if c == ' ' {
291            if prev_space {
292                continue;
293            }
294            prev_space = true;
295            out.push(' ');
296        } else {
297            prev_space = false;
298            out.push(c);
299        }
300    }
301    out
302}
303
304/// First non-empty `inner_html` among precompiled selectors (SCRAPE-R-004).
305pub(super) fn first_inner_html(document: &Html, selectors: &[&Selector]) -> Option<String> {
306    for sel in selectors {
307        if let Some(n) = document.select(sel).next() {
308            let inner = n.inner_html();
309            if !inner.trim().is_empty() {
310                return Some(inner);
311            }
312        }
313    }
314    None
315}
316
317/// Extract readme markdown from an already-parsed document (SCRAPE-S-003).
318///
319/// # Errors
320///
321/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
322pub fn extract_readme_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
323    let content = first_inner_html(
324        document,
325        &[
326            &SEL_RUSTDOC_DOCBLOCK,
327            &SEL_MAIN_DOCBLOCK,
328            &SEL_DOCBLOCK,
329            &SEL_RUSTDOC_MAIN_ITEM_DECL,
330            &SEL_ITEM_DECL,
331        ],
332    );
333    match content {
334        Some(h) => Ok((html_to_markdown(&h)?, false)),
335        None => Ok((String::new(), true)),
336    }
337}
338
339/// Extract readme markdown from raw HTML (offline tests / pure path).
340///
341/// # Errors
342///
343/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
344pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
345    extract_readme_markdown_from_document(&Html::parse_document(html))
346}
347
348/// Extract get-item markdown from an already-parsed document (SCRAPE-S-003).
349///
350/// # Errors
351///
352/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
353pub fn extract_item_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
354    let content = first_inner_html(
355        document,
356        &[&SEL_MAIN_CONTENT, &SEL_ITEM_DECL, &SEL_DOCBLOCK],
357    )
358    .unwrap_or_default();
359    let empty = content.trim().is_empty();
360    if empty {
361        Ok((String::new(), true))
362    } else {
363        Ok((html_to_markdown(&content)?, false))
364    }
365}
366
367/// Extract get-item markdown from raw HTML (offline tests / pure path).
368///
369/// # Errors
370///
371/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
372pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
373    extract_item_markdown_from_document(&Html::parse_document(html))
374}
375
376/// Extract markdown for a single associated method by rustdoc `id="method.X"`.
377///
378/// Locates the method anchor and prefers the enclosing `details.method-toggle`
379/// (signature + docblock). **Fail-closed:** missing or empty anchors return
380/// [`ErrorKind::NotFound`] — never the full parent page as a method success
381/// (Camada Y / GAP-W-001).
382///
383/// # Errors
384///
385/// Returns [`ErrorKind::NotFound`] when the method anchor is absent or empty.
386/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
387pub fn extract_method_markdown_from_html(
388    html: &str,
389    method_name: &str,
390) -> AppResult<(String, bool)> {
391    let (md, empty, _) = extract_method_markdown_scoped(html, method_name)?;
392    Ok((md, empty))
393}
394
395/// Like [`extract_method_markdown_from_html`] but reports extraction scope.
396///
397/// On success `scope` is always `"method"`. Missing anchors are errors, not
398/// `item_page` success payloads.
399///
400/// # Errors
401///
402/// Returns [`ErrorKind::NotFound`] when the method anchor is absent or empty.
403/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
404pub fn extract_method_markdown_scoped(
405    html: &str,
406    method_name: &str,
407) -> AppResult<(String, bool, &'static str)> {
408    extract_method_markdown_scoped_from_document(&Html::parse_document(html), method_name)
409}
410
411/// Method extract against an already-parsed document (SCRAPE-S-002/S-003).
412///
413/// Matches `id="method.{name}"` via static `[id]` selector + attribute equality
414/// — never builds a dynamic CSS selector string.
415///
416/// # Errors
417///
418/// Returns [`ErrorKind::NotFound`] when the method anchor is absent or empty.
419/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
420pub fn extract_method_markdown_scoped_from_document(
421    document: &Html,
422    method_name: &str,
423) -> AppResult<(String, bool, &'static str)> {
424    let want_id = format!("method.{method_name}");
425    for el in document.select(&SEL_WITH_ID) {
426        if el.value().attr("id") == Some(want_id.as_str()) {
427            let frag_html = method_container_html(el);
428            let md = html_to_markdown(&frag_html)?;
429            let empty = md.trim().is_empty();
430            if empty {
431                return Err(AppError::new(
432                    ErrorKind::NotFound,
433                    format!("method anchor empty: method.{method_name}"),
434                ));
435            }
436            return Ok((md, empty, "method"));
437        }
438    }
439    // Fail-closed: parent page HTTP 200 is not a method hit (GAP-W-001 / X-001).
440    Err(AppError::new(
441        ErrorKind::NotFound,
442        format!("method anchor not found: method.{method_name}"),
443    ))
444}
445
446/// List rustdoc associated-method anchor names (`id="method.X"` → `X`).
447///
448/// Used by `--suggest` on method miss (one DOM pass, no network). Pre-sizes the
449/// output vec from a first count-free push path with a modest capacity hint.
450pub fn list_method_anchor_names(document: &Html) -> Vec<String> {
451    let mut out = Vec::with_capacity(32);
452    for el in document.select(&SEL_WITH_ID) {
453        if let Some(id) = el.value().attr("id") {
454            if let Some(name) = id.strip_prefix("method.") {
455                if !name.is_empty() {
456                    out.push(name.to_string());
457                }
458            }
459        }
460    }
461    out
462}
463
464/// Parse HTML and list method anchor names (CPU-bound helper for ops recovery).
465pub fn list_method_anchor_names_from_html(html: &str) -> Vec<String> {
466    list_method_anchor_names(&Html::parse_document(html))
467}
468
469/// Prefer the enclosing rustdoc `details.method-toggle` (signature + docs).
470pub(super) fn method_container_html(anchor: scraper::ElementRef<'_>) -> String {
471    let mut node = anchor.parent();
472    while let Some(n) = node {
473        if let Some(el) = scraper::ElementRef::wrap(n) {
474            if el.value().name() == "details" {
475                return el.html();
476            }
477            node = el.parent();
478        } else {
479            break;
480        }
481    }
482    // Fall back to the section/element that carries the method id.
483    anchor.html()
484}