Skip to main content

docsrs_cli/docs_rs/html/
extract.rs

1//! Docblock, item, and associated-item markdown extraction from rustdoc HTML.
2
3use std::sync::LazyLock;
4
5use scraper::{Html, Selector};
6
7use crate::docs_rs::assoc::{AssocAnchorKind, METHOD_ANCHOR_PREFIXES, strip_assoc_anchor_prefix};
8use crate::error::{AppError, AppResult, ErrorDetail};
9
10use super::sanitize::html_to_markdown;
11
12/// Process-static CSS selectors (valid by construction — panic on init is a bug).
13static SEL_RUSTDOC_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
14    Selector::parse(".rustdoc .docblock")
15        .expect("hardcoded scraper selector '.rustdoc .docblock' is valid by construction")
16});
17static SEL_MAIN_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
18    Selector::parse("#main-content .docblock")
19        .expect("hardcoded scraper selector '#main-content .docblock' is valid by construction")
20});
21static SEL_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
22    Selector::parse(".docblock")
23        .expect("hardcoded scraper selector '.docblock' is valid by construction")
24});
25static SEL_RUSTDOC_MAIN_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
26    Selector::parse(".rustdoc-main .item-decl")
27        .expect("hardcoded scraper selector '.rustdoc-main .item-decl' is valid by construction")
28});
29static SEL_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
30    Selector::parse(".item-decl")
31        .expect("hardcoded scraper selector '.item-decl' is valid by construction")
32});
33static SEL_MAIN_CONTENT: LazyLock<Selector> = LazyLock::new(|| {
34    Selector::parse("#main-content")
35        .expect("hardcoded scraper selector '#main-content' is valid by construction")
36});
37/// Any element with an `id` attribute — used for method anchors without dynamic CSS (SCRAPE-S-002).
38static SEL_WITH_ID: LazyLock<Selector> = LazyLock::new(|| {
39    Selector::parse("[id]").expect("hardcoded scraper selector '[id]' is valid by construction")
40});
41
42/// First non-empty `inner_html` among precompiled selectors (SCRAPE-R-004).
43fn first_inner_html(document: &Html, selectors: &[&Selector]) -> Option<String> {
44    for sel in selectors {
45        if let Some(n) = document.select(sel).next() {
46            let inner = n.inner_html();
47            if !inner.trim().is_empty() {
48                return Some(inner);
49            }
50        }
51    }
52    None
53}
54
55/// Extract readme markdown from an already-parsed document (SCRAPE-S-003).
56///
57/// # Errors
58///
59/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
60pub fn extract_readme_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
61    let content = first_inner_html(
62        document,
63        &[
64            &SEL_RUSTDOC_DOCBLOCK,
65            &SEL_MAIN_DOCBLOCK,
66            &SEL_DOCBLOCK,
67            &SEL_RUSTDOC_MAIN_ITEM_DECL,
68            &SEL_ITEM_DECL,
69        ],
70    );
71    match content {
72        Some(h) => Ok((html_to_markdown(&h)?, false)),
73        None => Ok((String::new(), true)),
74    }
75}
76
77/// Extract readme markdown from raw HTML (offline tests / pure path).
78///
79/// # Errors
80///
81/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
82pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
83    extract_readme_markdown_from_document(&Html::parse_document(html))
84}
85
86/// Extract get-item markdown from an already-parsed document (SCRAPE-S-003).
87///
88/// # Errors
89///
90/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
91pub fn extract_item_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
92    let content = first_inner_html(
93        document,
94        &[&SEL_MAIN_CONTENT, &SEL_ITEM_DECL, &SEL_DOCBLOCK],
95    )
96    .unwrap_or_default();
97    let empty = content.trim().is_empty();
98    if empty {
99        Ok((String::new(), true))
100    } else {
101        Ok((html_to_markdown(&content)?, false))
102    }
103}
104
105/// Extract get-item markdown from raw HTML (offline tests / pure path).
106///
107/// # Errors
108///
109/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
110pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
111    extract_item_markdown_from_document(&Html::parse_document(html))
112}
113
114/// Extract markdown for a single associated method by rustdoc `id="method.X"`.
115///
116/// Locates the method anchor and prefers the enclosing `details.method-toggle`
117/// (signature + docblock). **Fail-closed:** missing or empty anchors return
118/// [`crate::error::ErrorKind::NotFound`] — never the full parent page as a method success
119/// (Camada Y / GAP-W-001).
120///
121/// # Errors
122///
123/// Returns [`crate::error::ErrorKind::NotFound`] when the method anchor is absent or empty.
124/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
125pub fn extract_method_markdown_from_html(
126    html: &str,
127    method_name: &str,
128) -> AppResult<(String, bool)> {
129    let found = extract_method_markdown_scoped(html, method_name)?;
130    Ok((found.markdown, found.empty))
131}
132
133/// A located associated-item anchor plus its rendered markdown.
134///
135/// `anchor_id` is the id that actually matched, so callers can echo a fragment
136/// that exists on the page instead of the one they guessed.
137#[derive(Debug, Clone)]
138pub struct MethodExtract {
139    /// Markdown for the member signature plus its docblock.
140    pub markdown: String,
141    /// Whether the rendered markdown is blank (always `false` on success).
142    pub empty: bool,
143    /// Extraction scope reported on the wire; always `"method"`.
144    ///
145    /// The value means "the markdown came from the member anchor", never "the
146    /// member is a function". Associated types and constants report it too:
147    /// agents have asserted `extraction == "method"` to reject parent-page
148    /// fallback success since 1.2.0, and that fail-closed guarantee holds for
149    /// every anchor family.
150    pub scope: &'static str,
151    /// Anchor id that matched, e.g. `tymethod.next` or `associatedtype.Item`.
152    pub anchor_id: String,
153}
154
155/// Like [`extract_method_markdown_from_html`] but reports scope and anchor id.
156///
157/// # Errors
158///
159/// Returns [`crate::error::ErrorKind::NotFound`] when no anchor variant is present, or when
160/// the matched anchor renders empty.
161/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
162pub fn extract_method_markdown_scoped(html: &str, method_name: &str) -> AppResult<MethodExtract> {
163    extract_assoc_markdown_scoped_from_document(
164        &Html::parse_document(html),
165        METHOD_ANCHOR_PREFIXES,
166        method_name,
167    )
168}
169
170/// Associated-item extract against an already-parsed document (SCRAPE-S-002/S-003).
171///
172/// Matches every id built from `prefixes` via the static `[id]` selector plus
173/// attribute equality — never builds a dynamic CSS selector string.
174///
175/// Walks the DOM **once**: a second full pass per prefix would double the CPU
176/// cost of the hot member path, and a trait page carries hundreds of ids.
177/// Document order does not decide the winner — prefix order does, so `method.`
178/// still beats `tymethod.` when a page somehow carries both for one name. The
179/// scan short-circuits on a rank-0 hit.
180///
181/// Suffixed duplicates rustdoc emits for `impl` blocks (`associatedtype.Item-3`)
182/// never match, because comparison is full-id equality rather than prefix test.
183///
184/// # Errors
185///
186/// Returns [`crate::error::ErrorKind::NotFound`] when no anchor variant is present, or when
187/// the matched anchor renders empty.
188/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
189pub fn extract_assoc_markdown_scoped_from_document(
190    document: &Html,
191    prefixes: &[&str],
192    member_name: &str,
193) -> AppResult<MethodExtract> {
194    let candidates: Vec<String> = prefixes
195        .iter()
196        .map(|prefix| format!("{prefix}{member_name}"))
197        .collect();
198    let mut best: Option<(usize, scraper::ElementRef<'_>)> = None;
199    for el in document.select(&SEL_WITH_ID) {
200        let Some(id) = el.value().attr("id") else {
201            continue;
202        };
203        let Some(rank) = candidates.iter().position(|c| c == id) else {
204            continue;
205        };
206        if best.as_ref().is_none_or(|(seen, _)| rank < *seen) {
207            best = Some((rank, el));
208            if rank == 0 {
209                break;
210            }
211        }
212    }
213    let Some((rank, el)) = best else {
214        // Fail-closed: parent page HTTP 200 is not a member hit (GAP-W-001 / X-001).
215        // Naming every id tried keeps the miss diagnosable without a second fetch.
216        return Err(AppError::of(ErrorDetail::AssocAnchorMissing {
217            anchors: candidates.join(", "),
218        }));
219    };
220    let anchor_id = candidates[rank].clone();
221    let frag_html = method_container_html(el);
222    let markdown = html_to_markdown(&frag_html)?;
223    if markdown.trim().is_empty() {
224        return Err(AppError::of(ErrorDetail::AssocAnchorEmpty {
225            anchor_id: anchor_id.to_string(),
226        }));
227    }
228    Ok(MethodExtract {
229        markdown,
230        empty: false,
231        scope: "method",
232        anchor_id,
233    })
234}
235
236/// Strip any [`METHOD_ANCHOR_PREFIXES`] entry from a rustdoc anchor id.
237///
238/// Returns the bare method name, or `None` when the id names something else
239/// (`associatedtype.Item`, `variant.Some`, a section heading, …).
240pub fn strip_method_anchor_prefix(id: &str) -> Option<&str> {
241    strip_assoc_anchor_prefix(id, METHOD_ANCHOR_PREFIXES)
242}
243
244/// List rustdoc associated-method anchor names (`method.X` / `tymethod.X` → `X`).
245///
246/// Used by `--suggest` on method miss (one DOM pass, no network). Required trait
247/// methods are included: omitting them made `--suggest` blind on trait pages.
248pub fn list_method_anchor_names(document: &Html) -> Vec<String> {
249    list_assoc_anchor_names(document, METHOD_ANCHOR_PREFIXES)
250}
251
252/// List rustdoc anchor names of one family (`associatedtype.X` → `X`).
253///
254/// Backs `--suggest` for every member category. Suggestions come from the
255/// **parent page already in hand**, never from `all.html` — which is why they
256/// work at all: the crate index lists the trait but none of its members.
257///
258/// Rustdoc appends a `-<n>` disambiguator when the same member name appears in
259/// several impl blocks on one page (`associatedtype.Item-1`, `Item-10`, …).
260/// A Rust identifier can never contain `-`, so any hyphen here is that suffix
261/// and nothing else — the name is truncated at it and duplicates collapse.
262/// Without this, `Iterator::item` answered with `Item`, `Item-1`, `Item-10`,
263/// `Item-100` and `Item-101`: four of five suggestions were noise, and the
264/// edit-distance ranking favours them precisely because they are near-identical.
265pub fn list_assoc_anchor_names(document: &Html, prefixes: &[&str]) -> Vec<String> {
266    let mut out = Vec::with_capacity(32);
267    let mut seen = std::collections::HashSet::with_capacity(32);
268    for el in document.select(&SEL_WITH_ID) {
269        if let Some(name) = el
270            .value()
271            .attr("id")
272            .and_then(|id| strip_assoc_anchor_prefix(id, prefixes))
273        {
274            let canonical = name.split('-').next().unwrap_or(name);
275            if !canonical.is_empty() && seen.insert(canonical.to_string()) {
276                out.push(canonical.to_string());
277            }
278        }
279    }
280    out
281}
282
283/// Parse HTML and list method anchor names (CPU-bound helper for ops recovery).
284pub fn list_method_anchor_names_from_html(html: &str) -> Vec<String> {
285    list_method_anchor_names(&Html::parse_document(html))
286}
287
288/// Parse HTML and list one family's anchor names (CPU-bound helper for ops recovery).
289pub fn list_assoc_anchor_names_from_html(html: &str, family: AssocAnchorKind) -> Vec<String> {
290    list_assoc_anchor_names(&Html::parse_document(html), family.anchor_prefixes())
291}
292
293/// Prefer the enclosing rustdoc `details.method-toggle` (signature + docs).
294fn method_container_html(anchor: scraper::ElementRef<'_>) -> String {
295    let mut node = anchor.parent();
296    while let Some(n) = node {
297        if let Some(el) = scraper::ElementRef::wrap(n) {
298            if el.value().name() == "details" {
299                return el.html();
300            }
301            node = el.parent();
302        } else {
303            break;
304        }
305    }
306    // Fall back to the section/element that carries the method id.
307    anchor.html()
308}