docsrs_cli/docs_rs/html/extract.rs
1//! Docblock, item, and associated-item markdown extraction from rustdoc HTML.
2
3use std::sync::LazyLock;
4
5use scraper::{Html, Selector};
6
7use crate::docs_rs::assoc::{AssocAnchorKind, METHOD_ANCHOR_PREFIXES, strip_assoc_anchor_prefix};
8use crate::error::{AppError, AppResult, ErrorDetail};
9
10use super::sanitize::html_to_markdown;
11
12/// Process-static CSS selectors (valid by construction — panic on init is a bug).
13static SEL_RUSTDOC_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
14 Selector::parse(".rustdoc .docblock")
15 .expect("hardcoded scraper selector '.rustdoc .docblock' is valid by construction")
16});
17static SEL_MAIN_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
18 Selector::parse("#main-content .docblock")
19 .expect("hardcoded scraper selector '#main-content .docblock' is valid by construction")
20});
21static SEL_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
22 Selector::parse(".docblock")
23 .expect("hardcoded scraper selector '.docblock' is valid by construction")
24});
25static SEL_RUSTDOC_MAIN_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
26 Selector::parse(".rustdoc-main .item-decl")
27 .expect("hardcoded scraper selector '.rustdoc-main .item-decl' is valid by construction")
28});
29static SEL_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
30 Selector::parse(".item-decl")
31 .expect("hardcoded scraper selector '.item-decl' is valid by construction")
32});
33static SEL_MAIN_CONTENT: LazyLock<Selector> = LazyLock::new(|| {
34 Selector::parse("#main-content")
35 .expect("hardcoded scraper selector '#main-content' is valid by construction")
36});
37/// Any element with an `id` attribute — used for method anchors without dynamic CSS (SCRAPE-S-002).
38static SEL_WITH_ID: LazyLock<Selector> = LazyLock::new(|| {
39 Selector::parse("[id]").expect("hardcoded scraper selector '[id]' is valid by construction")
40});
41
42/// First non-empty `inner_html` among precompiled selectors (SCRAPE-R-004).
43fn first_inner_html(document: &Html, selectors: &[&Selector]) -> Option<String> {
44 for sel in selectors {
45 if let Some(n) = document.select(sel).next() {
46 let inner = n.inner_html();
47 if !inner.trim().is_empty() {
48 return Some(inner);
49 }
50 }
51 }
52 None
53}
54
55/// Extract readme markdown from an already-parsed document (SCRAPE-S-003).
56///
57/// # Errors
58///
59/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
60pub fn extract_readme_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
61 let content = first_inner_html(
62 document,
63 &[
64 &SEL_RUSTDOC_DOCBLOCK,
65 &SEL_MAIN_DOCBLOCK,
66 &SEL_DOCBLOCK,
67 &SEL_RUSTDOC_MAIN_ITEM_DECL,
68 &SEL_ITEM_DECL,
69 ],
70 );
71 match content {
72 Some(h) => Ok((html_to_markdown(&h)?, false)),
73 None => Ok((String::new(), true)),
74 }
75}
76
77/// Extract readme markdown from raw HTML (offline tests / pure path).
78///
79/// # Errors
80///
81/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
82pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
83 extract_readme_markdown_from_document(&Html::parse_document(html))
84}
85
86/// Extract get-item markdown from an already-parsed document (SCRAPE-S-003).
87///
88/// # Errors
89///
90/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
91pub fn extract_item_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
92 let content = first_inner_html(
93 document,
94 &[&SEL_MAIN_CONTENT, &SEL_ITEM_DECL, &SEL_DOCBLOCK],
95 )
96 .unwrap_or_default();
97 let empty = content.trim().is_empty();
98 if empty {
99 Ok((String::new(), true))
100 } else {
101 Ok((html_to_markdown(&content)?, false))
102 }
103}
104
105/// Extract get-item markdown from raw HTML (offline tests / pure path).
106///
107/// # Errors
108///
109/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
110pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
111 extract_item_markdown_from_document(&Html::parse_document(html))
112}
113
114/// Extract markdown for a single associated method by rustdoc `id="method.X"`.
115///
116/// Locates the method anchor and prefers the enclosing `details.method-toggle`
117/// (signature + docblock). **Fail-closed:** missing or empty anchors return
118/// [`crate::error::ErrorKind::NotFound`] — never the full parent page as a method success
119/// (Camada Y / GAP-W-001).
120///
121/// # Errors
122///
123/// Returns [`crate::error::ErrorKind::NotFound`] when the method anchor is absent or empty.
124/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
125pub fn extract_method_markdown_from_html(
126 html: &str,
127 method_name: &str,
128) -> AppResult<(String, bool)> {
129 let found = extract_method_markdown_scoped(html, method_name)?;
130 Ok((found.markdown, found.empty))
131}
132
133/// A located associated-item anchor plus its rendered markdown.
134///
135/// `anchor_id` is the id that actually matched, so callers can echo a fragment
136/// that exists on the page instead of the one they guessed.
137#[derive(Debug, Clone)]
138pub struct MethodExtract {
139 /// Markdown for the member signature plus its docblock.
140 pub markdown: String,
141 /// Whether the rendered markdown is blank (always `false` on success).
142 pub empty: bool,
143 /// Extraction scope reported on the wire; always `"method"`.
144 ///
145 /// The value means "the markdown came from the member anchor", never "the
146 /// member is a function". Associated types and constants report it too:
147 /// agents have asserted `extraction == "method"` to reject parent-page
148 /// fallback success since 1.2.0, and that fail-closed guarantee holds for
149 /// every anchor family.
150 pub scope: &'static str,
151 /// Anchor id that matched, e.g. `tymethod.next` or `associatedtype.Item`.
152 pub anchor_id: String,
153}
154
155/// Like [`extract_method_markdown_from_html`] but reports scope and anchor id.
156///
157/// # Errors
158///
159/// Returns [`crate::error::ErrorKind::NotFound`] when no anchor variant is present, or when
160/// the matched anchor renders empty.
161/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
162pub fn extract_method_markdown_scoped(html: &str, method_name: &str) -> AppResult<MethodExtract> {
163 extract_assoc_markdown_scoped_from_document(
164 &Html::parse_document(html),
165 METHOD_ANCHOR_PREFIXES,
166 method_name,
167 )
168}
169
170/// Associated-item extract against an already-parsed document (SCRAPE-S-002/S-003).
171///
172/// Matches every id built from `prefixes` via the static `[id]` selector plus
173/// attribute equality — never builds a dynamic CSS selector string.
174///
175/// Walks the DOM **once**: a second full pass per prefix would double the CPU
176/// cost of the hot member path, and a trait page carries hundreds of ids.
177/// Document order does not decide the winner — prefix order does, so `method.`
178/// still beats `tymethod.` when a page somehow carries both for one name. The
179/// scan short-circuits on a rank-0 hit.
180///
181/// Suffixed duplicates rustdoc emits for `impl` blocks (`associatedtype.Item-3`)
182/// never match, because comparison is full-id equality rather than prefix test.
183///
184/// # Errors
185///
186/// Returns [`crate::error::ErrorKind::NotFound`] when no anchor variant is present, or when
187/// the matched anchor renders empty.
188/// Returns [`crate::error::ErrorKind::Parse`] when HTML→Markdown conversion fails.
189pub fn extract_assoc_markdown_scoped_from_document(
190 document: &Html,
191 prefixes: &[&str],
192 member_name: &str,
193) -> AppResult<MethodExtract> {
194 let candidates: Vec<String> = prefixes
195 .iter()
196 .map(|prefix| format!("{prefix}{member_name}"))
197 .collect();
198 let mut best: Option<(usize, scraper::ElementRef<'_>)> = None;
199 for el in document.select(&SEL_WITH_ID) {
200 let Some(id) = el.value().attr("id") else {
201 continue;
202 };
203 let Some(rank) = candidates.iter().position(|c| c == id) else {
204 continue;
205 };
206 if best.as_ref().is_none_or(|(seen, _)| rank < *seen) {
207 best = Some((rank, el));
208 if rank == 0 {
209 break;
210 }
211 }
212 }
213 let Some((rank, el)) = best else {
214 // Fail-closed: parent page HTTP 200 is not a member hit (GAP-W-001 / X-001).
215 // Naming every id tried keeps the miss diagnosable without a second fetch.
216 return Err(AppError::of(ErrorDetail::AssocAnchorMissing {
217 anchors: candidates.join(", "),
218 }));
219 };
220 let anchor_id = candidates[rank].clone();
221 let frag_html = method_container_html(el);
222 let markdown = html_to_markdown(&frag_html)?;
223 if markdown.trim().is_empty() {
224 return Err(AppError::of(ErrorDetail::AssocAnchorEmpty {
225 anchor_id: anchor_id.to_string(),
226 }));
227 }
228 Ok(MethodExtract {
229 markdown,
230 empty: false,
231 scope: "method",
232 anchor_id,
233 })
234}
235
236/// Strip any [`METHOD_ANCHOR_PREFIXES`] entry from a rustdoc anchor id.
237///
238/// Returns the bare method name, or `None` when the id names something else
239/// (`associatedtype.Item`, `variant.Some`, a section heading, …).
240pub fn strip_method_anchor_prefix(id: &str) -> Option<&str> {
241 strip_assoc_anchor_prefix(id, METHOD_ANCHOR_PREFIXES)
242}
243
244/// List rustdoc associated-method anchor names (`method.X` / `tymethod.X` → `X`).
245///
246/// Used by `--suggest` on method miss (one DOM pass, no network). Required trait
247/// methods are included: omitting them made `--suggest` blind on trait pages.
248pub fn list_method_anchor_names(document: &Html) -> Vec<String> {
249 list_assoc_anchor_names(document, METHOD_ANCHOR_PREFIXES)
250}
251
252/// List rustdoc anchor names of one family (`associatedtype.X` → `X`).
253///
254/// Backs `--suggest` for every member category. Suggestions come from the
255/// **parent page already in hand**, never from `all.html` — which is why they
256/// work at all: the crate index lists the trait but none of its members.
257///
258/// Rustdoc appends a `-<n>` disambiguator when the same member name appears in
259/// several impl blocks on one page (`associatedtype.Item-1`, `Item-10`, …).
260/// A Rust identifier can never contain `-`, so any hyphen here is that suffix
261/// and nothing else — the name is truncated at it and duplicates collapse.
262/// Without this, `Iterator::item` answered with `Item`, `Item-1`, `Item-10`,
263/// `Item-100` and `Item-101`: four of five suggestions were noise, and the
264/// edit-distance ranking favours them precisely because they are near-identical.
265pub fn list_assoc_anchor_names(document: &Html, prefixes: &[&str]) -> Vec<String> {
266 let mut out = Vec::with_capacity(32);
267 let mut seen = std::collections::HashSet::with_capacity(32);
268 for el in document.select(&SEL_WITH_ID) {
269 if let Some(name) = el
270 .value()
271 .attr("id")
272 .and_then(|id| strip_assoc_anchor_prefix(id, prefixes))
273 {
274 let canonical = name.split('-').next().unwrap_or(name);
275 if !canonical.is_empty() && seen.insert(canonical.to_string()) {
276 out.push(canonical.to_string());
277 }
278 }
279 }
280 out
281}
282
283/// Parse HTML and list method anchor names (CPU-bound helper for ops recovery).
284pub fn list_method_anchor_names_from_html(html: &str) -> Vec<String> {
285 list_method_anchor_names(&Html::parse_document(html))
286}
287
288/// Parse HTML and list one family's anchor names (CPU-bound helper for ops recovery).
289pub fn list_assoc_anchor_names_from_html(html: &str, family: AssocAnchorKind) -> Vec<String> {
290 list_assoc_anchor_names(&Html::parse_document(html), family.anchor_prefixes())
291}
292
293/// Prefer the enclosing rustdoc `details.method-toggle` (signature + docs).
294fn method_container_html(anchor: scraper::ElementRef<'_>) -> String {
295 let mut node = anchor.parent();
296 while let Some(n) = node {
297 if let Some(el) = scraper::ElementRef::wrap(n) {
298 if el.value().name() == "details" {
299 return el.html();
300 }
301 node = el.parent();
302 } else {
303 break;
304 }
305 }
306 // Fall back to the section/element that carries the method id.
307 anchor.html()
308}