1use std::sync::LazyLock;
13
14use regex::Regex;
15use scraper::{Html, Selector};
16use url::Url;
17
18use crate::config::{HOST_DOC_RUST_LANG_ORG, is_stdlib_crate};
19use crate::domain::compile_bounded_regex;
20use crate::error::{AppError, AppResult, ErrorKind};
21
22static SEL_A_HREF: LazyLock<Selector> = LazyLock::new(|| {
24 Selector::parse("a[href]").expect("hardcoded scraper selector 'a[href]' is valid by construction")
25});
26static SEL_TITLE: LazyLock<Selector> = LazyLock::new(|| {
27 Selector::parse("title").expect("hardcoded scraper selector 'title' is valid by construction")
28});
29static SEL_DROP_SCRIPT_STYLE: LazyLock<Selector> = LazyLock::new(|| {
30 Selector::parse("script, style, noscript")
31 .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction")
32});
33static SEL_RUSTDOC_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
34 Selector::parse(".rustdoc .docblock")
35 .expect("hardcoded scraper selector '.rustdoc .docblock' is valid by construction")
36});
37static SEL_MAIN_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
38 Selector::parse("#main-content .docblock")
39 .expect("hardcoded scraper selector '#main-content .docblock' is valid by construction")
40});
41static SEL_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
42 Selector::parse(".docblock")
43 .expect("hardcoded scraper selector '.docblock' is valid by construction")
44});
45static SEL_RUSTDOC_MAIN_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
46 Selector::parse(".rustdoc-main .item-decl")
47 .expect("hardcoded scraper selector '.rustdoc-main .item-decl' is valid by construction")
48});
49static SEL_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
50 Selector::parse(".item-decl")
51 .expect("hardcoded scraper selector '.item-decl' is valid by construction")
52});
53static SEL_MAIN_CONTENT: LazyLock<Selector> = LazyLock::new(|| {
54 Selector::parse("#main-content")
55 .expect("hardcoded scraper selector '#main-content' is valid by construction")
56});
57static SEL_WITH_ID: LazyLock<Selector> = LazyLock::new(|| {
59 Selector::parse("[id]").expect("hardcoded scraper selector '[id]' is valid by construction")
60});
61
62#[cfg_attr(not(test), allow(dead_code))]
63pub(super) fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
64 extract_resolved_version_for_crate(final_url, requested, None)
65}
66
67pub(super) fn extract_resolved_version_for_crate(
72 final_url: &Url,
73 requested: &str,
74 crate_name: Option<&str>,
75) -> Option<String> {
76 let host = final_url.host_str().unwrap_or("");
77 let segs: Vec<&str> = final_url
78 .path_segments()?
79 .filter(|s| !s.is_empty())
80 .collect();
81 if segs.is_empty() {
82 return None;
83 }
84
85 let is_stdlib_host = host.eq_ignore_ascii_case(HOST_DOC_RUST_LANG_ORG)
86 || crate_name.is_some_and(is_stdlib_crate)
87 || segs
88 .get(1)
89 .is_some_and(|s| matches!(*s, "std" | "core" | "alloc"));
90
91 if is_stdlib_host {
92 let channel = segs.first()?;
94 if matches!(*channel, "std" | "core" | "alloc") {
95 return None;
96 }
97 return Some((*channel).to_string());
98 }
99
100 let ver = segs.get(1)?;
102 if *ver == "latest" {
103 return None;
104 }
105 if requested == "latest" || *ver != requested {
106 return Some((*ver).to_string());
107 }
108 Some((*ver).to_string())
109}
110
111#[cfg_attr(not(test), allow(dead_code))]
118pub(super) fn scrape_docs_rs_version_from_html(html: &str, crate_name: &str) -> Option<String> {
119 let document = Html::parse_document(html);
120 scrape_docs_rs_version_from_document(&document, crate_name)
121}
122
123pub(super) fn scrape_docs_rs_version_from_document(
125 document: &Html,
126 crate_name: &str,
127) -> Option<String> {
128 let crate_name = crate_name.trim();
129 if crate_name.is_empty() || is_stdlib_crate(crate_name) {
130 return None;
131 }
132 for a in document.select(&SEL_A_HREF) {
133 if let Some(href) = a.value().attr("href")
134 && let Some(ver) = version_from_docs_rs_href(href, crate_name)
135 {
136 return Some(ver);
137 }
138 }
139 if let Some(title) = document.select(&SEL_TITLE).next() {
141 let text: String = title.text().collect();
142 if let Some(ver) = version_token_near_crate_in_text(&text, crate_name) {
143 return Some(ver);
144 }
145 }
146 None
147}
148
149fn version_from_docs_rs_href(href: &str, crate_name: &str) -> Option<String> {
151 let href = href.trim();
152 if href.is_empty() {
153 return None;
154 }
155 let path = if href.starts_with("http://") || href.starts_with("https://") {
157 Url::parse(href).ok()?.path().to_string()
158 } else {
159 let path = href.split('#').next().unwrap_or(href);
161 path.split('?').next().unwrap_or(path).to_string()
162 };
163 let segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
164 for i in 0..segs.len().saturating_sub(1) {
165 if segs[i].eq_ignore_ascii_case(crate_name)
166 && let Some(ver) = segs.get(i + 1)
167 && looks_like_semver(ver)
168 {
169 return Some((*ver).to_string());
170 }
171 }
172 None
173}
174
175fn looks_like_semver(s: &str) -> bool {
177 if s.is_empty() || s.starts_with('v') || s.starts_with('V') || s.contains('+') {
178 return false;
179 }
180 let (core, pre) = match s.split_once('-') {
181 Some((c, p)) => (c, Some(p)),
182 None => (s, None),
183 };
184 let mut parts = core.split('.');
185 let (Some(a), Some(b), Some(c), None) = (parts.next(), parts.next(), parts.next(), parts.next())
186 else {
187 return false;
188 };
189 if !a.bytes().all(|b| b.is_ascii_digit())
190 || !b.bytes().all(|b| b.is_ascii_digit())
191 || !c.bytes().all(|b| b.is_ascii_digit())
192 || a.is_empty()
193 || b.is_empty()
194 || c.is_empty()
195 {
196 return false;
197 }
198 if let Some(pre) = pre {
199 if pre.is_empty() {
200 return false;
201 }
202 return pre
203 .bytes()
204 .all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'-');
205 }
206 true
207}
208
209fn version_token_near_crate_in_text(text: &str, crate_name: &str) -> Option<String> {
211 let lower = text.to_ascii_lowercase();
212 let crate_l = crate_name.to_ascii_lowercase();
213 let idx = lower.find(&crate_l)?;
214 let after = &text[idx + crate_name.len()..];
215 for token in after.split(|c: char| !(c.is_ascii_alphanumeric() || c == '.' || c == '-')) {
216 if looks_like_semver(token) {
217 return Some(token.to_string());
218 }
219 }
220 None
221}
222
223static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
225 compile_bounded_regex(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
226 .expect("hardcoded on* handler strip regex is valid by construction")
227});
228
229static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
231 compile_bounded_regex(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
232 .expect("hardcoded javascript: URL strip regex is valid by construction")
233});
234
235pub fn sanitize_html_fragment(html: &str) -> String {
246 let fragment = Html::parse_fragment(html);
247 let mut drop_html: Vec<String> = fragment
250 .select(&SEL_DROP_SCRIPT_STYLE)
251 .map(|el| el.html())
252 .collect();
253 drop_html.sort_by_key(|s| std::cmp::Reverse(s.len()));
254 let mut out = if drop_html.is_empty() {
256 html.to_string()
257 } else {
258 let mut buf = String::with_capacity(html.len());
259 buf.push_str(html);
260 for tag_html in drop_html {
261 if !tag_html.is_empty() {
262 buf = buf.replace(&tag_html, "");
263 }
264 }
265 buf
266 };
267 out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
269 out = RE_JS_URLS.replace_all(&out, "").into_owned();
271 out
272}
273
274pub(super) fn html_to_markdown(html: &str) -> AppResult<String> {
275 let cleaned = sanitize_html_fragment(html);
276 let md = htmd::convert(&cleaned).map_err(|e| {
278 AppError::with_source(ErrorKind::Parse, "html to markdown conversion failed", e)
279 })?;
280 Ok(scrub_rustdoc_chrome(&md))
281}
282
283pub fn scrub_rustdoc_chrome(md: &str) -> String {
285 let stripped = md.replace('§', "").replace("Copy item path", "");
286 let mut out = String::with_capacity(stripped.len());
288 let mut prev_space = false;
289 for c in stripped.chars() {
290 if c == ' ' {
291 if prev_space {
292 continue;
293 }
294 prev_space = true;
295 out.push(' ');
296 } else {
297 prev_space = false;
298 out.push(c);
299 }
300 }
301 out
302}
303
304pub(super) fn first_inner_html(document: &Html, selectors: &[&Selector]) -> Option<String> {
306 for sel in selectors {
307 if let Some(n) = document.select(sel).next() {
308 let inner = n.inner_html();
309 if !inner.trim().is_empty() {
310 return Some(inner);
311 }
312 }
313 }
314 None
315}
316
317pub fn extract_readme_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
323 let content = first_inner_html(
324 document,
325 &[
326 &SEL_RUSTDOC_DOCBLOCK,
327 &SEL_MAIN_DOCBLOCK,
328 &SEL_DOCBLOCK,
329 &SEL_RUSTDOC_MAIN_ITEM_DECL,
330 &SEL_ITEM_DECL,
331 ],
332 );
333 match content {
334 Some(h) => Ok((html_to_markdown(&h)?, false)),
335 None => Ok((String::new(), true)),
336 }
337}
338
339pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
345 extract_readme_markdown_from_document(&Html::parse_document(html))
346}
347
348pub fn extract_item_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
354 let content = first_inner_html(
355 document,
356 &[&SEL_MAIN_CONTENT, &SEL_ITEM_DECL, &SEL_DOCBLOCK],
357 )
358 .unwrap_or_default();
359 let empty = content.trim().is_empty();
360 if empty {
361 Ok((String::new(), true))
362 } else {
363 Ok((html_to_markdown(&content)?, false))
364 }
365}
366
367pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
373 extract_item_markdown_from_document(&Html::parse_document(html))
374}
375
376pub fn extract_method_markdown_from_html(
388 html: &str,
389 method_name: &str,
390) -> AppResult<(String, bool)> {
391 let (md, empty, _) = extract_method_markdown_scoped(html, method_name)?;
392 Ok((md, empty))
393}
394
395pub fn extract_method_markdown_scoped(
405 html: &str,
406 method_name: &str,
407) -> AppResult<(String, bool, &'static str)> {
408 extract_method_markdown_scoped_from_document(&Html::parse_document(html), method_name)
409}
410
411pub fn extract_method_markdown_scoped_from_document(
421 document: &Html,
422 method_name: &str,
423) -> AppResult<(String, bool, &'static str)> {
424 let want_id = format!("method.{method_name}");
425 for el in document.select(&SEL_WITH_ID) {
426 if el.value().attr("id") == Some(want_id.as_str()) {
427 let frag_html = method_container_html(el);
428 let md = html_to_markdown(&frag_html)?;
429 let empty = md.trim().is_empty();
430 if empty {
431 return Err(AppError::new(
432 ErrorKind::NotFound,
433 format!("method anchor empty: method.{method_name}"),
434 ));
435 }
436 return Ok((md, empty, "method"));
437 }
438 }
439 Err(AppError::new(
441 ErrorKind::NotFound,
442 format!("method anchor not found: method.{method_name}"),
443 ))
444}
445
446pub fn list_method_anchor_names(document: &Html) -> Vec<String> {
451 let mut out = Vec::with_capacity(32);
452 for el in document.select(&SEL_WITH_ID) {
453 if let Some(id) = el.value().attr("id") {
454 if let Some(name) = id.strip_prefix("method.") {
455 if !name.is_empty() {
456 out.push(name.to_string());
457 }
458 }
459 }
460 }
461 out
462}
463
464pub fn list_method_anchor_names_from_html(html: &str) -> Vec<String> {
466 list_method_anchor_names(&Html::parse_document(html))
467}
468
469pub(super) fn method_container_html(anchor: scraper::ElementRef<'_>) -> String {
471 let mut node = anchor.parent();
472 while let Some(n) = node {
473 if let Some(el) = scraper::ElementRef::wrap(n) {
474 if el.value().name() == "details" {
475 return el.html();
476 }
477 node = el.parent();
478 } else {
479 break;
480 }
481 }
482 anchor.html()
484}