use std::sync::LazyLock;
use regex::Regex;
use scraper::{Html, Selector};
use url::Url;
use crate::config::{HOST_DOC_RUST_LANG_ORG, is_stdlib_crate};
use crate::domain::compile_bounded_regex;
use crate::error::{AppError, AppResult, ErrorKind};
static SEL_A_HREF: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("a[href]").expect("hardcoded scraper selector 'a[href]' is valid by construction")
});
static SEL_TITLE: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("title").expect("hardcoded scraper selector 'title' is valid by construction")
});
static SEL_DROP_SCRIPT_STYLE: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("script, style, noscript")
.expect("hardcoded scraper selector 'script, style, noscript' is valid by construction")
});
static SEL_RUSTDOC_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(".rustdoc .docblock")
.expect("hardcoded scraper selector '.rustdoc .docblock' is valid by construction")
});
static SEL_MAIN_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("#main-content .docblock")
.expect("hardcoded scraper selector '#main-content .docblock' is valid by construction")
});
static SEL_DOCBLOCK: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(".docblock")
.expect("hardcoded scraper selector '.docblock' is valid by construction")
});
static SEL_RUSTDOC_MAIN_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(".rustdoc-main .item-decl")
.expect("hardcoded scraper selector '.rustdoc-main .item-decl' is valid by construction")
});
static SEL_ITEM_DECL: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse(".item-decl")
.expect("hardcoded scraper selector '.item-decl' is valid by construction")
});
static SEL_MAIN_CONTENT: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("#main-content")
.expect("hardcoded scraper selector '#main-content' is valid by construction")
});
static SEL_WITH_ID: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("[id]").expect("hardcoded scraper selector '[id]' is valid by construction")
});
#[cfg_attr(not(test), allow(dead_code))]
pub(super) fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
extract_resolved_version_for_crate(final_url, requested, None)
}
pub(super) fn extract_resolved_version_for_crate(
final_url: &Url,
requested: &str,
crate_name: Option<&str>,
) -> Option<String> {
let host = final_url.host_str().unwrap_or("");
let segs: Vec<&str> = final_url
.path_segments()?
.filter(|s| !s.is_empty())
.collect();
if segs.is_empty() {
return None;
}
let is_stdlib_host = host.eq_ignore_ascii_case(HOST_DOC_RUST_LANG_ORG)
|| crate_name.is_some_and(is_stdlib_crate)
|| segs
.get(1)
.is_some_and(|s| matches!(*s, "std" | "core" | "alloc"));
if is_stdlib_host {
let channel = segs.first()?;
if matches!(*channel, "std" | "core" | "alloc") {
return None;
}
return Some((*channel).to_string());
}
let ver = segs.get(1)?;
if *ver == "latest" {
return None;
}
if requested == "latest" || *ver != requested {
return Some((*ver).to_string());
}
Some((*ver).to_string())
}
#[cfg_attr(not(test), allow(dead_code))]
pub(super) fn scrape_docs_rs_version_from_html(html: &str, crate_name: &str) -> Option<String> {
let document = Html::parse_document(html);
scrape_docs_rs_version_from_document(&document, crate_name)
}
pub(super) fn scrape_docs_rs_version_from_document(
document: &Html,
crate_name: &str,
) -> Option<String> {
let crate_name = crate_name.trim();
if crate_name.is_empty() || is_stdlib_crate(crate_name) {
return None;
}
for a in document.select(&SEL_A_HREF) {
if let Some(href) = a.value().attr("href")
&& let Some(ver) = version_from_docs_rs_href(href, crate_name)
{
return Some(ver);
}
}
if let Some(title) = document.select(&SEL_TITLE).next() {
let text: String = title.text().collect();
if let Some(ver) = version_token_near_crate_in_text(&text, crate_name) {
return Some(ver);
}
}
None
}
fn version_from_docs_rs_href(href: &str, crate_name: &str) -> Option<String> {
let href = href.trim();
if href.is_empty() {
return None;
}
let path = if href.starts_with("http://") || href.starts_with("https://") {
Url::parse(href).ok()?.path().to_string()
} else {
let path = href.split('#').next().unwrap_or(href);
path.split('?').next().unwrap_or(path).to_string()
};
let segs: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
for i in 0..segs.len().saturating_sub(1) {
if segs[i].eq_ignore_ascii_case(crate_name)
&& let Some(ver) = segs.get(i + 1)
&& looks_like_semver(ver)
{
return Some((*ver).to_string());
}
}
None
}
fn looks_like_semver(s: &str) -> bool {
if s.is_empty() || s.starts_with('v') || s.starts_with('V') || s.contains('+') {
return false;
}
let (core, pre) = match s.split_once('-') {
Some((c, p)) => (c, Some(p)),
None => (s, None),
};
let mut parts = core.split('.');
let (Some(a), Some(b), Some(c), None) = (parts.next(), parts.next(), parts.next(), parts.next())
else {
return false;
};
if !a.bytes().all(|b| b.is_ascii_digit())
|| !b.bytes().all(|b| b.is_ascii_digit())
|| !c.bytes().all(|b| b.is_ascii_digit())
|| a.is_empty()
|| b.is_empty()
|| c.is_empty()
{
return false;
}
if let Some(pre) = pre {
if pre.is_empty() {
return false;
}
return pre
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'-');
}
true
}
fn version_token_near_crate_in_text(text: &str, crate_name: &str) -> Option<String> {
let lower = text.to_ascii_lowercase();
let crate_l = crate_name.to_ascii_lowercase();
let idx = lower.find(&crate_l)?;
let after = &text[idx + crate_name.len()..];
for token in after.split(|c: char| !(c.is_ascii_alphanumeric() || c == '.' || c == '-')) {
if looks_like_semver(token) {
return Some(token.to_string());
}
}
None
}
static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
compile_bounded_regex(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
.expect("hardcoded on* handler strip regex is valid by construction")
});
static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
compile_bounded_regex(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
.expect("hardcoded javascript: URL strip regex is valid by construction")
});
pub fn sanitize_html_fragment(html: &str) -> String {
let fragment = Html::parse_fragment(html);
let mut drop_html: Vec<String> = fragment
.select(&SEL_DROP_SCRIPT_STYLE)
.map(|el| el.html())
.collect();
drop_html.sort_by_key(|s| std::cmp::Reverse(s.len()));
let mut out = if drop_html.is_empty() {
html.to_string()
} else {
let mut buf = String::with_capacity(html.len());
buf.push_str(html);
for tag_html in drop_html {
if !tag_html.is_empty() {
buf = buf.replace(&tag_html, "");
}
}
buf
};
out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
out = RE_JS_URLS.replace_all(&out, "").into_owned();
out
}
pub(super) fn html_to_markdown(html: &str) -> AppResult<String> {
let cleaned = sanitize_html_fragment(html);
let md = htmd::convert(&cleaned).map_err(|e| {
AppError::with_source(ErrorKind::Parse, "html to markdown conversion failed", e)
})?;
Ok(scrub_rustdoc_chrome(&md))
}
pub fn scrub_rustdoc_chrome(md: &str) -> String {
let stripped = md.replace('§', "").replace("Copy item path", "");
let mut out = String::with_capacity(stripped.len());
let mut prev_space = false;
for c in stripped.chars() {
if c == ' ' {
if prev_space {
continue;
}
prev_space = true;
out.push(' ');
} else {
prev_space = false;
out.push(c);
}
}
out
}
pub(super) fn first_inner_html(document: &Html, selectors: &[&Selector]) -> Option<String> {
for sel in selectors {
if let Some(n) = document.select(sel).next() {
let inner = n.inner_html();
if !inner.trim().is_empty() {
return Some(inner);
}
}
}
None
}
pub fn extract_readme_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
let content = first_inner_html(
document,
&[
&SEL_RUSTDOC_DOCBLOCK,
&SEL_MAIN_DOCBLOCK,
&SEL_DOCBLOCK,
&SEL_RUSTDOC_MAIN_ITEM_DECL,
&SEL_ITEM_DECL,
],
);
match content {
Some(h) => Ok((html_to_markdown(&h)?, false)),
None => Ok((String::new(), true)),
}
}
pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
extract_readme_markdown_from_document(&Html::parse_document(html))
}
pub fn extract_item_markdown_from_document(document: &Html) -> AppResult<(String, bool)> {
let content = first_inner_html(
document,
&[&SEL_MAIN_CONTENT, &SEL_ITEM_DECL, &SEL_DOCBLOCK],
)
.unwrap_or_default();
let empty = content.trim().is_empty();
if empty {
Ok((String::new(), true))
} else {
Ok((html_to_markdown(&content)?, false))
}
}
pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
extract_item_markdown_from_document(&Html::parse_document(html))
}
pub fn extract_method_markdown_from_html(
html: &str,
method_name: &str,
) -> AppResult<(String, bool)> {
let (md, empty, _) = extract_method_markdown_scoped(html, method_name)?;
Ok((md, empty))
}
pub fn extract_method_markdown_scoped(
html: &str,
method_name: &str,
) -> AppResult<(String, bool, &'static str)> {
extract_method_markdown_scoped_from_document(&Html::parse_document(html), method_name)
}
pub fn extract_method_markdown_scoped_from_document(
document: &Html,
method_name: &str,
) -> AppResult<(String, bool, &'static str)> {
let want_id = format!("method.{method_name}");
for el in document.select(&SEL_WITH_ID) {
if el.value().attr("id") == Some(want_id.as_str()) {
let frag_html = method_container_html(el);
let md = html_to_markdown(&frag_html)?;
let empty = md.trim().is_empty();
if empty {
return Err(AppError::new(
ErrorKind::NotFound,
format!("method anchor empty: method.{method_name}"),
));
}
return Ok((md, empty, "method"));
}
}
Err(AppError::new(
ErrorKind::NotFound,
format!("method anchor not found: method.{method_name}"),
))
}
pub fn list_method_anchor_names(document: &Html) -> Vec<String> {
let mut out = Vec::with_capacity(32);
for el in document.select(&SEL_WITH_ID) {
if let Some(id) = el.value().attr("id") {
if let Some(name) = id.strip_prefix("method.") {
if !name.is_empty() {
out.push(name.to_string());
}
}
}
}
out
}
pub fn list_method_anchor_names_from_html(html: &str) -> Vec<String> {
list_method_anchor_names(&Html::parse_document(html))
}
pub(super) fn method_container_html(anchor: scraper::ElementRef<'_>) -> String {
let mut node = anchor.parent();
while let Some(n) = node {
if let Some(el) = scraper::ElementRef::wrap(n) {
if el.value().name() == "details" {
return el.html();
}
node = el.parent();
} else {
break;
}
}
anchor.html()
}