use url::Url;
use crate::domain::{AllowedOrigin, CrateName, VersionArg};
use crate::error::{AppError, AppResult, ErrorDetail, ErrorKind};
use crate::http::{HttpClient, decode_utf8, require_content_type_html};
use crate::item_kind::ItemKind;
use scraper::Html;
use super::assoc::{ASSOC_ANCHOR_MISS_PREFIX, assoc_from_fragment, associated_item_path};
use super::html::{
extract_assoc_markdown_scoped_from_document, extract_item_markdown_from_document,
extract_readme_markdown_from_document, extract_resolved_version_for_crate,
scrape_docs_rs_version_from_document,
};
use super::types::{GetItemData, ReadmeData};
use super::urls::{
get_item_url_on_origin, get_item_url_on_origin_with_parent_kind, readme_url_on_origin,
strip_crate_prefix_segments,
};
pub async fn fetch_readme(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
) -> AppResult<ReadmeData> {
fetch_readme_on_origin(http, &AllowedOrigin::docs_rs_default(), crate_name, version).await
}
pub async fn fetch_readme_on_origin(
http: &HttpClient,
origin: &AllowedOrigin,
crate_name: &CrateName,
version: &VersionArg,
) -> AppResult<ReadmeData> {
let url = readme_url_on_origin(origin, crate_name, version)?;
fetch_readme_at(http, crate_name, version, &url).await
}
pub async fn fetch_readme_at(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
url: &Url,
) -> AppResult<ReadmeData> {
let resp = http.get_html(url).await?;
if resp.status.as_u16() == 404 {
return Err(AppError::from_http_status(
404,
&format!("crate or version not found: {crate_name}@{version}"),
));
}
if !resp.status.is_success() {
return Err(AppError::from_http_status(
resp.status.as_u16(),
"rustdoc readme",
));
}
require_content_type_html(resp.content_type.as_deref())?;
let body = resp.body;
let crate_for_scrape = crate_name.as_str().to_string();
let (markdown, empty, scraped_ver) = http
.budget()
.run_cpu_bound(move || {
let text = decode_utf8(&body)?;
let doc = Html::parse_document(&text);
let (md, empty) = extract_readme_markdown_from_document(&doc)?;
let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
Ok((md, empty, scraped))
})
.await?;
let resolved = extract_resolved_version_for_crate(
&resp.final_url,
version.as_str(),
Some(crate_name.as_str()),
)
.or(scraped_ver);
Ok(ReadmeData {
crate_name: crate_name.as_str().to_string(),
version: version.as_str().to_string(),
resolved_version: resolved,
markdown,
empty,
truncated: false,
source_url: resp.final_url.to_string(),
cache_hit: resp.cache_hit,
})
}
pub async fn fetch_item(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
kind: ItemKind,
segments: &[String],
) -> AppResult<GetItemData> {
fetch_item_on_origin(
http,
&AllowedOrigin::docs_rs_default(),
crate_name,
version,
kind,
segments,
)
.await
}
pub async fn fetch_item_on_origin(
http: &HttpClient,
origin: &AllowedOrigin,
crate_name: &CrateName,
version: &VersionArg,
kind: ItemKind,
segments: &[String],
) -> AppResult<GetItemData> {
fetch_item_on_origin_with_echo(
http,
origin,
crate_name,
version,
kind,
kind.as_str(),
segments,
)
.await
}
pub async fn fetch_item_on_origin_with_echo(
http: &HttpClient,
origin: &AllowedOrigin,
crate_name: &CrateName,
version: &VersionArg,
kind: ItemKind,
item_type_echo: &str,
segments: &[String],
) -> AppResult<GetItemData> {
let segs_for_detect = strip_crate_prefix_segments(crate_name, kind, segments);
if let Some(assoc) = associated_item_path(kind, &segs_for_detect) {
let mut first_err: Option<AppError> = None;
for parent_kind in assoc.parent_kind_probe() {
let url = get_item_url_on_origin_with_parent_kind(
origin,
crate_name,
version,
kind,
segments,
Some(*parent_kind),
)?;
match fetch_item_at_with_echo(
http,
crate_name,
version,
kind,
item_type_echo,
segments,
&url,
)
.await
{
Ok(data) => return Ok(data),
Err(e)
if e.kind() == ErrorKind::NotFound
&& e.message().starts_with(ASSOC_ANCHOR_MISS_PREFIX) =>
{
return Err(e);
}
Err(e) if e.kind() == ErrorKind::NotFound => {
if first_err.is_none() {
first_err = Some(e);
}
continue;
}
Err(e) => return Err(e),
}
}
return Err(first_err.unwrap_or_else(|| AppError::of(ErrorDetail::AssocParentPageNotFound)));
}
let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
fetch_item_at_with_echo(
http,
crate_name,
version,
kind,
item_type_echo,
segments,
&url,
)
.await
}
pub async fn fetch_item_at(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
kind: ItemKind,
segments: &[String],
url: &Url,
) -> AppResult<GetItemData> {
fetch_item_at_with_echo(
http,
crate_name,
version,
kind,
kind.as_str(),
segments,
url,
)
.await
}
pub async fn fetch_item_at_with_echo(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
kind: ItemKind,
item_type_echo: &str,
segments: &[String],
url: &Url,
) -> AppResult<GetItemData> {
let _ = kind; let mut fetch_url = url.clone();
let fragment = fetch_url.fragment().map(str::to_string);
fetch_url.set_fragment(None);
let resp = http.get_html(&fetch_url).await?;
if resp.status.as_u16() == 404 {
return Err(AppError::from_http_status(
404,
&format!("item not found at {url}"),
));
}
if !resp.status.is_success() {
return Err(AppError::from_http_status(
resp.status.as_u16(),
"rustdoc get-item",
));
}
require_content_type_html(resp.content_type.as_deref())?;
let assoc_anchor = fragment
.as_deref()
.and_then(assoc_from_fragment)
.map(|(family, name)| (family, name.to_string()));
let body = resp.body;
let assoc_anchor_cpu = assoc_anchor;
let crate_for_scrape = crate_name.as_str().to_string();
let ((markdown, empty), extraction, matched_anchor, scraped_ver) = http
.budget()
.run_cpu_bound(move || {
let text = decode_utf8(&body)?;
let doc = Html::parse_document(&text);
let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
if let Some((family, ref m)) = assoc_anchor_cpu {
let found =
extract_assoc_markdown_scoped_from_document(&doc, family.anchor_prefixes(), m)?;
Ok((
(found.markdown, found.empty),
Some(found.scope.to_string()),
Some(found.anchor_id),
scraped,
))
} else {
let (md, empty) = extract_item_markdown_from_document(&doc)?;
Ok(((md, empty), None, None, scraped))
}
})
.await?;
let item_path = segments.join("::");
let item_name = segments
.last()
.cloned()
.unwrap_or_else(|| item_path.clone());
let title = format!("{item_path} ({item_type_echo})");
let resolved = extract_resolved_version_for_crate(
&resp.final_url,
version.as_str(),
Some(crate_name.as_str()),
)
.or(scraped_ver);
let anchor_family = matched_anchor
.as_deref()
.and_then(|a| a.split_once('.'))
.map(|(family, _)| family.to_string());
let mut source_url = resp.final_url.clone();
if let Some(anchor) = matched_anchor {
source_url.set_fragment(Some(&anchor));
} else if let Some(f) = fragment {
source_url.set_fragment(Some(&f));
}
Ok(GetItemData {
crate_name: crate_name.as_str().to_string(),
item_type: item_type_echo.to_string(),
item_path,
item_name,
version: version.as_str().to_string(),
resolved_version: resolved,
markdown,
empty,
truncated: false,
source_url: source_url.to_string(),
title,
cache_hit: resp.cache_hit,
extraction,
anchor_family,
resolved_item_path: None,
})
}