1use url::Url;
4
5use crate::domain::{AllowedOrigin, CrateName, VersionArg};
6use crate::error::{AppError, AppResult, ErrorDetail, ErrorKind};
7use crate::http::{HttpClient, decode_utf8, require_content_type_html};
8use crate::item_kind::ItemKind;
9
10use scraper::Html;
11
12use super::assoc::{ASSOC_ANCHOR_MISS_PREFIX, assoc_from_fragment, associated_item_path};
13use super::html::{
14 extract_assoc_markdown_scoped_from_document, extract_item_markdown_from_document,
15 extract_readme_markdown_from_document, extract_resolved_version_for_crate,
16 scrape_docs_rs_version_from_document,
17};
18use super::types::{GetItemData, ReadmeData};
19use super::urls::{
20 get_item_url_on_origin, get_item_url_on_origin_with_parent_kind, readme_url_on_origin,
21 strip_crate_prefix_segments,
22};
23
24pub async fn fetch_readme(
31 http: &HttpClient,
32 crate_name: &CrateName,
33 version: &VersionArg,
34) -> AppResult<ReadmeData> {
35 fetch_readme_on_origin(http, &AllowedOrigin::docs_rs_default(), crate_name, version).await
36}
37
38pub async fn fetch_readme_on_origin(
45 http: &HttpClient,
46 origin: &AllowedOrigin,
47 crate_name: &CrateName,
48 version: &VersionArg,
49) -> AppResult<ReadmeData> {
50 let url = readme_url_on_origin(origin, crate_name, version)?;
51 fetch_readme_at(http, crate_name, version, &url).await
52}
53
54pub async fn fetch_readme_at(
62 http: &HttpClient,
63 crate_name: &CrateName,
64 version: &VersionArg,
65 url: &Url,
66) -> AppResult<ReadmeData> {
67 let resp = http.get_html(url).await?;
68 if resp.status.as_u16() == 404 {
69 return Err(AppError::from_http_status(
70 404,
71 &format!("crate or version not found: {crate_name}@{version}"),
72 ));
73 }
74 if !resp.status.is_success() {
75 return Err(AppError::from_http_status(
76 resp.status.as_u16(),
77 "rustdoc readme",
78 ));
79 }
80 require_content_type_html(resp.content_type.as_deref())?;
82
83 let body = resp.body;
85 let crate_for_scrape = crate_name.as_str().to_string();
86 let (markdown, empty, scraped_ver) = http
87 .budget()
88 .run_cpu_bound(move || {
89 let text = decode_utf8(&body)?;
90 let doc = Html::parse_document(&text);
91 let (md, empty) = extract_readme_markdown_from_document(&doc)?;
92 let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
93 Ok((md, empty, scraped))
94 })
95 .await?;
96 let resolved = extract_resolved_version_for_crate(
97 &resp.final_url,
98 version.as_str(),
99 Some(crate_name.as_str()),
100 )
101 .or(scraped_ver);
102 Ok(ReadmeData {
103 crate_name: crate_name.as_str().to_string(),
104 version: version.as_str().to_string(),
105 resolved_version: resolved,
106 markdown,
107 empty,
108 truncated: false,
109 source_url: resp.final_url.to_string(),
110 cache_hit: resp.cache_hit,
111 })
112}
113
114pub async fn fetch_item(
121 http: &HttpClient,
122 crate_name: &CrateName,
123 version: &VersionArg,
124 kind: ItemKind,
125 segments: &[String],
126) -> AppResult<GetItemData> {
127 fetch_item_on_origin(
128 http,
129 &AllowedOrigin::docs_rs_default(),
130 crate_name,
131 version,
132 kind,
133 segments,
134 )
135 .await
136}
137
138pub async fn fetch_item_on_origin(
148 http: &HttpClient,
149 origin: &AllowedOrigin,
150 crate_name: &CrateName,
151 version: &VersionArg,
152 kind: ItemKind,
153 segments: &[String],
154) -> AppResult<GetItemData> {
155 fetch_item_on_origin_with_echo(
156 http,
157 origin,
158 crate_name,
159 version,
160 kind,
161 kind.as_str(),
162 segments,
163 )
164 .await
165}
166
167pub async fn fetch_item_on_origin_with_echo(
172 http: &HttpClient,
173 origin: &AllowedOrigin,
174 crate_name: &CrateName,
175 version: &VersionArg,
176 kind: ItemKind,
177 item_type_echo: &str,
178 segments: &[String],
179) -> AppResult<GetItemData> {
180 let segs_for_detect = strip_crate_prefix_segments(crate_name, kind, segments);
185
186 if let Some(assoc) = associated_item_path(kind, &segs_for_detect) {
187 let mut first_err: Option<AppError> = None;
209 for parent_kind in assoc.parent_kind_probe() {
210 let url = get_item_url_on_origin_with_parent_kind(
211 origin,
212 crate_name,
213 version,
214 kind,
215 segments,
216 Some(*parent_kind),
217 )?;
218 match fetch_item_at_with_echo(
219 http,
220 crate_name,
221 version,
222 kind,
223 item_type_echo,
224 segments,
225 &url,
226 )
227 .await
228 {
229 Ok(data) => return Ok(data),
230 Err(e)
232 if e.kind() == ErrorKind::NotFound
233 && e.message().starts_with(ASSOC_ANCHOR_MISS_PREFIX) =>
234 {
235 return Err(e);
236 }
237 Err(e) if e.kind() == ErrorKind::NotFound => {
238 if first_err.is_none() {
239 first_err = Some(e);
240 }
241 continue;
242 }
243 Err(e) => return Err(e),
244 }
245 }
246 return Err(first_err.unwrap_or_else(|| AppError::of(ErrorDetail::AssocParentPageNotFound)));
247 }
248
249 let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
250 fetch_item_at_with_echo(
251 http,
252 crate_name,
253 version,
254 kind,
255 item_type_echo,
256 segments,
257 &url,
258 )
259 .await
260}
261
262pub async fn fetch_item_at(
270 http: &HttpClient,
271 crate_name: &CrateName,
272 version: &VersionArg,
273 kind: ItemKind,
274 segments: &[String],
275 url: &Url,
276) -> AppResult<GetItemData> {
277 fetch_item_at_with_echo(
278 http,
279 crate_name,
280 version,
281 kind,
282 kind.as_str(),
283 segments,
284 url,
285 )
286 .await
287}
288
289pub async fn fetch_item_at_with_echo(
294 http: &HttpClient,
295 crate_name: &CrateName,
296 version: &VersionArg,
297 kind: ItemKind,
298 item_type_echo: &str,
299 segments: &[String],
300 url: &Url,
301) -> AppResult<GetItemData> {
302 let _ = kind; let mut fetch_url = url.clone();
305 let fragment = fetch_url.fragment().map(str::to_string);
306 fetch_url.set_fragment(None);
307
308 let resp = http.get_html(&fetch_url).await?;
309 if resp.status.as_u16() == 404 {
310 return Err(AppError::from_http_status(
311 404,
312 &format!("item not found at {url}"),
313 ));
314 }
315 if !resp.status.is_success() {
316 return Err(AppError::from_http_status(
317 resp.status.as_u16(),
318 "rustdoc get-item",
319 ));
320 }
321 require_content_type_html(resp.content_type.as_deref())?;
322
323 let assoc_anchor = fragment
328 .as_deref()
329 .and_then(assoc_from_fragment)
330 .map(|(family, name)| (family, name.to_string()));
331 let body = resp.body;
333 let assoc_anchor_cpu = assoc_anchor;
334 let crate_for_scrape = crate_name.as_str().to_string();
335 let ((markdown, empty), extraction, matched_anchor, scraped_ver) = http
336 .budget()
337 .run_cpu_bound(move || {
338 let text = decode_utf8(&body)?;
339 let doc = Html::parse_document(&text);
340 let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
341 if let Some((family, ref m)) = assoc_anchor_cpu {
342 let found =
343 extract_assoc_markdown_scoped_from_document(&doc, family.anchor_prefixes(), m)?;
344 Ok((
345 (found.markdown, found.empty),
346 Some(found.scope.to_string()),
347 Some(found.anchor_id),
348 scraped,
349 ))
350 } else {
351 let (md, empty) = extract_item_markdown_from_document(&doc)?;
352 Ok(((md, empty), None, None, scraped))
353 }
354 })
355 .await?;
356 let item_path = segments.join("::");
357 let item_name = segments
358 .last()
359 .cloned()
360 .unwrap_or_else(|| item_path.clone());
361 let title = format!("{item_path} ({item_type_echo})");
362 let resolved = extract_resolved_version_for_crate(
363 &resp.final_url,
364 version.as_str(),
365 Some(crate_name.as_str()),
366 )
367 .or(scraped_ver);
368
369 let anchor_family = matched_anchor
373 .as_deref()
374 .and_then(|a| a.split_once('.'))
375 .map(|(family, _)| family.to_string());
376
377 let mut source_url = resp.final_url.clone();
378 if let Some(anchor) = matched_anchor {
382 source_url.set_fragment(Some(&anchor));
383 } else if let Some(f) = fragment {
384 source_url.set_fragment(Some(&f));
385 }
386
387 Ok(GetItemData {
388 crate_name: crate_name.as_str().to_string(),
389 item_type: item_type_echo.to_string(),
390 item_path,
391 item_name,
392 version: version.as_str().to_string(),
393 resolved_version: resolved,
394 markdown,
395 empty,
396 truncated: false,
397 source_url: source_url.to_string(),
398 title,
399 cache_hit: resp.cache_hit,
400 extraction,
401 anchor_family,
402 resolved_item_path: None,
403 })
404}