1use url::Url;
4
5use crate::domain::{AllowedOrigin, CrateName, VersionArg};
6use crate::error::{AppError, AppResult, ErrorKind};
7use crate::http::{HttpClient, decode_utf8, require_content_type_html};
8use crate::item_kind::{ItemKind, rustc_crate_name};
9
10use scraper::Html;
11
12use super::html::{
13 extract_item_markdown_from_document, extract_method_markdown_scoped_from_document,
14 extract_readme_markdown_from_document, extract_resolved_version_for_crate,
15 scrape_docs_rs_version_from_document,
16};
17use super::types::{GetItemData, ReadmeData};
18use super::urls::{
19 METHOD_PARENT_KINDS, get_item_url_on_origin, get_item_url_on_origin_with_parent_kind,
20 is_method_path, readme_url_on_origin,
21};
22
23pub async fn fetch_readme(
30 http: &HttpClient,
31 crate_name: &CrateName,
32 version: &VersionArg,
33) -> AppResult<ReadmeData> {
34 fetch_readme_on_origin(http, &AllowedOrigin::docs_rs_default(), crate_name, version).await
35}
36
37pub async fn fetch_readme_on_origin(
44 http: &HttpClient,
45 origin: &AllowedOrigin,
46 crate_name: &CrateName,
47 version: &VersionArg,
48) -> AppResult<ReadmeData> {
49 let url = readme_url_on_origin(origin, crate_name, version)?;
50 fetch_readme_at(http, crate_name, version, &url).await
51}
52
53pub async fn fetch_readme_at(
61 http: &HttpClient,
62 crate_name: &CrateName,
63 version: &VersionArg,
64 url: &Url,
65) -> AppResult<ReadmeData> {
66 let resp = http.get_html(url).await?;
67 if resp.status.as_u16() == 404 {
68 return Err(AppError::from_http_status(
69 404,
70 &format!("crate or version not found: {crate_name}@{version}"),
71 ));
72 }
73 if !resp.status.is_success() {
74 return Err(AppError::from_http_status(
75 resp.status.as_u16(),
76 "rustdoc readme",
77 ));
78 }
79 require_content_type_html(resp.content_type.as_deref())?;
81
82 let body = resp.body;
84 let crate_for_scrape = crate_name.as_str().to_string();
85 let (markdown, empty, scraped_ver) = http
86 .budget()
87 .run_cpu_bound(move || {
88 let text = decode_utf8(&body)?;
89 let doc = Html::parse_document(&text);
90 let (md, empty) = extract_readme_markdown_from_document(&doc)?;
91 let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
92 Ok((md, empty, scraped))
93 })
94 .await?;
95 let resolved = extract_resolved_version_for_crate(
96 &resp.final_url,
97 version.as_str(),
98 Some(crate_name.as_str()),
99 )
100 .or(scraped_ver);
101 Ok(ReadmeData {
102 crate_name: crate_name.as_str().to_string(),
103 version: version.as_str().to_string(),
104 resolved_version: resolved,
105 markdown,
106 empty,
107 truncated: false,
108 source_url: resp.final_url.to_string(),
109 cache_hit: resp.cache_hit,
110 })
111}
112
113pub async fn fetch_item(
120 http: &HttpClient,
121 crate_name: &CrateName,
122 version: &VersionArg,
123 kind: ItemKind,
124 segments: &[String],
125) -> AppResult<GetItemData> {
126 fetch_item_on_origin(
127 http,
128 &AllowedOrigin::docs_rs_default(),
129 crate_name,
130 version,
131 kind,
132 segments,
133 )
134 .await
135}
136
137pub async fn fetch_item_on_origin(
147 http: &HttpClient,
148 origin: &AllowedOrigin,
149 crate_name: &CrateName,
150 version: &VersionArg,
151 kind: ItemKind,
152 segments: &[String],
153) -> AppResult<GetItemData> {
154 fetch_item_on_origin_with_echo(
155 http,
156 origin,
157 crate_name,
158 version,
159 kind,
160 kind.as_str(),
161 segments,
162 )
163 .await
164}
165
166pub async fn fetch_item_on_origin_with_echo(
168 http: &HttpClient,
169 origin: &AllowedOrigin,
170 crate_name: &CrateName,
171 version: &VersionArg,
172 kind: ItemKind,
173 item_type_echo: &str,
174 segments: &[String],
175) -> AppResult<GetItemData> {
176 let rustc_root = rustc_crate_name(crate_name.as_str());
178 let segs_for_detect: Vec<String> = {
179 let mut s = segments.to_vec();
180 if let Some(first) = s.first() {
181 let f = first.as_str();
182 let is_crate_prefix = f == crate_name.as_str() || f == rustc_root.as_str();
183 if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
184 s.remove(0);
185 }
186 }
187 s
188 };
189
190 if is_method_path(kind, &segs_for_detect) {
191 let mut first_err: Option<AppError> = None;
194 for parent_kind in METHOD_PARENT_KINDS {
195 let url = get_item_url_on_origin_with_parent_kind(
196 origin,
197 crate_name,
198 version,
199 kind,
200 segments,
201 Some(*parent_kind),
202 )?;
203 match fetch_item_at_with_echo(
204 http,
205 crate_name,
206 version,
207 kind,
208 item_type_echo,
209 segments,
210 &url,
211 )
212 .await
213 {
214 Ok(data) => return Ok(data),
215 Err(e)
217 if e.kind() == ErrorKind::NotFound
218 && e.message().starts_with("method anchor ") =>
219 {
220 return Err(e);
221 }
222 Err(e) if e.kind() == ErrorKind::NotFound => {
223 if first_err.is_none() {
224 first_err = Some(e);
225 }
226 continue;
227 }
228 Err(e) => return Err(e),
229 }
230 }
231 return Err(first_err.unwrap_or_else(|| {
232 AppError::new(ErrorKind::NotFound, "method parent type page not found")
233 }));
234 }
235
236 let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
237 fetch_item_at_with_echo(
238 http,
239 crate_name,
240 version,
241 kind,
242 item_type_echo,
243 segments,
244 &url,
245 )
246 .await
247}
248
249pub async fn fetch_item_at(
257 http: &HttpClient,
258 crate_name: &CrateName,
259 version: &VersionArg,
260 kind: ItemKind,
261 segments: &[String],
262 url: &Url,
263) -> AppResult<GetItemData> {
264 fetch_item_at_with_echo(
265 http,
266 crate_name,
267 version,
268 kind,
269 kind.as_str(),
270 segments,
271 url,
272 )
273 .await
274}
275
276pub async fn fetch_item_at_with_echo(
278 http: &HttpClient,
279 crate_name: &CrateName,
280 version: &VersionArg,
281 kind: ItemKind,
282 item_type_echo: &str,
283 segments: &[String],
284 url: &Url,
285) -> AppResult<GetItemData> {
286 let _ = kind; let mut fetch_url = url.clone();
289 let fragment = fetch_url.fragment().map(str::to_string);
290 fetch_url.set_fragment(None);
291
292 let resp = http.get_html(&fetch_url).await?;
293 if resp.status.as_u16() == 404 {
294 return Err(AppError::from_http_status(
295 404,
296 &format!("item not found at {url}"),
297 ));
298 }
299 if !resp.status.is_success() {
300 return Err(AppError::from_http_status(
301 resp.status.as_u16(),
302 "rustdoc get-item",
303 ));
304 }
305 require_content_type_html(resp.content_type.as_deref())?;
306
307 let method_anchor = fragment
308 .as_deref()
309 .and_then(|f| f.strip_prefix("method."))
310 .map(str::to_string);
311 let body = resp.body;
313 let method_anchor_cpu = method_anchor.clone();
314 let crate_for_scrape = crate_name.as_str().to_string();
315 let ((markdown, empty), extraction, scraped_ver) = http
316 .budget()
317 .run_cpu_bound(move || {
318 let text = decode_utf8(&body)?;
319 let doc = Html::parse_document(&text);
320 let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
321 if let Some(ref m) = method_anchor_cpu {
322 let (md, empty, scope) =
323 extract_method_markdown_scoped_from_document(&doc, m)?;
324 Ok(((md, empty), Some(scope.to_string()), scraped))
325 } else {
326 let (md, empty) = extract_item_markdown_from_document(&doc)?;
327 Ok(((md, empty), None, scraped))
328 }
329 })
330 .await?;
331 let item_path = segments.join("::");
332 let item_name = segments
333 .last()
334 .cloned()
335 .unwrap_or_else(|| item_path.clone());
336 let title = format!("{item_path} ({item_type_echo})");
337 let resolved = extract_resolved_version_for_crate(
338 &resp.final_url,
339 version.as_str(),
340 Some(crate_name.as_str()),
341 )
342 .or(scraped_ver);
343
344 let mut source_url = resp.final_url.clone();
345 if let Some(f) = fragment {
346 source_url.set_fragment(Some(&f));
347 }
348
349 Ok(GetItemData {
350 crate_name: crate_name.as_str().to_string(),
351 item_type: item_type_echo.to_string(),
352 item_path,
353 item_name,
354 version: version.as_str().to_string(),
355 resolved_version: resolved,
356 markdown,
357 empty,
358 truncated: false,
359 source_url: source_url.to_string(),
360 title,
361 cache_hit: resp.cache_hit,
362 extraction,
363 resolved_item_path: None,
364 })
365}