Skip to main content

docsrs_cli/docs_rs/
fetch.rs

1//! Async HTTP fetch for readme / get-item (I/O on Tokio; parse on spawn_blocking).
2
3use url::Url;
4
5use crate::domain::{AllowedOrigin, CrateName, VersionArg};
6use crate::error::{AppError, AppResult, ErrorKind};
7use crate::http::{HttpClient, decode_utf8, require_content_type_html};
8use crate::item_kind::{ItemKind, rustc_crate_name};
9
10use scraper::Html;
11
12use super::html::{
13    extract_item_markdown_from_document, extract_method_markdown_scoped_from_document,
14    extract_readme_markdown_from_document, extract_resolved_version_for_crate,
15    scrape_docs_rs_version_from_document,
16};
17use super::types::{GetItemData, ReadmeData};
18use super::urls::{
19    METHOD_PARENT_KINDS, get_item_url_on_origin, get_item_url_on_origin_with_parent_kind,
20    is_method_path, readme_url_on_origin,
21};
22
23/// Fetch and convert crate overview docblock (production docs.rs).
24///
25/// # Errors
26///
27/// Propagates URL, HTTP, decode, and HTML conversion failures from
28/// [`fetch_readme_on_origin`].
29pub async fn fetch_readme(
30    http: &HttpClient,
31    crate_name: &CrateName,
32    version: &VersionArg,
33) -> AppResult<ReadmeData> {
34    fetch_readme_on_origin(http, &AllowedOrigin::docs_rs_default(), crate_name, version).await
35}
36
37/// Fetch readme against a configurable origin (offline mocks).
38///
39/// # Errors
40///
41/// Returns [`ErrorKind::Internal`] when the origin URL is invalid.
42/// Propagates fetch failures from [`fetch_readme_at`].
43pub async fn fetch_readme_on_origin(
44    http: &HttpClient,
45    origin: &AllowedOrigin,
46    crate_name: &CrateName,
47    version: &VersionArg,
48) -> AppResult<ReadmeData> {
49    let url = readme_url_on_origin(origin, crate_name, version)?;
50    fetch_readme_at(http, crate_name, version, &url).await
51}
52
53/// Fetch readme from a prebuilt URL (wiremock).
54///
55/// # Errors
56///
57/// Propagates HTTP errors from [`HttpClient::get_html`].
58/// Maps non-success statuses via [`AppError::from_http_status`].
59/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
60pub async fn fetch_readme_at(
61    http: &HttpClient,
62    crate_name: &CrateName,
63    version: &VersionArg,
64    url: &Url,
65) -> AppResult<ReadmeData> {
66    let resp = http.get_html(url).await?;
67    if resp.status.as_u16() == 404 {
68        return Err(AppError::from_http_status(
69            404,
70            &format!("crate or version not found: {crate_name}@{version}"),
71        ));
72    }
73    if !resp.status.is_success() {
74        return Err(AppError::from_http_status(
75            resp.status.as_u16(),
76            "rustdoc readme",
77        ));
78    }
79    // Fail closed when origin advertises a non-HTML type (MIME confusion).
80    require_content_type_html(resp.content_type.as_deref())?;
81
82    // One decode + one DOM parse for extract + version scrape (SCRAPE-Q-004/009, S-003).
83    let body = resp.body;
84    let crate_for_scrape = crate_name.as_str().to_string();
85    let (markdown, empty, scraped_ver) = http
86        .budget()
87        .run_cpu_bound(move || {
88            let text = decode_utf8(&body)?;
89            let doc = Html::parse_document(&text);
90            let (md, empty) = extract_readme_markdown_from_document(&doc)?;
91            let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
92            Ok((md, empty, scraped))
93        })
94        .await?;
95    let resolved = extract_resolved_version_for_crate(
96        &resp.final_url,
97        version.as_str(),
98        Some(crate_name.as_str()),
99    )
100    .or(scraped_ver);
101    Ok(ReadmeData {
102        crate_name: crate_name.as_str().to_string(),
103        version: version.as_str().to_string(),
104        resolved_version: resolved,
105        markdown,
106        empty,
107        truncated: false,
108        source_url: resp.final_url.to_string(),
109        cache_hit: resp.cache_hit,
110    })
111}
112
113/// Fetch and convert a typed rustdoc item page (production docs.rs).
114///
115/// # Errors
116///
117/// Propagates URL, HTTP, decode, and HTML conversion failures from
118/// [`fetch_item_on_origin`].
119pub async fn fetch_item(
120    http: &HttpClient,
121    crate_name: &CrateName,
122    version: &VersionArg,
123    kind: ItemKind,
124    segments: &[String],
125) -> AppResult<GetItemData> {
126    fetch_item_on_origin(
127        http,
128        &AllowedOrigin::docs_rs_default(),
129        crate_name,
130        version,
131        kind,
132        segments,
133    )
134    .await
135}
136
137/// Fetch item against a configurable origin (offline mocks).
138///
139/// For associated methods, tries parent type pages (struct → enum → trait → …)
140/// until one returns HTTP 200.
141///
142/// # Errors
143///
144/// Propagates URL build errors from [`get_item_url_on_origin`] and fetch errors
145/// from [`fetch_item_at`].
146pub async fn fetch_item_on_origin(
147    http: &HttpClient,
148    origin: &AllowedOrigin,
149    crate_name: &CrateName,
150    version: &VersionArg,
151    kind: ItemKind,
152    segments: &[String],
153) -> AppResult<GetItemData> {
154    fetch_item_on_origin_with_echo(
155        http,
156        origin,
157        crate_name,
158        version,
159        kind,
160        kind.as_str(),
161        segments,
162    )
163    .await
164}
165
166/// Like [`fetch_item_on_origin`] with a wire `item_type` echo (`method` vs `fn`).
167pub async fn fetch_item_on_origin_with_echo(
168    http: &HttpClient,
169    origin: &AllowedOrigin,
170    crate_name: &CrateName,
171    version: &VersionArg,
172    kind: ItemKind,
173    item_type_echo: &str,
174    segments: &[String],
175) -> AppResult<GetItemData> {
176    // Strip crate prefix the same way URL builder does for method detection.
177    let rustc_root = rustc_crate_name(crate_name.as_str());
178    let segs_for_detect: Vec<String> = {
179        let mut s = segments.to_vec();
180        if let Some(first) = s.first() {
181            let f = first.as_str();
182            let is_crate_prefix = f == crate_name.as_str() || f == rustc_root.as_str();
183            if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
184                s.remove(0);
185            }
186        }
187        s
188    };
189
190    if is_method_path(kind, &segs_for_detect) {
191        // Keep the *first* parent-page NotFound for diagnostics (X-010): dry-run
192        // plans `struct.*` first; reporting the last probe (`union.*`) misleads agents.
193        let mut first_err: Option<AppError> = None;
194        for parent_kind in METHOD_PARENT_KINDS {
195            let url = get_item_url_on_origin_with_parent_kind(
196                origin,
197                crate_name,
198                version,
199                kind,
200                segments,
201                Some(*parent_kind),
202            )?;
203            match fetch_item_at_with_echo(
204                http,
205                crate_name,
206                version,
207                kind,
208                item_type_echo,
209                segments,
210                &url,
211            )
212            .await
213            {
214                Ok(data) => return Ok(data),
215                // Parent page found but method anchor missing → stop probing kinds.
216                Err(e)
217                    if e.kind() == ErrorKind::NotFound
218                        && e.message().starts_with("method anchor ") =>
219                {
220                    return Err(e);
221                }
222                Err(e) if e.kind() == ErrorKind::NotFound => {
223                    if first_err.is_none() {
224                        first_err = Some(e);
225                    }
226                    continue;
227                }
228                Err(e) => return Err(e),
229            }
230        }
231        return Err(first_err.unwrap_or_else(|| {
232            AppError::new(ErrorKind::NotFound, "method parent type page not found")
233        }));
234    }
235
236    let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
237    fetch_item_at_with_echo(
238        http,
239        crate_name,
240        version,
241        kind,
242        item_type_echo,
243        segments,
244        &url,
245    )
246    .await
247}
248
249/// Fetch item from a prebuilt URL (wiremock).
250///
251/// # Errors
252///
253/// Propagates HTTP errors from [`HttpClient::get_html`].
254/// Maps non-success statuses via [`AppError::from_http_status`].
255/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
256pub async fn fetch_item_at(
257    http: &HttpClient,
258    crate_name: &CrateName,
259    version: &VersionArg,
260    kind: ItemKind,
261    segments: &[String],
262    url: &Url,
263) -> AppResult<GetItemData> {
264    fetch_item_at_with_echo(
265        http,
266        crate_name,
267        version,
268        kind,
269        kind.as_str(),
270        segments,
271        url,
272    )
273    .await
274}
275
276/// Like [`fetch_item_at`] but echoes a requested kind string (e.g. `method` vs `fn`).
277pub async fn fetch_item_at_with_echo(
278    http: &HttpClient,
279    crate_name: &CrateName,
280    version: &VersionArg,
281    kind: ItemKind,
282    item_type_echo: &str,
283    segments: &[String],
284    url: &Url,
285) -> AppResult<GetItemData> {
286    let _ = kind; // URL kind already baked into `url`; echo drives item_type/title.
287    // HTTP does not send fragments; strip for request, keep for source_url.
288    let mut fetch_url = url.clone();
289    let fragment = fetch_url.fragment().map(str::to_string);
290    fetch_url.set_fragment(None);
291
292    let resp = http.get_html(&fetch_url).await?;
293    if resp.status.as_u16() == 404 {
294        return Err(AppError::from_http_status(
295            404,
296            &format!("item not found at {url}"),
297        ));
298    }
299    if !resp.status.is_success() {
300        return Err(AppError::from_http_status(
301            resp.status.as_u16(),
302            "rustdoc get-item",
303        ));
304    }
305    require_content_type_html(resp.content_type.as_deref())?;
306
307    let method_anchor = fragment
308        .as_deref()
309        .and_then(|f| f.strip_prefix("method."))
310        .map(str::to_string);
311    // Move body once: one DOM for markdown + version scrape (SCRAPE-S-003).
312    let body = resp.body;
313    let method_anchor_cpu = method_anchor.clone();
314    let crate_for_scrape = crate_name.as_str().to_string();
315    let ((markdown, empty), extraction, scraped_ver) = http
316        .budget()
317        .run_cpu_bound(move || {
318            let text = decode_utf8(&body)?;
319            let doc = Html::parse_document(&text);
320            let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
321            if let Some(ref m) = method_anchor_cpu {
322                let (md, empty, scope) =
323                    extract_method_markdown_scoped_from_document(&doc, m)?;
324                Ok(((md, empty), Some(scope.to_string()), scraped))
325            } else {
326                let (md, empty) = extract_item_markdown_from_document(&doc)?;
327                Ok(((md, empty), None, scraped))
328            }
329        })
330        .await?;
331    let item_path = segments.join("::");
332    let item_name = segments
333        .last()
334        .cloned()
335        .unwrap_or_else(|| item_path.clone());
336    let title = format!("{item_path} ({item_type_echo})");
337    let resolved = extract_resolved_version_for_crate(
338        &resp.final_url,
339        version.as_str(),
340        Some(crate_name.as_str()),
341    )
342    .or(scraped_ver);
343
344    let mut source_url = resp.final_url.clone();
345    if let Some(f) = fragment {
346        source_url.set_fragment(Some(&f));
347    }
348
349    Ok(GetItemData {
350        crate_name: crate_name.as_str().to_string(),
351        item_type: item_type_echo.to_string(),
352        item_path,
353        item_name,
354        version: version.as_str().to_string(),
355        resolved_version: resolved,
356        markdown,
357        empty,
358        truncated: false,
359        source_url: source_url.to_string(),
360        title,
361        cache_hit: resp.cache_hit,
362        extraction,
363        resolved_item_path: None,
364    })
365}