Skip to main content

docsrs_cli/docs_rs/
fetch.rs

1//! Async HTTP fetch for readme / get-item (I/O on Tokio; parse on spawn_blocking).
2
3use url::Url;
4
5use crate::domain::{AllowedOrigin, CrateName, VersionArg};
6use crate::error::{AppError, AppResult, ErrorDetail, ErrorKind};
7use crate::http::{HttpClient, decode_utf8, require_content_type_html};
8use crate::item_kind::ItemKind;
9
10use scraper::Html;
11
12use super::assoc::{ASSOC_ANCHOR_MISS_PREFIX, assoc_from_fragment, associated_item_path};
13use super::html::{
14    extract_assoc_markdown_scoped_from_document, extract_item_markdown_from_document,
15    extract_readme_markdown_from_document, extract_resolved_version_for_crate,
16    scrape_docs_rs_version_from_document,
17};
18use super::types::{GetItemData, ReadmeData};
19use super::urls::{
20    get_item_url_on_origin, get_item_url_on_origin_with_parent_kind, readme_url_on_origin,
21    strip_crate_prefix_segments,
22};
23
24/// Fetch and convert crate overview docblock (production docs.rs).
25///
26/// # Errors
27///
28/// Propagates URL, HTTP, decode, and HTML conversion failures from
29/// [`fetch_readme_on_origin`].
30pub async fn fetch_readme(
31    http: &HttpClient,
32    crate_name: &CrateName,
33    version: &VersionArg,
34) -> AppResult<ReadmeData> {
35    fetch_readme_on_origin(http, &AllowedOrigin::docs_rs_default(), crate_name, version).await
36}
37
38/// Fetch readme against a configurable origin (offline mocks).
39///
40/// # Errors
41///
42/// Returns [`crate::error::ErrorKind::Internal`] when the origin URL is invalid.
43/// Propagates fetch failures from [`fetch_readme_at`].
44pub async fn fetch_readme_on_origin(
45    http: &HttpClient,
46    origin: &AllowedOrigin,
47    crate_name: &CrateName,
48    version: &VersionArg,
49) -> AppResult<ReadmeData> {
50    let url = readme_url_on_origin(origin, crate_name, version)?;
51    fetch_readme_at(http, crate_name, version, &url).await
52}
53
54/// Fetch readme from a prebuilt URL (wiremock).
55///
56/// # Errors
57///
58/// Propagates HTTP errors from [`HttpClient::get_html`].
59/// Maps non-success statuses via [`AppError::from_http_status`].
60/// Returns [`crate::error::ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
61pub async fn fetch_readme_at(
62    http: &HttpClient,
63    crate_name: &CrateName,
64    version: &VersionArg,
65    url: &Url,
66) -> AppResult<ReadmeData> {
67    let resp = http.get_html(url).await?;
68    if resp.status.as_u16() == 404 {
69        return Err(AppError::from_http_status(
70            404,
71            &format!("crate or version not found: {crate_name}@{version}"),
72        ));
73    }
74    if !resp.status.is_success() {
75        return Err(AppError::from_http_status(
76            resp.status.as_u16(),
77            "rustdoc readme",
78        ));
79    }
80    // Fail closed when origin advertises a non-HTML type (MIME confusion).
81    require_content_type_html(resp.content_type.as_deref())?;
82
83    // One decode + one DOM parse for extract + version scrape (SCRAPE-Q-004/009, S-003).
84    let body = resp.body;
85    let crate_for_scrape = crate_name.as_str().to_string();
86    let (markdown, empty, scraped_ver) = http
87        .budget()
88        .run_cpu_bound(move || {
89            let text = decode_utf8(&body)?;
90            let doc = Html::parse_document(&text);
91            let (md, empty) = extract_readme_markdown_from_document(&doc)?;
92            let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
93            Ok((md, empty, scraped))
94        })
95        .await?;
96    let resolved = extract_resolved_version_for_crate(
97        &resp.final_url,
98        version.as_str(),
99        Some(crate_name.as_str()),
100    )
101    .or(scraped_ver);
102    Ok(ReadmeData {
103        crate_name: crate_name.as_str().to_string(),
104        version: version.as_str().to_string(),
105        resolved_version: resolved,
106        markdown,
107        empty,
108        truncated: false,
109        source_url: resp.final_url.to_string(),
110        cache_hit: resp.cache_hit,
111    })
112}
113
114/// Fetch and convert a typed rustdoc item page (production docs.rs).
115///
116/// # Errors
117///
118/// Propagates URL, HTTP, decode, and HTML conversion failures from
119/// [`fetch_item_on_origin`].
120pub async fn fetch_item(
121    http: &HttpClient,
122    crate_name: &CrateName,
123    version: &VersionArg,
124    kind: ItemKind,
125    segments: &[String],
126) -> AppResult<GetItemData> {
127    fetch_item_on_origin(
128        http,
129        &AllowedOrigin::docs_rs_default(),
130        crate_name,
131        version,
132        kind,
133        segments,
134    )
135    .await
136}
137
138/// Fetch item against a configurable origin (offline mocks).
139///
140/// For associated methods, tries parent type pages (struct → enum → trait → …)
141/// until one returns HTTP 200.
142///
143/// # Errors
144///
145/// Propagates URL build errors from [`get_item_url_on_origin`] and fetch errors
146/// from [`fetch_item_at`].
147pub async fn fetch_item_on_origin(
148    http: &HttpClient,
149    origin: &AllowedOrigin,
150    crate_name: &CrateName,
151    version: &VersionArg,
152    kind: ItemKind,
153    segments: &[String],
154) -> AppResult<GetItemData> {
155    fetch_item_on_origin_with_echo(
156        http,
157        origin,
158        crate_name,
159        version,
160        kind,
161        kind.as_str(),
162        segments,
163    )
164    .await
165}
166
167/// Like [`fetch_item_on_origin`] with a wire `item_type` echo (`method` vs `fn`).
168///
169/// # Errors
170/// Same as [`fetch_item_on_origin`].
171pub async fn fetch_item_on_origin_with_echo(
172    http: &HttpClient,
173    origin: &AllowedOrigin,
174    crate_name: &CrateName,
175    version: &VersionArg,
176    kind: ItemKind,
177    item_type_echo: &str,
178    segments: &[String],
179) -> AppResult<GetItemData> {
180    // Shared with the URL builder on purpose. This block used to be a hand copy
181    // carrying the comment "the same way URL builder does" — a promise the
182    // compiler could not keep. Detection and URL construction reading different
183    // rules is the exact shape of GAP-ASSOCITEM-001.
184    let segs_for_detect = strip_crate_prefix_segments(crate_name, kind, segments);
185
186    if let Some(assoc) = associated_item_path(kind, &segs_for_detect) {
187        // Keep the *first* parent-page NotFound for diagnostics (X-010): dry-run
188        // plans the family's leading kind first; reporting the last probe
189        // (`union.*`) misleads agents.
190        //
191        // Sequential on purpose, reconfirmed by measurement in 2026-08 after the
192        // decision was reopened (GAP-PROBE-001).
193        //
194        // The original rejection cited a 12.6x regression without naming the
195        // mechanism, which is why it kept getting reopened. The mechanism is the
196        // politeness delay: `rate_limit_delay_ms` defaults to 1000 and is keyed
197        // by host, while every kind in `parent_kind_probe` — up to five — resolves
198        // against that same host. Measured on this host: `Duration::MAX`, which
199        // hits on its first probe, completes in 104 ms; `Iterator::Item`, which
200        // needs a second request, takes 1083 ms. The extra second is the throttle,
201        // not the network.
202        //
203        // So the floor for N sequential probes is (N-1) x delay whether they are
204        // issued serially or concurrently: a fan-out would queue behind the same
205        // limiter and add coordination on top. Concurrency can only pay off once
206        // the probes stop sharing a rate-limited host, which is not a property of
207        // rustdoc URLs.
208        let mut first_err: Option<AppError> = None;
209        for parent_kind in assoc.parent_kind_probe() {
210            let url = get_item_url_on_origin_with_parent_kind(
211                origin,
212                crate_name,
213                version,
214                kind,
215                segments,
216                Some(*parent_kind),
217            )?;
218            match fetch_item_at_with_echo(
219                http,
220                crate_name,
221                version,
222                kind,
223                item_type_echo,
224                segments,
225                &url,
226            )
227            .await
228            {
229                Ok(data) => return Ok(data),
230                // Parent page found but member anchor missing → stop probing kinds.
231                Err(e)
232                    if e.kind() == ErrorKind::NotFound
233                        && e.message().starts_with(ASSOC_ANCHOR_MISS_PREFIX) =>
234                {
235                    return Err(e);
236                }
237                Err(e) if e.kind() == ErrorKind::NotFound => {
238                    if first_err.is_none() {
239                        first_err = Some(e);
240                    }
241                    continue;
242                }
243                Err(e) => return Err(e),
244            }
245        }
246        return Err(first_err.unwrap_or_else(|| AppError::of(ErrorDetail::AssocParentPageNotFound)));
247    }
248
249    let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
250    fetch_item_at_with_echo(
251        http,
252        crate_name,
253        version,
254        kind,
255        item_type_echo,
256        segments,
257        &url,
258    )
259    .await
260}
261
262/// Fetch item from a prebuilt URL (wiremock).
263///
264/// # Errors
265///
266/// Propagates HTTP errors from [`HttpClient::get_html`].
267/// Maps non-success statuses via [`AppError::from_http_status`].
268/// Returns [`crate::error::ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
269pub async fn fetch_item_at(
270    http: &HttpClient,
271    crate_name: &CrateName,
272    version: &VersionArg,
273    kind: ItemKind,
274    segments: &[String],
275    url: &Url,
276) -> AppResult<GetItemData> {
277    fetch_item_at_with_echo(
278        http,
279        crate_name,
280        version,
281        kind,
282        kind.as_str(),
283        segments,
284        url,
285    )
286    .await
287}
288
289/// Like [`fetch_item_at`] but echoes a requested kind string (e.g. `method` vs `fn`).
290///
291/// # Errors
292/// Same as [`fetch_item_at`].
293pub async fn fetch_item_at_with_echo(
294    http: &HttpClient,
295    crate_name: &CrateName,
296    version: &VersionArg,
297    kind: ItemKind,
298    item_type_echo: &str,
299    segments: &[String],
300    url: &Url,
301) -> AppResult<GetItemData> {
302    let _ = kind; // URL kind already baked into `url`; echo drives item_type/title.
303    // HTTP does not send fragments; strip for request, keep for source_url.
304    let mut fetch_url = url.clone();
305    let fragment = fetch_url.fragment().map(str::to_string);
306    fetch_url.set_fragment(None);
307
308    let resp = http.get_html(&fetch_url).await?;
309    if resp.status.as_u16() == 404 {
310        return Err(AppError::from_http_status(
311            404,
312            &format!("item not found at {url}"),
313        ));
314    }
315    if !resp.status.is_success() {
316        return Err(AppError::from_http_status(
317            resp.status.as_u16(),
318            "rustdoc get-item",
319        ));
320    }
321    require_content_type_html(resp.content_type.as_deref())?;
322
323    // A fragment names its own anchor family without ambiguity, so the family is
324    // read back off the URL already in hand instead of threading an extra
325    // parameter through every layer. The extractor still decides which prefix of
326    // that family the page actually carries (`method.` vs `tymethod.`).
327    let assoc_anchor = fragment
328        .as_deref()
329        .and_then(assoc_from_fragment)
330        .map(|(family, name)| (family, name.to_string()));
331    // Move body once: one DOM for markdown + version scrape (SCRAPE-S-003).
332    let body = resp.body;
333    let assoc_anchor_cpu = assoc_anchor;
334    let crate_for_scrape = crate_name.as_str().to_string();
335    let ((markdown, empty), extraction, matched_anchor, scraped_ver) = http
336        .budget()
337        .run_cpu_bound(move || {
338            let text = decode_utf8(&body)?;
339            let doc = Html::parse_document(&text);
340            let scraped = scrape_docs_rs_version_from_document(&doc, &crate_for_scrape);
341            if let Some((family, ref m)) = assoc_anchor_cpu {
342                let found =
343                    extract_assoc_markdown_scoped_from_document(&doc, family.anchor_prefixes(), m)?;
344                Ok((
345                    (found.markdown, found.empty),
346                    Some(found.scope.to_string()),
347                    Some(found.anchor_id),
348                    scraped,
349                ))
350            } else {
351                let (md, empty) = extract_item_markdown_from_document(&doc)?;
352                Ok(((md, empty), None, None, scraped))
353            }
354        })
355        .await?;
356    let item_path = segments.join("::");
357    let item_name = segments
358        .last()
359        .cloned()
360        .unwrap_or_else(|| item_path.clone());
361    let title = format!("{item_path} ({item_type_echo})");
362    let resolved = extract_resolved_version_for_crate(
363        &resp.final_url,
364        version.as_str(),
365        Some(crate_name.as_str()),
366    )
367    .or(scraped_ver);
368
369    // The anchor id carries the family in its prefix (`variant.Some`), so the
370    // real family is already in hand here and never has to be re-derived from
371    // the requested kind. Split before the id is moved into the fragment below.
372    let anchor_family = matched_anchor
373        .as_deref()
374        .and_then(|a| a.split_once('.'))
375        .map(|(family, _)| family.to_string());
376
377    let mut source_url = resp.final_url.clone();
378    // Echo the anchor that exists on the page, not the one the URL builder
379    // guessed: a required trait method lives at `#tymethod.X`, and pointing an
380    // agent at `#method.X` would send it to a fragment the page never defines.
381    if let Some(anchor) = matched_anchor {
382        source_url.set_fragment(Some(&anchor));
383    } else if let Some(f) = fragment {
384        source_url.set_fragment(Some(&f));
385    }
386
387    Ok(GetItemData {
388        crate_name: crate_name.as_str().to_string(),
389        item_type: item_type_echo.to_string(),
390        item_path,
391        item_name,
392        version: version.as_str().to_string(),
393        resolved_version: resolved,
394        markdown,
395        empty,
396        truncated: false,
397        source_url: source_url.to_string(),
398        title,
399        cache_hit: resp.cache_hit,
400        extraction,
401        anchor_family,
402        resolved_item_path: None,
403    })
404}