Skip to main content

docsrs_cli/docs_rs/
search.rs

1//! all.html search: async fetch + parallel/sequential hit classification.
2//!
3//! Parallelism: `rayon` only above [`RAYON_HIT_THRESHOLD`]; small lists stay sequential.
4
5use std::collections::HashSet;
6use std::sync::LazyLock;
7
8use rayon::prelude::*;
9use scraper::{Html, Selector};
10use url::Url;
11
12/// Compiled once: all.html symbol links under main content (SCRAPE-R-004).
13static SEL_MAIN_CONTENT_A: LazyLock<Selector> = LazyLock::new(|| {
14    Selector::parse("#main-content a")
15        .expect("hardcoded scraper selector '#main-content a' is valid by construction")
16});
17
18use crate::config::{MAX_SEARCH_IN_CRATE_LIMIT, RAYON_HIT_THRESHOLD};
19use crate::domain::{AllowedOrigin, CrateName, MatchMode, SearchQuery, VersionArg};
20use crate::error::{AppError, AppResult, ErrorKind};
21use crate::http::{HttpClient, decode_utf8, require_content_type_html};
22use crate::item_kind::ItemKind;
23use crate::shutdown::CancelFlag;
24
25use super::types::{SearchInCrateData, SearchInCrateHit};
26use super::urls::all_html_url_on_origin;
27
28/// Parse all.html and filter symbols (production docs.rs).
29///
30/// # Errors
31///
32/// Propagates URL, HTTP, decode, and parse failures from
33/// [`search_in_crate_on_origin`].
34pub async fn search_in_crate(
35    http: &HttpClient,
36    crate_name: &CrateName,
37    version: &VersionArg,
38    query: &SearchQuery,
39    item_type: Option<ItemKind>,
40    limit: u32,
41    match_mode: MatchMode,
42) -> AppResult<SearchInCrateData> {
43    // Prefer stdlib origin for std/core/alloc (same rule as ops::docs_origin_for_crate).
44    let origin = if crate_name.is_stdlib() {
45        AllowedOrigin::stdlib_docs_default()
46    } else {
47        AllowedOrigin::docs_rs_default()
48    };
49    search_in_crate_on_origin(
50        http,
51        &origin,
52        crate_name,
53        version,
54        query,
55        item_type,
56        limit,
57        match_mode,
58    )
59    .await
60}
61
62/// search-in-crate against a configurable origin (offline mocks).
63///
64/// # Errors
65///
66/// Propagates URL build errors from [`all_html_url_on_origin`] and fetch errors
67/// from [`search_in_crate_at`].
68#[allow(clippy::too_many_arguments)] // domain triple + filter/limit/mode are intentional
69pub async fn search_in_crate_on_origin(
70    http: &HttpClient,
71    origin: &AllowedOrigin,
72    crate_name: &CrateName,
73    version: &VersionArg,
74    query: &SearchQuery,
75    item_type: Option<ItemKind>,
76    limit: u32,
77    match_mode: MatchMode,
78) -> AppResult<SearchInCrateData> {
79    let url = all_html_url_on_origin(origin, crate_name, version)?;
80    search_in_crate_at(
81        http, crate_name, version, query, item_type, limit, match_mode, &url,
82    )
83    .await
84}
85
86/// search-in-crate against a prebuilt all.html URL (wiremock).
87///
88/// # Errors
89///
90/// Propagates HTTP errors from [`HttpClient::get_html`].
91/// Maps non-success statuses via [`AppError::from_http_status`].
92/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or href join failures.
93/// Propagates parse errors from [`search_in_crate_from_html`].
94#[allow(clippy::too_many_arguments)] // domain triple + filter/limit/mode + url
95pub async fn search_in_crate_at(
96    http: &HttpClient,
97    crate_name: &CrateName,
98    version: &VersionArg,
99    query: &SearchQuery,
100    item_type: Option<ItemKind>,
101    limit: u32,
102    match_mode: MatchMode,
103    url: &Url,
104) -> AppResult<SearchInCrateData> {
105    let resp = http.get_html(url).await?;
106    if resp.status.as_u16() == 404 {
107        return Err(AppError::from_http_status(
108            404,
109            &format!("all.html not found for {crate_name}@{version}"),
110        ));
111    }
112    if !resp.status.is_success() {
113        return Err(AppError::from_http_status(
114            resp.status.as_u16(),
115            "rustdoc all.html",
116        ));
117    }
118    require_content_type_html(resp.content_type.as_deref())?;
119
120    // Move body into the CPU worker (one-shot memory: no clone for a second decode).
121    let body = resp.body;
122    let source_url = resp.final_url.to_string();
123    let cache_hit = resp.cache_hit;
124    let crate_name = crate_name.clone();
125    let version = version.clone();
126    let query = query.clone();
127    let cancel = http.cancel_flag().clone();
128    http.budget()
129        .run_cpu_bound(move || {
130            let text = decode_utf8(&body)?;
131            search_in_crate_from_html(
132                &text,
133                &crate_name,
134                &version,
135                &query,
136                item_type,
137                limit,
138                match_mode,
139                &source_url,
140                cache_hit,
141                &cancel,
142            )
143        })
144        .await
145}
146
147/// Resolve an all.html `href` against `base` (`source_url`), same-origin only.
148///
149/// - Relative and path-absolute hrefs use [`Url::join`] against `base` (never a
150///   hardcoded docs.rs host — SCRAPE-Q-001).
151/// - Absolute `http(s)://` hrefs are accepted only when **scheme + host** match
152///   `base` (host compared case-insensitively).
153/// - Off-origin absolute URLs and join/parse failures return [`None`] so callers
154///   can soft-skip the hit without failing the entire search (SCRAPE-S-001).
155#[must_use]
156pub fn resolve_hit_url(base: &Url, href: &str) -> Option<String> {
157    let href = href.trim();
158    if href.is_empty() {
159        return None;
160    }
161    let joined = if href.starts_with("http://") || href.starts_with("https://") {
162        Url::parse(href).ok()?
163    } else {
164        // Url::join handles relative segments and absolute paths (`/…`).
165        base.join(href).ok()?
166    };
167    if !same_origin(base, &joined) {
168        return None;
169    }
170    Some(joined.to_string())
171}
172
173/// True when `candidate` shares scheme and host with `base` (SCRAPE-S-001).
174fn same_origin(base: &Url, candidate: &Url) -> bool {
175    if candidate.scheme() != base.scheme() {
176        return false;
177    }
178    match (base.host_str(), candidate.host_str()) {
179        (Some(a), Some(b)) => a.eq_ignore_ascii_case(b),
180        (None, None) => true,
181        _ => false,
182    }
183}
184
185/// Join relative/absolute href against crate rustdoc base (same-origin).
186///
187/// Prefer [`resolve_hit_url`] in hit classification (soft-skip). This wrapper
188/// maps off-origin / join failure to [`ErrorKind::Parse`] for callers that want
189/// a hard error.
190///
191/// # Errors
192///
193/// Returns [`ErrorKind::Parse`] when the href is off-origin or not joinable.
194pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
195    resolve_hit_url(base, href).ok_or_else(|| {
196        AppError::new(
197            ErrorKind::Parse,
198            "failed to join href (off-origin or invalid against source_url base)",
199        )
200    })
201}
202
203/// One classified hit candidate before dedup/limit (index preserves document order).
204/// One classified hit candidate before dedup/limit: (index, name, kind, url, score).
205type ClassifiedHit = (usize, String, ItemKind, String, u8);
206
207/// Pure parse of all.html body for offline tests and CPU workers.
208///
209/// `base` must be the final all.html URL (or its directory). Relative and
210/// absolute-path hrefs resolve via [`resolve_hit_url`] against this URL — never
211/// against a hardcoded `docs.rs` host — so stdlib (`doc.rust-lang.org`) and
212/// mock origins keep correct hit URLs (SCRAPE-Q-001). Off-origin absolute hrefs
213/// are soft-skipped (SCRAPE-S-001).
214///
215/// Large candidate lists use `rayon`; small lists stay sequential.
216/// Results are scored and sorted (exact leaf first) before applying `limit`.
217///
218/// # Errors
219///
220/// Currently infallible for join failures (soft-skip); reserved for future
221/// hard parse failures of the document itself.
222pub fn parse_all_html_hits(
223    html: &str,
224    base: &Url,
225    query: &SearchQuery,
226    item_type: Option<ItemKind>,
227    limit: usize,
228    match_mode: MatchMode,
229    cancel: &CancelFlag,
230) -> AppResult<Vec<SearchInCrateHit>> {
231    if limit == 0 {
232        return Ok(Vec::new());
233    }
234    cancel.check()?;
235    let document = Html::parse_document(html);
236    let q = query.as_str().trim().to_ascii_lowercase();
237
238    let candidates: Vec<(String, String)> = document
239        .select(&SEL_MAIN_CONTENT_A)
240        .filter_map(|a| {
241            let name = a.text().collect::<String>().trim().to_string();
242            if name.is_empty() {
243                return None;
244            }
245            let href = a.value().attr("href")?.to_string();
246            if href.is_empty() {
247                return None;
248            }
249            Some((name, href))
250        })
251        .collect();
252
253    if candidates.len() < RAYON_HIT_THRESHOLD {
254        return filter_hits_sequential(candidates, base, &q, item_type, limit, match_mode, cancel);
255    }
256    filter_hits_parallel(candidates, base, &q, item_type, limit, match_mode, cancel)
257}
258
259pub(super) fn filter_hits_sequential(
260    candidates: Vec<(String, String)>,
261    base: &Url,
262    q: &str,
263    item_type: Option<ItemKind>,
264    limit: usize,
265    match_mode: MatchMode,
266    cancel: &CancelFlag,
267) -> AppResult<Vec<SearchInCrateHit>> {
268    // Pre-size for memory discipline (Rules Rust — gerenciamento de memória).
269    let mut rows: Vec<ClassifiedHit> = Vec::with_capacity(candidates.len());
270    for (idx, (name, href)) in candidates.into_iter().enumerate() {
271        cancel.check()?;
272        if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
273            rows.push(hit);
274        }
275    }
276    finalize_hits(rows, limit)
277}
278
279pub(super) fn filter_hits_parallel(
280    candidates: Vec<(String, String)>,
281    base: &Url,
282    q: &str,
283    item_type: Option<ItemKind>,
284    limit: usize,
285    match_mode: MatchMode,
286    cancel: &CancelFlag,
287) -> AppResult<Vec<SearchInCrateHit>> {
288    // Clone Arc-backed flag once for the rayon pool (cheap; no product env).
289    let cancel = cancel.clone();
290    let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
291        .into_par_iter()
292        .enumerate()
293        .map(|(idx, (name, href))| {
294            // Cooperative cancel: first cancelled worker returns Interrupted/Terminated.
295            cancel.check()?;
296            classify_hit_row(idx, name, &href, base, q, item_type, match_mode)
297        })
298        .collect();
299    let rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
300    finalize_hits(rows, limit)
301}
302
303pub(super) fn finalize_hits(mut rows: Vec<ClassifiedHit>, limit: usize) -> AppResult<Vec<SearchInCrateHit>> {
304    // Score ascending, then original document order for stability.
305    rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
306    // Capacity: min(limit, rows) so `usize::MAX` total-scan never overflows (SCRAPE-S-005).
307    let mut hits = Vec::with_capacity(limit.min(rows.len()));
308    let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
309    for (_idx, name, kind, abs, score) in rows {
310        if !seen.insert((name.clone(), kind)) {
311            continue;
312        }
313        hits.push(SearchInCrateHit {
314            name,
315            kind: kind.as_str().to_string(),
316            url: abs,
317            score: Some(score),
318        });
319        if hits.len() >= limit {
320            break;
321        }
322    }
323    Ok(hits)
324}
325
326pub(super) fn classify_hit_row(
327    idx: usize,
328    name: String,
329    href: &str,
330    base: &Url,
331    q: &str,
332    item_type: Option<ItemKind>,
333    match_mode: MatchMode,
334) -> AppResult<Option<ClassifiedHit>> {
335    let Some(kind) = ItemKind::from_href(href) else {
336        return Ok(None);
337    };
338    if let Some(filter) = item_type
339        && kind != filter
340    {
341        return Ok(None);
342    }
343    let Some(score) = match_mode.score(&name, q) else {
344        return Ok(None);
345    };
346    // Soft-skip off-origin / unjoinable hrefs — never abort the whole search (SCRAPE-S-001).
347    let Some(abs) = resolve_hit_url(base, href) else {
348        return Ok(None);
349    };
350    Ok(Some((idx, name, kind, abs, score)))
351}
352
353/// Build SearchInCrateData from HTML body without network (offline tests).
354///
355/// `--limit 0` is honoured: `emitted = 0`, `hits = []`, and `truncated` is true
356/// when the unfiltered total is greater than zero. Values above
357/// [`MAX_SEARCH_IN_CRATE_LIMIT`] are capped.
358///
359/// # Errors
360///
361/// Returns [`ErrorKind::Parse`] when `source_url` is not a valid URL base.
362/// Propagates parse failures from [`parse_all_html_hits`].
363#[allow(clippy::too_many_arguments)] // pure offline path mirrors the async API surface
364pub fn search_in_crate_from_html(
365    html: &str,
366    crate_name: &CrateName,
367    version: &VersionArg,
368    query: &SearchQuery,
369    item_type: Option<ItemKind>,
370    limit: u32,
371    match_mode: MatchMode,
372    source_url: &str,
373    cache_hit: bool,
374    cancel: &CancelFlag,
375) -> AppResult<SearchInCrateData> {
376    let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
377    // Join base = final all.html URL (stdlib / mock / docs.rs). Never rebuild host.
378    let base = Url::parse(source_url).map_err(|e| {
379        AppError::with_source(ErrorKind::Parse, "invalid source_url for hit join base", e)
380    })?;
381    let all = parse_all_html_hits(html, &base, query, item_type, usize::MAX, match_mode, cancel)?;
382    let total = all.len();
383    let hits: Vec<_> = all.into_iter().take(limit).collect();
384    let emitted = hits.len();
385    // Empty query: scores are 0 — omit score noise on wire for cleaner lists.
386    let hits = if query.as_str().trim().is_empty() {
387        hits.into_iter()
388            .map(|mut h| {
389                h.score = None;
390                h
391            })
392            .collect()
393    } else {
394        hits
395    };
396    Ok(SearchInCrateData {
397        crate_name: crate_name.as_str().to_string(),
398        query: query.as_str().to_string(),
399        version: version.as_str().to_string(),
400        item_type: item_type.map(|k| k.as_str().to_string()),
401        match_mode: match_mode.as_str().to_string(),
402        total,
403        emitted,
404        hits,
405        truncated: total > emitted,
406        source_url: source_url.to_string(),
407        cache_hit,
408    })
409}