Skip to main content

docsrs_cli/docs_rs/
search.rs

1//! all.html search: async fetch + sequential hit classification.
2//!
3//! The scan is sequential by measurement, not by omission. Per-item work is one
4//! URL resolve plus a string match — too small to amortize thread fan-out and
5//! join. A rayon path was benchmarked from 16 to 32768 candidates and lost at
6//! every size (best case 0.85x, i.e. 18% slower), so it was removed.
7
8use std::collections::HashSet;
9use std::sync::LazyLock;
10
11use scraper::{Html, Selector};
12use url::Url;
13
14/// Compiled once: all.html symbol links under main content (SCRAPE-R-004).
15static SEL_MAIN_CONTENT_A: LazyLock<Selector> = LazyLock::new(|| {
16    Selector::parse("#main-content a")
17        .expect("hardcoded scraper selector '#main-content a' is valid by construction")
18});
19
20use crate::config::MAX_SEARCH_IN_CRATE_LIMIT;
21use crate::domain::{AllowedOrigin, CrateName, MatchMode, SearchQuery, VersionArg};
22use crate::error::{AppError, AppResult, ErrorDetail};
23use crate::http::{HttpClient, decode_utf8, require_content_type_html};
24use crate::item_kind::ItemKind;
25use crate::shutdown::CancelFlag;
26
27use super::types::{SearchInCrateData, SearchInCrateHit};
28use super::urls::all_html_url_on_origin;
29
30/// Parse all.html and filter symbols (production docs.rs).
31///
32/// # Errors
33///
34/// Propagates URL, HTTP, decode, and parse failures from
35/// [`search_in_crate_on_origin`].
36pub async fn search_in_crate(
37    http: &HttpClient,
38    crate_name: &CrateName,
39    version: &VersionArg,
40    query: &SearchQuery,
41    item_type: Option<ItemKind>,
42    limit: u32,
43    match_mode: MatchMode,
44) -> AppResult<SearchInCrateData> {
45    // Prefer stdlib origin for std/core/alloc (same rule as ops::docs_origin_for_crate).
46    let origin = if crate_name.is_stdlib() {
47        AllowedOrigin::stdlib_docs_default()
48    } else {
49        AllowedOrigin::docs_rs_default()
50    };
51    search_in_crate_on_origin(
52        http, &origin, crate_name, version, query, item_type, limit, match_mode,
53    )
54    .await
55}
56
57/// search-in-crate against a configurable origin (offline mocks).
58///
59/// # Errors
60///
61/// Propagates URL build errors from [`all_html_url_on_origin`] and fetch errors
62/// from [`search_in_crate_at`].
63#[allow(clippy::too_many_arguments)] // domain triple + filter/limit/mode are intentional
64pub async fn search_in_crate_on_origin(
65    http: &HttpClient,
66    origin: &AllowedOrigin,
67    crate_name: &CrateName,
68    version: &VersionArg,
69    query: &SearchQuery,
70    item_type: Option<ItemKind>,
71    limit: u32,
72    match_mode: MatchMode,
73) -> AppResult<SearchInCrateData> {
74    let url = all_html_url_on_origin(origin, crate_name, version)?;
75    search_in_crate_at(
76        http, crate_name, version, query, item_type, limit, match_mode, &url,
77    )
78    .await
79}
80
81/// search-in-crate against a prebuilt all.html URL (wiremock).
82///
83/// # Errors
84///
85/// Propagates HTTP errors from [`HttpClient::get_html`].
86/// Maps non-success statuses via [`AppError::from_http_status`].
87/// Returns [`crate::error::ErrorKind::Parse`] on non-UTF-8 bodies or href join failures.
88/// Propagates parse errors from [`search_in_crate_from_html`].
89#[allow(clippy::too_many_arguments)] // domain triple + filter/limit/mode + url
90pub async fn search_in_crate_at(
91    http: &HttpClient,
92    crate_name: &CrateName,
93    version: &VersionArg,
94    query: &SearchQuery,
95    item_type: Option<ItemKind>,
96    limit: u32,
97    match_mode: MatchMode,
98    url: &Url,
99) -> AppResult<SearchInCrateData> {
100    let resp = http.get_html(url).await?;
101    if resp.status.as_u16() == 404 {
102        return Err(AppError::from_http_status(
103            404,
104            &format!("all.html not found for {crate_name}@{version}"),
105        ));
106    }
107    if !resp.status.is_success() {
108        return Err(AppError::from_http_status(
109            resp.status.as_u16(),
110            "rustdoc all.html",
111        ));
112    }
113    require_content_type_html(resp.content_type.as_deref())?;
114
115    // Move body into the CPU worker (one-shot memory: no clone for a second decode).
116    let body = resp.body;
117    let source_url = resp.final_url.to_string();
118    let cache_hit = resp.cache_hit;
119    let crate_name = crate_name.clone();
120    let version = version.clone();
121    let query = query.clone();
122    let cancel = http.cancel_flag().clone();
123    http.budget()
124        .run_cpu_bound(move || {
125            let text = decode_utf8(&body)?;
126            search_in_crate_from_html(
127                &text,
128                &crate_name,
129                &version,
130                &query,
131                item_type,
132                limit,
133                match_mode,
134                &source_url,
135                cache_hit,
136                &cancel,
137            )
138        })
139        .await
140}
141
142/// Resolve an all.html `href` against `base` (`source_url`), same-origin only.
143///
144/// - Relative and path-absolute hrefs use [`Url::join`] against `base` (never a
145///   hardcoded docs.rs host — SCRAPE-Q-001).
146/// - Absolute `http(s)://` hrefs are accepted only when **scheme + host** match
147///   `base` (host compared case-insensitively).
148/// - Off-origin absolute URLs and join/parse failures return [`None`] so callers
149///   can soft-skip the hit without failing the entire search (SCRAPE-S-001).
150#[must_use]
151pub fn resolve_hit_url(base: &Url, href: &str) -> Option<String> {
152    let href = href.trim();
153    if href.is_empty() {
154        return None;
155    }
156    let joined = if href.starts_with("http://") || href.starts_with("https://") {
157        Url::parse(href).ok()?
158    } else {
159        // Url::join handles relative segments and absolute paths (`/…`).
160        base.join(href).ok()?
161    };
162    if !same_origin(base, &joined) {
163        return None;
164    }
165    Some(joined.to_string())
166}
167
168/// True when `candidate` shares scheme and host with `base` (SCRAPE-S-001).
169fn same_origin(base: &Url, candidate: &Url) -> bool {
170    if candidate.scheme() != base.scheme() {
171        return false;
172    }
173    match (base.host_str(), candidate.host_str()) {
174        (Some(a), Some(b)) => a.eq_ignore_ascii_case(b),
175        (None, None) => true,
176        _ => false,
177    }
178}
179
180/// Join relative/absolute href against crate rustdoc base (same-origin).
181///
182/// Prefer [`resolve_hit_url`] in hit classification (soft-skip). This wrapper
183/// maps off-origin / join failure to [`crate::error::ErrorKind::Parse`] for callers that want
184/// a hard error.
185///
186/// # Errors
187///
188/// Returns [`crate::error::ErrorKind::Parse`] when the href is off-origin or not joinable.
189pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
190    resolve_hit_url(base, href).ok_or_else(|| AppError::of(ErrorDetail::HitJoinFailed))
191}
192
193/// One classified hit candidate before dedup/limit (index preserves document order).
194/// One classified hit candidate before dedup/limit: (index, name, kind, url, score).
195type ClassifiedHit = (usize, String, ItemKind, String, u8);
196
197/// Pure parse of all.html body for offline tests and CPU workers.
198///
199/// `base` must be the final all.html URL (or its directory). Relative and
200/// absolute-path hrefs resolve via [`resolve_hit_url`] against this URL — never
201/// against a hardcoded `docs.rs` host — so stdlib (`doc.rust-lang.org`) and
202/// mock origins keep correct hit URLs (SCRAPE-Q-001). Off-origin absolute hrefs
203/// are soft-skipped (SCRAPE-S-001).
204///
205/// The scan is sequential at every size (see module docs).
206/// Results are scored and sorted (exact leaf first) before applying `limit`.
207///
208/// # Errors
209///
210/// Currently infallible for join failures (soft-skip); reserved for future
211/// hard parse failures of the document itself.
212pub fn parse_all_html_hits(
213    html: &str,
214    base: &Url,
215    query: &SearchQuery,
216    item_type: Option<ItemKind>,
217    limit: usize,
218    match_mode: MatchMode,
219    cancel: &CancelFlag,
220) -> AppResult<Vec<SearchInCrateHit>> {
221    if limit == 0 {
222        return Ok(Vec::new());
223    }
224    cancel.check()?;
225    let document = Html::parse_document(html);
226    let q = query.as_str().trim().to_ascii_lowercase();
227
228    let candidates: Vec<(String, String)> = document
229        .select(&SEL_MAIN_CONTENT_A)
230        .filter_map(|a| {
231            let name = a.text().collect::<String>().trim().to_string();
232            if name.is_empty() {
233                return None;
234            }
235            let href = a.value().attr("href")?.to_string();
236            if href.is_empty() {
237                return None;
238            }
239            Some((name, href))
240        })
241        .collect();
242
243    filter_hits_sequential(candidates, base, &q, item_type, limit, match_mode, cancel)
244}
245
246pub(super) fn filter_hits_sequential(
247    candidates: Vec<(String, String)>,
248    base: &Url,
249    q: &str,
250    item_type: Option<ItemKind>,
251    limit: usize,
252    match_mode: MatchMode,
253    cancel: &CancelFlag,
254) -> AppResult<Vec<SearchInCrateHit>> {
255    // Pre-size for memory discipline (Rules Rust — gerenciamento de memória).
256    let mut rows: Vec<ClassifiedHit> = Vec::with_capacity(candidates.len());
257    for (idx, (name, href)) in candidates.into_iter().enumerate() {
258        cancel.check()?;
259        if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
260            rows.push(hit);
261        }
262    }
263    finalize_hits(rows, limit)
264}
265
266pub(super) fn finalize_hits(
267    mut rows: Vec<ClassifiedHit>,
268    limit: usize,
269) -> AppResult<Vec<SearchInCrateHit>> {
270    // Score ascending, then original document order for stability.
271    rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
272
273    // Two passes so the dedup key borrows from the rows instead of cloning every
274    // name into the set (GAP-ALLOC-001). The single-pass version had to clone,
275    // because the same `String` was needed by both the `HashSet` and the hit it
276    // moved into; splitting the decision from the construction removes that
277    // conflict without changing the result or its order.
278    //
279    // The borrow forces the scope: `seen` holds `&str` into `rows`, so it must
280    // die before `rows` is consumed below. The block is what guarantees that,
281    // and `keep` carries the decision out as plain bytes.
282    let mut kept = 0usize;
283    let keep: Vec<bool> = {
284        let mut seen: HashSet<(&str, ItemKind)> = HashSet::with_capacity(rows.len());
285        rows.iter()
286            .map(|(_idx, name, kind, _abs, _score)| {
287                // Stop admitting once the limit is met, mirroring the original
288                // `break`: rows past that point are neither emitted nor recorded.
289                if kept >= limit {
290                    return false;
291                }
292                let fresh = seen.insert((name.as_str(), *kind));
293                if fresh {
294                    kept += 1;
295                }
296                fresh
297            })
298            .collect()
299    };
300
301    // Capacity is the measured survivor count, not `limit.min(rows.len())`: the
302    // exact number is known now, so the vector never over-allocates for a query
303    // whose matches were mostly duplicates (SCRAPE-S-005 still bounds the scan).
304    let mut hits = Vec::with_capacity(kept);
305    for ((_idx, name, kind, abs, score), keep_row) in rows.into_iter().zip(keep) {
306        if !keep_row {
307            continue;
308        }
309        hits.push(SearchInCrateHit {
310            name,
311            kind: kind.as_str().to_string(),
312            url: abs,
313            score: Some(score),
314        });
315    }
316    Ok(hits)
317}
318
319pub(super) fn classify_hit_row(
320    idx: usize,
321    name: String,
322    href: &str,
323    base: &Url,
324    q: &str,
325    item_type: Option<ItemKind>,
326    match_mode: MatchMode,
327) -> AppResult<Option<ClassifiedHit>> {
328    let Some(kind) = ItemKind::from_href(href) else {
329        return Ok(None);
330    };
331    if let Some(filter) = item_type
332        && kind != filter
333    {
334        return Ok(None);
335    }
336    let Some(score) = match_mode.score(&name, q) else {
337        return Ok(None);
338    };
339    // Soft-skip off-origin / unjoinable hrefs — never abort the whole search (SCRAPE-S-001).
340    let Some(abs) = resolve_hit_url(base, href) else {
341        return Ok(None);
342    };
343    Ok(Some((idx, name, kind, abs, score)))
344}
345
346/// Build SearchInCrateData from HTML body without network (offline tests).
347///
348/// `--limit 0` is honoured: `emitted = 0`, `hits = []`, and `truncated` is true
349/// when the unfiltered total is greater than zero. Values above
350/// [`MAX_SEARCH_IN_CRATE_LIMIT`] are capped.
351///
352/// # Errors
353///
354/// Returns [`crate::error::ErrorKind::Parse`] when `source_url` is not a valid URL base.
355/// Propagates parse failures from [`parse_all_html_hits`].
356#[allow(clippy::too_many_arguments)] // pure offline path mirrors the async API surface
357pub fn search_in_crate_from_html(
358    html: &str,
359    crate_name: &CrateName,
360    version: &VersionArg,
361    query: &SearchQuery,
362    item_type: Option<ItemKind>,
363    limit: u32,
364    match_mode: MatchMode,
365    source_url: &str,
366    cache_hit: bool,
367    cancel: &CancelFlag,
368) -> AppResult<SearchInCrateData> {
369    let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
370    // Join base = final all.html URL (stdlib / mock / docs.rs). Never rebuild host.
371    let base = Url::parse(source_url)
372        .map_err(|e| AppError::of_with_source(ErrorDetail::HitBaseInvalid, e))?;
373    let all = parse_all_html_hits(
374        html,
375        &base,
376        query,
377        item_type,
378        usize::MAX,
379        match_mode,
380        cancel,
381    )?;
382    let total = all.len();
383    let hits: Vec<_> = all.into_iter().take(limit).collect();
384    let emitted = hits.len();
385    // Empty query: scores are 0 — omit score noise on wire for cleaner lists.
386    let hits = if query.as_str().trim().is_empty() {
387        hits.into_iter()
388            .map(|mut h| {
389                h.score = None;
390                h
391            })
392            .collect()
393    } else {
394        hits
395    };
396    Ok(SearchInCrateData {
397        crate_name: crate_name.as_str().to_string(),
398        query: query.as_str().to_string(),
399        version: version.as_str().to_string(),
400        item_type: item_type.map(|k| k.as_str().to_string()),
401        match_mode: match_mode.as_str().to_string(),
402        total,
403        emitted,
404        hits,
405        truncated: total > emitted,
406        source_url: source_url.to_string(),
407        cache_hit,
408    })
409}
410
411#[cfg(test)]
412mod threshold_tests {
413    use super::*;
414
415    fn candidates(count: usize) -> Vec<(String, String)> {
416        (0..count)
417            .map(|i| {
418                (
419                    format!("symbol_name_{i}"),
420                    format!("struct.symbol_name_{i}.html"),
421                )
422            })
423            .collect()
424    }
425
426    fn base() -> Url {
427        Url::parse("https://docs.rs/tokio/latest/tokio/all.html").expect("static test URL is valid")
428    }
429
430    #[test]
431    fn sequential_scan_keeps_every_matching_candidate() {
432        // Size sweep bracketing the removed rayon threshold (63/64): the scan
433        // must classify every candidate whatever the list size. This replaces
434        // the old sequential-vs-parallel equality test, which could only prove
435        // the two paths agreed, never that either was right.
436        let cancel = CancelFlag::new();
437        for count in [1usize, 63, 64, 500] {
438            let hits = filter_hits_sequential(
439                candidates(count),
440                &base(),
441                "symbol",
442                None,
443                usize::MAX,
444                MatchMode::Substring,
445                &cancel,
446            )
447            .expect("sequential classification succeeds");
448            assert_eq!(hits.len(), count, "hit count mismatch at count={count}");
449        }
450    }
451
452    #[test]
453    fn sequential_scan_applies_limit() {
454        let cancel = CancelFlag::new();
455        let hits = filter_hits_sequential(
456            candidates(500),
457            &base(),
458            "symbol",
459            None,
460            10,
461            MatchMode::Substring,
462            &cancel,
463        )
464        .expect("sequential classification succeeds");
465        assert_eq!(hits.len(), 10);
466    }
467}