1use std::collections::HashSet;
6use std::sync::LazyLock;
7
8use rayon::prelude::*;
9use scraper::{Html, Selector};
10use url::Url;
11
12static SEL_MAIN_CONTENT_A: LazyLock<Selector> = LazyLock::new(|| {
14 Selector::parse("#main-content a")
15 .expect("hardcoded scraper selector '#main-content a' is valid by construction")
16});
17
18use crate::config::{MAX_SEARCH_IN_CRATE_LIMIT, RAYON_HIT_THRESHOLD};
19use crate::domain::{AllowedOrigin, CrateName, MatchMode, SearchQuery, VersionArg};
20use crate::error::{AppError, AppResult, ErrorKind};
21use crate::http::{HttpClient, decode_utf8, require_content_type_html};
22use crate::item_kind::ItemKind;
23use crate::shutdown::CancelFlag;
24
25use super::types::{SearchInCrateData, SearchInCrateHit};
26use super::urls::all_html_url_on_origin;
27
28pub async fn search_in_crate(
35 http: &HttpClient,
36 crate_name: &CrateName,
37 version: &VersionArg,
38 query: &SearchQuery,
39 item_type: Option<ItemKind>,
40 limit: u32,
41 match_mode: MatchMode,
42) -> AppResult<SearchInCrateData> {
43 let origin = if crate_name.is_stdlib() {
45 AllowedOrigin::stdlib_docs_default()
46 } else {
47 AllowedOrigin::docs_rs_default()
48 };
49 search_in_crate_on_origin(
50 http,
51 &origin,
52 crate_name,
53 version,
54 query,
55 item_type,
56 limit,
57 match_mode,
58 )
59 .await
60}
61
62#[allow(clippy::too_many_arguments)] pub async fn search_in_crate_on_origin(
70 http: &HttpClient,
71 origin: &AllowedOrigin,
72 crate_name: &CrateName,
73 version: &VersionArg,
74 query: &SearchQuery,
75 item_type: Option<ItemKind>,
76 limit: u32,
77 match_mode: MatchMode,
78) -> AppResult<SearchInCrateData> {
79 let url = all_html_url_on_origin(origin, crate_name, version)?;
80 search_in_crate_at(
81 http, crate_name, version, query, item_type, limit, match_mode, &url,
82 )
83 .await
84}
85
86#[allow(clippy::too_many_arguments)] pub async fn search_in_crate_at(
96 http: &HttpClient,
97 crate_name: &CrateName,
98 version: &VersionArg,
99 query: &SearchQuery,
100 item_type: Option<ItemKind>,
101 limit: u32,
102 match_mode: MatchMode,
103 url: &Url,
104) -> AppResult<SearchInCrateData> {
105 let resp = http.get_html(url).await?;
106 if resp.status.as_u16() == 404 {
107 return Err(AppError::from_http_status(
108 404,
109 &format!("all.html not found for {crate_name}@{version}"),
110 ));
111 }
112 if !resp.status.is_success() {
113 return Err(AppError::from_http_status(
114 resp.status.as_u16(),
115 "rustdoc all.html",
116 ));
117 }
118 require_content_type_html(resp.content_type.as_deref())?;
119
120 let body = resp.body;
122 let source_url = resp.final_url.to_string();
123 let cache_hit = resp.cache_hit;
124 let crate_name = crate_name.clone();
125 let version = version.clone();
126 let query = query.clone();
127 let cancel = http.cancel_flag().clone();
128 http.budget()
129 .run_cpu_bound(move || {
130 let text = decode_utf8(&body)?;
131 search_in_crate_from_html(
132 &text,
133 &crate_name,
134 &version,
135 &query,
136 item_type,
137 limit,
138 match_mode,
139 &source_url,
140 cache_hit,
141 &cancel,
142 )
143 })
144 .await
145}
146
147#[must_use]
156pub fn resolve_hit_url(base: &Url, href: &str) -> Option<String> {
157 let href = href.trim();
158 if href.is_empty() {
159 return None;
160 }
161 let joined = if href.starts_with("http://") || href.starts_with("https://") {
162 Url::parse(href).ok()?
163 } else {
164 base.join(href).ok()?
166 };
167 if !same_origin(base, &joined) {
168 return None;
169 }
170 Some(joined.to_string())
171}
172
173fn same_origin(base: &Url, candidate: &Url) -> bool {
175 if candidate.scheme() != base.scheme() {
176 return false;
177 }
178 match (base.host_str(), candidate.host_str()) {
179 (Some(a), Some(b)) => a.eq_ignore_ascii_case(b),
180 (None, None) => true,
181 _ => false,
182 }
183}
184
185pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
195 resolve_hit_url(base, href).ok_or_else(|| {
196 AppError::new(
197 ErrorKind::Parse,
198 "failed to join href (off-origin or invalid against source_url base)",
199 )
200 })
201}
202
203type ClassifiedHit = (usize, String, ItemKind, String, u8);
206
207pub fn parse_all_html_hits(
223 html: &str,
224 base: &Url,
225 query: &SearchQuery,
226 item_type: Option<ItemKind>,
227 limit: usize,
228 match_mode: MatchMode,
229 cancel: &CancelFlag,
230) -> AppResult<Vec<SearchInCrateHit>> {
231 if limit == 0 {
232 return Ok(Vec::new());
233 }
234 cancel.check()?;
235 let document = Html::parse_document(html);
236 let q = query.as_str().trim().to_ascii_lowercase();
237
238 let candidates: Vec<(String, String)> = document
239 .select(&SEL_MAIN_CONTENT_A)
240 .filter_map(|a| {
241 let name = a.text().collect::<String>().trim().to_string();
242 if name.is_empty() {
243 return None;
244 }
245 let href = a.value().attr("href")?.to_string();
246 if href.is_empty() {
247 return None;
248 }
249 Some((name, href))
250 })
251 .collect();
252
253 if candidates.len() < RAYON_HIT_THRESHOLD {
254 return filter_hits_sequential(candidates, base, &q, item_type, limit, match_mode, cancel);
255 }
256 filter_hits_parallel(candidates, base, &q, item_type, limit, match_mode, cancel)
257}
258
259pub(super) fn filter_hits_sequential(
260 candidates: Vec<(String, String)>,
261 base: &Url,
262 q: &str,
263 item_type: Option<ItemKind>,
264 limit: usize,
265 match_mode: MatchMode,
266 cancel: &CancelFlag,
267) -> AppResult<Vec<SearchInCrateHit>> {
268 let mut rows: Vec<ClassifiedHit> = Vec::with_capacity(candidates.len());
270 for (idx, (name, href)) in candidates.into_iter().enumerate() {
271 cancel.check()?;
272 if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
273 rows.push(hit);
274 }
275 }
276 finalize_hits(rows, limit)
277}
278
279pub(super) fn filter_hits_parallel(
280 candidates: Vec<(String, String)>,
281 base: &Url,
282 q: &str,
283 item_type: Option<ItemKind>,
284 limit: usize,
285 match_mode: MatchMode,
286 cancel: &CancelFlag,
287) -> AppResult<Vec<SearchInCrateHit>> {
288 let cancel = cancel.clone();
290 let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
291 .into_par_iter()
292 .enumerate()
293 .map(|(idx, (name, href))| {
294 cancel.check()?;
296 classify_hit_row(idx, name, &href, base, q, item_type, match_mode)
297 })
298 .collect();
299 let rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
300 finalize_hits(rows, limit)
301}
302
303pub(super) fn finalize_hits(mut rows: Vec<ClassifiedHit>, limit: usize) -> AppResult<Vec<SearchInCrateHit>> {
304 rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
306 let mut hits = Vec::with_capacity(limit.min(rows.len()));
308 let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
309 for (_idx, name, kind, abs, score) in rows {
310 if !seen.insert((name.clone(), kind)) {
311 continue;
312 }
313 hits.push(SearchInCrateHit {
314 name,
315 kind: kind.as_str().to_string(),
316 url: abs,
317 score: Some(score),
318 });
319 if hits.len() >= limit {
320 break;
321 }
322 }
323 Ok(hits)
324}
325
326pub(super) fn classify_hit_row(
327 idx: usize,
328 name: String,
329 href: &str,
330 base: &Url,
331 q: &str,
332 item_type: Option<ItemKind>,
333 match_mode: MatchMode,
334) -> AppResult<Option<ClassifiedHit>> {
335 let Some(kind) = ItemKind::from_href(href) else {
336 return Ok(None);
337 };
338 if let Some(filter) = item_type
339 && kind != filter
340 {
341 return Ok(None);
342 }
343 let Some(score) = match_mode.score(&name, q) else {
344 return Ok(None);
345 };
346 let Some(abs) = resolve_hit_url(base, href) else {
348 return Ok(None);
349 };
350 Ok(Some((idx, name, kind, abs, score)))
351}
352
353#[allow(clippy::too_many_arguments)] pub fn search_in_crate_from_html(
365 html: &str,
366 crate_name: &CrateName,
367 version: &VersionArg,
368 query: &SearchQuery,
369 item_type: Option<ItemKind>,
370 limit: u32,
371 match_mode: MatchMode,
372 source_url: &str,
373 cache_hit: bool,
374 cancel: &CancelFlag,
375) -> AppResult<SearchInCrateData> {
376 let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
377 let base = Url::parse(source_url).map_err(|e| {
379 AppError::with_source(ErrorKind::Parse, "invalid source_url for hit join base", e)
380 })?;
381 let all = parse_all_html_hits(html, &base, query, item_type, usize::MAX, match_mode, cancel)?;
382 let total = all.len();
383 let hits: Vec<_> = all.into_iter().take(limit).collect();
384 let emitted = hits.len();
385 let hits = if query.as_str().trim().is_empty() {
387 hits.into_iter()
388 .map(|mut h| {
389 h.score = None;
390 h
391 })
392 .collect()
393 } else {
394 hits
395 };
396 Ok(SearchInCrateData {
397 crate_name: crate_name.as_str().to_string(),
398 query: query.as_str().to_string(),
399 version: version.as_str().to_string(),
400 item_type: item_type.map(|k| k.as_str().to_string()),
401 match_mode: match_mode.as_str().to_string(),
402 total,
403 emitted,
404 hits,
405 truncated: total > emitted,
406 source_url: source_url.to_string(),
407 cache_hit,
408 })
409}