1use std::collections::HashSet;
9use std::sync::LazyLock;
10
11use scraper::{Html, Selector};
12use url::Url;
13
14static SEL_MAIN_CONTENT_A: LazyLock<Selector> = LazyLock::new(|| {
16 Selector::parse("#main-content a")
17 .expect("hardcoded scraper selector '#main-content a' is valid by construction")
18});
19
20use crate::config::MAX_SEARCH_IN_CRATE_LIMIT;
21use crate::domain::{AllowedOrigin, CrateName, MatchMode, SearchQuery, VersionArg};
22use crate::error::{AppError, AppResult, ErrorDetail};
23use crate::http::{HttpClient, decode_utf8, require_content_type_html};
24use crate::item_kind::ItemKind;
25use crate::shutdown::CancelFlag;
26
27use super::types::{SearchInCrateData, SearchInCrateHit};
28use super::urls::all_html_url_on_origin;
29
30pub async fn search_in_crate(
37 http: &HttpClient,
38 crate_name: &CrateName,
39 version: &VersionArg,
40 query: &SearchQuery,
41 item_type: Option<ItemKind>,
42 limit: u32,
43 match_mode: MatchMode,
44) -> AppResult<SearchInCrateData> {
45 let origin = if crate_name.is_stdlib() {
47 AllowedOrigin::stdlib_docs_default()
48 } else {
49 AllowedOrigin::docs_rs_default()
50 };
51 search_in_crate_on_origin(
52 http, &origin, crate_name, version, query, item_type, limit, match_mode,
53 )
54 .await
55}
56
57#[allow(clippy::too_many_arguments)] pub async fn search_in_crate_on_origin(
65 http: &HttpClient,
66 origin: &AllowedOrigin,
67 crate_name: &CrateName,
68 version: &VersionArg,
69 query: &SearchQuery,
70 item_type: Option<ItemKind>,
71 limit: u32,
72 match_mode: MatchMode,
73) -> AppResult<SearchInCrateData> {
74 let url = all_html_url_on_origin(origin, crate_name, version)?;
75 search_in_crate_at(
76 http, crate_name, version, query, item_type, limit, match_mode, &url,
77 )
78 .await
79}
80
81#[allow(clippy::too_many_arguments)] pub async fn search_in_crate_at(
91 http: &HttpClient,
92 crate_name: &CrateName,
93 version: &VersionArg,
94 query: &SearchQuery,
95 item_type: Option<ItemKind>,
96 limit: u32,
97 match_mode: MatchMode,
98 url: &Url,
99) -> AppResult<SearchInCrateData> {
100 let resp = http.get_html(url).await?;
101 if resp.status.as_u16() == 404 {
102 return Err(AppError::from_http_status(
103 404,
104 &format!("all.html not found for {crate_name}@{version}"),
105 ));
106 }
107 if !resp.status.is_success() {
108 return Err(AppError::from_http_status(
109 resp.status.as_u16(),
110 "rustdoc all.html",
111 ));
112 }
113 require_content_type_html(resp.content_type.as_deref())?;
114
115 let body = resp.body;
117 let source_url = resp.final_url.to_string();
118 let cache_hit = resp.cache_hit;
119 let crate_name = crate_name.clone();
120 let version = version.clone();
121 let query = query.clone();
122 let cancel = http.cancel_flag().clone();
123 http.budget()
124 .run_cpu_bound(move || {
125 let text = decode_utf8(&body)?;
126 search_in_crate_from_html(
127 &text,
128 &crate_name,
129 &version,
130 &query,
131 item_type,
132 limit,
133 match_mode,
134 &source_url,
135 cache_hit,
136 &cancel,
137 )
138 })
139 .await
140}
141
142#[must_use]
151pub fn resolve_hit_url(base: &Url, href: &str) -> Option<String> {
152 let href = href.trim();
153 if href.is_empty() {
154 return None;
155 }
156 let joined = if href.starts_with("http://") || href.starts_with("https://") {
157 Url::parse(href).ok()?
158 } else {
159 base.join(href).ok()?
161 };
162 if !same_origin(base, &joined) {
163 return None;
164 }
165 Some(joined.to_string())
166}
167
168fn same_origin(base: &Url, candidate: &Url) -> bool {
170 if candidate.scheme() != base.scheme() {
171 return false;
172 }
173 match (base.host_str(), candidate.host_str()) {
174 (Some(a), Some(b)) => a.eq_ignore_ascii_case(b),
175 (None, None) => true,
176 _ => false,
177 }
178}
179
180pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
190 resolve_hit_url(base, href).ok_or_else(|| AppError::of(ErrorDetail::HitJoinFailed))
191}
192
193type ClassifiedHit = (usize, String, ItemKind, String, u8);
196
197pub fn parse_all_html_hits(
213 html: &str,
214 base: &Url,
215 query: &SearchQuery,
216 item_type: Option<ItemKind>,
217 limit: usize,
218 match_mode: MatchMode,
219 cancel: &CancelFlag,
220) -> AppResult<Vec<SearchInCrateHit>> {
221 if limit == 0 {
222 return Ok(Vec::new());
223 }
224 cancel.check()?;
225 let document = Html::parse_document(html);
226 let q = query.as_str().trim().to_ascii_lowercase();
227
228 let candidates: Vec<(String, String)> = document
229 .select(&SEL_MAIN_CONTENT_A)
230 .filter_map(|a| {
231 let name = a.text().collect::<String>().trim().to_string();
232 if name.is_empty() {
233 return None;
234 }
235 let href = a.value().attr("href")?.to_string();
236 if href.is_empty() {
237 return None;
238 }
239 Some((name, href))
240 })
241 .collect();
242
243 filter_hits_sequential(candidates, base, &q, item_type, limit, match_mode, cancel)
244}
245
246pub(super) fn filter_hits_sequential(
247 candidates: Vec<(String, String)>,
248 base: &Url,
249 q: &str,
250 item_type: Option<ItemKind>,
251 limit: usize,
252 match_mode: MatchMode,
253 cancel: &CancelFlag,
254) -> AppResult<Vec<SearchInCrateHit>> {
255 let mut rows: Vec<ClassifiedHit> = Vec::with_capacity(candidates.len());
257 for (idx, (name, href)) in candidates.into_iter().enumerate() {
258 cancel.check()?;
259 if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
260 rows.push(hit);
261 }
262 }
263 finalize_hits(rows, limit)
264}
265
266pub(super) fn finalize_hits(
267 mut rows: Vec<ClassifiedHit>,
268 limit: usize,
269) -> AppResult<Vec<SearchInCrateHit>> {
270 rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
272
273 let mut kept = 0usize;
283 let keep: Vec<bool> = {
284 let mut seen: HashSet<(&str, ItemKind)> = HashSet::with_capacity(rows.len());
285 rows.iter()
286 .map(|(_idx, name, kind, _abs, _score)| {
287 if kept >= limit {
290 return false;
291 }
292 let fresh = seen.insert((name.as_str(), *kind));
293 if fresh {
294 kept += 1;
295 }
296 fresh
297 })
298 .collect()
299 };
300
301 let mut hits = Vec::with_capacity(kept);
305 for ((_idx, name, kind, abs, score), keep_row) in rows.into_iter().zip(keep) {
306 if !keep_row {
307 continue;
308 }
309 hits.push(SearchInCrateHit {
310 name,
311 kind: kind.as_str().to_string(),
312 url: abs,
313 score: Some(score),
314 });
315 }
316 Ok(hits)
317}
318
319pub(super) fn classify_hit_row(
320 idx: usize,
321 name: String,
322 href: &str,
323 base: &Url,
324 q: &str,
325 item_type: Option<ItemKind>,
326 match_mode: MatchMode,
327) -> AppResult<Option<ClassifiedHit>> {
328 let Some(kind) = ItemKind::from_href(href) else {
329 return Ok(None);
330 };
331 if let Some(filter) = item_type
332 && kind != filter
333 {
334 return Ok(None);
335 }
336 let Some(score) = match_mode.score(&name, q) else {
337 return Ok(None);
338 };
339 let Some(abs) = resolve_hit_url(base, href) else {
341 return Ok(None);
342 };
343 Ok(Some((idx, name, kind, abs, score)))
344}
345
346#[allow(clippy::too_many_arguments)] pub fn search_in_crate_from_html(
358 html: &str,
359 crate_name: &CrateName,
360 version: &VersionArg,
361 query: &SearchQuery,
362 item_type: Option<ItemKind>,
363 limit: u32,
364 match_mode: MatchMode,
365 source_url: &str,
366 cache_hit: bool,
367 cancel: &CancelFlag,
368) -> AppResult<SearchInCrateData> {
369 let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
370 let base = Url::parse(source_url)
372 .map_err(|e| AppError::of_with_source(ErrorDetail::HitBaseInvalid, e))?;
373 let all = parse_all_html_hits(
374 html,
375 &base,
376 query,
377 item_type,
378 usize::MAX,
379 match_mode,
380 cancel,
381 )?;
382 let total = all.len();
383 let hits: Vec<_> = all.into_iter().take(limit).collect();
384 let emitted = hits.len();
385 let hits = if query.as_str().trim().is_empty() {
387 hits.into_iter()
388 .map(|mut h| {
389 h.score = None;
390 h
391 })
392 .collect()
393 } else {
394 hits
395 };
396 Ok(SearchInCrateData {
397 crate_name: crate_name.as_str().to_string(),
398 query: query.as_str().to_string(),
399 version: version.as_str().to_string(),
400 item_type: item_type.map(|k| k.as_str().to_string()),
401 match_mode: match_mode.as_str().to_string(),
402 total,
403 emitted,
404 hits,
405 truncated: total > emitted,
406 source_url: source_url.to_string(),
407 cache_hit,
408 })
409}
410
411#[cfg(test)]
412mod threshold_tests {
413 use super::*;
414
415 fn candidates(count: usize) -> Vec<(String, String)> {
416 (0..count)
417 .map(|i| {
418 (
419 format!("symbol_name_{i}"),
420 format!("struct.symbol_name_{i}.html"),
421 )
422 })
423 .collect()
424 }
425
426 fn base() -> Url {
427 Url::parse("https://docs.rs/tokio/latest/tokio/all.html").expect("static test URL is valid")
428 }
429
430 #[test]
431 fn sequential_scan_keeps_every_matching_candidate() {
432 let cancel = CancelFlag::new();
437 for count in [1usize, 63, 64, 500] {
438 let hits = filter_hits_sequential(
439 candidates(count),
440 &base(),
441 "symbol",
442 None,
443 usize::MAX,
444 MatchMode::Substring,
445 &cancel,
446 )
447 .expect("sequential classification succeeds");
448 assert_eq!(hits.len(), count, "hit count mismatch at count={count}");
449 }
450 }
451
452 #[test]
453 fn sequential_scan_applies_limit() {
454 let cancel = CancelFlag::new();
455 let hits = filter_hits_sequential(
456 candidates(500),
457 &base(),
458 "symbol",
459 None,
460 10,
461 MatchMode::Substring,
462 &cancel,
463 )
464 .expect("sequential classification succeeds");
465 assert_eq!(hits.len(), 10);
466 }
467}