use std::collections::HashSet;
use std::sync::LazyLock;
use rayon::prelude::*;
use scraper::{Html, Selector};
use url::Url;
static SEL_MAIN_CONTENT_A: LazyLock<Selector> = LazyLock::new(|| {
Selector::parse("#main-content a")
.expect("hardcoded scraper selector '#main-content a' is valid by construction")
});
use crate::config::{MAX_SEARCH_IN_CRATE_LIMIT, RAYON_HIT_THRESHOLD};
use crate::domain::{AllowedOrigin, CrateName, MatchMode, SearchQuery, VersionArg};
use crate::error::{AppError, AppResult, ErrorKind};
use crate::http::{HttpClient, decode_utf8, require_content_type_html};
use crate::item_kind::ItemKind;
use crate::shutdown::CancelFlag;
use super::types::{SearchInCrateData, SearchInCrateHit};
use super::urls::all_html_url_on_origin;
pub async fn search_in_crate(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
query: &SearchQuery,
item_type: Option<ItemKind>,
limit: u32,
match_mode: MatchMode,
) -> AppResult<SearchInCrateData> {
let origin = if crate_name.is_stdlib() {
AllowedOrigin::stdlib_docs_default()
} else {
AllowedOrigin::docs_rs_default()
};
search_in_crate_on_origin(
http,
&origin,
crate_name,
version,
query,
item_type,
limit,
match_mode,
)
.await
}
#[allow(clippy::too_many_arguments)] pub async fn search_in_crate_on_origin(
http: &HttpClient,
origin: &AllowedOrigin,
crate_name: &CrateName,
version: &VersionArg,
query: &SearchQuery,
item_type: Option<ItemKind>,
limit: u32,
match_mode: MatchMode,
) -> AppResult<SearchInCrateData> {
let url = all_html_url_on_origin(origin, crate_name, version)?;
search_in_crate_at(
http, crate_name, version, query, item_type, limit, match_mode, &url,
)
.await
}
#[allow(clippy::too_many_arguments)] pub async fn search_in_crate_at(
http: &HttpClient,
crate_name: &CrateName,
version: &VersionArg,
query: &SearchQuery,
item_type: Option<ItemKind>,
limit: u32,
match_mode: MatchMode,
url: &Url,
) -> AppResult<SearchInCrateData> {
let resp = http.get_html(url).await?;
if resp.status.as_u16() == 404 {
return Err(AppError::from_http_status(
404,
&format!("all.html not found for {crate_name}@{version}"),
));
}
if !resp.status.is_success() {
return Err(AppError::from_http_status(
resp.status.as_u16(),
"rustdoc all.html",
));
}
require_content_type_html(resp.content_type.as_deref())?;
let body = resp.body;
let source_url = resp.final_url.to_string();
let cache_hit = resp.cache_hit;
let crate_name = crate_name.clone();
let version = version.clone();
let query = query.clone();
let cancel = http.cancel_flag().clone();
http.budget()
.run_cpu_bound(move || {
let text = decode_utf8(&body)?;
search_in_crate_from_html(
&text,
&crate_name,
&version,
&query,
item_type,
limit,
match_mode,
&source_url,
cache_hit,
&cancel,
)
})
.await
}
#[must_use]
pub fn resolve_hit_url(base: &Url, href: &str) -> Option<String> {
let href = href.trim();
if href.is_empty() {
return None;
}
let joined = if href.starts_with("http://") || href.starts_with("https://") {
Url::parse(href).ok()?
} else {
base.join(href).ok()?
};
if !same_origin(base, &joined) {
return None;
}
Some(joined.to_string())
}
fn same_origin(base: &Url, candidate: &Url) -> bool {
if candidate.scheme() != base.scheme() {
return false;
}
match (base.host_str(), candidate.host_str()) {
(Some(a), Some(b)) => a.eq_ignore_ascii_case(b),
(None, None) => true,
_ => false,
}
}
pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
resolve_hit_url(base, href).ok_or_else(|| {
AppError::new(
ErrorKind::Parse,
"failed to join href (off-origin or invalid against source_url base)",
)
})
}
type ClassifiedHit = (usize, String, ItemKind, String, u8);
pub fn parse_all_html_hits(
html: &str,
base: &Url,
query: &SearchQuery,
item_type: Option<ItemKind>,
limit: usize,
match_mode: MatchMode,
cancel: &CancelFlag,
) -> AppResult<Vec<SearchInCrateHit>> {
if limit == 0 {
return Ok(Vec::new());
}
cancel.check()?;
let document = Html::parse_document(html);
let q = query.as_str().trim().to_ascii_lowercase();
let candidates: Vec<(String, String)> = document
.select(&SEL_MAIN_CONTENT_A)
.filter_map(|a| {
let name = a.text().collect::<String>().trim().to_string();
if name.is_empty() {
return None;
}
let href = a.value().attr("href")?.to_string();
if href.is_empty() {
return None;
}
Some((name, href))
})
.collect();
if candidates.len() < RAYON_HIT_THRESHOLD {
return filter_hits_sequential(candidates, base, &q, item_type, limit, match_mode, cancel);
}
filter_hits_parallel(candidates, base, &q, item_type, limit, match_mode, cancel)
}
pub(super) fn filter_hits_sequential(
candidates: Vec<(String, String)>,
base: &Url,
q: &str,
item_type: Option<ItemKind>,
limit: usize,
match_mode: MatchMode,
cancel: &CancelFlag,
) -> AppResult<Vec<SearchInCrateHit>> {
let mut rows: Vec<ClassifiedHit> = Vec::with_capacity(candidates.len());
for (idx, (name, href)) in candidates.into_iter().enumerate() {
cancel.check()?;
if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
rows.push(hit);
}
}
finalize_hits(rows, limit)
}
pub(super) fn filter_hits_parallel(
candidates: Vec<(String, String)>,
base: &Url,
q: &str,
item_type: Option<ItemKind>,
limit: usize,
match_mode: MatchMode,
cancel: &CancelFlag,
) -> AppResult<Vec<SearchInCrateHit>> {
let cancel = cancel.clone();
let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
.into_par_iter()
.enumerate()
.map(|(idx, (name, href))| {
cancel.check()?;
classify_hit_row(idx, name, &href, base, q, item_type, match_mode)
})
.collect();
let rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
finalize_hits(rows, limit)
}
pub(super) fn finalize_hits(mut rows: Vec<ClassifiedHit>, limit: usize) -> AppResult<Vec<SearchInCrateHit>> {
rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
let mut hits = Vec::with_capacity(limit.min(rows.len()));
let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
for (_idx, name, kind, abs, score) in rows {
if !seen.insert((name.clone(), kind)) {
continue;
}
hits.push(SearchInCrateHit {
name,
kind: kind.as_str().to_string(),
url: abs,
score: Some(score),
});
if hits.len() >= limit {
break;
}
}
Ok(hits)
}
pub(super) fn classify_hit_row(
idx: usize,
name: String,
href: &str,
base: &Url,
q: &str,
item_type: Option<ItemKind>,
match_mode: MatchMode,
) -> AppResult<Option<ClassifiedHit>> {
let Some(kind) = ItemKind::from_href(href) else {
return Ok(None);
};
if let Some(filter) = item_type
&& kind != filter
{
return Ok(None);
}
let Some(score) = match_mode.score(&name, q) else {
return Ok(None);
};
let Some(abs) = resolve_hit_url(base, href) else {
return Ok(None);
};
Ok(Some((idx, name, kind, abs, score)))
}
#[allow(clippy::too_many_arguments)] pub fn search_in_crate_from_html(
html: &str,
crate_name: &CrateName,
version: &VersionArg,
query: &SearchQuery,
item_type: Option<ItemKind>,
limit: u32,
match_mode: MatchMode,
source_url: &str,
cache_hit: bool,
cancel: &CancelFlag,
) -> AppResult<SearchInCrateData> {
let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
let base = Url::parse(source_url).map_err(|e| {
AppError::with_source(ErrorKind::Parse, "invalid source_url for hit join base", e)
})?;
let all = parse_all_html_hits(html, &base, query, item_type, usize::MAX, match_mode, cancel)?;
let total = all.len();
let hits: Vec<_> = all.into_iter().take(limit).collect();
let emitted = hits.len();
let hits = if query.as_str().trim().is_empty() {
hits.into_iter()
.map(|mut h| {
h.score = None;
h
})
.collect()
} else {
hits
};
Ok(SearchInCrateData {
crate_name: crate_name.as_str().to_string(),
query: query.as_str().to_string(),
version: version.as_str().to_string(),
item_type: item_type.map(|k| k.as_str().to_string()),
match_mode: match_mode.as_str().to_string(),
total,
emitted,
hits,
truncated: total > emitted,
source_url: source_url.to_string(),
cache_hit,
})
}