1use std::collections::HashSet;
27use std::sync::LazyLock;
28
29use rayon::prelude::*;
30use regex::Regex;
31use scraper::{Html, Selector};
32use serde::{Deserialize, Serialize};
33use url::Url;
34
35use crate::config::{
36 HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS, MAX_SEARCH_IN_CRATE_LIMIT, is_stdlib_crate,
37 stdlib_channel,
38};
39use crate::domain::MatchMode;
40use crate::error::{AppError, AppResult, ErrorKind};
41use crate::http::{HttpClient, content_type_looks_html, decode_utf8};
42use crate::item_kind::{ItemKind, rustc_crate_name};
43
44#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct ReadmeData {
47 pub crate_name: String,
49 pub version: String,
51 #[serde(skip_serializing_if = "Option::is_none")]
53 pub resolved_version: Option<String>,
54 pub markdown: String,
56 pub empty: bool,
58 pub truncated: bool,
60 pub source_url: String,
62 pub cache_hit: bool,
64}
65
66#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct GetItemData {
69 pub crate_name: String,
71 pub item_type: String,
73 pub item_path: String,
75 pub item_name: String,
77 pub version: String,
79 #[serde(skip_serializing_if = "Option::is_none")]
81 pub resolved_version: Option<String>,
82 pub markdown: String,
84 pub empty: bool,
86 pub truncated: bool,
88 pub source_url: String,
90 pub title: String,
92 pub cache_hit: bool,
94}
95
96#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
98pub struct SearchInCrateHit {
99 pub name: String,
101 pub kind: String,
103 pub url: String,
105 #[serde(skip_serializing_if = "Option::is_none")]
107 pub score: Option<u8>,
108}
109
110#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct SearchInCrateData {
113 pub crate_name: String,
115 pub query: String,
117 pub version: String,
119 #[serde(skip_serializing_if = "Option::is_none")]
121 pub item_type: Option<String>,
122 pub match_mode: String,
124 pub total: usize,
126 pub emitted: usize,
128 pub hits: Vec<SearchInCrateHit>,
130 pub truncated: bool,
132 pub source_url: String,
134 pub cache_hit: bool,
136}
137
138fn default_docs_origin(crate_name: &str) -> String {
139 if is_stdlib_crate(crate_name) {
140 format!("https://{HOST_DOC_RUST_LANG_ORG}")
141 } else {
142 format!("https://{HOST_DOCS_RS}")
143 }
144}
145
146pub fn readme_url(crate_name: &str, version: &str) -> AppResult<Url> {
152 readme_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
153}
154
155pub fn readme_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
161 let origin = origin.trim_end_matches('/');
162 let s = if is_stdlib_crate(crate_name) {
163 let channel = stdlib_channel(version);
165 format!("{origin}/{channel}/{crate_name}/index.html")
166 } else {
167 let rustc = rustc_crate_name(crate_name);
168 format!("{origin}/{crate_name}/{version}/{rustc}/index.html")
169 };
170 Url::parse(&s).map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid readme URL", e))
171}
172
173pub fn get_item_url(
180 crate_name: &str,
181 version: &str,
182 kind: ItemKind,
183 segments: &[String],
184) -> AppResult<Url> {
185 get_item_url_on_origin(
186 &default_docs_origin(crate_name),
187 crate_name,
188 version,
189 kind,
190 segments,
191 )
192}
193
194fn is_method_path(kind: ItemKind, segs: &[String]) -> bool {
197 if segs.len() < 2 {
198 return false;
199 }
200 if kind != ItemKind::Fn {
201 return false;
202 }
203 let parent = segs[segs.len() - 2].as_str();
204 let method = segs[segs.len() - 1].as_str();
205 let parent_type = parent
206 .chars()
207 .next()
208 .is_some_and(|c| c.is_ascii_uppercase());
209 let method_fn = method
210 .chars()
211 .next()
212 .is_some_and(|c| c.is_ascii_lowercase() || c == '_');
213 parent_type && method_fn
214}
215
216const METHOD_PARENT_KINDS: &[ItemKind] = &[
218 ItemKind::Struct,
219 ItemKind::Enum,
220 ItemKind::Trait,
221 ItemKind::Type,
222 ItemKind::Union,
223];
224
225pub fn get_item_url_on_origin(
235 origin: &str,
236 crate_name: &str,
237 version: &str,
238 kind: ItemKind,
239 segments: &[String],
240) -> AppResult<Url> {
241 get_item_url_on_origin_with_parent_kind(origin, crate_name, version, kind, segments, None)
242}
243
244pub fn get_item_url_on_origin_with_parent_kind(
246 origin: &str,
247 crate_name: &str,
248 version: &str,
249 kind: ItemKind,
250 segments: &[String],
251 parent_kind_override: Option<ItemKind>,
252) -> AppResult<Url> {
253 let origin = origin.trim_end_matches('/');
254 let rustc_root = rustc_crate_name(crate_name);
255 let segs: Vec<String> = {
256 let mut s = segments.to_vec();
257 if let Some(first) = s.first() {
258 let f = first.as_str();
259 let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
260 if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
261 s.remove(0);
262 }
263 }
264 s
265 };
266
267 if is_method_path(kind, &segs) {
269 let method_name = segs.last().expect("is_method_path requires >=2 segs").clone();
270 let parent_name = segs[segs.len() - 2].clone();
271 let parent_kind = parent_kind_override.unwrap_or(ItemKind::Struct);
272 let mod_parts: Vec<String> = if segs.len() == 2 {
273 if is_stdlib_crate(crate_name) {
274 Vec::new()
275 } else {
276 vec![rustc_root.clone()]
277 }
278 } else {
279 let mut m = if is_stdlib_crate(crate_name) {
280 Vec::new()
281 } else {
282 vec![rustc_root.clone()]
283 };
284 for p in &segs[..segs.len() - 2] {
285 m.push(rustc_crate_name(p));
286 }
287 m
288 };
289 let url_str = if is_stdlib_crate(crate_name) {
290 let channel = stdlib_channel(version);
291 if mod_parts.is_empty() {
292 format!(
293 "{origin}/{channel}/{crate_name}/{}.{}.html#method.{method_name}",
294 parent_kind.file_prefix(),
295 parent_name
296 )
297 } else {
298 format!(
299 "{origin}/{channel}/{crate_name}/{}/{}.{}.html#method.{method_name}",
300 mod_parts.join("/"),
301 parent_kind.file_prefix(),
302 parent_name
303 )
304 }
305 } else if mod_parts.is_empty() {
306 format!(
307 "{origin}/{crate_name}/{version}/{}/{}.{}.html#method.{method_name}",
308 rustc_root,
309 parent_kind.file_prefix(),
310 parent_name
311 )
312 } else {
313 format!(
314 "{origin}/{crate_name}/{version}/{}/{}.{}.html#method.{method_name}",
315 mod_parts.join("/"),
316 parent_kind.file_prefix(),
317 parent_name
318 )
319 };
320 return Url::parse(&url_str)
321 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e));
322 }
323
324 let url_str = if is_stdlib_crate(crate_name) {
325 let channel = stdlib_channel(version);
326 if kind == ItemKind::Module {
327 let mut parts: Vec<String> = Vec::new();
328 for p in &segs {
329 parts.push(rustc_crate_name(p));
330 }
331 if parts.is_empty() {
332 format!("{origin}/{channel}/{crate_name}/index.html")
333 } else {
334 format!(
335 "{origin}/{channel}/{crate_name}/{}/index.html",
336 parts.join("/")
337 )
338 }
339 } else {
340 if segs.is_empty() {
341 return Err(AppError::new(
342 ErrorKind::InvalidInput,
343 "item path missing item name",
344 ));
345 }
346 let item_name = segs.last().ok_or_else(|| {
347 AppError::new(ErrorKind::InvalidInput, "item path missing item name")
348 })?;
349 let mod_parts: Vec<String> = segs[..segs.len().saturating_sub(1)]
350 .iter()
351 .map(|p| rustc_crate_name(p))
352 .collect();
353 if mod_parts.is_empty() {
354 format!(
355 "{origin}/{channel}/{crate_name}/{}.{}.html",
356 kind.file_prefix(),
357 item_name
358 )
359 } else {
360 format!(
361 "{origin}/{channel}/{crate_name}/{}/{}.{}.html",
362 mod_parts.join("/"),
363 kind.file_prefix(),
364 item_name
365 )
366 }
367 }
368 } else if kind == ItemKind::Module {
369 let mut parts: Vec<String> = vec![rustc_root];
370 for p in &segs {
371 parts.push(rustc_crate_name(p));
372 }
373 format!(
374 "{origin}/{crate_name}/{version}/{}/index.html",
375 parts.join("/")
376 )
377 } else {
378 if segs.is_empty() {
379 return Err(AppError::new(
380 ErrorKind::InvalidInput,
381 "item path missing item name",
382 ));
383 }
384 let item_name = segs
385 .last()
386 .ok_or_else(|| AppError::new(ErrorKind::InvalidInput, "item path missing item name"))?;
387 let mod_parts: Vec<String> = if segs.len() == 1 {
388 vec![rustc_root]
389 } else {
390 let mut m = vec![rustc_root];
391 for p in &segs[..segs.len() - 1] {
392 m.push(rustc_crate_name(p));
393 }
394 m
395 };
396 format!(
397 "{origin}/{crate_name}/{version}/{}/{}.{}.html",
398 mod_parts.join("/"),
399 kind.file_prefix(),
400 item_name
401 )
402 };
403
404 Url::parse(&url_str)
405 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e))
406}
407
408pub fn all_html_url(crate_name: &str, version: &str) -> AppResult<Url> {
414 all_html_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
415}
416
417pub fn all_html_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
423 let origin = origin.trim_end_matches('/');
424 let s = if is_stdlib_crate(crate_name) {
425 let channel = stdlib_channel(version);
426 format!("{origin}/{channel}/{crate_name}/all.html")
427 } else {
428 let rustc = rustc_crate_name(crate_name);
429 format!("{origin}/{crate_name}/{version}/{rustc}/all.html")
430 };
431 Url::parse(&s)
432 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html URL", e))
433}
434
435#[cfg_attr(not(test), allow(dead_code))]
436fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
437 extract_resolved_version_for_crate(final_url, requested, None)
438}
439
440fn extract_resolved_version_for_crate(
445 final_url: &Url,
446 requested: &str,
447 crate_name: Option<&str>,
448) -> Option<String> {
449 let host = final_url.host_str().unwrap_or("");
450 let segs: Vec<&str> = final_url
451 .path_segments()?
452 .filter(|s| !s.is_empty())
453 .collect();
454 if segs.is_empty() {
455 return None;
456 }
457
458 let is_stdlib_host = host.eq_ignore_ascii_case(HOST_DOC_RUST_LANG_ORG)
459 || crate_name.is_some_and(is_stdlib_crate)
460 || segs
461 .get(1)
462 .is_some_and(|s| matches!(*s, "std" | "core" | "alloc"));
463
464 if is_stdlib_host {
465 let channel = segs.first()?;
467 if matches!(*channel, "std" | "core" | "alloc") {
468 return None;
469 }
470 return Some((*channel).to_string());
471 }
472
473 let ver = segs.get(1)?;
475 if *ver == "latest" {
476 return None;
477 }
478 if requested == "latest" || *ver != requested {
479 return Some((*ver).to_string());
480 }
481 Some((*ver).to_string())
482}
483
484fn scrape_docs_rs_version_from_html(html: &str, crate_name: &str) -> Option<String> {
488 let crate_name = crate_name.trim();
489 if crate_name.is_empty() || is_stdlib_crate(crate_name) {
490 return None;
491 }
492 let escaped = regex::escape(crate_name);
494 let re_path = Regex::new(&format!(
496 r"(?i)(?:https?://docs\.rs/|/){escaped}/([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?)\b"
497 ))
498 .ok()?;
499 if let Some(c) = re_path.captures(html) {
500 return c.get(1).map(|m| m.as_str().to_string());
501 }
502 let re_title = Regex::new(&format!(
504 r"(?i)<title>[^<]*?\b{escaped}\b[^<]*?\b([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?)\b"
505 ))
506 .ok()?;
507 re_title
508 .captures(html)
509 .and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
510}
511
512static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
514 Regex::new(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
515 .expect("hardcoded on* handler strip regex is valid by construction")
516});
517
518static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
520 Regex::new(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
521 .expect("hardcoded javascript: URL strip regex is valid by construction")
522});
523
524pub fn sanitize_html_fragment(html: &str) -> String {
530 let fragment = Html::parse_fragment(html);
531 let drop_sel = Selector::parse("script, style, noscript")
532 .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction");
533 if let Some(root) = fragment.tree.root().children().next() {
535 let _ = root;
536 }
537 let mut out = String::with_capacity(html.len());
541 out.push_str(html);
542 for el in fragment.select(&drop_sel) {
543 let tag_html = el.html();
544 out = out.replace(&tag_html, "");
545 }
546 out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
548 out = RE_JS_URLS.replace_all(&out, "").into_owned();
550 out
551}
552
553fn html_to_markdown(html: &str) -> AppResult<String> {
554 let cleaned = sanitize_html_fragment(html);
555 htmd::convert(&cleaned).map_err(|e| {
556 AppError::new(
557 ErrorKind::Parse,
558 format!("HTML to Markdown conversion failed: {e}"),
559 )
560 })
561}
562
563fn first_inner_html(document: &Html, selectors: &[&str]) -> Option<String> {
564 for sel in selectors {
565 if let Ok(s) = Selector::parse(sel)
566 && let Some(n) = document.select(&s).next()
567 {
568 let inner = n.inner_html();
569 if !inner.trim().is_empty() {
570 return Some(inner);
571 }
572 }
573 }
574 None
575}
576
577pub async fn fetch_readme(
584 http: &HttpClient,
585 crate_name: &str,
586 version: &str,
587) -> AppResult<ReadmeData> {
588 fetch_readme_on_origin(
589 http,
590 &format!("https://{HOST_DOCS_RS}"),
591 crate_name,
592 version,
593 )
594 .await
595}
596
597pub async fn fetch_readme_on_origin(
604 http: &HttpClient,
605 origin: &str,
606 crate_name: &str,
607 version: &str,
608) -> AppResult<ReadmeData> {
609 let url = readme_url_on_origin(origin, crate_name, version)?;
610 fetch_readme_at(http, crate_name, version, &url).await
611}
612
613pub async fn fetch_readme_at(
621 http: &HttpClient,
622 crate_name: &str,
623 version: &str,
624 url: &Url,
625) -> AppResult<ReadmeData> {
626 let resp = http.get_html(url).await?;
627 if resp.status.as_u16() == 404 {
628 return Err(AppError::from_http_status(
629 404,
630 &format!("crate or version not found: {crate_name}@{version}"),
631 ));
632 }
633 if !resp.status.is_success() {
634 return Err(AppError::from_http_status(
635 resp.status.as_u16(),
636 "docs.rs readme",
637 ));
638 }
639 if resp.content_type.is_some() && !content_type_looks_html(resp.content_type.as_deref()) {
640 tracing::warn!(content_type = ?resp.content_type, "unexpected Content-Type from docs.rs");
641 }
642
643 let body = resp.body.clone();
644 let (markdown, empty) = http
645 .budget()
646 .run_cpu_bound(move || {
647 let text = decode_utf8(&body)?;
648 extract_readme_markdown_from_html(&text)
649 })
650 .await?;
651 let mut resolved =
652 extract_resolved_version_for_crate(&resp.final_url, version, Some(crate_name));
653 if resolved.is_none() {
654 if let Ok(text) = decode_utf8(&resp.body) {
656 resolved = scrape_docs_rs_version_from_html(&text, crate_name);
657 }
658 }
659 Ok(ReadmeData {
660 crate_name: crate_name.to_string(),
661 version: version.to_string(),
662 resolved_version: resolved,
663 markdown,
664 empty,
665 truncated: false,
666 source_url: resp.final_url.to_string(),
667 cache_hit: resp.cache_hit,
668 })
669}
670
671pub async fn fetch_item(
678 http: &HttpClient,
679 crate_name: &str,
680 version: &str,
681 kind: ItemKind,
682 segments: &[String],
683) -> AppResult<GetItemData> {
684 fetch_item_on_origin(
685 http,
686 &format!("https://{HOST_DOCS_RS}"),
687 crate_name,
688 version,
689 kind,
690 segments,
691 )
692 .await
693}
694pub async fn fetch_item_on_origin(
704 http: &HttpClient,
705 origin: &str,
706 crate_name: &str,
707 version: &str,
708 kind: ItemKind,
709 segments: &[String],
710) -> AppResult<GetItemData> {
711 let rustc_root = rustc_crate_name(crate_name);
713 let segs_for_detect: Vec<String> = {
714 let mut s = segments.to_vec();
715 if let Some(first) = s.first() {
716 let f = first.as_str();
717 let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
718 if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
719 s.remove(0);
720 }
721 }
722 s
723 };
724
725 if is_method_path(kind, &segs_for_detect) {
726 let mut last_err: Option<AppError> = None;
727 for parent_kind in METHOD_PARENT_KINDS {
728 let url = get_item_url_on_origin_with_parent_kind(
729 origin,
730 crate_name,
731 version,
732 kind,
733 segments,
734 Some(*parent_kind),
735 )?;
736 match fetch_item_at(http, crate_name, version, kind, segments, &url).await {
737 Ok(data) => return Ok(data),
738 Err(e) if e.kind() == ErrorKind::NotFound => {
739 last_err = Some(e);
740 continue;
741 }
742 Err(e) => return Err(e),
743 }
744 }
745 return Err(last_err.unwrap_or_else(|| {
746 AppError::new(ErrorKind::NotFound, "method parent type page not found")
747 }));
748 }
749
750 let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
751 fetch_item_at(http, crate_name, version, kind, segments, &url).await
752}
753
754pub async fn fetch_item_at(
762 http: &HttpClient,
763 crate_name: &str,
764 version: &str,
765 kind: ItemKind,
766 segments: &[String],
767 url: &Url,
768) -> AppResult<GetItemData> {
769 let mut fetch_url = url.clone();
771 let fragment = fetch_url.fragment().map(str::to_string);
772 fetch_url.set_fragment(None);
773
774 let resp = http.get_html(&fetch_url).await?;
775 if resp.status.as_u16() == 404 {
776 return Err(AppError::from_http_status(
777 404,
778 &format!("item not found at {url}"),
779 ));
780 }
781 if !resp.status.is_success() {
782 return Err(AppError::from_http_status(
783 resp.status.as_u16(),
784 "docs.rs get-item",
785 ));
786 }
787
788 let method_anchor = fragment
789 .as_deref()
790 .and_then(|f| f.strip_prefix("method."))
791 .map(str::to_string);
792 let body = resp.body.clone();
793 let method_anchor_cpu = method_anchor.clone();
794 let (markdown, empty) = http
795 .budget()
796 .run_cpu_bound(move || {
797 let text = decode_utf8(&body)?;
798 if let Some(ref m) = method_anchor_cpu {
799 extract_method_markdown_from_html(&text, m)
800 } else {
801 extract_item_markdown_from_html(&text)
802 }
803 })
804 .await?;
805 let item_path = segments.join("::");
806 let item_name = segments
807 .last()
808 .cloned()
809 .unwrap_or_else(|| item_path.clone());
810 let title = format!("{item_path} ({})", kind.as_str());
811 let mut resolved =
812 extract_resolved_version_for_crate(&resp.final_url, version, Some(crate_name));
813 if resolved.is_none() {
814 if let Ok(text) = decode_utf8(&resp.body) {
815 resolved = scrape_docs_rs_version_from_html(&text, crate_name);
816 }
817 }
818
819 let mut source_url = resp.final_url.clone();
820 if let Some(f) = fragment {
821 source_url.set_fragment(Some(&f));
822 }
823
824 Ok(GetItemData {
825 crate_name: crate_name.to_string(),
826 item_type: kind.as_str().to_string(),
827 item_path,
828 item_name,
829 version: version.to_string(),
830 resolved_version: resolved,
831 markdown,
832 empty,
833 truncated: false,
834 source_url: source_url.to_string(),
835 title,
836 cache_hit: resp.cache_hit,
837 })
838}
839pub async fn search_in_crate(
846 http: &HttpClient,
847 crate_name: &str,
848 version: &str,
849 query: &str,
850 item_type: Option<ItemKind>,
851 limit: u32,
852 match_mode: MatchMode,
853) -> AppResult<SearchInCrateData> {
854 search_in_crate_on_origin(
855 http,
856 &format!("https://{HOST_DOCS_RS}"),
857 crate_name,
858 version,
859 query,
860 item_type,
861 limit,
862 match_mode,
863 )
864 .await
865}
866
867#[allow(clippy::too_many_arguments)]
874pub async fn search_in_crate_on_origin(
875 http: &HttpClient,
876 origin: &str,
877 crate_name: &str,
878 version: &str,
879 query: &str,
880 item_type: Option<ItemKind>,
881 limit: u32,
882 match_mode: MatchMode,
883) -> AppResult<SearchInCrateData> {
884 let url = all_html_url_on_origin(origin, crate_name, version)?;
885 search_in_crate_at(
886 http,
887 crate_name,
888 version,
889 query,
890 item_type,
891 limit,
892 match_mode,
893 &url,
894 )
895 .await
896}
897
898#[allow(clippy::too_many_arguments)]
907pub async fn search_in_crate_at(
908 http: &HttpClient,
909 crate_name: &str,
910 version: &str,
911 query: &str,
912 item_type: Option<ItemKind>,
913 limit: u32,
914 match_mode: MatchMode,
915 url: &Url,
916) -> AppResult<SearchInCrateData> {
917 let resp = http.get_html(url).await?;
918 if resp.status.as_u16() == 404 {
919 return Err(AppError::from_http_status(
920 404,
921 &format!("all.html not found for {crate_name}@{version}"),
922 ));
923 }
924 if !resp.status.is_success() {
925 return Err(AppError::from_http_status(
926 resp.status.as_u16(),
927 "docs.rs all.html",
928 ));
929 }
930
931 let body = resp.body.clone();
932 let source_url = resp.final_url.to_string();
933 let cache_hit = resp.cache_hit;
934 let crate_name = crate_name.to_string();
935 let version = version.to_string();
936 let query = query.to_string();
937 http.budget()
938 .run_cpu_bound(move || {
939 let text = decode_utf8(&body)?;
940 search_in_crate_from_html(
941 &text,
942 &crate_name,
943 &version,
944 &query,
945 item_type,
946 limit,
947 match_mode,
948 &source_url,
949 cache_hit,
950 )
951 })
952 .await
953}
954
955
956pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
962 if href.starts_with("http://") || href.starts_with("https://") {
963 return Ok(href.to_string());
964 }
965 if href.starts_with('/') {
966 let joined = Url::parse(&format!("https://docs.rs{href}")).map_err(|e| {
967 AppError::with_source(ErrorKind::Parse, "failed to join absolute path href", e)
968 })?;
969 return Ok(joined.to_string());
970 }
971 base.join(href)
972 .map(|u| u.to_string())
973 .map_err(|e| AppError::with_source(ErrorKind::Parse, "failed to join relative href", e))
974}
975
976const RAYON_HIT_THRESHOLD: usize = 64;
978
979type ClassifiedHit = (usize, String, ItemKind, String, u8);
982
983pub fn parse_all_html_hits(
993 html: &str,
994 crate_name: &str,
995 version: &str,
996 query: &str,
997 item_type: Option<ItemKind>,
998 limit: usize,
999 match_mode: MatchMode,
1000) -> AppResult<Vec<SearchInCrateHit>> {
1001 if limit == 0 {
1002 return Ok(Vec::new());
1003 }
1004 let document = Html::parse_document(html);
1005 let a_sel = Selector::parse("#main-content a")
1006 .expect("hardcoded scraper selector '#main-content a' is valid by construction");
1007 let rustc = rustc_crate_name(crate_name);
1008 let base = Url::parse(&format!(
1009 "https://{HOST_DOCS_RS}/{crate_name}/{version}/{rustc}/"
1010 ))
1011 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html base", e))?;
1012 let q = query.trim().to_ascii_lowercase();
1013
1014 let candidates: Vec<(String, String)> = document
1015 .select(&a_sel)
1016 .filter_map(|a| {
1017 let name = a.text().collect::<String>().trim().to_string();
1018 if name.is_empty() {
1019 return None;
1020 }
1021 let href = a.value().attr("href")?.to_string();
1022 if href.is_empty() {
1023 return None;
1024 }
1025 Some((name, href))
1026 })
1027 .collect();
1028
1029 if candidates.len() < RAYON_HIT_THRESHOLD {
1030 return filter_hits_sequential(candidates, &base, &q, item_type, limit, match_mode);
1031 }
1032 filter_hits_parallel(candidates, &base, &q, item_type, limit, match_mode)
1033}
1034
1035fn filter_hits_sequential(
1036 candidates: Vec<(String, String)>,
1037 base: &Url,
1038 q: &str,
1039 item_type: Option<ItemKind>,
1040 limit: usize,
1041 match_mode: MatchMode,
1042) -> AppResult<Vec<SearchInCrateHit>> {
1043 let mut rows: Vec<ClassifiedHit> = Vec::new();
1044 for (idx, (name, href)) in candidates.into_iter().enumerate() {
1045 if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
1046 rows.push(hit);
1047 }
1048 }
1049 finalize_hits(rows, limit)
1050}
1051
1052fn filter_hits_parallel(
1053 candidates: Vec<(String, String)>,
1054 base: &Url,
1055 q: &str,
1056 item_type: Option<ItemKind>,
1057 limit: usize,
1058 match_mode: MatchMode,
1059) -> AppResult<Vec<SearchInCrateHit>> {
1060 let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
1061 .into_par_iter()
1062 .enumerate()
1063 .map(|(idx, (name, href))| classify_hit_row(idx, name, &href, base, q, item_type, match_mode))
1064 .collect();
1065 let rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
1066 finalize_hits(rows, limit)
1067}
1068
1069fn finalize_hits(mut rows: Vec<ClassifiedHit>, limit: usize) -> AppResult<Vec<SearchInCrateHit>> {
1070 rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
1072 let mut hits = Vec::with_capacity(limit.min(256));
1073 let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
1074 for (_idx, name, kind, abs, score) in rows {
1075 if !seen.insert((name.clone(), kind)) {
1076 continue;
1077 }
1078 hits.push(SearchInCrateHit {
1079 name,
1080 kind: kind.as_str().to_string(),
1081 url: abs,
1082 score: Some(score),
1083 });
1084 if hits.len() >= limit {
1085 break;
1086 }
1087 }
1088 Ok(hits)
1089}
1090
1091fn classify_hit_row(
1092 idx: usize,
1093 name: String,
1094 href: &str,
1095 base: &Url,
1096 q: &str,
1097 item_type: Option<ItemKind>,
1098 match_mode: MatchMode,
1099) -> AppResult<Option<ClassifiedHit>> {
1100 let Some(kind) = ItemKind::from_href(href) else {
1101 return Ok(None);
1102 };
1103 if let Some(filter) = item_type
1104 && kind != filter
1105 {
1106 return Ok(None);
1107 }
1108 let Some(score) = match_mode.score(&name, q) else {
1109 return Ok(None);
1110 };
1111 let abs = join_href(base, href)?;
1112 Ok(Some((idx, name, kind, abs, score)))
1113}
1114pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
1120 let document = Html::parse_document(html);
1121 let content = first_inner_html(
1122 &document,
1123 &[
1124 ".rustdoc .docblock",
1125 "#main-content .docblock",
1126 ".docblock",
1127 ".rustdoc-main .item-decl",
1128 ".item-decl",
1129 ],
1130 );
1131 match content {
1132 Some(h) => Ok((html_to_markdown(&h)?, false)),
1133 None => Ok((String::new(), true)),
1134 }
1135}
1136
1137pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
1143 let document = Html::parse_document(html);
1144 let content = first_inner_html(&document, &["#main-content", ".item-decl", ".docblock"])
1145 .unwrap_or_default();
1146 let empty = content.trim().is_empty();
1147 if empty {
1148 Ok((String::new(), true))
1149 } else {
1150 Ok((html_to_markdown(&content)?, false))
1151 }
1152}
1153
1154pub fn extract_method_markdown_from_html(html: &str, method_name: &str) -> AppResult<(String, bool)> {
1170 let needle_dq = format!("id=\"method.{method_name}\"");
1171 let needle_sq = format!("id='method.{method_name}'");
1172 if !html.contains(&needle_dq) && !html.contains(&needle_sq) {
1173 return extract_item_markdown_from_html(html);
1174 }
1175 let document = Html::parse_document(html);
1177 if let Ok(sel) = Selector::parse("#main-content") {
1179 if let Some(main) = document.select(&sel).next() {
1180 let frag = main.html();
1181 if frag.contains(&needle_dq) || frag.contains(&needle_sq) {
1182 let md = html_to_markdown(&frag)?;
1183 let empty = md.trim().is_empty();
1184 return Ok((md, empty));
1185 }
1186 }
1187 }
1188 extract_item_markdown_from_html(html)
1189}
1190
1191#[allow(clippy::too_many_arguments)]
1202pub fn search_in_crate_from_html(
1203 html: &str,
1204 crate_name: &str,
1205 version: &str,
1206 query: &str,
1207 item_type: Option<ItemKind>,
1208 limit: u32,
1209 match_mode: MatchMode,
1210 source_url: &str,
1211 cache_hit: bool,
1212) -> AppResult<SearchInCrateData> {
1213 let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
1214 let all = parse_all_html_hits(
1215 html,
1216 crate_name,
1217 version,
1218 query,
1219 item_type,
1220 usize::MAX,
1221 match_mode,
1222 )?;
1223 let total = all.len();
1224 let hits: Vec<_> = all.into_iter().take(limit).collect();
1225 let emitted = hits.len();
1226 let hits = if query.trim().is_empty() {
1228 hits.into_iter()
1229 .map(|mut h| {
1230 h.score = None;
1231 h
1232 })
1233 .collect()
1234 } else {
1235 hits
1236 };
1237 Ok(SearchInCrateData {
1238 crate_name: crate_name.to_string(),
1239 query: query.to_string(),
1240 version: version.to_string(),
1241 item_type: item_type.map(|k| k.as_str().to_string()),
1242 match_mode: match_mode.as_str().to_string(),
1243 total,
1244 emitted,
1245 hits,
1246 truncated: total > emitted,
1247 source_url: source_url.to_string(),
1248 cache_hit,
1249 })
1250}
1251#[cfg(test)]
1252mod tests {
1253 use super::*;
1254
1255 #[test]
1256 fn readme_url_hyphen() {
1257 let u = readme_url("async-trait", "latest").unwrap();
1258 assert_eq!(
1259 u.as_str(),
1260 "https://docs.rs/async-trait/latest/async_trait/index.html"
1261 );
1262 }
1263
1264 #[test]
1265 fn get_item_nested() {
1266 let segs = vec!["tokio".into(), "runtime".into(), "Runtime".into()];
1267 let u = get_item_url("tokio", "latest", ItemKind::Struct, &segs).unwrap();
1268 assert!(u.as_str().contains("/tokio/runtime/struct.Runtime.html"));
1269 }
1270
1271 #[test]
1272 fn get_item_without_crate_prefix() {
1273 let segs = vec!["Parser".into()];
1274 let u = get_item_url("clap", "latest", ItemKind::Trait, &segs).unwrap();
1275 assert!(
1276 u.as_str().ends_with("/clap/trait.Parser.html")
1277 || u.path().ends_with("/clap/trait.Parser.html")
1278 );
1279 }
1280
1281 #[test]
1282 fn get_item_keeps_single_segment_equal_to_crate_name() {
1283 let segs = vec!["async_trait".into()];
1285 let u = get_item_url("async-trait", "latest", ItemKind::Attribute, &segs).unwrap();
1286 assert!(
1287 u.as_str()
1288 .ends_with("/async-trait/latest/async_trait/attr.async_trait.html"),
1289 "url={u}"
1290 );
1291 }
1292
1293 #[test]
1294 fn get_item_module_single_segment_crate_name_is_root() {
1295 let segs = vec!["async_trait".into()];
1296 let u = get_item_url("async-trait", "latest", ItemKind::Module, &segs).unwrap();
1297 assert!(
1298 u.as_str()
1299 .ends_with("/async-trait/latest/async_trait/index.html"),
1300 "url={u}"
1301 );
1302 }
1303
1304 #[test]
1305 fn join_relative() {
1306 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1307 let j = join_href(&base, "de/trait.Deserialize.html").unwrap();
1308 assert!(j.contains("/serde/de/trait.Deserialize.html"));
1309 }
1310
1311 #[test]
1312 fn join_absolute_path_and_fragment() {
1313 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1314 let j = join_href(&base, "/serde/latest/serde/struct.Error.html").unwrap();
1315 assert!(j.starts_with("https://docs.rs/serde/"));
1316 let f = join_href(&base, "struct.Error.html#method.to_string").unwrap();
1317 assert!(f.contains("#method.to_string"));
1318 }
1319
1320 #[test]
1321 fn sanitize_strips_script() {
1322 let raw = r#"<div>ok<script>alert(1)</script><p onclick="x()">hi</p></div>"#;
1323 let s = sanitize_html_fragment(raw);
1324 assert!(!s.contains("script"));
1325 assert!(!s.contains("onclick"));
1326 assert!(s.contains("hi") || s.contains("ok"));
1327 }
1328
1329 #[test]
1330 fn readme_fallback_item_decl_converts() {
1331 let html = r#"<!DOCTYPE html><html><body class="rustdoc-main">
1332 <div class="item-decl"><pre>pub use foo;</pre></div>
1333 </body></html>"#;
1334 let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1335 assert!(!empty);
1336 assert!(!md.trim().is_empty());
1337 }
1338
1339 #[test]
1340 fn all_html_order_stable() {
1341 let html = r#"<!DOCTYPE html><html><body>
1342 <div id="main-content">
1343 <a href="struct.A.html">A</a>
1344 <a href="trait.B.html">B</a>
1345 <a href="fn.c.html">c</a>
1346 <a href="union.U.html">U</a>
1347 <a href="attr.x.html">x</a>
1348 <a href="derive.D.html">D</a>
1349 </div></body></html>"#;
1350 let hits = parse_all_html_hits(html, "demo", "1.0.0", "", None, 100, crate::domain::MatchMode::Prefix).unwrap();
1351 assert_eq!(hits.len(), 6);
1352 assert_eq!(hits[0].name, "A");
1353 assert_eq!(hits[0].kind, "struct");
1354 assert_eq!(hits[3].kind, "union");
1355 assert_eq!(hits[4].kind, "attribute");
1356 assert_eq!(hits[5].kind, "derive");
1357 }
1358
1359 #[test]
1360 fn std_core_alloc_urls() {
1361 let u = readme_url("std", "latest").unwrap();
1362 assert_eq!(
1363 u.as_str(),
1364 "https://doc.rust-lang.org/stable/std/index.html"
1365 );
1366 let u = get_item_url(
1367 "std",
1368 "latest",
1369 ItemKind::Struct,
1370 &["option".into(), "Option".into()],
1371 )
1372 .unwrap();
1373 assert_eq!(
1374 u.as_str(),
1375 "https://doc.rust-lang.org/stable/std/option/struct.Option.html"
1376 );
1377 let a = all_html_url("core", "nightly").unwrap();
1378 assert_eq!(
1379 a.as_str(),
1380 "https://doc.rust-lang.org/nightly/core/all.html"
1381 );
1382 }
1383
1384 #[test]
1385 fn extract_item_from_fixture() {
1386 let html = include_str!("../tests/fixtures/docs_rs/get_item_main.html");
1387 let (md, empty) = extract_item_markdown_from_html(html).unwrap();
1388 assert!(!empty);
1389 assert!(md.contains("Runtime") || md.contains("Tokio") || md.contains("runtime"));
1390 }
1391
1392 #[test]
1393 fn extract_readme_primary_docblock_fixture() {
1394 let html = include_str!("../tests/fixtures/docs_rs/readme_docblock.html");
1395 let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1396 assert!(!empty);
1397 assert!(!md.to_ascii_lowercase().contains("alert"));
1398 }
1399
1400 #[test]
1401 fn search_in_crate_from_html_limit_and_filter() {
1402 let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1403 let data = search_in_crate_from_html(
1404 html,
1405 "demo",
1406 "1.0.0",
1407 "",
1408 Some(ItemKind::Struct),
1409 1,
1410 MatchMode::Prefix,
1411 "https://docs.rs/demo/1.0.0/demo/all.html",
1412 false,
1413 )
1414 .unwrap();
1415 assert_eq!(data.total, 1);
1416 assert_eq!(data.emitted, 1);
1417 assert_eq!(data.hits[0].kind, "struct");
1418 assert!(!data.truncated);
1419 }
1420
1421 #[test]
1422 fn search_in_crate_truncated_when_limit_cuts_hits() {
1423 let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1424 let data = search_in_crate_from_html(
1425 html,
1426 "demo",
1427 "1.0.0",
1428 "",
1429 None,
1430 2,
1431 MatchMode::Prefix,
1432 "https://docs.rs/demo/1.0.0/demo/all.html",
1433 false,
1434 )
1435 .unwrap();
1436 assert!(data.total > 2, "fixture must have more than 2 hits");
1437 assert_eq!(data.emitted, 2);
1438 assert!(data.truncated);
1439 let full = search_in_crate_from_html(
1440 html,
1441 "demo",
1442 "1.0.0",
1443 "",
1444 None,
1445 1000,
1446 MatchMode::Prefix,
1447 "https://docs.rs/demo/1.0.0/demo/all.html",
1448 false,
1449 )
1450 .unwrap();
1451 assert_eq!(full.total, full.emitted);
1452 assert!(!full.truncated);
1453 }
1454
1455 #[test]
1456 fn search_in_crate_limit_zero_emits_nothing() {
1457 let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1458 let data = search_in_crate_from_html(
1459 html,
1460 "demo",
1461 "1.0.0",
1462 "",
1463 None,
1464 0,
1465 MatchMode::Prefix,
1466 "https://docs.rs/demo/1.0.0/demo/all.html",
1467 false,
1468 )
1469 .unwrap();
1470 assert!(data.total > 0, "fixture must have hits for total");
1471 assert_eq!(data.emitted, 0);
1472 assert!(data.hits.is_empty());
1473 assert!(data.truncated);
1474 }
1475
1476 #[test]
1477 fn serialize_exact_match_beats_deserializer() {
1478 let html = r#"<!DOCTYPE html><html><body>
1479 <div id="main-content">
1480 <a href="trait.Deserialize.html">Deserialize</a>
1481 <a href="trait.Serialize.html">Serialize</a>
1482 <a href="struct.Deserializer.html">Deserializer</a>
1483 </div></body></html>"#;
1484 let hits = parse_all_html_hits(
1485 html,
1486 "serde",
1487 "1.0.0",
1488 "Serialize",
1489 Some(ItemKind::Trait),
1490 10,
1491 MatchMode::Prefix,
1492 )
1493 .unwrap();
1494 assert_eq!(hits[0].name, "Serialize");
1495 assert!(hits.iter().all(|h| h.name == "Serialize"));
1496 }
1497
1498 #[test]
1499 fn method_url_uses_anchor() {
1500 let segs = vec!["runtime".into(), "Runtime".into(), "new".into()];
1501 let u = get_item_url("tokio", "latest", ItemKind::Fn, &segs).unwrap();
1502 assert!(
1503 u.as_str().contains("struct.Runtime.html#method.new"),
1504 "url={u}"
1505 );
1506 }
1507
1508 #[test]
1509 fn stdlib_resolved_version_is_channel() {
1510 let u = Url::parse("https://doc.rust-lang.org/stable/std/index.html").unwrap();
1511 assert_eq!(
1512 extract_resolved_version_for_crate(&u, "latest", Some("std")).as_deref(),
1513 Some("stable")
1514 );
1515 }
1516
1517 #[test]
1518 fn resolved_version_from_url() {
1519 let u = Url::parse("https://docs.rs/serde/1.0.210/serde/index.html").unwrap();
1520 assert_eq!(
1521 extract_resolved_version(&u, "latest").as_deref(),
1522 Some("1.0.210")
1523 );
1524 let u2 = Url::parse("https://docs.rs/serde/latest/serde/index.html").unwrap();
1525 assert!(extract_resolved_version(&u2, "latest").is_none());
1526 }
1527
1528 #[test]
1529 fn scrape_version_prefers_target_crate_not_deps() {
1530 let html = r#"
1531 <title>tokio - Rust</title>
1532 <a href="https://docs.rs/bytes/1.2.0/bytes/">bytes</a>
1533 <a href="/tokio/1.53.0/tokio/runtime/index.html">runtime</a>
1534 <a href="https://docs.rs/mio/0.8.0/mio/">mio</a>
1535 "#;
1536 assert_eq!(
1537 scrape_docs_rs_version_from_html(html, "tokio").as_deref(),
1538 Some("1.53.0")
1539 );
1540 let html2 = r#"<title>x</title><a href="https://docs.rs/bytes/1.2.0/bytes/">b</a>"#;
1542 assert_eq!(scrape_docs_rs_version_from_html(html2, "tokio"), None);
1543 }
1544
1545 #[test]
1546 fn module_url_template() {
1547 let segs = vec!["serde".into(), "de".into()];
1548 let u = get_item_url("serde", "latest", ItemKind::Module, &segs).unwrap();
1549 assert!(u.as_str().ends_with("/serde/de/index.html"));
1550 }
1551
1552 #[test]
1553 fn empty_html_readme() {
1554 let (md, empty) = extract_readme_markdown_from_html("<html><body></body></html>").unwrap();
1555 assert!(empty);
1556 assert!(md.is_empty());
1557 }
1558
1559 #[test]
1560 fn get_item_empty_segments_errors() {
1561 let err = get_item_url("clap", "latest", ItemKind::Struct, &[]).unwrap_err();
1562 assert_eq!(err.kind(), ErrorKind::InvalidInput);
1563 }
1564
1565 #[test]
1566 fn join_full_http_url_passthrough() {
1567 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1568 let full = "https://docs.rs/serde/latest/serde/struct.Error.html";
1569 assert_eq!(join_href(&base, full).unwrap(), full);
1570 let http = "http://example.invalid/x.html";
1571 assert_eq!(join_href(&base, http).unwrap(), http);
1572 }
1573
1574 #[test]
1575 fn origin_builders_trim_trailing_slash() {
1576 let u = readme_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1577 assert_eq!(u.as_str(), "https://docs.rs/demo/1.0.0/demo/index.html");
1578 let a = all_html_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1579 assert_eq!(a.as_str(), "https://docs.rs/demo/1.0.0/demo/all.html");
1580 }
1581
1582 #[test]
1583 fn empty_item_html_extract() {
1584 let (md, empty) = extract_item_markdown_from_html("<html><body></body></html>").unwrap();
1585 assert!(empty);
1586 assert!(md.is_empty());
1587 }
1588}