1use std::collections::HashSet;
27use std::sync::LazyLock;
28
29use rayon::prelude::*;
30use regex::Regex;
31use scraper::{Html, Selector};
32use serde::{Deserialize, Serialize};
33use url::Url;
34
35use crate::config::{
36 HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS, MAX_SEARCH_IN_CRATE_LIMIT, is_stdlib_crate,
37 stdlib_channel,
38};
39use crate::error::{AppError, AppResult, ErrorKind};
40use crate::http::{HttpClient, content_type_looks_html, decode_utf8};
41use crate::item_kind::{ItemKind, rustc_crate_name};
42
43#[derive(Debug, Clone, Serialize, Deserialize)]
45pub struct ReadmeData {
46 pub crate_name: String,
48 pub version: String,
50 #[serde(skip_serializing_if = "Option::is_none")]
52 pub resolved_version: Option<String>,
53 pub markdown: String,
55 pub empty: bool,
57 pub truncated: bool,
59 pub source_url: String,
61}
62
63#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct GetItemData {
66 pub crate_name: String,
68 pub item_type: String,
70 pub item_path: String,
72 pub version: String,
74 #[serde(skip_serializing_if = "Option::is_none")]
76 pub resolved_version: Option<String>,
77 pub markdown: String,
79 pub empty: bool,
81 pub truncated: bool,
83 pub source_url: String,
85 pub title: String,
87}
88
89#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
91pub struct SearchInCrateHit {
92 pub name: String,
94 pub kind: String,
96 pub url: String,
98}
99
100#[derive(Debug, Clone, Serialize, Deserialize)]
102pub struct SearchInCrateData {
103 pub crate_name: String,
105 pub query: String,
107 pub version: String,
109 pub total: usize,
111 pub emitted: usize,
113 pub hits: Vec<SearchInCrateHit>,
115 pub truncated: bool,
117 pub source_url: String,
119}
120
121fn default_docs_origin(crate_name: &str) -> String {
122 if is_stdlib_crate(crate_name) {
123 format!("https://{HOST_DOC_RUST_LANG_ORG}")
124 } else {
125 format!("https://{HOST_DOCS_RS}")
126 }
127}
128
129pub fn readme_url(crate_name: &str, version: &str) -> AppResult<Url> {
135 readme_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
136}
137
138pub fn readme_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
144 let origin = origin.trim_end_matches('/');
145 let s = if is_stdlib_crate(crate_name) {
146 let channel = stdlib_channel(version);
148 format!("{origin}/{channel}/{crate_name}/index.html")
149 } else {
150 let rustc = rustc_crate_name(crate_name);
151 format!("{origin}/{crate_name}/{version}/{rustc}/index.html")
152 };
153 Url::parse(&s).map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid readme URL", e))
154}
155
156pub fn get_item_url(
163 crate_name: &str,
164 version: &str,
165 kind: ItemKind,
166 segments: &[String],
167) -> AppResult<Url> {
168 get_item_url_on_origin(
169 &default_docs_origin(crate_name),
170 crate_name,
171 version,
172 kind,
173 segments,
174 )
175}
176
177pub fn get_item_url_on_origin(
184 origin: &str,
185 crate_name: &str,
186 version: &str,
187 kind: ItemKind,
188 segments: &[String],
189) -> AppResult<Url> {
190 let origin = origin.trim_end_matches('/');
191 let rustc_root = rustc_crate_name(crate_name);
192 let segs: Vec<String> = {
197 let mut s = segments.to_vec();
198 if let Some(first) = s.first() {
199 let f = first.as_str();
200 let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
201 if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
202 s.remove(0);
203 }
204 }
205 s
206 };
207
208 let url_str = if is_stdlib_crate(crate_name) {
209 let channel = stdlib_channel(version);
211 if kind == ItemKind::Module {
212 let mut parts: Vec<String> = Vec::new();
213 for p in &segs {
214 parts.push(rustc_crate_name(p));
215 }
216 if parts.is_empty() {
217 format!("{origin}/{channel}/{crate_name}/index.html")
218 } else {
219 format!(
220 "{origin}/{channel}/{crate_name}/{}/index.html",
221 parts.join("/")
222 )
223 }
224 } else {
225 if segs.is_empty() {
226 return Err(AppError::new(
227 ErrorKind::InvalidInput,
228 "item path missing item name",
229 ));
230 }
231 let item_name = segs.last().ok_or_else(|| {
232 AppError::new(ErrorKind::InvalidInput, "item path missing item name")
233 })?;
234 let mod_parts: Vec<String> = segs[..segs.len().saturating_sub(1)]
235 .iter()
236 .map(|p| rustc_crate_name(p))
237 .collect();
238 if mod_parts.is_empty() {
239 format!(
240 "{origin}/{channel}/{crate_name}/{}.{}.html",
241 kind.file_prefix(),
242 item_name
243 )
244 } else {
245 format!(
246 "{origin}/{channel}/{crate_name}/{}/{}.{}.html",
247 mod_parts.join("/"),
248 kind.file_prefix(),
249 item_name
250 )
251 }
252 }
253 } else if kind == ItemKind::Module {
254 let mut parts: Vec<String> = vec![rustc_root];
255 for p in &segs {
256 parts.push(rustc_crate_name(p));
257 }
258 format!(
259 "{origin}/{crate_name}/{version}/{}/index.html",
260 parts.join("/")
261 )
262 } else {
263 if segs.is_empty() {
264 return Err(AppError::new(
265 ErrorKind::InvalidInput,
266 "item path missing item name",
267 ));
268 }
269 let item_name = segs
270 .last()
271 .ok_or_else(|| AppError::new(ErrorKind::InvalidInput, "item path missing item name"))?;
272 let mod_parts: Vec<String> = if segs.len() == 1 {
273 vec![rustc_root]
274 } else {
275 let mut m = vec![rustc_root];
276 for p in &segs[..segs.len() - 1] {
277 m.push(rustc_crate_name(p));
278 }
279 m
280 };
281 format!(
282 "{origin}/{crate_name}/{version}/{}/{}.{}.html",
283 mod_parts.join("/"),
284 kind.file_prefix(),
285 item_name
286 )
287 };
288
289 Url::parse(&url_str)
290 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e))
291}
292
293pub fn all_html_url(crate_name: &str, version: &str) -> AppResult<Url> {
299 all_html_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
300}
301
302pub fn all_html_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
308 let origin = origin.trim_end_matches('/');
309 let s = if is_stdlib_crate(crate_name) {
310 let channel = stdlib_channel(version);
311 format!("{origin}/{channel}/{crate_name}/all.html")
312 } else {
313 let rustc = rustc_crate_name(crate_name);
314 format!("{origin}/{crate_name}/{version}/{rustc}/all.html")
315 };
316 Url::parse(&s)
317 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html URL", e))
318}
319
320fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
321 let mut segs = final_url.path_segments()?.filter(|s| !s.is_empty());
322 let _pkg = segs.next()?;
323 let ver = segs.next()?;
324 if ver == "latest" {
325 return None;
326 }
327 if requested == "latest" || ver != requested {
328 return Some(ver.to_string());
329 }
330 Some(ver.to_string())
331}
332
333static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
335 Regex::new(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
336 .expect("hardcoded on* handler strip regex is valid by construction")
337});
338
339static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
341 Regex::new(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
342 .expect("hardcoded javascript: URL strip regex is valid by construction")
343});
344
345pub fn sanitize_html_fragment(html: &str) -> String {
351 let fragment = Html::parse_fragment(html);
352 let drop_sel = Selector::parse("script, style, noscript")
353 .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction");
354 if let Some(root) = fragment.tree.root().children().next() {
356 let _ = root;
357 }
358 let mut out = String::with_capacity(html.len());
362 out.push_str(html);
363 for el in fragment.select(&drop_sel) {
364 let tag_html = el.html();
365 out = out.replace(&tag_html, "");
366 }
367 out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
369 out = RE_JS_URLS.replace_all(&out, "").into_owned();
371 out
372}
373
374fn html_to_markdown(html: &str) -> AppResult<String> {
375 let cleaned = sanitize_html_fragment(html);
376 htmd::convert(&cleaned).map_err(|e| {
377 AppError::new(
378 ErrorKind::Parse,
379 format!("HTML to Markdown conversion failed: {e}"),
380 )
381 })
382}
383
384fn first_inner_html(document: &Html, selectors: &[&str]) -> Option<String> {
385 for sel in selectors {
386 if let Ok(s) = Selector::parse(sel)
387 && let Some(n) = document.select(&s).next()
388 {
389 let inner = n.inner_html();
390 if !inner.trim().is_empty() {
391 return Some(inner);
392 }
393 }
394 }
395 None
396}
397
398pub async fn fetch_readme(
405 http: &HttpClient,
406 crate_name: &str,
407 version: &str,
408) -> AppResult<ReadmeData> {
409 fetch_readme_on_origin(
410 http,
411 &format!("https://{HOST_DOCS_RS}"),
412 crate_name,
413 version,
414 )
415 .await
416}
417
418pub async fn fetch_readme_on_origin(
425 http: &HttpClient,
426 origin: &str,
427 crate_name: &str,
428 version: &str,
429) -> AppResult<ReadmeData> {
430 let url = readme_url_on_origin(origin, crate_name, version)?;
431 fetch_readme_at(http, crate_name, version, &url).await
432}
433
434pub async fn fetch_readme_at(
442 http: &HttpClient,
443 crate_name: &str,
444 version: &str,
445 url: &Url,
446) -> AppResult<ReadmeData> {
447 let resp = http.get_html(url).await?;
448 if resp.status.as_u16() == 404 {
449 return Err(AppError::from_http_status(
450 404,
451 &format!("crate or version not found: {crate_name}@{version}"),
452 ));
453 }
454 if !resp.status.is_success() {
455 return Err(AppError::from_http_status(
456 resp.status.as_u16(),
457 "docs.rs readme",
458 ));
459 }
460 if resp.content_type.is_some() && !content_type_looks_html(resp.content_type.as_deref()) {
461 tracing::warn!(content_type = ?resp.content_type, "unexpected Content-Type from docs.rs");
462 }
463
464 let body = resp.body.clone();
465 let (markdown, empty) = http
466 .budget()
467 .run_cpu_bound(move || {
468 let text = decode_utf8(&body)?;
469 extract_readme_markdown_from_html(&text)
470 })
471 .await?;
472 let resolved = extract_resolved_version(&resp.final_url, version);
473 Ok(ReadmeData {
474 crate_name: crate_name.to_string(),
475 version: version.to_string(),
476 resolved_version: resolved,
477 markdown,
478 empty,
479 truncated: false,
480 source_url: resp.final_url.to_string(),
481 })
482}
483
484pub async fn fetch_item(
491 http: &HttpClient,
492 crate_name: &str,
493 version: &str,
494 kind: ItemKind,
495 segments: &[String],
496) -> AppResult<GetItemData> {
497 fetch_item_on_origin(
498 http,
499 &format!("https://{HOST_DOCS_RS}"),
500 crate_name,
501 version,
502 kind,
503 segments,
504 )
505 .await
506}
507
508pub async fn fetch_item_on_origin(
515 http: &HttpClient,
516 origin: &str,
517 crate_name: &str,
518 version: &str,
519 kind: ItemKind,
520 segments: &[String],
521) -> AppResult<GetItemData> {
522 let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
523 fetch_item_at(http, crate_name, version, kind, segments, &url).await
524}
525
526pub async fn fetch_item_at(
534 http: &HttpClient,
535 crate_name: &str,
536 version: &str,
537 kind: ItemKind,
538 segments: &[String],
539 url: &Url,
540) -> AppResult<GetItemData> {
541 let resp = http.get_html(url).await?;
542 if resp.status.as_u16() == 404 {
543 return Err(AppError::from_http_status(
544 404,
545 &format!("item not found at {url}"),
546 ));
547 }
548 if !resp.status.is_success() {
549 return Err(AppError::from_http_status(
550 resp.status.as_u16(),
551 "docs.rs get-item",
552 ));
553 }
554
555 let body = resp.body.clone();
556 let (markdown, empty) = http
557 .budget()
558 .run_cpu_bound(move || {
559 let text = decode_utf8(&body)?;
560 extract_item_markdown_from_html(&text)
561 })
562 .await?;
563 let item_path = segments.join("::");
564 let title = format!("{item_path} ({})", kind.as_str());
565 let resolved = extract_resolved_version(&resp.final_url, version);
566
567 Ok(GetItemData {
568 crate_name: crate_name.to_string(),
569 item_type: kind.as_str().to_string(),
570 item_path,
571 version: version.to_string(),
572 resolved_version: resolved,
573 markdown,
574 empty,
575 truncated: false,
576 source_url: resp.final_url.to_string(),
577 title,
578 })
579}
580
581pub async fn search_in_crate(
588 http: &HttpClient,
589 crate_name: &str,
590 version: &str,
591 query: &str,
592 item_type: Option<ItemKind>,
593 limit: u32,
594) -> AppResult<SearchInCrateData> {
595 search_in_crate_on_origin(
596 http,
597 &format!("https://{HOST_DOCS_RS}"),
598 crate_name,
599 version,
600 query,
601 item_type,
602 limit,
603 )
604 .await
605}
606
607pub async fn search_in_crate_on_origin(
614 http: &HttpClient,
615 origin: &str,
616 crate_name: &str,
617 version: &str,
618 query: &str,
619 item_type: Option<ItemKind>,
620 limit: u32,
621) -> AppResult<SearchInCrateData> {
622 let url = all_html_url_on_origin(origin, crate_name, version)?;
623 search_in_crate_at(http, crate_name, version, query, item_type, limit, &url).await
624}
625
626pub async fn search_in_crate_at(
635 http: &HttpClient,
636 crate_name: &str,
637 version: &str,
638 query: &str,
639 item_type: Option<ItemKind>,
640 limit: u32,
641 url: &Url,
642) -> AppResult<SearchInCrateData> {
643 let resp = http.get_html(url).await?;
644 if resp.status.as_u16() == 404 {
645 return Err(AppError::from_http_status(
646 404,
647 &format!("all.html not found for {crate_name}@{version}"),
648 ));
649 }
650 if !resp.status.is_success() {
651 return Err(AppError::from_http_status(
652 resp.status.as_u16(),
653 "docs.rs all.html",
654 ));
655 }
656
657 let body = resp.body.clone();
658 let source_url = resp.final_url.to_string();
659 let crate_name = crate_name.to_string();
660 let version = version.to_string();
661 let query = query.to_string();
662 http.budget()
663 .run_cpu_bound(move || {
664 let text = decode_utf8(&body)?;
665 search_in_crate_from_html(
666 &text,
667 &crate_name,
668 &version,
669 &query,
670 item_type,
671 limit,
672 &source_url,
673 )
674 })
675 .await
676}
677
678pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
684 if href.starts_with("http://") || href.starts_with("https://") {
685 return Ok(href.to_string());
686 }
687 if href.starts_with('/') {
688 let joined = Url::parse(&format!("https://docs.rs{href}")).map_err(|e| {
689 AppError::with_source(ErrorKind::Parse, "failed to join absolute path href", e)
690 })?;
691 return Ok(joined.to_string());
692 }
693 base.join(href)
694 .map(|u| u.to_string())
695 .map_err(|e| AppError::with_source(ErrorKind::Parse, "failed to join relative href", e))
696}
697
698const RAYON_HIT_THRESHOLD: usize = 64;
700
701type ClassifiedHit = (usize, String, ItemKind, String);
703
704pub fn parse_all_html_hits(
713 html: &str,
714 crate_name: &str,
715 version: &str,
716 query: &str,
717 item_type: Option<ItemKind>,
718 limit: usize,
719) -> AppResult<Vec<SearchInCrateHit>> {
720 if limit == 0 {
721 return Ok(Vec::new());
722 }
723 let document = Html::parse_document(html);
724 let a_sel = Selector::parse("#main-content a")
725 .expect("hardcoded scraper selector '#main-content a' is valid by construction");
726 let rustc = rustc_crate_name(crate_name);
727 let base = Url::parse(&format!(
728 "https://{HOST_DOCS_RS}/{crate_name}/{version}/{rustc}/"
729 ))
730 .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html base", e))?;
731 let q = query.trim().to_ascii_lowercase();
732
733 let candidates: Vec<(String, String)> = document
735 .select(&a_sel)
736 .filter_map(|a| {
737 let name = a.text().collect::<String>().trim().to_string();
738 if name.is_empty() {
739 return None;
740 }
741 let href = a.value().attr("href")?.to_string();
742 if href.is_empty() {
743 return None;
744 }
745 Some((name, href))
746 })
747 .collect();
748
749 if candidates.len() < RAYON_HIT_THRESHOLD {
750 return filter_hits_sequential(candidates, &base, &q, item_type, limit);
751 }
752 filter_hits_parallel(candidates, &base, &q, item_type, limit)
753}
754
755fn filter_hits_sequential(
756 candidates: Vec<(String, String)>,
757 base: &Url,
758 q: &str,
759 item_type: Option<ItemKind>,
760 limit: usize,
761) -> AppResult<Vec<SearchInCrateHit>> {
762 let mut hits = Vec::with_capacity(limit.min(256));
763 let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
764 for (name, href) in candidates {
765 if let Some(hit) = classify_hit(name, &href, base, q, item_type, &mut seen)? {
766 hits.push(hit);
767 if hits.len() >= limit {
768 break;
769 }
770 }
771 }
772 Ok(hits)
773}
774
775fn filter_hits_parallel(
776 candidates: Vec<(String, String)>,
777 base: &Url,
778 q: &str,
779 item_type: Option<ItemKind>,
780 limit: usize,
781) -> AppResult<Vec<SearchInCrateHit>> {
782 let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
784 .into_par_iter()
785 .enumerate()
786 .map(|(idx, (name, href))| {
787 let Some(kind) = ItemKind::from_href(&href) else {
788 return Ok(None);
789 };
790 if let Some(filter) = item_type
791 && kind != filter
792 {
793 return Ok(None);
794 }
795 if !q.is_empty() && !name.to_ascii_lowercase().contains(q) {
796 return Ok(None);
797 }
798 let abs = join_href(base, &href)?;
799 Ok(Some((idx, name, kind, abs)))
800 })
801 .collect();
802 let mut rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
803 rows.par_sort_unstable_by_key(|(idx, _, _, _)| *idx);
804
805 let mut hits = Vec::with_capacity(limit.min(256));
806 let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
807 for (_idx, name, kind, abs) in rows {
808 if !seen.insert((name.clone(), kind)) {
809 continue;
810 }
811 hits.push(SearchInCrateHit {
812 name,
813 kind: kind.as_str().to_string(),
814 url: abs,
815 });
816 if hits.len() >= limit {
817 break;
818 }
819 }
820 Ok(hits)
821}
822
823fn classify_hit(
824 name: String,
825 href: &str,
826 base: &Url,
827 q: &str,
828 item_type: Option<ItemKind>,
829 seen: &mut HashSet<(String, ItemKind)>,
830) -> AppResult<Option<SearchInCrateHit>> {
831 let Some(kind) = ItemKind::from_href(href) else {
832 return Ok(None);
833 };
834 if let Some(filter) = item_type
835 && kind != filter
836 {
837 return Ok(None);
838 }
839 if !q.is_empty() && !name.to_ascii_lowercase().contains(q) {
840 return Ok(None);
841 }
842 let abs = join_href(base, href)?;
843 if !seen.insert((name.clone(), kind)) {
844 return Ok(None);
845 }
846 Ok(Some(SearchInCrateHit {
847 name,
848 kind: kind.as_str().to_string(),
849 url: abs,
850 }))
851}
852
853pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
859 let document = Html::parse_document(html);
860 let content = first_inner_html(
861 &document,
862 &[
863 ".rustdoc .docblock",
864 "#main-content .docblock",
865 ".docblock",
866 ".rustdoc-main .item-decl",
867 ".item-decl",
868 ],
869 );
870 match content {
871 Some(h) => Ok((html_to_markdown(&h)?, false)),
872 None => Ok((String::new(), true)),
873 }
874}
875
876pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
882 let document = Html::parse_document(html);
883 let content = first_inner_html(&document, &["#main-content", ".item-decl", ".docblock"])
884 .unwrap_or_default();
885 let empty = content.trim().is_empty();
886 if empty {
887 Ok((String::new(), true))
888 } else {
889 Ok((html_to_markdown(&content)?, false))
890 }
891}
892
893pub fn search_in_crate_from_html(
903 html: &str,
904 crate_name: &str,
905 version: &str,
906 query: &str,
907 item_type: Option<ItemKind>,
908 limit: u32,
909 source_url: &str,
910) -> AppResult<SearchInCrateData> {
911 let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
912 let all = parse_all_html_hits(html, crate_name, version, query, item_type, usize::MAX)?;
913 let total = all.len();
914 let hits: Vec<_> = all.into_iter().take(limit).collect();
915 let emitted = hits.len();
916 Ok(SearchInCrateData {
917 crate_name: crate_name.to_string(),
918 query: query.to_string(),
919 version: version.to_string(),
920 total,
921 emitted,
922 hits,
923 truncated: total > emitted,
924 source_url: source_url.to_string(),
925 })
926}
927
928#[cfg(test)]
929mod tests {
930 use super::*;
931
932 #[test]
933 fn readme_url_hyphen() {
934 let u = readme_url("async-trait", "latest").unwrap();
935 assert_eq!(
936 u.as_str(),
937 "https://docs.rs/async-trait/latest/async_trait/index.html"
938 );
939 }
940
941 #[test]
942 fn get_item_nested() {
943 let segs = vec!["tokio".into(), "runtime".into(), "Runtime".into()];
944 let u = get_item_url("tokio", "latest", ItemKind::Struct, &segs).unwrap();
945 assert!(u.as_str().contains("/tokio/runtime/struct.Runtime.html"));
946 }
947
948 #[test]
949 fn get_item_without_crate_prefix() {
950 let segs = vec!["Parser".into()];
951 let u = get_item_url("clap", "latest", ItemKind::Trait, &segs).unwrap();
952 assert!(
953 u.as_str().ends_with("/clap/trait.Parser.html")
954 || u.path().ends_with("/clap/trait.Parser.html")
955 );
956 }
957
958 #[test]
959 fn get_item_keeps_single_segment_equal_to_crate_name() {
960 let segs = vec!["async_trait".into()];
962 let u = get_item_url("async-trait", "latest", ItemKind::Attribute, &segs).unwrap();
963 assert!(
964 u.as_str()
965 .ends_with("/async-trait/latest/async_trait/attr.async_trait.html"),
966 "url={u}"
967 );
968 }
969
970 #[test]
971 fn get_item_module_single_segment_crate_name_is_root() {
972 let segs = vec!["async_trait".into()];
973 let u = get_item_url("async-trait", "latest", ItemKind::Module, &segs).unwrap();
974 assert!(
975 u.as_str()
976 .ends_with("/async-trait/latest/async_trait/index.html"),
977 "url={u}"
978 );
979 }
980
981 #[test]
982 fn join_relative() {
983 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
984 let j = join_href(&base, "de/trait.Deserialize.html").unwrap();
985 assert!(j.contains("/serde/de/trait.Deserialize.html"));
986 }
987
988 #[test]
989 fn join_absolute_path_and_fragment() {
990 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
991 let j = join_href(&base, "/serde/latest/serde/struct.Error.html").unwrap();
992 assert!(j.starts_with("https://docs.rs/serde/"));
993 let f = join_href(&base, "struct.Error.html#method.to_string").unwrap();
994 assert!(f.contains("#method.to_string"));
995 }
996
997 #[test]
998 fn sanitize_strips_script() {
999 let raw = r#"<div>ok<script>alert(1)</script><p onclick="x()">hi</p></div>"#;
1000 let s = sanitize_html_fragment(raw);
1001 assert!(!s.contains("script"));
1002 assert!(!s.contains("onclick"));
1003 assert!(s.contains("hi") || s.contains("ok"));
1004 }
1005
1006 #[test]
1007 fn readme_fallback_item_decl_converts() {
1008 let html = r#"<!DOCTYPE html><html><body class="rustdoc-main">
1009 <div class="item-decl"><pre>pub use foo;</pre></div>
1010 </body></html>"#;
1011 let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1012 assert!(!empty);
1013 assert!(!md.trim().is_empty());
1014 }
1015
1016 #[test]
1017 fn all_html_order_stable() {
1018 let html = r#"<!DOCTYPE html><html><body>
1019 <div id="main-content">
1020 <a href="struct.A.html">A</a>
1021 <a href="trait.B.html">B</a>
1022 <a href="fn.c.html">c</a>
1023 <a href="union.U.html">U</a>
1024 <a href="attr.x.html">x</a>
1025 <a href="derive.D.html">D</a>
1026 </div></body></html>"#;
1027 let hits = parse_all_html_hits(html, "demo", "1.0.0", "", None, 100).unwrap();
1028 assert_eq!(hits.len(), 6);
1029 assert_eq!(hits[0].name, "A");
1030 assert_eq!(hits[0].kind, "struct");
1031 assert_eq!(hits[3].kind, "union");
1032 assert_eq!(hits[4].kind, "attribute");
1033 assert_eq!(hits[5].kind, "derive");
1034 }
1035
1036 #[test]
1037 fn std_core_alloc_urls() {
1038 let u = readme_url("std", "latest").unwrap();
1039 assert_eq!(
1040 u.as_str(),
1041 "https://doc.rust-lang.org/stable/std/index.html"
1042 );
1043 let u = get_item_url(
1044 "std",
1045 "latest",
1046 ItemKind::Struct,
1047 &["option".into(), "Option".into()],
1048 )
1049 .unwrap();
1050 assert_eq!(
1051 u.as_str(),
1052 "https://doc.rust-lang.org/stable/std/option/struct.Option.html"
1053 );
1054 let a = all_html_url("core", "nightly").unwrap();
1055 assert_eq!(
1056 a.as_str(),
1057 "https://doc.rust-lang.org/nightly/core/all.html"
1058 );
1059 }
1060
1061 #[test]
1062 fn extract_item_from_fixture() {
1063 let html = include_str!("../tests/fixtures/docs_rs/get_item_main.html");
1064 let (md, empty) = extract_item_markdown_from_html(html).unwrap();
1065 assert!(!empty);
1066 assert!(md.contains("Runtime") || md.contains("Tokio") || md.contains("runtime"));
1067 }
1068
1069 #[test]
1070 fn extract_readme_primary_docblock_fixture() {
1071 let html = include_str!("../tests/fixtures/docs_rs/readme_docblock.html");
1072 let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1073 assert!(!empty);
1074 assert!(!md.to_ascii_lowercase().contains("alert"));
1075 }
1076
1077 #[test]
1078 fn search_in_crate_from_html_limit_and_filter() {
1079 let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1080 let data = search_in_crate_from_html(
1081 html,
1082 "demo",
1083 "1.0.0",
1084 "",
1085 Some(ItemKind::Struct),
1086 1,
1087 "https://docs.rs/demo/1.0.0/demo/all.html",
1088 )
1089 .unwrap();
1090 assert_eq!(data.total, 1);
1091 assert_eq!(data.emitted, 1);
1092 assert_eq!(data.hits[0].kind, "struct");
1093 assert!(!data.truncated);
1094 }
1095
1096 #[test]
1097 fn search_in_crate_truncated_when_limit_cuts_hits() {
1098 let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1099 let data = search_in_crate_from_html(
1100 html,
1101 "demo",
1102 "1.0.0",
1103 "",
1104 None,
1105 2,
1106 "https://docs.rs/demo/1.0.0/demo/all.html",
1107 )
1108 .unwrap();
1109 assert!(data.total > 2, "fixture must have more than 2 hits");
1110 assert_eq!(data.emitted, 2);
1111 assert!(data.truncated);
1112 let full = search_in_crate_from_html(
1113 html,
1114 "demo",
1115 "1.0.0",
1116 "",
1117 None,
1118 1000,
1119 "https://docs.rs/demo/1.0.0/demo/all.html",
1120 )
1121 .unwrap();
1122 assert_eq!(full.total, full.emitted);
1123 assert!(!full.truncated);
1124 }
1125
1126 #[test]
1127 fn search_in_crate_limit_zero_emits_nothing() {
1128 let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1129 let data = search_in_crate_from_html(
1130 html,
1131 "demo",
1132 "1.0.0",
1133 "",
1134 None,
1135 0,
1136 "https://docs.rs/demo/1.0.0/demo/all.html",
1137 )
1138 .unwrap();
1139 assert!(data.total > 0, "fixture must have hits for total");
1140 assert_eq!(data.emitted, 0);
1141 assert!(data.hits.is_empty());
1142 assert!(data.truncated);
1143 assert!(
1144 parse_all_html_hits(html, "demo", "1.0.0", "", None, 0)
1145 .unwrap()
1146 .is_empty()
1147 );
1148 }
1149
1150 #[test]
1151 fn resolved_version_from_url() {
1152 let u = Url::parse("https://docs.rs/serde/1.0.210/serde/index.html").unwrap();
1153 assert_eq!(
1154 extract_resolved_version(&u, "latest").as_deref(),
1155 Some("1.0.210")
1156 );
1157 let u2 = Url::parse("https://docs.rs/serde/latest/serde/index.html").unwrap();
1158 assert!(extract_resolved_version(&u2, "latest").is_none());
1159 }
1160
1161 #[test]
1162 fn module_url_template() {
1163 let segs = vec!["serde".into(), "de".into()];
1164 let u = get_item_url("serde", "latest", ItemKind::Module, &segs).unwrap();
1165 assert!(u.as_str().ends_with("/serde/de/index.html"));
1166 }
1167
1168 #[test]
1169 fn empty_html_readme() {
1170 let (md, empty) = extract_readme_markdown_from_html("<html><body></body></html>").unwrap();
1171 assert!(empty);
1172 assert!(md.is_empty());
1173 }
1174
1175 #[test]
1176 fn get_item_empty_segments_errors() {
1177 let err = get_item_url("clap", "latest", ItemKind::Struct, &[]).unwrap_err();
1178 assert_eq!(err.kind(), ErrorKind::InvalidInput);
1179 }
1180
1181 #[test]
1182 fn join_full_http_url_passthrough() {
1183 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1184 let full = "https://docs.rs/serde/latest/serde/struct.Error.html";
1185 assert_eq!(join_href(&base, full).unwrap(), full);
1186 let http = "http://example.invalid/x.html";
1187 assert_eq!(join_href(&base, http).unwrap(), http);
1188 }
1189
1190 #[test]
1191 fn origin_builders_trim_trailing_slash() {
1192 let u = readme_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1193 assert_eq!(u.as_str(), "https://docs.rs/demo/1.0.0/demo/index.html");
1194 let a = all_html_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1195 assert_eq!(a.as_str(), "https://docs.rs/demo/1.0.0/demo/all.html");
1196 }
1197
1198 #[test]
1199 fn empty_item_html_extract() {
1200 let (md, empty) = extract_item_markdown_from_html("<html><body></body></html>").unwrap();
1201 assert!(empty);
1202 assert!(md.is_empty());
1203 }
1204}