1mod types;
32pub(crate) mod hits;
33mod urls;
34mod html;
35mod fetch;
36mod search;
37
38pub use types::{GetItemData, ReadmeData, SearchInCrateData, SearchInCrateHit};
39pub use urls::{
40 METHOD_PARENT_KIND_NAMES, all_html_url, all_html_url_on_origin, get_item_url,
41 get_item_url_on_origin, get_item_url_on_origin_with_parent_kind, is_method_path,
42 method_segments_from_parent, pick_unique_type_path, readme_url, readme_url_on_origin,
43 strip_crate_prefix_segments,
44};
45pub use html::{
46 extract_item_markdown_from_html, extract_method_markdown_from_html,
47 extract_method_markdown_scoped, extract_readme_markdown_from_html,
48 list_method_anchor_names, list_method_anchor_names_from_html, sanitize_html_fragment,
49 scrub_rustdoc_chrome,
50};
51pub use fetch::{
52 fetch_item, fetch_item_at, fetch_item_at_with_echo, fetch_item_on_origin,
53 fetch_item_on_origin_with_echo, fetch_readme, fetch_readme_at, fetch_readme_on_origin,
54};
55pub use search::{
56 join_href, parse_all_html_hits, resolve_hit_url, search_in_crate, search_in_crate_at,
57 search_in_crate_from_html, search_in_crate_on_origin,
58};
59
60#[cfg(test)]
62#[allow(unused_imports)]
63use html::{
64 extract_resolved_version, extract_resolved_version_for_crate, scrape_docs_rs_version_from_html,
65};
66
67#[cfg(test)]
68mod tests {
69 use super::*;
70 use crate::domain::{AllowedOrigin, CrateName, MatchMode, SearchQuery, VersionArg};
71 use crate::error::ErrorKind;
72 use crate::item_kind::ItemKind;
73 use url::Url;
74
75 #[test]
76 fn readme_url_hyphen() {
77 let u = readme_url(&CrateName::parse("async-trait").unwrap(), &VersionArg::parse("latest").unwrap()).unwrap();
78 assert_eq!(
79 u.as_str(),
80 "https://docs.rs/async-trait/latest/async_trait/index.html"
81 );
82 }
83
84 #[test]
85 fn get_item_nested() {
86 let segs = vec!["tokio".into(), "runtime".into(), "Runtime".into()];
87 let u = get_item_url(&CrateName::parse("tokio").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Struct, &segs).unwrap();
88 assert!(u.as_str().contains("/tokio/runtime/struct.Runtime.html"));
89 }
90
91 #[test]
92 fn get_item_without_crate_prefix() {
93 let segs = vec!["Parser".into()];
94 let u = get_item_url(&CrateName::parse("clap").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Trait, &segs).unwrap();
95 assert!(
96 u.as_str().ends_with("/clap/trait.Parser.html")
97 || u.path().ends_with("/clap/trait.Parser.html")
98 );
99 }
100
101 #[test]
102 fn get_item_keeps_single_segment_equal_to_crate_name() {
103 let segs = vec!["async_trait".into()];
105 let u = get_item_url(&CrateName::parse("async-trait").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Attribute, &segs).unwrap();
106 assert!(
107 u.as_str()
108 .ends_with("/async-trait/latest/async_trait/attr.async_trait.html"),
109 "url={u}"
110 );
111 }
112
113 #[test]
114 fn get_item_module_single_segment_crate_name_is_root() {
115 let segs = vec!["async_trait".into()];
116 let u = get_item_url(&CrateName::parse("async-trait").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Module, &segs).unwrap();
117 assert!(
118 u.as_str()
119 .ends_with("/async-trait/latest/async_trait/index.html"),
120 "url={u}"
121 );
122 }
123
124 #[test]
125 fn join_relative() {
126 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
127 let j = join_href(&base, "de/trait.Deserialize.html").unwrap();
128 assert!(j.contains("/serde/de/trait.Deserialize.html"));
129 }
130
131 #[test]
132 fn join_absolute_path_and_fragment() {
133 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
134 let j = join_href(&base, "/serde/latest/serde/struct.Error.html").unwrap();
135 assert!(j.starts_with("https://docs.rs/serde/"));
136 let f = join_href(&base, "struct.Error.html#method.to_string").unwrap();
137 assert!(f.contains("#method.to_string"));
138 let std_base = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
140 let std_j = join_href(&std_base, "/stable/std/option/struct.Option.html").unwrap();
141 assert!(
142 std_j.starts_with("https://doc.rust-lang.org/"),
143 "must not hardcode docs.rs for absolute paths: {std_j}"
144 );
145 assert!(std_j.contains("struct.Option.html"));
146 }
147
148 #[test]
149 fn sanitize_strips_script() {
150 let raw = r#"<div>ok<script>alert(1)</script><p onclick="x()">hi</p></div>"#;
151 let s = sanitize_html_fragment(raw);
152 assert!(!s.contains("script"));
153 assert!(!s.contains("onclick"));
154 assert!(s.contains("hi") || s.contains("ok"));
155 }
156
157 #[test]
158 fn readme_fallback_item_decl_converts() {
159 let html = r#"<!DOCTYPE html><html><body class="rustdoc-main">
160 <div class="item-decl"><pre>pub use foo;</pre></div>
161 </body></html>"#;
162 let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
163 assert!(!empty);
164 assert!(!md.trim().is_empty());
165 }
166
167 #[test]
168 fn all_html_order_stable() {
169 let html = r#"<!DOCTYPE html><html><body>
170 <div id="main-content">
171 <a href="struct.A.html">A</a>
172 <a href="trait.B.html">B</a>
173 <a href="fn.c.html">c</a>
174 <a href="union.U.html">U</a>
175 <a href="attr.x.html">x</a>
176 <a href="derive.D.html">D</a>
177 </div></body></html>"#;
178 let base = Url::parse("https://docs.rs/demo/1.0.0/demo/all.html").unwrap();
179 let hits = parse_all_html_hits(
180 html,
181 &base,
182 &SearchQuery::parse("", true).unwrap(),
183 None,
184 100,
185 crate::domain::MatchMode::Prefix,
186 &crate::shutdown::CancelFlag::new(),
187 )
188 .unwrap();
189 assert_eq!(hits.len(), 6);
190 assert_eq!(hits[0].name, "A");
191 assert_eq!(hits[0].kind, "struct");
192 assert_eq!(hits[3].kind, "union");
193 assert_eq!(hits[4].kind, "attribute");
194 assert_eq!(hits[5].kind, "derive");
195 }
196
197 #[test]
198 fn std_core_alloc_urls() {
199 let u = readme_url(&CrateName::parse("std").unwrap(), &VersionArg::parse("latest").unwrap()).unwrap();
200 assert_eq!(
201 u.as_str(),
202 "https://doc.rust-lang.org/stable/std/index.html"
203 );
204 let u = get_item_url(
205 &CrateName::parse("std").unwrap(),
206 &VersionArg::parse("latest").unwrap(),
207 ItemKind::Struct,
208 &["option".into(), "Option".into()],
209 )
210 .unwrap();
211 assert_eq!(
212 u.as_str(),
213 "https://doc.rust-lang.org/stable/std/option/struct.Option.html"
214 );
215 let a = all_html_url(&CrateName::parse("core").unwrap(), &VersionArg::parse("nightly").unwrap()).unwrap();
216 assert_eq!(
217 a.as_str(),
218 "https://doc.rust-lang.org/nightly/core/all.html"
219 );
220 }
221
222 #[test]
223 fn extract_item_from_fixture() {
224 let html = include_str!("../../tests/fixtures/docs_rs/get_item_main.html");
225 let (md, empty) = extract_item_markdown_from_html(html).unwrap();
226 assert!(!empty);
227 assert!(md.contains("Runtime") || md.contains("Tokio") || md.contains("runtime"));
228 }
229
230 #[test]
231 fn extract_readme_primary_docblock_fixture() {
232 let html = include_str!("../../tests/fixtures/docs_rs/readme_docblock.html");
233 let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
234 assert!(!empty);
235 assert!(!md.to_ascii_lowercase().contains("alert"));
236 }
237
238 #[test]
239 fn parse_all_html_hits_honors_cancel_flag() {
240 let html = include_str!("../../tests/fixtures/docs_rs/all_html_sample.html");
241 let base = url::Url::parse("https://docs.rs/demo/1.0.0/demo/all.html").unwrap();
242 let cancel = crate::shutdown::CancelFlag::new();
243 cancel.cancel();
244 let err = parse_all_html_hits(
245 html,
246 &base,
247 &SearchQuery::parse("", true).unwrap(),
248 None,
249 100,
250 MatchMode::Prefix,
251 &cancel,
252 )
253 .expect_err("cancelled flag must abort scrape");
254 assert!(
255 matches!(
256 err.kind(),
257 crate::error::ErrorKind::Interrupted | crate::error::ErrorKind::Terminated
258 ),
259 "kind={:?}",
260 err.kind()
261 );
262 }
263
264 #[test]
265 fn search_in_crate_from_html_limit_and_filter() {
266 let html = include_str!("../../tests/fixtures/docs_rs/all_html_sample.html");
267 let data = search_in_crate_from_html(
268 html,
269 &CrateName::parse("demo").unwrap(),
270 &VersionArg::parse("1.0.0").unwrap(),
271 &SearchQuery::parse("", true).unwrap(),
272 Some(ItemKind::Struct),
273 1,
274 MatchMode::Prefix,
275 "https://docs.rs/demo/1.0.0/demo/all.html",
276 false,
277 &crate::shutdown::CancelFlag::new(),
278 )
279 .unwrap();
280 assert_eq!(data.total, 1);
281 assert_eq!(data.emitted, 1);
282 assert_eq!(data.hits[0].kind, "struct");
283 assert!(!data.truncated);
284 }
285
286 #[test]
287 fn search_in_crate_truncated_when_limit_cuts_hits() {
288 let html = include_str!("../../tests/fixtures/docs_rs/all_html_sample.html");
289 let data = search_in_crate_from_html(
290 html,
291 &CrateName::parse("demo").unwrap(),
292 &VersionArg::parse("1.0.0").unwrap(),
293 &SearchQuery::parse("", true).unwrap(),
294 None,
295 2,
296 MatchMode::Prefix,
297 "https://docs.rs/demo/1.0.0/demo/all.html",
298 false,
299 &crate::shutdown::CancelFlag::new(),
300 )
301 .unwrap();
302 assert!(data.total > 2, "fixture must have more than 2 hits");
303 assert_eq!(data.emitted, 2);
304 assert!(data.truncated);
305 let full = search_in_crate_from_html(
306 html,
307 &CrateName::parse("demo").unwrap(),
308 &VersionArg::parse("1.0.0").unwrap(),
309 &SearchQuery::parse("", true).unwrap(),
310 None,
311 1000,
312 MatchMode::Prefix,
313 "https://docs.rs/demo/1.0.0/demo/all.html",
314 false,
315 &crate::shutdown::CancelFlag::new(),
316 )
317 .unwrap();
318 assert_eq!(full.total, full.emitted);
319 assert!(!full.truncated);
320 }
321
322 #[test]
323 fn search_in_crate_limit_zero_emits_nothing() {
324 let html = include_str!("../../tests/fixtures/docs_rs/all_html_sample.html");
325 let data = search_in_crate_from_html(
326 html,
327 &CrateName::parse("demo").unwrap(),
328 &VersionArg::parse("1.0.0").unwrap(),
329 &SearchQuery::parse("", true).unwrap(),
330 None,
331 0,
332 MatchMode::Prefix,
333 "https://docs.rs/demo/1.0.0/demo/all.html",
334 false,
335 &crate::shutdown::CancelFlag::new(),
336 )
337 .unwrap();
338 assert!(data.total > 0, "fixture must have hits for total");
339 assert_eq!(data.emitted, 0);
340 assert!(data.hits.is_empty());
341 assert!(data.truncated);
342 }
343
344 #[test]
345 fn serialize_exact_match_beats_deserializer() {
346 let html = r#"<!DOCTYPE html><html><body>
347 <div id="main-content">
348 <a href="trait.Deserialize.html">Deserialize</a>
349 <a href="trait.Serialize.html">Serialize</a>
350 <a href="struct.Deserializer.html">Deserializer</a>
351 </div></body></html>"#;
352 let base = Url::parse("https://docs.rs/serde/1.0.0/serde/all.html").unwrap();
353 let hits = parse_all_html_hits(
354 html,
355 &base,
356 &SearchQuery::parse("Serialize", false).unwrap(),
357 Some(ItemKind::Trait),
358 10,
359 MatchMode::Prefix,
360 &crate::shutdown::CancelFlag::new(),
361 )
362 .unwrap();
363 assert_eq!(hits[0].name, "Serialize");
364 assert!(hits.iter().all(|h| h.name == "Serialize"));
365 }
366
367 #[test]
368 fn hit_urls_follow_source_url_host_stdlib_and_mock() {
369 let html = r#"<!DOCTYPE html><html><body>
371 <div id="main-content">
372 <a href="option/enum.Option.html">option::Option</a>
373 </div></body></html>"#;
374 let std_base =
375 Url::parse("https://doc.rust-lang.org/stable/std/all.html").unwrap();
376 let std_hits = parse_all_html_hits(
377 html,
378 &std_base,
379 &SearchQuery::parse("Option", false).unwrap(),
380 None,
381 10,
382 MatchMode::Prefix,
383 &crate::shutdown::CancelFlag::new(),
384 )
385 .unwrap();
386 assert_eq!(std_hits.len(), 1);
387 assert!(
388 std_hits[0].url.starts_with("https://doc.rust-lang.org/"),
389 "stdlib hit must stay on doc.rust-lang.org: {}",
390 std_hits[0].url
391 );
392 assert!(
393 !std_hits[0].url.contains("docs.rs"),
394 "stdlib hit must not rewrite host to docs.rs: {}",
395 std_hits[0].url
396 );
397
398 let mock_base =
399 Url::parse("http://127.0.0.1:9/demo/1.0.0/demo/all.html").unwrap();
400 let mock_html = r#"<!DOCTYPE html><html><body>
401 <div id="main-content">
402 <a href="struct.Client.html">Client</a>
403 </div></body></html>"#;
404 let mock_hits = parse_all_html_hits(
405 mock_html,
406 &mock_base,
407 &SearchQuery::parse("Client", false).unwrap(),
408 None,
409 10,
410 MatchMode::Prefix,
411 &crate::shutdown::CancelFlag::new(),
412 )
413 .unwrap();
414 assert_eq!(mock_hits.len(), 1);
415 assert!(
416 mock_hits[0].url.starts_with("http://127.0.0.1:9/"),
417 "mock hit must preserve loopback origin: {}",
418 mock_hits[0].url
419 );
420 }
421
422 #[test]
423 fn method_url_uses_anchor() {
424 let segs = vec!["runtime".into(), "Runtime".into(), "new".into()];
425 let u = get_item_url(&CrateName::parse("tokio").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Fn, &segs).unwrap();
426 assert!(
427 u.as_str().contains("struct.Runtime.html#method.new"),
428 "url={u}"
429 );
430 }
431
432 #[test]
433 fn stdlib_resolved_version_is_channel() {
434 let u = Url::parse("https://doc.rust-lang.org/stable/std/index.html").unwrap();
435 assert_eq!(
436 extract_resolved_version_for_crate(&u, "latest", Some("std")).as_deref(),
437 Some("stable")
438 );
439 }
440
441 #[test]
442 fn resolved_version_from_url() {
443 let u = Url::parse("https://docs.rs/serde/1.0.210/serde/index.html").unwrap();
444 assert_eq!(
445 extract_resolved_version(&u, "latest").as_deref(),
446 Some("1.0.210")
447 );
448 let u2 = Url::parse("https://docs.rs/serde/latest/serde/index.html").unwrap();
449 assert!(extract_resolved_version(&u2, "latest").is_none());
450 }
451
452 #[test]
453 fn scrape_version_prefers_target_crate_not_deps() {
454 let html = r#"
455 <title>tokio - Rust</title>
456 <a href="https://docs.rs/bytes/1.2.0/bytes/">bytes</a>
457 <a href="/tokio/1.53.0/tokio/runtime/index.html">runtime</a>
458 <a href="https://docs.rs/mio/0.8.0/mio/">mio</a>
459 "#;
460 assert_eq!(
461 scrape_docs_rs_version_from_html(html, "tokio").as_deref(),
462 Some("1.53.0")
463 );
464 let html2 = r#"<title>x</title><a href="https://docs.rs/bytes/1.2.0/bytes/">b</a>"#;
466 assert_eq!(scrape_docs_rs_version_from_html(html2, "tokio"), None);
467 }
468
469 #[test]
470 fn module_url_template() {
471 let segs = vec!["serde".into(), "de".into()];
472 let u = get_item_url(&CrateName::parse("serde").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Module, &segs).unwrap();
473 assert!(u.as_str().ends_with("/serde/de/index.html"));
474 }
475
476 #[test]
477 fn empty_html_readme() {
478 let (md, empty) = extract_readme_markdown_from_html("<html><body></body></html>").unwrap();
479 assert!(empty);
480 assert!(md.is_empty());
481 }
482
483 #[test]
484 fn get_item_empty_segments_errors() {
485 let err = get_item_url(&CrateName::parse("clap").unwrap(), &VersionArg::parse("latest").unwrap(), ItemKind::Struct, &[]).unwrap_err();
486 assert_eq!(err.kind(), ErrorKind::InvalidInput);
487 }
488
489 #[test]
490 fn join_same_origin_absolute_ok_off_origin_skipped() {
491 let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
493 let full = "https://docs.rs/serde/latest/serde/struct.Error.html";
494 assert_eq!(resolve_hit_url(&base, full).as_deref(), Some(full));
495 assert_eq!(join_href(&base, full).unwrap(), full);
496 let http = "http://example.invalid/x.html";
497 assert!(resolve_hit_url(&base, http).is_none());
498 assert!(join_href(&base, http).is_err());
499 assert!(resolve_hit_url(&base, "https://evil.example/struct.Foo.html").is_none());
501 assert!(resolve_hit_url(&base, "http://docs.rs/serde/latest/serde/struct.Error.html").is_none());
503 }
504
505 #[test]
506 fn poison_absolute_off_origin_href_soft_skipped_in_search() {
507 let html = r#"<!DOCTYPE html><html><body>
509 <div id="main-content">
510 <a href="https://evil.example/struct.Poison.html">Poison</a>
511 <a href="struct.Client.html">Client</a>
512 </div></body></html>"#;
513 let base = Url::parse("https://docs.rs/demo/1.0.0/demo/all.html").unwrap();
514 let hits = parse_all_html_hits(
516 html,
517 &base,
518 &SearchQuery::parse("i", false).unwrap(),
519 None,
520 10,
521 MatchMode::Substring,
522 &crate::shutdown::CancelFlag::new(),
523 )
524 .unwrap();
525 assert_eq!(hits.len(), 1, "only same-origin Client: {hits:?}");
526 assert_eq!(hits[0].name, "Client");
527 assert!(
528 hits[0].url.starts_with("https://docs.rs/"),
529 "hit must stay on base host: {}",
530 hits[0].url
531 );
532 assert!(
533 hits.iter().all(|h| !h.url.contains("evil.example")),
534 "poison host must not appear: {hits:?}"
535 );
536 }
537
538 #[test]
539 fn origin_builders_trim_trailing_slash() {
540 let origin = AllowedOrigin::parse("https://docs.rs/").unwrap();
541 let u = readme_url_on_origin(
542 &origin,
543 &CrateName::parse("demo").unwrap(),
544 &VersionArg::parse("1.0.0").unwrap(),
545 )
546 .unwrap();
547 assert_eq!(u.as_str(), "https://docs.rs/demo/1.0.0/demo/index.html");
548 let a = all_html_url_on_origin(
549 &origin,
550 &CrateName::parse("demo").unwrap(),
551 &VersionArg::parse("1.0.0").unwrap(),
552 )
553 .unwrap();
554 assert_eq!(a.as_str(), "https://docs.rs/demo/1.0.0/demo/all.html");
555 }
556
557 #[test]
558 fn empty_item_html_extract() {
559 let (md, empty) = extract_item_markdown_from_html("<html><body></body></html>").unwrap();
560 assert!(empty);
561 assert!(md.is_empty());
562 }
563
564 #[test]
565 fn method_extract_scopes_to_details_not_full_page() {
566 let html = include_str!("../../tests/fixtures/docs_rs/method_runtime_new.html");
567 let (md, empty, scope) = extract_method_markdown_scoped(html, "new").unwrap();
568 assert!(!empty);
569 assert_eq!(scope, "method");
570 assert!(
571 !md.contains("parent page noise"),
572 "must not dump full parent page: {md}"
573 );
574 assert!(
575 md.to_ascii_lowercase().contains("new") || md.contains("Creates a new runtime"),
576 "expected method content: {md}"
577 );
578 assert!(!md.contains('§'));
579 assert!(!md.contains("Copy item path"));
580 }
581
582 #[test]
583 fn scrub_rustdoc_chrome_strips_section_sign() {
584 let dirty = "## [§](#serde)Serde\nCopy item path\nbody";
585 let clean = scrub_rustdoc_chrome(dirty);
586 assert!(!clean.contains('§'));
587 assert!(!clean.contains("Copy item path"));
588 assert!(clean.contains("Serde"));
589 }
590
591 #[test]
592 fn method_missing_anchor_is_not_found() {
593 let html = r#"<html><body><div id="main-content"><h1>Struct Runtime</h1><p>only parent</p></div></body></html>"#;
595 let err = extract_method_markdown_scoped(html, "new").unwrap_err();
596 assert_eq!(err.kind(), ErrorKind::NotFound);
597 assert!(
598 err.message().contains("method anchor not found"),
599 "message={}",
600 err.message()
601 );
602 }
603
604 #[test]
605 fn list_method_anchors_from_fixture() {
606 let html = include_str!("../../tests/fixtures/docs_rs/method_runtime_new.html");
607 let names = list_method_anchor_names_from_html(html);
608 assert!(
609 names.iter().any(|n| n == "new"),
610 "expected method.new in fixture: {names:?}"
611 );
612 }
613
614 #[test]
615 fn pick_unique_type_path_resolves_runtime() {
616 let hits = vec![
617 SearchInCrateHit {
618 name: "runtime::LocalRuntime".into(),
619 kind: "struct".into(),
620 url: "u1".into(),
621 score: Some(1),
622 },
623 SearchInCrateHit {
624 name: "runtime::Runtime".into(),
625 kind: "struct".into(),
626 url: "u2".into(),
627 score: Some(0),
628 },
629 SearchInCrateHit {
630 name: "runtime::RuntimeFlavor".into(),
631 kind: "enum".into(),
632 url: "u3".into(),
633 score: Some(2),
634 },
635 SearchInCrateHit {
636 name: "task::spawn".into(),
637 kind: "fn".into(),
638 url: "u4".into(),
639 score: Some(0),
640 },
641 ];
642 assert_eq!(
643 pick_unique_type_path("Runtime", &hits).as_deref(),
644 Some("runtime::Runtime")
645 );
646 assert_eq!(
647 method_segments_from_parent("runtime::Runtime", "new"),
648 vec!["runtime".to_string(), "Runtime".to_string(), "new".to_string()]
649 );
650 assert!(is_method_path(
651 ItemKind::Fn,
652 &["Runtime".into(), "new".into()]
653 ));
654 assert!(!is_method_path(ItemKind::Fn, &["spawn".into()]));
655 }
656
657 #[test]
658 fn pick_unique_type_path_ambiguous_returns_none() {
659 let hits = vec![
660 SearchInCrateHit {
661 name: "a::Client".into(),
662 kind: "struct".into(),
663 url: "u1".into(),
664 score: Some(0),
665 },
666 SearchInCrateHit {
667 name: "b::Client".into(),
668 kind: "struct".into(),
669 url: "u2".into(),
670 score: Some(0),
671 },
672 ];
673 assert!(pick_unique_type_path("Client", &hits).is_none());
674 }
675}