Skip to main content

docsrs_cli/
docs_rs.rs

1//! docs.rs operations: readme, get-item, search-in-crate.
2//!
3//! # Extraction posture (Rules Rust — web scraping / data extraction)
4//!
5//! - **Transport** is HTTP GET ([`HttpClient`]); this module only parses bytes.
6//! - **Structure** uses `scraper` (html5ever) + CSS selectors — never regex for
7//!   field extraction from HTML.
8//! - **Sanitization** may use process-static regex only to strip `on*` handlers
9//!   and `javascript:` URLs after DOM script/style removal (XSS hygiene, not
10//!   content discovery).
11//! - **Markdown** conversion via `htmd` on the sanitized fragment.
12//! - **Provenance:** every success payload includes `source_url` (final URL).
13//! - **Encoding:** UTF-8 only after BOM strip ([`crate::http::decode_utf8`]);
14//!   docs.rs / doc.rust-lang.org ship UTF-8. `encoding_rs` multi-charset is OOS.
15//! - **Not a crawl:** no link frontier, no sitemap, no RSS/Atom, no robots meta
16//!   index policy (agent one-shot read, not dataset indexing).
17//!
18//! # Workload classification (Rules Rust — parallelism)
19//!
20//! - **I/O stage:** HTTPS GET via [`HttpClient`] on multi-thread Tokio workers.
21//! - **CPU stage:** HTML scrape + markdown conversion via
22//!   [`ConcurrencyBudget::run_cpu_bound`] (`spawn_blocking` + Semaphore).
23//! - **Hit scan:** large `all.html` candidate lists use `rayon` after a size
24//!   threshold; small lists stay sequential to avoid pool overhead.
25
26use std::collections::HashSet;
27use std::sync::LazyLock;
28
29use rayon::prelude::*;
30use regex::Regex;
31use scraper::{Html, Selector};
32use serde::{Deserialize, Serialize};
33use url::Url;
34
35use crate::config::{
36    HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS, MAX_SEARCH_IN_CRATE_LIMIT, is_stdlib_crate,
37    stdlib_channel,
38};
39use crate::domain::MatchMode;
40use crate::error::{AppError, AppResult, ErrorKind};
41use crate::http::{HttpClient, content_type_looks_html, decode_utf8};
42use crate::item_kind::{ItemKind, rustc_crate_name};
43
44/// Crate overview (rustdoc crate docblock, not git README).
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct ReadmeData {
47    /// Requested crate name.
48    pub crate_name: String,
49    /// Requested version token.
50    pub version: String,
51    /// Version resolved by docs.rs redirects when known.
52    #[serde(skip_serializing_if = "Option::is_none")]
53    pub resolved_version: Option<String>,
54    /// Markdown extracted from the crate overview.
55    pub markdown: String,
56    /// True when the overview body is empty.
57    pub empty: bool,
58    /// True when output was truncated by `max_output_bytes`.
59    pub truncated: bool,
60    /// Final source URL of the HTML page.
61    pub source_url: String,
62    /// True when the HTTP body was served from the local disk cache.
63    pub cache_hit: bool,
64}
65
66/// Typed item documentation payload.
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct GetItemData {
69    /// Requested crate name.
70    pub crate_name: String,
71    /// Normalized item kind string.
72    pub item_type: String,
73    /// Requested item path.
74    pub item_path: String,
75    /// Last path segment (leaf name) for agent convenience.
76    pub item_name: String,
77    /// Requested version token.
78    pub version: String,
79    /// Version resolved by docs.rs redirects when known.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub resolved_version: Option<String>,
82    /// Markdown extracted from the item page.
83    pub markdown: String,
84    /// True when the item body is empty.
85    pub empty: bool,
86    /// True when output was truncated by `max_output_bytes`.
87    pub truncated: bool,
88    /// Final source URL of the HTML page.
89    pub source_url: String,
90    /// Page title when available.
91    pub title: String,
92    /// True when the HTTP body was served from the local disk cache.
93    pub cache_hit: bool,
94}
95
96/// Single all.html hit.
97#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
98pub struct SearchInCrateHit {
99    /// Symbol or path name from all.html.
100    pub name: String,
101    /// Item kind string (`struct`, `fn`, …).
102    pub kind: String,
103    /// Absolute URL of the item documentation page.
104    pub url: String,
105    /// Match quality score (0 = best). Omitted when query is empty.
106    #[serde(skip_serializing_if = "Option::is_none")]
107    pub score: Option<u8>,
108}
109
110/// search-in-crate result set (`truncated` is true when `total > emitted`).
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct SearchInCrateData {
113    /// Requested crate name.
114    pub crate_name: String,
115    /// Text filter applied to hits.
116    pub query: String,
117    /// Requested version token.
118    pub version: String,
119    /// Echo of the applied item-type filter when set.
120    #[serde(skip_serializing_if = "Option::is_none")]
121    pub item_type: Option<String>,
122    /// Match mode applied (`exact` | `prefix` | `substring`).
123    pub match_mode: String,
124    /// Total classified hits before the limit.
125    pub total: usize,
126    /// Number of hits actually emitted.
127    pub emitted: usize,
128    /// Emitted hits.
129    pub hits: Vec<SearchInCrateHit>,
130    /// True when the hit list was cut by `--limit` (`total > emitted`).
131    pub truncated: bool,
132    /// Final source URL of the all.html page.
133    pub source_url: String,
134    /// True when the HTTP body was served from the local disk cache.
135    pub cache_hit: bool,
136}
137
138fn default_docs_origin(crate_name: &str) -> String {
139    if is_stdlib_crate(crate_name) {
140        format!("https://{HOST_DOC_RUST_LANG_ORG}")
141    } else {
142        format!("https://{HOST_DOCS_RS}")
143    }
144}
145
146/// Build crate index URL with rustc hyphen→underscore segment.
147///
148/// # Errors
149///
150/// Propagates [`ErrorKind::Internal`] from [`readme_url_on_origin`] when the URL is invalid.
151pub fn readme_url(crate_name: &str, version: &str) -> AppResult<Url> {
152    readme_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
153}
154
155/// Build readme URL against a custom origin (wiremock tests).
156///
157/// # Errors
158///
159/// Returns [`ErrorKind::Internal`] when `origin` / path segments do not form a valid URL.
160pub fn readme_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
161    let origin = origin.trim_end_matches('/');
162    let s = if is_stdlib_crate(crate_name) {
163        // doc.rust-lang.org/{channel}/{crate}/index.html
164        let channel = stdlib_channel(version);
165        format!("{origin}/{channel}/{crate_name}/index.html")
166    } else {
167        let rustc = rustc_crate_name(crate_name);
168        format!("{origin}/{crate_name}/{version}/{rustc}/index.html")
169    };
170    Url::parse(&s).map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid readme URL", e))
171}
172
173/// Build rustdoc item or module URL.
174///
175/// # Errors
176///
177/// Propagates [`ErrorKind::InvalidInput`] or [`ErrorKind::Internal`] from
178/// [`get_item_url_on_origin`].
179pub fn get_item_url(
180    crate_name: &str,
181    version: &str,
182    kind: ItemKind,
183    segments: &[String],
184) -> AppResult<Url> {
185    get_item_url_on_origin(
186        &default_docs_origin(crate_name),
187        crate_name,
188        version,
189        kind,
190        segments,
191    )
192}
193
194/// Detect associated method / inherent method path: `Type::method` where Type is
195/// UpperCamel and method starts lowercase (or kind forced as method via parse alias).
196fn is_method_path(kind: ItemKind, segs: &[String]) -> bool {
197    if segs.len() < 2 {
198        return false;
199    }
200    if kind != ItemKind::Fn {
201        return false;
202    }
203    let parent = segs[segs.len() - 2].as_str();
204    let method = segs[segs.len() - 1].as_str();
205    let parent_type = parent
206        .chars()
207        .next()
208        .is_some_and(|c| c.is_ascii_uppercase());
209    let method_fn = method
210        .chars()
211        .next()
212        .is_some_and(|c| c.is_ascii_lowercase() || c == '_');
213    parent_type && method_fn
214}
215
216/// Parent type kinds tried when resolving a method URL (struct first — common case).
217const METHOD_PARENT_KINDS: &[ItemKind] = &[
218    ItemKind::Struct,
219    ItemKind::Enum,
220    ItemKind::Trait,
221    ItemKind::Type,
222    ItemKind::Union,
223];
224
225/// Build get-item URL against a custom origin (wiremock tests).
226///
227/// Associated methods (`Runtime::new`) resolve to the parent type page plus
228/// `#method.{name}` (rustdoc layout). Free functions keep `{kind}.{name}.html`.
229///
230/// # Errors
231///
232/// Returns [`ErrorKind::InvalidInput`] when a non-module path lacks an item name.
233/// Returns [`ErrorKind::Internal`] when the assembled path is not a valid URL.
234pub fn get_item_url_on_origin(
235    origin: &str,
236    crate_name: &str,
237    version: &str,
238    kind: ItemKind,
239    segments: &[String],
240) -> AppResult<Url> {
241    get_item_url_on_origin_with_parent_kind(origin, crate_name, version, kind, segments, None)
242}
243
244/// Like [`get_item_url_on_origin`] but forces the parent type kind for methods.
245pub fn get_item_url_on_origin_with_parent_kind(
246    origin: &str,
247    crate_name: &str,
248    version: &str,
249    kind: ItemKind,
250    segments: &[String],
251    parent_kind_override: Option<ItemKind>,
252) -> AppResult<Url> {
253    let origin = origin.trim_end_matches('/');
254    let rustc_root = rustc_crate_name(crate_name);
255    let segs: Vec<String> = {
256        let mut s = segments.to_vec();
257        if let Some(first) = s.first() {
258            let f = first.as_str();
259            let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
260            if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
261                s.remove(0);
262            }
263        }
264        s
265    };
266
267    // Associated method: Type::method → parent page + #method.name
268    if is_method_path(kind, &segs) {
269        let method_name = segs.last().expect("is_method_path requires >=2 segs").clone();
270        let parent_name = segs[segs.len() - 2].clone();
271        let parent_kind = parent_kind_override.unwrap_or(ItemKind::Struct);
272        let mod_parts: Vec<String> = if segs.len() == 2 {
273            if is_stdlib_crate(crate_name) {
274                Vec::new()
275            } else {
276                vec![rustc_root.clone()]
277            }
278        } else {
279            let mut m = if is_stdlib_crate(crate_name) {
280                Vec::new()
281            } else {
282                vec![rustc_root.clone()]
283            };
284            for p in &segs[..segs.len() - 2] {
285                m.push(rustc_crate_name(p));
286            }
287            m
288        };
289        let url_str = if is_stdlib_crate(crate_name) {
290            let channel = stdlib_channel(version);
291            if mod_parts.is_empty() {
292                format!(
293                    "{origin}/{channel}/{crate_name}/{}.{}.html#method.{method_name}",
294                    parent_kind.file_prefix(),
295                    parent_name
296                )
297            } else {
298                format!(
299                    "{origin}/{channel}/{crate_name}/{}/{}.{}.html#method.{method_name}",
300                    mod_parts.join("/"),
301                    parent_kind.file_prefix(),
302                    parent_name
303                )
304            }
305        } else if mod_parts.is_empty() {
306            format!(
307                "{origin}/{crate_name}/{version}/{}/{}.{}.html#method.{method_name}",
308                rustc_root,
309                parent_kind.file_prefix(),
310                parent_name
311            )
312        } else {
313            format!(
314                "{origin}/{crate_name}/{version}/{}/{}.{}.html#method.{method_name}",
315                mod_parts.join("/"),
316                parent_kind.file_prefix(),
317                parent_name
318            )
319        };
320        return Url::parse(&url_str)
321            .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e));
322    }
323
324    let url_str = if is_stdlib_crate(crate_name) {
325        let channel = stdlib_channel(version);
326        if kind == ItemKind::Module {
327            let mut parts: Vec<String> = Vec::new();
328            for p in &segs {
329                parts.push(rustc_crate_name(p));
330            }
331            if parts.is_empty() {
332                format!("{origin}/{channel}/{crate_name}/index.html")
333            } else {
334                format!(
335                    "{origin}/{channel}/{crate_name}/{}/index.html",
336                    parts.join("/")
337                )
338            }
339        } else {
340            if segs.is_empty() {
341                return Err(AppError::new(
342                    ErrorKind::InvalidInput,
343                    "item path missing item name",
344                ));
345            }
346            let item_name = segs.last().ok_or_else(|| {
347                AppError::new(ErrorKind::InvalidInput, "item path missing item name")
348            })?;
349            let mod_parts: Vec<String> = segs[..segs.len().saturating_sub(1)]
350                .iter()
351                .map(|p| rustc_crate_name(p))
352                .collect();
353            if mod_parts.is_empty() {
354                format!(
355                    "{origin}/{channel}/{crate_name}/{}.{}.html",
356                    kind.file_prefix(),
357                    item_name
358                )
359            } else {
360                format!(
361                    "{origin}/{channel}/{crate_name}/{}/{}.{}.html",
362                    mod_parts.join("/"),
363                    kind.file_prefix(),
364                    item_name
365                )
366            }
367        }
368    } else if kind == ItemKind::Module {
369        let mut parts: Vec<String> = vec![rustc_root];
370        for p in &segs {
371            parts.push(rustc_crate_name(p));
372        }
373        format!(
374            "{origin}/{crate_name}/{version}/{}/index.html",
375            parts.join("/")
376        )
377    } else {
378        if segs.is_empty() {
379            return Err(AppError::new(
380                ErrorKind::InvalidInput,
381                "item path missing item name",
382            ));
383        }
384        let item_name = segs
385            .last()
386            .ok_or_else(|| AppError::new(ErrorKind::InvalidInput, "item path missing item name"))?;
387        let mod_parts: Vec<String> = if segs.len() == 1 {
388            vec![rustc_root]
389        } else {
390            let mut m = vec![rustc_root];
391            for p in &segs[..segs.len() - 1] {
392                m.push(rustc_crate_name(p));
393            }
394            m
395        };
396        format!(
397            "{origin}/{crate_name}/{version}/{}/{}.{}.html",
398            mod_parts.join("/"),
399            kind.file_prefix(),
400            item_name
401        )
402    };
403
404    Url::parse(&url_str)
405        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e))
406}
407
408/// Build all.html index URL.
409///
410/// # Errors
411///
412/// Propagates [`ErrorKind::Internal`] from [`all_html_url_on_origin`].
413pub fn all_html_url(crate_name: &str, version: &str) -> AppResult<Url> {
414    all_html_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
415}
416
417/// Build all.html URL against a custom origin (wiremock tests).
418///
419/// # Errors
420///
421/// Returns [`ErrorKind::Internal`] when `origin` / path segments do not form a valid URL.
422pub fn all_html_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
423    let origin = origin.trim_end_matches('/');
424    let s = if is_stdlib_crate(crate_name) {
425        let channel = stdlib_channel(version);
426        format!("{origin}/{channel}/{crate_name}/all.html")
427    } else {
428        let rustc = rustc_crate_name(crate_name);
429        format!("{origin}/{crate_name}/{version}/{rustc}/all.html")
430    };
431    Url::parse(&s)
432        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html URL", e))
433}
434
435#[cfg_attr(not(test), allow(dead_code))]
436fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
437    extract_resolved_version_for_crate(final_url, requested, None)
438}
439
440/// Resolve version/channel from the final URL, aware of docs.rs vs stdlib layout.
441///
442/// - docs.rs: `/{pkg}/{version}/…` — second path segment is the version
443/// - doc.rust-lang.org: `/{channel}/{crate}/…` — first path segment is channel/version
444fn extract_resolved_version_for_crate(
445    final_url: &Url,
446    requested: &str,
447    crate_name: Option<&str>,
448) -> Option<String> {
449    let host = final_url.host_str().unwrap_or("");
450    let segs: Vec<&str> = final_url
451        .path_segments()?
452        .filter(|s| !s.is_empty())
453        .collect();
454    if segs.is_empty() {
455        return None;
456    }
457
458    let is_stdlib_host = host.eq_ignore_ascii_case(HOST_DOC_RUST_LANG_ORG)
459        || crate_name.is_some_and(is_stdlib_crate)
460        || segs
461            .get(1)
462            .is_some_and(|s| matches!(*s, "std" | "core" | "alloc"));
463
464    if is_stdlib_host {
465        // /{channel}/{crate}/… — never emit crate name as version
466        let channel = segs.first()?;
467        if matches!(*channel, "std" | "core" | "alloc") {
468            return None;
469        }
470        return Some((*channel).to_string());
471    }
472
473    // docs.rs: /{pkg}/{version}/…
474    let ver = segs.get(1)?;
475    if *ver == "latest" {
476        return None;
477    }
478    if requested == "latest" || *ver != requested {
479        return Some((*ver).to_string());
480    }
481    Some((*ver).to_string())
482}
483
484/// Try to scrape a concrete SemVer from docs.rs HTML when the URL still says `/latest/`.
485///
486/// Only accepts versions that belong to `crate_name` (never a dependency hit).
487fn scrape_docs_rs_version_from_html(html: &str, crate_name: &str) -> Option<String> {
488    let crate_name = crate_name.trim();
489    if crate_name.is_empty() || is_stdlib_crate(crate_name) {
490        return None;
491    }
492    // Crate names are already validated (`[A-Za-z][A-Za-z0-9_-]*`); escape for safety.
493    let escaped = regex::escape(crate_name);
494    // Prefer paths for THIS crate: `/tokio/1.53.0/` or `docs.rs/tokio/1.53.0`
495    let re_path = Regex::new(&format!(
496        r"(?i)(?:https?://docs\.rs/|/){escaped}/([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?)\b"
497    ))
498    .ok()?;
499    if let Some(c) = re_path.captures(html) {
500        return c.get(1).map(|m| m.as_str().to_string());
501    }
502    // Title may include "crate version" on some rustdoc skins
503    let re_title = Regex::new(&format!(
504        r"(?i)<title>[^<]*?\b{escaped}\b[^<]*?\b([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?)\b"
505    ))
506    .ok()?;
507    re_title
508        .captures(html)
509        .and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
510}
511
512/// Compiled once: strip `on*` event handlers from HTML fragments (hot path for every HTML→MD).
513static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
514    Regex::new(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
515        .expect("hardcoded on* handler strip regex is valid by construction")
516});
517
518/// Compiled once: strip `javascript:` URLs from href/src attributes.
519static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
520    Regex::new(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
521        .expect("hardcoded javascript: URL strip regex is valid by construction")
522});
523
524/// Remove script/style nodes and on* handlers before Markdown conversion.
525///
526/// Extraction of doc content uses CSS selectors elsewhere in this module.
527/// Regexes here are **security scrub only** (`on*` / `javascript:`), compiled
528/// once via process-static [`LazyLock`] — not used to scrape fields from HTML.
529pub fn sanitize_html_fragment(html: &str) -> String {
530    let fragment = Html::parse_fragment(html);
531    let drop_sel = Selector::parse("script, style, noscript")
532        .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction");
533    // Prefer reconstructing from root children excluding dropped tags.
534    if let Some(root) = fragment.tree.root().children().next() {
535        let _ = root;
536    }
537    // Collect HTML of non-script/style nodes by stripping matches from a working copy.
538    // Capacity ≈ input length: script/style removals only shrink the buffer.
539    // Precondition: `html` is already body-capped (HARD_MAX_BODY_BYTES) and resident in RAM.
540    let mut out = String::with_capacity(html.len());
541    out.push_str(html);
542    for el in fragment.select(&drop_sel) {
543        let tag_html = el.html();
544        out = out.replace(&tag_html, "");
545    }
546    // Strip event handlers: onload=, onclick=, etc.
547    out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
548    // Drop javascript: URLs in href/src
549    out = RE_JS_URLS.replace_all(&out, "").into_owned();
550    out
551}
552
553fn html_to_markdown(html: &str) -> AppResult<String> {
554    let cleaned = sanitize_html_fragment(html);
555    htmd::convert(&cleaned).map_err(|e| {
556        AppError::new(
557            ErrorKind::Parse,
558            format!("HTML to Markdown conversion failed: {e}"),
559        )
560    })
561}
562
563fn first_inner_html(document: &Html, selectors: &[&str]) -> Option<String> {
564    for sel in selectors {
565        if let Ok(s) = Selector::parse(sel)
566            && let Some(n) = document.select(&s).next()
567        {
568            let inner = n.inner_html();
569            if !inner.trim().is_empty() {
570                return Some(inner);
571            }
572        }
573    }
574    None
575}
576
577/// Fetch and convert crate overview docblock (production docs.rs).
578///
579/// # Errors
580///
581/// Propagates URL, HTTP, decode, and HTML conversion failures from
582/// [`fetch_readme_on_origin`].
583pub async fn fetch_readme(
584    http: &HttpClient,
585    crate_name: &str,
586    version: &str,
587) -> AppResult<ReadmeData> {
588    fetch_readme_on_origin(
589        http,
590        &format!("https://{HOST_DOCS_RS}"),
591        crate_name,
592        version,
593    )
594    .await
595}
596
597/// Fetch readme against a configurable origin (offline mocks).
598///
599/// # Errors
600///
601/// Returns [`ErrorKind::Internal`] when the origin URL is invalid.
602/// Propagates fetch failures from [`fetch_readme_at`].
603pub async fn fetch_readme_on_origin(
604    http: &HttpClient,
605    origin: &str,
606    crate_name: &str,
607    version: &str,
608) -> AppResult<ReadmeData> {
609    let url = readme_url_on_origin(origin, crate_name, version)?;
610    fetch_readme_at(http, crate_name, version, &url).await
611}
612
613/// Fetch readme from a prebuilt URL (wiremock).
614///
615/// # Errors
616///
617/// Propagates HTTP errors from [`HttpClient::get_html`].
618/// Maps non-success statuses via [`AppError::from_http_status`].
619/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
620pub async fn fetch_readme_at(
621    http: &HttpClient,
622    crate_name: &str,
623    version: &str,
624    url: &Url,
625) -> AppResult<ReadmeData> {
626    let resp = http.get_html(url).await?;
627    if resp.status.as_u16() == 404 {
628        return Err(AppError::from_http_status(
629            404,
630            &format!("crate or version not found: {crate_name}@{version}"),
631        ));
632    }
633    if !resp.status.is_success() {
634        return Err(AppError::from_http_status(
635            resp.status.as_u16(),
636            "docs.rs readme",
637        ));
638    }
639    if resp.content_type.is_some() && !content_type_looks_html(resp.content_type.as_deref()) {
640        tracing::warn!(content_type = ?resp.content_type, "unexpected Content-Type from docs.rs");
641    }
642
643    let body = resp.body.clone();
644    let (markdown, empty) = http
645        .budget()
646        .run_cpu_bound(move || {
647            let text = decode_utf8(&body)?;
648            extract_readme_markdown_from_html(&text)
649        })
650        .await?;
651    let mut resolved =
652        extract_resolved_version_for_crate(&resp.final_url, version, Some(crate_name));
653    if resolved.is_none() {
654        // Body still available only as markdown path; re-decode for scrape when latest.
655        if let Ok(text) = decode_utf8(&resp.body) {
656            resolved = scrape_docs_rs_version_from_html(&text, crate_name);
657        }
658    }
659    Ok(ReadmeData {
660        crate_name: crate_name.to_string(),
661        version: version.to_string(),
662        resolved_version: resolved,
663        markdown,
664        empty,
665        truncated: false,
666        source_url: resp.final_url.to_string(),
667        cache_hit: resp.cache_hit,
668    })
669}
670
671/// Fetch and convert a typed rustdoc item page (production docs.rs).
672///
673/// # Errors
674///
675/// Propagates URL, HTTP, decode, and HTML conversion failures from
676/// [`fetch_item_on_origin`].
677pub async fn fetch_item(
678    http: &HttpClient,
679    crate_name: &str,
680    version: &str,
681    kind: ItemKind,
682    segments: &[String],
683) -> AppResult<GetItemData> {
684    fetch_item_on_origin(
685        http,
686        &format!("https://{HOST_DOCS_RS}"),
687        crate_name,
688        version,
689        kind,
690        segments,
691    )
692    .await
693}
694/// Fetch item against a configurable origin (offline mocks).
695///
696/// For associated methods, tries parent type pages (struct → enum → trait → …)
697/// until one returns HTTP 200.
698///
699/// # Errors
700///
701/// Propagates URL build errors from [`get_item_url_on_origin`] and fetch errors
702/// from [`fetch_item_at`].
703pub async fn fetch_item_on_origin(
704    http: &HttpClient,
705    origin: &str,
706    crate_name: &str,
707    version: &str,
708    kind: ItemKind,
709    segments: &[String],
710) -> AppResult<GetItemData> {
711    // Strip crate prefix the same way URL builder does for method detection.
712    let rustc_root = rustc_crate_name(crate_name);
713    let segs_for_detect: Vec<String> = {
714        let mut s = segments.to_vec();
715        if let Some(first) = s.first() {
716            let f = first.as_str();
717            let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
718            if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
719                s.remove(0);
720            }
721        }
722        s
723    };
724
725    if is_method_path(kind, &segs_for_detect) {
726        let mut last_err: Option<AppError> = None;
727        for parent_kind in METHOD_PARENT_KINDS {
728            let url = get_item_url_on_origin_with_parent_kind(
729                origin,
730                crate_name,
731                version,
732                kind,
733                segments,
734                Some(*parent_kind),
735            )?;
736            match fetch_item_at(http, crate_name, version, kind, segments, &url).await {
737                Ok(data) => return Ok(data),
738                Err(e) if e.kind() == ErrorKind::NotFound => {
739                    last_err = Some(e);
740                    continue;
741                }
742                Err(e) => return Err(e),
743            }
744        }
745        return Err(last_err.unwrap_or_else(|| {
746            AppError::new(ErrorKind::NotFound, "method parent type page not found")
747        }));
748    }
749
750    let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
751    fetch_item_at(http, crate_name, version, kind, segments, &url).await
752}
753
754/// Fetch item from a prebuilt URL (wiremock).
755///
756/// # Errors
757///
758/// Propagates HTTP errors from [`HttpClient::get_html`].
759/// Maps non-success statuses via [`AppError::from_http_status`].
760/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
761pub async fn fetch_item_at(
762    http: &HttpClient,
763    crate_name: &str,
764    version: &str,
765    kind: ItemKind,
766    segments: &[String],
767    url: &Url,
768) -> AppResult<GetItemData> {
769    // HTTP does not send fragments; strip for request, keep for source_url.
770    let mut fetch_url = url.clone();
771    let fragment = fetch_url.fragment().map(str::to_string);
772    fetch_url.set_fragment(None);
773
774    let resp = http.get_html(&fetch_url).await?;
775    if resp.status.as_u16() == 404 {
776        return Err(AppError::from_http_status(
777            404,
778            &format!("item not found at {url}"),
779        ));
780    }
781    if !resp.status.is_success() {
782        return Err(AppError::from_http_status(
783            resp.status.as_u16(),
784            "docs.rs get-item",
785        ));
786    }
787
788    let method_anchor = fragment
789        .as_deref()
790        .and_then(|f| f.strip_prefix("method."))
791        .map(str::to_string);
792    let body = resp.body.clone();
793    let method_anchor_cpu = method_anchor.clone();
794    let (markdown, empty) = http
795        .budget()
796        .run_cpu_bound(move || {
797            let text = decode_utf8(&body)?;
798            if let Some(ref m) = method_anchor_cpu {
799                extract_method_markdown_from_html(&text, m)
800            } else {
801                extract_item_markdown_from_html(&text)
802            }
803        })
804        .await?;
805    let item_path = segments.join("::");
806    let item_name = segments
807        .last()
808        .cloned()
809        .unwrap_or_else(|| item_path.clone());
810    let title = format!("{item_path} ({})", kind.as_str());
811    let mut resolved =
812        extract_resolved_version_for_crate(&resp.final_url, version, Some(crate_name));
813    if resolved.is_none() {
814        if let Ok(text) = decode_utf8(&resp.body) {
815            resolved = scrape_docs_rs_version_from_html(&text, crate_name);
816        }
817    }
818
819    let mut source_url = resp.final_url.clone();
820    if let Some(f) = fragment {
821        source_url.set_fragment(Some(&f));
822    }
823
824    Ok(GetItemData {
825        crate_name: crate_name.to_string(),
826        item_type: kind.as_str().to_string(),
827        item_path,
828        item_name,
829        version: version.to_string(),
830        resolved_version: resolved,
831        markdown,
832        empty,
833        truncated: false,
834        source_url: source_url.to_string(),
835        title,
836        cache_hit: resp.cache_hit,
837    })
838}
839/// Parse all.html and filter symbols (production docs.rs).
840///
841/// # Errors
842///
843/// Propagates URL, HTTP, decode, and parse failures from
844/// [`search_in_crate_on_origin`].
845pub async fn search_in_crate(
846    http: &HttpClient,
847    crate_name: &str,
848    version: &str,
849    query: &str,
850    item_type: Option<ItemKind>,
851    limit: u32,
852    match_mode: MatchMode,
853) -> AppResult<SearchInCrateData> {
854    search_in_crate_on_origin(
855        http,
856        &format!("https://{HOST_DOCS_RS}"),
857        crate_name,
858        version,
859        query,
860        item_type,
861        limit,
862        match_mode,
863    )
864    .await
865}
866
867/// search-in-crate against a configurable origin (offline mocks).
868///
869/// # Errors
870///
871/// Propagates URL build errors from [`all_html_url_on_origin`] and fetch errors
872/// from [`search_in_crate_at`].
873#[allow(clippy::too_many_arguments)]
874pub async fn search_in_crate_on_origin(
875    http: &HttpClient,
876    origin: &str,
877    crate_name: &str,
878    version: &str,
879    query: &str,
880    item_type: Option<ItemKind>,
881    limit: u32,
882    match_mode: MatchMode,
883) -> AppResult<SearchInCrateData> {
884    let url = all_html_url_on_origin(origin, crate_name, version)?;
885    search_in_crate_at(
886        http,
887        crate_name,
888        version,
889        query,
890        item_type,
891        limit,
892        match_mode,
893        &url,
894    )
895    .await
896}
897
898/// search-in-crate against a prebuilt all.html URL (wiremock).
899///
900/// # Errors
901///
902/// Propagates HTTP errors from [`HttpClient::get_html`].
903/// Maps non-success statuses via [`AppError::from_http_status`].
904/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or href join failures.
905/// Propagates parse errors from [`search_in_crate_from_html`].
906#[allow(clippy::too_many_arguments)]
907pub async fn search_in_crate_at(
908    http: &HttpClient,
909    crate_name: &str,
910    version: &str,
911    query: &str,
912    item_type: Option<ItemKind>,
913    limit: u32,
914    match_mode: MatchMode,
915    url: &Url,
916) -> AppResult<SearchInCrateData> {
917    let resp = http.get_html(url).await?;
918    if resp.status.as_u16() == 404 {
919        return Err(AppError::from_http_status(
920            404,
921            &format!("all.html not found for {crate_name}@{version}"),
922        ));
923    }
924    if !resp.status.is_success() {
925        return Err(AppError::from_http_status(
926            resp.status.as_u16(),
927            "docs.rs all.html",
928        ));
929    }
930
931    let body = resp.body.clone();
932    let source_url = resp.final_url.to_string();
933    let cache_hit = resp.cache_hit;
934    let crate_name = crate_name.to_string();
935    let version = version.to_string();
936    let query = query.to_string();
937    http.budget()
938        .run_cpu_bound(move || {
939            let text = decode_utf8(&body)?;
940            search_in_crate_from_html(
941                &text,
942                &crate_name,
943                &version,
944                &query,
945                item_type,
946                limit,
947                match_mode,
948                &source_url,
949                cache_hit,
950            )
951        })
952        .await
953}
954
955
956/// Join relative/absolute/full href against crate rustdoc base.
957///
958/// # Errors
959///
960/// Returns [`ErrorKind::Parse`] when a relative or absolute-path href cannot be joined.
961pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
962    if href.starts_with("http://") || href.starts_with("https://") {
963        return Ok(href.to_string());
964    }
965    if href.starts_with('/') {
966        let joined = Url::parse(&format!("https://docs.rs{href}")).map_err(|e| {
967            AppError::with_source(ErrorKind::Parse, "failed to join absolute path href", e)
968        })?;
969        return Ok(joined.to_string());
970    }
971    base.join(href)
972        .map(|u| u.to_string())
973        .map_err(|e| AppError::with_source(ErrorKind::Parse, "failed to join relative href", e))
974}
975
976/// Minimum candidate count before `rayon` fan-out (overhead below this is pure loss).
977const RAYON_HIT_THRESHOLD: usize = 64;
978
979/// One classified hit candidate before dedup/limit (index preserves document order).
980/// One classified hit candidate before dedup/limit: (index, name, kind, url, score).
981type ClassifiedHit = (usize, String, ItemKind, String, u8);
982
983/// Pure parse of all.html body for offline tests and CPU workers.
984///
985/// Large candidate lists use `rayon`; small lists stay sequential.
986/// Results are scored and sorted (exact leaf first) before applying `limit`.
987///
988/// # Errors
989///
990/// Returns [`ErrorKind::Internal`] when the synthetic base URL is invalid.
991/// Returns [`ErrorKind::Parse`] when an href cannot be joined via [`join_href`].
992pub fn parse_all_html_hits(
993    html: &str,
994    crate_name: &str,
995    version: &str,
996    query: &str,
997    item_type: Option<ItemKind>,
998    limit: usize,
999    match_mode: MatchMode,
1000) -> AppResult<Vec<SearchInCrateHit>> {
1001    if limit == 0 {
1002        return Ok(Vec::new());
1003    }
1004    let document = Html::parse_document(html);
1005    let a_sel = Selector::parse("#main-content a")
1006        .expect("hardcoded scraper selector '#main-content a' is valid by construction");
1007    let rustc = rustc_crate_name(crate_name);
1008    let base = Url::parse(&format!(
1009        "https://{HOST_DOCS_RS}/{crate_name}/{version}/{rustc}/"
1010    ))
1011    .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html base", e))?;
1012    let q = query.trim().to_ascii_lowercase();
1013
1014    let candidates: Vec<(String, String)> = document
1015        .select(&a_sel)
1016        .filter_map(|a| {
1017            let name = a.text().collect::<String>().trim().to_string();
1018            if name.is_empty() {
1019                return None;
1020            }
1021            let href = a.value().attr("href")?.to_string();
1022            if href.is_empty() {
1023                return None;
1024            }
1025            Some((name, href))
1026        })
1027        .collect();
1028
1029    if candidates.len() < RAYON_HIT_THRESHOLD {
1030        return filter_hits_sequential(candidates, &base, &q, item_type, limit, match_mode);
1031    }
1032    filter_hits_parallel(candidates, &base, &q, item_type, limit, match_mode)
1033}
1034
1035fn filter_hits_sequential(
1036    candidates: Vec<(String, String)>,
1037    base: &Url,
1038    q: &str,
1039    item_type: Option<ItemKind>,
1040    limit: usize,
1041    match_mode: MatchMode,
1042) -> AppResult<Vec<SearchInCrateHit>> {
1043    let mut rows: Vec<ClassifiedHit> = Vec::new();
1044    for (idx, (name, href)) in candidates.into_iter().enumerate() {
1045        if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
1046            rows.push(hit);
1047        }
1048    }
1049    finalize_hits(rows, limit)
1050}
1051
1052fn filter_hits_parallel(
1053    candidates: Vec<(String, String)>,
1054    base: &Url,
1055    q: &str,
1056    item_type: Option<ItemKind>,
1057    limit: usize,
1058    match_mode: MatchMode,
1059) -> AppResult<Vec<SearchInCrateHit>> {
1060    let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
1061        .into_par_iter()
1062        .enumerate()
1063        .map(|(idx, (name, href))| classify_hit_row(idx, name, &href, base, q, item_type, match_mode))
1064        .collect();
1065    let rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
1066    finalize_hits(rows, limit)
1067}
1068
1069fn finalize_hits(mut rows: Vec<ClassifiedHit>, limit: usize) -> AppResult<Vec<SearchInCrateHit>> {
1070    // Score ascending, then original document order for stability.
1071    rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
1072    let mut hits = Vec::with_capacity(limit.min(256));
1073    let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
1074    for (_idx, name, kind, abs, score) in rows {
1075        if !seen.insert((name.clone(), kind)) {
1076            continue;
1077        }
1078        hits.push(SearchInCrateHit {
1079            name,
1080            kind: kind.as_str().to_string(),
1081            url: abs,
1082            score: Some(score),
1083        });
1084        if hits.len() >= limit {
1085            break;
1086        }
1087    }
1088    Ok(hits)
1089}
1090
1091fn classify_hit_row(
1092    idx: usize,
1093    name: String,
1094    href: &str,
1095    base: &Url,
1096    q: &str,
1097    item_type: Option<ItemKind>,
1098    match_mode: MatchMode,
1099) -> AppResult<Option<ClassifiedHit>> {
1100    let Some(kind) = ItemKind::from_href(href) else {
1101        return Ok(None);
1102    };
1103    if let Some(filter) = item_type
1104        && kind != filter
1105    {
1106        return Ok(None);
1107    }
1108    let Some(score) = match_mode.score(&name, q) else {
1109        return Ok(None);
1110    };
1111    let abs = join_href(base, href)?;
1112    Ok(Some((idx, name, kind, abs, score)))
1113}
1114/// Extract readme markdown from raw HTML (offline tests / pure path).
1115///
1116/// # Errors
1117///
1118/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1119pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
1120    let document = Html::parse_document(html);
1121    let content = first_inner_html(
1122        &document,
1123        &[
1124            ".rustdoc .docblock",
1125            "#main-content .docblock",
1126            ".docblock",
1127            ".rustdoc-main .item-decl",
1128            ".item-decl",
1129        ],
1130    );
1131    match content {
1132        Some(h) => Ok((html_to_markdown(&h)?, false)),
1133        None => Ok((String::new(), true)),
1134    }
1135}
1136
1137/// Extract get-item markdown from raw HTML (offline tests / pure path).
1138///
1139/// # Errors
1140///
1141/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1142pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
1143    let document = Html::parse_document(html);
1144    let content = first_inner_html(&document, &["#main-content", ".item-decl", ".docblock"])
1145        .unwrap_or_default();
1146    let empty = content.trim().is_empty();
1147    if empty {
1148        Ok((String::new(), true))
1149    } else {
1150        Ok((html_to_markdown(&content)?, false))
1151    }
1152}
1153
1154/// Extract markdown for a single associated method by rustdoc `id="method.X"`.
1155///
1156/// Falls back to the full item page when the anchor is missing.
1157///
1158/// # Errors
1159///
1160/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1161/// Extract markdown for a single associated method by rustdoc `id="method.X"`.
1162///
1163/// When the method anchor is present, converts the parent section if found;
1164/// otherwise falls back to the full item page (still useful with `#method.` URL).
1165///
1166/// # Errors
1167///
1168/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1169pub fn extract_method_markdown_from_html(html: &str, method_name: &str) -> AppResult<(String, bool)> {
1170    let needle_dq = format!("id=\"method.{method_name}\"");
1171    let needle_sq = format!("id='method.{method_name}'");
1172    if !html.contains(&needle_dq) && !html.contains(&needle_sq) {
1173        return extract_item_markdown_from_html(html);
1174    }
1175    // Prefer a section/details/div that wraps the method id when scrapable.
1176    let document = Html::parse_document(html);
1177    // Attribute selectors with dots need care; use main-content fallback.
1178    if let Ok(sel) = Selector::parse("#main-content") {
1179        if let Some(main) = document.select(&sel).next() {
1180            let frag = main.html();
1181            if frag.contains(&needle_dq) || frag.contains(&needle_sq) {
1182                let md = html_to_markdown(&frag)?;
1183                let empty = md.trim().is_empty();
1184                return Ok((md, empty));
1185            }
1186        }
1187    }
1188    extract_item_markdown_from_html(html)
1189}
1190
1191/// Build SearchInCrateData from HTML body without network (offline tests).
1192/// Build SearchInCrateData from HTML body without network (offline tests).
1193///
1194/// `--limit 0` is honoured: `emitted = 0`, `hits = []`, and `truncated` is true
1195/// when the unfiltered total is greater than zero. Values above
1196/// [`MAX_SEARCH_IN_CRATE_LIMIT`] are capped.
1197///
1198/// # Errors
1199///
1200/// Propagates parse failures from [`parse_all_html_hits`].
1201#[allow(clippy::too_many_arguments)]
1202pub fn search_in_crate_from_html(
1203    html: &str,
1204    crate_name: &str,
1205    version: &str,
1206    query: &str,
1207    item_type: Option<ItemKind>,
1208    limit: u32,
1209    match_mode: MatchMode,
1210    source_url: &str,
1211    cache_hit: bool,
1212) -> AppResult<SearchInCrateData> {
1213    let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
1214    let all = parse_all_html_hits(
1215        html,
1216        crate_name,
1217        version,
1218        query,
1219        item_type,
1220        usize::MAX,
1221        match_mode,
1222    )?;
1223    let total = all.len();
1224    let hits: Vec<_> = all.into_iter().take(limit).collect();
1225    let emitted = hits.len();
1226    // Empty query: scores are 0 — omit score noise on wire for cleaner lists.
1227    let hits = if query.trim().is_empty() {
1228        hits.into_iter()
1229            .map(|mut h| {
1230                h.score = None;
1231                h
1232            })
1233            .collect()
1234    } else {
1235        hits
1236    };
1237    Ok(SearchInCrateData {
1238        crate_name: crate_name.to_string(),
1239        query: query.to_string(),
1240        version: version.to_string(),
1241        item_type: item_type.map(|k| k.as_str().to_string()),
1242        match_mode: match_mode.as_str().to_string(),
1243        total,
1244        emitted,
1245        hits,
1246        truncated: total > emitted,
1247        source_url: source_url.to_string(),
1248        cache_hit,
1249    })
1250}
1251#[cfg(test)]
1252mod tests {
1253    use super::*;
1254
1255    #[test]
1256    fn readme_url_hyphen() {
1257        let u = readme_url("async-trait", "latest").unwrap();
1258        assert_eq!(
1259            u.as_str(),
1260            "https://docs.rs/async-trait/latest/async_trait/index.html"
1261        );
1262    }
1263
1264    #[test]
1265    fn get_item_nested() {
1266        let segs = vec!["tokio".into(), "runtime".into(), "Runtime".into()];
1267        let u = get_item_url("tokio", "latest", ItemKind::Struct, &segs).unwrap();
1268        assert!(u.as_str().contains("/tokio/runtime/struct.Runtime.html"));
1269    }
1270
1271    #[test]
1272    fn get_item_without_crate_prefix() {
1273        let segs = vec!["Parser".into()];
1274        let u = get_item_url("clap", "latest", ItemKind::Trait, &segs).unwrap();
1275        assert!(
1276            u.as_str().ends_with("/clap/trait.Parser.html")
1277                || u.path().ends_with("/clap/trait.Parser.html")
1278        );
1279    }
1280
1281    #[test]
1282    fn get_item_keeps_single_segment_equal_to_crate_name() {
1283        // Attribute/macro items often share the rustc crate name.
1284        let segs = vec!["async_trait".into()];
1285        let u = get_item_url("async-trait", "latest", ItemKind::Attribute, &segs).unwrap();
1286        assert!(
1287            u.as_str()
1288                .ends_with("/async-trait/latest/async_trait/attr.async_trait.html"),
1289            "url={u}"
1290        );
1291    }
1292
1293    #[test]
1294    fn get_item_module_single_segment_crate_name_is_root() {
1295        let segs = vec!["async_trait".into()];
1296        let u = get_item_url("async-trait", "latest", ItemKind::Module, &segs).unwrap();
1297        assert!(
1298            u.as_str()
1299                .ends_with("/async-trait/latest/async_trait/index.html"),
1300            "url={u}"
1301        );
1302    }
1303
1304    #[test]
1305    fn join_relative() {
1306        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1307        let j = join_href(&base, "de/trait.Deserialize.html").unwrap();
1308        assert!(j.contains("/serde/de/trait.Deserialize.html"));
1309    }
1310
1311    #[test]
1312    fn join_absolute_path_and_fragment() {
1313        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1314        let j = join_href(&base, "/serde/latest/serde/struct.Error.html").unwrap();
1315        assert!(j.starts_with("https://docs.rs/serde/"));
1316        let f = join_href(&base, "struct.Error.html#method.to_string").unwrap();
1317        assert!(f.contains("#method.to_string"));
1318    }
1319
1320    #[test]
1321    fn sanitize_strips_script() {
1322        let raw = r#"<div>ok<script>alert(1)</script><p onclick="x()">hi</p></div>"#;
1323        let s = sanitize_html_fragment(raw);
1324        assert!(!s.contains("script"));
1325        assert!(!s.contains("onclick"));
1326        assert!(s.contains("hi") || s.contains("ok"));
1327    }
1328
1329    #[test]
1330    fn readme_fallback_item_decl_converts() {
1331        let html = r#"<!DOCTYPE html><html><body class="rustdoc-main">
1332        <div class="item-decl"><pre>pub use foo;</pre></div>
1333        </body></html>"#;
1334        let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1335        assert!(!empty);
1336        assert!(!md.trim().is_empty());
1337    }
1338
1339    #[test]
1340    fn all_html_order_stable() {
1341        let html = r#"<!DOCTYPE html><html><body>
1342        <div id="main-content">
1343          <a href="struct.A.html">A</a>
1344          <a href="trait.B.html">B</a>
1345          <a href="fn.c.html">c</a>
1346          <a href="union.U.html">U</a>
1347          <a href="attr.x.html">x</a>
1348          <a href="derive.D.html">D</a>
1349        </div></body></html>"#;
1350        let hits = parse_all_html_hits(html, "demo", "1.0.0", "", None, 100, crate::domain::MatchMode::Prefix).unwrap();
1351        assert_eq!(hits.len(), 6);
1352        assert_eq!(hits[0].name, "A");
1353        assert_eq!(hits[0].kind, "struct");
1354        assert_eq!(hits[3].kind, "union");
1355        assert_eq!(hits[4].kind, "attribute");
1356        assert_eq!(hits[5].kind, "derive");
1357    }
1358
1359    #[test]
1360    fn std_core_alloc_urls() {
1361        let u = readme_url("std", "latest").unwrap();
1362        assert_eq!(
1363            u.as_str(),
1364            "https://doc.rust-lang.org/stable/std/index.html"
1365        );
1366        let u = get_item_url(
1367            "std",
1368            "latest",
1369            ItemKind::Struct,
1370            &["option".into(), "Option".into()],
1371        )
1372        .unwrap();
1373        assert_eq!(
1374            u.as_str(),
1375            "https://doc.rust-lang.org/stable/std/option/struct.Option.html"
1376        );
1377        let a = all_html_url("core", "nightly").unwrap();
1378        assert_eq!(
1379            a.as_str(),
1380            "https://doc.rust-lang.org/nightly/core/all.html"
1381        );
1382    }
1383
1384    #[test]
1385    fn extract_item_from_fixture() {
1386        let html = include_str!("../tests/fixtures/docs_rs/get_item_main.html");
1387        let (md, empty) = extract_item_markdown_from_html(html).unwrap();
1388        assert!(!empty);
1389        assert!(md.contains("Runtime") || md.contains("Tokio") || md.contains("runtime"));
1390    }
1391
1392    #[test]
1393    fn extract_readme_primary_docblock_fixture() {
1394        let html = include_str!("../tests/fixtures/docs_rs/readme_docblock.html");
1395        let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1396        assert!(!empty);
1397        assert!(!md.to_ascii_lowercase().contains("alert"));
1398    }
1399
1400    #[test]
1401    fn search_in_crate_from_html_limit_and_filter() {
1402        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1403        let data = search_in_crate_from_html(
1404            html,
1405            "demo",
1406            "1.0.0",
1407            "",
1408            Some(ItemKind::Struct),
1409            1,
1410            MatchMode::Prefix,
1411            "https://docs.rs/demo/1.0.0/demo/all.html",
1412        false,
1413        )
1414        .unwrap();
1415        assert_eq!(data.total, 1);
1416        assert_eq!(data.emitted, 1);
1417        assert_eq!(data.hits[0].kind, "struct");
1418        assert!(!data.truncated);
1419    }
1420
1421    #[test]
1422    fn search_in_crate_truncated_when_limit_cuts_hits() {
1423        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1424        let data = search_in_crate_from_html(
1425            html,
1426            "demo",
1427            "1.0.0",
1428            "",
1429            None,
1430            2,
1431            MatchMode::Prefix,
1432            "https://docs.rs/demo/1.0.0/demo/all.html",
1433        false,
1434        )
1435        .unwrap();
1436        assert!(data.total > 2, "fixture must have more than 2 hits");
1437        assert_eq!(data.emitted, 2);
1438        assert!(data.truncated);
1439        let full = search_in_crate_from_html(
1440            html,
1441            "demo",
1442            "1.0.0",
1443            "",
1444            None,
1445            1000,
1446            MatchMode::Prefix,
1447            "https://docs.rs/demo/1.0.0/demo/all.html",
1448        false,
1449        )
1450        .unwrap();
1451        assert_eq!(full.total, full.emitted);
1452        assert!(!full.truncated);
1453    }
1454
1455    #[test]
1456    fn search_in_crate_limit_zero_emits_nothing() {
1457        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1458        let data = search_in_crate_from_html(
1459            html,
1460            "demo",
1461            "1.0.0",
1462            "",
1463            None,
1464            0,
1465            MatchMode::Prefix,
1466            "https://docs.rs/demo/1.0.0/demo/all.html",
1467        false,
1468        )
1469        .unwrap();
1470        assert!(data.total > 0, "fixture must have hits for total");
1471        assert_eq!(data.emitted, 0);
1472        assert!(data.hits.is_empty());
1473        assert!(data.truncated);
1474    }
1475
1476    #[test]
1477    fn serialize_exact_match_beats_deserializer() {
1478        let html = r#"<!DOCTYPE html><html><body>
1479        <div id="main-content">
1480          <a href="trait.Deserialize.html">Deserialize</a>
1481          <a href="trait.Serialize.html">Serialize</a>
1482          <a href="struct.Deserializer.html">Deserializer</a>
1483        </div></body></html>"#;
1484        let hits = parse_all_html_hits(
1485            html,
1486            "serde",
1487            "1.0.0",
1488            "Serialize",
1489            Some(ItemKind::Trait),
1490            10,
1491            MatchMode::Prefix,
1492        )
1493        .unwrap();
1494        assert_eq!(hits[0].name, "Serialize");
1495        assert!(hits.iter().all(|h| h.name == "Serialize"));
1496    }
1497
1498    #[test]
1499    fn method_url_uses_anchor() {
1500        let segs = vec!["runtime".into(), "Runtime".into(), "new".into()];
1501        let u = get_item_url("tokio", "latest", ItemKind::Fn, &segs).unwrap();
1502        assert!(
1503            u.as_str().contains("struct.Runtime.html#method.new"),
1504            "url={u}"
1505        );
1506    }
1507
1508    #[test]
1509    fn stdlib_resolved_version_is_channel() {
1510        let u = Url::parse("https://doc.rust-lang.org/stable/std/index.html").unwrap();
1511        assert_eq!(
1512            extract_resolved_version_for_crate(&u, "latest", Some("std")).as_deref(),
1513            Some("stable")
1514        );
1515    }
1516
1517    #[test]
1518    fn resolved_version_from_url() {
1519        let u = Url::parse("https://docs.rs/serde/1.0.210/serde/index.html").unwrap();
1520        assert_eq!(
1521            extract_resolved_version(&u, "latest").as_deref(),
1522            Some("1.0.210")
1523        );
1524        let u2 = Url::parse("https://docs.rs/serde/latest/serde/index.html").unwrap();
1525        assert!(extract_resolved_version(&u2, "latest").is_none());
1526    }
1527
1528    #[test]
1529    fn scrape_version_prefers_target_crate_not_deps() {
1530        let html = r#"
1531        <title>tokio - Rust</title>
1532        <a href="https://docs.rs/bytes/1.2.0/bytes/">bytes</a>
1533        <a href="/tokio/1.53.0/tokio/runtime/index.html">runtime</a>
1534        <a href="https://docs.rs/mio/0.8.0/mio/">mio</a>
1535        "#;
1536        assert_eq!(
1537            scrape_docs_rs_version_from_html(html, "tokio").as_deref(),
1538            Some("1.53.0")
1539        );
1540        // Must not pick dependency when crate path missing
1541        let html2 = r#"<title>x</title><a href="https://docs.rs/bytes/1.2.0/bytes/">b</a>"#;
1542        assert_eq!(scrape_docs_rs_version_from_html(html2, "tokio"), None);
1543    }
1544
1545    #[test]
1546    fn module_url_template() {
1547        let segs = vec!["serde".into(), "de".into()];
1548        let u = get_item_url("serde", "latest", ItemKind::Module, &segs).unwrap();
1549        assert!(u.as_str().ends_with("/serde/de/index.html"));
1550    }
1551
1552    #[test]
1553    fn empty_html_readme() {
1554        let (md, empty) = extract_readme_markdown_from_html("<html><body></body></html>").unwrap();
1555        assert!(empty);
1556        assert!(md.is_empty());
1557    }
1558
1559    #[test]
1560    fn get_item_empty_segments_errors() {
1561        let err = get_item_url("clap", "latest", ItemKind::Struct, &[]).unwrap_err();
1562        assert_eq!(err.kind(), ErrorKind::InvalidInput);
1563    }
1564
1565    #[test]
1566    fn join_full_http_url_passthrough() {
1567        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1568        let full = "https://docs.rs/serde/latest/serde/struct.Error.html";
1569        assert_eq!(join_href(&base, full).unwrap(), full);
1570        let http = "http://example.invalid/x.html";
1571        assert_eq!(join_href(&base, http).unwrap(), http);
1572    }
1573
1574    #[test]
1575    fn origin_builders_trim_trailing_slash() {
1576        let u = readme_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1577        assert_eq!(u.as_str(), "https://docs.rs/demo/1.0.0/demo/index.html");
1578        let a = all_html_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1579        assert_eq!(a.as_str(), "https://docs.rs/demo/1.0.0/demo/all.html");
1580    }
1581
1582    #[test]
1583    fn empty_item_html_extract() {
1584        let (md, empty) = extract_item_markdown_from_html("<html><body></body></html>").unwrap();
1585        assert!(empty);
1586        assert!(md.is_empty());
1587    }
1588}